All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Public Member Functions | Public Attributes | List of all members
manageDataRunDefinitions.SampleProcessClass Class Reference

Public Member Functions

def __init__
 
def __call__
 
def doDescribe
 
def doQuery
 
def doCheck
 
def doCreateDef
 
def doDeleteDef
 
def buildDefName
 
def getSummary
 
def defNameCount
 
def describeSample
 

Public Attributes

 actions
 
 fake
 
 force
 
 prependUser
 
 samweb
 
 buildQuery
 
 SAMuser
 

Detailed Description

Definition at line 240 of file manageDataRunDefinitions.py.

Constructor & Destructor Documentation

def manageDataRunDefinitions.SampleProcessClass.__init__ (   self,
  samweb,
  create = False,
  query = False,
  printDefs = False,
  describe = False,
  check = False,
  delete = False,
  fake = False,
  force = False,
  prependUser = True 
)

Definition at line 245 of file manageDataRunDefinitions.py.

245 
246  ):
247  # action collection
248  self.actions = []
249  if check: self.actions.append("check")
250  if create: self.actions.append("create")
251  if query: self.actions.append("query")
252  if printDefs: self.actions.append("printDefs")
253  if describe: self.actions.append("describe")
254  if delete: self.actions.append("delete")
255  if not self.actions:
256  raise RuntimeError("At least one action needs to be enabled.")
257  self.fake = fake
258  self.force = force
259  self.prependUser = prependUser
260 
261  self.samweb = samweb if samweb else SAMWebClient()
262  self.buildQuery = DimensionQueryMaker()
263 
264  try: self.SAMuser = samweb.get_user()
265  except samexcpt.Error as e:
266  if self.prependUser:
267  logging.error("Could not find out your name! %s", e)
268  raise
269  self.SAMuser = None
270  #

Member Function Documentation

def manageDataRunDefinitions.SampleProcessClass.__call__ (   self,
  info 
)

Definition at line 274 of file manageDataRunDefinitions.py.

274 
275  def __call__(self, info):
276  assert self.samweb, "SAM web client not initialized. We do not go anywhere."
277 
278  if not info.isComplete():
279  raise RuntimeError(f"Can't process incomplete specification: {info}")
280 
281  dim = self.buildQuery(info)
282 
283  logging.debug("Info: %s => dim='%s'", info, dim)
284  defName = self.buildDefName(info)
285 
286  for action in self.actions:
287 
288  if action == "printDefs":
289  print(defName)
290 
291  elif action == "check":
292  self.doCheck(defName)
293 
294  elif action == "describe":
295  self.doDescribe(defName)
296 
297  elif action == "query":
298  self.doQuery(info=info, defName=defName, dim=dim)
299 
300  elif action == "create":
301  self.doCreateDef(info, defName=defName, dim=dim)
302 
303  elif action == "delete":
304  self.doDeleteDef(info, defName=defName, dim=dim)
305 
306  else:
307  raise RuntimeError(f"LOGIC ERROR: action {action} not implemented.")
308 
309  # for
310 
return defName
print
do one_file $F done echo for F in find $TOP name CMakeLists txt print
Definition: update_sources.sh:94
def manageDataRunDefinitions.SampleProcessClass.buildDefName (   self,
  info 
)

Definition at line 481 of file manageDataRunDefinitions.py.

481 
482  def buildDefName(self, info):
483  if self.prependUser: return self.SAMuser + "_" + info.defName()
else: return info.defName()
def manageDataRunDefinitions.SampleProcessClass.defNameCount (   self,
  defName 
)

Definition at line 504 of file manageDataRunDefinitions.py.

504 
505  def defNameCount(self, defName: "SAM definition name"):
506  """Returns the count of files of `defName`, `None` if not found.
507 
508  Throws exception in all other error situations.
509  """
510  try: return self.samweb.countFiles(defname=defName)
except samexcpt.DefinitionNotFound: return None
def manageDataRunDefinitions.SampleProcessClass.describeSample (   self,
  info 
)

Definition at line 513 of file manageDataRunDefinitions.py.

513 
514  def describeSample(self, info): return "ICARUS data " + str(info)
515 
516 # class SampleProcessClass
517 
518 
# ------------------------------------------------------------------------------
def manageDataRunDefinitions.SampleProcessClass.doCheck (   self,
  defName 
)

Definition at line 343 of file manageDataRunDefinitions.py.

343 
344  def doCheck(self, defName):
345  assert defName
346  if self.fake:
347  print(f"DRYRUN> countFiles(defname={defName!r})")
348  return True
349  count = self.defNameCount(defName)
350  if count is None:
351  print(f"{defName} not available")
352  return False
353  else:
354  print(f"{defName} available ({count} files)")
return True
print
do one_file $F done echo for F in find $TOP name CMakeLists txt print
Definition: update_sources.sh:94
def manageDataRunDefinitions.SampleProcessClass.doCreateDef (   self,
  info,
  defName = None,
  dim = None 
)

Definition at line 357 of file manageDataRunDefinitions.py.

357 
358  def doCreateDef(self, info, defName=None, dim=None):
359  assert info
360  if not defName: defName = self.buildDefName(info)
361  if not dim: dim = self.buildQuery(info)
362  descr = self.describeSample(info)
363 
364  count = self.defNameCount(defName)
365  if count is not None:
366  logging.error(f"Definition {defName!r} already exists (and matches {count} files).")
367  return None
368 
369  try:
370  count = self.samweb.countFiles(dimensions=dim)
371  except samexcpt.Error as e:
372  logging.error(f"Attempt to count matches with {dim!r} failed: %s", e)
373  return None
374  if count == 0:
375  print(f"Definition {defName} NOT created as it would match no file (query: {dim!r})")
376  return None
377  logging.debug(f"Creating {defName!r}, now matching {count} files")
378  if self.fake:
379  print(f"DRYRUN> createDefinition(defname={defName!r}, dims={dim!r}, description={descr!r})")
380  else:
381  try:
382  self.samweb.createDefinition(defname=defName, dims=dim, description=descr)
383  print(f"{defName} created ({count} files)")
384  except samexcpt.Error as e:
385  logging.error \
386  (f"Failed to create definition {defName} from query='{dim}': %s", e)
387  return None
388  # if
return defName
print
do one_file $F done echo for F in find $TOP name CMakeLists txt print
Definition: update_sources.sh:94
def manageDataRunDefinitions.SampleProcessClass.doDeleteDef (   self,
  info,
  defName = None,
  dim = None 
)

Definition at line 391 of file manageDataRunDefinitions.py.

391 
392  def doDeleteDef(self, info, defName=None, dim=None):
393  assert self.samweb
394  assert info
395  if not defName: defName = self.buildDefName(info)
396  if not dim: dim = self.buildQuery(info)
397  descr = self.describeSample(info)
398 
399  try:
400  defInfo = self.samweb.descDefinitionDict(defName)
401  except samexcpt.DefinitionNotFound:
402  print(f"Definition Name: {defName} => NOT FOUND") # we are ok... I guess
403  return None
404 
405  # we perform many checks to make sure this deletion is proper
406  ForcedMsg = { True: "forced to delete it anyway", False: "won't delete unless forced to", }
407  checksOk = True
408 
409  if not self.SAMuser:
410  try: self.SAMuser = self.samweb.get_user()
411  except samexcpt.Error as e:
412  logging.error("Could not find out your name! %s", e)
413  # if not cached already
414  try: SAMgroup = self.samweb.get_group()
415  except samexcpt.Error as e:
416  logging.error("Could not find out the name of your group! %s", e)
417  logging.debug(f"You appear to be {SAMuser!r} of group {SAMgroup!r}")
418 
419  if defInfo['username'] != SAMuser:
420  logging.warning(
421  f"Definition {defName!r} was created on {defInfo['create_time']}"
422  f" by {defInfo['username']}/{defInfo['group']}, not by you ({SAMuser})"
423  f": won't delete."
424  )
425  checksOk = False
426  # if
427 
428  if defInfo['group'] != SAMgroup:
429  logging.warning(
430  f"Definition {defName!r} was created on {defInfo['create_time']}"
431  f" by {defInfo['username']}/{defInfo['group']}, not by your group ({SAMgroup})"
432  f": won't delete."
433  )
434  checksOk = False
435  # if
436 
437  if defInfo['dimensions'] != dim:
438  logging.warning(f"Definition {defName!r} has unexpected query:"
439  f" ({defInfo['dimensions']!r}, expected: {dim!r}); {ForcedMsg[self.force]}."
440  )
441  if not self.force: checksOk = False
442  #
443 
444  if defInfo['description'] != descr:
445  logging.warning(
446  f"Definition {defName!r} appears not to be created with this program:"
447  f" description mismatch"
448  f" ({defInfo['description']!r}, expected: {descr!r}); {ForcedMsg[self.force]}."
449  )
450  if not self.force: checksOk = False
451  # if
452 
453  if not checksOk:
454  logging.error(f"Definition {defName!r} will NOT be deleted.")
455  return None
456 
457  if self.fake:
458  print(f"DRYRUN> deleteDefinition({defName!r})")
459  return defName
460  try:
461  self.samweb.deleteDefinition(defName)
462  except samexcpt.DefinitionNotFound:
463  logging.error(f"Definition {defName} NOT FOUND (can't be deleted)")
464  # except samexcpt.DefinitionNotFound: # which exception to match?
465  # logging.error(f"Definition {defName!r} has already been used and can't be deleted.")
466  except samexcpt.NoAuthorizationCredentials as e:
467  logging.error(f"Failed to delete definition {defName!r} for lack of credentials: %s", e)
468  except samexcpt.Error as e:
469  logging.error(f"Failed to delete definition {defName!r}: %s", e)
470 
471  count = self.defNameCount(defName)
472  if count is not None:
473  logging.error(f"Deletion of definition {defName!r} silently FAILED"
474  f" (still there with its own {count} files).")
475  return None
476  # if still there
477  print(f"Definition {defName} successfully deleted.")
return defName
print
do one_file $F done echo for F in find $TOP name CMakeLists txt print
Definition: update_sources.sh:94
def manageDataRunDefinitions.SampleProcessClass.doDescribe (   self,
  defName 
)

Definition at line 314 of file manageDataRunDefinitions.py.

314 
315  def doDescribe(self, defName):
316  if self.fake:
317  print(f"DRYRUN> descDefinition({defName!r})")
318  return
319  try:
320  print(self.samweb.descDefinition(defName))
321  except samexcpt.DefinitionNotFound:
print(f"Definition Name: {defName} => NOT FOUND")
print
do one_file $F done echo for F in find $TOP name CMakeLists txt print
Definition: update_sources.sh:94
def manageDataRunDefinitions.SampleProcessClass.doQuery (   self,
  info,
  defName,
  dim 
)

Definition at line 324 of file manageDataRunDefinitions.py.

324 
325  def doQuery(self, info, defName, dim):
326  if self.fake:
327  print(f"DRYRUN> {dim}")
328  return None
329  try:
330  summary = self.getSummary(info=info, defName=defName, dims=dim)
331  except samexcpt.Error as e:
332  logging.error(f"Query of definition {defName} (query: '{dim}') failed: %s", e)
333  summary = None
334  else:
335  print(f"{info}: {(summary['total_event_count'] if summary['total_event_count'] else 'unknown')} events"
336  f" in {summary['file_count']} files"
337  f" ({summary['total_file_size']/(1 << 30):g} GiB)"
338  )
339  e = None
340  # try ... else
return summary if summary else e
print
do one_file $F done echo for F in find $TOP name CMakeLists txt print
Definition: update_sources.sh:94
def manageDataRunDefinitions.SampleProcessClass.getSummary (   self,
  info = None,
  defName = None,
  dims = None 
)

Definition at line 486 of file manageDataRunDefinitions.py.

486 
487  def getSummary(self, info=None, defName=None, dims=None):
488  assert info or defName or dims
489  e = RuntimeError("Insufficient parameters to get summary") # should not happen
490 
491  if not defName and info: defName = self.buildDefName(info)
492  if defName:
493  try: return self.samweb.listFilesSummary(defname=defName)
494  except samexcpt.DefinitionNotFound as e: queryError = e
495 
496  if not dims and info: dims = self.buildQuery(info)
497  if dims:
498  try: return self.samweb.listFilesSummary(dims)
499  except samexcpt.Error as e: queryError = e
500 
raise e

Member Data Documentation

manageDataRunDefinitions.SampleProcessClass.actions

Definition at line 247 of file manageDataRunDefinitions.py.

manageDataRunDefinitions.SampleProcessClass.buildQuery

Definition at line 261 of file manageDataRunDefinitions.py.

manageDataRunDefinitions.SampleProcessClass.fake

Definition at line 256 of file manageDataRunDefinitions.py.

manageDataRunDefinitions.SampleProcessClass.force

Definition at line 257 of file manageDataRunDefinitions.py.

manageDataRunDefinitions.SampleProcessClass.prependUser

Definition at line 258 of file manageDataRunDefinitions.py.

manageDataRunDefinitions.SampleProcessClass.SAMuser

Definition at line 263 of file manageDataRunDefinitions.py.

manageDataRunDefinitions.SampleProcessClass.samweb

Definition at line 260 of file manageDataRunDefinitions.py.

The documentation for this class was generated from the following file:
Generated by   doxygen 1.8.5