All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Functions | Variables
python.project_utilities Namespace Reference

Functions

def fast_isdir
 
def wait_for_stat
 
def get_scratch_dir
 
def mountpoint
 
def dollar_escape
 
def parseInt
 
def create_limited_dataset
 
def samweb
 
def start_project
 
def active_projects2
 
def active_projects
 
def make_active_project_dataset
 
def defExists
 
def makeDummyDef
 
def addLayerTwo
 
def default_jobsub_submit_options
 
def batch_status_check
 
def path_to_srm_url
 
def safeexist
 
def saferead
 
def safecopy
 
def path_to_url
 
def path_to_local
 
def SafeTFile
 
def expandDefnames
 
def tokenizeRPN
 
def listFiles
 
def makeFileListDefinition
 
def get_singularity
 

Variables

 myargv = sys.argv
 
 samweb_obj = None
 
dictionary samcache = {}
 

Function Documentation

def python.project_utilities.active_projects (   defname = '')

Definition at line 402 of file project_utilities.py.

402 
403 def active_projects(defname = ''):
404 
405  result = set()
406 
407  # Get project name stem.
408 
409  s = samweb()
410  prjstem = ''
411  if defname != '':
412  prjstem = '%s_' % s.makeProjectName(defname).rsplit('_',1)[0]
413 
414  # Dump station
415 
416  url = '%s/dumpStation?station=%s' % (s.get_baseurl(), get_experiment())
417  furl = urlrequest.urlopen(url)
418 
419  # Parse response.
420 
421  for line in furl.readlines():
422  words = line.split()
423  if len(words) > 5:
424  prjname = convert_str(words[0])
425  if prjstem == '' or prjname.startswith(prjstem):
426  result.add(prjname)
427 
428  # Done.
429 
430  return result
431 
432 # Make active projects dataset definition.
433 #
434 # defname - Dataset definition associated with active projects.
435 # dropbixwait - Dropbox wait interval (float days).
436 # active_defname - Name of dataset definition to create.
437 # wait_defname - Name of dropbox waiting dataset to create.
python.project_utilities.samweb
def samweb
Definition: project_utilities.py:258
python.project_utilities.active_projects
def active_projects
Definition: project_utilities.py:402
python.larbatch_utilities.get_experiment
def get_experiment
Definition: larbatch_utilities.py:876
python.larbatch_utilities.convert_str
def convert_str
Definition: larbatch_utilities.py:1171
def python.project_utilities.active_projects2 (   defname = '',
  dropboxwait = 0. 
)

Definition at line 349 of file project_utilities.py.

349 
350 def active_projects2(defname = '', dropboxwait = 0.):
351 
352  result = set()
353 
354  # Get project name stem.
355 
356  s = samweb()
357  prjstem = ''
358  if defname != '':
359  prjstem = '%s_' % s.makeProjectName(defname).rsplit('_',1)[0]
360 
361  # Query a list of projects started within the last 72 hours.
362 
363  dt = datetime.timedelta(3, 0)
364  tmin = datetime.datetime.utcnow() - dt
365  tminstr = tmin.strftime('%Y-%m-%dT%H:%M:%S')
366  prjnames = s.listProjects(started_after = tminstr)
367 
368  # Loop over projects to check end times.
369 
370  for prjname in prjnames:
371  if prjstem == '' or prjname.startswith(prjstem):
372 
373  # This project is a candidate for inclusion in result.
374  # Check end time.
375 
376  age = 0
377  prjurl = s.findProject(project=prjname, station=get_experiment())
378  if prjurl != '':
379  prjsum = s.projectSummary(prjurl)
380  if 'project_end_time' in prjsum:
381  tendstr = prjsum['project_end_time']
382  if len(tendstr) >= 19:
383  try:
384  tend = datetime.datetime.strptime(tendstr[:19], '%Y-%m-%dT%H:%M:%S')
385  tage = datetime.datetime.utcnow() - tend
386  age = tage.total_seconds()
387  except:
388  pass
389 
390  # Keep this project if there is no end time.
391 
392  if age <= dropboxwait * 86400:
393  result.add(prjname)
394 
395  # Done.
396 
397  return result
398 
399 
400 # Return a list of active projects associated with a particular dataset definition stem.
401 # If the definition argument is the empty string, return all active projects.
python.project_utilities.samweb
def samweb
Definition: project_utilities.py:258
python.larbatch_utilities.get_experiment
def get_experiment
Definition: larbatch_utilities.py:876
python.project_utilities.active_projects2
def active_projects2
Definition: project_utilities.py:349
def python.project_utilities.addLayerTwo (   path,
  recreate = True 
)

Definition at line 539 of file project_utilities.py.

539 
540 def addLayerTwo(path, recreate=True):
541 
542  # Don't do anything if this file is not located in dCache (/pnfs/...)
543  # or has nonzero size.
544 
545  if larbatch_posix.exists(path) and path[0:6] == '/pnfs/' and larbatch_posix.stat(path).st_size == 0:
546 
547  if recreate:
548  print('Adding layer two for path %s.' % path)
549  else:
550  print('Deleting empty file %s.' % path)
551 
552  # Now we got a zero size file in dCache, which kind of files may be
553  # missing layer two.
554  # Delete the file and recreate it using ifdh.
555 
556  larbatch_posix.remove(path)
557  if not recreate:
558  return
559  test_proxy()
560 
561  # Make sure environment variables X509_USER_CERT and X509_USER_KEY
562  # are not defined (they confuse ifdh).
563 
564  save_vars = {}
565  for var in ('X509_USER_CERT', 'X509_USER_KEY'):
566  if var in os.environ:
567  save_vars[var] = os.environ[var]
568  del os.environ[var]
569 
570  # Do ifdh cp.
571 
572  command = ['ifdh', 'cp', '/dev/null', path]
573  jobinfo = subprocess.Popen(command, stdout=subprocess.PIPE,
574  stderr=subprocess.PIPE)
575  q = queue.Queue()
576  thread = threading.Thread(target=wait_for_subprocess, args=[jobinfo, q])
577  thread.start()
578  thread.join(timeout=60)
579  if thread.is_alive():
580  print('Terminating subprocess.')
581  jobinfo.terminate()
582  thread.join()
583  rc = q.get()
584  jobout = convert_str(q.get())
585  joberr = convert_str(q.get())
586  if rc != 0:
587  for var in list(save_vars.keys()):
588  os.environ[var] = save_vars[var]
589  raise IFDHError(command, rc, jobout, joberr)
590 
591  # Restore environment variables.
592 
593  for var in list(save_vars.keys()):
594  os.environ[var] = save_vars[var]
595 
596 # This function returns jobsub_submit options that should be included for
597 # all batch submissions.
python.larbatch_utilities.test_proxy
def test_proxy
Definition: larbatch_utilities.py:682
print
do one_file $F done echo for F in find $TOP name CMakeLists txt print
Definition: update_sources.sh:94
python.larbatch_utilities.convert_str
def convert_str
Definition: larbatch_utilities.py:1171
python.project_utilities.addLayerTwo
def addLayerTwo
Definition: project_utilities.py:539
list
list
Definition: file_to_url.sh:28
def python.project_utilities.batch_status_check ( )

Definition at line 609 of file project_utilities.py.

609 
610 def batch_status_check():
611  return True
612 
613 # The following functions are included for backward compatibility.
614 # The actual implementations have been moved to larbatch_posix or
615 # larbatch_utilities, with a different name.
python.project_utilities.batch_status_check
def batch_status_check
Definition: project_utilities.py:609
def python.project_utilities.create_limited_dataset (   defname,
  run,
  subruns 
)

Definition at line 216 of file project_utilities.py.

216 
217 def create_limited_dataset(defname, run, subruns):
218 
219  if len(subruns) == 0:
220  return ''
221 
222  # Construct comma-separated list of run-subrun pairs in a form that is
223  # acceptable as sam dimension constraint.
224 
225  run_subrun_dim = ''
226  for subrun in subruns:
227  if run_subrun_dim != '':
228  run_subrun_dim += ','
229  run_subrun_dim += "%d.%d" % (run, subrun)
230 
231  # Construct dimension including run and subrun constraints.
232 
233  dim = "defname: %s and run_number %s" % (defname, run_subrun_dim)
234 
235  # Test the new dimension.
236 
237  nfiles = samweb().countFiles(dimensions=dim)
238  if nfiles == 0:
239  return ''
240 
241  # Make sure we have a kca certificate.
242 
243  test_kca()
244 
245  # Construct a new unique definition name.
246 
247  newdefname = defname + '_' + str(uuid.uuid4())
248 
249  # Create definition.
250 
251  samweb().createDefinition(newdefname, dim, user=get_user(), group=get_experiment())
252 
253  # Done (return definition name).
254 
255  return newdefname
256 
257 # Return initialized SAMWebClient object.
python.project_utilities.samweb
def samweb
Definition: project_utilities.py:258
python.larbatch_utilities.test_kca
def test_kca
Definition: larbatch_utilities.py:630
python.larbatch_utilities.get_experiment
def get_experiment
Definition: larbatch_utilities.py:876
python.larbatch_utilities.get_user
def get_user
Definition: larbatch_utilities.py:1005
python.project_utilities.create_limited_dataset
def create_limited_dataset
Definition: project_utilities.py:216
def python.project_utilities.default_jobsub_submit_options ( )

Definition at line 598 of file project_utilities.py.

598 
599 def default_jobsub_submit_options():
600  opt = ''
601  return opt
602 
603 # Check the health status of the batch system and any other resources that
604 # are required to submit batch jobs successfully. The idea is that this
605 # function may be called before submitting batch jobs. If this function
606 # returns false, batch jobs should not be submitted, and this failure should
607 # not be counted as an error. The default implementation here always returns
608 # true, but may be overridden in experiment_utilities.
python.project_utilities.default_jobsub_submit_options
def default_jobsub_submit_options
Definition: project_utilities.py:598
def python.project_utilities.defExists (   defname)

Definition at line 513 of file project_utilities.py.

513 
514 def defExists(defname):
515  def_exists = False
516  try:
517  samweb().descDefinition(defname)
518  def_exists = True
519  except:
520  def_exists = False
521  return def_exists
522 
523 
524 # Function to make a dummy sam dataset definition (doesn't match files) in case one doesn't exist.
python.project_utilities.samweb
def samweb
Definition: project_utilities.py:258
python.project_utilities.defExists
def defExists
Definition: project_utilities.py:513
def python.project_utilities.dollar_escape (   s)

Definition at line 162 of file project_utilities.py.

162 
163 def dollar_escape(s):
164 
165  result = ''
166  for c in s:
167  if c == '$' and ( len(result) == 0 or result[-1] != '\\'):
168  result += '\\'
169  result += c
170  return result
171 
172 
173 # Function to parse a string containing a comma- and hyphen-separated
174 # representation of a collection of positive integers into a sorted list
175 # of ints. Raise ValueError excpetion in case of unparseable string.
python.project_utilities.dollar_escape
def dollar_escape
Definition: project_utilities.py:162
def python.project_utilities.expandDefnames (   dim)

Definition at line 650 of file project_utilities.py.

650 
651 def expandDefnames(dim):
652 
653  result = ''
654  isdefname = False
655  words = dim.split()
656 
657  for word in words:
658  if isdefname:
659  isdefname = False
660  desc = samweb().descDefinitionDict(word)
661  descdim = desc['dimensions']
662 
663  # If this definition doesn't contain a top level or" or "minus" clause,
664  # leave it unexpanded.
665 
666  if descdim.find(' or ') < 0 and descdim.find(' minus ') < 0:
667  result += ' defname: %s' % word
668  else:
669  result += ' ( %s )' % desc['dimensions']
670 
671  else:
672  if word == 'defname:':
673  isdefname = True
674  else:
675  result += ' %s' % word
676 
677  return result
678 
679 # This function converts a sam dimension into a tokenized rpn list.
680 #
681 # The following kinds of tokens are recognized.
682 #
683 # 1. Grouping symbols "(", ")", "isparentof:(", "ischildof:("
684 #
685 # 2. Operators "or", "minus". Operators have equal precedence and
686 # associate from left to right.
687 #
688 # 3. "with limit N" clause (must come at end).
689 #
690 # 4. Any string expression that does not fall in above categories.
691 #
692 # The returned value of this function is a list consisting of sam dimensions,
693 # "or" and "minus" operators, and possibly a final "with limit" clause.
python.project_utilities.samweb
def samweb
Definition: project_utilities.py:258
python.project_utilities.expandDefnames
def expandDefnames
Definition: project_utilities.py:650
def python.project_utilities.fast_isdir (   path)

Definition at line 67 of file project_utilities.py.

67 
68 def fast_isdir(path):
69  result = False
70  if path[-5:] != '.list' and \
71  path[-5:] != '.root' and \
72  path[-4:] != '.txt' and \
73  path[-4:] != '.fcl' and \
74  path[-4:] != '.out' and \
75  path[-4:] != '.err' and \
76  path[-3:] != '.sh' and \
77  path[-5:] != '.stat' and \
78  larbatch_posix.isdir(path):
79  result = True
80  return result
81 
82 # Wait for file to appear on local filesystem.
python.project_utilities.fast_isdir
def fast_isdir
Definition: project_utilities.py:67
def python.project_utilities.get_scratch_dir ( )

Definition at line 115 of file project_utilities.py.

115 
116 def get_scratch_dir():
117  scratch = ''
118 
119  # Get scratch directory path.
120 
121  if 'TMPDIR' in os.environ:
122  scratch = os.environ['TMPDIR']
123 
124  elif 'SCRATCH' in os.environ:
125  scratch = os.environ['SCRATCH']
126 
127  else:
128  scratch = '/scratch/%s/%s' % (get_experiment(), get_user())
129  if not larbatch_posix.isdir(scratch) or not larbatch_posix.access(scratch, os.W_OK):
130  scratch = '/%s/data/users/%s' % (get_experiment(), get_user())
131 
132  # Checkout.
133 
134  if scratch == '':
135  raise RuntimeError('No scratch directory specified.')
136 
137  if not larbatch_posix.isdir(scratch) or not larbatch_posix.access(scratch, os.W_OK):
138  raise RuntimeError('Scratch directory %s does not exist or is not writeable.' % scratch)
139 
140  return scratch
141 
142 # Function to return the mountpoint of a given path.
python.project_utilities.get_scratch_dir
def get_scratch_dir
Definition: project_utilities.py:115
python.larbatch_utilities.get_experiment
def get_experiment
Definition: larbatch_utilities.py:876
python.larbatch_utilities.get_user
def get_user
Definition: larbatch_utilities.py:1005
def python.project_utilities.get_singularity (   name)

Definition at line 932 of file project_utilities.py.

932 
933 def get_singularity(name):
934 
935  result = ''
936  dir = '/cvmfs/singularity.opensciencegrid.org/fermilab'
937  lcname = name.lower()
938 
939  # See if the argument makes sense as a path.
940 
941  if os.path.exists(os.path.abspath(name)):
942  result = os.path.abspath(name)
943 
944  # Otherwise, try to interpret the argument as an alias.
945 
946  elif os.path.exists('%s/%s' % (dir, lcname)):
947  result = '%s/%s' % (dir, lcname)
948  elif os.path.exists('%s/fnal-wn-%s' % (dir, lcname)):
949  result = '%s/fnal-wn-%s' % (dir, lcname)
950  elif os.path.exists('%s/fnal-wn-%s:latest' % (dir, lcname)):
951  result = '%s/fnal-wn-%s:latest' % (dir, lcname)
952 
953  # Done.
954 
955  return result
python.project_utilities.get_singularity
def get_singularity
Definition: project_utilities.py:932
def python.project_utilities.listFiles (   dim)

Definition at line 800 of file project_utilities.py.

800 
801 def listFiles(dim):
802 
803  global samcache
804 
805  print('Generating completed set of files using dimension "%s".' % dim)
806 
807  # Check cache.
808 
809  if dim in samcache:
810  print('Fetching result from sam cache.')
811  return samcache[dim]
812 
813  # As a first step, expand out "defname:" clauses containing top level "or" or "minus"
814  # clauses.
815 
816  done = False
817  while not done:
818  newdim = expandDefnames(dim)
819  if newdim == dim:
820  done = True
821  else:
822  dim = newdim
823 
824  # Parse dimension into rpn list of sam dimensions and set operations.
825 
826  rpn = tokenizeRPN(dim)
827 
828  # Evaluate rpn.
829 
830  stack = []
831  for item in rpn:
832 
833  if item == 'or':
834 
835  # Take the set union of the top two items on the stack.
836 
837  set1 = stack.pop()
838  set2 = stack.pop()
839  union = set1 | set2
840  print('Set union %d files' % len(union))
841  stack.append(union)
842 
843  elif item == 'minus':
844 
845  # Take the set difference of the top two items on the stack.
846 
847  set1 = stack.pop()
848  set2 = stack.pop()
849  diff = set2 - set1
850  print('Set difference %d files' % len(diff))
851  stack.append(diff)
852 
853  elif item.startswith('with limit'):
854 
855  # Truncate set on top of stack.
856 
857  n = int(item[10:])
858  while len(stack[-1]) > n:
859  stack[-1].pop()
860  print('Truncated to %d files' % len(stack[-1]))
861 
862  else:
863 
864  # Treat this item as a sam dimension.
865  # Evaluate this dimension as a completed set, and push this set
866  # onto the stack.
867 
868  print('Evaluating "%s"' % item)
869  if item in samcache:
870  print('Fetching result from cache.')
871  files = samcache[item]
872  else:
873  files = set(samweb().listFiles(item))
874  samcache[item] = files
875  print('Result %d files' % len(files))
876  stack.append(files)
877 
878  # Done.
879 
880  print('Final result %d files' % len(stack[-1]))
881  samcache[dim] = stack[-1]
882  return stack[-1]
883 
884 # Make a sam dataset definition consisting of a list of files. The file
885 # list can be passed directly as an argument, or be evaluated by function
886 # listFiles. The name of the newly created dataset definition
887 # is returned as the return value of the function.
python.project_utilities.samweb
def samweb
Definition: project_utilities.py:258
python.project_utilities.expandDefnames
def expandDefnames
Definition: project_utilities.py:650
print
do one_file $F done echo for F in find $TOP name CMakeLists txt print
Definition: update_sources.sh:94
python.project_utilities.tokenizeRPN
def tokenizeRPN
Definition: project_utilities.py:694
python.project_utilities.listFiles
def listFiles
Definition: project_utilities.py:800
def python.project_utilities.make_active_project_dataset (   defname,
  dropboxwait,
  active_defname,
  wait_defname 
)

Definition at line 438 of file project_utilities.py.

438 
439 def make_active_project_dataset(defname, dropboxwait, active_defname, wait_defname):
440 
441  s = samweb()
442  test_kca()
443 
444  # Get list of active projects.
445 
446  prjs = active_projects(defname) | active_projects2(defname, dropboxwait)
447 
448  # Make sam dimension.
449 
450  dim = ''
451  for prj in prjs:
452  if dim == '':
453  dim = 'snapshot_for_project_name %s' % prj
454  else:
455  dim += ',%s' % prj
456 
457  # If there were no matching projects, make up some legal dimension that won't
458  # match any files.
459 
460  if dim == '':
461  dim = 'file_id 0'
462 
463  # Create or update active_defname.
464 
465  if defExists(active_defname):
466  print('Updating dataset definition %s' % active_defname)
467  s.deleteDefinition(active_defname)
468  else:
469  print('Creating dataset definition %s' % active_defname)
470 
471  s.createDefinition(active_defname, dim, user=get_user(), group=get_experiment())
472 
473  # If the dropbox waiting interval is nonzero, create a dataset for
474  # dropbox waiting files.
475 
476  dim = ''
477  if dropboxwait > 0.:
478 
479  # Convert the waiting interval to a datetime.timedelta object.
480 
481  dt = datetime.timedelta(int(dropboxwait), int(dropboxwait % 1 * 86400))
482 
483  # Get the earliest allowed time.
484 
485  tmin = datetime.datetime.utcnow() - dt
486 
487  # Format time in a form acceptable to sam.
488 
489  tminstr = tmin.strftime('%Y-%m-%dT%H:%M:%S')
490 
491  # Append sam dimension.
492 
493  dim = "isparentof: (create_date > '%s' and availability: virtual)" % tminstr
494 
495  else:
496 
497  # Otherwise make dummy dataset.
498 
499  dim = 'file_id 0'
500 
501  # Create or update active_defname.
502 
503  if defExists(wait_defname):
504  print('Updating dataset definition %s' % wait_defname)
505  s.deleteDefinition(wait_defname)
506  else:
507  print('Creating dataset definition %s' % wait_defname)
508 
509  s.createDefinition(wait_defname, dim, user=get_user(), group=get_experiment())
510 
511 
512 # Function to check whether a sam dataset definition exists.
python.project_utilities.samweb
def samweb
Definition: project_utilities.py:258
python.larbatch_utilities.test_kca
def test_kca
Definition: larbatch_utilities.py:630
python.project_utilities.defExists
def defExists
Definition: project_utilities.py:513
print
do one_file $F done echo for F in find $TOP name CMakeLists txt print
Definition: update_sources.sh:94
python.project_utilities.active_projects
def active_projects
Definition: project_utilities.py:402
python.larbatch_utilities.get_experiment
def get_experiment
Definition: larbatch_utilities.py:876
python.project_utilities.active_projects2
def active_projects2
Definition: project_utilities.py:349
python.project_utilities.make_active_project_dataset
def make_active_project_dataset
Definition: project_utilities.py:438
python.larbatch_utilities.get_user
def get_user
Definition: larbatch_utilities.py:1005
def python.project_utilities.makeDummyDef (   defname)

Definition at line 525 of file project_utilities.py.

525 
526 def makeDummyDef(defname):
527 
528  if not defExists(defname):
529 
530  # Make dummy definition.
531 
532  print('Making dummy dataset definition %s' % defname)
533  test_kca()
534  samweb().createDefinition(defname, 'file_id 0', user=get_user(), group=get_experiment())
535 
536 
537 # Function to ensure that files in dCache have layer two.
538 # This function is included here as a workaround for bugs in the dCache nfs interface.
python.project_utilities.samweb
def samweb
Definition: project_utilities.py:258
python.larbatch_utilities.test_kca
def test_kca
Definition: larbatch_utilities.py:630
python.project_utilities.defExists
def defExists
Definition: project_utilities.py:513
print
do one_file $F done echo for F in find $TOP name CMakeLists txt print
Definition: update_sources.sh:94
python.project_utilities.makeDummyDef
def makeDummyDef
Definition: project_utilities.py:525
python.larbatch_utilities.get_experiment
def get_experiment
Definition: larbatch_utilities.py:876
python.larbatch_utilities.get_user
def get_user
Definition: larbatch_utilities.py:1005
def python.project_utilities.makeFileListDefinition (   list_or_dim)

Definition at line 888 of file project_utilities.py.

888 
889 def makeFileListDefinition(list_or_dim):
890 
891  # Make sure we have a kca certificate.
892 
893  test_kca()
894 
895  # Make file list dimension.
896 
897  flist = []
898  if type(list_or_dim) == type([]) or type(list_or_dim) == type(set()):
899  flist = list_or_dim
900  print('Making file list definition from %s with %d elements.' % (type(list_or_dim),
901  len(list_or_dim)))
902  else:
903  flist = listFiles(list_or_dim)
904  print('Making file list definition using dimension "%s"' % list_or_dim)
905 
906  listdim=''
907  for filename in flist:
908  if listdim == '':
909  listdim = 'file_name %s' % filename
910  else:
911  listdim += ', %s' % filename
912  if listdim == '':
913  listdim = 'file_id 0'
914 
915  # Maybe construct a new unique definition name.
916 
917  defname = get_user() + '_filelist_' + str(uuid.uuid4())
918 
919  # Create definition.
920 
921  samweb().createDefinition(defname, listdim, user=get_user(), group=get_experiment())
922 
923  # Done.
924 
925  return defname
926 
927 # Get full path of specified singularity container image file.
928 # The argument can be an absolute or relative path of the image file,
929 # or the argument can be an alias, such as 'sl7'.
930 # Alias arguments are used to find standard Fermilab singularity images.
931 # If no image file can be found, return the empty string.
python.project_utilities.samweb
def samweb
Definition: project_utilities.py:258
python.larbatch_utilities.test_kca
def test_kca
Definition: larbatch_utilities.py:630
print
do one_file $F done echo for F in find $TOP name CMakeLists txt print
Definition: update_sources.sh:94
python.larbatch_utilities.get_experiment
def get_experiment
Definition: larbatch_utilities.py:876
python.project_utilities.listFiles
def listFiles
Definition: project_utilities.py:800
python.project_utilities.makeFileListDefinition
def makeFileListDefinition
Definition: project_utilities.py:888
python.larbatch_utilities.get_user
def get_user
Definition: larbatch_utilities.py:1005
def python.project_utilities.mountpoint (   path)

Definition at line 143 of file project_utilities.py.

143 
144 def mountpoint(path):
145 
146  # Handle symbolic links and relative paths.
147 
148  path = os.path.realpath(path)
149 
150  # Find mountpoint.
151 
152  while not os.path.ismount(path):
153  dir = os.path.dirname(path)
154  if len(dir) >= len(path):
155  return dir
156  path = dir
157 
158  return path
159 
160 
161 # Function to escape dollar signs in string by prepending backslash (\).
python.project_utilities.mountpoint
def mountpoint
Definition: project_utilities.py:143
def python.project_utilities.parseInt (   s)

Definition at line 176 of file project_utilities.py.

176 
177 def parseInt(s):
178 
179  result = set()
180 
181  # First split string into tokens separated by commas.
182 
183  for token in s.split(','):
184 
185  # Plain integers handled here.
186 
187  if token.strip().isdigit():
188  result.add(int(token))
189  continue
190 
191  # Hyphenenated ranges handled here.
192 
193  limits = token.split('-')
194  if len(limits) == 2 and limits[0].strip().isdigit() and limits[1].strip().isdigit():
195  result |= set(range(int(limits[0]), int(limits[1])+1))
196  continue
197 
198  # Don't understand.
199 
200  raise ValueError('Unparseable range token %s.' % token)
201 
202  # Return result in form of a sorted list.
203 
204  return sorted(result)
205 
206 
207 # Function to construct a new dataset definition from an existing definition
208 # such that the new dataset definition will be limited to a specified run and
209 # set of subruns.
210 #
211 # The name of the new definition is returned as the return value of
212 # the function.
213 #
214 # If the new query does not return any files, the new dataset is not created,
215 # and the function returns the empty string ('').
python.project_utilities.parseInt
def parseInt
Definition: project_utilities.py:176
gen_crt_frags.strip
def strip
Definition: gen_crt_frags.py:292
def python.project_utilities.path_to_local (   path)

Definition at line 637 of file project_utilities.py.

637 
638 def path_to_local(path):
639  return path
640 
641 # Class SafeTFile is retired. For compatibility, calls to the former
642 # constructor of class SafeTFile are now simply passed to the ROOT
643 # TFile open method. Note that class SafeTFile only ever supported
644 # opening root files for reading.
python.project_utilities.path_to_local
def path_to_local
Definition: project_utilities.py:637
def python.project_utilities.path_to_srm_url (   path)

Definition at line 616 of file project_utilities.py.

616 
617 def path_to_srm_url(path):
618  return larbatch_utilities.srm_uri(path)
python.project_utilities.path_to_srm_url
def path_to_srm_url
Definition: project_utilities.py:616
def python.project_utilities.path_to_url (   path)

Definition at line 634 of file project_utilities.py.

634 
635 def path_to_url(path):
636  return path
python.project_utilities.path_to_url
def path_to_url
Definition: project_utilities.py:634
def python.project_utilities.safecopy (   src,
  dest 
)

Definition at line 628 of file project_utilities.py.

628 
629 def safecopy(src, dest):
630  return larbatch_posix.copy(src, dest)
631 
632 # The following functions are depracated and function as no-ops.
633 # They are included for backward compatibility.
python.project_utilities.safecopy
def safecopy
Definition: project_utilities.py:628
def python.project_utilities.safeexist (   path)

Definition at line 619 of file project_utilities.py.

619 
620 def safeexist(path):
621  return larbatch_posix.exists(path)
python.project_utilities.safeexist
def safeexist
Definition: project_utilities.py:619
def python.project_utilities.saferead (   path)

Definition at line 622 of file project_utilities.py.

622 
623 def saferead(path):
624  if safeexist(path):
625  return larbatch_posix.readlines(path)
626  else:
627  return []
python.project_utilities.safeexist
def safeexist
Definition: project_utilities.py:619
python.project_utilities.saferead
def saferead
Definition: project_utilities.py:622
def python.project_utilities.SafeTFile (   path)

Definition at line 645 of file project_utilities.py.

645 
646 def SafeTFile(path):
647  return ROOT.TFile.Open(path)
648 
649 # Expand "defname:" clauses in a sam dimension.
python.project_utilities.SafeTFile
def SafeTFile
Definition: project_utilities.py:645
def python.project_utilities.samweb ( )

Definition at line 258 of file project_utilities.py.

258 
259 def samweb():
260 
261  global samweb_obj
262 
263  if samweb_obj == None:
264  samweb_obj = samweb_cli.SAMWebClient(experiment=get_experiment())
265 
266  os.environ['SSL_CERT_DIR'] = '/etc/grid-security/certificates'
267 
268  return samweb_obj
269 
270 # Start sam project.
python.project_utilities.samweb
def samweb
Definition: project_utilities.py:258
python.larbatch_utilities.get_experiment
def get_experiment
Definition: larbatch_utilities.py:876
def python.project_utilities.start_project (   defname,
  default_prjname,
  max_files,
  force_snapshot,
  filelistdef 
)

Definition at line 271 of file project_utilities.py.

271 
272 def start_project(defname, default_prjname, max_files, force_snapshot, filelistdef):
273 
274  # Check project name.
275 
276  s = samweb()
277  prjname = default_prjname
278  if prjname == '':
279  prjname = s.makeProjectName(defname)
280  print('Starting project %s' % prjname)
281 
282  # Make sure we have a certificate.
283 
284  test_kca()
285 
286  # Figure out how many files are in the input dataset.
287 
288  nf = 0
289  if filelistdef:
290  files = listFiles('defname: %s' % defname)
291  nf = len(files)
292  else:
293  nf = s.countFiles('defname: %s' % defname)
294  print('Input dataset has %d files.' % nf)
295  if nf == 0:
296  return 1
297 
298  # Make limited dataset?
299 
300  if max_files > 0 and nf > max_files:
301  limitdef = '%s_limit_%d' % (prjname, max_files)
302 
303  # Figure out whether limitdef already exists.
304 
305  if defExists(limitdef) and not filelistdef:
306  print('Using already created limited dataset definition %s.' % limitdef)
307  else:
308  dim = 'defname: %s with limit %d' % (defname, max_files)
309  if filelistdef:
310  limitdef = makeFileListDefinition(dim)
311  else:
312  print('Creating limited dataset definition %s.' % limitdef)
313  s.createDefinition(limitdef, dim, user=get_user(), group=get_experiment())
314 
315  defname = limitdef
316  nf = max_files
317 
318  elif filelistdef:
319 
320  dim = 'defname: %s' % defname
321  defname = makeFileListDefinition(dim)
322 
323  # Force snapshot?
324 
325  if force_snapshot:
326  print('Forcing snapthot.')
327  defname = '%s:force' % defname
328 
329  # Start the project.
330 
331  print('Starting project %s.' % prjname)
332  s.startProject(prjname,
333  defname=defname,
334  station=get_experiment(),
335  group=get_experiment(),
336  user=get_user())
337 
338  # Done.
339 
340  return 0
341 
342 # Return a list of active projects associated with a particular dataset definition stem
343 # based on project start and end times. The particular criteria used in this function
344 # are:
345 #
346 # 1. Project started within the last 72 hours.
347 #
348 # 2. Project no end time.
python.project_utilities.samweb
def samweb
Definition: project_utilities.py:258
python.larbatch_utilities.test_kca
def test_kca
Definition: larbatch_utilities.py:630
python.project_utilities.defExists
def defExists
Definition: project_utilities.py:513
print
do one_file $F done echo for F in find $TOP name CMakeLists txt print
Definition: update_sources.sh:94
python.larbatch_utilities.get_experiment
def get_experiment
Definition: larbatch_utilities.py:876
python.project_utilities.listFiles
def listFiles
Definition: project_utilities.py:800
python.project_utilities.makeFileListDefinition
def makeFileListDefinition
Definition: project_utilities.py:888
python.larbatch_utilities.get_user
def get_user
Definition: larbatch_utilities.py:1005
python.project_utilities.start_project
def start_project
Definition: project_utilities.py:271
def python.project_utilities.tokenizeRPN (   dim)

Definition at line 694 of file project_utilities.py.

694 
695 def tokenizeRPN(dim):
696 
697  temp = []
698  result = []
699  exp = ''
700 
701  # Split of final "with limit" clause, if any.
702 
703  head = dim
704  tail = ''
705  n = dim.find('with limit')
706  if n >= 0:
707  head = dim[:n]
708  tail = dim[n:]
709 
710  # Space out parentheses.
711 
712  head = head.replace('(', ' ( ')
713  head = head.replace(')', ' ) ')
714 
715  # But not isxxx:
716 
717  head = head.replace('isparentof: ', 'isparentof:')
718  head = head.replace('ischildof: ', 'ischildof:')
719 
720  for word in head.split():
721 
722  if word == '(' or word == 'isparentof:(' or word == 'ischildof:(':
723  if len(exp) > 0:
724  result.append(exp)
725  exp = ''
726  temp.append(word)
727 
728  elif word == 'or' or word == 'minus':
729 
730  if len(exp) > 0:
731  result.append(exp)
732  exp = ''
733 
734  done = False
735  while len(temp) > 0 and not done:
736  last = temp.pop()
737  if last == '(' or last == 'isparentof:(' or last == 'ischildof:':
738  temp.append(last)
739  done = True
740  else:
741  result.append(last)
742  temp.append(word)
743 
744  elif word == ')':
745 
746  if len(exp) > 0:
747  result.append(exp)
748  exp = ''
749 
750  done = False
751  while not done:
752  last = temp.pop()
753  if last == '(':
754  done = True
755  elif last == 'isparentof:(':
756  if len(result) == 0 or result[-1] == 'or' or result[-1] == 'minus':
757  raise RuntimeError('isparentof: parse error')
758  last = result.pop()
759  result.append('isparentof:( %s )' % last)
760  done = True
761  elif last == 'ischildof:(':
762  if len(result) == 0 or result[-1] == 'or' or result[-1] == 'minus':
763  raise RuntimeError('ischildof: parse error')
764  last = result.pop()
765  result.append('ischildof:( %s )' % last)
766  done = True
767  else:
768  result.append(last)
769 
770  else:
771  if len(exp) == 0:
772  exp = word
773  else:
774  exp += ' %s' % word
775 
776  # Clear remaining items.
777 
778  if len(exp) > 0:
779  result.append(exp)
780  while len(temp) > 0:
781  result.append(temp.pop())
782 
783  # Add final "with limit" clause, if any.
784 
785  if len(tail) > 0:
786  result.append(tail)
787 
788  return result
789 
790 
791 # This function mostly mimics the samweb listFiles function. It evaluates a sam dimension
792 # and returns a completed list of files in the form of a python set.
793 #
794 # This function exists to work around inefficiencies in the default sam implementation
795 # of listFiles by performing various set operations (set unions and set differences, as
796 # indicated sam "or" and "minus" clauses) on completed python sets, rather than as database
797 # queries.
798 #
799 # Additionally, this function caches the results of queries.
python.project_utilities.tokenizeRPN
def tokenizeRPN
Definition: project_utilities.py:694
def python.project_utilities.wait_for_stat (   path)

Definition at line 83 of file project_utilities.py.

83 
84 def wait_for_stat(path):
85 
86  ntry = 60
87  while ntry > 0:
88  if larbatch_posix.access(path, os.R_OK):
89  return 0
90  print('Waiting ...')
91 
92  # Reading the parent directory seems to make files be visible faster.
93 
94  larbatch_posix.listdir(os.path.dirname(path))
95  time.sleep(1)
96  ntry = ntry - 1
97 
98  # Timed out.
99 
100  return 1
101 
102 # Function to return the path of a scratch directory which can be used
103 # for creating large temporary files. The scratch directory should not
104 # be in dCache. The default implementation here uses the following algorithm.
105 #
106 # 1. Environment variable TMPDIR.
107 #
108 # 2. Environment variable SCRATCH.
109 #
110 # 3. Path /scratch/<experiment>/<user>
111 #
112 # 4. Path /<experiment>/data/users/<user>
113 #
114 # Raise an exception if the scratch directory doesn't exist or is not writeable.
python.project_utilities.wait_for_stat
def wait_for_stat
Definition: project_utilities.py:83
print
do one_file $F done echo for F in find $TOP name CMakeLists txt print
Definition: update_sources.sh:94

Variable Documentation

python.project_utilities.myargv = sys.argv

Definition at line 53 of file project_utilities.py.

dictionary python.project_utilities.samcache = {}

Definition at line 62 of file project_utilities.py.

python.project_utilities.samweb_obj = None

Definition at line 61 of file project_utilities.py.

Generated by   doxygen 1.8.5