All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Classes | Functions | Variables
python.larbatch_posix Namespace Reference

Classes

class  dcache_file
 

Functions

def use_grid
 
def open
 
def readlines
 
def copy
 
def listdir
 
def exists
 
def isdir
 
def stat
 
def access
 
def walk
 
def mkdir
 
def makedirs
 
def rename
 
def remove
 
def rmdir
 
def rmtree
 
def chmod
 
def symlink
 
def readlink
 
def root_stream
 

Variables

tuple pnfs_is_mounted = os.path.isdir('/pnfs')
 
string prefer_grid = 'LARBATCH_GRID'
 
string debug = 'LARBATCH_DEBUG'
 

Function Documentation

def python.larbatch_posix.access (   path,
  mode 
)

Definition at line 538 of file larbatch_posix.py.

538 
539 def access(path, mode):
540 
541  result = False
542  if path.startswith('/pnfs/') and (prefer_grid or not pnfs_is_mounted):
543  if debug:
544  print('*** Larbatch_posix: Check access for %s using ifdh.' % path)
545  sr = stat(path)
546  if sr.st_mode != 0:
547 
548  # File exists.
549 
550  result = True
551 
552  # Test permissions.
553 
554  if mode & os.R_OK:
555  if not sr.st_mode & statmod.S_IRUSR:
556  result = False
557  if mode & os.W_OK:
558  if not sr.st_mode & statmod.S_IWUSR:
559  result = False
560  if mode & os.X_OK:
561  if not sr.st_mode & statmod.S_IXUSR:
562  result = False
563 
564  else:
565  if debug:
566  print('*** Larbatch_posix: Check access for %s using posix.' % path)
567  result = os.access(path, mode)
568 
569  # Done.
570 
571  return result
572 
573 
574 # Walk directory tree. Like os.walk, this function returns an iterator over
575 # 3-tuples, one for each directory in the tree rooted in the specified
576 # top directory. Each 3-tuple contains the following information.
577 #
578 # 1. Path of directory.
579 # 2. List of dictory names in this directory.
580 # 3. List of non-directory files in this directory.
581 #
582 # In case of posix mode, this function includes its own implementation
583 # of the walking algorithm so that we can take advantage of optimzations
584 # contained in this module's implementation of isdir.
python.larbatch_posix.stat
def stat
Definition: larbatch_posix.py:482
print
do one_file $F done echo for F in find $TOP name CMakeLists txt print
Definition: update_sources.sh:94
python.larbatch_posix.access
def access
Definition: larbatch_posix.py:538
def python.larbatch_posix.chmod (   path,
  mode 
)

Definition at line 804 of file larbatch_posix.py.

804 
805 def chmod(path, mode):
806  if path.startswith('/pnfs/') and (prefer_grid or not pnfs_is_mounted):
807  if debug:
808  print('*** Larbatch_posix: Change mode for %s using ifdh.' % path)
809  larbatch_utilities.ifdh_chmod(path, mode)
810  else:
811  if debug:
812  print('*** Larbatch_posix: Change mode for %s using posix.' % path)
813  try:
814  os.chmod(path, mode)
815  except:
816  print('Warning: chmod failed for path %s' % path)
817 
818 
819 # Make symbolic link.
820 # Use nfs.
python.larbatch_posix.chmod
def chmod
Definition: larbatch_posix.py:804
print
do one_file $F done echo for F in find $TOP name CMakeLists txt print
Definition: update_sources.sh:94
def python.larbatch_posix.copy (   src,
  dest 
)

Definition at line 294 of file larbatch_posix.py.

294 
295 def copy(src, dest):
296  if exists(dest):
297  remove(dest)
298  if (src.startswith('/pnfs/') or dest.startswith('/pnfs/')):
299  if prefer_grid or not pnfs_is_mounted:
300  if debug:
301  print('*** Larbatch_posix: Copy %s to %s using ifdh.' % (src, dest))
302  larbatch_utilities.ifdh_cp(src, dest)
303  else:
304  if debug:
305  print('*** Larbatch_posix: Copy %s to %s using posix with timeout.' % (src, dest))
306  larbatch_utilities.posix_cp(src, dest)
307  else:
308  if debug:
309  print('*** Larbatch_posix: Copy %s to %s using posix.' % (src, dest))
310  shutil.copy(src, dest)
311 
312  # Done
313 
314  return
315 
316 
317 # List directory contents.
print
do one_file $F done echo for F in find $TOP name CMakeLists txt print
Definition: update_sources.sh:94
python.larbatch_posix.exists
def exists
Definition: larbatch_posix.py:379
python.larbatch_posix.remove
def remove
Definition: larbatch_posix.py:715
python.larbatch_posix.copy
def copy
Definition: larbatch_posix.py:294
def python.larbatch_posix.exists (   path)

Definition at line 379 of file larbatch_posix.py.

379 
380 def exists(path):
381 
382  result = False
383  if path.startswith('/pnfs/') and (prefer_grid or not pnfs_is_mounted):
384  if debug:
385  print('*** Larbatch_posix: Check existence of %s using ifdh.' % path)
386 
387  # Do "ifdh ls."
388 
389  try:
390  larbatch_utilities.ifdh_ls(path, 0)
391  result = True
392  except:
393  result = False
394 
395  else:
396  if debug:
397  print('*** Larbatch_posix: Check existence of %s using posix.' % path)
398  #result = os.path.exists(path)
399 
400  # In order to reduce hang risk from stat'ing file,
401  # check existence by getting contents of parent directory.
402 
403  npath = os.path.normpath(path) # Strip trailing '/'.
404  dir = os.path.dirname(npath)
405  base = os.path.basename(npath)
406  if dir == '':
407  dir = '.'
408  if isdir(dir):
409  files = listdir(dir)
410  for filename in files:
411  if base == filename:
412  result = True
413 
414  # Done.
415 
416  return result
417 
418 
419 # Test existence if directory.
print
do one_file $F done echo for F in find $TOP name CMakeLists txt print
Definition: update_sources.sh:94
python.larbatch_posix.isdir
def isdir
Definition: larbatch_posix.py:420
python.larbatch_posix.exists
def exists
Definition: larbatch_posix.py:379
python.larbatch_posix.listdir
def listdir
Definition: larbatch_posix.py:318
def python.larbatch_posix.isdir (   path)

Definition at line 420 of file larbatch_posix.py.

420 
421 def isdir(path):
422 
423  result = False
424 
425  # Optimizations for speed and to reduce hang risk by not stat'ing every file.
426 
427  if path[-5:] == '.list' or \
428  path[-5:] == '.root' or \
429  path[-5:] == '.json' or \
430  path[-4:] == '.txt' or \
431  path[-4:] == '.fcl' or \
432  path[-4:] == '.out' or \
433  path[-4:] == '.err' or \
434  path[-3:] == '.sh' or \
435  path[-5:] == '.stat':
436  return False
437 
438  if path.startswith('/pnfs/') and (prefer_grid or not pnfs_is_mounted):
439  if debug:
440  print('*** Larbatch_posix: Check existence of directory %s using ifdh.' % path)
441 
442  # Make sure path exists before trying to determine if it is a directory.
443 
444  if exists(path):
445 
446  # The only reliable way to get information about a directory is
447  # to get a listing of the parent directory.
448 
449  npath = os.path.normpath(path) # Strip trailing '/'
450  name = os.path.basename(npath)
451  dir = os.path.dirname(npath)
452  lines = larbatch_utilities.ifdh_ll(dir, 1)
453  for line in lines:
454  words = line.split()
455  if len(words) > 5 and words[-1] == name:
456  if words[0][0] == 'd':
457  result = True
458 
459 
460  else:
461  if debug:
462  print('*** Larbatch_posix: Check existence of directory %s using posix.' % path)
463  result = os.path.isdir(path)
464 
465  # Done.
466 
467  return result
468 
469 
470 # Get file status.
471 #
472 # This function is a partial emulation of os.stat. The return value
473 # is a type os.stat_result, which is a 10-tuple of values. The following
474 # values are filled by this function, since information is not always
475 # available from grid tools.
476 #
477 # mode - File type and permissions.
478 # uid - Owner uid. For compatibility, always set to the process uid.
479 # gid - Owner gid. For compatibility, always set to the process gid.
480 # nlink - Number of links.
481 # size - Object size.
print
do one_file $F done echo for F in find $TOP name CMakeLists txt print
Definition: update_sources.sh:94
python.larbatch_posix.isdir
def isdir
Definition: larbatch_posix.py:420
python.larbatch_posix.exists
def exists
Definition: larbatch_posix.py:379
def python.larbatch_posix.listdir (   path)

Definition at line 318 of file larbatch_posix.py.

318 
319 def listdir(path):
320 
321  if not isdir(path):
322  raise OSError('%s is not a directory.' % path)
323  result = []
324  if path.startswith('/pnfs/') and (prefer_grid or not pnfs_is_mounted):
325  if debug:
326  print('*** Larbatch_posix: Listdir %s using ifdh.' % path)
327 
328  # Get normalized tail.
329 
330  tail = os.path.normpath(path[-6:])
331 
332  # Call "ifdh ls".
333 
334  contents = larbatch_utilities.ifdh_ls(path, 1)
335 
336  # Loop over contents returned by ifdh.
337  # Normalize the paths returned by "ifdh ls", which in this context mainly
338  # has the effect of stripping off trailing '/' on directories.
339  # Filter out parent directory, which ifdh sometimes (usually?) includes in result.
340 
341  for c in contents:
342  nc = os.path.normpath(c.strip())
343  if not nc.endswith(tail):
344  result.append(os.path.basename(nc))
345 
346  else:
347  if debug:
348  print('*** Larbatch_posix: Listdir %s using posix.' % path)
349  #result = os.listdir(path)
350 
351  # To reduce hang risk, read contents of directory in a subprocess with
352  # a timeout.
353 
354  cmd = ['ls', path]
355  jobinfo = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
356 
357  q = queue.Queue()
358  thread = threading.Thread(target=larbatch_utilities.wait_for_subprocess, args=[jobinfo, q])
359  thread.start()
360  thread.join(timeout=60)
361  if thread.is_alive():
362  if debug:
363  print('*** Larbatch_posix: Terminating subprocess.')
364  jobinfo.terminate()
365  thread.join()
366  rc = q.get()
367  jobout = convert_str(q.get())
368  joberr = convert_str(q.get())
369  if rc == 0:
370  for word in jobout.split():
371  result.append(word)
372 
373  # Done.
374 
375  return result
376 
377 
378 # Test existence. Works for files and directories.
print
do one_file $F done echo for F in find $TOP name CMakeLists txt print
Definition: update_sources.sh:94
python.larbatch_posix.isdir
def isdir
Definition: larbatch_posix.py:420
python.larbatch_posix.listdir
def listdir
Definition: larbatch_posix.py:318
python.larbatch_utilities.convert_str
def convert_str
Definition: larbatch_utilities.py:1171
def python.larbatch_posix.makedirs (   path,
  mode = 0o777 
)

Definition at line 658 of file larbatch_posix.py.

658 
659 def makedirs(path, mode=0o777):
660  if path.startswith('/pnfs/') and (prefer_grid or not pnfs_is_mounted):
661  if debug:
662  print('*** Larbatch_posix: Make directory recursively for %s using ifdh.' % path)
663 
664  # Make sure parent directory exists.
665 
666  np = os.path.normpath(path) # Stip trailing '/', if present.
667  parent = os.path.dirname(np)
668  if not isdir(parent):
669  makedirs(parent, mode)
670 
671  # Now make directory itself.
672 
673  larbatch_utilities.ifdh_mkdir(path)
674  else:
675  if debug:
676  print('*** Larbatch_posix: Make directory recursively for %s using posix.' % path)
677  os.makedirs(path, mode)
678 
679 
680 # Rename file.
681 # "ifdh mv" seems to be buggy, so use uberftp.
print
do one_file $F done echo for F in find $TOP name CMakeLists txt print
Definition: update_sources.sh:94
python.larbatch_posix.isdir
def isdir
Definition: larbatch_posix.py:420
python.larbatch_posix.makedirs
def makedirs
Definition: larbatch_posix.py:658
def python.larbatch_posix.mkdir (   path,
  mode = 0o777 
)

Definition at line 642 of file larbatch_posix.py.

642 
643 def mkdir(path, mode=0o777):
644  if path.startswith('/pnfs/') and (prefer_grid or not pnfs_is_mounted):
645  if debug:
646  print('*** Larbatch_posix: Make directory for %s using ifdh.' % path)
647  larbatch_utilities.ifdh_mkdir(path)
648  else:
649  if debug:
650  print('*** Larbatch_posix: Make directory for %s using posix.' % path)
651  os.mkdir(path, mode)
652 
653 
654 # Make directory and parents.
655 # Mode argument is ignored for dCache files, but is passed to os.mkdir
656 # for non-dCache files.
657 # "ifdh mkdir_p" is buggy, so we do the recursion locally.
print
do one_file $F done echo for F in find $TOP name CMakeLists txt print
Definition: update_sources.sh:94
python.larbatch_posix.mkdir
def mkdir
Definition: larbatch_posix.py:642
def python.larbatch_posix.open (   path,
  mode = 'r',
  buf = -1 
)

Definition at line 275 of file larbatch_posix.py.

275 
276 def open(path, mode='r', buf=-1):
277  if path.startswith('/pnfs/') and (prefer_grid or not pnfs_is_mounted):
278  if debug:
279  print('*** Larbatch_posix: Opening dcache_file %s using mode %s.' % (path, mode))
280  return dcache_file(path, mode, buf)
281  else:
282  if debug:
283  print('*** Larbatch_posix: Opening normal file %s using mode %s.' % (path, mode))
284  return __builtins__['open'](path, mode, buf)
285 
286 
287 # Read lines from a file.
print
do one_file $F done echo for F in find $TOP name CMakeLists txt print
Definition: update_sources.sh:94
python.larbatch_posix.dcache_file
Definition: larbatch_posix.py:156
python.larbatch_posix.open
def open
Definition: larbatch_posix.py:275
def python.larbatch_posix.readlines (   path)

Definition at line 288 of file larbatch_posix.py.

288 
289 def readlines(path):
290  return open(path).readlines()
291 
292 
293 # Copy file.
python.larbatch_posix.readlines
def readlines
Definition: larbatch_posix.py:288
python.larbatch_posix.open
def open
Definition: larbatch_posix.py:275
def python.larbatch_posix.readlink (   path)

Definition at line 856 of file larbatch_posix.py.

856 
857 def readlink(path):
858 
859  result = ''
860 
861  # Make sure we have a kerberos ticket.
862 
863  if path.startswith('/pnfs/') and not_pnfs_is_mounted:
864  if debug:
865  print('*** Larbatch_posix: Read symbolic link %s using nfs server.' % path)
866  larbatch_utilities.test_ticket()
867  cmd = ['ssh', larbatch_utilities.nfs_server(), 'readlink', path]
868  jobinfo = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
869 
870  q = queue.Queue()
871  thread = threading.Thread(target=larbatch_utilities.wait_for_subprocess, args=[jobinfo, q])
872  thread.start()
873  thread.join(timeout=60)
874  if thread.is_alive():
875  if debug:
876  print('*** Larbatch_posix: Terminating subprocess.')
877  jobinfo.terminate()
878  thread.join()
879  rc = q.get()
880  jobout = convert_str(q.get())
881  joberr = convert_str(q.get())
882  if rc != 0:
883  raise IFDHError(cmd, rc, jobout, joberr)
884  result = jobout.strip()
885 
886  else:
887  if debug:
888  print('*** Larbatch_posix: Read symbolic link %s using posix.' % path)
889  result = os.readlink(path)
890 
891  # Done.
892 
893  return result
894 
895 
896 # Convert a root file path to a streamable path or uri that can be opened
897 # using TFile::Open.
898 # Non-dCache paths (paths not starting with '/pnfs/') are not changed.
899 # dCache paths (patsh starting with '/pnfs/') may be converted to an xrootd uri.
print
do one_file $F done echo for F in find $TOP name CMakeLists txt print
Definition: update_sources.sh:94
python.larbatch_posix.readlink
def readlink
Definition: larbatch_posix.py:856
python.larbatch_utilities.convert_str
def convert_str
Definition: larbatch_utilities.py:1171
def python.larbatch_posix.remove (   path)

Definition at line 715 of file larbatch_posix.py.

715 
716 def remove(path):
717  if path.startswith('/pnfs/') and (prefer_grid or not pnfs_is_mounted):
718  if debug:
719  print('*** Larbatch_posix: Delete file %s using ifdh.' % path)
720  larbatch_utilities.ifdh_rm(path)
721  else:
722  if debug:
723  print('*** Larbatch_posix: Delete file %s using posix.' % path)
724 
725  # Deleting a file is a hang risk, especially, but not only, in dCache.
726  # Therefore, use the following procedure.
727  #
728  # 1. Rename file to a random name (this can usually be done, even
729  # for undeletable files).
730  #
731  # 2. Delete renamed file in a subprocess. No need to wait for
732  # subprocess to finish, or check its exit status.
733 
734  #os.remove(path)
735  newpath = path + '_' + str(uuid.uuid4())
736  try:
737  os.rename(path, newpath)
738  os.system('rm -f %s &' % newpath)
739  return
740  except:
741  pass
742  os.remove(path)
743 
744 # Delete empty directory.
print
do one_file $F done echo for F in find $TOP name CMakeLists txt print
Definition: update_sources.sh:94
python.larbatch_posix.remove
def remove
Definition: larbatch_posix.py:715
def python.larbatch_posix.rename (   src,
  dest 
)

Definition at line 682 of file larbatch_posix.py.

682 
683 def rename(src, dest):
684  if (src.startswith('/pnfs/') or
685  dest.startswith('/pnfs/')) and (prefer_grid or not pnfs_is_mounted):
686  if debug:
687  print('*** Larbatch_posix: Rename %s to %s using ifdh.' % (src, dest))
688 
689  src_uri = larbatch_utilities.gridftp_uri(src)
690  dest_path = larbatch_utilities.dcache_path(dest)
691  cmd = ['uberftp', '-rename', src_uri, dest_path]
692  jobinfo = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
693 
694  q = queue.Queue()
695  thread = threading.Thread(target=larbatch_utilities.wait_for_subprocess, args=[jobinfo, q])
696  thread.start()
697  thread.join(timeout=60)
698  if thread.is_alive():
699  if debug:
700  print('*** Larbatch_posix: Terminating subprocess.')
701  jobinfo.terminate()
702  thread.join()
703  rc = q.get()
704  jobout = convert_str(q.get())
705  joberr = convert_str(q.get())
706  if rc != 0:
707  raise IFDHError(cmd, rc, jobout, joberr)
708  else:
709  if debug:
710  print('*** Larbatch_posix: Rename %s to %s using posix.' % (src, dest))
711  os.rename(src, dest)
712 
713 
714 # Delete file.
print
do one_file $F done echo for F in find $TOP name CMakeLists txt print
Definition: update_sources.sh:94
python.larbatch_posix.rename
def rename
Definition: larbatch_posix.py:682
python.larbatch_utilities.convert_str
def convert_str
Definition: larbatch_utilities.py:1171
def python.larbatch_posix.rmdir (   path)

Definition at line 745 of file larbatch_posix.py.

745 
746 def rmdir(path):
747  if path.startswith('/pnfs/') and (prefer_grid or not pnfs_is_mounted):
748  if debug:
749  print('*** Larbatch_posix: Delete directoroy %s using ifdh.' % path)
750  larbatch_utilities.ifdh_rmdir(path)
751  else:
752  if debug:
753  print('*** Larbatch_posix: Delete directoroy %s using posix.' % path)
754  os.rmdir(path)
755 
756 
757 # Delete directory tree.
python.larbatch_posix.rmdir
def rmdir
Definition: larbatch_posix.py:745
print
do one_file $F done echo for F in find $TOP name CMakeLists txt print
Definition: update_sources.sh:94
def python.larbatch_posix.rmtree (   path)

Definition at line 758 of file larbatch_posix.py.

758 
759 def rmtree(path):
760  if path.startswith('/pnfs/') and (prefer_grid or not pnfs_is_mounted):
761  if debug:
762  print('*** Larbatch_posix: Delete directoroy tree %s using ifdh.' % path)
763 
764  # Delete contents recursively.
765 
766  lines = larbatch_utilities.ifdh_ll(path, 1)
767  for line in lines:
768  words = line.split()
769  if len(words) > 5:
770  if words[0][0] == 'd':
771  rmtree(os.path.join(path, words[-1]))
772  else:
773  remove(os.path.join(path, words[-1]))
774 
775  # Directory should be empty when we get to here.
776 
777  rmdir(path)
778 
779  else:
780  if debug:
781  print('*** Larbatch_posix: Delete directoroy tree %s using posix.' % path)
782 
783  # Deleting a directory tree is a hang risk, especially, but not only, in dCache.
784  # Therefore, use the following procedure.
785  #
786  # 1. Rename directory to a random name (this can usually be done, even
787  # for undeletable directories).
788  #
789  # 2. Delete renamed directory in a subprocess. No need to wait for
790  # subprocess to finish, or check its exit status.
791 
792  #shutil.rmtree(path)
793  npath = os.path.normpath(path) # Strip trailing '/'
794  newpath = npath + '_' + str(uuid.uuid4())
795  os.rename(npath, newpath)
796  os.system('rm -rf %s &' % newpath)
797 
798  # Done
799 
800  return
801 
802 
803 # Change mode.
python.larbatch_posix.rmdir
def rmdir
Definition: larbatch_posix.py:745
print
do one_file $F done echo for F in find $TOP name CMakeLists txt print
Definition: update_sources.sh:94
python.larbatch_posix.rmtree
def rmtree
Definition: larbatch_posix.py:758
python.larbatch_posix.remove
def remove
Definition: larbatch_posix.py:715
def python.larbatch_posix.root_stream (   path)

Definition at line 900 of file larbatch_posix.py.

900 
901 def root_stream(path):
902 
903  stream = path
904  if path.startswith('/pnfs/') and (prefer_grid or not pnfs_is_mounted):
905  if debug:
906  print('*** Larbatch_posix: Stream path %s using xrootd.' % path)
907  larbatch_utilities.test_proxy()
908  stream = larbatch_utilities.xrootd_uri(path)
909  else:
910  if debug:
911  print('*** Larbatch_posix: Stream path %s as normal file.' % path)
912  return stream
print
do one_file $F done echo for F in find $TOP name CMakeLists txt print
Definition: update_sources.sh:94
python.larbatch_posix.root_stream
def root_stream
Definition: larbatch_posix.py:900
def python.larbatch_posix.stat (   path)

Definition at line 482 of file larbatch_posix.py.

482 
483 def stat(path):
484 
485  result = None
486  if path.startswith('/pnfs/') and (prefer_grid or not pnfs_is_mounted):
487  if debug:
488  print('*** Larbatch_posix: Stat %s using ifdh.' % path)
489 
490  # The only reliable way to get information about a directory is
491  # to get a listing of the parent directory.
492 
493  npath = os.path.normpath(path) # Strip trailing '/'
494  name = os.path.basename(npath)
495  dir = os.path.dirname(npath)
496  lines = larbatch_utilities.ifdh_ll(dir, 1)
497  for line in lines:
498  words = line.split()
499  if len(words) >5 and words[-1] == name:
500 
501  # Found thie path. Fill result.
502 
503  # Interpret mode string.
504 
505  mode = larbatch_utilities.parse_mode(words[0])
506 
507  # Get remaining fields.
508 
509  nlinks = int(words[1])
510  size = int(words[4])
511  result = os.stat_result((mode, # Mode
512  0, # Inode
513  0, # Device
514  nlinks, # Number of links
515  os.getuid(), # Uid
516  os.getgid(), # Gid
517  size, # Size
518  0, # Access time
519  0, # Mod time
520  0)) # Creation time
521 
522  else:
523  if debug:
524  print('*** Larbatch_posix: Stat %s using posix.' % path)
525  result = os.stat(path)
526 
527  if result == None:
528  raise OSError('No such file or directory.')
529 
530  # Done.
531 
532  return result
533 
534 
535 # Test file access.
536 # This implementation only tests access to dCache files via the
537 # user permission, which is a limitation of grid tools.
python.larbatch_posix.stat
def stat
Definition: larbatch_posix.py:482
print
do one_file $F done echo for F in find $TOP name CMakeLists txt print
Definition: update_sources.sh:94
def python.larbatch_posix.symlink (   src,
  dest 
)

Definition at line 821 of file larbatch_posix.py.

821 
822 def symlink(src, dest):
823 
824  # Make sure we have a kerberos ticket.
825 
826  if src.startswith('/pnfs/') and not pnfs_is_mounted:
827  if debug:
828  print('*** Larbatch_posix: Make symbolic link from %s to %s using nfs server.' % (src, dest))
829  larbatch_utilities.test_ticket()
830  cmd = ['ssh', larbatch_utilities.nfs_server(), 'ln', '-s', src, dest]
831  jobinfo = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
832 
833  q = queue.Queue()
834  thread = threading.Thread(target=larbatch_utilities.wait_for_subprocess, args=[jobinfo, q])
835  thread.start()
836  thread.join(timeout=60)
837  if thread.is_alive():
838  if debug:
839  print('*** Larbatch_posix: Terminating subprocess.')
840  jobinfo.terminate()
841  thread.join()
842  rc = q.get()
843  jobout = convert_str(q.get())
844  joberr = convert_str(q.get())
845  if rc != 0:
846  raise IFDHError(cmd, rc, jobout, joberr)
847 
848  else:
849  if debug:
850  print('*** Larbatch_posix: Make symbolic link from %s to %s using posix.' % (src, dest))
851  os.symlink(src, dest)
852 
853 
854 # Read a symbolic link.
855 # Use nfs.
print
do one_file $F done echo for F in find $TOP name CMakeLists txt print
Definition: update_sources.sh:94
python.larbatch_utilities.convert_str
def convert_str
Definition: larbatch_utilities.py:1171
python.larbatch_posix.symlink
def symlink
Definition: larbatch_posix.py:821
def python.larbatch_posix.use_grid (   force = True)

Definition at line 150 of file larbatch_posix.py.

150 
151 def use_grid(force=True):
152  prefer_grid = force
153 
154 
155 # File-like class for dCache files.
python.larbatch_posix.use_grid
def use_grid
Definition: larbatch_posix.py:150
def python.larbatch_posix.walk (   top,
  topdown = True 
)

Definition at line 585 of file larbatch_posix.py.

585 
586 def walk(top, topdown=True):
587 
588  # Quit if top directory doesn't exist.
589 
590  if not exists(top):
591  return
592 
593  # Get contents of top directory using either ifdh or posix.
594 
595  dirs = []
596  files = []
597  if top.startswith('/pnfs/') and (prefer_grid or not pnfs_is_mounted):
598  if debug:
599  print('*** Larbatch_posix: Walk directory tree for %s using ifdh.' % top)
600 
601  # Retrieve the contents of this directory using ifdh.
602 
603  lines = larbatch_utilities.ifdh_ll(top, 1)
604  for line in lines:
605  words = line.split()
606  if len(words) > 5 and words[-1] != '__pycache__':
607  if words[0][0] == 'd':
608  dirs.append(words[-1])
609  else:
610  files.append(words[-1])
611  else:
612  if debug:
613  print('*** Larbatch_posix: Walk directory tree for %s using posix.' % top)
614  contents = listdir(top)
615  for obj in contents:
616  if obj != '__pycache__':
617  if isdir(os.path.join(top, obj)):
618  dirs.append(obj)
619  else:
620  files.append(obj)
621 
622  if topdown:
623  yield top, dirs, files
624 
625  # Recursively descend into subdirectories.
626 
627  for dir in dirs:
628  for result in walk(os.path.join(top, dir), topdown):
629  yield result
630 
631  if not topdown:
632  yield top, dirs, files
633 
634  # Done.
635 
636  return
637 
638 
639 # Make directory (parent directory must exist).
640 # Mode argument is ignored for dCache files, but is passed to os.mkdir
641 # for non-dCache files.
print
do one_file $F done echo for F in find $TOP name CMakeLists txt print
Definition: update_sources.sh:94
python.larbatch_posix.walk
def walk
Definition: larbatch_posix.py:585
python.larbatch_posix.isdir
def isdir
Definition: larbatch_posix.py:420
python.larbatch_posix.exists
def exists
Definition: larbatch_posix.py:379
python.larbatch_posix.listdir
def listdir
Definition: larbatch_posix.py:318

Variable Documentation

string python.larbatch_posix.debug = 'LARBATCH_DEBUG'

Definition at line 142 of file larbatch_posix.py.

tuple python.larbatch_posix.pnfs_is_mounted = os.path.isdir('/pnfs')

Definition at line 140 of file larbatch_posix.py.

string python.larbatch_posix.prefer_grid = 'LARBATCH_GRID'

Definition at line 141 of file larbatch_posix.py.

Generated by   doxygen 1.8.5