All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Functions | Variables
python.root_metadata Namespace Reference

Functions

def convert_1_adler32_to_0_adler32
 
def enstoreChecksum
 
def fileEnstoreChecksum
 
def get_external_metadata
 

Variables

 myargv = sys.argv
 
 Parser = argparse.ArgumentParser\
 
string help = "JSON file to write the output to [default: screen]"
 
tuple args = Parser.parse_args()
 
tuple md = get_external_metadata(args.InputFile)
 
tuple mdtext = json.dumps(md, indent=2, sort_keys=True)
 
tuple outputFile = open(args.OutputFile, 'w')
 

Function Documentation

def python.root_metadata.convert_1_adler32_to_0_adler32 (   crc,
  filesize 
)

Definition at line 33 of file root_metadata.py.

33 
34 def convert_1_adler32_to_0_adler32(crc, filesize):
35  crc = int(crc)
36  filesize = int(filesize)
37  size = int(filesize % 65521)
38  s1 = (crc & 0xffff)
39  s2 = ((crc >> 16) & 0xffff)
40  s1 = (s1 + 65521 - 1) % 65521
41  s2 = (s2 + 65521 - size) % 65521
42  return (s2 << 16) + s1
43 
44 
45 # Checksum utilities copied from sam_web_client
python.root_metadata.convert_1_adler32_to_0_adler32
def convert_1_adler32_to_0_adler32
Definition: root_metadata.py:33
def python.root_metadata.enstoreChecksum (   fileobj)

Definition at line 46 of file root_metadata.py.

46 
47 def enstoreChecksum(fileobj):
48  import zlib
49  readblocksize = 1024*1024
50  crc = 0
51  while 1:
52  try:
53  s = fileobj.read(readblocksize)
54  except (OSError, IOError) as ex:
55  raise Error(str(ex))
56  if not s: break
57  crc = zlib.adler32(s,crc)
58  crc = int(crc)
59  if crc < 0:
60  # Return 32 bit unsigned value
61  crc = (crc & 0x7FFFFFFF) | 0x80000000
62  return { "crc_value" : str(crc), "crc_type" : "adler 32 crc type" }
python.root_metadata.enstoreChecksum
def enstoreChecksum
Definition: root_metadata.py:46
def python.root_metadata.fileEnstoreChecksum (   path) 
Calculate enstore compatible CRC value

Definition at line 63 of file root_metadata.py.

63 
64 def fileEnstoreChecksum(path):
65  """Calculate enstore compatible CRC value"""
66 
67  crc = {}
68  srm_url = project_utilities.path_to_srm_url(path)
69 
70  if srm_url == path:
71  try:
72  f = larbatch_posix.open(path,'rb')
73  crc = enstoreChecksum(f)
74  except (IOError, OSError) as ex:
75  raise Error(str(ex))
76  finally:
77  f.close()
78  else:
79  try:
80  # Following commented commands are old way of calculating checksum by
81  # transferring entire file over network.
82  # Should work again if uncommented (if srm way breaks).
83 
84  #cmd = ['ifdh', 'cp', path, '/dev/fd/1']
85  #p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
86  #f = p.stdout
87  #crc = enstoreChecksum(f)
88 
89  # New (clever, efficient, obscure...) way of accessing dCache
90  # stored checksum using srm.
91  project_utilities.test_proxy()
92  cmd = ['srmls', '-2', '-l', srm_url]
93  srmout = convert_str(subprocess.check_output(cmd))
94  first = True
95  crc0 = 0
96  for line in srmout.split('\n'):
97  if first:
98  size = int(line[2:line.find('/')-1])
99  first = False
100  continue
101  if line.find("Checksum value:") > 0:
102  ssum = line[line.find(':') + 2:]
103  crc1 = int( ssum , base = 16 )
104  crc0 = convert_1_adler32_to_0_adler32(crc1, size)
105  break
106 
107  crc = {"crc_value": str(crc0), "crc_type": "adler 32 crc type"}
108 
109  except:
110  # Try the old method
111  cmd = ['ifdh', 'cp', path, '/dev/fd/1']
112  p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
113  f = p.stdout
114  crc = enstoreChecksum(f)
115  return crc
python.root_metadata.convert_1_adler32_to_0_adler32
def convert_1_adler32_to_0_adler32
Definition: root_metadata.py:33
python.root_metadata.enstoreChecksum
def enstoreChecksum
Definition: root_metadata.py:46
python.larbatch_utilities.convert_str
def convert_str
Definition: larbatch_utilities.py:1171
python.root_metadata.fileEnstoreChecksum
def fileEnstoreChecksum
Definition: root_metadata.py:63
def python.root_metadata.get_external_metadata (   inputfile)

Definition at line 116 of file root_metadata.py.

116 
117 def get_external_metadata(inputfile):
118 
119  # define an empty python dictionary
120  md = {}
121 
122  # Check whether this file exists.
123  if not os.path.exists(inputfile):
124  return md
125 
126  # Get the other meta data field parameters
127  md['file_name'] = os.path.basename(inputfile)
128  md['file_size'] = str(os.path.getsize(inputfile))
129  md['crc'] = fileEnstoreChecksum(inputfile)
130 
131  # Quit here if file type is not ".root"
132 
133  if not inputfile.endswith('.root'):
134  return md
135 
136  # Root checks.
137 
138  ROOT.gEnv.SetValue('RooFit.Banner', '0')
139  file = ROOT.TFile.Open(larbatch_posix.root_stream(inputfile))
140  if file and file.IsOpen() and not file.IsZombie():
141 
142  # Root file opened successfully.
143  # Get number of events.
144 
145  obj = file.Get('Events')
146  if obj and obj.InheritsFrom('TTree'):
147 
148  # This has a TTree named Events.
149 
150  nev = obj.GetEntriesFast()
151  md['events'] = str(nev)
152 
153  # Get runs and subruns fro SubRuns tree.
154 
155  subrun_tree = file.Get('SubRuns')
156  if subrun_tree and subrun_tree.InheritsFrom('TTree'):
157  md['subruns'] = []
158  nsubruns = subrun_tree.GetEntriesFast()
159  tfr = ROOT.TTreeFormula('subruns',
160  'SubRunAuxiliary.id_.run_.run_',
161  subrun_tree)
162  tfs = ROOT.TTreeFormula('subruns',
163  'SubRunAuxiliary.id_.subRun_',
164  subrun_tree)
165  for entry in range(nsubruns):
166  subrun_tree.GetEntry(entry)
167  run = tfr.EvalInstance64()
168  subrun = tfs.EvalInstance64()
169  run_subrun = (run, subrun)
170  if not run_subrun in md['subruns']:
171  md['subruns'].append(run_subrun)
172 
173  # Get stream name.
174 
175  try:
176  stream_name = stream.get_stream(inputfile)
177  md['data_stream'] = stream_name
178  except:
179  pass
180 
181  return md
python.root_metadata.get_external_metadata
def get_external_metadata
Definition: root_metadata.py:116
python.root_metadata.fileEnstoreChecksum
def fileEnstoreChecksum
Definition: root_metadata.py:63

Variable Documentation

tuple python.root_metadata.args = Parser.parse_args()

Definition at line 194 of file root_metadata.py.

string python.root_metadata.help = "JSON file to write the output to [default: screen]"

Definition at line 191 of file root_metadata.py.

tuple python.root_metadata.md = get_external_metadata(args.InputFile)

Definition at line 196 of file root_metadata.py.

tuple python.root_metadata.mdtext = json.dumps(md, indent=2, sort_keys=True)

Definition at line 197 of file root_metadata.py.

python.root_metadata.myargv = sys.argv

Definition at line 16 of file root_metadata.py.

tuple python.root_metadata.outputFile = open(args.OutputFile, 'w')

Definition at line 199 of file root_metadata.py.

python.root_metadata.Parser = argparse.ArgumentParser\

Definition at line 186 of file root_metadata.py.

Generated by   doxygen 1.8.5