util::MetadataSBN Class Reference

#include <MetadataSBN.h>

Classes
struct	metadata

Public Member Functions
	MetadataSBN (fhicl::ParameterSet const &pset, art::ActivityRegistry &reg)
void	GetMetadataMaps (std::map< std::string, std::string > &strs, std::map< std::string, int > &ints, std::map< std::string, double > &doubles, std::map< std::string, std::string > &objs)

Public Attributes
metadata	md
std::set< art::SubRunID >	fSubRunNumbers

Private Member Functions
void	postBeginJob ()
void	postOpenInputFile (std::string const &fn)
void	postEvent (art::Event const &ev, art::ScheduleContext)
void	postBeginSubRun (art::SubRun const &subrun)
void	postEndSubRun (art::SubRun const &subrun)
void	postCloseInputFile ()
std::string	GetParentsString () const
std::string	GetRunsString () const

Private Attributes
std::map< std::string, std::string >	mdmapStr
std::map< std::string, int >	mdmapInt
std::map< std::string, double >	mdmapDouble
std::map< std::string, std::string >	mdmapObj
std::string	fExperiment
std::string	frunType
std::vector< std::string >	fJSONFileName
std::vector< std::string >	fDataTier
std::vector< std::string >	fFileFormat
std::vector< int >	fMerge
std::string	fPOTModuleLabel
art::FileStatsCollector	fFileStats
art::PostCloseFileRenamer	fRenamer {fFileStats}

Detailed Description

Definition at line 32 of file MetadataSBN.h.

Constructor & Destructor Documentation

util::MetadataSBN::MetadataSBN	(	fhicl::ParameterSet const &	pset,
		art::ActivityRegistry &	reg
	)

Definition at line 84 of file MetadataSBN_service.cc.

                                                        :
  fFileStats{"", art::ServiceHandle<art::TriggerNamesService const>{}->getProcessName()}

Member Function Documentation

void util::MetadataSBN::GetMetadataMaps	(	std::map< std::string, std::string > &	strs,
		std::map< std::string, int > &	ints,
		std::map< std::string, double > &	doubles,
		std::map< std::string, std::string > &	objs
	)

Definition at line 310 of file MetadataSBN_service.cc.

{
  strs.clear(); ints.clear(); doubles.clear(); objs.clear();

  objs["application"] = "{\"family\": \""+std::get<0>(md.fapplication)+"\", \"name\": \""+std::get<1>(md.fapplication)+"\", \"version\": \""+std::get<2>(md.fapplication)+"\"}";

  if(!md.fParents.empty()) objs["parents"] = GetParentsString();
  if(!md.fruns.empty()) objs["runs"] = GetRunsString();

  // convert start and end times into time format: Year-Month-DayTHours:Minutes:Seconds
  char endbuf[80], startbuf[80];
  struct tm tstruct;
  tstruct = *localtime(&md.fend_time);
  strftime(endbuf,sizeof(endbuf),"%Y-%m-%dT%H:%M:%S",&tstruct);
  tstruct = *localtime(&md.fstart_time);
  strftime(startbuf,sizeof(startbuf),"%Y-%m-%dT%H:%M:%S",&tstruct);

  strs["start_time"] = startbuf;
  strs["end_time"] = endbuf;

  strs["data_tier"] = md.fdata_tier;
  ints["event_count"] = md.fevent_count;
  strs["file_format"] = md.ffile_format;
  ints["first_event"] = md.ffirst_event;
  ints["last_event"] = md.flast_event;

  const std::string proj = fExperiment+"_project";
  MaybeCopyToMap(md.fFCLName, "fcl.name", strs);
  MaybeCopyToMap(md.fProjectName, proj+".name", strs);
  MaybeCopyToMap(md.fProjectStage, proj+".stage", strs);
  MaybeCopyToMap(md.fProjectVersion, proj+".version", strs);
  MaybeCopyToMap(md.fProjectSoftware, proj+".software", strs);
  MaybeCopyToMap(md.fProductionName, "production.name", strs);
  MaybeCopyToMap(md.fProductionType, "production.type", strs);

  MaybeCopyToMap(md.fgroup, "group", strs);
  MaybeCopyToMap(md.ffile_type, "file_type", strs);

  if(md.merge >= 0) {
    ints["merge.merge"] = (md.merge==0 ? 0 : 1);
    ints["merge.merged"] = 0;
  }
  doubles["mc.pot"] = md.fTotPOT;  
}

std::string util::MetadataSBN::GetParentsString ( ) const

private

Definition at line 272 of file MetadataSBN_service.cc.

{
  if(md.fParents.empty()) return "";

  unsigned int c = 0;

  std::string ret = "[\n";
  for(auto parent: md.fParents) {
    std::cout<<"Parent " << c << ": " << parent << std::endl;
    c++;
    size_t n = parent.find_last_of('/');
    size_t f1 = (n == std::string::npos ? 0 : n+1);
    ret += "    {\n     \"file_name\": \"" + parent.substr(f1) + "\"\n    }";
    if(md.fParents.size() == 1 || c == md.fParents.size()) ret += "\n";
    else ret += ",\n";
  }

  ret += "  ]";
  return ret;
}

std::string util::MetadataSBN::GetRunsString ( ) const

private

Definition at line 294 of file MetadataSBN_service.cc.

{
  unsigned int c = 0;

  std::string ret = "[\n";
  for(auto&t :md.fruns){
    c++;
    ret += "    [\n     " + std::to_string(std::get<0>(t)) + ",\n     " + std::to_string(std::get<1>(t)) + ",\n     \"" + std::get<2>(t) + "\"\n    ]";
    if(md.fruns.size() == 1 || c == md.fruns.size()) ret += "\n";
    else ret += ",\n";
  }
  ret += "  ]";
  return ret;
}

void util::MetadataSBN::postBeginJob ( )

private

Definition at line 174 of file MetadataSBN_service.cc.

{
  // get the start time
  md.fstart_time = time(0);

  // Get art metadata service and extract paramters from there
  art::ServiceHandle<art::FileCatalogMetadata> artmds;

  art::FileCatalogMetadata::collection_type artmd;
  artmds->getMetadata(artmd);

  std::map<std::string, std::string> mdmap;
  for(const auto& d: artmd)
    mdmap[d.first] = UnQuoteString(d.second);

  // if a certain paramter/key is not found, assign an empty string value to it
  MaybeCopyFromMap(mdmap, "application.family",  std::get<0>(md.fapplication));
  MaybeCopyFromMap(mdmap, "art.process_name",    std::get<1>(md.fapplication));
  MaybeCopyFromMap(mdmap, "application.version", std::get<2>(md.fapplication));
  MaybeCopyFromMap(mdmap, "group",        md.fgroup);
  MaybeCopyFromMap(mdmap, "file_type",    md.ffile_type);
  MaybeCopyFromMap(mdmap, "art.run_type", frunType);
}

void util::MetadataSBN::postBeginSubRun ( art::SubRun const &  subrun )

private

Definition at line 234 of file MetadataSBN_service.cc.

{
  art::RunNumber_t run = sr.run();
  art::SubRunNumber_t subrun = sr.subRun();
  art::SubRunID srid = sr.id();

  // save run, subrun and runType information once every subrun
  if (fSubRunNumbers.count(srid) == 0){
    fSubRunNumbers.insert(srid);
    md.fruns.push_back(make_tuple(run, subrun, frunType));
  }
}

void util::MetadataSBN::postCloseInputFile ( )

private

Definition at line 360 of file MetadataSBN_service.cc.

{
  //update end time
  md.fend_time = time(0);

  // Loop over files.

  for(unsigned int i=0; i<fJSONFileName.size(); ++i) {

    // Update per-file metadata.

    md.fdata_tier = fDataTier[i];
    md.ffile_format = fFileFormat[i];
    if(fMerge.size() > i)
      md.merge = fMerge[i];
    else
      md.merge = -1;


    std::map<std::string, std::string> strs;
    std::map<std::string, int> ints;
    std::map<std::string, double> doubles;
    std::map<std::string, std::string> objs;
    GetMetadataMaps(strs, ints, doubles, objs);

    // open a json file and write everything from the struct md complying to the
    // samweb json format. This json file holds the below information temporarily.
    // If you submitted a grid job invoking this service, the information from
    // this file is appended to a final json file and this file will be removed

    if(!fJSONFileName[i].empty()){
      std::ofstream jsonfile;
      jsonfile.open(fJSONFileName[i]);
      jsonfile << "{\n";

      bool once = true;
      for(auto& it: objs){
        if(!once) jsonfile << ",\n";
        once = false;
        jsonfile << "  \"" << it.first << "\": " << it.second;
      }
      for(auto& it: strs){
        // Have to escape string outputs
        jsonfile << ",\n  \"" << it.first << "\": \"" << it.second << "\"";
      }
      for(auto& it: ints){
        jsonfile << ",\n  \"" << it.first << "\": " << it.second;
      }
      for(auto& it: doubles){
        jsonfile << ",\n  \"" << it.first << "\": " << it.second;
      }

      jsonfile<<"\n}\n";
      jsonfile.close();
    }
  }

  fFileStats.recordFileClose();
  //TODO figure out how to make the name identical to the TFile
  //std::string new_name = fRenamer.maybeRenameFile("myjson.json",fJSONFileName);
}

void util::MetadataSBN::postEndSubRun ( art::SubRun const &  subrun )

private

Definition at line 249 of file MetadataSBN_service.cc.

{
  art::Handle< sumdata::POTSummary > potListHandle;
  double fTotPOT = 0;
  if(sr.getByLabel(fPOTModuleLabel,potListHandle)){
    fTotPOT+=potListHandle->totpot;
  }

  md.fTotPOT += fTotPOT;
}

void util::MetadataSBN::postEvent ( art::Event const &  ev, art::ScheduleContext    )

private

Definition at line 211 of file MetadataSBN_service.cc.

{
  art::RunNumber_t run = evt.run();
  art::SubRunNumber_t subrun = evt.subRun();
  art::EventNumber_t event = evt.event();
  art::SubRunID srid = evt.id().subRunID();

  // save run, subrun and runType information once every subrun
  if (fSubRunNumbers.count(srid) == 0){
    fSubRunNumbers.insert(srid);
    md.fruns.push_back(make_tuple(run, subrun, frunType));
  }

  // save the first event
  if (md.fevent_count == 0) md.ffirst_event = event;
  md.flast_event = event;
  // event counter
  ++md.fevent_count;

}

void util::MetadataSBN::postOpenInputFile ( std::string const &  fn )

private

Definition at line 201 of file MetadataSBN_service.cc.

{
  // save parent input files here
  // 08/06 DBrailsford: Only save the parent string if the string is filled.  The string still exists (with 0 characters) for generation stage files.  See redmine issue 20124
  if (fn.length() > 0) md.fParents.insert(fn);
  fFileStats.recordInputFile(fn);
}

Member Data Documentation

std::vector<std::string> util::MetadataSBN::fDataTier

private

Definition at line 93 of file MetadataSBN.h.

std::string util::MetadataSBN::fExperiment

private

Definition at line 90 of file MetadataSBN.h.

std::vector<std::string> util::MetadataSBN::fFileFormat

private

Definition at line 94 of file MetadataSBN.h.

art::FileStatsCollector util::MetadataSBN::fFileStats

private

Definition at line 97 of file MetadataSBN.h.

std::vector<std::string> util::MetadataSBN::fJSONFileName

private

Definition at line 92 of file MetadataSBN.h.

std::vector<int> util::MetadataSBN::fMerge

private

Definition at line 95 of file MetadataSBN.h.

std::string util::MetadataSBN::fPOTModuleLabel

private

Definition at line 96 of file MetadataSBN.h.

art::PostCloseFileRenamer util::MetadataSBN::fRenamer {fFileStats}

private

Definition at line 98 of file MetadataSBN.h.

std::string util::MetadataSBN::frunType

private

Definition at line 91 of file MetadataSBN.h.

std::set<art::SubRunID> util::MetadataSBN::fSubRunNumbers

Definition at line 64 of file MetadataSBN.h.

metadata util::MetadataSBN::md

Definition at line 63 of file MetadataSBN.h.

std::map<std::string, double> util::MetadataSBN::mdmapDouble

private

Definition at line 86 of file MetadataSBN.h.

std::map<std::string, int> util::MetadataSBN::mdmapInt

private

Definition at line 85 of file MetadataSBN.h.

std::map<std::string, std::string> util::MetadataSBN::mdmapObj

private

Definition at line 87 of file MetadataSBN.h.

std::map<std::string,std::string> util::MetadataSBN::mdmapStr

private

Definition at line 84 of file MetadataSBN.h.

---

The documentation for this class was generated from the following files:

• MetadataSBN.h
• MetadataSBN_service.cc