Create smaller CAFs. More...

#include <FileReducer.h>

Inheritance diagram for ana::FileReducer:

Public Types
typedef void(	ReductionFunc )(caf::StandardRecord *)

Public Member Functions
	FileReducer (const std::string &wildcard, const std::string &outfile)

	FileReducer (const std::vector< std::string > &fnames, const std::string &outfile)

virtual	~FileReducer ()

void	AddSpillCut (const SpillCut &cut)
	Only copy records to the output file if they pass this cut. More...

void	AddSliceCut (const SliceCut &cut)

void	SetEventList (const std::string &fname)
	If called, only events whose run/subrun/event occur in fname will be retained. More...

void	AddReductionStep (const std::function< ReductionFunc > &f)
	Run the specified reduction function over each event. More...

void	SetMetadata (const std::string &key, const std::string &val)
	Override any metadata key in the output file. More...

virtual void	Go () override
	Load all the registered spectra. More...

Protected Member Functions
void	HandleFile (TFile fin, TFile fout, TTree &trOut, Progress prog, long &nRecSeen, long &nRecPassed)

void	HandleNestedTree (TFile fout, TTree recTree, TTree &trOut, Progress prog, long &nRecSeen, long &nRecPassed)

void	HandleFlatTree (TFile fout, TTree recTree, TTree &trOut, Progress prog, long &nRecSeen, long &nRecPassed)

void	CopyGlobalTree (TFile fin, TFile fout)

void	UpdateMetadata (std::map< std::string, std::string > &meta, const std::set< std::string > &mask, const std::vector< std::string > &fnames) const

void	Huskify (caf::StandardRecord *sr) const
	Strip all information out of this record and tag it as a husk. More...

Protected Member Functions inherited from ana::SpectrumLoaderBase
	SpectrumLoaderBase (DataSource src=kBeam)
	Component of other constructors. More...

	SpectrumLoaderBase (const std::string &wildcard, DataSource src=kBeam)
	Construct from a filename, wildcard, SAM definition, or SAM query. More...

	SpectrumLoaderBase (const std::vector< std::string > &fnames, DataSource src=kBeam)
	Construct from an explicit list of files. More...

	SpectrumLoaderBase (SpectrumLoaderBase &&)=default

SpectrumLoaderBase &	operator= (SpectrumLoaderBase &&)=default

	SpectrumLoaderBase (const SpectrumLoaderBase &)=delete

SpectrumLoaderBase &	operator= (const SpectrumLoaderBase &)=delete

IFileSource *	WildcardOrSAMQuery (const std::string &str) const
	Figure out if str is a wildcard or SAM query and return a source. More...

virtual void	RemoveSpectrum (Spectrum *)

virtual void	RemoveReweightableSpectrum (ReweightableSpectrum *)

int	NFiles () const
	Forwards to fFileSource. More...

TFile *	GetNextFile ()

virtual	~SpectrumLoaderBase ()

virtual void	AddSpectrum (Spectrum &spect, const Var &var, const SpillCut &spillcut, const Cut &cut, const SystShifts &shift, const Var &wei=kUnweighted)
	For use by the Spectrum constructor. More...

virtual void	AddSpectrum (Spectrum &spect, const MultiVar &var, const SpillCut &spillcut, const Cut &cut, const SystShifts &shift, const Var &wei=kUnweighted)
	For use by the Spectrum constructor. More...

virtual void	AddSpectrum (Spectrum &spect, const SpillVar &var, const SpillCut &cut, const SpillVar &wei=kSpillUnweighted)
	For use by the Spectrum constructor. More...

virtual void	AddSpectrum (Spectrum &spect, const SpillMultiVar &var, const SpillCut &cut, const SpillVar &wei=kSpillUnweighted)
	For use by the Spectrum constructor. More...

virtual void	AddReweightableSpectrum (ReweightableSpectrum &spect, const Var &var, const Cut &cut, const SystShifts &shift, const Var &wei)
	For use by the constructors of ReweightableSpectrum subclasses. More...

virtual void	AddReweightableSpectrum (ReweightableSpectrum &spect, const Var &var, const SpillCut &spillcut, const SliceCut &slicecut, const SystShifts &shift, const Var &wei)
	For use by the constructors of ReweightableSpectrum subclasses. More...

virtual bool	Gone () const
	Indicate whether or not Go has been called. More...

Protected Attributes
std::string	fOutfile

SpillCut *	fSpillCut

SliceCut *	fSliceCut

std::set< std::tuple< int, int, int > >	fEventList

std::vector< std::function < ReductionFunc > >	fReductionFuncs

std::map< std::string, std::string >	fMetaMap

Protected Attributes inherited from ana::SpectrumLoaderBase
std::string	fWildcard

std::unique_ptr< IFileSource >	fFileSource

DataSource	fSource

bool	fGone
	Has Go() been called? Can't add more histograms after that. More...

double	fPOT

double	fPOTFromHist
	Accumulated by calls to GetNextFile. More...

int	fNReadouts

IDMap< SpillCut, IDMap < SystShifts, IDMap< Cut, IDMap< Var, IDMap < VarOrMultiVar, SpectList > > > > >	fHistDefs
	All the spectra that need to be filled. More...

IDMap< SpillCut, IDMap < SpillVar, IDMap < SpillVarOrMultiVar, SpectList > > >	fSpillHistDefs
	[spillcut][spillwei][spillvar] More...

Additional Inherited Members
Protected Types inherited from ana::SpectrumLoaderBase
typedef _VarOrMultiVar < caf::SRSliceProxy >	VarOrMultiVar

typedef _VarOrMultiVar < caf::SRSpillProxy >	SpillVarOrMultiVar

Detailed Description

Create smaller CAFs.

This class produces new CAFs, removing entries that fail a cut. It also allows the event record to be edited in custom ways.

Definition at line 19 of file FileReducer.h.

Member Typedef Documentation

typedef void( ana::FileReducer::ReductionFunc)(caf::StandardRecord *)

Definition at line 36 of file FileReducer.h.

Constructor & Destructor Documentation

ana::FileReducer::FileReducer	(	const std::string &	wildcard,
		const std::string &	outfile
	)

Definition at line 32 of file FileReducer.cxx.

     : SpectrumLoaderBase(wildcard),
       fOutfile(outfile),
       fSpillCut(nullptr), fSliceCut(nullptr)
   {
   }

ana::FileReducer::FileReducer	(	const std::vector< std::string > &	fnames,
		const std::string &	outfile
	)

Definition at line 41 of file FileReducer.cxx.

     : SpectrumLoaderBase(fnames),
       fOutfile(outfile),
       fSpillCut(nullptr),
       fSliceCut(nullptr)
   {
   }

ana::FileReducer::~FileReducer ( )

virtual

Definition at line 51 of file FileReducer.cxx.

   {
     delete fSpillCut;
     delete fSliceCut;
   }

Member Function Documentation

void ana::FileReducer::AddReductionStep ( const std::function< ReductionFunc > & f )

inline

Run the specified reduction function over each event.

Definition at line 41 of file FileReducer.h.

41 {fReductionFuncs.push_back(f);}

ana::FileReducer::fReductionFuncs

std::vector< std::function< ReductionFunc > > fReductionFuncs

Definition: FileReducer.h:80

void ana::FileReducer::AddSliceCut ( const SliceCut & cut )

Definition at line 69 of file FileReducer.cxx.

   {
     if(fSliceCut){
       *fSliceCut = *fSliceCut && cut;
     }
     else{
       fSliceCut = new SliceCut(cut);
     }
   }

void ana::FileReducer::AddSpillCut ( const SpillCut & cut )

Only copy records to the output file if they pass this cut.

Definition at line 58 of file FileReducer.cxx.

   {
     if(fSpillCut){
       *fSpillCut = *fSpillCut && cut;
     }
     else{
       fSpillCut = new SpillCut(cut);
     }
   }

void ana::FileReducer::CopyGlobalTree	(	TFile *	fin,
		TFile *	fout
	)

protected

Definition at line 322 of file FileReducer.cxx.

   {
     TTree* globalIn = (TTree*)fin->Get("globalTree");
     if(!globalIn) return;
 
     // Copy globalTree verbatim from input to output
     caf::SRGlobal global;
     caf::SRGlobal* pglobal = &global;
     globalIn->SetBranchAddress("global", &pglobal);
     fout->cd();
     TTree globalOut("globalTree", "globalTree");
     globalOut.Branch("global", "caf::SRGlobal", &global);
     assert(globalIn->GetEntries() == 1);
     // TODO check that the globalTree is the same in every file
     globalIn->GetEntry(0);
     globalOut.Fill();
     globalOut.Write();
   }

void ana::FileReducer::Go ( )

overridevirtual

Load all the registered spectra.

Implements ana::SpectrumLoaderBase.

Definition at line 95 of file FileReducer.cxx.

   {
     //    FloatingExceptionOnNaN fpnan;
 
     // Don't want overflow to happen. Set to 1 petabyte: effectively infinite.
     TTree::SetMaxTreeSize(1e15);
 
     if(fGone){
       std::cerr << "Error: can only call Go() once on a FileReducer" << std::endl;
       abort();
     }
     fGone = true;
 
     const int Nfiles = NFiles();
 
     Progress prog(TString::Format("Filling from %d files matching '%s'", Nfiles, fWildcard.c_str()).Data());
 
     TFile fout(fOutfile.c_str(), "RECREATE");
     TTree* trOut = 0; // created in first call to HandleFile()
 
     TH1* hPOTOut = new TH1F("TotalPOT", "", 1, 0, 1);
     TH1* hEventsOut = new TH1F("TotalEvents", "", 1, 0, 1);
 
     std::vector<std::string> fnames;
 
     std::map<std::string, std::string> meta;
     std::set<std::string> meta_mask;
 
     long nRecSeen = 0;
     long nRecPassed = 0;
 
     int fileIdx = -1;
     while(TFile* fin = GetNextFile()){
       ++fileIdx;
 
       assert(!fin->IsZombie());
 
       fnames.push_back(fin->GetName());
 
       TH1* hPOT = (TH1*)fin->Get("TotalPOT");
       assert(hPOT);
       hPOTOut->Add(hPOT);
 
       TDirectory* meta_dir = (TDirectory*)fin->Get("metadata");
       assert(meta_dir);
       CombineMetadata(meta, GetCAFMetadata(meta_dir), meta_mask);
 
       TH1* hEvents = (TH1*)fin->Get("TotalEvents");
       assert(hEvents);
       hEventsOut->Add(hEvents);
 
       HandleFile(fin, &fout, trOut,
                  Nfiles == 1 ? &prog : 0,
                  nRecSeen, nRecPassed);
 
       if(fileIdx == 0) CopyGlobalTree(fin, &fout);
 
       prog.SetProgress((fileIdx+1.)/Nfiles);
     } // end while GetNextFile
 
     fout.cd();
     trOut->Write();
     hPOTOut->Write();
     hEventsOut->Write();
 
     UpdateMetadata(meta, meta_mask, fnames);
     WriteCAFMetadata(fout.mkdir("metadata"), meta);
 
     fout.Close();
 
     prog.Done();
 
     std::cout << "Passed " << nRecPassed << " / " << nRecSeen << " records";
     std::cout << std::endl;
   }

void ana::FileReducer::HandleFile	(	TFile *	fin,
		TFile *	fout,
		TTree *&	trOut,
		Progress *	prog,
		long &	nRecSeen,
		long &	nRecPassed
	)

protected

Definition at line 172 of file FileReducer.cxx.

   {
     TTree* recTree = (TTree*)fin->Get("recTree");
     assert(recTree);
 
     const caf::CAFType type = caf::GetCAFType(recTree);
 
     if(trOut){
       const caf::CAFType outtype = caf::GetCAFType(trOut);
       if(type != outtype){
         std::cerr << "FileReducer: Error: dataset contains mixed CAF types (flat vs nested)" << std::endl;
         abort();
       }
     }
 
     if(type == caf::kNested) HandleNestedTree(fout, recTree, trOut,
                                               prog,
                                               nRecSeen, nRecPassed);
     else if(type == caf::kFlat) HandleFlatTree(fout, recTree, trOut,
                                                prog,
                                                nRecSeen, nRecPassed);
 
     else{
       std::cerr << "FileReducer: Error: Unrecognized file type: "
                 << fin->GetName() << std::endl;
       abort();
     }
   }

void ana::FileReducer::HandleFlatTree	(	TFile *	fout,
		TTree *	recTree,
		TTree *&	trOut,
		Progress *	prog,
		long &	nRecSeen,
		long &	nRecPassed
	)

protected

Definition at line 282 of file FileReducer.cxx.

   {
     if(!fEventList.empty()){
       std::cerr << "FileReducer: Event list not supported for FlatCAFs (yet)" << std::endl;
       abort();
     }
 
     if(fSpillCut){
       std::cerr << "FileReducer: Spill cuts not supported for FlatCAFs" << std::endl;
       abort();
     }
 
     if(fSliceCut){
       std::cerr << "FileReducer: Slice cuts not supported for FlatCAFs" << std::endl;
       abort();
     }
 
     if(!fReductionFuncs.empty()){
       std::cerr << "FileReducer: Reduction functions not supported for FlatCAFs" << std::endl;
       abort();
     }
 
     // Do the actual copy
     if(!trOut){
       fout->cd();
       trOut = recTree->CloneTree();
     }
     else{
       fout->cd();
       recTree->CopyAddresses(trOut);
       trOut->CopyEntries(recTree);
     }
 
     nRecSeen += recTree->GetEntries();
     nRecPassed += recTree->GetEntries();
   }

void ana::FileReducer::HandleNestedTree	(	TFile *	fout,
		TTree *	recTree,
		TTree *&	trOut,
		Progress *	prog,
		long &	nRecSeen,
		long &	nRecPassed
	)

protected

Do we need to include the event? Either based on the selection...

Definition at line 204 of file FileReducer.cxx.

   {
     if(!trOut){
       fout->cd();
       trOut = new TTree("recTree", "recTree");
       //      FloatingExceptionOnNaN fpnan(false);
       caf::StandardRecord dummy;
       trOut->Branch("rec", &dummy);
     }
 
     // Use this one for assessing cuts etc
     caf::Proxy<caf::StandardRecord> srProxy(recTree, "rec");
 
     // And if they pass load into this one for writing out
     caf::StandardRecord* sr = 0;
     recTree->SetBranchAddress("rec", &sr);
 
     caf::StandardRecord* oldsr = 0;
 
     const int Nentries = recTree->GetEntries();
     for(int n = 0; n < Nentries; ++n){
       ++nRecSeen;
       recTree->LoadTree(n);
 
       // Apply EventList cut if it's been enabled
       if(!fEventList.empty() &&
          !fEventList.count(std::make_tuple(srProxy.hdr.run,
                                            srProxy.hdr.subrun,
                                            srProxy.hdr.evt))) continue;
 
       /// Do we need to include the event? Either based on the selection...
       const bool passesSpillCut = !fSpillCut || (*fSpillCut)(&srProxy);
       // Or if it carries POT or spill counting information
       if(passesSpillCut || srProxy.hdr.pot > 0 || srProxy.hdr.ngenevt > 0){
         recTree->GetEntry(n);
 
         if(sr != oldsr){
           trOut->SetBranchAddress("rec", &sr);
           oldsr = sr;
         }
 
         if(!passesSpillCut){
           // We're only keeping this one for the exposure info
           Huskify(sr);
           trOut->Fill();
           continue;
         }
         // Otherwise, need to proceed to keep/reject individual slices
 
         std::vector<int> tocut;
         for(unsigned int i = 0; i < srProxy.slc.size(); ++i){
           if(fSliceCut && !(*fSliceCut)(&srProxy.slc[i])) tocut.push_back(i);
         }
 
         // Remove slices in reverse order so that the indices remain valid
         for(auto it = tocut.rbegin(); it != tocut.rend(); ++it){
           sr->slc.erase(sr->slc.begin() + *it);
         }
 
         // Apply any additional reduction steps
         for(const auto& f: fReductionFuncs) f(sr);
         // This is kind of problematic since the proxy and actual record
         // could be out of sync. Let's just disable this option for now.
         //          for(const auto & f: fReductionFuncsWithProxy) f(sr, &srProxy);
 
         ++nRecPassed;
         trOut->Fill();
       }
 
       // prog won't be passed if the caller doesn't want per-event updates
       if(n%100 == 0 && prog) prog->SetProgress(double(n)/Nentries);
     } // end for n
   }

void ana::FileReducer::Huskify ( caf::StandardRecord * sr ) const

protected

Strip all information out of this record and tag it as a husk.

Definition at line 384 of file FileReducer.cxx.

   {
     sr->hdr.husk = true;
 
     // It's not actually necessary for correctness to clear all these fields
     // out, so not a disaster if we happen to miss one that gets added in
     // future, for example. But it does help to minimize the size of the
     // resulting CAF.
 
     sr->slc.clear();
     sr->nslc = 0;
     sr->fake_reco.clear();
     sr->nfake_reco = 0;
     sr->true_particles.clear();
     sr->ntrue_particles = 0;
     sr->crt_hits.clear();
     sr->ncrt_hits = 0;
     sr->crt_tracks.clear();
     sr->ncrt_tracks = 0;
 
     sr->reco.trk.clear();
     sr->reco.ntrk = 0;
     sr->reco.shw.clear();
     sr->reco.nshw = 0;
     sr->reco.stub.clear();
     sr->reco.nstub = 0;
 
     sr->mc.nu.clear();
     sr->mc.nnu = 0;
     sr->mc.prtl.clear();
     sr->mc.nprtl = 0;
   }

void ana::FileReducer::SetEventList ( const std::string & fname )

If called, only events whose run/subrun/event occur in fname will be retained.

Definition at line 80 of file FileReducer.cxx.

   {
     FILE* f = fopen(fname.c_str(), "r");
     assert(f);
 
     while(!feof(f)){
       int run, subrun, event;
       fscanf(f, "%d %d %d", &run, &subrun, &event);
       fEventList.emplace(run, subrun, event);
     }
 
     fclose(f);
   }

void ana::FileReducer::SetMetadata	(	const std::string &	key,
		const std::string &	val
	)

inline

Override any metadata key in the output file.

Definition at line 45 of file FileReducer.h.

     {
       // Let's assume the user is just setting string keys
       fMetaMap[key] = "\""+val+"\"";
     }

void ana::FileReducer::UpdateMetadata	(	std::map< std::string, std::string > &	meta,
		const std::set< std::string > &	mask,
		const std::vector< std::string > &	fnames
	)		const

protected

Definition at line 342 of file FileReducer.cxx.

   {
     for(const std::string& m: mask){
       std::cerr << "Warning: metadata parameter '" << m << "' differs between input files and has been dropped from the output." << std::endl;
       meta.erase(m);
     }
 
     /*
     // change caf -> decaf in the metadata field, if parents have data_tier
     if(meta.find("data_tier") != meta.end()){
       std::string decaf_tier = meta["data_tier"];
       assert(decaf_tier.size() >= 3);
       // For indexing: note that all of this is surrounded by quotes
       assert(decaf_tier.substr(decaf_tier.size()-4,3) == "caf");
       // don't make 'decaf' into 'dedecaf', however
       if (decaf_tier.size() < 6 || decaf_tier.substr(decaf_tier.size()-6,5) != "decaf")
         decaf_tier.replace(decaf_tier.size()-4,3,"decaf");
       meta["data_tier"] = decaf_tier;
     }
     */
 
     std::string parents = "[";
     for(const std::string& f: fnames){
       parents += "{\"file_name\": \""+std::string(basename((char *)f.c_str()))+"\"}, ";
     }
     if(!fnames.empty()) parents.resize(parents.size()-2); // drop trailing ", "
     parents += "]";
 
     meta["parents"] = parents;
 
     // if there's one more than one parent, this is a concat
     if(fnames.size() > 1 && meta.find("data_tier") != meta.end()){
       meta["data_tier"] = "\"concat_caf\"";
     }
 
     // Allow user to override any metadata
     for(auto it: fMetaMap) meta[it.first] = it.second;
   }

Member Data Documentation

std::set<std::tuple<int, int, int> > ana::FileReducer::fEventList

protected

Definition at line 78 of file FileReducer.h.

std::map<std::string, std::string> ana::FileReducer::fMetaMap

protected

Definition at line 83 of file FileReducer.h.

std::string ana::FileReducer::fOutfile

protected

Definition at line 74 of file FileReducer.h.

std::vector<std::function<ReductionFunc> > ana::FileReducer::fReductionFuncs

protected

Definition at line 80 of file FileReducer.h.

SliceCut* ana::FileReducer::fSliceCut

protected

Definition at line 76 of file FileReducer.h.

SpillCut* ana::FileReducer::fSpillCut

protected

Definition at line 75 of file FileReducer.h.

The documentation for this class was generated from the following files:

Public Types

Public Member Functions

Protected Member Functions

Protected Attributes

Additional Inherited Members

Detailed Description

Member Typedef Documentation

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation