diff --git a/parameters/config.xml b/parameters/config.xml index 01df49d..17d4f2b 100644 --- a/parameters/config.xml +++ b/parameters/config.xml @@ -1,226 +1,229 @@ - + - + + + + - + Using z-expansion - + diff --git a/src/FCN/JointFCN.cxx b/src/FCN/JointFCN.cxx index 6e01efd..fcd5fac 100755 --- a/src/FCN/JointFCN.cxx +++ b/src/FCN/JointFCN.cxx @@ -1,1128 +1,1126 @@ #include "JointFCN.h" #include "FitUtils.h" #include //*************************************************** JointFCN::JointFCN(TFile *outfile) { //*************************************************** fOutputDir = gDirectory; if (outfile) Config::Get().out = outfile; std::vector samplekeys = Config::QueryKeys("sample"); LoadSamples(samplekeys); std::vector covarkeys = Config::QueryKeys("covar"); LoadPulls(covarkeys); fCurIter = 0; fMCFilled = false; fIterationTree = false; fDialVals = NULL; fNDials = 0; fUsingEventManager = FitPar::Config().GetParB("EventManager"); fOutputDir->cd(); } //*************************************************** JointFCN::JointFCN(std::vector samplekeys, TFile *outfile) { //*************************************************** fOutputDir = gDirectory; if (outfile) Config::Get().out = outfile; LoadSamples(samplekeys); fCurIter = 0; fMCFilled = false; fOutputDir->cd(); fIterationTree = false; fDialVals = NULL; fNDials = 0; fUsingEventManager = FitPar::Config().GetParB("EventManager"); fOutputDir->cd(); } //*************************************************** JointFCN::~JointFCN() { //*************************************************** // Delete Samples for (MeasListConstIter iter = fSamples.begin(); iter != fSamples.end(); iter++) { MeasurementBase *exp = *iter; delete exp; } for (PullListConstIter iter = fPulls.begin(); iter != fPulls.end(); iter++) { ParamPull *pull = *iter; delete pull; } // Sort Tree if (fIterationTree) DestroyIterationTree(); if (fDialVals) delete fDialVals; if (fSampleLikes) delete fSampleLikes; }; //*************************************************** void JointFCN::CreateIterationTree(std::string name, FitWeight *rw) { //*************************************************** LOG(FIT) << " Creating new iteration container! " << std::endl; DestroyIterationTree(); fIterationTreeName = name; // Add sample likelihoods and ndof for (MeasListConstIter iter = fSamples.begin(); iter != fSamples.end(); iter++) { MeasurementBase *exp = *iter; std::string name = exp->GetName(); std::string liketag = name + "_likelihood"; fNameValues.push_back(liketag); fCurrentValues.push_back(0.0); std::string ndoftag = name + "_ndof"; fNameValues.push_back(ndoftag); fCurrentValues.push_back(0.0); } // Add Pull terms for (PullListConstIter iter = fPulls.begin(); iter != fPulls.end(); iter++) { ParamPull *pull = *iter; std::string name = pull->GetName(); std::string liketag = name + "_likelihood"; fNameValues.push_back(liketag); fCurrentValues.push_back(0.0); std::string ndoftag = name + "_ndof"; fNameValues.push_back(ndoftag); fCurrentValues.push_back(0.0); } // Add Likelihoods fNameValues.push_back("total_likelihood"); fCurrentValues.push_back(0.0); fNameValues.push_back("total_ndof"); fCurrentValues.push_back(0.0); // Setup Containers fSampleN = fSamples.size() + fPulls.size(); fSampleLikes = new double[fSampleN]; fSampleNDOF = new int[fSampleN]; // Add Dials std::vector dials = rw->GetDialNames(); for (size_t i = 0; i < dials.size(); i++) { fNameValues.push_back(dials[i]); fCurrentValues.push_back(0.0); } fNDials = dials.size(); fDialVals = new double[fNDials]; // Set IterationTree Flag fIterationTree = true; } //*************************************************** void JointFCN::DestroyIterationTree() { //*************************************************** fIterationCount.clear(); fCurrentValues.clear(); fNameValues.clear(); fIterationValues.clear(); } //*************************************************** void JointFCN::WriteIterationTree() { //*************************************************** LOG(FIT) << "Writing iteration tree" << std::endl; // Make a new TTree TTree *itree = new TTree(fIterationTreeName.c_str(), fIterationTreeName.c_str()); double *vals = new double[fNameValues.size()]; int count = 0; itree->Branch("iteration", &count, "Iteration/I"); for (size_t i = 0; i < fNameValues.size(); i++) { itree->Branch(fNameValues[i].c_str(), &vals[i], (fNameValues[i] + "/D").c_str()); } // Fill Iterations for (size_t i = 0; i < fIterationValues.size(); i++) { std::vector itervals = fIterationValues[i]; // Fill iteration state count = fIterationCount[i]; for (size_t j = 0; j < itervals.size(); j++) { vals[j] = itervals[j]; } // Save to TTree itree->Fill(); } // Write to file itree->Write(); } //*************************************************** void JointFCN::FillIterationTree(FitWeight *rw) { //*************************************************** // Loop over samples count int count = 0; for (int i = 0; i < fSampleN; i++) { fCurrentValues[count++] = fSampleLikes[i]; fCurrentValues[count++] = double(fSampleNDOF[i]); } // Fill Totals fCurrentValues[count++] = fLikelihood; fCurrentValues[count++] = double(fNDOF); // Loop Over Parameter Counts rw->GetAllDials(fDialVals, fNDials); for (int i = 0; i < fNDials; i++) { fCurrentValues[count++] = double(fDialVals[i]); } // Push Back Into Container fIterationCount.push_back(fCurIter); fIterationValues.push_back(fCurrentValues); } //*************************************************** double JointFCN::DoEval(const double *x) { //*************************************************** // WEIGHT ENGINE fDialChanged = FitBase::GetRW()->HasRWDialChanged(x); FitBase::GetRW()->UpdateWeightEngine(x); if (fDialChanged) { FitBase::GetRW()->Reconfigure(); FitBase::EvtManager().ResetWeightFlags(); } if (LOG_LEVEL(REC)) { FitBase::GetRW()->Print(); } // SORT SAMPLES ReconfigureSamples(); // GET TEST STAT fLikelihood = GetLikelihood(); fNDOF = GetNDOF(); // PRINT PROGRESS LOG(FIT) << "Current Stat (iter. " << this->fCurIter << ") = " << fLikelihood << std::endl; // UPDATE TREE - if (fIterationTree) - FillIterationTree(FitBase::GetRW()); + if (fIterationTree) FillIterationTree(FitBase::GetRW()); return fLikelihood; } //*************************************************** int JointFCN::GetNDOF() { //*************************************************** int totaldof = 0; int count = 0; // Total number of Free bins in each MC prediction for (MeasListConstIter iter = fSamples.begin(); iter != fSamples.end(); iter++) { MeasurementBase *exp = *iter; int dof = exp->GetNDOF(); // Save Seperate DOF if (fIterationTree) { fSampleNDOF[count] = dof; } // Add to total totaldof += dof; count++; } // Loop over pulls for (PullListConstIter iter = fPulls.begin(); iter != fPulls.end(); iter++) { ParamPull *pull = *iter; double dof = pull->GetLikelihood(); // Save seperate DOF if (fIterationTree) { fSampleNDOF[count] = dof; } // Add to total totaldof += dof; count++; } // Set Data Variable if (fIterationTree) { fSampleNDOF[count] = totaldof; } return totaldof; } //*************************************************** double JointFCN::GetLikelihood() { //*************************************************** LOG(MIN) << std::left << std::setw(43) << "Getting likelihoods..." << " : " << "-2logL" << std::endl; // Loop and add up likelihoods in an uncorrelated way double like = 0.0; int count = 0; for (MeasListConstIter iter = fSamples.begin(); iter != fSamples.end(); iter++) { MeasurementBase *exp = *iter; double newlike = exp->GetLikelihood(); int ndof = exp->GetNDOF(); // Save seperate likelihoods if (fIterationTree) { fSampleLikes[count] = newlike; } LOG(MIN) << "-> " << std::left << std::setw(40) << exp->GetName() << " : " << newlike << "/" << ndof << std::endl; // Add Weight Scaling // like *= FitBase::GetRW()->GetSampleLikelihoodWeight(exp->GetName()); // Add to total like += newlike; count++; } // Loop over pulls for (PullListConstIter iter = fPulls.begin(); iter != fPulls.end(); iter++) { ParamPull *pull = *iter; double newlike = pull->GetLikelihood(); // Save seperate likelihoods if (fIterationTree) { fSampleLikes[count] = newlike; } // Add to total like += newlike; count++; } // Set Data Variable fLikelihood = like; if (fIterationTree) { fSampleLikes[count] = fLikelihood; } return like; }; void JointFCN::LoadSamples(std::vector samplekeys) { LOG(MIN) << "Loading Samples : " << samplekeys.size() << std::endl; for (size_t i = 0; i < samplekeys.size(); i++) { nuiskey key = samplekeys[i]; // Get Sample Options std::string samplename = key.GetS("name"); std::string samplefile = key.GetS("input"); std::string sampletype = key.GetS("type"); std::string fakeData = ""; LOG(MIN) << "Loading Sample : " << samplename << std::endl; fOutputDir->cd(); MeasurementBase *NewLoadedSample = SampleUtils::CreateSample(key); if (!NewLoadedSample) { ERR(FTL) << "Could not load sample provided: " << samplename << std::endl; ERR(FTL) << "Check spelling with that in src/FCN/SampleList.cxx" << std::endl; throw; } else { fSamples.push_back(NewLoadedSample); } } } //*************************************************** void JointFCN::LoadPulls(std::vector pullkeys) { //*************************************************** for (size_t i = 0; i < pullkeys.size(); i++) { nuiskey key = pullkeys[i]; std::string pullname = key.GetS("name"); std::string pullfile = key.GetS("input"); std::string pulltype = key.GetS("type"); fOutputDir->cd(); fPulls.push_back(new ParamPull(pullname, pullfile, pulltype)); } } //*************************************************** void JointFCN::ReconfigureSamples(bool fullconfig) { //*************************************************** int starttime = time(NULL); LOG(REC) << "Starting Reconfigure iter. " << this->fCurIter << std::endl; // std::cout << fUsingEventManager << " " << fullconfig << " " << fMCFilled << // std::endl; // Event Manager Reconf if (fUsingEventManager) { - if (!fullconfig and fMCFilled) + if (!fullconfig && fMCFilled) ReconfigureFastUsingManager(); else ReconfigureUsingManager(); } else { // Loop over all Measurement Classes for (MeasListConstIter iter = fSamples.begin(); iter != fSamples.end(); iter++) { MeasurementBase *exp = *iter; // If RW Either do signal or full reconfigure. if (fDialChanged or !fMCFilled or fullconfig) { if (!fullconfig and fMCFilled) exp->ReconfigureFast(); else exp->Reconfigure(); // If RW Not needed just do normalisation } else { exp->Renormalise(); } } } // Loop over pulls and update for (PullListConstIter iter = fPulls.begin(); iter != fPulls.end(); iter++) { ParamPull *pull = *iter; pull->Reconfigure(); } fMCFilled = true; LOG(MIN) << "Finished Reconfigure iter. " << fCurIter << " in " << time(NULL) - starttime << "s" << std::endl; fCurIter++; } //*************************************************** void JointFCN::ReconfigureSignal() { //*************************************************** ReconfigureSamples(false); } //*************************************************** void JointFCN::ReconfigureAllEvents() { //*************************************************** FitBase::GetRW()->Reconfigure(); FitBase::EvtManager().ResetWeightFlags(); ReconfigureSamples(true); } std::vector JointFCN::GetInputList() { std::vector InputList; fIsAllSplines = true; MeasListConstIter iterSam = fSamples.begin(); for (; iterSam != fSamples.end(); iterSam++) { MeasurementBase *exp = (*iterSam); std::vector subsamples = exp->GetSubSamples(); for (size_t i = 0; i < subsamples.size(); i++) { InputHandlerBase *inp = subsamples[i]->GetInput(); if (std::find(InputList.begin(), InputList.end(), inp) == InputList.end()) { if (subsamples[i]->GetInput()->GetType() != kSPLINEPARAMETER) fIsAllSplines = false; InputList.push_back(subsamples[i]->GetInput()); } } } return InputList; } std::vector JointFCN::GetSubSampleList() { std::vector SampleList; MeasListConstIter iterSam = fSamples.begin(); for (; iterSam != fSamples.end(); iterSam++) { MeasurementBase *exp = (*iterSam); std::vector subsamples = exp->GetSubSamples(); for (size_t i = 0; i < subsamples.size(); i++) { SampleList.push_back(subsamples[i]); } } return SampleList; } //*************************************************** void JointFCN::ReconfigureUsingManager() { //*************************************************** // 'Slow' Event Manager Reconfigure LOG(REC) << "Event Manager Reconfigure" << std::endl; int timestart = time(NULL); // Reset all samples MeasListConstIter iterSam = fSamples.begin(); for (; iterSam != fSamples.end(); iterSam++) { MeasurementBase *exp = (*iterSam); exp->ResetAll(); } // If we are siving signal, reset all containers. bool savesignal = (FitPar::Config().GetParB("SignalReconfigures")); if (savesignal) { // Reset all of our event signal vectors fSignalEventBoxes.clear(); fSignalEventFlags.clear(); fSampleSignalFlags.clear(); fSignalEventSplines.clear(); } // Make sure we have a list of inputs if (fInputList.empty()) { fInputList = GetInputList(); fSubSampleList = GetSubSampleList(); } // If all inputs are splines make sure the readers are told // they need to be reconfigured. std::vector::iterator inp_iter = fInputList.begin(); if (fIsAllSplines) { for (; inp_iter != fInputList.end(); inp_iter++) { InputHandlerBase *curinput = (*inp_iter); // Tell reader in each BaseEvent it needs a Reconfigure next weight calc. BaseFitEvt *curevent = curinput->FirstBaseEvent(); if (curevent->fSplineRead) { curevent->fSplineRead->SetNeedsReconfigure(true); } } } // MAIN INPUT LOOP ==================== int fillcount = 0; int inputcount = 0; inp_iter = fInputList.begin(); // Loop over each input in manager for (; inp_iter != fInputList.end(); inp_iter++) { InputHandlerBase *curinput = (*inp_iter); // Get event information FitEvent *curevent = curinput->FirstNuisanceEvent(); curinput->CreateCache(); int i = 0; int nevents = curinput->GetNEvents(); int countwidth = nevents / 10; // Start event loop iterating until we get a NULL pointer. while (curevent) { // Get Event Weight // The reweighting weight curevent->RWWeight = FitBase::GetRW()->CalcWeight(curevent); // The Custom weight and reweight curevent->Weight = curevent->RWWeight * curevent->InputWeight * curevent->CustomWeight; - // double rwweight = curevent->Weight; - // std::cout << "RWWeight = " << curevent->RWWeight << " " << - // curevent->InputWeight << std::endl; - // Logging - // std::cout << CHECKLOG(1) << std::endl; if (LOGGING(REC)) { if (countwidth && (i % countwidth == 0)) { QLOG(REC, curinput->GetName() << " : Processed " << i << " events. [M, W] = [" << curevent->Mode << ", " << curevent->Weight << "]"); } } // Setup flag for if signal found in at least one sample bool foundsignal = false; // Create a new signal bitset for this event std::vector signalbitset(fSubSampleList.size()); // Create a new signal box vector for this event std::vector signalboxes; // Start measurement iterator size_t measitercount = 0; std::vector::iterator meas_iter = fSubSampleList.begin(); // Loop over all subsamples (sub in JointMeas) for (; meas_iter != fSubSampleList.end(); meas_iter++) { MeasurementBase *curmeas = (*meas_iter); // Compare input pointers, to current input, skip if not. // Pointer tells us if it matches without doing ID checks. if (curinput != curmeas->GetInput()) { if (savesignal) { // Set bit to 0 as definitely not signal signalbitset[measitercount] = 0; } // Count up what measurement we are on. measitercount++; // Skip sample as input not signal. continue; } // Fill events for matching inputs. MeasurementVariableBox *box = curmeas->FillVariableBox(curevent); bool signal = curmeas->isSignal(curevent); curmeas->SetSignal(signal); curmeas->FillHistograms(curevent->Weight); // If its Signal tally up fills if (signal) { fillcount++; } // If we are saving signal/splines fill the bitset if (savesignal) { signalbitset[measitercount] = signal; } // If signal save a clone of the event box for use later. if (savesignal and signal) { foundsignal = true; signalboxes.push_back(box->CloneSignalBox()); } // Keep track of Measurement we are on. measitercount++; } // Once we've filled the measurements, if saving signal // push back if any sample flagged this event as signal if (savesignal) { fSignalEventFlags.push_back(foundsignal); } // Save the vector of signal boxes for this event - if (savesignal and foundsignal) { + if (savesignal && foundsignal) { fSignalEventBoxes.push_back(signalboxes); fSampleSignalFlags.push_back(signalbitset); } // If all inputs are splines we can save the spline coefficients // for fast in memory reconfigures later. - if (fIsAllSplines and savesignal and foundsignal) { + if (fIsAllSplines && savesignal && foundsignal) { // Make temp vector to push back with std::vector coeff; for (size_t l = 0; l < (UInt_t)curevent->fSplineRead->GetNPar(); l++) { coeff.push_back(curevent->fSplineCoeff[l]); } // Push back to signal event splines. Kept in sync with // fSignalEventBoxes size. // int splinecount = fSignalEventSplines.size(); fSignalEventSplines.push_back(coeff); // if (splinecount % 1000 == 0) { // std::cout << "Pushed Back Coeff " << splinecount << " : "; // for (size_t l = 0; l < fSignalEventSplines[splinecount].size(); l++) // { // std::cout << " " << fSignalEventSplines[splinecount][l]; // } // std::cout << std::endl; // } } // Clean up vectors once done with this event signalboxes.clear(); signalbitset.clear(); // Iterate to the next event. curevent = curinput->NextNuisanceEvent(); i++; } // curinput->RemoveCache(); // Keep track of what input we are on. inputcount++; } // End of Event Loop =============================== // Now event loop is finished loop over all Measurements // Converting Binned events to XSec Distributions iterSam = fSamples.begin(); for (; iterSam != fSamples.end(); iterSam++) { MeasurementBase *exp = (*iterSam); exp->ConvertEventRates(); } // Print out statements on approximate memory usage for profiling. LOG(REC) << "Filled " << fillcount << " signal events." << std::endl; if (savesignal) { int mem = ( // sizeof(fSignalEventBoxes) + // fSignalEventBoxes.size() * sizeof(fSignalEventBoxes.at(0)) + sizeof(MeasurementVariableBox1D) * fillcount) * 1E-6; LOG(REC) << " -> Saved " << fillcount << " signal boxes for faster access. (~" << mem << " MB)" << std::endl; if (fIsAllSplines and !fSignalEventSplines.empty()) { int splmem = sizeof(float) * fSignalEventSplines.size() * fSignalEventSplines[0].size() * 1E-6; LOG(REC) << " -> Saved " << fillcount << " " << fSignalEventSplines.size() << " spline sets into memory. (~" << splmem << " MB)" << std::endl; } } LOG(REC) << "Time taken ReconfigureUsingManager() : " << time(NULL) - timestart << std::endl; // Check SignalReconfigures works for all samples if (savesignal) { double likefull = GetLikelihood(); ReconfigureFastUsingManager(); double likefast = GetLikelihood(); if (fabs(likefull - likefast) > 0.0001) { ERROR(FTL, "Fast and Full Likelihoods DIFFER! : " << likefull << " : " << likefast); ERROR(FTL, "This means some samples you are using are not setup to use " "SignalReconfigures=1"); ERROR(FTL, "Please turn OFF signal reconfigures."); throw; } else { LOG(FIT) << "Likelihoods for FULL and FAST match. Will use FAST next time." << std::endl; } } - - // End of reconfigure - return; }; //*************************************************** void JointFCN::ReconfigureFastUsingManager() { //*************************************************** LOG(FIT) << " -> Doing FAST using manager" << std::endl; // Get Start time for profilling int timestart = time(NULL); // Reset all samples MeasListConstIter iterSam = fSamples.begin(); for (; iterSam != fSamples.end(); iterSam++) { MeasurementBase *exp = (*iterSam); exp->ResetAll(); } // Check for saved variables if not do a full reconfigure. if (fSignalEventFlags.empty()) { ERR(WRN) << "Signal Flags Empty! Using normal manager." << std::endl; ReconfigureUsingManager(); return; } bool fFillNuisanceEvent = FitPar::Config().GetParB("FullEventOnSignalReconfigure"); // Setup fast vector iterators. std::vector::iterator inpsig_iter = fSignalEventFlags.begin(); std::vector >::iterator box_iter = fSignalEventBoxes.begin(); std::vector >::iterator spline_iter = fSignalEventSplines.begin(); std::vector >::iterator samsig_iter = fSampleSignalFlags.begin(); int splinecount = 0; // Setup stuff for logging int fillcount = 0; - int nevents = fSignalEventFlags.size(); + // This is just the total number of events + //int nevents = fSignalEventFlags.size(); + // This is the number of events that are signal + int nevents = fSignalEventBoxes.size(); int countwidth = nevents / 10; // If All Splines tell splines they need a reconfigure. std::vector::iterator inp_iter = fInputList.begin(); if (fIsAllSplines) { LOG(REC) << "All Spline Inputs so using fast spline loop." << std::endl; for (; inp_iter != fInputList.end(); inp_iter++) { InputHandlerBase *curinput = (*inp_iter); // Tell each fSplineRead in BaseFitEvent to reconf next weight calc BaseFitEvt *curevent = curinput->FirstBaseEvent(); if (curevent->fSplineRead) curevent->fSplineRead->SetNeedsReconfigure(true); } } // Loop over all possible spline inputs double *coreeventweights = new double[fSignalEventBoxes.size()]; splinecount = 0; inp_iter = fInputList.begin(); inpsig_iter = fSignalEventFlags.begin(); spline_iter = fSignalEventSplines.begin(); // Loop over all signal flags // For each valid signal flag add one to splinecount // Get Splines from that count and add to weight // Add splinecount int sigcount = 0; - splinecount = 0; // #pragma omp parallel for shared(splinecount,sigcount) for (uint iinput = 0; iinput < fInputList.size(); iinput++) { InputHandlerBase *curinput = fInputList[iinput]; BaseFitEvt *curevent = curinput->FirstBaseEvent(); + // Loop over the events in each input for (int i = 0; i < curinput->GetNEvents(); i++) { double rwweight = 0.0; + + // If the event is a signal event if (fSignalEventFlags[sigcount]) { // Get Event Info if (!fIsAllSplines) { if (fFillNuisanceEvent) { curevent = curinput->GetNuisanceEvent(i); } else { curevent = curinput->GetBaseEvent(i); } } else { curevent->fSplineCoeff = &fSignalEventSplines[splinecount][0]; } curevent->RWWeight = FitBase::GetRW()->CalcWeight(curevent); curevent->Weight = curevent->RWWeight * curevent->InputWeight * curevent->CustomWeight; rwweight = curevent->Weight; coreeventweights[splinecount] = rwweight; if (countwidth && ((splinecount % countwidth) == 0)) { - LOG(REC) << "Processed " << splinecount - << " event weights. W = " << rwweight << std::endl; + QLOG(REC, curinput->GetName() + << " : Processed " << i << " events. W = " + << curevent->Weight << std::endl); } // #pragma omp atomic splinecount++; } // #pragma omp atomic sigcount++; } } + LOG(SAM) << "Processed event weights." << std::endl; // #pragma omp barrier // Reset Iterators inpsig_iter = fSignalEventFlags.begin(); spline_iter = fSignalEventSplines.begin(); box_iter = fSignalEventBoxes.begin(); samsig_iter = fSampleSignalFlags.begin(); int nsplineweights = splinecount; splinecount = 0; // Start of Fast Event Loop ============================ // Start input iterators // Loop over number of inputs for (int ispline = 0; ispline < nsplineweights; ispline++) { double rwweight = coreeventweights[ispline]; // Get iterators for this event std::vector::iterator subsamsig_iter = (*samsig_iter).begin(); std::vector::iterator subbox_iter = (*box_iter).begin(); // Loop over all sub measurements. std::vector::iterator meas_iter = fSubSampleList.begin(); for (; meas_iter != fSubSampleList.end(); meas_iter++, subsamsig_iter++) { MeasurementBase *curmeas = (*meas_iter); // If event flagged as signal for this sample fill from the box. if (*subsamsig_iter) { curmeas->SetSignal(true); curmeas->FillHistogramsFromBox((*subbox_iter), rwweight); // Move onto next box if there is one. subbox_iter++; fillcount++; } } if (ispline % countwidth == 0) { LOG(REC) << "Filled " << ispline << " sample weights." << std::endl; } // Iterate over the main signal event containers. samsig_iter++; box_iter++; spline_iter++; splinecount++; } // End of Fast Event Loop =================== LOG(SAM) << "Filled sample distributions." << std::endl; // Now loop over all Measurements // Convert Binned events iterSam = fSamples.begin(); for (; iterSam != fSamples.end(); iterSam++) { MeasurementBase *exp = (*iterSam); exp->ConvertEventRates(); } // Cleanup coreeventweights delete coreeventweights; // Print some reconfigure profiling. LOG(REC) << "Filled " << fillcount << " signal events." << std::endl; LOG(REC) << "Time taken ReconfigureFastUsingManager() : " << time(NULL) - timestart << std::endl; } //*************************************************** void JointFCN::Write() { //*************************************************** // Save a likelihood/ndof plot LOG(MIN) << "Writing likelihood plot.." << std::endl; std::vector likes; std::vector ndofs; std::vector names; for (MeasListConstIter iter = fSamples.begin(); iter != fSamples.end(); iter++) { MeasurementBase *exp = *iter; double like = exp->GetLikelihood(); double ndof = exp->GetNDOF(); std::string name = exp->GetName(); likes.push_back(like); ndofs.push_back(ndof); names.push_back(name); } if (likes.size()) { TH1D likehist = TH1D("likelihood_hist", "likelihood_hist", likes.size(), 0.0, double(likes.size())); TH1D ndofhist = TH1D("ndof_hist", "ndof_hist", ndofs.size(), 0.0, double(ndofs.size())); TH1D divhist = TH1D("likedivndof_hist", "likedivndof_hist", likes.size(), 0.0, double(likes.size())); for (int i = 0; i < likehist.GetNbinsX(); i++) { likehist.SetBinContent(i + 1, likes[i]); ndofhist.SetBinContent(i + 1, ndofs[i]); if (ndofs[i] != 0.0) { divhist.SetBinContent(i + 1, likes[i] / ndofs[i]); } likehist.GetXaxis()->SetBinLabel(i + 1, names[i].c_str()); ndofhist.GetXaxis()->SetBinLabel(i + 1, names[i].c_str()); divhist.GetXaxis()->SetBinLabel(i + 1, names[i].c_str()); } likehist.Write(); ndofhist.Write(); divhist.Write(); } // Loop over individual experiments and call Write LOG(MIN) << "Writing each of the data classes..." << std::endl; for (MeasListConstIter iter = fSamples.begin(); iter != fSamples.end(); iter++) { MeasurementBase *exp = *iter; exp->Write(); } // Save Pull Terms for (PullListConstIter iter = fPulls.begin(); iter != fPulls.end(); iter++) { ParamPull *pull = *iter; pull->Write(); } if (FitPar::Config().GetParB("EventManager")) { // Get list of inputs std::map fInputs = FitBase::EvtManager().GetInputs(); std::map::const_iterator iterInp; for (iterInp = fInputs.begin(); iterInp != fInputs.end(); iterInp++) { InputHandlerBase *input = (iterInp->second); input->GetFluxHistogram()->Write(); input->GetXSecHistogram()->Write(); input->GetEventHistogram()->Write(); } } }; //*************************************************** void JointFCN::SetFakeData(std::string fakeinput) { //*************************************************** LOG(MIN) << "Setting fake data from " << fakeinput << std::endl; for (MeasListConstIter iter = fSamples.begin(); iter != fSamples.end(); iter++) { MeasurementBase *exp = *iter; exp->SetFakeDataValues(fakeinput); } return; } //*************************************************** void JointFCN::ThrowDataToy() { //*************************************************** for (MeasListConstIter iter = fSamples.begin(); iter != fSamples.end(); iter++) { MeasurementBase *exp = *iter; exp->ThrowDataToy(); } return; } //*************************************************** std::vector JointFCN::GetAllNames() { //*************************************************** // Vect of all likelihoods and total std::vector namevect; // Loop over samples first for (MeasListConstIter iter = fSamples.begin(); iter != fSamples.end(); iter++) { MeasurementBase *exp = *iter; // Get Likelihoods and push to vector namevect.push_back(exp->GetName()); } // Loop over pulls second for (PullListConstIter iter = fPulls.begin(); iter != fPulls.end(); iter++) { ParamPull *pull = *iter; // Push back to vector namevect.push_back(pull->GetName()); } // Finally add the total namevect.push_back("total"); return namevect; } //*************************************************** std::vector JointFCN::GetAllLikelihoods() { //*************************************************** // Vect of all likelihoods and total std::vector likevect; double total_likelihood = 0.0; LOG(MIN) << "Likelihoods : " << std::endl; // Loop over samples first for (MeasListConstIter iter = fSamples.begin(); iter != fSamples.end(); iter++) { MeasurementBase *exp = *iter; // Get Likelihoods and push to vector double singlelike = exp->GetLikelihood(); likevect.push_back(singlelike); total_likelihood += singlelike; // Print Out LOG(MIN) << "-> " << std::left << std::setw(40) << exp->GetName() << " : " << singlelike << std::endl; } // Loop over pulls second for (PullListConstIter iter = fPulls.begin(); iter != fPulls.end(); iter++) { ParamPull *pull = *iter; // Push back to vector double singlelike = pull->GetLikelihood(); likevect.push_back(singlelike); total_likelihood += singlelike; // Print Out LOG(MIN) << "-> " << std::left << std::setw(40) << pull->GetName() << " : " << singlelike << std::endl; } // Finally add the total likelihood likevect.push_back(total_likelihood); return likevect; } //*************************************************** std::vector JointFCN::GetAllNDOF() { //*************************************************** // Vect of all ndof and total std::vector ndofvect; int total_ndof = 0; // Loop over samples first for (MeasListConstIter iter = fSamples.begin(); iter != fSamples.end(); iter++) { MeasurementBase *exp = *iter; // Get Likelihoods and push to vector int singlendof = exp->GetNDOF(); ndofvect.push_back(singlendof); total_ndof += singlendof; } // Loop over pulls second for (PullListConstIter iter = fPulls.begin(); iter != fPulls.end(); iter++) { ParamPull *pull = *iter; // Push back to vector int singlendof = pull->GetNDOF(); ndofvect.push_back(singlendof); total_ndof += singlendof; } // Finally add the total ndof ndofvect.push_back(total_ndof); return ndofvect; } diff --git a/src/FitBase/Measurement1D.cxx b/src/FitBase/Measurement1D.cxx index 2e2a64b..e6737c7 100644 --- a/src/FitBase/Measurement1D.cxx +++ b/src/FitBase/Measurement1D.cxx @@ -1,1900 +1,1899 @@ // Copyright 2016 L. Pickering, P. Stowell, R. Terri, C. Wilkinson, C. Wret /******************************************************************************* * This ile is part of NUISANCE. * * NUISANCE is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * NUISANCE is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with NUISANCE. If not, see . *******************************************************************************/ #include "Measurement1D.h" //******************************************************************** Measurement1D::Measurement1D(void) { //******************************************************************** // XSec Scalings fScaleFactor = -1.0; fCurrentNorm = 1.0; // Histograms fDataHist = NULL; fDataTrue = NULL; fMCHist = NULL; fMCFine = NULL; fMCWeighted = NULL; fMaskHist = NULL; // Covar covar = NULL; fFullCovar = NULL; fShapeCovar = NULL; fCovar = NULL; fInvert = NULL; fDecomp = NULL; // Fake Data fFakeDataInput = ""; fFakeDataFile = NULL; // Options fDefaultTypes = "FIX/FULL/CHI2"; fAllowedTypes = "FIX,FREE,SHAPE/FULL,DIAG/CHI2/NORM/ENUCORR/Q2CORR/ENU1D/MASK/NOWIDTH"; fIsFix = false; fIsShape = false; fIsFree = false; fIsDiag = false; fIsFull = false; fAddNormPen = false; fIsMask = false; fIsChi2SVD = false; fIsRawEvents = false; fIsNoWidth = false; fIsDifXSec = false; fIsEnu1D = false; // Inputs fInput = NULL; fRW = NULL; // Extra Histograms fMCHist_Modes = NULL; } //******************************************************************** Measurement1D::~Measurement1D(void) { //******************************************************************** if (fDataHist) delete fDataHist; if (fDataTrue) delete fDataTrue; if (fMCHist) delete fMCHist; if (fMCFine) delete fMCFine; if (fMCWeighted) delete fMCWeighted; if (fMaskHist) delete fMaskHist; if (covar) delete covar; if (fFullCovar) delete fFullCovar; if (fShapeCovar) delete fShapeCovar; if (fCovar) delete fCovar; if (fInvert) delete fInvert; if (fDecomp) delete fDecomp; } //******************************************************************** void Measurement1D::FinaliseSampleSettings() { //******************************************************************** MeasurementBase::FinaliseSampleSettings(); // Setup naming + renaming fName = fSettings.GetName(); fSettings.SetS("originalname", fName); if (fSettings.Has("rename")) { fName = fSettings.GetS("rename"); fSettings.SetS("name", fName); } // Setup all other options LOG(SAM) << "Finalising Sample Settings: " << fName << std::endl; if ((fSettings.GetS("originalname").find("Evt") != std::string::npos)) { fIsRawEvents = true; LOG(SAM) << "Found event rate measurement but using poisson likelihoods." << std::endl; } if (fSettings.GetS("originalname").find("XSec_1DEnu") != std::string::npos) { fIsEnu1D = true; LOG(SAM) << "::" << fName << "::" << std::endl; LOG(SAM) << "Found XSec Enu measurement, applying flux integrated scaling, " << "not flux averaged!" << std::endl; } if (fIsEnu1D && fIsRawEvents) { LOG(SAM) << "Found 1D Enu XSec distribution AND fIsRawEvents, is this " "really correct?!" << std::endl; LOG(SAM) << "Check experiment constructor for " << fName << " and correct this!" << std::endl; LOG(SAM) << "I live in " << __FILE__ << ":" << __LINE__ << std::endl; exit(-1); } if (!fRW) fRW = FitBase::GetRW(); if (!fInput and !fIsJoint) SetupInputs(fSettings.GetS("input")); // Setup options SetFitOptions(fDefaultTypes); // defaults SetFitOptions(fSettings.GetS("type")); // user specified EnuMin = GeneralUtils::StrToDbl(fSettings.GetS("enu_min")); EnuMax = GeneralUtils::StrToDbl(fSettings.GetS("enu_max")); if (fAddNormPen) { if (fNormError <= 0.0) { ERR(WRN) << "Norm error for class " << fName << " is 0.0!" << std::endl; ERR(WRN) << "If you want to use it please add fNormError=VAL" << std::endl; throw; } } } //******************************************************************** void Measurement1D::CreateDataHistogram(int dimx, double* binx) { //******************************************************************** if (fDataHist) delete fDataHist; fDataHist = new TH1D( (fSettings.GetName() + "_data").c_str(), (fSettings.GetFullTitles()).c_str(), dimx, binx) ; } //******************************************************************** void Measurement1D::SetDataFromTextFile(std::string datafile) { //******************************************************************** LOG(SAM) << "Reading data from text file: " << datafile << std::endl; fDataHist = PlotUtils::GetTH1DFromFile(datafile, fSettings.GetName() + "_data", fSettings.GetFullTitles()); } //******************************************************************** void Measurement1D::SetDataFromRootFile(std::string datafile, std::string histname) { //******************************************************************** LOG(SAM) << "Reading data from root file: " << datafile << ";" << histname << std::endl; fDataHist = PlotUtils::GetTH1DFromRootFile(datafile, histname); fDataHist->SetNameTitle((fSettings.GetName() + "_data").c_str(), (fSettings.GetFullTitles()).c_str()); return; }; //******************************************************************** void Measurement1D::SetEmptyData(){ //******************************************************************** fDataHist = new TH1D("EMPTY_DATA","EMPTY_DATA",1,0.0,1.0); } //******************************************************************** void Measurement1D::SetPoissonErrors() { //******************************************************************** if (!fDataHist) { ERR(FTL) << "Need a data hist to setup possion errors! " << std::endl; ERR(FTL) << "Setup Data First!" << std::endl; throw; } for (int i = 0; i < fDataHist->GetNbinsX() + 1; i++) { fDataHist->SetBinError(i + 1, sqrt(fDataHist->GetBinContent(i + 1))); } } //******************************************************************** void Measurement1D::SetCovarFromDiagonal(TH1D* data) { //******************************************************************** if (!data and fDataHist) { data = fDataHist; } if (data) { LOG(SAM) << "Setting diagonal covariance for: " << data->GetName() << std::endl; fFullCovar = StatUtils::MakeDiagonalCovarMatrix(data); covar = StatUtils::GetInvert(fFullCovar); fDecomp = StatUtils::GetDecomp(fFullCovar); } else { ERR(FTL) << "No data input provided to set diagonal covar from!" << std::endl; } // if (!fIsDiag) { // ERR(FTL) << "SetCovarMatrixFromDiag called for measurement " // << "that is not set as diagonal." << std::endl; // throw; // } } //******************************************************************** void Measurement1D::SetCovarFromTextFile(std::string covfile, int dim) { //******************************************************************** if (dim == -1) { dim = fDataHist->GetNbinsX(); } LOG(SAM) << "Reading covariance from text file: " << covfile << std::endl; fFullCovar = StatUtils::GetCovarFromTextFile(covfile, dim); covar = StatUtils::GetInvert(fFullCovar); fDecomp = StatUtils::GetDecomp(fFullCovar); } //******************************************************************** void Measurement1D::SetCovarFromMultipleTextFiles(std::string covfiles, int dim) { //******************************************************************** if (dim == -1) { dim = fDataHist->GetNbinsX(); } std::vector covList = GeneralUtils::ParseToStr(covfiles, ";"); fFullCovar = new TMatrixDSym(dim); for (uint i = 0; i < covList.size(); ++i){ LOG(SAM) << "Reading covariance from text file: " << covList[i] << std::endl; TMatrixDSym* temp_cov = StatUtils::GetCovarFromTextFile(covList[i], dim); (*fFullCovar) += (*temp_cov); delete temp_cov; } covar = StatUtils::GetInvert(fFullCovar); fDecomp = StatUtils::GetDecomp(fFullCovar); } //******************************************************************** void Measurement1D::SetCovarFromRootFile(std::string covfile, std::string histname) { //******************************************************************** LOG(SAM) << "Reading covariance from text file: " << covfile << ";" << histname << std::endl; fFullCovar = StatUtils::GetCovarFromRootFile(covfile, histname); covar = StatUtils::GetInvert(fFullCovar); fDecomp = StatUtils::GetDecomp(fFullCovar); } //******************************************************************** void Measurement1D::SetCovarInvertFromTextFile(std::string covfile, int dim) { //******************************************************************** if (dim == -1) { dim = fDataHist->GetNbinsX(); } LOG(SAM) << "Reading inverted covariance from text file: " << covfile << std::endl; covar = StatUtils::GetCovarFromTextFile(covfile, dim); fFullCovar = StatUtils::GetInvert(covar); fDecomp = StatUtils::GetDecomp(fFullCovar); } //******************************************************************** void Measurement1D::SetCovarInvertFromRootFile(std::string covfile, std::string histname) { //******************************************************************** LOG(SAM) << "Reading inverted covariance from text file: " << covfile << ";" << histname << std::endl; covar = StatUtils::GetCovarFromRootFile(covfile, histname); fFullCovar = StatUtils::GetInvert(covar); fDecomp = StatUtils::GetDecomp(fFullCovar); } //******************************************************************** void Measurement1D::SetCorrelationFromTextFile(std::string covfile, int dim) { //******************************************************************** if (dim == -1) dim = fDataHist->GetNbinsX(); LOG(SAM) << "Reading data correlations from text file: " << covfile << ";" << dim << std::endl; TMatrixDSym* correlation = StatUtils::GetCovarFromTextFile(covfile, dim); if (!fDataHist) { ERR(FTL) << "Trying to set correlations from text file but there is no data to build it from. \n" << "In constructor make sure data is set before SetCorrelationFromTextFile is called. \n" << std::endl; throw; } // Fill covar from data errors and correlations fFullCovar = new TMatrixDSym(dim); for (int i = 0; i < fDataHist->GetNbinsX(); i++) { for (int j = 0; j < fDataHist->GetNbinsX(); j++) { (*fFullCovar)(i, j) = (*correlation)(i, j) * fDataHist->GetBinError(i + 1) * fDataHist->GetBinError(j + 1) * 1.E76; } } // Fill other covars. covar = StatUtils::GetInvert(fFullCovar); fDecomp = StatUtils::GetDecomp(fFullCovar); delete correlation; } //******************************************************************** void Measurement1D::SetCorrelationFromMultipleTextFiles(std::string corrfiles, int dim) { //******************************************************************** if (dim == -1) { dim = fDataHist->GetNbinsX(); } std::vector corrList = GeneralUtils::ParseToStr(corrfiles, ";"); fFullCovar = new TMatrixDSym(dim); for (uint i = 0; i < corrList.size(); ++i){ LOG(SAM) << "Reading covariance from text file: " << corrList[i] << std::endl; TMatrixDSym* temp_cov = StatUtils::GetCovarFromTextFile(corrList[i], dim); for (int i = 0; i < fDataHist->GetNbinsX(); i++) { for (int j = 0; j < fDataHist->GetNbinsX(); j++) { (*temp_cov)(i, j) = (*temp_cov)(i, j) * fDataHist->GetBinError(i + 1) * fDataHist->GetBinError(j + 1) * 1.E76; } } (*fFullCovar) += (*temp_cov); delete temp_cov; } covar = StatUtils::GetInvert(fFullCovar); fDecomp = StatUtils::GetDecomp(fFullCovar); } //******************************************************************** void Measurement1D::SetCorrelationFromRootFile(std::string covfile, std::string histname) { //******************************************************************** LOG(SAM) << "Reading data correlations from text file: " << covfile << ";" << histname << std::endl; TMatrixDSym* correlation = StatUtils::GetCovarFromRootFile(covfile, histname); if (!fDataHist) { ERR(FTL) << "Trying to set correlations from text file but there is no data to build it from. \n" << "In constructor make sure data is set before SetCorrelationFromTextFile is called. \n" << std::endl; throw; } // Fill covar from data errors and correlations fFullCovar = new TMatrixDSym(fDataHist->GetNbinsX()); for (int i = 0; i < fDataHist->GetNbinsX(); i++) { for (int j = 0; j < fDataHist->GetNbinsX(); j++) { (*fFullCovar)(i, j) = (*correlation)(i, j) * fDataHist->GetBinError(i + 1) * fDataHist->GetBinError(j + 1) * 1.E76; } } // Fill other covars. covar = StatUtils::GetInvert(fFullCovar); fDecomp = StatUtils::GetDecomp(fFullCovar); delete correlation; } //******************************************************************** void Measurement1D::SetCholDecompFromTextFile(std::string covfile, int dim) { //******************************************************************** if (dim == -1) { dim = fDataHist->GetNbinsX(); } LOG(SAM) << "Reading cholesky from text file: " << covfile << std::endl; TMatrixD* temp = StatUtils::GetMatrixFromTextFile(covfile, dim, dim); TMatrixD* trans = (TMatrixD*)temp->Clone(); trans->T(); (*trans) *= (*temp); fFullCovar = new TMatrixDSym(dim, trans->GetMatrixArray(), ""); covar = StatUtils::GetInvert(fFullCovar); fDecomp = StatUtils::GetDecomp(fFullCovar); delete temp; delete trans; } //******************************************************************** void Measurement1D::SetCholDecompFromRootFile(std::string covfile, std::string histname) { //******************************************************************** LOG(SAM) << "Reading cholesky decomp from root file: " << covfile << ";" << histname << std::endl; TMatrixD* temp = StatUtils::GetMatrixFromRootFile(covfile, histname); TMatrixD* trans = (TMatrixD*)temp->Clone(); trans->T(); (*trans) *= (*temp); fFullCovar = new TMatrixDSym(temp->GetNrows(), trans->GetMatrixArray(), ""); covar = StatUtils::GetInvert(fFullCovar); fDecomp = StatUtils::GetDecomp(fFullCovar); delete temp; delete trans; } void Measurement1D::SetShapeCovar(){ // Return if this is missing any pre-requisites if (!fFullCovar) return; if (!fDataHist) return; // Also return if it's bloody stupid under the circumstances if (fIsDiag) return; fShapeCovar = StatUtils::ExtractShapeOnlyCovar(fFullCovar, fDataHist); return; } //******************************************************************** void Measurement1D::ScaleData(double scale) { //******************************************************************** fDataHist->Scale(scale); } //******************************************************************** void Measurement1D::ScaleDataErrors(double scale) { //******************************************************************** for (int i = 0; i < fDataHist->GetNbinsX(); i++) { fDataHist->SetBinError(i + 1, fDataHist->GetBinError(i + 1) * scale); } } //******************************************************************** void Measurement1D::ScaleCovar(double scale) { //******************************************************************** (*fFullCovar) *= scale; (*covar) *= 1.0 / scale; (*fDecomp) *= sqrt(scale); } //******************************************************************** void Measurement1D::SetBinMask(std::string maskfile) { //******************************************************************** if (!fIsMask) return; LOG(SAM) << "Reading bin mask from file: " << maskfile << std::endl; // Create a mask histogram with dim of data int nbins = fDataHist->GetNbinsX(); fMaskHist = new TH1I((fSettings.GetName() + "_BINMASK").c_str(), (fSettings.GetName() + "_BINMASK; Bin; Mask?").c_str(), nbins, 0, nbins); std::string line; std::ifstream mask(maskfile.c_str(), std::ifstream::in); if (!mask.is_open()) { LOG(FTL) << " Cannot find mask file." << std::endl; throw; } while (std::getline(mask >> std::ws, line, '\n')) { std::vector entries = GeneralUtils::ParseToInt(line, " "); // Skip lines with poorly formatted lines if (entries.size() < 2) { LOG(WRN) << "Measurement1D::SetBinMask(), couldn't parse line: " << line << std::endl; continue; } // The first index should be the bin number, the second should be the mask // value. int val = 0; if (entries[1] > 0) val = 1; fMaskHist->SetBinContent(entries[0], val); } // Apply masking by setting masked data bins to zero PlotUtils::MaskBins(fDataHist, fMaskHist); return; } //******************************************************************** void Measurement1D::FinaliseMeasurement() { //******************************************************************** LOG(SAM) << "Finalising Measurement: " << fName << std::endl; if (fSettings.GetB("onlymc")){ if (fDataHist) delete fDataHist; fDataHist = new TH1D("empty_data","empty_data",1,0.0,1.0); } // Make sure data is setup if (!fDataHist) { ERR(FTL) << "No data has been setup inside " << fName << " constructor!" << std::endl; throw; } // Make sure covariances are setup if (!fFullCovar) { fIsDiag = true; SetCovarFromDiagonal(fDataHist); } if (!covar) { covar = StatUtils::GetInvert(fFullCovar); } if (!fDecomp) { fDecomp = StatUtils::GetDecomp(fFullCovar); } // Push the diagonals of fFullCovar onto the data histogram // Comment this out until the covariance/data scaling is consistent! StatUtils::SetDataErrorFromCov(fDataHist, fFullCovar, 1E-38); // If shape only, set covar and fDecomp using the shape-only matrix (if set) if (fIsShape && fShapeCovar and FitPar::Config().GetParB("UseShapeCovar")){ if (covar) delete covar; covar = StatUtils::GetInvert(fShapeCovar); if (fDecomp) delete fDecomp; fDecomp = StatUtils::GetDecomp(fFullCovar); } // Setup fMCHist from data fMCHist = (TH1D*)fDataHist->Clone(); fMCHist->SetNameTitle((fSettings.GetName() + "_MC").c_str(), (fSettings.GetFullTitles()).c_str()); fMCHist->Reset(); // Setup fMCFine fMCFine = new TH1D("mcfine", "mcfine", fDataHist->GetNbinsX() * 8, fMCHist->GetBinLowEdge(1), fMCHist->GetBinLowEdge(fDataHist->GetNbinsX() + 1)); fMCFine->SetNameTitle((fSettings.GetName() + "_MC_FINE").c_str(), (fSettings.GetFullTitles()).c_str()); fMCFine->Reset(); // Setup MC Stat fMCStat = (TH1D*)fMCHist->Clone(); fMCStat->Reset(); // Search drawopts for possible types to include by default std::string drawopts = FitPar::Config().GetParS("drawopts"); if (drawopts.find("MODES") != std::string::npos) { fMCHist_Modes = new TrueModeStack( (fSettings.GetName() + "_MODES").c_str(), ("True Channels"), fMCHist); SetAutoProcessTH1(fMCHist_Modes, kCMD_Reset, kCMD_Norm, kCMD_Write); } // Setup bin masks using sample name if (fIsMask) { std::string curname = fName; std::string origname = fSettings.GetS("originalname"); // Check rename.mask std::string maskloc = FitPar::Config().GetParDIR(curname + ".mask"); // Check origname.mask if (maskloc.empty()) maskloc = FitPar::Config().GetParDIR(origname + ".mask"); // Check database if (maskloc.empty()) { maskloc = FitPar::GetDataBase() + "/masks/" + origname + ".mask"; } // Setup Bin Mask SetBinMask(maskloc); } if (fScaleFactor < 0) { ERR(FTL) << "I found a negative fScaleFactor in " << __FILE__ << ":" << __LINE__ << std::endl; ERR(FTL) << "fScaleFactor = " << fScaleFactor << std::endl; ERR(FTL) << "EXITING" << std::endl; throw; } // Create and fill Weighted Histogram if (!fMCWeighted) { fMCWeighted = (TH1D*)fMCHist->Clone(); fMCWeighted->SetNameTitle((fName + "_MCWGHTS").c_str(), (fName + "_MCWGHTS" + fPlotTitles).c_str()); fMCWeighted->GetYaxis()->SetTitle("Weighted Events"); } } //******************************************************************** void Measurement1D::SetFitOptions(std::string opt) { //******************************************************************** // Do nothing if default given if (opt == "DEFAULT") return; // CHECK Conflicting Fit Options std::vector fit_option_allow = GeneralUtils::ParseToStr(fAllowedTypes, "/"); for (UInt_t i = 0; i < fit_option_allow.size(); i++) { std::vector fit_option_section = GeneralUtils::ParseToStr(fit_option_allow.at(i), ","); bool found_option = false; for (UInt_t j = 0; j < fit_option_section.size(); j++) { std::string av_opt = fit_option_section.at(j); if (!found_option and opt.find(av_opt) != std::string::npos) { found_option = true; } else if (found_option and opt.find(av_opt) != std::string::npos) { ERR(FTL) << "ERROR: Conflicting fit options provided: " << opt << std::endl << "Conflicting group = " << fit_option_section.at(i) << std::endl << "You should only supply one of these options in card file." << std::endl; throw; } } } // Check all options are allowed std::vector fit_options_input = GeneralUtils::ParseToStr(opt, "/"); for (UInt_t i = 0; i < fit_options_input.size(); i++) { if (fAllowedTypes.find(fit_options_input.at(i)) == std::string::npos) { ERR(FTL) << "ERROR: Fit Option '" << fit_options_input.at(i) << "' Provided is not allowed for this measurement." << std::endl; ERR(FTL) << "Fit Options should be provided as a '/' seperated list " "(e.g. FREE/DIAG/NORM)" << std::endl; ERR(FTL) << "Available options for " << fName << " are '" << fAllowedTypes << "'" << std::endl; throw; } } // Set TYPE fFitType = opt; // FIX,SHAPE,FREE if (opt.find("FIX") != std::string::npos) { fIsFree = fIsShape = false; fIsFix = true; } else if (opt.find("SHAPE") != std::string::npos) { fIsFree = fIsFix = false; fIsShape = true; } else if (opt.find("FREE") != std::string::npos) { fIsFix = fIsShape = false; fIsFree = true; } // DIAG,FULL (or default to full) if (opt.find("DIAG") != std::string::npos) { fIsDiag = true; fIsFull = false; } else if (opt.find("FULL") != std::string::npos) { fIsDiag = false; fIsFull = true; } // CHI2/LL (OTHERS?) if (opt.find("LOG") != std::string::npos) { fIsChi2 = false; ERR(FTL) << "No other LIKELIHOODS properly supported!" << std::endl; ERR(FTL) << "Try to use a chi2!" << std::endl; throw; } else { fIsChi2 = true; } // EXTRAS if (opt.find("RAW") != std::string::npos) fIsRawEvents = true; if (opt.find("NOWIDTH") != std::string::npos) fIsNoWidth = true; if (opt.find("DIF") != std::string::npos) fIsDifXSec = true; if (opt.find("ENU1D") != std::string::npos) fIsEnu1D = true; if (opt.find("NORM") != std::string::npos) fAddNormPen = true; if (opt.find("MASK") != std::string::npos) fIsMask = true; return; }; //******************************************************************** void Measurement1D::SetSmearingMatrix(std::string smearfile, int truedim, int recodim) { //******************************************************************** // The smearing matrix describes the migration from true bins (rows) to reco // bins (columns) // Counter over the true bins! int row = 0; std::string line; std::ifstream smear(smearfile.c_str(), std::ifstream::in); // Note that the smearing matrix may be rectangular. fSmearMatrix = new TMatrixD(truedim, recodim); if (smear.is_open()) LOG(SAM) << "Reading smearing matrix from file: " << smearfile << std::endl; else ERR(FTL) << "Smearing matrix provided is incorrect: " << smearfile << std::endl; while (std::getline(smear >> std::ws, line, '\n')) { int column = 0; std::vector entries = GeneralUtils::ParseToDbl(line, " "); for (std::vector::iterator iter = entries.begin(); iter != entries.end(); iter++) { (*fSmearMatrix)(row, column) = (*iter) / 100.; // Convert to fraction from // percentage (this may not be // general enough) column++; } row++; } return; } //******************************************************************** void Measurement1D::ApplySmearingMatrix() { //******************************************************************** if (!fSmearMatrix) { ERR(WRN) << fName << ": attempted to apply smearing matrix, but none was set" << std::endl; return; } TH1D* unsmeared = (TH1D*)fMCHist->Clone(); TH1D* smeared = (TH1D*)fMCHist->Clone(); smeared->Reset(); // Loop over reconstructed bins // true = row; reco = column for (int rbin = 0; rbin < fSmearMatrix->GetNcols(); ++rbin) { // Sum up the constributions from all true bins double rBinVal = 0; // Loop over true bins for (int tbin = 0; tbin < fSmearMatrix->GetNrows(); ++tbin) { rBinVal += (*fSmearMatrix)(tbin, rbin) * unsmeared->GetBinContent(tbin + 1); } smeared->SetBinContent(rbin + 1, rBinVal); } fMCHist = (TH1D*)smeared->Clone(); return; } /* Reconfigure LOOP */ //******************************************************************** void Measurement1D::ResetAll() { //******************************************************************** fMCHist->Reset(); fMCFine->Reset(); fMCStat->Reset(); return; }; //******************************************************************** void Measurement1D::FillHistograms() { //******************************************************************** if (Signal) { QLOG(DEB, "Fill MCHist: " << fXVar << ", " << Weight); fMCHist->Fill(fXVar, Weight); fMCFine->Fill(fXVar, Weight); fMCStat->Fill(fXVar, 1.0); if (fMCHist_Modes) fMCHist_Modes->Fill(Mode, fXVar, Weight); } return; }; //******************************************************************** void Measurement1D::ScaleEvents() { //******************************************************************** // Fill MCWeighted; // for (int i = 0; i < fMCHist->GetNbinsX(); i++) { // fMCWeighted->SetBinContent(i + 1, fMCHist->GetBinContent(i + 1)); // fMCWeighted->SetBinError(i + 1, fMCHist->GetBinError(i + 1)); // } // Setup Stat ratios for MC and MC Fine double* statratio = new double[fMCHist->GetNbinsX()]; for (int i = 0; i < fMCHist->GetNbinsX(); i++) { if (fMCHist->GetBinContent(i + 1) != 0) { statratio[i] = fMCHist->GetBinError(i + 1) / fMCHist->GetBinContent(i + 1); } else { statratio[i] = 0.0; } } double* statratiofine = new double[fMCFine->GetNbinsX()]; for (int i = 0; i < fMCFine->GetNbinsX(); i++) { if (fMCFine->GetBinContent(i + 1) != 0) { statratiofine[i] = fMCFine->GetBinError(i + 1) / fMCFine->GetBinContent(i + 1); } else { statratiofine[i] = 0.0; } } // Scaling for raw event rates if (fIsRawEvents) { double datamcratio = fDataHist->Integral() / fMCHist->Integral(); fMCHist->Scale(datamcratio); fMCFine->Scale(datamcratio); if (fMCHist_Modes) fMCHist_Modes->Scale(datamcratio); // Scaling for XSec as function of Enu } else if (fIsEnu1D) { PlotUtils::FluxUnfoldedScaling(fMCHist, GetFluxHistogram(), GetEventHistogram(), fScaleFactor, fNEvents); PlotUtils::FluxUnfoldedScaling(fMCFine, GetFluxHistogram(), GetEventHistogram(), fScaleFactor, fNEvents); if (fMCHist_Modes) { // Loop over the modes fMCHist_Modes->FluxUnfold(GetFluxHistogram(), GetEventHistogram(), fScaleFactor, fNEvents); //PlotUtils::FluxUnfoldedScaling(fMCHist_Modes, GetFluxHistogram(), //GetEventHistogram(), fScaleFactor, //fNEvents); } } else if (fIsNoWidth) { fMCHist->Scale(fScaleFactor); fMCFine->Scale(fScaleFactor); if (fMCHist_Modes) fMCHist_Modes->Scale(fScaleFactor); // Any other differential scaling } else { fMCHist->Scale(fScaleFactor, "width"); fMCFine->Scale(fScaleFactor, "width"); if (fMCHist_Modes) fMCHist_Modes->Scale(fScaleFactor, "width"); } // Proper error scaling - ROOT Freaks out with xsec weights sometimes for (int i = 0; i < fMCStat->GetNbinsX(); i++) { fMCHist->SetBinError(i + 1, fMCHist->GetBinContent(i + 1) * statratio[i]); } for (int i = 0; i < fMCFine->GetNbinsX(); i++) { fMCFine->SetBinError(i + 1, fMCFine->GetBinContent(i + 1) * statratiofine[i]); } // Clean up delete[] statratio; delete[] statratiofine; return; }; //******************************************************************** void Measurement1D::ApplyNormScale(double norm) { //******************************************************************** fCurrentNorm = norm; fMCHist->Scale(1.0 / norm); fMCFine->Scale(1.0 / norm); return; }; /* Statistic Functions - Outsources to StatUtils */ //******************************************************************** int Measurement1D::GetNDOF() { //******************************************************************** int ndof = fDataHist->GetNbinsX(); if (fMaskHist and fIsMask) ndof -= fMaskHist->Integral(); return ndof; } //******************************************************************** double Measurement1D::GetLikelihood() { //******************************************************************** // If this is for a ratio, there is no data histogram to compare to! if (fNoData || !fDataHist) return 0.; // Apply Masking to MC if Required. if (fIsMask and fMaskHist) { PlotUtils::MaskBins(fMCHist, fMaskHist); } - // Sort Shape Scaling double scaleF = 0.0; // TODO Include !fIsRawEvents if (fIsShape) { if (fMCHist->Integral(1, fMCHist->GetNbinsX(), "width")) { scaleF = fDataHist->Integral(1, fDataHist->GetNbinsX(), "width") / fMCHist->Integral(1, fMCHist->GetNbinsX(), "width"); fMCHist->Scale(scaleF); fMCFine->Scale(scaleF); } } // Likelihood Calculation double stat = 0.; if (fIsChi2) { if (fIsRawEvents) { stat = StatUtils::GetChi2FromEventRate(fDataHist, fMCHist, fMaskHist); } else if (fIsDiag) { stat = StatUtils::GetChi2FromDiag(fDataHist, fMCHist, fMaskHist); } else if (!fIsDiag and !fIsRawEvents) { stat = StatUtils::GetChi2FromCov(fDataHist, fMCHist, covar, fMaskHist); } } // Sort Penalty Terms if (fAddNormPen) { double penalty = (1. - fCurrentNorm) * (1. - fCurrentNorm) / (fNormError * fNormError); stat += penalty; } // Return to normal scaling if (fIsShape) { // and !FitPar::Config().GetParB("saveshapescaling")) { fMCHist->Scale(1. / scaleF); fMCFine->Scale(1. / scaleF); } fLikelihood = stat; return stat; } /* Fake Data Functions */ //******************************************************************** void Measurement1D::SetFakeDataValues(std::string fakeOption) { //******************************************************************** // Setup original/datatrue TH1D* tempdata = (TH1D*) fDataHist->Clone(); if (!fIsFakeData) { fIsFakeData = true; // Make a copy of the original data histogram. if (!fDataOrig) fDataOrig = (TH1D*)fDataHist->Clone((fName + "_data_original").c_str()); } else { ResetFakeData(); } // Setup Inputs fFakeDataInput = fakeOption; LOG(SAM) << "Setting fake data from : " << fFakeDataInput << std::endl; // From MC if (fFakeDataInput.compare("MC") == 0) { fDataHist = (TH1D*)fMCHist->Clone((fName + "_MC").c_str()); // Fake File } else { if (!fFakeDataFile) fFakeDataFile = new TFile(fFakeDataInput.c_str(), "READ"); fDataHist = (TH1D*)fFakeDataFile->Get((fName + "_MC").c_str()); } // Setup Data Hist fDataHist->SetNameTitle((fName + "_FAKE").c_str(), (fName + fPlotTitles).c_str()); // Replace Data True if (fDataTrue) delete fDataTrue; fDataTrue = (TH1D*)fDataHist->Clone(); fDataTrue->SetNameTitle((fName + "_FAKE_TRUE").c_str(), (fName + fPlotTitles).c_str()); // Make a new covariance for fake data hist. int nbins = fDataHist->GetNbinsX(); double alpha_i = 0.0; double alpha_j = 0.0; for (int i = 0; i < nbins; i++) { for (int j = 0; j < nbins; j++) { alpha_i = fDataHist->GetBinContent(i + 1) / tempdata->GetBinContent(i + 1); alpha_j = fDataHist->GetBinContent(j + 1) / tempdata->GetBinContent(j + 1); (*fFullCovar)(i, j) = alpha_i * alpha_j * (*fFullCovar)(i, j); } } // Setup Covariances if (covar) delete covar; covar = StatUtils::GetInvert(fFullCovar); if (fDecomp) delete fDecomp; fDecomp = StatUtils::GetInvert(fFullCovar); delete tempdata; return; }; //******************************************************************** void Measurement1D::ResetFakeData() { //******************************************************************** if (fIsFakeData) { if (fDataHist) delete fDataHist; fDataHist = (TH1D*)fDataTrue->Clone((fSettings.GetName() + "_FKDAT").c_str()); } } //******************************************************************** void Measurement1D::ResetData() { //******************************************************************** if (fIsFakeData) { if (fDataHist) delete fDataHist; fDataHist = (TH1D*)fDataOrig->Clone((fSettings.GetName() + "_data").c_str()); } fIsFakeData = false; } //******************************************************************** void Measurement1D::ThrowCovariance() { //******************************************************************** // Take a fDecomposition and use it to throw the current dataset. // Requires fDataTrue also be set incase used repeatedly. if (!fDataTrue) fDataTrue = (TH1D*) fDataHist->Clone(); if (fDataHist) delete fDataHist; fDataHist = StatUtils::ThrowHistogram(fDataTrue, fFullCovar); return; }; //******************************************************************** void Measurement1D::ThrowDataToy(){ //******************************************************************** if (!fDataTrue) fDataTrue = (TH1D*) fDataHist->Clone(); if (fMCHist) delete fMCHist; fMCHist = StatUtils::ThrowHistogram(fDataTrue, fFullCovar); } /* Access Functions */ //******************************************************************** TH1D* Measurement1D::GetMCHistogram() { //******************************************************************** if (!fMCHist) return fMCHist; std::ostringstream chi2; chi2 << std::setprecision(5) << this->GetLikelihood(); int linecolor = kRed; int linestyle = 1; int linewidth = 1; int fillcolor = 0; int fillstyle = 1001; // if (fSettings.Has("linecolor")) linecolor = fSettings.GetI("linecolor"); // if (fSettings.Has("linestyle")) linestyle = fSettings.GetI("linestyle"); // if (fSettings.Has("linewidth")) linewidth = fSettings.GetI("linewidth"); // if (fSettings.Has("fillcolor")) fillcolor = fSettings.GetI("fillcolor"); // if (fSettings.Has("fillstyle")) fillstyle = fSettings.GetI("fillstyle"); fMCHist->SetTitle(chi2.str().c_str()); fMCHist->SetLineColor(linecolor); fMCHist->SetLineStyle(linestyle); fMCHist->SetLineWidth(linewidth); fMCHist->SetFillColor(fillcolor); fMCHist->SetFillStyle(fillstyle); return fMCHist; }; //******************************************************************** TH1D* Measurement1D::GetDataHistogram() { //******************************************************************** if (!fDataHist) return fDataHist; int datacolor = kBlack; int datastyle = 1; int datawidth = 1; // if (fSettings.Has("datacolor")) datacolor = fSettings.GetI("datacolor"); // if (fSettings.Has("datastyle")) datastyle = fSettings.GetI("datastyle"); // if (fSettings.Has("datawidth")) datawidth = fSettings.GetI("datawidth"); fDataHist->SetLineColor(datacolor); fDataHist->SetLineWidth(datawidth); fDataHist->SetMarkerStyle(datastyle); return fDataHist; }; /* Write Functions */ // Save all the histograms at once //******************************************************************** void Measurement1D::Write(std::string drawOpt) { //******************************************************************** // Get Draw Options drawOpt = FitPar::Config().GetParS("drawopts"); // Write Settigns if (drawOpt.find("SETTINGS") != std::string::npos){ fSettings.Set("#chi^{2}",fLikelihood); fSettings.Set("NDOF", this->GetNDOF() ); fSettings.Set("#chi^{2}/NDOF", fLikelihood / this->GetNDOF() ); fSettings.Write(); } // Write Data/MC if (drawOpt.find("DATA") != std::string::npos) GetDataList().at(0)->Write(); if (drawOpt.find("MC") != std::string::npos) { GetMCList().at(0)->Write(); if((fEvtRateScaleFactor != 0xdeadbeef) && GetMCList().at(0)){ TH1D * PredictedEvtRate = static_cast(GetMCList().at(0)->Clone()); PredictedEvtRate->Scale(fEvtRateScaleFactor); PredictedEvtRate->GetYaxis()->SetTitle("Predicted event rate"); PredictedEvtRate->Write(); } } // Write Fine Histogram if (drawOpt.find("FINE") != std::string::npos) GetFineList().at(0)->Write(); // Write Weighted Histogram if (drawOpt.find("WEIGHTS") != std::string::npos && fMCWeighted) fMCWeighted->Write(); // Save Flux/Evt if no event manager if (!FitPar::Config().GetParB("EventManager")) { if (drawOpt.find("FLUX") != std::string::npos && GetFluxHistogram()) GetFluxHistogram()->Write(); if (drawOpt.find("EVT") != std::string::npos && GetEventHistogram()) GetEventHistogram()->Write(); if (drawOpt.find("XSEC") != std::string::npos && GetEventHistogram()) GetXSecHistogram()->Write(); } // Write Mask if (fIsMask && (drawOpt.find("MASK") != std::string::npos)) { fMaskHist->Write(); } // Write Covariances if (drawOpt.find("COV") != std::string::npos && fFullCovar) { PlotUtils::GetFullCovarPlot(fFullCovar, fSettings.GetName()); } if (drawOpt.find("INVCOV") != std::string::npos && covar) { PlotUtils::GetInvCovarPlot(covar, fSettings.GetName()); } if (drawOpt.find("DECOMP") != std::string::npos && fDecomp) { PlotUtils::GetDecompCovarPlot(fDecomp, fSettings.GetName()); } // // Likelihood residual plots // if (drawOpt.find("RESIDUAL") != std::string::npos) { // WriteResidualPlots(); // } // Ratio and Shape Plots if (drawOpt.find("RATIO") != std::string::npos) { WriteRatioPlot(); } if (drawOpt.find("SHAPE") != std::string::npos) { WriteShapePlot(); if (drawOpt.find("RATIO") != std::string::npos) WriteShapeRatioPlot(); } // // RATIO // if (drawOpt.find("CANVMC") != std::string::npos) { // TCanvas* c1 = WriteMCCanvas(fDataHist, fMCHist); // c1->Write(); // delete c1; // } // // PDG // if (drawOpt.find("CANVPDG") != std::string::npos && fMCHist_Modes) { // TCanvas* c2 = WritePDGCanvas(fDataHist, fMCHist, fMCHist_Modes); // c2->Write(); // delete c2; // } // Write Extra Histograms AutoWriteExtraTH1(); WriteExtraHistograms(); // Returning LOG(SAM) << "Written Histograms: " << fName << std::endl; return; } //******************************************************************** void Measurement1D::WriteRatioPlot() { //******************************************************************** // Setup mc data ratios TH1D* dataRatio = (TH1D*)fDataHist->Clone((fName + "_data_RATIO").c_str()); TH1D* mcRatio = (TH1D*)fMCHist->Clone((fName + "_MC_RATIO").c_str()); // Extra MC Data Ratios for (int i = 0; i < mcRatio->GetNbinsX(); i++) { dataRatio->SetBinContent(i + 1, fDataHist->GetBinContent(i + 1) / fMCHist->GetBinContent(i + 1)); dataRatio->SetBinError(i + 1, fDataHist->GetBinError(i + 1) / fMCHist->GetBinContent(i + 1)); mcRatio->SetBinContent(i + 1, fMCHist->GetBinContent(i + 1) / fMCHist->GetBinContent(i + 1)); mcRatio->SetBinError(i + 1, fMCHist->GetBinError(i + 1) / fMCHist->GetBinContent(i + 1)); } // Write ratios mcRatio->Write(); dataRatio->Write(); delete mcRatio; delete dataRatio; } //******************************************************************** void Measurement1D::WriteShapePlot() { //******************************************************************** TH1D* mcShape = (TH1D*)fMCHist->Clone((fName + "_MC_SHAPE").c_str()); TH1D* dataShape = (TH1D*)fDataHist->Clone((fName + "_data_SHAPE").c_str()); // Don't check error if (fShapeCovar) StatUtils::SetDataErrorFromCov(dataShape, fShapeCovar, 1E-38, false); double shapeScale = 1.0; if (fIsRawEvents) { shapeScale = fDataHist->Integral() / fMCHist->Integral(); } else { shapeScale = fDataHist->Integral("width") / fMCHist->Integral("width"); } mcShape->Scale(shapeScale); std::stringstream ss; ss << shapeScale; mcShape->SetTitle(ss.str().c_str()); mcShape->SetLineWidth(3); mcShape->SetLineStyle(7); mcShape->Write(); dataShape->Write(); delete mcShape; } //******************************************************************** void Measurement1D::WriteShapeRatioPlot() { //******************************************************************** // Get a mcshape histogram TH1D* mcShape = (TH1D*)fMCHist->Clone((fName + "_MC_SHAPE").c_str()); double shapeScale = 1.0; if (fIsRawEvents) { shapeScale = fDataHist->Integral() / fMCHist->Integral(); } else { shapeScale = fDataHist->Integral("width") / fMCHist->Integral("width"); } mcShape->Scale(shapeScale); // Create shape ratio histograms TH1D* mcShapeRatio = (TH1D*)mcShape->Clone((fName + "_MC_SHAPE_RATIO").c_str()); TH1D* dataShapeRatio = (TH1D*)fDataHist->Clone((fName + "_data_SHAPE_RATIO").c_str()); // Divide the histograms mcShapeRatio->Divide(mcShape); dataShapeRatio->Divide(mcShape); // Colour the shape ratio plots mcShapeRatio->SetLineWidth(3); mcShapeRatio->SetLineStyle(7); mcShapeRatio->Write(); dataShapeRatio->Write(); delete mcShapeRatio; delete dataShapeRatio; } //// CRAP TO BE REMOVED //******************************************************************** void Measurement1D::SetupMeasurement(std::string inputfile, std::string type, FitWeight * rw, std::string fkdt) { //******************************************************************** nuiskey samplekey = Config::CreateKey("sample"); samplekey.Set("name", fName); samplekey.Set("type",type); samplekey.Set("input",inputfile); fSettings = LoadSampleSettings(samplekey); // Reset everything to NULL // Init(); // Check if name contains Evt, indicating that it is a raw number of events // measurements and should thus be treated as once fIsRawEvents = false; if ((fName.find("Evt") != std::string::npos) && fIsRawEvents == false) { fIsRawEvents = true; LOG(SAM) << "Found event rate measurement but fIsRawEvents == false!" << std::endl; LOG(SAM) << "Overriding this and setting fIsRawEvents == true!" << std::endl; } fIsEnu1D = false; if (fName.find("XSec_1DEnu") != std::string::npos) { fIsEnu1D = true; LOG(SAM) << "::" << fName << "::" << std::endl; LOG(SAM) << "Found XSec Enu measurement, applying flux integrated scaling, " "not flux averaged!" << std::endl; } if (fIsEnu1D && fIsRawEvents) { LOG(SAM) << "Found 1D Enu XSec distribution AND fIsRawEvents, is this " "really correct?!" << std::endl; LOG(SAM) << "Check experiment constructor for " << fName << " and correct this!" << std::endl; LOG(SAM) << "I live in " << __FILE__ << ":" << __LINE__ << std::endl; throw; } fRW = rw; if (!fInput and !fIsJoint) SetupInputs(inputfile); // Set Default Options SetFitOptions(fDefaultTypes); // Set Passed Options SetFitOptions(type); // Still adding support for flat flux inputs // // Set Enu Flux Scaling // if (isFlatFluxFolding) this->Input()->ApplyFluxFolding( // this->defaultFluxHist ); // FinaliseMeasurement(); } //******************************************************************** void Measurement1D::SetupDefaultHist() { //******************************************************************** // Setup fMCHist fMCHist = (TH1D*)fDataHist->Clone(); fMCHist->SetNameTitle((fName + "_MC").c_str(), (fName + "_MC" + fPlotTitles).c_str()); // Setup fMCFine Int_t nBins = fMCHist->GetNbinsX(); fMCFine = new TH1D( (fName + "_MC_FINE").c_str(), (fName + "_MC_FINE" + fPlotTitles).c_str(), nBins * 6, fMCHist->GetBinLowEdge(1), fMCHist->GetBinLowEdge(nBins + 1)); fMCStat = (TH1D*)fMCHist->Clone(); fMCStat->Reset(); fMCHist->Reset(); fMCFine->Reset(); // Setup the NEUT Mode Array PlotUtils::CreateNeutModeArray((TH1D*)fMCHist, (TH1**)fMCHist_PDG); PlotUtils::ResetNeutModeArray((TH1**)fMCHist_PDG); // Setup bin masks using sample name if (fIsMask) { std::string maskloc = FitPar::Config().GetParDIR(fName + ".mask"); if (maskloc.empty()) { maskloc = FitPar::GetDataBase() + "/masks/" + fName + ".mask"; } SetBinMask(maskloc); } fMCHist_Modes = new TrueModeStack( (fName + "_MODES").c_str(), ("True Channels"), fMCHist); SetAutoProcessTH1(fMCHist_Modes, kCMD_Reset, kCMD_Norm, kCMD_Write); return; } //******************************************************************** void Measurement1D::SetDataValues(std::string dataFile) { //******************************************************************** // Override this function if the input file isn't in a suitable format LOG(SAM) << "Reading data from: " << dataFile.c_str() << std::endl; fDataHist = PlotUtils::GetTH1DFromFile(dataFile, (fName + "_data"), fPlotTitles); fDataTrue = (TH1D*)fDataHist->Clone(); // Number of data points is number of bins fNDataPointsX = fDataHist->GetXaxis()->GetNbins(); return; }; //******************************************************************** void Measurement1D::SetDataFromDatabase(std::string inhistfile, std::string histname) { //******************************************************************** LOG(SAM) << "Filling histogram from " << inhistfile << "->" << histname << std::endl; fDataHist = PlotUtils::GetTH1DFromRootFile( (GeneralUtils::GetTopLevelDir() + "/data/" + inhistfile), histname); fDataHist->SetNameTitle((fName + "_data").c_str(), (fName + "_data").c_str()); return; }; //******************************************************************** void Measurement1D::SetDataFromFile(std::string inhistfile, std::string histname) { //******************************************************************** LOG(SAM) << "Filling histogram from " << inhistfile << "->" << histname << std::endl; fDataHist = PlotUtils::GetTH1DFromRootFile((inhistfile), histname); fDataHist->SetNameTitle((fName + "_data").c_str(), (fName + "_data").c_str()); return; }; //******************************************************************** void Measurement1D::SetCovarMatrix(std::string covarFile) { //******************************************************************** // Covariance function, only really used when reading in the MB Covariances. TFile* tempFile = new TFile(covarFile.c_str(), "READ"); TH2D* covarPlot = new TH2D(); TH2D* fFullCovarPlot = new TH2D(); std::string covName = ""; std::string covOption = FitPar::Config().GetParS("thrown_covariance"); if (fIsShape || fIsFree) covName = "shp_"; if (fIsDiag) covName += "diag"; else covName += "full"; covarPlot = (TH2D*)tempFile->Get((covName + "cov").c_str()); if (!covOption.compare("SUB")) fFullCovarPlot = (TH2D*)tempFile->Get((covName + "cov").c_str()); else if (!covOption.compare("FULL")) fFullCovarPlot = (TH2D*)tempFile->Get("fullcov"); else ERR(WRN) << "Incorrect thrown_covariance option in parameters." << std::endl; int dim = int(fDataHist->GetNbinsX()); //-this->masked->Integral()); int covdim = int(fDataHist->GetNbinsX()); this->covar = new TMatrixDSym(dim); fFullCovar = new TMatrixDSym(dim); fDecomp = new TMatrixDSym(dim); int row, column = 0; row = 0; column = 0; for (Int_t i = 0; i < covdim; i++) { // if (this->masked->GetBinContent(i+1) > 0) continue; for (Int_t j = 0; j < covdim; j++) { // if (this->masked->GetBinContent(j+1) > 0) continue; (*this->covar)(row, column) = covarPlot->GetBinContent(i + 1, j + 1); (*fFullCovar)(row, column) = fFullCovarPlot->GetBinContent(i + 1, j + 1); column++; } column = 0; row++; } // Set bin errors on data if (!fIsDiag) { StatUtils::SetDataErrorFromCov(fDataHist, fFullCovar); } // Get Deteriminant and inverse matrix // fCovDet = this->covar->Determinant(); TDecompSVD LU = TDecompSVD(*this->covar); this->covar = new TMatrixDSym(dim, LU.Invert().GetMatrixArray(), ""); return; }; //******************************************************************** // Sets the covariance matrix from a provided file in a text format // scale is a multiplicative pre-factor to apply in the case where the // covariance is given in some unit (e.g. 1E-38) void Measurement1D::SetCovarMatrixFromText(std::string covarFile, int dim, double scale) { //******************************************************************** // Make a counter to track the line number int row = 0; std::string line; std::ifstream covarread(covarFile.c_str(), std::ifstream::in); this->covar = new TMatrixDSym(dim); fFullCovar = new TMatrixDSym(dim); if (covarread.is_open()) LOG(SAM) << "Reading covariance matrix from file: " << covarFile << std::endl; else ERR(FTL) << "Covariance matrix provided is incorrect: " << covarFile << std::endl; // Loop over the lines in the file while (std::getline(covarread >> std::ws, line, '\n')) { int column = 0; // Loop over entries and insert them into matrix std::vector entries = GeneralUtils::ParseToDbl(line, " "); if (entries.size() <= 1) { ERR(WRN) << "SetCovarMatrixFromText -> Covariance matrix only has <= 1 " "entries on this line: " << row << std::endl; } for (std::vector::iterator iter = entries.begin(); iter != entries.end(); iter++) { (*covar)(row, column) = *iter; (*fFullCovar)(row, column) = *iter; column++; } row++; } covarread.close(); // Scale the actualy covariance matrix by some multiplicative factor (*fFullCovar) *= scale; // Robust matrix inversion method TDecompSVD LU = TDecompSVD(*this->covar); // THIS IS ACTUALLY THE INVERSE COVARIANCE MATRIXA AAAAARGH delete this->covar; this->covar = new TMatrixDSym(dim, LU.Invert().GetMatrixArray(), ""); // Now need to multiply by the scaling factor // If the covariance (*this->covar) *= 1. / (scale); return; }; //******************************************************************** void Measurement1D::SetCovarMatrixFromCorrText(std::string corrFile, int dim) { //******************************************************************** // Make a counter to track the line number int row = 0; std::string line; std::ifstream corr(corrFile.c_str(), std::ifstream::in); this->covar = new TMatrixDSym(dim); this->fFullCovar = new TMatrixDSym(dim); if (corr.is_open()) LOG(SAM) << "Reading and converting correlation matrix from file: " << corrFile << std::endl; else { ERR(FTL) << "Correlation matrix provided is incorrect: " << corrFile << std::endl; exit(-1); } while (std::getline(corr >> std::ws, line, '\n')) { int column = 0; // Loop over entries and insert them into matrix // Multiply by the errors to get the covariance, rather than the correlation // matrix std::vector entries = GeneralUtils::ParseToDbl(line, " "); for (std::vector::iterator iter = entries.begin(); iter != entries.end(); iter++) { double val = (*iter) * this->fDataHist->GetBinError(row + 1) * 1E38 * this->fDataHist->GetBinError(column + 1) * 1E38; if (val == 0) { ERR(FTL) << "Found a zero value in the covariance matrix, assuming " "this is an error!" << std::endl; exit(-1); } (*this->covar)(row, column) = val; (*this->fFullCovar)(row, column) = val; column++; } row++; } // Robust matrix inversion method TDecompSVD LU = TDecompSVD(*this->covar); delete this->covar; this->covar = new TMatrixDSym(dim, LU.Invert().GetMatrixArray(), ""); return; }; //******************************************************************** // FullUnits refers to if we have "real" unscaled units in the covariance matrix, e.g. 1E-76. // If this is the case we need to scale it so that the chi2 contribution is correct // NUISANCE internally assumes the covariance matrix has units of 1E76 void Measurement1D::SetCovarFromDataFile(std::string covarFile, std::string covName, bool FullUnits) { //******************************************************************** LOG(SAM) << "Getting covariance from " << covarFile << "->" << covName << std::endl; TFile* tempFile = new TFile(covarFile.c_str(), "READ"); TH2D* covPlot = (TH2D*)tempFile->Get(covName.c_str()); covPlot->SetDirectory(0); // Scale the covariance matrix if it comes in normal units if (FullUnits) { covPlot->Scale(1.E76); } int dim = covPlot->GetNbinsX(); fFullCovar = new TMatrixDSym(dim); for (int i = 0; i < dim; i++) { for (int j = 0; j < dim; j++) { (*fFullCovar)(i, j) = covPlot->GetBinContent(i + 1, j + 1); } } this->covar = (TMatrixDSym*)fFullCovar->Clone(); fDecomp = (TMatrixDSym*)fFullCovar->Clone(); TDecompSVD LU = TDecompSVD(*this->covar); this->covar = new TMatrixDSym(dim, LU.Invert().GetMatrixArray(), ""); TDecompChol LUChol = TDecompChol(*fDecomp); LUChol.Decompose(); fDecomp = new TMatrixDSym(dim, LU.GetU().GetMatrixArray(), ""); return; }; // //******************************************************************** // void Measurement1D::SetBinMask(std::string maskFile) { // //******************************************************************** // // Create a mask histogram. // int nbins = fDataHist->GetNbinsX(); // fMaskHist = // new TH1I((fName + "_fMaskHist").c_str(), // (fName + "_fMaskHist; Bin; Mask?").c_str(), nbins, 0, nbins); // std::string line; // std::ifstream mask(maskFile.c_str(), std::ifstream::in); // if (mask.is_open()) // LOG(SAM) << "Reading bin mask from file: " << maskFile << std::endl; // else // LOG(FTL) << " Cannot find mask file." << std::endl; // while (std::getline(mask >> std::ws, line, '\n')) { // std::vector entries = GeneralUtils::ParseToInt(line, " "); // // Skip lines with poorly formatted lines // if (entries.size() < 2) { // LOG(WRN) << "Measurement1D::SetBinMask(), couldn't parse line: " << line // << std::endl; // continue; // } // // The first index should be the bin number, the second should be the mask // // value. // fMaskHist->SetBinContent(entries[0], entries[1]); // } // // Set masked data bins to zero // PlotUtils::MaskBins(fDataHist, fMaskHist); // return; // } // //******************************************************************** // void Measurement1D::GetBinContents(std::vector& cont, // std::vector& err) { // //******************************************************************** // // Return a vector of the main bin contents // for (int i = 0; i < fMCHist->GetNbinsX(); i++) { // cont.push_back(fMCHist->GetBinContent(i + 1)); // err.push_back(fMCHist->GetBinError(i + 1)); // } // return; // }; /* XSec Functions */ // //******************************************************************** // void Measurement1D::SetFluxHistogram(std::string fluxFile, int minE, int // maxE, // double fluxNorm) { // //******************************************************************** // // Note this expects the flux bins to be given in terms of MeV // LOG(SAM) << "Reading flux from file: " << fluxFile << std::endl; // TGraph f(fluxFile.c_str(), "%lg %lg"); // fFluxHist = // new TH1D((fName + "_flux").c_str(), (fName + "; E_{#nu} (GeV)").c_str(), // f.GetN() - 1, minE, maxE); // Double_t* yVal = f.GetY(); // for (int i = 0; i < fFluxHist->GetNbinsX(); ++i) // fFluxHist->SetBinContent(i + 1, yVal[i] * fluxNorm); // }; // //******************************************************************** // double Measurement1D::TotalIntegratedFlux(std::string intOpt, double low, // double high) { // //******************************************************************** // if (fInput->GetType() == kGiBUU) { // return 1.0; // } // // The default case of low = -9999.9 and high = -9999.9 // if (low == -9999.9) low = this->EnuMin; // if (high == -9999.9) high = this->EnuMax; // int minBin = fFluxHist->GetXaxis()->FindBin(low); // int maxBin = fFluxHist->GetXaxis()->FindBin(high); // // Get integral over custom range // double integral = fFluxHist->Integral(minBin, maxBin + 1, intOpt.c_str()); // return integral; // }; diff --git a/src/InputHandler/InputHandler.cxx b/src/InputHandler/InputHandler.cxx index d606944..e7b3adb 100644 --- a/src/InputHandler/InputHandler.cxx +++ b/src/InputHandler/InputHandler.cxx @@ -1,298 +1,299 @@ // Copyright 2016 L. Pickering, P Stowell, R. Terri, C. Wilkinson, C. Wret /******************************************************************************* * This file is part of NUISANCE. * * NUISANCE is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * NUISANCE is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with NUISANCE. If not, see . *******************************************************************************/ #include "InputHandler.h" #include "InputUtils.h" InputHandlerBase::InputHandlerBase() { fName = ""; fFluxHist = NULL; fEventHist = NULL; fNEvents = 0; fNUISANCEEvent = NULL; fBaseEvent = NULL; kRemoveUndefParticles = FitPar::Config().GetParB("RemoveUndefParticles"); kRemoveFSIParticles = FitPar::Config().GetParB("RemoveFSIParticles"); kRemoveNuclearParticles = FitPar::Config().GetParB("RemoveNuclearParticles"); fMaxEvents = FitPar::Config().GetParI("MAXEVENTS"); fTTreePerformance = NULL; }; InputHandlerBase::~InputHandlerBase() { if (fFluxHist) delete fFluxHist; if (fEventHist) delete fEventHist; // if (fXSecHist) delete fXSecHist; // if (fNUISANCEEvent) delete fNUISANCEEvent; jointfluxinputs.clear(); jointeventinputs.clear(); jointindexlow.clear(); jointindexhigh.clear(); jointindexallowed.clear(); jointindexscale.clear(); // if (fTTreePerformance) { // fTTreePerformance->SaveAs(("ttreeperfstats_" + fName + // ".root").c_str()); // } } void InputHandlerBase::Print(){}; TH1D* InputHandlerBase::GetXSecHistogram(void) { fXSecHist = (TH1D*)fFluxHist->Clone(); fXSecHist->Divide(fEventHist); return fXSecHist; }; double InputHandlerBase::PredictedEventRate(double low, double high, std::string intOpt) { Int_t minBin = fEventHist->GetXaxis()->FindFixBin(low); Int_t maxBin = fEventHist->GetXaxis()->FindFixBin(high); if ((fEventHist->IsBinOverflow(minBin) && (low != -9999.9))) { minBin = 1; } if ((fEventHist->IsBinOverflow(maxBin) && (high != -9999.9))) { maxBin = fEventHist->GetXaxis()->GetNbins() + 1; } // If we are within a single bin if (minBin == maxBin) { // Get the contained fraction of the single bin's width return ((high - low) / fEventHist->GetXaxis()->GetBinWidth(minBin)) * fEventHist->Integral(minBin, minBin, intOpt.c_str()); } double lowBinUpEdge = fEventHist->GetXaxis()->GetBinUpEdge(minBin); double highBinLowEdge = fEventHist->GetXaxis()->GetBinLowEdge(maxBin); double lowBinfracIntegral = ((lowBinUpEdge - low) / fEventHist->GetXaxis()->GetBinWidth(minBin)) * fEventHist->Integral(minBin, minBin, intOpt.c_str()); double highBinfracIntegral = ((high - highBinLowEdge) / fEventHist->GetXaxis()->GetBinWidth(maxBin)) * fEventHist->Integral(maxBin, maxBin, intOpt.c_str()); // If they are neighbouring bins if ((minBin + 1) == maxBin) { std::cout << "Get lowfrac + highfrac" << std::endl; // Get the contained fraction of the two bin's width return lowBinfracIntegral + highBinfracIntegral; } double ContainedIntegral = fEventHist->Integral(minBin + 1, maxBin - 1, intOpt.c_str()); // If there are filled bins between them return lowBinfracIntegral + highBinfracIntegral + ContainedIntegral; }; double InputHandlerBase::TotalIntegratedFlux(double low, double high, std::string intOpt) { Int_t minBin = fFluxHist->GetXaxis()->FindFixBin(low); Int_t maxBin = fFluxHist->GetXaxis()->FindFixBin(high); if ((fFluxHist->IsBinOverflow(minBin) && (low != -9999.9))) { minBin = 1; } if ((fFluxHist->IsBinOverflow(maxBin) && (high != -9999.9))) { maxBin = fFluxHist->GetXaxis()->GetNbins(); high = fFluxHist->GetXaxis()->GetBinLowEdge(maxBin+1); } // If we are within a single bin if (minBin == maxBin) { // Get the contained fraction of the single bin's width return ((high - low) / fFluxHist->GetXaxis()->GetBinWidth(minBin)) * fFluxHist->Integral(minBin, minBin, intOpt.c_str()); } double lowBinUpEdge = fFluxHist->GetXaxis()->GetBinUpEdge(minBin); double highBinLowEdge = fFluxHist->GetXaxis()->GetBinLowEdge(maxBin); double lowBinfracIntegral = ((lowBinUpEdge - low) / fFluxHist->GetXaxis()->GetBinWidth(minBin)) * fFluxHist->Integral(minBin, minBin, intOpt.c_str()); double highBinfracIntegral = ((high - highBinLowEdge) / fFluxHist->GetXaxis()->GetBinWidth(maxBin)) * fFluxHist->Integral(maxBin, maxBin, intOpt.c_str()); // If they are neighbouring bins if ((minBin + 1) == maxBin) { std::cout << "Get lowfrac + highfrac" << std::endl; // Get the contained fraction of the two bin's width return lowBinfracIntegral + highBinfracIntegral; } double ContainedIntegral = fFluxHist->Integral(minBin + 1, maxBin - 1, intOpt.c_str()); // If there are filled bins between them return lowBinfracIntegral + highBinfracIntegral + ContainedIntegral; } std::vector InputHandlerBase::GetFluxList(void) { return std::vector(1, fFluxHist); }; std::vector InputHandlerBase::GetEventList(void) { return std::vector(1, fEventHist); }; std::vector InputHandlerBase::GetXSecList(void) { return std::vector(1, GetXSecHistogram()); }; FitEvent* InputHandlerBase::FirstNuisanceEvent() { fCurrentIndex = 0; return GetNuisanceEvent(fCurrentIndex); }; FitEvent* InputHandlerBase::NextNuisanceEvent() { fCurrentIndex++; if ((fMaxEvents != -1) && (fCurrentIndex > fMaxEvents)) { return NULL; } return GetNuisanceEvent(fCurrentIndex); }; BaseFitEvt* InputHandlerBase::FirstBaseEvent() { fCurrentIndex = 0; return GetBaseEvent(fCurrentIndex); }; BaseFitEvt* InputHandlerBase::NextBaseEvent() { fCurrentIndex++; if (jointinput and fMaxEvents != -1) { while (fCurrentIndex < jointindexlow[jointindexswitch] || fCurrentIndex >= jointindexhigh[jointindexswitch]) { jointindexswitch++; // Loop Around if (jointindexswitch == jointindexlow.size()) { jointindexswitch = 0; } } if (fCurrentIndex > jointindexlow[jointindexswitch] + jointindexallowed[jointindexswitch]) { fCurrentIndex = jointindexlow[jointindexswitch]; } } return GetBaseEvent(fCurrentIndex); }; void InputHandlerBase::RegisterJointInput(std::string input, int n, TH1D* f, TH1D* e) { if (jointfluxinputs.size() == 0) { jointindexswitch = 0; fNEvents = 0; } // Push into individual input vectors jointfluxinputs.push_back((TH1D*)f->Clone()); jointeventinputs.push_back((TH1D*)e->Clone()); jointindexlow.push_back(fNEvents); jointindexhigh.push_back(fNEvents + n); fNEvents += n; // Add to the total flux/event hist if (!fFluxHist) fFluxHist = (TH1D*)f->Clone(); else fFluxHist->Add(f); if (!fEventHist) fEventHist = (TH1D*)e->Clone(); else fEventHist->Add(e); } void InputHandlerBase::SetupJointInputs() { if (jointeventinputs.size() <= 1) { jointinput = false; } else if (jointeventinputs.size() > 1) { jointinput = true; jointindexswitch = 0; } fMaxEvents = FitPar::Config().GetParI("MAXEVENTS"); if (fMaxEvents != -1 and jointeventinputs.size() > 1) { THROW("Can only handle joint inputs when config MAXEVENTS = -1!"); } if (jointeventinputs.size() > 1) { ERROR(WRN, "GiBUU sample contains multiple inputs. This will only work for " "samples that expect multi-species inputs. If this sample does, you " "can ignore this warning."); } for (size_t i = 0; i < jointeventinputs.size(); i++) { double scale = double(fNEvents) / fEventHist->Integral("width"); scale *= jointeventinputs.at(i)->Integral("width"); scale /= double(jointindexhigh[i] - jointindexlow[i]); jointindexscale.push_back(scale); } fEventHist->SetNameTitle((fName + "_EVT").c_str(), (fName + "_EVT").c_str()); fFluxHist->SetNameTitle((fName + "_FLUX").c_str(), (fName + "_FLUX").c_str()); // Setup Max Events if (fMaxEvents > 1 && fMaxEvents < fNEvents) { if (LOG_LEVEL(SAM)) { std::cout << "\t\t|-> Read Max Entries : " << fMaxEvents << std::endl; } fNEvents = fMaxEvents; } // Print out Status if (LOG_LEVEL(SAM)) { std::cout << "\t\t|-> Total Entries : " << fNEvents << std::endl << "\t\t|-> Event Integral : " << fEventHist->Integral("width") * 1.E-38 << " events/nucleon" << std::endl << "\t\t|-> Flux Integral : " << fFluxHist->Integral("width") << " /cm2" << std::endl << "\t\t|-> Event/Flux : " << fEventHist->Integral("width") * 1.E-38 / fFluxHist->Integral("width") << " cm2/nucleon" << std::endl; } } BaseFitEvt* InputHandlerBase::GetBaseEvent(const UInt_t entry) { + // Do some light processing: don't calculate the kinematics return static_cast(GetNuisanceEvent(entry, true)); } double InputHandlerBase::GetInputWeight(int entry) { if (!jointinput) return 1.0; // Find Switch Scale while (entry < jointindexlow[jointindexswitch] || entry >= jointindexhigh[jointindexswitch]) { jointindexswitch++; // Loop Around if (jointindexswitch >= jointindexlow.size()) { jointindexswitch = 0; } } return jointindexscale[jointindexswitch]; }; diff --git a/src/Routines/SystematicRoutines.cxx b/src/Routines/SystematicRoutines.cxx index 04c98f6..24066b8 100755 --- a/src/Routines/SystematicRoutines.cxx +++ b/src/Routines/SystematicRoutines.cxx @@ -1,1437 +1,1418 @@ // Copyright 2016 L. Pickering, P Stowell, R. Terri, C. Wilkinson, C. Wret /******************************************************************************* * This file is part of NUISANCE. * * NUISANCE is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * NUISANCE is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with NUISANCE. If not, see . *******************************************************************************/ #include "SystematicRoutines.h" void SystematicRoutines::Init(){ fInputFile = ""; fInputRootFile = NULL; fOutputFile = ""; fOutputRootFile = NULL; fCovar = fCovarFree = NULL; fCorrel = fCorrelFree = NULL; fDecomp = fDecompFree = NULL; fStrategy = "ErrorBands"; fRoutines.clear(); fRoutines.push_back("ErrorBands"); fCardFile = ""; fFakeDataInput = ""; fSampleFCN = NULL; fAllowedRoutines = ("ErrorBands,PlotLimits"); }; SystematicRoutines::~SystematicRoutines(){ }; SystematicRoutines::SystematicRoutines(int argc, char* argv[]){ // Initialise Defaults Init(); nuisconfig configuration = Config::Get(); // Default containers std::string cardfile = ""; std::string maxevents = "-1"; int errorcount = 0; int verbocount = 0; std::vector xmlcmds; std::vector configargs; fNThrows = 250; fStartThrows = 0; fThrowString = ""; // Make easier to handle arguments. std::vector args = GeneralUtils::LoadCharToVectStr(argc, argv); ParserUtils::ParseArgument(args, "-c", fCardFile, true); ParserUtils::ParseArgument(args, "-o", fOutputFile, false, false); ParserUtils::ParseArgument(args, "-n", maxevents, false, false); ParserUtils::ParseArgument(args, "-f", fStrategy, false, false); ParserUtils::ParseArgument(args, "-d", fFakeDataInput, false, false); ParserUtils::ParseArgument(args, "-s", fStartThrows, false, false); ParserUtils::ParseArgument(args, "-t", fNThrows, false, false); ParserUtils::ParseArgument(args, "-p", fThrowString, false, false); ParserUtils::ParseArgument(args, "-i", xmlcmds); ParserUtils::ParseArgument(args, "-q", configargs); ParserUtils::ParseCounter(args, "e", errorcount); ParserUtils::ParseCounter(args, "v", verbocount); ParserUtils::CheckBadArguments(args); // Add extra defaults if none given if (fCardFile.empty() and xmlcmds.empty()) { ERR(FTL) << "No input supplied!" << std::endl; throw; } if (fOutputFile.empty() and !fCardFile.empty()) { fOutputFile = fCardFile + ".root"; ERR(WRN) << "No output supplied so saving it to: " << fOutputFile << std::endl; } else if (fOutputFile.empty()) { ERR(FTL) << "No output file or cardfile supplied!" << std::endl; throw; } // Configuration Setup ============================= // Check no comp key is available if (Config::Get().GetNodes("nuiscomp").empty()) { fCompKey = Config::Get().CreateNode("nuiscomp"); } else { fCompKey = Config::Get().GetNodes("nuiscomp")[0]; } if (!fCardFile.empty()) fCompKey.Set("cardfile", fCardFile); if (!fOutputFile.empty()) fCompKey.Set("outputfile", fOutputFile); if (!fStrategy.empty()) fCompKey.Set("strategy", fStrategy); // Load XML Cardfile configuration.LoadSettings( fCompKey.GetS("cardfile"), ""); // Add Config Args for (size_t i = 0; i < configargs.size(); i++) { configuration.OverrideConfig(configargs[i]); } if (maxevents.compare("-1")){ configuration.OverrideConfig("MAXEVENTS=" + maxevents); } // Finish configuration XML configuration.FinaliseSettings(fCompKey.GetS("outputfile") + ".xml"); // Add Error Verbo Lines verbocount += Config::GetParI("VERBOSITY"); errorcount += Config::GetParI("ERROR"); std::cout << "[ NUISANCE ]: Setting VERBOSITY=" << verbocount << std::endl; std::cout << "[ NUISANCE ]: Setting ERROR=" << errorcount << std::endl; SETVERBOSITY(verbocount); // Proper Setup if (fStrategy.find("ErrorBands") != std::string::npos || fStrategy.find("MergeErrors") != std::string::npos){ fOutputRootFile = new TFile(fCompKey.GetS("outputfile").c_str(), "RECREATE"); } // fOutputRootFile = new TFile(fCompKey.GetS("outputfile").c_str(), "RECREATE"); SetupSystematicsFromXML(); SetupCovariance(); SetupRWEngine(); SetupFCN(); GetCovarFromFCN(); // Run(); return; }; void SystematicRoutines::SetupSystematicsFromXML(){ LOG(FIT) << "Setting up nuismin" << std::endl; // Setup Parameters ------------------------------------------ std::vector parkeys = Config::QueryKeys("parameter"); if (!parkeys.empty()) { LOG(FIT) << "Number of parameters : " << parkeys.size() << std::endl; } for (size_t i = 0; i < parkeys.size(); i++) { nuiskey key = parkeys.at(i); // Check for type,name,nom if (!key.Has("type")) { ERR(FTL) << "No type given for parameter " << i << std::endl; throw; } else if (!key.Has("name")) { ERR(FTL) << "No name given for parameter " << i << std::endl; throw; } else if (!key.Has("nominal")) { ERR(FTL) << "No nominal given for parameter " << i << std::endl; throw; } // Get Inputs std::string partype = key.GetS("type"); std::string parname = key.GetS("name"); double parnom = key.GetD("nominal"); double parlow = parnom - 1; double parhigh = parnom + 1; double parstep = 1; // Override if state not given if (!key.Has("state")){ key.SetS("state","FIX"); } std::string parstate = key.GetS("state"); // Extra limits if (key.Has("low")) { parlow = key.GetD("low"); parhigh = key.GetD("high"); parstep = key.GetD("step"); LOG(FIT) << "Read " << partype << " : " << parname << " = " << parnom << " : " << parlow << " < p < " << parhigh << " : " << parstate << std::endl; } else { LOG(FIT) << "Read " << partype << " : " << parname << " = " << parnom << " : " << parstate << std::endl; } // Run Parameter Conversion if needed if (parstate.find("ABS") != std::string::npos) { parnom = FitBase::RWAbsToSigma( partype, parname, parnom ); parlow = FitBase::RWAbsToSigma( partype, parname, parlow ); parhigh = FitBase::RWAbsToSigma( partype, parname, parhigh ); parstep = FitBase::RWAbsToSigma( partype, parname, parstep ); } else if (parstate.find("FRAC") != std::string::npos) { parnom = FitBase::RWFracToSigma( partype, parname, parnom ); parlow = FitBase::RWFracToSigma( partype, parname, parlow ); parhigh = FitBase::RWFracToSigma( partype, parname, parhigh ); parstep = FitBase::RWFracToSigma( partype, parname, parstep ); } // Push into vectors fParams.push_back(parname); fTypeVals[parname] = FitBase::ConvDialType(partype);; fStartVals[parname] = parnom; fCurVals[parname] = parnom; fErrorVals[parname] = 0.0; fStateVals[parname] = parstate; bool fixstate = parstate.find("FIX") != std::string::npos; fFixVals[parname] = fixstate; fStartFixVals[parname] = fFixVals[parname]; fMinVals[parname] = parlow; fMaxVals[parname] = parhigh; fStepVals[parname] = parstep; - } // Setup Samples ---------------------------------------------- std::vector samplekeys = Config::QueryKeys("sample"); if (!samplekeys.empty()) { LOG(FIT) << "Number of samples : " << samplekeys.size() << std::endl; } for (size_t i = 0; i < samplekeys.size(); i++) { nuiskey key = samplekeys.at(i); // Get Sample Options std::string samplename = key.GetS("name"); std::string samplefile = key.GetS("input"); std::string sampletype = key.Has("type") ? key.GetS("type") : "DEFAULT"; double samplenorm = key.Has("norm") ? key.GetD("norm") : 1.0; // Print out LOG(FIT) << "Read sample info " << i << " : " << samplename << std::endl << "\t\t input -> " << samplefile << std::endl << "\t\t state -> " << sampletype << std::endl << "\t\t norm -> " << samplenorm << std::endl; // If FREE add to parameters otherwise continue if (sampletype.find("FREE") == std::string::npos) { continue; } // Form norm dial from samplename + sampletype + "_norm"; std::string normname = samplename + "_norm"; // Check normname not already present if (fTypeVals.find(normname) != fTypeVals.end()) { continue; } // Add new norm dial to list if its passed above checks fParams.push_back(normname); fTypeVals[normname] = kNORM; fStateVals[normname] = sampletype; fCurVals[normname] = samplenorm; fErrorVals[normname] = 0.0; fMinVals[normname] = 0.1; fMaxVals[normname] = 10.0; fStepVals[normname] = 0.5; bool state = sampletype.find("FREE") == std::string::npos; fFixVals[normname] = state; fStartFixVals[normname] = state; } // Setup Fake Parameters ----------------------------- std::vector fakekeys = Config::QueryKeys("fakeparameter"); if (!fakekeys.empty()) { LOG(FIT) << "Number of fake parameters : " << fakekeys.size() << std::endl; } for (size_t i = 0; i < fakekeys.size(); i++) { nuiskey key = fakekeys.at(i); // Check for type,name,nom if (!key.Has("name")) { ERR(FTL) << "No name given for fakeparameter " << i << std::endl; throw; } else if (!key.Has("nom")) { ERR(FTL) << "No nominal given for fakeparameter " << i << std::endl; throw; } // Get Inputs std::string parname = key.GetS("name"); double parnom = key.GetD("nom"); // Push into vectors fFakeVals[parname] = parnom; } - } /* Setup Functions */ //************************************* void SystematicRoutines::SetupRWEngine(){ //************************************* for (UInt_t i = 0; i < fParams.size(); i++){ std::string name = fParams[i]; FitBase::GetRW() -> IncludeDial(name, fTypeVals.at(name) ); } UpdateRWEngine(fStartVals); return; } //************************************* void SystematicRoutines::SetupFCN(){ //************************************* LOG(FIT)<<"Making the jointFCN"<Reconfigure(); fSampleFCN->ReconfigureAllEvents(); fSampleFCN->SetFakeData("MC"); UpdateRWEngine(fCurVals); LOG(FIT)<<"Set all data to fake MC predictions."<SetFakeData(fFakeDataInput); } return; } //***************************************** -void SystematicRoutines::GetCovarFromFCN(){ +// Setup the parameter covariances from the FCN +void SystematicRoutines::GetCovarFromFCN() { //***************************************** LOG(FIT) << "Loading ParamPull objects from FCN to build covariance..." << std::endl; // Make helperstring std::ostringstream helperstr; // Keep track of what is being thrown std::map dialthrowhandle; // Get Covariance Objects from FCN std::list inputpulls = fSampleFCN->GetPullList(); for (PullListConstIter iter = inputpulls.begin(); iter != inputpulls.end(); iter++){ ParamPull* pull = (*iter); if (pull->GetType().find("THROW") != std::string::npos){ fInputThrows.push_back(pull); fInputCovar.push_back(pull->GetFullCovarMatrix()); fInputDials.push_back(pull->GetDataHist()); LOG(FIT) << "Read ParamPull: " << pull->GetName() << " " << pull->GetType() << std::endl; } TH1D dialhist = pull->GetDataHist(); TH1D minhist = pull->GetMinHist(); TH1D maxhist = pull->GetMaxHist(); TH1I typehist = pull->GetDialTypes(); for (int i = 0; i < dialhist.GetNbinsX(); i++){ std::string name = std::string(dialhist.GetXaxis()->GetBinLabel(i+1)); dialthrowhandle[name] = pull->GetName(); + // Add to Containers + // Set the starting values etc to the postfit + fParams.push_back(name); + fCurVals[name] = dialhist.GetBinContent(i+1); + // Set the starting values to be nominal MC + fStartVals[name] = 0.0; + fMinVals[name] = minhist.GetBinContent(i+1); + fMaxVals[name] = maxhist.GetBinContent(i+1); + fStepVals[name] = 1.0; + fFixVals[name] = false; + fStartFixVals[name] = false; + fTypeVals[name] = typehist.GetBinContent(i+1); + fStateVals[name] = "FREE," + pull->GetType(); + + + // If we find the string if (fCurVals.find(name) == fCurVals.end()){ - - // Add to Containers - fParams.push_back(name); - fCurVals[name] = dialhist.GetBinContent(i+1); - fStartVals[name] = dialhist.GetBinContent(i+1); - fMinVals[name] = minhist.GetBinContent(i+1); - fMaxVals[name] = maxhist.GetBinContent(i+1); - fStepVals[name] = 1.0; - fFixVals[name] = false; - fStartFixVals[name] = false; - fTypeVals[name] = typehist.GetBinContent(i+1); - fStateVals[name] = "FREE" + pull->GetType(); - - // Maker Helper - helperstr << std::string(16, ' ' ) << FitBase::ConvDialType(fTypeVals[name]) << " " - << name << " " << fMinVals[name] << " " - << fMaxVals[name] << " " << fStepVals[name] << " " << fStateVals[name] - << std::endl; + // Maker Helper + helperstr << std::string(16, ' ' ) << FitBase::ConvDialType(fTypeVals[name]) << " " + << name << " " << fMinVals[name] << " " + << fMaxVals[name] << " " << fStepVals[name] << " " << fStateVals[name] + << std::endl; } } } // Check if no throws given if (fInputThrows.empty()){ ERR(WRN) << "No covariances given to nuissyst" << std::endl; ERR(WRN) << "Pushing back an uncorrelated gaussian throw error for each free parameter using step size" << std::endl; for (UInt_t i = 0; i < fParams.size(); i++){ std::string syst = fParams[i]; if (fFixVals[syst]) continue; // Make Terms std::string name = syst + "_pull"; std::ostringstream pullterm; pullterm << "DIAL:" << syst << ";" - << fStartVals[syst] << ";" - << fStepVals[syst]; + << fStartVals[syst] << ";" + << fStepVals[syst]; std::string type = "GAUSTHROW/NEUT"; // Push Back Pulls ParamPull* pull = new ParamPull( name, pullterm.str(), type ); fInputThrows.push_back(pull); fInputCovar.push_back(pull->GetFullCovarMatrix()); fInputDials.push_back(pull->GetDataHist()); // Print Whats added LOG(FIT) << "Added ParamPull : " << name << " " << pullterm.str() << " " << type << std::endl; // Add helper string for future fits helperstr << std::string(16, ' ' ) << "covar " << name << " " << pullterm.str() << " " << type << std::endl; // Keep Track of Throws dialthrowhandle[syst] = pull->GetName(); } } // Print Helper String if (!helperstr.str().empty()){ LOG(FIT) << "To remove these statements in future studies, add the lines below to your card:" << std::endl; // Can't use the logger properly because this can be multi-line. Use cout and added spaces to look better! std::cout << helperstr.str(); sleep(2); } - - // Print Throw State for (UInt_t i = 0; i < fParams.size(); i++){ std::string syst = fParams[i]; if (dialthrowhandle.find(syst) != dialthrowhandle.end()){ - LOG(FIT) << "Dial " << i << ". " << setw(40) << syst << " = THROWING with " << dialthrowhandle[syst] << std::endl; + LOG(FIT) << "Dial " << i << ". " << setw(20) << syst << " = THROWing with " << dialthrowhandle[syst] << std::endl; } else { - LOG(FIT) << "Dial " << i << ". " << setw(40) << syst << " = FIXED" << std::endl; + LOG(FIT) << "Dial " << i << ". " << setw(20) << syst << " = is FIXED" << std::endl; } } - - // Pause anyway - sleep(1); - return; } - - - /* - Fitting Functions -*/ + Fitting Functions + */ //************************************* void SystematicRoutines::UpdateRWEngine(std::map& updateVals){ -//************************************* + //************************************* for (UInt_t i = 0; i < fParams.size(); i++){ std::string name = fParams[i]; - if (updateVals.find(name) == updateVals.end()) continue; FitBase::GetRW()->SetDialValue(name,updateVals.at(name)); } FitBase::GetRW()->Reconfigure(); - return; } //************************************* void SystematicRoutines::PrintState(){ -//************************************* + //************************************* LOG(FIT)<<"------------"<GetLikelihood(); LOG(FIT) << std::left << std::setw(46) << "Likelihood for JointFCN: " << like << std::endl; LOG(FIT)<<"------------"<cd(); SaveCurrentState(); } //************************************* void SystematicRoutines::SaveCurrentState(std::string subdir){ -//************************************* + //************************************* LOG(FIT)<<"Saving current full FCN predictions" <mkdir(subdir.c_str()); newdir->cd(); } FitBase::GetRW()->Reconfigure(); fSampleFCN->ReconfigureAllEvents(); fSampleFCN->Write(); // Change back to current DIR curdir->cd(); return; } //************************************* void SystematicRoutines::SaveNominal(){ -//************************************* + //************************************* if (!fOutputRootFile) fOutputRootFile = new TFile(fCompKey.GetS("outputfile").c_str(), "RECREATE"); fOutputRootFile->cd(); LOG(FIT)<<"Saving Nominal Predictions (be cautious with this)" <Reconfigure(); SaveCurrentState("nominal"); - }; //************************************* void SystematicRoutines::SavePrefit(){ -//************************************* + //************************************* if (!fOutputRootFile) fOutputRootFile = new TFile(fCompKey.GetS("outputfile").c_str(), "RECREATE"); fOutputRootFile->cd(); LOG(FIT)<<"Saving Prefit Predictions"< 0){ fCovarFree = new TH2D("covariance_free", - "covariance_free", - NFREE,0,NFREE, - NFREE,0,NFREE); + "covariance_free", + NFREE,0,NFREE, + NFREE,0,NFREE); } // Set Bin Labels int countall = 0; int countfree = 0; for (UInt_t i = 0; i < fParams.size(); i++){ fCovar->GetXaxis()->SetBinLabel(countall+1,fParams[i].c_str()); fCovar->GetYaxis()->SetBinLabel(countall+1,fParams[i].c_str()); countall++; if (!fFixVals[fParams[i]] and NFREE > 0){ fCovarFree->GetXaxis()->SetBinLabel(countfree+1,fParams[i].c_str()); fCovarFree->GetYaxis()->SetBinLabel(countfree+1,fParams[i].c_str()); countfree++; } } fCorrel = PlotUtils::GetCorrelationPlot(fCovar,"correlation"); fDecomp = PlotUtils::GetDecompPlot(fCovar,"decomposition"); if (NFREE > 0)fCorrelFree = PlotUtils::GetCorrelationPlot(fCovarFree, "correlation_free"); if (NFREE > 0)fDecompFree = PlotUtils::GetDecompPlot(fCovarFree,"decomposition_free"); return; }; //************************************* void SystematicRoutines::ThrowCovariance(bool uniformly){ -//************************************* + //************************************* // Set fThrownVals to all values in currentVals for (UInt_t i = 0; i < fParams.size(); i++){ std::string name = fParams.at(i); fThrownVals[name] = fCurVals[name]; } for (PullListConstIter iter = fInputThrows.begin(); - iter != fInputThrows.end(); iter++){ + iter != fInputThrows.end(); iter++){ ParamPull* pull = *iter; pull->ThrowCovariance(); TH1D dialhist = pull->GetDataHist(); for (int i = 0; i < dialhist.GetNbinsX(); i++){ std::string name = std::string(dialhist.GetXaxis()->GetBinLabel(i+1)); if (fCurVals.find(name) != fCurVals.end()){ - fThrownVals[name] = dialhist.GetBinContent(i+1); + fThrownVals[name] = dialhist.GetBinContent(i+1); } } // Reset throw in case pulls are calculated. pull->ResetToy(); } }; //************************************* void SystematicRoutines::PlotLimits(){ -//************************************* + //************************************* std::cout << "Plotting Limits" << std::endl; if (!fOutputRootFile) fOutputRootFile = new TFile(fCompKey.GetS("outputfile").c_str(), "RECREATE"); TDirectory* limfolder = (TDirectory*) fOutputRootFile->mkdir("Limits"); limfolder->cd(); // Set all parameters at their starting values for (UInt_t i = 0; i < fParams.size(); i++){ fCurVals[fParams[i]] = fStartVals[fParams[i]]; } TDirectory* nomfolder = (TDirectory*) limfolder->mkdir("nominal"); nomfolder->cd(); UpdateRWEngine(fCurVals); fSampleFCN->ReconfigureAllEvents(); fSampleFCN->Write(); limfolder->cd(); std::vector allfolders; // Loop through each parameter for (UInt_t i = 0; i < fParams.size(); i++){ std::string syst = fParams[i]; std::cout << "Starting Param " << syst << std::endl; if (fFixVals[syst]) continue; // Loop Downwards while (fCurVals[syst] > fMinVals[syst]){ fCurVals[syst] = fCurVals[syst] - fStepVals[syst]; // Check Limit if (fCurVals[syst] < fMinVals[syst]) - fCurVals[syst] = fMinVals[syst]; + fCurVals[syst] = fMinVals[syst]; // Check folder exists std::string curvalstring = std::string( Form( (syst + "_%f").c_str(), fCurVals[syst] ) ); if (std::find(allfolders.begin(), allfolders.end(), curvalstring) != allfolders.end()) - break; + break; // Make new folder for variation TDirectory* minfolder = (TDirectory*) limfolder->mkdir(Form( (syst + "_%f").c_str(), fCurVals[syst] ) ); minfolder->cd(); allfolders.push_back(curvalstring); // Update Iterations double *vals = FitUtils::GetArrayFromMap( fParams, fCurVals ); fSampleFCN->DoEval( vals ); delete vals; // Save to folder fSampleFCN->Write(); } // Reset before next loop fCurVals[syst] = fStartVals[syst]; // Loop Upwards now while (fCurVals[syst] < fMaxVals[syst]){ fCurVals[syst] = fCurVals[syst] + fStepVals[syst]; // Check Limit if (fCurVals[syst] > fMaxVals[syst]) - fCurVals[syst] = fMaxVals[syst]; + fCurVals[syst] = fMaxVals[syst]; // Check folder exists std::string curvalstring = std::string( Form( (syst + "_%f").c_str(), fCurVals[syst] ) ); if (std::find(allfolders.begin(), allfolders.end(), curvalstring) != allfolders.end()) - break; + break; // Make new folder TDirectory* maxfolder = (TDirectory*) limfolder->mkdir(Form( (syst + "_%f").c_str(), fCurVals[syst] ) ); maxfolder->cd(); allfolders.push_back(curvalstring); // Update Iterations double *vals = FitUtils::GetArrayFromMap( fParams, fCurVals ); fSampleFCN->DoEval( vals ); delete vals; // Save to file fSampleFCN->Write(); } // Reset before leaving fCurVals[syst] = fStartVals[syst]; UpdateRWEngine(fCurVals); } return; } //************************************* void SystematicRoutines::Run(){ -//************************************* + //************************************* - std::cout << "Running routines "<< std::endl; fRoutines = GeneralUtils::ParseToStr(fStrategy,","); for (UInt_t i = 0; i < fRoutines.size(); i++){ std::string routine = fRoutines.at(i); int fitstate = kFitUnfinished; LOG(FIT)<<"Running Routine: "<cd(); // For generating throws we check with the config int nthrows = Config::GetParI("error_throws"); int startthrows = fStartThrows; int endthrows = startthrows + nthrows; if (nthrows < 0) nthrows = endthrows; if (startthrows < 0) startthrows = 0; if (endthrows < 0) endthrows = startthrows + nthrows; // Setting Seed // Matteo Mazzanti's Fix struct timeval mytime; gettimeofday(&mytime, NULL); Double_t seed = time(NULL) + int(getpid())+ (mytime.tv_sec * 1000.) + (mytime.tv_usec / 1000.); gRandom->SetSeed(seed); // int seed = (gRandom->Uniform(0.0,1.0)*100000 + 100000000*(startthrows + endthrows) + time(NULL) + int(getpid()) ); // gRandom->SetSeed(seed); LOG(FIT) << "Using Seed : " << seed << std::endl; LOG(FIT) << "nthrows = " << nthrows << std::endl; LOG(FIT) << "startthrows = " << startthrows << std::endl; LOG(FIT) << "endthrows = " << endthrows << std::endl; - UpdateRWEngine(fCurVals); + UpdateRWEngine(fStartVals); fSampleFCN->ReconfigureAllEvents(); - if (startthrows == 0){ + // Make the nominal + if (startthrows == 0) { LOG(FIT) << "Making nominal " << std::endl; TDirectory* nominal = (TDirectory*) tempfile->mkdir("nominal"); nominal->cd(); fSampleFCN->Write(); + + // Make the postfit reading from the pull + LOG(FIT) << "Making postfit " << std::endl; + TDirectory* postfit = (TDirectory*) tempfile->mkdir("postfit"); + postfit->cd(); + UpdateRWEngine(fCurVals); + fSampleFCN->ReconfigureSignal(); + fSampleFCN->Write(); } - TTree* parameterTree = new TTree("throws","throws"); - double chi2; - for (UInt_t i = 0; i < fParams.size(); i++) - parameterTree->Branch(fParams[i].c_str(), &fThrownVals[fParams[i]], (fParams[i] + "/D").c_str()); - parameterTree->Branch("chi2",&chi2,"chi2/D"); + fSampleFCN->CreateIterationTree("error_iterations", FitBase::GetRW()); // Would anybody actually want to do uniform throws of any parameter?? bool uniformly = FitPar::Config().GetParB("error_uniform"); // Run Throws and save for (Int_t i = 0; i < endthrows+1; i++){ + // Generate Random Parameter Throw ThrowCovariance(uniformly); if (i < startthrows) continue; if (i == 0) continue; LOG(FIT) << "Throw " << i << "/" << endthrows << " ================================" << std::endl; - // Generate Random Parameter Throw - // ThrowCovariance(uniformly); TDirectory* throwfolder = (TDirectory*)tempfile->mkdir(Form("throw_%i",i)); throwfolder->cd(); // Run Eval double *vals = FitUtils::GetArrayFromMap( fParams, fThrownVals ); - chi2 = fSampleFCN->DoEval( vals ); + fSampleFCN->DoEval( vals ); delete vals; // Save the FCN fSampleFCN->Write(); - - parameterTree->Fill(); } tempfile->cd(); fSampleFCN->WriteIterationTree(); tempfile->Close(); } +// Merge throws together into one summary void SystematicRoutines::MergeThrows() { fOutputRootFile = new TFile(fCompKey.GetS("outputfile").c_str(), "RECREATE"); fOutputRootFile->cd(); // Make a container folder TDirectory* errorDIR = (TDirectory*) fOutputRootFile->mkdir("error_bands"); errorDIR->cd(); TDirectory* outnominal = (TDirectory*) fOutputRootFile->mkdir("nominal_throw"); outnominal->cd(); // Split Input Files if (!fThrowString.empty()) fThrowList = GeneralUtils::ParseToStr(fThrowString,","); // Add default if no throwlist given if (fThrowList.size() < 1) fThrowList.push_back( fOutputFile + ".throws.root" ); /// Save location of file containing nominal std::string nominalfile; bool nominalfound; // Loop over files and check they exist. for (uint i = 0; i < fThrowList.size(); i++){ std::string file = fThrowList[i]; bool found = false; // normal std::string newfile = file; TFile* throwfile = new TFile(file.c_str(),"READ"); if (throwfile and !throwfile->IsZombie()){ found = true; } // normal.throws.root if (!found){ newfile = file + ".throws.root"; throwfile = new TFile((file + ".throws.root").c_str(),"READ"); if (throwfile and !throwfile->IsZombie()) { found = true; } } // If its found save to throwlist, else save empty. // Also search for nominal if (found){ fThrowList[i] = newfile; LOG(FIT) << "Throws File :" << newfile << std::endl; // Find input which contains nominal if (throwfile->Get("nominal")){ nominalfound = true; nominalfile = newfile; } throwfile->Close(); } else { fThrowList[i] = ""; } delete throwfile; } // Make sure we have a nominal file if (!nominalfound or nominalfile.empty()){ ERR(FTL) << "No nominal found when mergining! Exiting!" << std::endl; throw; } // Get the nominal throws file TFile* tempfile = new TFile((nominalfile).c_str(),"READ"); tempfile->cd(); TDirectory* nominal = (TDirectory*)tempfile->Get("nominal"); bool uniformly = FitPar::Config().GetParB("error_uniform"); // Check percentage of bad files is okay. int badfilecount = 0; for (uint i = 0; i < fThrowList.size(); i++){ if (!fThrowList[i].empty()){ LOG(FIT) << "Loading Throws From File " << i << " : " << fThrowList[i] << std::endl; } else { badfilecount++; } } // Check we have at least one good file if ((uint)badfilecount == fThrowList.size()){ ERR(FTL) << "Found no good throw files for MergeThrows" << std::endl; throw; } else if (badfilecount > fThrowList.size()*0.25){ ERR(WRN) << "Over 25% of your throw files are dodgy. Please check this is okay!" << std::endl; ERR(WRN) << "Will continue for the time being..." << std::endl; sleep(5); } // Now go through the keys in the temporary file and look for TH1D, and TH2D plots TIter next(nominal->GetListOfKeys()); TKey *key; while ((key = (TKey*)next())) { TClass *cl = gROOT->GetClass(key->GetClassName()); if (!cl->InheritsFrom("TH1D") and !cl->InheritsFrom("TH2D")) continue; TH1* baseplot = (TH1D*)key->ReadObj(); std::string plotname = std::string(baseplot->GetName()); LOG(FIT) << "Creating error bands for " << plotname; if (LOG_LEVEL(FIT)){ if (!uniformly) std::cout << " : Using COVARIANCE Throws! " << std::endl; else std::cout << " : Using UNIFORM THROWS!!! " << std::endl; } int nbins = 0; if (cl->InheritsFrom("TH1D")) nbins = ((TH1D*)baseplot)->GetNbinsX(); else nbins = ((TH1D*)baseplot)->GetNbinsX()* ((TH1D*)baseplot)->GetNbinsY(); // Setup TProfile with RMS option TProfile* tprof = new TProfile((plotname + "_prof").c_str(),(plotname + "_prof").c_str(),nbins, 0, nbins, "S"); // Setup The TTREE double* bincontents; bincontents = new double[nbins]; double* binlowest; binlowest = new double[nbins]; double* binhighest; binhighest = new double[nbins]; errorDIR->cd(); TTree* bintree = new TTree((plotname + "_tree").c_str(), (plotname + "_tree").c_str()); for (Int_t i = 0; i < nbins; i++){ bincontents[i] = 0.0; binhighest[i] = 0.0; binlowest[i] = 0.0; bintree->Branch(Form("content_%i",i),&bincontents[i],Form("content_%i/D",i)); } // Make new throw plot TH1* newplot; // Run Throw Merging. for (UInt_t i = 0; i < fThrowList.size(); i++){ TFile* throwfile = new TFile(fThrowList[i].c_str(), "READ"); // Loop over all throws in a folder TIter nextthrow(throwfile->GetListOfKeys()); TKey *throwkey; while ((throwkey = (TKey*)nextthrow())) { // Skip non throw folders if (std::string(throwkey->GetName()).find("throw_") == std::string::npos) continue; // Get Throw DIR TDirectory* throwdir = (TDirectory*)throwkey->ReadObj(); // Get Plot From Throw newplot = (TH1*)throwdir->Get(plotname.c_str()); if (!newplot) continue; // Loop Over Plot for (Int_t j = 0; j < nbins; j++){ tprof->Fill(j+0.5, newplot->GetBinContent(j+1)); bincontents[j] = newplot->GetBinContent(j+1); if (bincontents[j] < binlowest[j] or i == 0) binlowest[j] = bincontents[j]; if (bincontents[j] > binhighest[j] or i == 0) binhighest[j] = bincontents[j]; } errorDIR->cd(); bintree->Fill(); } throwfile->Close(); delete throwfile; } errorDIR->cd(); if (uniformly){ LOG(FIT) << "Uniformly Calculating Plot Errors!" << std::endl; } TH1* statplot = (TH1*) baseplot->Clone(); for (Int_t j = 0; j < nbins; j++){ if (!uniformly){ // if ((baseplot->GetBinError(j+1)/baseplot->GetBinContent(j+1)) < 1.0) { // baseplot->SetBinError(j+1,sqrt(pow(tprof->GetBinError(j+1),2) + pow(baseplot->GetBinError(j+1),2))); // } else { - //baseplot->SetBinContent(j+1,tprof->GetBinContent(j+1)); - baseplot->SetBinError(j+1,tprof->GetBinError(j+1)); + baseplot->SetBinContent(j+1, tprof->GetBinContent(j+1)); + baseplot->SetBinError(j+1, tprof->GetBinError(j+1)); // } } else { baseplot->SetBinContent(j+1, 0.0);//(binlowest[j] + binhighest[j]) / 2.0); baseplot->SetBinError(j+1, 0.0); //(binhighest[j] - binlowest[j])/2.0); } } errorDIR->cd(); baseplot->Write(); tprof->Write(); bintree->Write(); outnominal->cd(); for (int i = 0; i < nbins; i++){ baseplot->SetBinError(i+1, sqrt(pow(statplot->GetBinError(i+1),2) + pow(baseplot->GetBinError(i+1),2))); } baseplot->Write(); delete statplot; delete baseplot; delete tprof; delete bintree; delete [] bincontents; } - - return; }; void SystematicRoutines::EigenErrors() { - fOutputRootFile = new TFile(fCompKey.GetS("outputfile").c_str(), "RECREATE"); fOutputRootFile->cd(); // Make Covariance TMatrixDSym* fullcovar = new TMatrixDSym( fParams.size() ); // Extract covariance from all loaded ParamPulls for (PullListConstIter iter = fInputThrows.begin(); iter != fInputThrows.end(); iter++){ ParamPull* pull = *iter; // Check pull is actualyl Gaussian std::string pulltype = pull->GetType(); if (pulltype.find("GAUSTHROW") == std::string::npos){ THROW("Can only calculate EigenErrors for Gaussian pulls!"); } // Get data and covariances TH1D dialhist = pull->GetDataHist(); TH2D covhist = pull->GetFullCovar(); // Loop over all dials and compare names for (size_t pari = 0; pari < fParams.size(); pari++){ for (size_t parj = 0; parj < fParams.size(); parj++){ std::string name_pari = fParams[pari]; std::string name_parj = fParams[parj]; // Compare names to those in the pull for (int pulli = 0; pulli < dialhist.GetNbinsX(); pulli++){ for (int pullj = 0; pullj < dialhist.GetNbinsX(); pullj++){ std::string name_pulli = dialhist.GetXaxis()->GetBinLabel(pulli+1); std::string name_pullj = dialhist.GetXaxis()->GetBinLabel(pullj+1); if (name_pulli == name_pari && name_pullj == name_parj){ (*fullcovar)[pari][parj] = covhist.GetBinContent(pulli+1, pullj+1); fCurVals[name_pari] = dialhist.GetBinContent(pulli+1); fCurVals[name_parj] = dialhist.GetBinContent(pullj+1); } } } } } } /* TFile* test = new TFile("testingcovar.root","RECREATE"); test->cd(); TH2D* joinedcov = new TH2D("COVAR","COVAR", fullcovar->GetNrows(), 0.0, float(fullcovar->GetNrows()), fullcovar->GetNrows(), 0.0, float(fullcovar->GetNrows())); for (int i = 0; i < fullcovar->GetNrows(); i++){ for (int j = 0; j < fullcovar->GetNcols(); j++){ joinedcov->SetBinContent(i+1, j+1, (*fullcovar)[i][j]); } } joinedcov->Write("COVAR"); test->Close(); */ // Calculator all EigenVectors and EigenValues TMatrixDSymEigen* eigen = new TMatrixDSymEigen(*fullcovar); const TVectorD eigenVals = eigen->GetEigenValues(); const TMatrixD eigenVect = eigen->GetEigenVectors(); eigenVals.Print(); eigenVect.Print(); TDirectory* outnominal = (TDirectory*) fOutputRootFile->mkdir("nominal"); outnominal->cd(); double *valst = FitUtils::GetArrayFromMap( fParams, fCurVals ); //double chi2 = fSampleFCN->DoEval( valst ); delete valst; fSampleFCN->Write(); // Loop over all throws TDirectory* throwsdir = (TDirectory*) fOutputRootFile->mkdir("throws"); throwsdir->cd(); int count = 0; // Produce all error throws. for (int i = 0; i < eigenVect.GetNrows(); i++){ TDirectory* throwfolder = (TDirectory*)throwsdir->mkdir(Form("throw_%i",count)); throwfolder->cd(); // Get New Parameter Vector LOG(FIT) << "Parameter Set " << count << std::endl; for (int j = 0; j < eigenVect.GetNrows(); j++){ std::string param = fParams[j]; LOG(FIT) << " " << j << ". " << param << " : " << fCurVals[param] + sqrt(eigenVals[i]) * eigenVect[j][i] << std::endl; fThrownVals[param] = fCurVals[param] + sqrt(eigenVals[i]) * eigenVect[j][i]; } // Run Eval double *vals = FitUtils::GetArrayFromMap( fParams, fThrownVals ); double chi2 = fSampleFCN->DoEval( vals ); delete vals; count++; fSampleFCN->Write(); throwfolder = (TDirectory*)throwsdir->mkdir(Form("throw_%i",count)); throwfolder->cd(); // Get New Parameter Vector LOG(FIT) << "Parameter Set " << count << std::endl; for (int j = 0; j < eigenVect.GetNrows(); j++){ std::string param = fParams[j]; LOG(FIT) << " " << j << ". " << param << " : " <DoEval( vals2 ); delete vals2; count++; // Save the FCN fSampleFCN->Write(); } fOutputRootFile->Close(); fOutputRootFile = new TFile(fCompKey.GetS("outputfile").c_str(), "UPDATE"); fOutputRootFile->cd(); throwsdir = (TDirectory*) fOutputRootFile->Get("throws"); outnominal = (TDirectory*) fOutputRootFile->Get("nominal"); // Loop through Error DIR TDirectory* outerr = (TDirectory*) fOutputRootFile->mkdir("errors"); outerr->cd(); TIter next(outnominal->GetListOfKeys()); TKey *key; while ((key = (TKey*)next())) { TClass *cl = gROOT->GetClass(key->GetClassName()); if (!cl->InheritsFrom("TH1D") and !cl->InheritsFrom("TH2D")) continue; LOG(FIT) << "Creating error bands for " << key->GetName() << std::endl; std::string plotname = std::string(key->GetName()); if (plotname.find("_EVT") != std::string::npos) continue; if (plotname.find("_FLUX") != std::string::npos) continue; if (plotname.find("_FLX") != std::string::npos) continue; TH1* baseplot = (TH1D*)key->ReadObj()->Clone(Form("%s_ORIGINAL",key->GetName())); TH1* errorplot_upper = (TH1D*)baseplot->Clone(Form("%s_ERROR_UPPER",key->GetName())); TH1* errorplot_lower = (TH1D*)baseplot->Clone(Form("%s_ERROR_LOWER", key->GetName())); TH1* meanplot = (TH1D*)baseplot->Clone(Form("%s_SET_MEAN", key->GetName())); TH1* systplot = (TH1D*)baseplot->Clone(Form("%s_SYST", key->GetName())); TH1* statplot = (TH1D*)baseplot->Clone(Form("%s_STAT", key->GetName())); TH1* totlplot = (TH1D*)baseplot->Clone(Form("%s_TOTAL", key->GetName())); int nbins = 0; if (cl->InheritsFrom("TH1D")) nbins = ((TH1D*)baseplot)->GetNbinsX(); else nbins = ((TH1D*)baseplot)->GetNbinsX()* ((TH1D*)baseplot)->GetNbinsY(); meanplot->Reset(); errorplot_upper->Reset(); errorplot_lower->Reset(); for (int j = 0; j < nbins; j++){ errorplot_upper->SetBinError(j+1, 0.0); errorplot_lower->SetBinError(j+1, 0.0); } // Loop over throws and calculate mean and error for +- throws int addcount = 0; // Add baseplot first to slightly bias to central value meanplot->Add(baseplot); addcount++; for (int i = 0; i < count; i++){ TH1* newplot = (TH1D*) throwsdir->Get(Form("throw_%i/%s",i,plotname.c_str())); if (!newplot){ ERR(WRN) << "Cannot find new plot : " << Form("throw_%i/%s",i,plotname.c_str()) << std::endl; ERR(WRN) << "This plot will not have the correct errors!" << std::endl; continue; } newplot->SetDirectory(0); nbins = newplot->GetNbinsX(); for (int j = 0; j < nbins; j++){ if (i % 2 == 0){ - // std::cout << plotname<< " : upper " << errorplot_upper->GetBinContent(j+1) << " adding " << pow(baseplot->GetBinContent(j+1) - newplot->GetBinContent(j+1),2) << std::endl; - // std::cout << " -> " << baseplot->GetBinContent(j+1) << " " <GetBinContent(j+1) << std::endl; errorplot_upper->SetBinContent(j+1, errorplot_upper->GetBinContent(j+1) + pow(baseplot->GetBinContent(j+1) - newplot->GetBinContent(j+1),2)); - // newplot->Print(); } else { - // std::cout << plotname << " : lower " << errorplot_lower->GetBinContent(j+1) << " adding " << pow(baseplot->GetBinContent(j+1) - newplot->GetBinContent(j+1),2) << std::endl; - // std::cout << " -> " << baseplot->GetBinContent(j+1) << " " << newplot->GetBinContent(j+1) << std::endl; errorplot_lower->SetBinContent(j+1, errorplot_lower->GetBinContent(j+1) + pow(baseplot->GetBinContent(j+1) - newplot->GetBinContent(j+1),2)); - // newplot->Print(); } meanplot->SetBinContent(j+1, meanplot->GetBinContent(j+1) + baseplot->GetBinContent(j+1)); } delete newplot; addcount++; } // Get mean Average for (int j = 0; j < nbins; j++){ meanplot->SetBinContent(j+1, meanplot->GetBinContent(j+1)/double(addcount)); } for (int j = 0; j < nbins; j++){ errorplot_upper->SetBinContent(j+1, sqrt(errorplot_upper->GetBinContent(j+1))); errorplot_lower->SetBinContent(j+1, sqrt(errorplot_lower->GetBinContent(j+1))); statplot->SetBinError(j+1, baseplot->GetBinError(j+1) ); systplot->SetBinError(j+1, (errorplot_upper->GetBinContent(j+1) + errorplot_lower->GetBinContent(j+1))/2.0); totlplot->SetBinError(j+1, sqrt( pow(statplot->GetBinError(j+1),2) + pow(systplot->GetBinError(j+1),2) ) ); meanplot->SetBinError(j+1, sqrt( pow(statplot->GetBinError(j+1),2) + pow(systplot->GetBinError(j+1),2) ) ); } outerr->cd(); errorplot_upper->Write(); errorplot_lower->Write(); baseplot->Write(); meanplot->Write(); statplot->Write(); systplot->Write(); totlplot->Write(); delete errorplot_upper; delete errorplot_lower; delete baseplot; delete meanplot; delete statplot; delete systplot; delete totlplot; } }