diff --git a/bin/rivet-diffhepdata-all b/bin/rivet-diffhepdata-all --- a/bin/rivet-diffhepdata-all +++ b/bin/rivet-diffhepdata-all @@ -1,144 +1,142 @@ #! /usr/bin/env python """\ %(prog)s [-h|--help] [-r|--rivet_directory RIVET_DIRECTORY] [-d|--hepdata_directory HEPDATA_DIRECTORY] [-o|--output_directory %OUTPUT_DIRECTORY] [-l|--lower_count LOWER_COUNT] [-u|--upper_count UPPER_COUNT] [-f|--force_download] [--sync] Check compatibility of YODA reference data files, distributed with Rivet, with the YODA file downloaded from HEPData. Optional arguments to specify Rivet analyses directory and directories to store HEPData .yoda files and yodadiff output. Optional arguments to loop over a range of INSPIRE IDs (for testing) and to force re-download of HEPData .yoda files. Optional argument to perform sync operation. Examples: # Specify Rivet directory, HEPData and yodadiff output directories, and redirect output to a text file. rivet-diffhepdata-all -r ../Rivet/analyses -d HEPDataYoda -o YodaDiffOutput > rivet-diffhepdata-all.txt # Loop over only the first 10 INSPIRE IDs (sorted by Rivet analysis name) and force HEPData re-download. rivet-diffhepdata-all -l 1 -u 10 -f # for Rivet .yoda files located in a subdirectory of current path """ from __future__ import print_function -import os, shutil, imp, glob, requests, argparse, importlib +import os, shutil, imp, glob, requests, argparse from rivet import hepdatautils +import yoda parser = argparse.ArgumentParser(usage='Check compatibility of YODA reference data files with HEPData for all Rivet analyses') parser.add_argument('-r', '--rivet_directory', nargs=1, default=['.'], help='directory to search for Rivet .yoda files') parser.add_argument('-d', '--hepdata_directory', nargs=1, default=['.'], help='directory to store downloaded HEPData .yoda files') parser.add_argument('-o', '--output_directory', nargs=1, default=['.'], help='directory to write yodadiff output') parser.add_argument('-l', '--lower_count', nargs=1, type=int, default=[0], help='minimum count for loop over INSPIRE IDs') parser.add_argument('-u', '--upper_count', nargs=1, type=int, default=[0], help='maximum count for loop over INSPIRE IDs') parser.add_argument('-f', '--force_download', action='store_true', help='force re-download of HEPData .yoda files') parser.add_argument('--sync', action='store_true', help='perform sync operation') args = parser.parse_args() rivet_directory = args.rivet_directory[0] hepdata_directory = args.hepdata_directory[0] output_name = args.output_directory[0] lower_count = args.lower_count[0] upper_count = args.upper_count[0] force_download = args.force_download perform_sync = args.sync print('Arguments specified:') print(' rivet_directory={}'.format(rivet_directory)) print(' hepdata_directory={}'.format(hepdata_directory)) print(' output={}'.format(output_name)) print(' lower_count={}'.format(lower_count)) print(' upper_count={}'.format(upper_count)) print(' force_download={}'.format(force_download)) print(' sync={}'.format(perform_sync)) def find(filename, path): """ Function to return first matching 'filename' by walking the directory tree top-down from 'path'. """ for root, dirs, files in os.walk(path): if filename in files: return os.path.join(root, filename) # Create output directories if they don't already exist. if not os.path.exists(hepdata_directory): os.makedirs(hepdata_directory) if not os.path.exists(output_name): os.makedirs(output_name) # Get mapping between INSPIRE IDs and Rivet analysis names. response = requests.get('http://rivet.hepforge.org/analyses.json') analyses = response.json() # Loop over INSPIRE IDs and collect compatible and incompatible analyses. # Sort analyses dict by the Rivet analysis name. compatible_analyses = [] incompatible_analyses = [] for count, inspire_id in enumerate(sorted(analyses, key=analyses.get)): # Loop over a restricted range of INSPIRE IDs (useful for testing). if count + 1 < lower_count or (upper_count and count + 1 > upper_count): continue print() num_analyses = len(analyses[inspire_id]) if num_analyses != 1: compatible = False print('Multiple (or zero) Rivet analyses {} for INSPIRE ID {}.'.format(analyses[inspire_id], inspire_id)) elif 'CONF' in analyses[inspire_id][0]: compatible = True print('Rivet analysis {} is a CONF note result. Skipping.'.format(analyses[inspire_id][0])) else: analysis = analyses[inspire_id][0] yodafile = find(analysis + '.yoda', rivet_directory) outfile = os.path.join(output_name, analysis + '.txt') # Check if .yoda file has already been downloaded from HEPData, otherwise download. Do anyway if force_download = True. matched_files = glob.glob(os.path.join(hepdata_directory, 'HEPData-ins' + inspire_id + '-v*-yoda.yoda')) if not matched_files or force_download: try: yodafile_from_hepdata = hepdatautils.download_from_hepdata(inspire_id, analysis) except: print('Download from HEPData failed for Rivet analysis {}.'.format(analysis)) if yodafile_from_hepdata: os.rename(yodafile_from_hepdata, os.path.join(hepdata_directory, yodafile_from_hepdata)) yodafile_from_hepdata = os.path.join(hepdata_directory, yodafile_from_hepdata) else: print('Missing YODA reference data file from HEPData for Rivet analysis {}.'.format(analysis)) else: yodafile_from_hepdata = sorted(matched_files)[-1] # sort in case of multiple versions (works for v1 up to v9) # Check if ref data needs post-processing, e.g. to patch zero bin widths # First check if a post-processed version already exists in HEPData directory: - hdPatchedVersions = glob.glob( os.path.join(hepdata_directory, '{}-post.yoda'.format(yodafile_from_hepdata[:-5])) ) + hdPatchedVersions = glob.glob( '{}-post.yoda'.format(yodafile_from_hepdata[:-5]) ) if hdPatchedVersions and not force_download: # use the latest one yodafile_from_hepdata = sorted(hdPatchedVersions)[-1] - else: # check if a post-processing script has been provided at all - hd_sync_dir = '{}/HDsync/'.format(rivet_dir) - hdPatchScript = find('HEPDATA_{}.yoda'.format(analysis}, hd_sync_dir) - if hdPatchScript: - hd_patcher = importlib.import_module(hdPatchScript) - patchedContent = hd_patcher.patch(yodafile_from_hepdata) # apply post-processing - # save the post-processed content in the HEPData directory and use it instead - yodafile_from_hepdata = '%s-post.yoda' % yodafile_from_hepdata[-5] - with open(yodafile_from_hepdata, 'w') as patchedFile: - yoda.writeYODA(patchedContent, patchedFile) + else: + patchedContent = hepdatautils.patch_yodaref(yodafile_from_hepdata) # apply post-processing + # save the post-processed content in the HEPData directory and use it instead + yodafile_from_hepdata = '{}-post.yoda'.format(yodafile_from_hepdata[:-5]) + with open(yodafile_from_hepdata, 'w') as patchedFile: + for tag in sorted(patchedContent.keys()): # could pass unsorted, but treat yo'self + yoda.writeYODA(patchedContent[tag], patchedFile) # Run yodadiff between the .yoda files from Rivet and HEPData. if yodafile: compatible = hepdatautils.compare_with_hepdata(yodafile, yodafile_from_hepdata=yodafile_from_hepdata, output=outfile) if perform_sync and compatible: shutil.copy(yodafile_from_hepdata, yodafile) else: print('Missing YODA reference data file from Rivet for analysis {}.'.format(analysis)) compatible = True if compatible: print('YODA reference data files from Rivet and HEPData are compatible!') compatible_analyses.append(inspire_id) else: print('YODA reference data files from Rivet and HEPData are NOT compatible!') incompatible_analyses.append(inspire_id) # Print out some summary information. print() print('Compatible Rivet analyses: {}'.format([analyses[inspire_id] for inspire_id in compatible_analyses])) print('Incompatible Rivet analyses: {}'.format([analyses[inspire_id] for inspire_id in incompatible_analyses])) print() print('Of {:d} Rivet analyses in {}, {:d} ({:.1f}%) were compatible and {:d} ({:.1f}%) were incompatible.'.format( len(analyses), rivet_directory, len(compatible_analyses), 100.*len(compatible_analyses)/len(analyses), len(incompatible_analyses), 100.*len(incompatible_analyses)/len(analyses))) diff --git a/pyext/rivet/hepdatapatches/ATLAS_2016_I1468168.py b/pyext/rivet/hepdatapatches/ATLAS_2016_I1468168.py new file mode 100644 --- /dev/null +++ b/pyext/rivet/hepdatapatches/ATLAS_2016_I1468168.py @@ -0,0 +1,7 @@ + +def patch(path, yodaobject): + if path == '/REF/ATLAS_2016_I1468168/d02-x01-y01': + for p in yodaobject.points: + p.xErrs = (0.5, 0.5) + return yodaobject + diff --git a/pyext/rivet/hepdatapatches/__init__.py b/pyext/rivet/hepdatapatches/__init__.py new file mode 100644 --- /dev/null +++ b/pyext/rivet/hepdatapatches/__init__.py @@ -0,0 +1,1 @@ +from . import ATLAS_2016_I1468168 diff --git a/pyext/rivet/hepdatautils.py b/pyext/rivet/hepdatautils.py old mode 100755 new mode 100644 --- a/pyext/rivet/hepdatautils.py +++ b/pyext/rivet/hepdatautils.py @@ -1,76 +1,100 @@ from __future__ import print_function def compare_with_hepdata(yodafile, inspire_id=0, yodafile_from_hepdata=None, output=None): """\ Compare a YODA reference data file, intended for inclusion in Rivet, with the YODA file downloaded from HEPData. Make the comparison using the yodadiff script distributed with YODA (https://yoda.hepforge.org/trac/browser/bin/yodadiff). :param yodafile: name of YODA reference data file (intended for inclusion in Rivet) :param inspire_id: INSPIRE ID (to download the YODA file from HEPData) :param yodafile_from_hepdata: name of YODA file already downloaded from HEPData :param output: name of output file for yodadiff instead of stdout (note: -o option of yodadiff not implemented) :return: True or False depending on whether YODA files are compatible """ if inspire_id: # Extract Rivet analysis name from last pathname component of yodafile (and discard extension). import os tail = os.path.split(yodafile)[1] rivet_analysis_name = os.path.splitext(tail)[0] yodafile_from_hepdata = download_from_hepdata(inspire_id, rivet_analysis_name) if yodafile_from_hepdata: print('Comparing {} with {}'.format(yodafile, yodafile_from_hepdata)) import subprocess args = ['yodadiff', yodafile, yodafile_from_hepdata] if output: print('Writing output of "{}" to {}'.format(' '.join(args), output)) with open(output, 'w') as out: # Note: -o|--output option of yodadiff is not implemented. exit_status = subprocess.call(args, stdout=out, stderr=out) else: exit_status = subprocess.call(args) if exit_status: return False else: print('No YODA file from HEPData specified!') return False return True def download_from_hepdata(inspire_id, rivet_analysis_name=None): """\ Download the latest YODA reference data file from HEPData identified by the INSPIRE ID. Optionally pass the Rivet analysis name to write in the YODA file exported from HEPData. :param inspire_id: INSPIRE ID :param rivet_analysis_name: Rivet analysis name to override default :return: name of YODA file downloaded from HEPData """ import tarfile, io, os, requests hdurl = 'https://hepdata.net/record/ins{}'.format(inspire_id) payload = {'format': 'yoda', 'rivet': rivet_analysis_name} response = requests.get(hdurl, params=payload) if response.history: print('Downloading from {}'.format(response.history[0].url)) else: print('Downloading from {}'.format(response.url)) if response.status_code != 200: print('Download failed ({} {}), does {} exist?'.format(response.status_code, response.reason, hdurl)) return None try: tar = tarfile.open(mode='r:gz', fileobj=io.BytesIO(response.content)) tar.extractall() yodafile_from_hepdata = tar.getnames()[0] os.chmod(yodafile_from_hepdata, 0o644) except tarfile.TarError as e: print('Error reading tarfile ({})'.format(str(e))) return None print('Downloaded {}'.format(yodafile_from_hepdata)) return yodafile_from_hepdata + + + +def patch_yodaref(yoda_from_hepdata): + """\ + Take a YODA file and check if the reference data contained in the file is in need of post-processing. + If so, apply relevant post-processing steps and return. + + :param yoda_from_hepdata: YODA file containing reference data from HEPData for post-processing + :return: YODA file with post-processed data from HEPData + """ + + + import importlib, yoda, hepdatapatches + hepdata_content = yoda.read( yoda_from_hepdata ) + for tag in hepdata_content: + if tag.startswith("/REF"): + routine, tableid = tag.rstrip("/")[5:].split('/') + if hasattr(hepdatapatches, routine): + # get relevant patch function for this routine and apply patch + routine_patcher = importlib.import_module("rivet.hepdatapatches." + routine) + hepdata_content[tag] = routine_patcher.patch(tag, hepdata_content[tag]) + return hepdata_content + diff --git a/pyext/setup.py.in b/pyext/setup.py.in --- a/pyext/setup.py.in +++ b/pyext/setup.py.in @@ -1,63 +1,63 @@ #! /usr/bin/env python from distutils.core import setup from distutils.extension import Extension from glob import glob ## Extension definition import os.path incdir1 = os.path.abspath("@abs_top_srcdir@/include") incdir2 = os.path.abspath("@abs_top_builddir@/include") incdir3 = os.path.abspath("@abs_srcdir@/rivet") incdir4 = os.path.abspath("@abs_builddir@/rivet") srcdir = os.path.abspath("@abs_top_srcdir@/src") libdir = os.path.abspath("@abs_top_builddir@/src/.libs") ## Assemble the library search dirs @ENABLE_HEPMC_3_TRUE@lookupdirs = [libdir, "@HEPMC3LIBPATH@", "@FASTJETLIBPATH@", "@YODALIBPATH@" ] @ENABLE_HEPMC_3_FALSE@lookupdirs = [libdir, "@HEPMCLIBPATH@", "@FASTJETLIBPATH@", "@YODALIBPATH@" ] if "RIVET_LOCAL" in os.environ: BASE_LINK_ARGS = ["-L@abs_top_builddir@/src/.libs"] else: BASE_LINK_ARGS = ["-L@prefix@/lib"] ## Be careful with extracting the GSL path from the flags string # import re # re_libdirflag = re.compile(r".*-L\s*(\S+).*") # re_match = re_libdirflag.search("@GSL_LDFLAGS@") # if re_match: # lookupdirs.append( re_match.group(1) ) ## A helper function def ext(name, depends=[], statics=[]): fullname = '@abs_builddir@/rivet/%s.cpp' % name if not os.path.isfile(fullname): # distcheck has it in srcdir fullname = os.path.relpath("@abs_srcdir@/rivet/%s.cpp" % name) return Extension( "rivet.%s" % name, [fullname] + statics, language="c++", # depends=depends, include_dirs=[incdir1, incdir2, incdir3, incdir4], # extra_compile_args="-I@prefix@/include @PYEXT_CXXFLAGS@ @HEPMCCPPFLAGS@ @FASTJETCPPFLAGS@ @YODACPPFLAGS@ @GSLCPPFLAGS@".split(), extra_compile_args="-I@prefix@/include @PYEXT_CXXFLAGS@ @HEPMCCPPFLAGS@ @HEPMC3CPPFLAGS@ @CPPFLAGS@ @FASTJETCPPFLAGS@ @YODACPPFLAGS@".split(), extra_link_args=BASE_LINK_ARGS, library_dirs=lookupdirs, runtime_library_dirs=lookupdirs[1:], @ENABLE_HEPMC_3_TRUE@libraries=["HepMC3", "fastjet", "YODA", "Rivet"]) @ENABLE_HEPMC_3_FALSE@libraries=["HepMC", "fastjet", "YODA", "Rivet"]) # libraries=["gsl", "HepMC", "fastjet", "YODA", "Rivet"]) #header_files = glob("../include/Rivet/*.h") + glob("../include/Rivet/Utils/*.h") extns = [ext("core")]#, header_files)] setup(name = "rivet", version="@PACKAGE_VERSION@", ext_modules = extns, - packages = ["rivet"]) + packages = ["rivet", "rivet.hepdatapatches"])