diff --git a/m4/ax_python_devel.m4 b/m4/ax_python_devel.m4 --- a/m4/ax_python_devel.m4 +++ b/m4/ax_python_devel.m4 @@ -1,351 +1,351 @@ # =========================================================================== # http://www.gnu.org/software/autoconf-archive/ax_python_devel.html # =========================================================================== # # SYNOPSIS # # AX_PYTHON_DEVEL([version]) # # DESCRIPTION # # Note: Defines as a precious variable "PYTHON_VERSION". Don't override it # in your configure.ac. # # This macro checks for Python and tries to get the include path to # 'Python.h'. It provides the $(PYTHON_CPPFLAGS) and $(PYTHON_LIBS) output # variables. It also exports $(PYTHON_EXTRA_LIBS) and # $(PYTHON_EXTRA_LDFLAGS) for embedding Python in your code. # # You can search for some particular version of Python by passing a # parameter to this macro, for example ">= '2.3.1'", or "== '2.4'". Please # note that you *have* to pass also an operator along with the version to # match, and pay special attention to the single quotes surrounding the # version number. Don't use "PYTHON_VERSION" for this: that environment # variable is declared as precious and thus reserved for the end-user. # # This macro should work for all versions of Python >= 2.1.0. As an end # user, you can disable the check for the python version by setting the # PYTHON_NOVERSIONCHECK environment variable to something else than the # empty string. # # If you need to use this macro for an older Python version, please # contact the authors. We're always open for feedback. # # LICENSE # # Copyright (c) 2009 Sebastian Huber # Copyright (c) 2009 Alan W. Irwin # Copyright (c) 2009 Rafael Laboissiere # Copyright (c) 2009 Andrew Collier # Copyright (c) 2009 Matteo Settenvini # Copyright (c) 2009 Horst Knorr # Copyright (c) 2013 Daniel Mullner # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the # Free Software Foundation, either version 3 of the License, or (at your # option) any later version. # # This program is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General # Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program. If not, see . # # As a special exception, the respective Autoconf Macro's copyright owner # gives unlimited permission to copy, distribute and modify the configure # scripts that are the output of Autoconf when processing the Macro. You # need not follow the terms of the GNU General Public License when using # or distributing such scripts, even though portions of the text of the # Macro appear in them. The GNU General Public License (GPL) does govern # all other use of the material that constitutes the Autoconf Macro. # # This special exception to the GPL applies to versions of the Autoconf # Macro released by the Autoconf Archive. When you make and distribute a # modified version of the Autoconf Macro, you may extend this special # exception to the GPL to apply to your modified version as well. #serial 18 AU_ALIAS([AC_PYTHON_DEVEL], [AX_PYTHON_DEVEL]) AC_DEFUN([AX_PYTHON_DEVEL],[ # # Allow the use of a (user set) custom python version # AC_ARG_VAR([PYTHON_VERSION],[The installed Python version to use, for example '2.3'. This string will be appended to the Python interpreter canonical name.]) AC_PATH_PROG([PYTHON],[python[$PYTHON_VERSION]]) if test -z "$PYTHON"; then AC_MSG_ERROR([Cannot find python$PYTHON_VERSION in your system path]) PYTHON_VERSION="" fi # # Check for a version of Python >= 2.1.0 # AC_MSG_CHECKING([for a version of Python >= '2.1.0']) ac_supports_python_ver=`$PYTHON -c "import sys; \ ver = sys.version.split ()[[0]]; \ print (ver >= '2.1.0')"` if test "$ac_supports_python_ver" != "True"; then if test -z "$PYTHON_NOVERSIONCHECK"; then AC_MSG_RESULT([no]) AC_MSG_FAILURE([ This version of the AC@&t@_PYTHON_DEVEL macro doesn't work properly with versions of Python before 2.1.0. You may need to re-run configure, setting the variables PYTHON_CPPFLAGS, PYTHON_LIBS, PYTHON_SITE_PKG, PYTHON_EXTRA_LIBS and PYTHON_EXTRA_LDFLAGS by hand. Moreover, to disable this check, set PYTHON_NOVERSIONCHECK to something else than an empty string. ]) else AC_MSG_RESULT([skip at user request]) fi else AC_MSG_RESULT([yes]) fi # # if the macro parameter ``version'' is set, honour it # if test -n "$1"; then AC_MSG_CHECKING([for a version of Python $1]) cat << EOF > ax_python_devel_vpy.py class VPy: def vtup(self, s): - return tuple(map(int, s.strip().split("."))) + return tuple(map(int, s.strip().replace("rc", ".").split("."))) def __init__(self): import platform self.vpy = self.vtup(platform.python_version()) def __eq__(self, s): return self.vpy == self.vtup(s) def __ne__(self, s): return self.vpy != self.vtup(s) def __lt__(self, s): return self.vpy < self.vtup(s) def __gt__(self, s): return self.vpy > self.vtup(s) def __le__(self, s): return self.vpy <= self.vtup(s) def __ge__(self, s): return self.vpy >= self.vtup(s) EOF ac_supports_python_ver=`$PYTHON -c "import ax_python_devel_vpy; \ ver = ax_python_devel_vpy.VPy(); \ print (ver $1)"` rm -rf ax_python_devel_vpy*.py* __pycache__/ax_python_devel_vpy*.py* if test "$ac_supports_python_ver" = "True"; then AC_MSG_RESULT([yes]) else AC_MSG_RESULT([no]) AC_MSG_ERROR([this package requires Python $1. If you have it installed, but it isn't the default Python interpreter in your system path, please pass the PYTHON_VERSION variable to configure. See ``configure --help'' for reference. ]) PYTHON_VERSION="" fi fi # # Check if you have distutils, else fail # AC_MSG_CHECKING([for the distutils Python package]) ac_distutils_result=`$PYTHON -c "import distutils" 2>&1` if test -z "$ac_distutils_result"; then AC_MSG_RESULT([yes]) else AC_MSG_RESULT([no]) AC_MSG_ERROR([cannot import Python module "distutils". Please check your Python installation. The error was: $ac_distutils_result]) PYTHON_VERSION="" fi # # Check for Python include path # AC_MSG_CHECKING([for Python include path]) if test -z "$PYTHON_CPPFLAGS"; then python_path=`$PYTHON -c "import distutils.sysconfig; \ print (distutils.sysconfig.get_python_inc ());"` plat_python_path=`$PYTHON -c "import distutils.sysconfig; \ print (distutils.sysconfig.get_python_inc (plat_specific=1));"` if test -n "${python_path}"; then if test "${plat_python_path}" != "${python_path}"; then python_path="-I$python_path -I$plat_python_path" else python_path="-I$python_path" fi fi PYTHON_CPPFLAGS=$python_path fi AC_MSG_RESULT([$PYTHON_CPPFLAGS]) AC_SUBST([PYTHON_CPPFLAGS]) # # Check for Python library path # AC_MSG_CHECKING([for Python library path]) if test -z "$PYTHON_LIBS"; then # (makes two attempts to ensure we've got a version number # from the interpreter) ac_python_version=`cat<]], [[Py_Initialize();]]) ],[pythonexists=yes],[pythonexists=no]) AC_LANG_POP([C]) # turn back to default flags CPPFLAGS="$ac_save_CPPFLAGS" LIBS="$ac_save_LIBS" LDFLAGS="$ac_save_LDFLAGS" AC_MSG_RESULT([$pythonexists]) if test ! "x$pythonexists" = "xyes"; then AC_MSG_FAILURE([ Could not link test program to Python. Maybe the main Python library has been installed in some non-standard library path. If so, pass it to configure, via the LIBS environment variable. Example: ./configure LIBS="-L/usr/non-standard-path/python/lib" ============================================================================ ERROR! You probably have to install the development version of the Python package for your distribution. The exact name of this package varies among them. ============================================================================ ]) PYTHON_VERSION="" fi # # all done! # ]) diff --git a/pyext/yoda/include/IO.pyx b/pyext/yoda/include/IO.pyx --- a/pyext/yoda/include/IO.pyx +++ b/pyext/yoda/include/IO.pyx @@ -1,251 +1,257 @@ # cython: c_string_type=unicode """Readers and writers The basic idea here is to provide Python IO semantics by using Python to do the IO. Otherwise we get C++ IO semantics in Python. It also means we can use dummy files, e.g. anything with read/write attributes. Generally a much better idea than just 'give this a filename', and well worth the inefficiencies and potential memory limits. """ import sys ## Check if a string matches any of the given patterns, and that it doesn't match any unpatterns (for path filtering) def _pattern_check(name, patterns, unpatterns): import re if patterns: if not isinstance(patterns, (list,tuple)): patterns = [patterns] ## Compile on the fly: works because compile(compiled_re) -> compiled_re if not any(re.compile(patt).search(name) for patt in patterns): return False if unpatterns: if not isinstance(unpatterns, (list,tuple)): unpatterns = [unpatterns] ## Compile on the fly: works because compile(compiled_re) -> compiled_re if any(re.compile(patt).search(name) for patt in unpatterns): return False return True ## Make a Python list of analysis objects from a C++ vector of them cdef list _aobjects_to_list(vector[c.AnalysisObject*]* aobjects, patterns, unpatterns): cdef list out = [] cdef c.AnalysisObject* ao cdef size_t i for i in range(aobjects.size()): ao = deref(aobjects)[i] ## NOTE: automatic type conversion by passing the type() as a key to globals() newao = cutil.new_owned_cls(globals()[ao.type().decode('utf-8')], ao) if _pattern_check(newao.path, patterns, unpatterns): out.append(newao) return out ## Make a Python dict of analysis objects from a C++ vector of them cdef dict _aobjects_to_dict(vector[c.AnalysisObject*]* aobjects, patterns, unpatterns): cdef dict out = {} cdef c.AnalysisObject* ao cdef size_t i for i in range(aobjects.size()): ao = deref(aobjects)[i] ## NOTE: automatic type conversion by passing the type() as a key to globals() newao = cutil.new_owned_cls( globals()[ao.type().decode('utf-8')], ao) if _pattern_check(newao.path, patterns, unpatterns): out[newao.path] = newao return out # ## Set a istringstream's string from a C/Python string # cdef void _make_iss(c.istringstream &iss, string s): # iss.str(s) -# ## Read a file's contents as a returned string -# ## The file argument can either be a file object, filename, or special "-" reference to stdin -# def _str_from_file(file_or_filename): -# if hasattr(file_or_filename, 'read'): -# s = file_or_filename.read() -# elif file_or_filename == "-": -# s = sys.stdin.read() -# else: -# with open(file_or_filename, "r") as f: -# s = f.read() -# return s +## Read a file's contents as a returned string +## The file argument can either be a file object, filename, or special "-" reference to stdin +def _str_from_file(file_or_filename): + if hasattr(file_or_filename, 'read'): + s = file_or_filename.read() + elif file_or_filename == "-": + s = sys.stdin.read() + else: + with open(file_or_filename, "r") as f: + s = f.read() + return s -# ## Write a string to a file -# ## The file argument can either be a file object, filename, or special "-" reference to stdout -# def _str_to_file(s, file_or_filename): -# s = s.decode('utf-8') -# if hasattr(file_or_filename, 'write'): -# file_or_filename.write(s) -# elif file_or_filename == "-": -# sys.stdout.write(s) -# else: -# with open(file_or_filename, "w") as f: -# f.write(s) +## Write a string to a file +## The file argument can either be a file object, filename, or special "-" reference to stdout +def _str_to_file(s, file_or_filename): + s = s.decode('utf-8') + if hasattr(file_or_filename, 'write'): + file_or_filename.write(s) + elif file_or_filename == "-": + sys.stdout.write(s) + else: + with open(file_or_filename, "w") as f: + f.write(s) ## ## Readers ## def read(filename, asdict=True, patterns=None, unpatterns=None): """ Read data objects from the provided filename, auto-determining the format from the file extension. The loaded data objects can be filtered on their path strings, using the optional patterns and unpatterns arguments. These can be strings, compiled regex objects with a 'match' method, or any iterable of those types. If given, only analyses with paths which match at least one pattern, and do not match any unpatterns, will be returned. Returns a dict or list of analysis objects depending on the asdict argument. """ # cdef c.istringstream iss # cdef vector[c.AnalysisObject*] aobjects # with open(filename, "r") as f: # s = f.read() # _make_iss(iss, s.encode('utf-8')) # c.Reader_create(filename.encode('utf-8')).read(iss, aobjects) # return _aobjects_to_dict(&aobjects, patterns, unpatterns) if asdict \ # else _aobjects_to_list(&aobjects, patterns, unpatterns) # cdef vector[c.AnalysisObject*] aobjects c.IO_read_from_file(filename.encode('utf-8'), aobjects) return _aobjects_to_dict(&aobjects, patterns, unpatterns) if asdict \ else _aobjects_to_list(&aobjects, patterns, unpatterns) def readYODA(filename, asdict=True, patterns=None, unpatterns=None): """ Read data objects from the provided YODA-format file. The loaded data objects can be filtered on their path strings, using the optional patterns and unpatterns arguments. These can be strings, compiled regex objects with a 'match' method, or any iterable of those types. If given, only analyses with paths which match at least one pattern, and do not match any unpatterns, will be returned. Returns a dict or list of analysis objects depending on the asdict argument. """ # cdef c.istringstream iss cdef vector[c.AnalysisObject*] aobjects # s = _str_from_file(file_or_filename) # _make_iss(iss, s.encode('utf-8')) # c.ReaderYODA_create().read(iss, aobjects) c.ReaderYODA_create().read_from_file(filename.encode('utf-8'), aobjects) return _aobjects_to_dict(&aobjects, patterns, unpatterns) if asdict \ else _aobjects_to_list(&aobjects, patterns, unpatterns) def readFLAT(filename, asdict=True, patterns=None, unpatterns=None): """ Read data objects from the provided FLAT-format file. The loaded data objects can be filtered on their path strings, using the optional patterns and unpatterns arguments. These can be strings, compiled regex objects with a 'match' method, or any iterable of those types. If given, only analyses with paths which match at least one pattern, and do not match any unpatterns, will be returned. Returns a dict or list of analysis objects depending on the asdict argument. """ # cdef c.istringstream iss cdef vector[c.AnalysisObject*] aobjects # s = _str_from_file(file_or_filename) # _make_iss(iss, s.encode('utf-8')) # c.ReaderFLAT_create().read(iss, aobjects) c.ReaderFLAT_create().read_from_file(filename.encode('utf-8'), aobjects) return _aobjects_to_dict(&aobjects, patterns, unpatterns) if asdict \ else _aobjects_to_list(&aobjects, patterns, unpatterns) def readAIDA(filename, asdict=True, patterns=None, unpatterns=None): """ Read data objects from the provided AIDA-format file. The loaded data objects can be filtered on their path strings, using the optional patterns and unpatterns arguments. These can be strings, compiled regex objects with a 'match' method, or any iterable of those types. If given, only analyses with paths which match at least one pattern, and do not match any unpatterns, will be returned. Returns a dict or list of analysis objects depending on the asdict argument. DEPRECATED: AIDA is a dead format. At some point we will stop supporting it. """ # cdef c.istringstream iss cdef vector[c.AnalysisObject*] aobjects # s = _str_from_file(file_or_filename) # _make_iss(iss, s.encode('utf-8')) # c.ReaderAIDA_create().read(iss, aobjects) c.ReaderAIDA_create().read_from_file(filename.encode('utf-8'), aobjects) return _aobjects_to_dict(&aobjects, patterns, unpatterns) if asdict \ else _aobjects_to_list(&aobjects, patterns, unpatterns) ## ## Writers ## def write(ana_objs, filename): """ Write data objects to the provided filename, auto-determining the format from the file extension. """ # cdef c.ostringstream oss cdef vector[c.AnalysisObject*] vec cdef AnalysisObject a aolist = ana_objs.values() if hasattr(ana_objs, "values") else ana_objs \ if hasattr(ana_objs, "__iter__") else [ana_objs] for a in aolist: vec.push_back(a._AnalysisObject()) c.IO_write_to_file(filename.encode('utf-8'), vec) - # _str_to_file(oss.str(), filename) + #_str_to_file(oss.str(), filename) -def writeYODA(ana_objs, filename): +def writeYODA(ana_objs, file_or_filename): """ Write data objects to the provided file in YODA format. """ - # cdef c.ostringstream oss + cdef c.ostringstream oss cdef vector[c.AnalysisObject*] vec cdef AnalysisObject a aolist = ana_objs.values() if hasattr(ana_objs, "values") else ana_objs \ if hasattr(ana_objs, "__iter__") else [ana_objs] for a in aolist: vec.push_back(a._AnalysisObject()) - c.WriterYODA_create().write_to_file(filename, vec) - # c.WriterYODA_create().write(oss, vec) - # _str_to_file(oss.str(), file_or_filename) + if type(file_or_filename) is str: + c.WriterYODA_create().write_to_file(file_or_filename, vec) + else: + c.WriterYODA_create().write(oss, vec) + _str_to_file(oss.str(), file_or_filename) -def writeFLAT(ana_objs, filename): +def writeFLAT(ana_objs, file_or_filename): """ Write data objects to the provided file in FLAT format. """ - # cdef c.ostringstream oss + cdef c.ostringstream oss cdef vector[c.AnalysisObject*] vec cdef AnalysisObject a aolist = ana_objs.values() if hasattr(ana_objs, "values") else ana_objs \ if hasattr(ana_objs, "__iter__") else [ana_objs] for a in aolist: vec.push_back(a._AnalysisObject()) - c.WriterFLAT_create().write_to_file(filename, vec) - # c.WriterFLAT_create().write(oss, vec) - # _str_to_file(oss.str(), file_or_filename) + if type(file_or_filename) is str: + c.WriterFLAT_create().write_to_file(file_or_filename, vec) + else: + c.WriterFLAT_create().write(oss, vec) + _str_to_file(oss.str(), file_or_filename) -def writeAIDA(ana_objs, filename): +def writeAIDA(ana_objs, file_or_filename): """ Write data objects to the provided file in AIDA format. """ - # cdef c.ostringstream oss + cdef c.ostringstream oss cdef vector[c.AnalysisObject*] vec cdef AnalysisObject a aolist = ana_objs.values() if hasattr(ana_objs, "values") else ana_objs \ if hasattr(ana_objs, "__iter__") else [ana_objs] for a in aolist: vec.push_back(a._AnalysisObject()) - c.WriterYODA_create().write_to_file(filename, vec) - # c.WriterAIDA_create().write(oss, vec) - # _str_to_file(oss.str(), file_or_filename) + if type(file_or_filename) is str: + c.WriterAIDA_create().write_to_file(file_or_filename, vec) + else: + c.WriterAIDA_create().write(oss, vec) + _str_to_file(oss.str(), file_or_filename) diff --git a/src/Writer.cc b/src/Writer.cc --- a/src/Writer.cc +++ b/src/Writer.cc @@ -1,126 +1,126 @@ // -*- C++ -*- // // This file is part of YODA -- Yet more Objects for Data Analysis // Copyright (C) 2008-2017 The YODA collaboration (see AUTHORS for details) // #include "YODA/Writer.h" #include "YODA/WriterYODA.h" #include "YODA/WriterAIDA.h" #include "YODA/WriterFLAT.h" #include "YODA/Config/BuildConfig.h" #ifdef HAVE_LIBZ #define _XOPEN_SOURCE 700 #include "zstr/zstr.hpp" #endif #include #include #include using namespace std; namespace YODA { Writer& mkWriter(const string& name) { // Determine the format from the string (a file or file extension) const size_t lastdot = name.find_last_of("."); string fmt = Utils::toLower(lastdot == string::npos ? name : name.substr(lastdot+1)); const bool compress = (fmt == "gz"); - cout << "***" << compress << endl; + //cout << "***" << compress << endl; if (compress) { #ifndef HAVE_LIBZ throw UserError("YODA was compiled without zlib support: can't write " + name); #endif const size_t lastbutonedot = (lastdot == string::npos) ? string::npos : name.find_last_of(".", lastdot-1); fmt = Utils::toLower(lastbutonedot == string::npos ? name : name.substr(lastbutonedot+1)); } // Create the appropriate Writer Writer* w = nullptr; if (Utils::startswith(fmt, "yoda")) w = &WriterYODA::create(); if (Utils::startswith(fmt, "aida")) w = &WriterAIDA::create(); if (Utils::startswith(fmt, "dat" )) w = &WriterFLAT::create(); ///< @todo Improve/remove... .ydat? if (Utils::startswith(fmt, "flat")) w = &WriterFLAT::create(); if (!w) throw UserError("Format cannot be identified from string '" + name + "'"); w->useCompression(compress); return *w; } void Writer::write(const std::string& filename, const AnalysisObject& ao) { std::vector vec{&ao}; write(filename, vec); } // Canonical writer function, including compression handling void Writer::write(ostream& stream, const vector& aos) { std::unique_ptr zos; std::ostream* os = &stream; // Wrap the stream if needed if (_compress) { #ifdef HAVE_LIBZ // Doesn't work to always create zstr wrapper: have to only create if compressing :-/ // zstr::ostream zstream(stream); // ostream& os = _compress ? zstream : stream; os = new zstr::ostream(stream); zos.reset(os); #else throw UserError("YODA was compiled without zlib support: can't write to a compressed stream"); #endif } // Write the data components /// @todo Remove the head/body/foot distinction? writeHead(*os); bool first = true; for (const AnalysisObject* aoptr : aos) { try { if (!first) *os << "\n"; //< blank line between items writeBody(*os, aoptr); first = false; } catch (const LowStatsError& ex) { /// @todo Why specifically LowStatsError? std::cerr << "LowStatsError in writing AnalysisObject " << aoptr->title() << ":\n" << ex.what() << "\n"; } } writeFoot(*os); *os << flush; } void Writer::writeBody(ostream& stream, const AnalysisObject* ao) { if (!ao) throw WriteError("Attempting to write a null AnalysisObject*"); writeBody(stream, *ao); } void Writer::writeBody(ostream& stream, const AnalysisObject& ao) { const string aotype = ao.type(); if (aotype == "Counter") { writeCounter(stream, dynamic_cast(ao)); } else if (aotype == "Histo1D") { writeHisto1D(stream, dynamic_cast(ao)); } else if (aotype == "Histo2D") { writeHisto2D(stream, dynamic_cast(ao)); } else if (aotype == "Profile1D") { writeProfile1D(stream, dynamic_cast(ao)); } else if (aotype == "Profile2D") { writeProfile2D(stream, dynamic_cast(ao)); } else if (aotype == "Scatter1D") { writeScatter1D(stream, dynamic_cast(ao)); } else if (aotype == "Scatter2D") { writeScatter2D(stream, dynamic_cast(ao)); } else if (aotype == "Scatter3D") { writeScatter3D(stream, dynamic_cast(ao)); } else if (aotype[0] == '_') { // Skip writing AO types with underscore prefixes (needed e.g. for Rivet wrappers) // maybe write a comment line in the output } else { ostringstream oss; oss << "Unrecognised analysis object type " << aotype << " in Writer::write"; throw Exception(oss.str()); } } }