No OneTemporary
Actions

Size

18 KB

Subscribers

None

View Options

	Index: trunk/bin/prof-plotcorrelations
	===================================================================
	--- trunk/bin/prof-plotcorrelations (revision 1293)
	+++ trunk/bin/prof-plotcorrelations (revision 1294)
	@@ -1,265 +1,266 @@
	#! /usr/bin/env python

	"""\
	%prog [options] results.pkl [...]

	Plot parameter-parameter correlations.

	Two covariance estimates are available:
	1. covariance matrix as estimated by the minimiser (MINUIT) for the
	minimisation that is based on the maximal number of runs.
	2. a sample covariance estimator that represents the parameter-parameter
	correlation between different choices of runs.

	The plots file names follow this naming scheme:
	OUTDIR/correlation/NAME-ESTIMATE.pdf
	where OUTDIR is the given output dir, NAME is the basename of the result
	file's path without the extension and ESTIMATE is either 'sample' or
	'minimiser', i.e. the origin of the estimation.
	"""

	import sys, os
	import numpy
	from matplotlib import pyplot, cm

	from professor import user as prof
	from professor.controlplots import makeplotshelper as mph
	from professor.tools.stringtools import latexEscape


	params = {
	'backend':'pdf',
	'figure.dpi': 200,
	'text.fontsize': 16,
	'legend.fontsize': 8,
	'axes.titlesize': 16,
	'axes.labelsize': 16,
	'xtick.labelsize': 16,
	'ytick.labelsize': 16,
	'lines.markersize':7,
	'figure.subplot.left' : 0.16,
	'figure.subplot.right' : 0.98,
	'figure.subplot.top' : 0.98,
	'figure.subplot.bottom' : 0.36,
	'text.usetex': False
	}
	pyplot.rcParams.update(params)


	import optparse
	parser = optparse.OptionParser(usage = __doc__, version=prof.version)
	#parser.add_option("-o", "--outdir",
	#dest = "OUTDIR",
	#default = "correlations",
	#help = "Output directory for correlation plots. [default: %default]")
	parser.add_option("--tex",
	dest = "TEX",
	action = "store_true",
	default = False,
	help = "Print out LaTeX table code to stdout for each correlation plot.")
	parser.add_option("--no-sample-correlation",
	dest = "USESAMPLE",
	action = "store_false",
	default = True,
	help = "Do not plot the sample correlation estimates")
	parser.add_option("--minimiser-correlation",
	dest = "USEMINIMISER",
	action = "store_true",
	default = False,
	help = "Plot the minimiser correlation estimates for the run"
	" combination with maximal number of runs.")
	parser.add_option("--format",
	dest = "FORMAT",
	type = "choice",
	choices = ["makeplots", "matplotlib"],
	default = "matplotlib",
	help = "Choose format of output files [default: %default]")
	#parser.add_option("--make-plots",
	#dest = "MAKEPLOTS",
	#action = "store_true",
	#default = False,
	#help = "Produce output file for make-plots rather than matplotlib.")
	+prof.addDataCLOptions(parser, mc=False, ref=True, ipol=True, scan=False)
	prof.addLoggingCLOptions(parser)
	prof.addOutputCLOptions(parser)
	opts, args = parser.parse_args()
	if not args:
	sys.stderr.write(parser.format_help() + "\n")
	sys.stderr.write("Error: No result files given!")
	sys.exit(1)

	prof.log.setPriority(opts)
	prof.writeGuideLine()


	## Check if specified outdir exists and create it otherwise.
	outdir = prof.getOutputDirectory(opts, "correlations")
	prof.log.debug("Using %s for sampling output." % outdir)
	prof.io.makeDir(outdir)


	# TODO: move this to professor/controlplots
	def plotCorrelations(fig, paramnames, corr):
	"""
	Parameters
	----------
	fig : matplotlib.Figure
	paramnames : list of str
	The sorted parameter names.
	corr : array_like
	The parameter-parameter correlation matrix.
	"""
	assert corr.shape == (len(paramnames), len(paramnames))
	# grid for pseudo color plots
	X, Y = numpy.meshgrid(numpy.arange(len(paramnames) + 1),
	numpy.arange(len(paramnames) + 1))
	sub = fig.add_subplot(1,1,1)
	# Reverse the y direction (i.e. axis 0 for the pcolor call) to get
	# variances on the 1st diagonal. A 'transpose()' is not necessary here.
	#
	# Convert correlation matrix to plain ndarray, because ParameterMatrix
	# does not allow reversing the order of rows/columns
	corr = numpy.asarray(corr)
	col = sub.pcolor(X, Y, corr[::-1], vmin=-1.0, vmax=1.0, cmap=cm.RdBu_r)
	colbar = fig.colorbar(col)
	colbar.set_label('$C_{ij}$', rotation=0)
	sub.set_xticks(numpy.arange(len(paramnames)) + 0.5)
	sub.set_xticklabels(paramnames, rotation=305,
	horizontalalignment="left")
	sub.set_yticks(numpy.arange(len(paramnames)) + 0.5)
	# sub.set_yticklabels(paramnames)
	# y tick labels are reversed.
	sub.set_yticklabels(paramnames[::-1])

	def writeLatexTable(stream, params, corr):
	# fix some tex issues with SHERPA parameter names
	params = map(latexEscape, params)
	# head of table
	stream.write(r"\begin{tabular}{l\|*{%i}{c}}" % (len(params))
	+ "\n")
	stream.write(" & " + " & ".join(params) + r" \\\hline" "\n")
	for i, p1 in enumerate(params):
	stream.write(p1)
	for j, p2 in enumerate(params):
	if j < i:
	stream.write(" & -- ")
	else:
	stream.write(" & %.2f" % (corr[i,j]))
	stream.write(r"\\" + "\n")
	stream.write(r"\end{tabular}" + "\n")

	def getMakePlotsHeader(resultsname, paramnames):
	header = "# BEGIN PLOT\n"
	header +="DrawOnly=%s\n"%resultsname
	header += "RatioPlot=0\n"
	header += "LogY=0\n"
	header += "Title=Parameter correlations for %s\n"%resultsname.split("-")[0]
	header += "ZMin=-1.0\n"
	header += "ZMax=1.0\n"
	header += "LeftMargin=1.7\n"
	header += "BottomMargin=1.5\n"
	header += getTicks(paramnames)
	header += "# END PLOT\n\n"
	return header

	def getTicks(paramnames):
	xticks="XCustomTicks="
	yticks="YCustomTicks="
	for num, param in enumerate(paramnames):
	if not num == len(paramnames)-1:
	xticks+="%.1f\t\\rput[r]{45}(0,0){\\strut %s}\t"%(num+.5, param)
	yticks+="%.1f\t\\strut %s\t"%(len(paramnames) -1 - num+.5, param)
	else:
	xticks+="%.1f\t\\rput[r]{45}(0,0){\\strut %s}"%(num+.5, param)
	yticks+="%.1f\t\\strut %s"%(len(paramnames) -1 - num+.5, param)

	return xticks + "\n" + yticks + "\n"

	def getMakePlotsHisto(correlations, paramnames, resultsname):
	# Some data preparation first
	data = []
	X, Y = numpy.meshgrid(numpy.arange(len(paramnames) + 1),
	numpy.arange(len(paramnames) + 1))
	X=X[0]
	Y=Y[:,0][::-1]
	for i in xrange(len(X)-1):
	for j in xrange(len(Y)-1):
	z=correlations[i][j]
	if not numpy.isnan(z):
	data.append((X[i], X[i+1], Y[j+1], Y[j], z))
	else:
	data.append((X[i], X[i+1], Y[j+1], Y[j], 0))

	# Get the histo
	histo = "# BEGIN HISTOGRAM %s\n"%resultsname
	histo += mph.get2DHisto(data)
	histo += "# END HISTOGRAM\n"
	return histo

	def writeMakePlots(fname, correlations, paramnames, resultsname):
	f = open(fname, "w")
	f.write(getMakePlotsHeader(resultsname, paramnames))
	f.write(getMakePlotsHisto(correlations, paramnames, resultsname))
	f.close()

	def cov2corr(covmat):
	"""Calculate correlation coefficients from a covariance matrix."""
	# code copied from numpy.corrcoeff() source code
	try:
	d = numpy.diag(covmat)
	except ValueError: # scalar covariance
	return 1
	return covmat/numpy.sqrt(numpy.multiply.outer(d,d))


	for path in args:
	prof.log.info("Working on file %s..." % (path))
	fbase = os.path.splitext(os.path.basename(path))[0]
	reslist = prof.ResultList.mkFromPickle(path)
	if opts.USESAMPLE:
	paramnames = reslist.getParamNames()
	fpath = os.path.join(outdir, fbase + "-sample.")
	corr = reslist.getSampleCorrelations()

	if opts.FORMAT == "matplotlib":
	fpath += "pdf"
	fig = pyplot.figure(1, facecolor="w", figsize=(10, 10))
	plotCorrelations(fig, paramnames, corr)
	pyplot.savefig(fpath)
	fig.clf()
	else:
	fpath += "dat"
	writeMakePlots(fpath, corr, paramnames, fbase+"-sample")

	prof.log.info(" saved sample correlations to %s" % (fpath))
	if opts.TEX:
	sys.stdout.write("%% Table for sample correlations of results in"
	" %s\n" % (path))
	writeLatexTable(sys.stdout, paramnames, corr)

	if opts.USEMINIMISER:
	fpath = os.path.join(outdir, fbase + "-minimiser.")
	t = reslist.getMaxRunsResults()
	if len(t) > 1:
	prof.log.warn("%i results with maximal number of runs in file"
	" %s!" % (len(t), path))
	prof.log.warn("Using the first result only!")
	correl = cov2corr(t[0].covariance)
	print correl

	if opts.FORMAT == "matplotlib":
	fpath += "pdf"
	fig = pyplot.figure(1, facecolor="w")
	plotCorrelations(fig, correl.names, correl)
	pyplot.savefig(fpath)
	fig.clf()

	elif opts.FORMAT == "makeplots":
	fpath += "dat"
	writeMakePlots(fpath, correl, correl.names, fbase+"-minimiser")

	prof.log.info(" saved minimiser correlations to %s" % fpath)

	if opts.TEX:
	# TODO: Really want raw use of stdout?
	sys.stdout.write("%% Table for minimiser correlations of results in %s\n" % path)
	writeLatexTable(sys.stdout, correl.names, correl)
	Index: trunk/prof-completion
	===================================================================
	--- trunk/prof-completion (revision 1293)
	+++ trunk/prof-completion (revision 1294)
	@@ -1,106 +1,107 @@
	## -- sh --
	## Analysis name completion for professor scripts

	(type _filedir &> /dev/null) \|\| \
	function _filedir() {
	local cur prev commands options command
	cur="${COMP_WORDS[COMP_CWORD]}"
	COMPREPLY=( $(compgen -W "$(ls ${cur}* 2> /dev/null)" -- ${cur}) )
	return 0
	}


	## Function for doing all the work, but configured to only handle certain functions via string args
	function _prof_complete() {
	local cur prev commands options command
	COMPREPLY=()
	cur="${COMP_WORDS[COMP_CWORD]}"
	prev="${COMP_WORDS[COMP_CWORD-1]}"

	## Shared options
	opts="--help --verbose --quiet --debug --version --no-logo"
	opts="$opts -h -l -v -q"
	echo "$*" \| grep -q dirs && opts="$opts --datadir --mcdir --ipoldir --refdir"
	echo "$*" \| grep -q outdir && opts="$opts --outdir -o"
	echo "$*" \| grep -q obsweights && opts="$opts --obs --obsfile --weights"
	echo "$*" \| grep -q runs && opts="$opts --runs --runsfile --runcombs"
	echo "$*" \| grep -q ipol && opts="$opts --ipol"

	## Script-specific options
	echo "$*" \| grep -q pinterpolate && opts="$opts --weave --noweave"
	echo "$*" \| grep -q ptune && {
	opts="$opts --no-ipolhistos --ipolhistodir --no-params --paramsdir --offset --runnum-offset"
	opts="$opts --minimizer --minos --print-minuit --limits --spmethods --manual-sp --fixed-parameters --error-tunes"
	opts="$opts --eigentunes --eigentunes-dgof"
	}
	echo "$*" \| grep -q penvelopes && opts="$opts --tune --cl --logy"
	echo "$*" \| grep -q pI && opts="$opts --paramdescrs --paramsfile"
	echo "$*" \| grep -q pipolhistos && opts="$opts --params --paramsfile --paramsdir"

	## Args requiring an ipol type
	if [[ ${prev} == "--ipol" ]]; then
	COMPREPLY=( $(compgen -W "quadratic cubic" -- ${cur}) )
	return 0
	fi

	## Args requiring a starting point enum
	if [[ ${prev} == "--spmethods" ]]; then
	COMPREPLY=( $(compgen -W "center random manual" -- ${cur}) )
	return 0
	fi

	## Args requiring a filename
	if (echo ${prev} \| egrep -q -- "--\<runs\>\|--\<runsfile\>\|--\<runcombs\>\|--\<limits\>\|--\<obs\>\|--\<obsfile\>\|--\<weights\>\|--\<outfile\>\|--\<paramdescrs\>\|--\<paramsfile\>"); then
	_filedir
	return 0
	fi

	## Args requiring a directory
	if (echo ${prev} \| egrep -q -- "--\<datadir\>\|--\<mcdir\>\|--\<ipoldir\>\|--\<refdir\>\|--\<outdir\>\|-\<o\>"); then
	_filedir -d
	return 0
	fi

	## Args requiring an uncalculable argument
	if (echo ${prev} \| egrep -q -- "--\<eigentunes-dgof\>"); then
	COMPREPLY=()
	return 0
	fi

	## Complete on all switches
	COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
	if test -n "$COMPREPLY"; then
	return 0
	fi

	+ _filedir
	return 0
	}



	## Dumbly apply this rough completion function to all prof-* scripts!
	function _prof_common { _prof_complete dirs; return $?; }
	for script in prof-batchtune prof-envelopes prof-ipolhistos prof-plotcorrelations prof-runcombs prof-sensitivities \
	prof-terror prof-checkspace prof-I prof-lsobs prof-plotpulls prof-sampleparams prof-showipol prof-tune \
	prof-compare-tunes prof-interpolate prof-mergeminresults prof-plotresultscatter prof-scanchi2 prof-showminresults; do
	complete -F _prof_common -o default $script
	done

	## Overload with specific versions
	function _prof_interpolate { _prof_complete dirs outdir obsweights runs ipol pinterpolate; return $?; }
	complete -F _prof_interpolate prof-interpolate

	function _prof_tune { _prof_complete dirs outdir obsweights runs ipol ptune; return $?; }
	complete -F _prof_tune prof-tune

	function _prof_I { _prof_complete dirs outdir obsweights runs ipol pI; return $?; }
	complete -F _prof_I prof-I

	function _prof_envelopes { _prof_complete dirs outdir obsweights runs ipol penvelopes; return $?; }
	complete -F _prof_envelopes prof-envelopes

	function _prof_showminresults { _prof_complete dirs runs ipol; test "$?" = "0" && return 0; _filedir pkl; return 0; }
	complete -F _prof_showminresults -o default prof-showminresults

	function _prof_ipolhistos { _prof_complete dirs runs ipol outdir obsweights pipolhistos return 0; }
	complete -F _prof_ipolhistos prof-ipolhistos
	Index: trunk/professor/tools/log.py
	===================================================================
	--- trunk/professor/tools/log.py (revision 1293)
	+++ trunk/professor/tools/log.py (revision 1294)
	@@ -1,117 +1,118 @@
	"""
	A simple wrapper around Python's logging module. It's to share logging
	initialisation between the prof-... scripts.
	"""

	## Default log message format
	MSGFORMAT = "%(module)s: %(levelname)s %(message)s"

	## Import logging module contents, with a workaround for OpenSuse 11.1/11.2
	import sys
	from logging import *
	try:
	basicConfig(level=INFO, format=MSGFORMAT)
	except NameError:
	from logging import basicConfig
	from logging import INFO, WARNING, ERROR, CRITICAL
	from logging import log, warn, warning, debug, info, error
	from logging import Formatter
	from logging import getLogger
	basicConfig(level=INFO, format=MSGFORMAT)

	## Extra-low logging level
	TRACE = 5


	## Add coloured logging facility
	## The background is set with 40 plus the number of the color, and the foreground with 30
	+DEFAULT = -1
	BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8)
	RESET_SEQ = "\033[0m"
	COLOR_SEQ = "\033[1;%dm"
	BOLD_SEQ = "\033[1m"
	COLORS = {
	'DEBUG': CYAN,
	- 'INFO': BLUE,
	+ 'INFO': DEFAULT,
	'WARNING': YELLOW,
	'CRITICAL': MAGENTA,
	'ERROR': RED
	}

	class ColoredFormatter(Formatter):
	def __init__(self, msg, use_color=True):
	Formatter.__init__(self, msg)
	self.use_color = use_color

	def format(self, record):
	levelname = record.levelname
	message = Formatter.format(self, record)
	- if self.use_color and sys.stderr.isatty() and levelname in COLORS:
	+ if self.use_color and sys.stderr.isatty() and levelname in COLORS and COLORS[levelname] != DEFAULT:
	message = COLOR_SEQ % (30 + COLORS[levelname]) + message + RESET_SEQ
	return message

	## Apply colourised formatting
	FORMATTER = Formatter(MSGFORMAT)
	try:
	MSGFORMAT = "%(message)s"
	FORMATTER = ColoredFormatter(MSGFORMAT)
	except:
	debug("Failed to configure coloured logging formatter")


	def addLoggingCLOptions(parser, logoswitch=False):
	"""
	Add logging control options to OptionParser parser.
	"""
	from optparse import OptionGroup
	g = OptionGroup(parser, "Logging")
	g.add_option("-q", "--quiet", dest="LOGLEVEL", default=INFO, action = "store_const", const=WARNING,
	help="Turn logging off except for warnings and errors")
	g.add_option("-v", "--verbose", dest="LOGLEVEL", default=INFO, action = "store_const", const=DEBUG,
	help="Turn on verbose logging")
	g.add_option("--debug", dest="LOGLEVEL", default=INFO, action = "store_const", const=TRACE,
	help="Turn on really verbose debug logging")
	if logoswitch:
	g.add_option("-l", "--no-logo", dest="SHOW_LOGO", default=True, action = "store_false",
	help="Don't show the amusing logos of professors")
	parser.add_option_group(g)


	def trace(msg, args, *kwargs):
	return log(TRACE, msg, args, *kwargs)


	def setPriority(values):
	"""Set the priority according to the values in options.

	The OptionParser that built the values must have one of the following
	option names:
	* a 'DEBUG' option, that is True or False (deprecated)
	* a 'LOGLEVEL' option, that evaluates to a log level constant of the
	logging module.

	Parameters
	----------
	values : optparse.Values
	"""
	global FORMATTER
	if hasattr(values, "DEBUG"):
	warn("Use of deprecated log level switch in command line. Fix the"
	" called script to use addLoggingCLOptions, please")
	if values.DEBUG:
	values.LOGLEVEL = DEBUG

	if hasattr(values, "LOGLEVEL"):
	logger = getLogger()
	logger.setLevel(values.LOGLEVEL)
	## Set a more informative logging format for verbose mode
	if values.LOGLEVEL < INFO:
	MSGFORMAT = "%(module)s: %(message)s"
	FORMATTER = ColoredFormatter(MSGFORMAT)
	for hndl in logger.handlers:
	hndl.setFormatter(FORMATTER)

	## This for old scripts that don't use addLoggingCLOptions
	else:
	error(str(values))
	error(str(dir(values)))
	raise ValueError("Values instance has no 'DEBUG' or 'LOGLEVEL' attribute!")

File Metadata

Mime Type: text/x-diff
Expires: Thu, Apr 24, 6:38 AM (1 d, 19 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 4887799
Default Alt Text: (18 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions