# @package      hubzero-submit-common
# @file         CommandParser.py
# @copyright    Copyright (c) 2012-2020 The Regents of the University of California.
# @license      http://opensource.org/licenses/MIT MIT
#
# Copyright (c) 2012-2020 The Regents of the University of California.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# HUBzero is a registered trademark of The Regents of the University of California.
#
from optparse import OptionParser,SUPPRESS_HELP,IndentedHelpFormatter,Option,OptionValueError
import sys
import textwrap
import os.path
import re
import math
import csv
from copy import copy
import fnmatch
import traceback
import logging

from hubzero.submit.LogMessage    import getLogIDMessage as getLogMessage
from hubzero.submit.CommentedFile import CommentedFile

if sys.version_info > (3,):
   long = int
ZERO = long(0)

try:
   iterRange = xrange
except NameError as e:
   iterRange = range

class CommandParser:
   class IndentedHelpFormatterWithNL(IndentedHelpFormatter):
      def _format_text(self, text):
         """
         Format a paragraph of free-form text for inclusion in the
         help output at the current indentation level.
         """
         result = []
         text_width = self.width - self.current_indent

         text_text = '\n'.join([x.strip('\n\x0b\x0c\r ') for x in text.split('\n')])
         for para in text_text.split('\n\n\n'):
            if '\n\n' in para:
               for block in para.split('\n\n'):
                  text_lines = textwrap.wrap(block.lstrip('\n'), text_width)
                  if len(text_lines):
                     for line in text_lines:
                        result.append("%*s%s\n" % (self.current_indent, "", line))
               result.append('\n')
            else:
               text_lines = textwrap.wrap(para.lstrip('\n'), text_width)
               if len(text_lines):
                  for line in text_lines:
                     result.append("%*s%s\n" % (self.current_indent, "", line))
                  result.append('\n')

         return "".join(result)


      def format_option(self, option):
         # The help for each option consists of two parts:
         #   * the opt strings and metavars
         #   eg. ("-x", or "-fFILENAME, --file=FILENAME")
         #   * the user-supplied help string
         #   eg. ("turn on expert mode", "read data from FILENAME")
         #
         # If possible, we write both of these on the same line:
         #   -x    turn on expert mode
         #
         # But if the opt string list is too long, we put the help
         # string on a second line, indented to the same column it would
         # start in if it fit on the first line.
         #   -fFILENAME, --file=FILENAME
         #           read data from FILENAME
         result = []
         opts = self.option_strings[option]
         opt_width = self.help_position - self.current_indent - 2
         if len(opts) > opt_width:
            opts = "%*s%s\n" % (self.current_indent, "", opts)
            indent_first = self.help_position
         else:                       # start help on same line as opts
            opts = "%*s%-*s  " % (self.current_indent, "", opt_width, opts)
            indent_first = 0
         result.append(opts)
         if option.help:
            help_text = self.expand_default(option)
            if '\n\n' in help_text:
               help_lines = []
               help_text = "\n".join([x.strip() for x in help_text.split("\n")])
               for para in help_text.split("\n\n"):
                  help_lines.extend(textwrap.wrap(para, self.help_width))
               if len(help_lines):
                  help_lines[-1] += "\n"
            else:
               help_lines = textwrap.wrap(help_text, self.help_width)
            result.append("%*s%s\n" % (indent_first, "", help_lines[0]))
            result.extend(["%*s%s\n" % (self.help_position, "", line) for line in help_lines[1:]])
         elif opts[-1] != "\n":
            result.append("\n")

         return "".join(result)


   class CustomOption(Option):
      def checkLongPositive(option,opt,value):
         try:
            longPositive = long(value)
         except ValueError:
            raise OptionValueError("option %s: invalid long positive value: %s" % (opt,str(value)))
         else:
            if longPositive < 1:
               raise OptionValueError("option %s: invalid long positive value: %ld" % (opt,longPositive))
            else:
               return(longPositive)

      TYPES = Option.TYPES + ("longPositive",)
      TYPE_CHECKER = copy(Option.TYPE_CHECKER)
      TYPE_CHECKER["longPositive"] = checkLongPositive


   def __grabMultipleStringArguments(self,option,opt_str,value,parser,isOptional):
      assert value is None
      value = getattr(parser.values,option.dest)
      if not value:
         value = []
      rargs = parser.rargs
      while rargs:
         arg = rargs[0]

         # Stop if we hit an arg like "--foo", "-a", "-fx", "--file=f",
         # etc.  Note that this also stops on "-3" or "-3.0", so if
         # your option takes numeric values, you will need to handle
         # this.
         if  (arg == "--"):
            del rargs[0]
            break
         elif((arg[:2] == "--" and len(arg) > 2) or
              (arg[:1] == "-" and len(arg) > 1 and arg[1] != "-")):
            break
         else:
            if not arg in value:
               value.append(arg)
            del rargs[0]

      if isOptional or (len(value) > 0):
         setattr(parser.values,option.dest,value)


   def __grabStringArgument(self,option,opt_str,value,parser,isOptional):
      assert value is None
      value = getattr(parser.values,option.dest)
      if not value:
         value = []
      rargs = parser.rargs
      if rargs:
         arg = rargs[0]

         # Stop if we hit an arg like "--foo", "-a", "-fx", "--file=f",
         # etc.  Note that this also stops on "-3" or "-3.0", so if
         # your option takes numeric values, you will need to handle
         # this.
         if((arg[:2] == "--" and len(arg) > 2) or
            (arg[:1] == "-" and len(arg) > 1 and arg[1] != "-")):
            pass
         else:
            if not arg in value:
               value.append(arg)
            del rargs[0]

      if isOptional or (len(value) > 0):
         setattr(parser.values,option.dest,value)


   def __grabMultipleJobIdArguments(self,option,opt_str,value,parser):
      assert value is None
      value = getattr(parser.values,option.dest)
      if not value:
         value = []
      rargs = parser.rargs
      while rargs:
         arg = rargs[0]

         # Stop if we hit an arg like "--foo", "-a", "-fx", "--file=f",
         # etc.  Note that this also stops on "-3" or "-3.0", so if
         # your option takes numeric values, you will need to handle
         # this.
         if  (arg == "--"):
            del rargs[0]
            break
         elif((arg[:2] == "--" and len(arg) > 2) or
              (arg[:1] == "-" and len(arg) > 1 and arg[1] != "-")):
            break
         else:
            try:
               jobId = int(arg)
               if not jobId in value:
                  value.append(jobId)
            except:
               pass
            del rargs[0]

      setattr(parser.values,option.dest,value)


   def __grabJobIdArgument(self,option,opt_str,value,parser):
      assert value is None
      value = getattr(parser.values,option.dest)
      if not value:
         value = []
      rargs = parser.rargs
      if rargs:
         arg = rargs[0]

         # Stop if we hit an arg like "--foo", "-a", "-fx", "--file=f",
         # etc.  Note that this also stops on "-3" or "-3.0", so if
         # your option takes numeric values, you will need to handle
         # this.
         if((arg[:2] == "--" and len(arg) > 2) or
            (arg[:1] == "-" and len(arg) > 1 and arg[1] != "-")):
            pass
         else:
            try:
               jobId = int(arg)
               if not jobId in value:
                  value.append(jobId)
            except:
               pass
            del rargs[0]

      setattr(parser.values,option.dest,value)


   def __grabMultipleParameterArguments(self,option,opt_str,value,parser):
      assert value is None
      value = getattr(parser.values,option.dest)
      if not value:
         value = []
      rargs = parser.rargs
      while rargs:
         arg = rargs[0]

         # Stop if we hit an arg like "--foo", "-a", "-fx", "--file=f",
         # etc.  Note that this also stops on "-3" or "-3.0", so if
         # your option takes numeric values, you will need to handle
         # this.
         if  (arg == "--"):
            del rargs[0]
            break
         elif((arg[:2] == "--" and len(arg) > 2) or
              (arg[:1] == "-" and len(arg) > 1 and arg[1] != "-")):
            break
         else:
            if not arg in value:
               sep = getattr(parser.values,'separator')
               value.append((sep,arg))
            del rargs[0]

      if len(value) > 0:
         setattr(parser.values,option.dest,value)


   def __grabParameterArgument(self,option,opt_str,value,parser):
      assert value is None
      value = getattr(parser.values,option.dest)
      if not value:
         value = []
      rargs = parser.rargs
      if rargs:
         arg = rargs[0]

         # Stop if we hit an arg like "--foo", "-a", "-fx", "--file=f",
         # etc.  Note that this also stops on "-3" or "-3.0", so if
         # your option takes numeric values, you will need to handle
         # this.
         if((arg[:2] == "--" and len(arg) > 2) or
            (arg[:1] == "-" and len(arg) > 1 and arg[1] != "-")):
            pass
         else:
            if not arg in value:
               sep = getattr(parser.values,'separator')
               value.append((sep,arg))
            del rargs[0]

      if len(value) > 0:
         setattr(parser.values,option.dest,value)


   def __grabMultipleHelpArguments(self,option,opt_str,value,parser,allowedHelpOptions):
      assert value is None
      value = []
      rargs = parser.rargs
      while rargs:
         arg = rargs[0]

         # Stop if we hit an arg like "--foo", "-a", "-fx", "--file=f",
         # etc.  Note that this also stops on "-3" or "-3.0", so if
         # your option takes numeric values, you will need to handle
         # this.
         if  (arg == "--"):
            del rargs[0]
            break
         elif((arg[:2] == "--" and len(arg) > 2) or
            (arg[:1] == "-" and len(arg) > 1 and arg[1] != "-")):
            break
         elif arg in allowedHelpOptions:
            if not arg in value:
               value.append(arg)
               del rargs[0]
         else:
            break

      if len(value) == 0:
         setattr(parser.values,'help',True)
      else:
         if 'managers' in value:
            setattr(parser.values,'helpManagers',True)
         if 'venues' in value:
            setattr(parser.values,'helpVenues',True)
         if 'tools' in value:
            setattr(parser.values,'helpTools',True)
         if 'examples' in value:
            setattr(parser.values,'helpExamples',True)


   def __grabHelpArgument(self,option,opt_str,value,parser,allowedHelpOptions):
      assert value is None
      value = []
      rargs = parser.rargs
      if rargs:
         arg = rargs[0]

         # Stop if we hit an arg like "--foo", "-a", "-fx", "--file=f",
         # etc.  Note that this also stops on "-3" or "-3.0", so if
         # your option takes numeric values, you will need to handle
         # this.
         if((arg[:2] == "--" and len(arg) > 2) or
            (arg[:1] == "-" and len(arg) > 1 and arg[1] != "-")):
            pass
         else:
            if arg in allowedHelpOptions:
               value.append(arg)
               del rargs[0]

      if len(value) == 0:
         setattr(parser.values,'help',True)
      elif 'managers' in value:
         setattr(parser.values,'helpManagers',True)
      elif 'venues' in value:
         setattr(parser.values,'helpVenues',True)
      elif 'tools' in value:
         setattr(parser.values,'helpTools',True)
      elif 'examples' in value:
         setattr(parser.values,'helpExamples',True)


   def __grabMultipleVersionArguments(self,option,opt_str,value,parser,allowedVersionOptions):
      assert value is None
      value = []
      rargs = parser.rargs
      while rargs:
         arg = rargs[0]

         # Stop if we hit an arg like "--foo", "-a", "-fx", "--file=f",
         # etc.  Note that this also stops on "-3" or "-3.0", so if
         # your option takes numeric values, you will need to handle
         # this.
         if  (arg == "--"):
            del rargs[0]
            break
         elif((arg[:2] == "--" and len(arg) > 2) or
            (arg[:1] == "-" and len(arg) > 1 and arg[1] != "-")):
            break
         elif arg in allowedVersionOptions:
            if not arg in value:
               value.append(arg)
               del rargs[0]
         else:
            break

      if len(value) == 0:
         setattr(parser.values,'version',True)
      else:
         if 'client' in value:
            setattr(parser.values,'versionClient',True)
         if 'server' in value:
            setattr(parser.values,'versionServer',True)
         if 'distributor' in value:
            setattr(parser.values,'versionDistributor',True)


   def __grabVersionArgument(self,option,opt_str,value,parser,allowedVersionOptions):
      assert value is None
      value = []
      rargs = parser.rargs
      if rargs:
         arg = rargs[0]

         # Stop if we hit an arg like "--foo", "-a", "-fx", "--file=f",
         # etc.  Note that this also stops on "-3" or "-3.0", so if
         # your option takes numeric values, you will need to handle
         # this.
         if((arg[:2] == "--" and len(arg) > 2) or
            (arg[:1] == "-" and len(arg) > 1 and arg[1] != "-")):
            pass
         else:
            if arg in allowedVersionOptions:
               value.append(arg)
               del rargs[0]

      if len(value) == 0:
         setattr(parser.values,'version',True)
      elif 'client' in value:
         setattr(parser.values,'versionClient',True)
      elif 'server' in value:
         setattr(parser.values,'versionServer',True)
      elif 'distributor' in value:
         setattr(parser.values,'versionDistributor',True)


   def __grabTailLengthArgument(self,option,opt_str,value,parser):
      assert value is None
      value = getattr(parser.values,option.dest)
      if not value:
         value = []
      tailLength = 10
      rargs = parser.rargs
      if rargs:
         arg = rargs[0]

         # Stop if we hit an arg like "--foo", "-a", "-fx", "--file=f",
         # etc.  Note that this also stops on "-3" or "-3.0", so if
         # your option takes numeric values, you will need to handle
         # this.
         if((arg[:2] == "--" and len(arg) > 2) or
            (arg[:1] == "-" and len(arg) > 1 and arg[1] != "-")):
            pass
         else:
            try:
               tailLength = int(arg)
               del rargs[0]
            except:
               pass

      value.append(tailLength)

      setattr(parser.values,option.dest,value)


   def __grabMultipleTailFileArguments(self,option,opt_str,value,parser):
      assert value is None
      value = getattr(parser.values,option.dest)
      if not value:
         value = []
      rargs = parser.rargs
      while rargs:
         arg = rargs[0]

         # Stop if we hit an arg like "--foo", "-a", "-fx", "--file=f",
         # etc.  Note that this also stops on "-3" or "-3.0", so if
         # your option takes numeric values, you will need to handle
         # this.
         if  (arg == "--"):
            del rargs[0]
            break
         elif((arg[:2] == "--" and len(arg) > 2) or
              (arg[:1] == "-" and len(arg) > 1 and arg[1] != "-")):
            break
         else:
            try:
               tailFile = arg
               if not tailFile in value:
                  value.append(tailFile)
            except:
               pass
            del rargs[0]

      setattr(parser.values,option.dest,value)


   def __grabTailFileArgument(self,option,opt_str,value,parser):
      assert value is None
      value = getattr(parser.values,option.dest)
      if not value:
         value = []
      rargs = parser.rargs
      if rargs:
         arg = rargs[0]

         # Stop if we hit an arg like "--foo", "-a", "-fx", "--file=f",
         # etc.  Note that this also stops on "-3" or "-3.0", so if
         # your option takes numeric values, you will need to handle
         # this.
         if((arg[:2] == "--" and len(arg) > 2) or
            (arg[:1] == "-" and len(arg) > 1 and arg[1] != "-")):
            pass
         else:
            tailFile = arg
            del rargs[0]

            value.append(tailFile)

      setattr(parser.values,option.dest,value)


   def __grabProgressArgument(self,option,opt_str,value,parser,allowedProgressOptions):
      assert value is None
      value = []
      rargs = parser.rargs
      if rargs:
         arg = rargs[0]

         # Stop if we hit an arg like "--foo", "-a", "-fx", "--file=f",
         # etc.  Note that this also stops on "-3" or "-3.0", so if
         # your option takes numeric values, you will need to handle
         # this.
         if((arg[:2] == "--" and len(arg) > 2) or
            (arg[:1] == "-" and len(arg) > 1 and arg[1] != "-")):
            pass
         else:
            if arg in allowedProgressOptions:
               value.append(arg)
               del rargs[0]

      if len(value) == 0:
         setattr(parser.values,'progress',True)
         setattr(parser.values,'progressAuto',True)
         setattr(parser.values,'progressCurses',False)
         setattr(parser.values,'progressSubmit',False)
         setattr(parser.values,'progressText',False)
         setattr(parser.values,'progressPegasus',False)
      elif 'auto' in value:
         setattr(parser.values,'progress',True)
         setattr(parser.values,'progressAuto',True)
         setattr(parser.values,'progressCurses',False)
         setattr(parser.values,'progressSubmit',False)
         setattr(parser.values,'progressText',False)
         setattr(parser.values,'progressPegasus',False)
      elif 'curses' in value:
         setattr(parser.values,'progress',True)
         setattr(parser.values,'progressAuto',False)
         setattr(parser.values,'progressCurses',True)
         setattr(parser.values,'progressSubmit',False)
         setattr(parser.values,'progressText',False)
         setattr(parser.values,'progressPegasus',False)
      elif 'submit' in value:
         setattr(parser.values,'progress',True)
         setattr(parser.values,'progressAuto',False)
         setattr(parser.values,'progressCurses',False)
         setattr(parser.values,'progressSubmit',True)
         setattr(parser.values,'progressText',False)
         setattr(parser.values,'progressPegasus',False)
      elif 'text' in value:
         setattr(parser.values,'progress',True)
         setattr(parser.values,'progressAuto',False)
         setattr(parser.values,'progressCurses',False)
         setattr(parser.values,'progressSubmit',False)
         setattr(parser.values,'progressText',True)
         setattr(parser.values,'progressPegasus',False)
      elif 'pegasus' in value:
         setattr(parser.values,'progress',True)
         setattr(parser.values,'progressAuto',False)
         setattr(parser.values,'progressCurses',False)
         setattr(parser.values,'progressSubmit',False)
         setattr(parser.values,'progressText',False)
         setattr(parser.values,'progressPegasus',True)
      elif 'silent' in value:
         setattr(parser.values,'progress',False)
         setattr(parser.values,'progressAuto',False)
         setattr(parser.values,'progressCurses',False)
         setattr(parser.values,'progressSubmit',False)
         setattr(parser.values,'progressText',False)
         setattr(parser.values,'progressPegasus',False)


   def __init__(self,
                doubleDashTerminator=False):
      self.logger               = logging.getLogger(__name__)
      self.commandOptions       = None
      self.commandArguments     = []
      self.sweepParameters      = {}
      self.csvDataParameters    = []
      self.csvDataHasHeader     = False
      self.nCSVDataCombinations = 0
      self.submitCommandFiles   = []

      self.exampleDescriptions = """Parameter examples:
\n\n
submit -p @@cap=10pf,100pf,1uf sim.exe @:indeck
\n\n
\tSubmit 3 jobs. The @:indeck means "use the file indeck as a template file."
Substitute the values 10pf, 100pf, and 1uf in place of @@cap within the file.
Send off one job for each of the values and bring back the results.
\n\n
submit -p @@vth=0:0.2:5 -p @@cap=10pf,100pf,1uf sim.exe @:indeck
\n\n
\tSubmit 78 jobs. The parameter @@vth goes from 0 to 5 in steps of 0.2, so there are 26 values for @@vth.
For each of those values, the parameter @@cap changes from 10pf to 100pf to 1uf. 26 x 3 = 78 jobs total.
Again @:indeck is treated as a template, and the values are substituted in place of @@vth and @@cap in that file.
\n\n
submit -p params sim.exe @:indeck
\n\n
\tIn this case, parameter definitions are taken from the file named params instead of the command line.
The file might have the following contents:
\n\n
\t\t# paramters for my job submission\n
\t\tparameter @@vth=0:0.2:5\n
\t\tparameter @@cap = 10pf,100pf,1uf\n\t
\n\n
submit -p "params;@@num=1-10;@@color=blue" job.sh @:job.data
\n\n
\tFor someone who loves syntax and complexity... The semicolon separates the parameters value into three parts.
The first says to load parameters from a file params.
The next part says add an additional parameter @@num that goes from 1 to 10.
The last part says add an additional parameter @@color with a single value blue.
The parameters @@num and @@color cannot override anything defined within params; they must be new parameter names.
\n\n
submit -d input.csv sim.exe @:indeck
\n\n
\tTakes parameters from the data file input.csv, which must be in comma-separated value format.
The first line of this file may contain a series of @@param names for each of the columns.
Whitespace is significant for all values entered in the csv file.
If it doesn't, then the columns are assumed to be called @@1, @@2, @@3, etc.
Each of the remaining lines represents a set of parameter values for one job; if there are 100 such lines, there will be 100 jobs.
For example, the file input.csv might look like this:
\n\n
\t\t@@vth,@@cap\n
\t\t1.1,1pf\n
\t\t2.2,1pf\n
\t\t1.1,10pf\n
\t\t2.2,10pf
\n\n
\tParameters are substituted as before into template files such as @:indeck.
\n\n
submit -d input.csv -p "@@doping=1e15-1e17 in 30 log" sim.exe @:infile
\n\n
\tTakes parameters from the data file input.csv, but
also adds another parameter @@doping which goes from 1e15 to 1e17 in 30 points on a log scale.
For each of these points, all values in the data file will be executed.
If the data file specifies 50 jobs, then this command would run 30 x 50 = 1500 jobs.
\n\n
submit -d input.csv -i @:extra/data.txt sim.exe @:indeck
\n\n
\tIn addition to the template indeck file, send along another file extra/data.txt with each job, and treat it as a template too.
\n\n
submit -s / -p @@address=23 Main St.,Hometown,Indiana/42 Broadway,Hometown,Indiana -s , -p @@color=red,green,blue job.sh @:job.data
\n\n
\tChange the separator to slash when defining the addresses, then
change back to comma for the @@color parameter and any remaining arguments.
We shouldn't have to change the separator often, but it might come in handy if the value strings themselves have commas.
\n\n
submit -p @@num=1:1000 sim.exe input@@num
\n\n
\tSubmit jobs 1,2,3,...,1000.
Parameter names such as @@num are recognized not only in template files, but also for arguments on the command line.
In this case, the numbers 1,2,3,...,1000 are substituted into the file name,
so the various jobs take their input from "input1", "input2", ..., "input1000".
\n\n
submit -p @@file=glob:indeck* sim.exe @@file
\n\n
\tLook for files matching indeck* and use the list of names as the parameter @@file.
Those values could be substituted into other template files, or used on the command line as in this example.
Suppose the directory contains files indeck1, indeck10, and indeck2.  The glob option will order the
files in a natural order: indeck1, indeck2, indeck10.  This example would launch three jobs using each
of those files as input for the job.
\n\n
submit -p @@file=globnat:indeck* sim.exe @@file
\n\n
\tThis option has been deprecated.  The functionality is now available with the glob option.
\n\n
"""
      indentedHelpFormatterWithNL = self.IndentedHelpFormatterWithNL()
      self.parser = OptionParser(prog="submit",add_help_option=False,
                                 formatter=indentedHelpFormatterWithNL,
                                 option_class=self.CustomOption)
      self.parser.disable_interspersed_args()
      self.parser.add_option("-j","--jobid",action="store",type="longPositive",dest="jobId",
                                            help=SUPPRESS_HELP)

      self.parser.set_defaults(help=False)
      self.parser.set_defaults(helpManagers=False)
      self.parser.set_defaults(helpVenues=False)
      self.parser.set_defaults(helpTools=False)
      self.parser.set_defaults(helpExamples=False)
      allowedHelpOptions = ['managers','venues','tools','examples']

      if doubleDashTerminator:
         self.parser.add_option("-h","--help",action="callback",
                                              callback=self.__grabMultipleHelpArguments,dest="help",
                                              callback_kwargs={'allowedHelpOptions':allowedHelpOptions},
                                              help="Report command usage." +
                                                   " Optionally request listing of managers, tools, venues, or examples.")
      else:
         self.parser.add_option("-h","--help",action="callback",
                                              callback=self.__grabHelpArgument,dest="help",
                                              callback_kwargs={'allowedHelpOptions':allowedHelpOptions},
                                              help="Report command usage." +
                                                   " Optionally request listing of managers, tools, venues, or examples.")

      self.parser.set_defaults(localExecution=False)
      self.parser.add_option("-l","--local",action="store_true",dest="localExecution",
                                            help="Execute command locally")
      self.parser.set_defaults(cacheMiss="")
      self.parser.add_option("--cacheMiss",action="store",type="string",dest="cacheMiss",
                                           help=SUPPRESS_HELP)
      self.parser.set_defaults(cacheHit="")
      self.parser.add_option("--cacheHit",action="store",type="string",dest="cacheHit",
                                          help=SUPPRESS_HELP)
      self.parser.set_defaults(cachePublish="")
      self.parser.add_option("--cachePublish",action="store",type="string",dest="cachePublish",
                                              help=SUPPRESS_HELP)

      if doubleDashTerminator:
         self.parser.add_option("--status",action="callback",
                                           callback=self.__grabMultipleJobIdArguments,dest="statusIds",
                                           help="Report status for runs executing remotely.")
         self.parser.add_option("-k","--kill",action="callback",
                                              callback=self.__grabMultipleJobIdArguments,dest="killIds",
                                              help="Kill runs executing remotely.")
         self.parser.add_option("--venueStatus",action="callback",
                                                callback=self.__grabMultipleStringArguments,dest="statusVenues",
                                                callback_kwargs={'isOptional':True},
                                                help="Report status for one or more venues.")
      else:
         self.parser.add_option("--status",action="callback",
                                           callback=self.__grabJobIdArgument,dest="statusIds",
                                           help="Report status for runs executing remotely.")
         self.parser.add_option("-k","--kill",action="callback",
                                              callback=self.__grabJobIdArgument,dest="killIds",
                                              help="Kill runs executing remotely.")
         self.parser.add_option("--venueStatus",action="callback",
                                                callback=self.__grabStringArgument,dest="statusVenues",
                                                callback_kwargs={'isOptional':True},
                                                help="Report venue status.")

      if doubleDashTerminator:
         self.parser.add_option("-v","--venue",action="callback",
                                               callback=self.__grabMultipleStringArguments,dest="destinations",
                                               callback_kwargs={'isOptional':False},
                                               help="Remote job destination list terminated by --")
         self.parser.add_option("-i","--inputfile",action="callback",
                                                   callback=self.__grabMultipleStringArguments,dest="inputFiles",
                                                   callback_kwargs={'isOptional':False},
                                                   help="Input file list terminated by --")
         self.parser.add_option("-o","--outputfile",action="callback",
                                                    callback=self.__grabMultipleStringArguments,dest="outputFiles",
                                                    callback_kwargs={'isOptional':False},
                                                    help=SUPPRESS_HELP)
         self.parser.add_option("-p","--parameters",action="callback",
                                                    callback=self.__grabMultipleParameterArguments,dest="parameters",
                                                    help="Parameter sweep variables terminated by --. See examples.")
         self.parser.add_option("--json",action="callback",
                                         callback=self.__grabMultipleStringArguments,dest="jsonArgs",
                                         callback_kwargs={'isOptional':False},
                                         help="File containing submit arguments, JSON format, terminated by --.")
      else:
         self.parser.add_option("-v","--venue",action="callback",
                                               callback=self.__grabStringArgument,dest="destinations",
                                               callback_kwargs={'isOptional':False},
                                               help="Remote job destination")
         self.parser.add_option("-i","--inputfile",action="callback",
                                                   callback=self.__grabStringArgument,dest="inputFiles",
                                                   callback_kwargs={'isOptional':False},
                                                   help="Input file")
         self.parser.add_option("-o","--outputfile",action="callback",
                                                    callback=self.__grabStringArgument,dest="outputFiles",
                                                    callback_kwargs={'isOptional':False},
                                                    help=SUPPRESS_HELP)
         self.parser.add_option("-p","--parameters",action="callback",
                                                    callback=self.__grabParameterArgument,dest="parameters",
                                                    help="Parameter sweep variables. See examples.")
         self.parser.add_option("--json",action="callback",
                                         callback=self.__grabStringArgument,dest="jsonArgs",
                                         callback_kwargs={'isOptional':False},
                                         help="File containing submit arguments, JSON format.")

      self.parser.add_option("--savejson",action="store",type="string",dest="saveJson",
                                          help="Save submit arguments to JSON file for possible reuse.")

      self.parser.add_option("-d","--data",action="callback",
                                           callback=self.__grabStringArgument,dest="csvData",
                                           callback_kwargs={'isOptional':False},
                                           help="Parametric variable data - csv format")

      self.parser.set_defaults(separator=',')
      self.parser.add_option("-s","--separator",action="store",type="string",dest="separator",
                                                help="Parameter sweep variable list separator")
      self.parser.add_option("-n","--nCpus",action="store",type="int",dest="nCpus",
                                            help="Number of processors for MPI execution")
      self.parser.add_option("-N","--ppn",action="store",type="int",dest="ppn",
                                          help="Number of processors/node for MPI execution")
      self.parser.add_option("-g","--nGpus",action="store",type="int",dest="nGpus",
                                            help="Number of GPUs")
      self.parser.add_option("-G","--gpn",action="store",type="int",dest="gpn",
                                          help="Number of GPUs/node")
      self.parser.add_option("--stripes",action="store",type="int",dest="nStripes",
                                         help="Number of parallel local jobs when doing parametric sweep")
      self.parser.add_option("-w","--wallTime",action="store",type="string",dest="wallTime",
                                               help="Estimated walltime hh:mm:ss or minutes")
      self.parser.add_option("--memory",action="store",type="int",dest="memoryMB",
                                        help="Memory requirement for each node (MB).")
      self.parser.add_option("-e","--env",action="callback",
                                          callback=self.__grabStringArgument,dest="environment",
                                          callback_kwargs={'isOptional':False},
                                          help="Variable=value")
      self.parser.add_option("--runName",action="store",type="string",dest="runName",
                                         help="Name used for directories and files created during the run." +
                                              " Restricted to alphanumeric characters")
      self.parser.add_option("-m","--manager",action="callback",
                                              callback=self.__grabStringArgument,dest="manager",
                                              callback_kwargs={'isOptional':False},
                                              help="Multiprocessor job manager")
      self.parser.add_option("-r","--redundancy",action="store",type="int",dest="nRedundant",
                                                 help="Number of identical simulations to execute in parallel")
      self.parser.set_defaults(metrics=False)
      self.parser.add_option("-M","--metrics",action="store_true",dest="metrics",
                                              help="Report resource usage on exit")
      self.parser.set_defaults(detach=False)
      self.parser.add_option("--detach",action="store_true",dest="detach",
                                        help="Detach client after launching run")
      self.parser.set_defaults(attachId=ZERO)
      self.parser.add_option("--attach",action="store",type="longPositive",dest="attachId",
                                        help="Attach to previously detached started server")
      self.parser.set_defaults(waitSwitch=False)
      self.parser.add_option("-W","--wait",action="store_true",dest="waitSwitch",
                                           help="Wait for reduced job load before submission")
      self.parser.set_defaults(debug=False)
      self.parser.add_option("--debug",action="store_true",dest="debug",
                                       help=SUPPRESS_HELP)
      self.parser.set_defaults(quotaCheck=True)
      self.parser.add_option("-Q","--quota",action="store_true",dest="quotaCheck",
                                            help="Enforce local user quota on remote execution host")
      self.parser.add_option("-q","--noquota",action="store_false",dest="quotaCheck",
                                              help="Do not enforce local user quota on remote execution host")

      self.parser.add_option("--tailStdout",action="callback",
                                            callback=self.__grabTailLengthArgument,dest="tailStdoutLength",
                                            help="Periodically report tail of stdout file.")
      self.parser.add_option("--tailStderr",action="callback",
                                            callback=self.__grabTailLengthArgument,dest="tailStderrLength",
                                            help="Periodically report tail of stderr file.")
      if doubleDashTerminator:
         self.parser.add_option("--tail",action="callback",
                                         callback=self.__grabMultipleTailFileArguments,dest="tailFiles",
                                         help="Periodically report tail of application files. (terminated by --)")
      else:
         self.parser.add_option("--tail",action="callback",
                                         callback=self.__grabTailFileArgument,dest="tailFiles",
                                         help="Periodically report tail of application file.")

      self.parser.set_defaults(progress=True)
      self.parser.set_defaults(progressAuto=True)
      self.parser.set_defaults(progressCurses=False)
      self.parser.set_defaults(progressSubmit=False)
      self.parser.set_defaults(progressText=False)
      self.parser.set_defaults(progressPegasus=False)
      allowedProgressOptions = ['auto','curses','submit','text','pegasus','silent']

      self.parser.add_option("--progress",action="callback",
                                          callback=self.__grabProgressArgument,dest="progress",
                                          callback_kwargs={'allowedProgressOptions':allowedProgressOptions},
                                          help="Show progress method." +
                                               " Choices are auto, curses, submit, text, pegasus, or silent.")

      self.parser.set_defaults(showCommand=False)
      self.parser.add_option("--show",action="store_true",dest="showCommand",
                                      help="Echo submit command to stdout.")

      self.parser.set_defaults(doHeartbeat=True)
      self.parser.add_option("--noHeartbeat",action="store_false",dest="doHeartbeat",
                                             help=SUPPRESS_HELP)
      self.parser.set_defaults(disableProbeCheck=False)
      self.parser.add_option("--disableProbeCheck",action="store_true",dest="disableProbeCheck",
                                                   help=SUPPRESS_HELP)
      self.parser.add_option("--userLogPath",action="store",type="string",dest="userLogPath",
                                             help=SUPPRESS_HELP)
      self.parser.add_option("--userJobidPath",action="store",type="string",dest="userJobidPath",
                                               help=SUPPRESS_HELP)
      self.parser.add_option("--x509SubmitProxy",action="store",type="string",dest="x509SubmitProxy",
                                                 help=SUPPRESS_HELP)
      self.parser.add_option("--requestCache",action="store",type="string",dest="requestCache",
                                              help=SUPPRESS_HELP)

      self.parser.set_defaults(disableJobMonitoring=False)
      self.parser.add_option("--disableJobMonitoring",action="store_true",dest="disableJobMonitoring",
                                                      help=SUPPRESS_HELP)
      self.parser.add_option("--harvestPath",action="store",type="string",dest="harvestPath",
                                             help=SUPPRESS_HELP)
      self.parser.set_defaults(harvestId=ZERO)
      self.parser.add_option("--harvest",action="store",type="longPositive",dest="harvestId",
                                         help=SUPPRESS_HELP)

      self.parser.add_option("--measurementsPath",action="store",type="string",dest="measurementsPath",
                                                  help=SUPPRESS_HELP)

      self.parser.set_defaults(asynchronous=False)
      self.parser.add_option("--asynchronous",action="store_true",dest="asynchronous",
                                              help="Asynchronous simulation - results will not be returned")

      self.parser.set_defaults(version=False)
      self.parser.set_defaults(versionClient=False)
      self.parser.set_defaults(versionServer=False)
      self.parser.set_defaults(versionDistributor=False)
      allowedVersionOptions = ['client','server','distributor']

      if doubleDashTerminator:
         self.parser.add_option("--version",action="callback",
                                            callback=self.__grabMultipleVersionArguments,dest="version",
                                            callback_kwargs={'allowedVersionOptions':allowedVersionOptions},
                                            help=SUPPRESS_HELP)
      else:
         self.parser.add_option("--version",action="callback",
                                            callback=self.__grabVersionArgument,dest="version",
                                            callback_kwargs={'allowedVersionOptions':allowedVersionOptions},
                                            help=SUPPRESS_HELP)


   def modifyCommandArguments(self,
                              args):
      modifiedArgs = []

      globExpr     = "(.*)glob:(.+)"
      globNatExpr  = "(.*)globnat:(.+)"
      reGlobExpr     = re.compile(globExpr)
      reGlobNatExpr  = re.compile(globNatExpr)

      separator = ','
      nextArgIsSeparator = False
      for arg in args:
         argVector = []
         if   arg.startswith('--separator='):
            separator = arg.split('=')[1]
         elif nextArgIsSeparator:
            separator = arg
            nextArgIsSeparator = False
         elif arg == '--separator':
            nextArgIsSeparator = True
         elif arg == '-s':
            nextArgIsSeparator = True
         elif reGlobExpr.match(arg):
         # look for files matching the glob pattern
            prefix  = reGlobExpr.match(arg).group(1)
            pattern = reGlobExpr.match(arg).group(2)
            dirname = os.path.dirname(pattern)
            if dirname == "":
               dirname = os.getcwd()
            basename = os.path.basename(pattern)
            try:
               dirFiles = os.listdir(dirname)
            except:
               dirFiles = []
            finally:
               matchingFiles = fnmatch.filter(dirFiles,basename)
               self.__natSort(matchingFiles)
               for matchingFile in matchingFiles:
                  argVector.append(os.path.join(dirname,matchingFile))
         elif reGlobNatExpr.match(arg):
         # look for files matching the globnat pattern
            prefix  = reGlobNatExpr.match(arg).group(1)
            pattern = reGlobNatExpr.match(arg).group(2)
            dirname = os.path.dirname(pattern)
            if dirname == "":
               dirname = os.getcwd()
            basename = os.path.basename(pattern)
            try:
               dirFiles = os.listdir(dirname)
            except:
               dirFiles = []
            finally:
               matchingFiles = fnmatch.filter(dirFiles,basename)
               self.__natSort(matchingFiles)
               for matchingFile in matchingFiles:
                  argVector.append(os.path.join(dirname,matchingFile))

         if argVector:
            newArg = prefix + separator.join(argVector)
            modifiedArgs.append(newArg)
         else:
            modifiedArgs.append(copy(arg))

      return(modifiedArgs)


   def parseArguments(self,
                      args):
      options,arguments = self.parser.parse_args(args)

      self.commandOptions   = options
      self.commandArguments = []
      for argument in arguments:
         self.commandArguments.append(argument)


   def getOption(self,
                 option):
      value = getattr(self.commandOptions,option)

      return(value)


   def getEnteredExecutable(self):
      enteredExecutable = self.commandArguments[0]

      return(enteredExecutable)


   def getEnteredCommand(self):
      enteredCommand = " ".join(self.commandArguments)

      return(enteredCommand)


   def getEnteredCommandArguments(self):

      return(self.commandArguments)


   def getParametricArgumentCount(self):
      parametricArgumentCount = 0
      if self.commandArguments:
         if len(self.commandArguments) > 1:
            for argument in self.commandArguments[1:]:
               if   '@:' in argument:
                  parametricArgumentCount += 1
               elif '@@' in argument:
                  parametricArgumentCount += 1

      return(parametricArgumentCount)


   def getParametricInputCount(self):
      parametricInputCount = 0
      if self.getOption('inputFiles'):
         for inputFile in self.getOption('inputFiles'):
            if   '@:' in inputFile:
               parametricInputCount += 1
            elif '@@' in inputFile:
               parametricInputCount += 1

      return(parametricInputCount)


   def getParametricEnvironmentCount(self):
      parametricEnvironmentCount = 0
      if self.getOption('environment'):
         for environmentVariableValue in self.getOption('environment'):
            if   '@:' in environmentVariableValue:
               parametricEnvironmentCount += 1
            elif '@@' in environmentVariableValue:
               parametricEnvironmentCount += 1

      return(parametricEnvironmentCount)


   def showUsage(self):
      if self.getOption('helpExamples'):
         self.parser.epilog = self.exampleDescriptions
      self.parser.print_help()


   OPERATIONMODEHELPUSAGE          = 1 << 0
   OPERATIONMODEHELPTOOLS          = 1 << 1
   OPERATIONMODEHELPVENUES         = 1 << 2
   OPERATIONMODEHELPMANAGERS       = 1 << 3
   OPERATIONMODEHELPEXAMPLES       = 1 << 4
   OPERATIONMODEVERSIONCLIENT      = 1 << 8
   OPERATIONMODEVERSIONSERVER      = 1 << 9
   OPERATIONMODEVERSIONDISTRIBUTOR = 1 << 10
   OPERATIONMODERUNNONE            = 1 << 15
   OPERATIONMODERUNCLIENTLOCAL     = 1 << 16
   OPERATIONMODERUNSERVER          = 1 << 17
   OPERATIONMODERUNPROXY           = 1 << 18
   OPERATIONMODERUNSTATUS          = 1 << 19
   OPERATIONMODERUNKILL            = 1 << 20
   OPERATIONMODERUNDISTRIBUTOR     = 1 << 21
   OPERATIONMODERUNDISTRIBUTORID   = 1 << 22
   OPERATIONMODERUNVENUESTATUS     = 1 << 23
   OPERATIONMODERUNCACHEHIT        = 1 << 24
   OPERATIONMODERUNCACHEMISS       = 1 << 25
   OPERATIONMODERUNCACHEPUBLISH    = 1 << 26
   OPERATIONMODERUNHARVESTID       = 1 << 27
   OPERATIONMODERUNMEASUREMENT     = 1 << 28


   def getOperationMode(self):
      operationMode = 0
      if self.getOption('help'):
         operationMode = operationMode | self.OPERATIONMODEHELPUSAGE
      if self.getOption('helpTools'):
         operationMode = operationMode | self.OPERATIONMODEHELPTOOLS
      if self.getOption('helpVenues'):
         operationMode = operationMode | self.OPERATIONMODEHELPVENUES
      if self.getOption('helpManagers'):
         operationMode = operationMode | self.OPERATIONMODEHELPMANAGERS
      if self.getOption('helpExamples'):
         operationMode = operationMode | self.OPERATIONMODEHELPEXAMPLES

      if self.getOption('version') or self.getOption('versionClient'):
         operationMode = operationMode | self.OPERATIONMODEVERSIONCLIENT
      if self.getOption('version') or self.getOption('versionServer'):
         operationMode = operationMode | self.OPERATIONMODEVERSIONSERVER
      if self.getOption('version') or self.getOption('versionDistributor'):
         operationMode = operationMode | self.OPERATIONMODEVERSIONDISTRIBUTOR

      enteredCommand = self.getEnteredCommand()
      if enteredCommand == "":
         if   self.getOption('statusIds') != None:
            operationMode = operationMode | self.OPERATIONMODERUNSTATUS
         elif self.getOption('killIds') != None:
            operationMode = operationMode | self.OPERATIONMODERUNKILL
         elif self.getOption('statusVenues') != None:
            operationMode = operationMode | self.OPERATIONMODERUNVENUESTATUS
         elif self.getOption('attachId') != ZERO:
            operationMode = operationMode | self.OPERATIONMODERUNPROXY
         elif self.getOption('cacheHit'):
            operationMode = operationMode | self.OPERATIONMODERUNCACHEHIT
         elif self.getOption('cacheMiss'):
            operationMode = operationMode | self.OPERATIONMODERUNCACHEMISS
         elif self.getOption('cachePublish'):
            operationMode = operationMode | self.OPERATIONMODERUNCACHEPUBLISH
         elif self.getOption('harvestId') != ZERO:
            operationMode = operationMode | self.OPERATIONMODERUNHARVESTID
         elif self.getOption('measurementsPath'):
            operationMode = operationMode | self.OPERATIONMODERUNMEASUREMENT
         else:
            if   operationMode & (self.OPERATIONMODEHELPTOOLS | self.OPERATIONMODEHELPVENUES | \
                                  self.OPERATIONMODEHELPMANAGERS | self.OPERATIONMODEVERSIONDISTRIBUTOR):
               operationMode = operationMode | self.OPERATIONMODERUNDISTRIBUTOR
            elif operationMode & self.OPERATIONMODEVERSIONSERVER:
               operationMode = operationMode | self.OPERATIONMODERUNSERVER
            else:
               operationMode = operationMode | self.OPERATIONMODERUNNONE
      else:
         if self.getOption('localExecution'):
            operationMode = operationMode | self.OPERATIONMODERUNCLIENTLOCAL
         else:
            operationMode = operationMode | self.OPERATIONMODERUNDISTRIBUTORID

      return(operationMode)


   def logOperationMode(self,
                        operationMode):
      operationModes = []
      if operationMode & self.OPERATIONMODEHELPUSAGE:
         operationModes.append("HELPUSAGE")
      if operationMode & self.OPERATIONMODEHELPTOOLS:
         operationModes.append("HELPTOOLS")
      if operationMode & self.OPERATIONMODEHELPVENUES:
         operationModes.append("HELPVENUES")
      if operationMode & self.OPERATIONMODEHELPMANAGERS:
         operationModes.append("HELPMANAGERS")
      if operationMode & self.OPERATIONMODEHELPEXAMPLES:
         operationModes.append("HELPEXAMPLES")
      if operationMode & self.OPERATIONMODEVERSIONCLIENT:
         operationModes.append("VERSIONCLIENT")
      if operationMode & self.OPERATIONMODEVERSIONSERVER:
         operationModes.append("VERSIONSERVER")
      if operationMode & self.OPERATIONMODEVERSIONDISTRIBUTOR:
         operationModes.append("VERSIONDISTRIBUTOR")
      if operationMode & self.OPERATIONMODERUNNONE:
         operationModes.append("RUNNONE")
      if operationMode & self.OPERATIONMODERUNCLIENTLOCAL:
         operationModes.append("RUNCLIENTLOCAL")
      if operationMode & self.OPERATIONMODERUNSERVER:
         operationModes.append("RUNSERVER")
      if operationMode & self.OPERATIONMODERUNPROXY:
         operationModes.append("RUNPROXY")
      if operationMode & self.OPERATIONMODERUNSTATUS:
         operationModes.append("RUNSTATUS")
      if operationMode & self.OPERATIONMODERUNKILL:
         operationModes.append("RUNKILL")
      if operationMode & self.OPERATIONMODERUNDISTRIBUTOR:
         operationModes.append("RUNDISTRIBUTOR")
      if operationMode & self.OPERATIONMODERUNDISTRIBUTORID:
         operationModes.append("RUNDISTRIBUTORID")
      if operationMode & self.OPERATIONMODERUNVENUESTATUS:
         operationModes.append("RUNVENUESTATUS")
      if operationMode & self.OPERATIONMODERUNCACHEHIT:
         operationModes.append("RUNCACHEHIT")
      if operationMode & self.OPERATIONMODERUNCACHEMISS:
         operationModes.append("RUNCACHEMISS")
      if operationMode & self.OPERATIONMODERUNCACHEPUBLISH:
         operationModes.append("RUNCACHEPUBLISH")
      if operationMode & self.OPERATIONMODERUNHARVESTID:
         operationModes.append("RUNHARVESTID")
      if operationMode & self.OPERATIONMODERUNMEASUREMENT:
         operationModes.append("RUNMEASUREMENT")

      self.logger.log(logging.INFO,getLogMessage("OperationModes(%d): %s" % (operationMode,operationModes)))


   def validateArguments(self):
      validArguments = True
      if self.getOption('localExecution') and self.getOption('destinations'):
         errorMessage = "Local execution cannot be combined with remote destinations"
         self.logger.log(logging.ERROR,getLogMessage(errorMessage))
         validArguments = False

      if self.getOption('detach'):
         if self.getOption('localExecution'):
            errorMessage = "Local execution cannot run detached"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('statusIds') != None:
            errorMessage = "Status reporting cannot be detached"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('killIds') != None:
            errorMessage = "Run termination cannot be detached"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('harvestId'):
            errorMessage = "Run harvesting cannot be detached"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('measurementsPath'):
            errorMessage = "Measurement cannot be detached"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('asynchronous'):
            errorMessage = "Asynchronous cannot be detached"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False

      if self.getOption('attachId'):
         if self.getOption('localExecution'):
            errorMessage = "Local execution cannot be combined with attach"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('statusIds') != None:
            errorMessage = "Status reporting cannot be combined with attach"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('killIds') != None:
            errorMessage = "Run termination cannot be combined with attach"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('harvestId'):
            errorMessage = "Run harvesting cannot be combined with attach"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('measurementsPath'):
            errorMessage = "Measurement cannot be combined with attach"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('asynchronous'):
            errorMessage = "Asynchronous cannot be combined with attach"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False

      if self.getOption('cacheHit') or self.getOption('cacheMiss') or self.getOption('cachePublish'):
         squidPattern = re.compile(r"[a-z0-9_\-]+/[0-9]+/[a-z0-9]{40}\Z")
         if self.getOption('cacheHit'):
            if not squidPattern.match(self.getOption('cacheHit')):
               errorMessage = "cacheHit pattern is not valid"
               self.logger.log(logging.ERROR,getLogMessage(errorMessage))
               validArguments = False
         if self.getOption('cacheMiss'):
            if not squidPattern.match(self.getOption('cacheMiss')):
               errorMessage = "cacheMiss pattern is not valid"
               self.logger.log(logging.ERROR,getLogMessage(errorMessage))
               validArguments = False
         if self.getOption('cachePublish'):
            if not squidPattern.match(self.getOption('cachePublish')):
               errorMessage = "cachePublish pattern is not valid"
               self.logger.log(logging.ERROR,getLogMessage(errorMessage))
               validArguments = False

      if self.getOption('cacheHit') or self.getOption('cacheMiss') or self.getOption('cachePublish'):
         if self.getOption('localExecution'):
            errorMessage = "Local execution cannot be combined with cache"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('statusIds') != None:
            errorMessage = "Status reporting cannot be combined with cache"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('killIds') != None:
            errorMessage = "Run termination cannot be combined with cache"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('harvestId'):
            errorMessage = "Run harvesting cannot be combined with cache"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('measurementsPath'):
            errorMessage = "Measurement cannot be combined with cache"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('asynchronous'):
            errorMessage = "Asynchronous cannot be combined with cache"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False

      if self.getOption('disableJobMonitoring'):
         if self.getOption('localExecution'):
            errorMessage = "Local execution cannot be combined with disableJobMonitoring"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('statusIds') != None:
            errorMessage = "Status reporting cannot be combined with disableJobMonitoring"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('killIds') != None:
            errorMessage = "Run termination cannot be combined with disableJobMonitoring"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('harvestId') or self.getOption('harvestPath'):
            errorMessage = "Run harvesting cannot be combined with disableJobMonitoring"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('measurementsPath'):
            errorMessage = "Run harvesting cannot be combined with disableJobMonitoring"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('asynchronous'):
            errorMessage = "Run harvesting cannot be combined with asynchronous"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False

      if self.getOption('help') or self.getOption('helpTools') or \
         self.getOption('helpVenues') or self.getOption('helpManagers') or self.getOption('helpExamples'):
         if self.getOption('detach'):
            errorMessage = "Help cannot be combined with the detach option"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('attachId'):
            errorMessage = "Help cannot be combined with the attach option"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('cacheHit') or self.getOption('cacheMiss') or self.getOption('cachePublish'):
            errorMessage = "Help cannot be combined with the cache option"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('harvestId'):
            errorMessage = "Help cannot be combined with the harvest option"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('measurementsPath'):
            errorMessage = "Help cannot be combined with the measurement option"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('asynchronous'):
            errorMessage = "Help cannot be combined with the asynchronous option"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False

      if self.getOption('version') or self.getOption('versionClient') or \
         self.getOption('versionServer') or self.getOption('versionDistributor'):
         if self.getOption('detach'):
            errorMessage = "version cannot be combined with the detach option"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('attachId'):
            errorMessage = "version cannot be combined with the attach option"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('cacheHit') or self.getOption('cacheMiss') or self.getOption('cachePublish'):
            errorMessage = "version cannot be combined with the cache option"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('harvestId'):
            errorMessage = "version cannot be combined with the harvest option"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('measurementsPath'):
            errorMessage = "version cannot be combined with the measurement option"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('asynchronous'):
            errorMessage = "version cannot be combined with the asynchronous option"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False

      if self.getOption('runName'):
         runName = self.getOption('runName')
         if not runName.isalnum():
            errorMessage = "runName contains non-alphanumeric characters"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False

      if self.getOption('harvestId'):
         if not self.getOption('harvestPath'):
            errorMessage = "The harvest option requires that harvestPath be specified"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
      if self.getOption('harvestPath'):
         if not self.getOption('harvestId'):
            errorMessage = "The harvestPath option requires that harvest be specified"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False

      if   self.getOption('statusIds') != None and self.getOption('killIds') != None:
         errorMessage = "Status reporting cannot be combined with run termination"
         self.logger.log(logging.ERROR,getLogMessage(errorMessage))
         validArguments = False
      elif self.getOption('statusIds') != None and self.getOption('statusVenues') != None:
         errorMessage = "Run status reporting cannot be combined with venue status reporting"
         self.logger.log(logging.ERROR,getLogMessage(errorMessage))
         validArguments = False
      elif self.getOption('killIds') != None and self.getOption('statusVenues') != None:
         errorMessage = "Run termination cannot be combined with venue status reporting"
         self.logger.log(logging.ERROR,getLogMessage(errorMessage))
         validArguments = False
      elif self.getOption('harvestId') and self.getOption('statusVenues') != None:
         errorMessage = "Run harvesting cannot be combined with venue status reporting"
         self.logger.log(logging.ERROR,getLogMessage(errorMessage))
         validArguments = False
      elif self.getOption('measurementsPath') and self.getOption('statusVenues') != None:
         errorMessage = "Measurement cannot be combined with venue status reporting"
         self.logger.log(logging.ERROR,getLogMessage(errorMessage))
         validArguments = False
      elif self.getOption('asynchronous') and self.getOption('statusVenues') != None:
         errorMessage = "Asynchronous cannot be combined with venue status reporting"
         self.logger.log(logging.ERROR,getLogMessage(errorMessage))
         validArguments = False
      elif self.getOption('killIds') != None and self.getOption('measurementsPath'):
         errorMessage = "Run termination cannot be combined with measurement"
         self.logger.log(logging.ERROR,getLogMessage(errorMessage))
         validArguments = False
      elif self.getOption('killIds') != None and self.getOption('asynchronous'):
         errorMessage = "Run termination cannot be combined with asynchronous"
         self.logger.log(logging.ERROR,getLogMessage(errorMessage))
         validArguments = False
      elif self.getOption('statusIds') != None and self.getOption('measurementsPath'):
         errorMessage = "Run status reporting cannot be combined with measurement"
         self.logger.log(logging.ERROR,getLogMessage(errorMessage))
         validArguments = False
      elif self.getOption('statusIds') != None and self.getOption('asynchronous'):
         errorMessage = "Run status reporting cannot be combined with asynchronous"
         self.logger.log(logging.ERROR,getLogMessage(errorMessage))
         validArguments = False
      elif len(self.commandArguments) > 0:
         if self.getOption('statusIds') != None:
            errorMessage = "Run status reporting cannot be combined with run submission"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('killIds') != None:
            errorMessage = "Run termination cannot be combined with run submission"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('statusVenues') != None:
            errorMessage = "Venue status reporting cannot be combined with run submission"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('help') or self.getOption('helpTools') or \
            self.getOption('helpVenues') or self.getOption('helpManagers') or self.getOption('helpExamples'):
            errorMessage = "Help cannot be combined with run submission"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('attachId') != ZERO:
            errorMessage = "Run attachment cannot be combined with run submission"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('cacheHit') or self.getOption('cacheMiss') or self.getOption('cachePublish'):
            errorMessage = "cache cannot be combined with run submission"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('harvestId') or self.getOption('harvestPath'):
            errorMessage = "harvest cannot be combined with run submission"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('measurementsPath'):
            errorMessage = "Measurement cannot be combined with run submission"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
      else:
         if self.getOption('disableJobMonitoring'):
            errorMessage = "disableJobMonitoring requires remote execution"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False
         if self.getOption('asynchronous'):
            errorMessage = "asynchronous requires remote execution"
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            validArguments = False

      return(validArguments)


   try:
      isinstance("", basestring)
      def __isArgString(self,
                        arg):
         return isinstance(arg,basestring)
   except NameError:
      def __isArgString(self,
                        arg):
         return isinstance(arg,str)


   def getParameterVector(self,
                          first,
                          last,
                          inc=1):
      parameterVector = []
      if self.__isArgString(first):
         try:
            first = int(first)
         except ValueError:
            first = float(first)
      if self.__isArgString(last):
         try:
            last = int(last)
         except ValueError:
            last = float(last)
      if self.__isArgString(inc):
         try:
            inc = int(inc)
         except ValueError:
            inc = float(inc)

      if float(last-first)*inc > 0.:
         nPoints = int(math.floor((last-first)/float(inc))) + 1

         for iPoint in iterRange(0,nPoints):
            value = first + inc*iPoint
            parameterVector.append(value)
      else:
         parameterVector.append(first)

      return(parameterVector)


   @staticmethod
   def __natSort(listValues):
      """ Sort the given list in the way that humans expect.
      """
      def tryint(stringValue):
         try:
            return int(stringValue)
         except:
            return stringValue.lower()

      def alphanumKey(stringValue):
         """ Turn a string into a list of string and number chunks.
            "z23a" -> ["z", 23, "a"]
         """
         return [ tryint(c) for c in re.split('([0-9]+)',stringValue) ]

      listValues.sort(key=alphanumKey)


   def parseParameter(self,
                      parameter,
                      separator=""):
      intExpr      = "[0-9]+"
      numExpr      = r"[-+]?(?:[0-9]+\.?[0-9]*|\.[0-9]+)(?:[eEdD][-+]?[0-9]+)?"
      matlabExpr   = "^(%s)[ \t]*:[ \t]*(%s)[ \t]*(?::[ \t]*(%s))?$" % (numExpr,numExpr,numExpr)
      rangeExpr    = "^(%s)[ \t]*-[ \t]*(%s)$" % (numExpr,numExpr)
      rangeByExpr  = "^(%s)[ \t]*[-:][ \t]*(%s)[ \t]+by[ \t]+(%s)$" % (numExpr,numExpr,numExpr)
      rangePtsExpr = "^(%s)[ \t]*[-:][ \t]*(%s)[ \t]+in[ \t]+(%s)(?:[ \t]*po?i?n?ts)?(?:[ \t]+(line?a?r?|log))?$" % \
                                                                      (numExpr,numExpr,intExpr)
      globExpr     = "^glob:(.+)"
      globNatExpr  = "^globnat:(.+)"

      reIntExpr      = re.compile(intExpr)
      reNumExpr      = re.compile(numExpr)
      reMatlabExpr   = re.compile(matlabExpr)
      reRangeExpr    = re.compile(rangeExpr)
      reRangeByExpr  = re.compile(rangeByExpr)
      reRangePtsExpr = re.compile(rangePtsExpr)
      reGlobExpr     = re.compile(globExpr)
      reGlobNatExpr  = re.compile(globNatExpr)

      if   reMatlabExpr.match(parameter):
      # matched: v1:v2 or v1:v2:v3
         v1,v2,v3 = reMatlabExpr.match(parameter).groups()
         if v3:
            parameterVector = self.getParameterVector(v1,v3,v2)
         else:
            parameterVector = self.getParameterVector(v1,v2)
      elif reRangeExpr.match(parameter):
      # matched: v1-v2
         v1,v2 = reRangeExpr.match(parameter).groups()
         parameterVector = self.getParameterVector(v1,v2)
      elif reRangeByExpr.match(parameter):
      # matched: v1-v2 by inc
         v1,v2,v3 = reRangeByExpr.match(parameter).groups()
         parameterVector = self.getParameterVector(v1,v2,v3)
      elif reRangePtsExpr.match(parameter):
      # matched: v1-v2/v3 log
         v1,v2,nPoints,how = reRangePtsExpr.match(parameter).groups()
         if int(nPoints) > 1:
            if how == 'log':
               if float(v1) > 0.:
                  first = math.log10(float(v1))
               else:
                  first = 0.
               if float(v2) > 0.:
                  last = math.log10(float(v2))
               else:
                  last = 0.
               inc = (last-first)/(float(nPoints)-1)
               parameterVector = []
               for value in self.getParameterVector(first,last,inc):
                  parameterVector.append(pow(10.0,value))
            else:
               v3 = (float(v2)-float(v1))/(float(nPoints)-1)
               parameterVector = self.getParameterVector(v1,v2,v3)
         else:
            parameterVector = []
      elif reGlobExpr.match(parameter):
      # look for files matching the glob pattern
         pattern = reGlobExpr.match(parameter).group(1)
         dirname = os.path.dirname(pattern)
         if dirname == "":
            dirname = os.getcwd()
         basename = os.path.basename(pattern)
         try:
            dirFiles = os.listdir(dirname)
         except:
            dirFiles = []
         finally:
            parameterVector = []
            matchingFiles = fnmatch.filter(dirFiles,basename)
            self.__natSort(matchingFiles)
            for matchingFile in matchingFiles:
               parameterVector.append(os.path.join(dirname,matchingFile))
      elif reGlobNatExpr.match(parameter):
      # look for files matching the globnat pattern
         pattern = reGlobNatExpr.match(parameter).group(1)
         dirname = os.path.dirname(pattern)
         if dirname == "":
            dirname = os.getcwd()
         basename = os.path.basename(pattern)
         try:
            dirFiles = os.listdir(dirname)
         except:
            dirFiles = []
         finally:
            parameterVector = []
            matchingFiles = fnmatch.filter(dirFiles,basename)
            self.__natSort(matchingFiles)
            for matchingFile in matchingFiles:
               parameterVector.append(os.path.join(dirname,matchingFile))
      elif separator == "":
      # there is no separator char -- keep whole value string as one element
         parameterVector = []
         parameterVector.append(parameter)
      else:
         parameterVector = parameter.split(separator)

      return(parameterVector)


   def setSubmitCommandFiles(self):
      parameterExpr   = r"^@@([a-zA-Z0-9_]+) *\= *(.+)"
      reParameterExpr = re.compile(parameterExpr)

      if self.commandOptions.parameters:
         for separator,sweepParameters in self.commandOptions.parameters:
            for sweepParameter in sweepParameters.split(';'):
               if not reParameterExpr.match(sweepParameter):
                  if os.path.isfile(sweepParameter):
                     self.submitCommandFiles.append(sweepParameter)

      if self.commandOptions.csvData:
         csvDataFile = self.commandOptions.csvData[0]
         if os.path.isfile(csvDataFile):
            self.submitCommandFiles.append(csvDataFile)

      if self.commandOptions.requestCache:
         cacheRequestFile = self.commandOptions.requestCache
         if not reParameterExpr.match(cacheRequestFile):
            if os.path.isfile(cacheRequestFile):
               self.submitCommandFiles.append(cacheRequestFile)


   def getSubmitCommandFiles(self):
      return(self.submitCommandFiles)


   def setSweepParameters(self):
      exitCode = 0
      if self.commandOptions.parameters:
         parameterPattern = re.compile(r'\s*parameter\s+(.*[^\s$])')
         separatorPattern = re.compile(r'\s*separator\s+(.*[^\s$])')
         commentPattern   = re.compile(r'\s*#.*')
         parameterExpr    = r"^@@([a-zA-Z0-9_]+) *\= *(.+)"
         reParameterExpr = re.compile(parameterExpr)
         for separator,sweepParameters in self.commandOptions.parameters:
            for sweepParameter in sweepParameters.split(';'):
               if   reParameterExpr.match(sweepParameter):
                  name,parameter = reParameterExpr.match(sweepParameter).groups()
                  if not name in self.sweepParameters:
                     self.sweepParameters[name] = self.parseParameter(parameter,separator)
                  else:
                     errorMessage = "Parameter @@%s defined more than once" % (name)
                     self.logger.log(logging.ERROR,getLogMessage(errorMessage))
                     exitCode = 1
               elif os.path.isfile(sweepParameter):
                  sep = ','
                  try:
                     fpParams = open(sweepParameter,'r')
                     try:
                        eof = False
                        while not eof:
                           record = fpParams.readline()
                           if record != "":
                              record = commentPattern.sub("",record)
                              if   parameterPattern.match(record):
                                 fileSweepParameters = parameterPattern.match(record).group(1)
                                 for sweepParameter in fileSweepParameters.split(';'):
                                    if reParameterExpr.match(sweepParameter):
                                       name,parameter = reParameterExpr.match(sweepParameter).groups()
                                       if not name in self.sweepParameters:
                                          self.sweepParameters[name] = self.parseParameter(parameter,sep)
                                       else:
                                          errorMessage = "Parameter @@%s defined more than once" % (name)
                                          self.logger.log(logging.ERROR,getLogMessage(errorMessage))
                                          exitCode = 1
                                    else:
                                       errorMessage = "Bad parameter definition \"%s\" -- should be @@NAME=expr" % (sweepParameter)
                                       self.logger.log(logging.ERROR,getLogMessage(errorMessage))
                                       exitCode = 1
                              elif separatorPattern.match(record):
                                 sep = separatorPattern.match(record).group(1)
                           else:
                              eof = True
                     except (IOError,OSError):
                        self.logger.log(logging.ERROR,getLogMessage("%s could not be read" % (sweepParameter)))
                     finally:
                        fpParams.close()
                  except (IOError,OSError):
                     self.logger.log(logging.ERROR,getLogMessage("%s could not be opened" % (sweepParameter)))
               else:
                  errorMessage = "Bad parameter definition \"%s\" -- should be @@NAME=expr or a file" % (sweepParameter)
                  self.logger.log(logging.ERROR,getLogMessage(errorMessage))
                  exitCode = 1

      return(exitCode)


   def setCSVDataParameters(self):
      exitCode = 0
      self.nCSVDataCombinations = 0
      if self.commandOptions.csvData:
         csvDataFile = self.commandOptions.csvData[0]
         headerExpr   = r"^\s*@@([a-zA-Z0-9_]+)\s*$"
         reHeaderExpr = re.compile(headerExpr)
         if os.path.isfile(csvDataFile):
            try:
               fpCSVData = open(csvDataFile,'r')
               try:
                  csvReader = csv.reader(fpCSVData)
                  try:
                     firstRow = next(csvReader)
                     try:
                        nHeaderLabels = 0
                        for headerLabel in firstRow:
                           if reHeaderExpr.match(headerLabel):
                              nHeaderLabels += 1
                        if   nHeaderLabels == 0:
                           nHeaderIdDigits = int(math.log10(len(firstRow))+1)
                           for headerId in iterRange(1,len(firstRow)+1):
                              headerLabel = str(headerId).zfill(nHeaderIdDigits)
                              self.csvDataParameters.append(headerLabel)
                           self.nCSVDataCombinations += 1
                        elif nHeaderLabels == len(firstRow):
                           for headerLabel in firstRow:
                              name = reHeaderExpr.match(headerLabel).group(1)
                              self.csvDataParameters.append(name)
                           self.csvDataHasHeader  = True
                        else:
                           errorMessage = "Parameter data file %s: header not properly specified" % (csvDataFile)
                           self.logger.log(logging.ERROR,getLogMessage(errorMessage))
                           exitCode = 1

                        for row in csvReader:
                           self.nCSVDataCombinations += 1
                     except:
                        self.logger.log(logging.ERROR,getLogMessage(traceback.format_exc()))
                  except StopIteration:
                     self.logger.log(logging.ERROR,getLogMessage("%s header row is missing" % (csvDataFile)))
               except csv.Error as e:
                  errorMessage = "Error in parameter data file %s, line %d: %s" % (csvDataFile,csvReader.line_num,e.args[0])
                  self.logger.log(logging.ERROR,getLogMessage(errorMessage))
                  exitCode = 1
               finally:
                  fpCSVData.close()
            except (IOError,OSError):
               self.logger.log(logging.ERROR,getLogMessage("%s could not be opened" % (csvDataFile)))
         else:
            errorMessage = "%s is not a file" % (csvDataFile)
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
            exitCode = 1

      return(exitCode)


   def __getNextCSVDataCombination(self):
      csvDataFile = self.commandOptions.csvData[0]
      try:
         fpCSV = open(csvDataFile,'r')
         try:
            csvReader = csv.reader(fpCSV)
            if self.csvDataHasHeader:
               parameterNames = next(csvReader)

            for csvDataCombination in csvReader:
               yield(tuple(csvDataCombination))
         except StopIteration:
            pass
         except csv.Error as e:
            errorMessage = "Error in parameter data file %s, line %d: %s" % (csvDataFile,csvReader.line_num,e.args[0])
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
         finally:
            fpCSV.close()
      except (IOError,OSError):
         self.logger.log(logging.ERROR,getLogMessage("%s could not be opened" % (csvDataFile)))


   def __getCSVDataCombinations(self):
      csvDataCombinations = []
      if len(self.csvDataParameters) > 0:
         csvDataCombinations = list(self.__getNextCSVDataCombination())

      return(self.csvDataParameters,csvDataCombinations)


   if sys.version_info < (2,7,):
      # http://stackoverflow.com/questions/1681269/how-code-a-function-similar-to-itertools-product-in-python-2-5
      @staticmethod
      def __product(*iterables):
         """ Equivalent of itertools.product for versions < 2.6,
             which does NOT build intermediate results.
             Omitted 'repeat' option.
             product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy
         """
         nIters = len(iterables)
         lstLenths = []
         lstRemaining = [1]
         for i in iterRange(nIters-1,-1,-1):
            m = len(iterables[i])
            lstLenths.insert(0, m)
            lstRemaining.insert(0, m * lstRemaining[0])
         nProducts = lstRemaining.pop(0)

         for p in iterRange(nProducts):
            lstVals = []

            for i in iterRange(nIters):
               j = p/lstRemaining[i]%lstLenths[i]
               lstVals.append(iterables[i][j])
            yield tuple(lstVals)
   else:
      from itertools import product as __product


   def getParameterCombinationCount(self):
      parameterNames = list(self.sweepParameters.keys())
      if len(parameterNames) > 0:
         nSweepCombinations = 1
         for parameterName in parameterNames:
            nSweepCombinations *= len(self.sweepParameters[parameterName])
      else:
         nSweepCombinations = 0

      if (nSweepCombinations > 0) and (self.nCSVDataCombinations > 0):
         nCombinations = nSweepCombinations*self.nCSVDataCombinations
      else:
         nCombinations = max(nSweepCombinations,self.nCSVDataCombinations)

      return(nCombinations)


   def __getSweepParameterCombinations(self):
      sweepParameterNames        = list(self.sweepParameters.keys())
      sweepParameterCombinations = None

      if len(sweepParameterNames) > 0:
         sweepParameterValues = self.sweepParameters.values()
         sweepParameterCombinations = list(self.__product(*sweepParameterValues))

      return((sweepParameterNames,sweepParameterCombinations))


   def writeParameterCombinations(self,
                                  csvPath):
      csvDataParameters,csvDataCombinations          = self.__getCSVDataCombinations()
      sweepParameterNames,sweepParameterCombinations = self.__getSweepParameterCombinations()

      try:
         fpCSV = open(csvPath,'wt')
         try:
            csvWriter = csv.writer(fpCSV)
            parameterCombinationCount = self.getParameterCombinationCount()
            nInstanceIdDigits = int(math.log10(parameterCombinationCount)+1)
            csvWriter.writerow(('# completed: 0/%d jobs' % (parameterCombinationCount),))
            if   len(csvDataParameters) > 0 and len(sweepParameterNames) > 0:
               parameters = ['Id','Status'] + csvDataParameters + sweepParameterNames
               csvWriter.writerow(parameters)
               instanceId = 0
               for csvDataCombination in csvDataCombinations:
                  for sweepParameterCombination in sweepParameterCombinations:
                     instanceId += 1
                     instanceLabel = str(instanceId).zfill(nInstanceIdDigits)
                     combination = (instanceLabel,'setting up') + csvDataCombination + sweepParameterCombination
                     csvWriter.writerow(combination)
            elif len(csvDataParameters) > 0:
               csvDataParameters.insert(0,'Status')
               csvDataParameters.insert(0,'Id')
               csvWriter.writerow(csvDataParameters)
               instanceId = 0
               for csvDataCombination in csvDataCombinations:
                  instanceId += 1
                  instanceLabel = str(instanceId).zfill(nInstanceIdDigits)
                  csvWriter.writerow((instanceLabel,'setting up') + csvDataCombination)
            elif len(sweepParameterNames) > 0:
               sweepParameterNames.insert(0,'Status')
               sweepParameterNames.insert(0,'Id')
               csvWriter.writerow(sweepParameterNames)
               instanceId = 0
               for sweepParameterCombination in sweepParameterCombinations:
                  instanceId += 1
                  instanceLabel = str(instanceId).zfill(nInstanceIdDigits)
                  csvWriter.writerow((instanceLabel,'setting up') + sweepParameterCombination)
         except csv.Error:
            self.logger.log(logging.ERROR,getLogMessage("csv writer failed on %s" % (csvPath)))
         finally:
            fpCSV.close()
      except (IOError,OSError):
         self.logger.log(logging.ERROR,getLogMessage("%s could not be opened" % (csvPath)))


   def getNextParameterCombination(self):
      csvDataParameters,csvDataCombinations          = self.__getCSVDataCombinations()
      sweepParameterNames,sweepParameterCombinations = self.__getSweepParameterCombinations()

      if   len(csvDataParameters) > 0 and len(sweepParameterNames) > 0:
         parameters = csvDataParameters + sweepParameterNames
         for csvDataCombination in csvDataCombinations:
            for sweepParameterCombination in sweepParameterCombinations:
               combination = csvDataCombination + sweepParameterCombination
               substitutions = {}
               for parameterName,parameterValue in zip(parameters,combination):
                  substitutions[parameterName] = parameterValue
               yield(substitutions)
      elif len(csvDataParameters) > 0:
         parameters = csvDataParameters
         for csvDataCombination in csvDataCombinations:
            combination = csvDataCombination
            substitutions = {}
            for parameterName,parameterValue in zip(parameters,combination):
               substitutions[parameterName] = parameterValue
            yield(substitutions)
      elif len(sweepParameterNames) > 0:
         parameters = sweepParameterNames
         for sweepParameterCombination in sweepParameterCombinations:
            combination = sweepParameterCombination
            substitutions = {}
            for parameterName,parameterValue in zip(parameters,combination):
               substitutions[parameterName] = parameterValue
            yield(substitutions)


   def getNextParameterCombinationFromCSV(self,
                                          csvPath):
      try:
         fpCSV = open(csvPath,'r')
         try:
            csvReader = csv.reader(CommentedFile(fpCSV))
            parameterNames = next(csvReader)
            del parameterNames[0]
            del parameterNames[0]

            for parameterCombination in csvReader:
               del parameterCombination[0]
               del parameterCombination[0]
               substitutions = {}
               for parameterName,parameterValue in zip(parameterNames,parameterCombination):
                  substitutions[parameterName] = parameterValue.replace(' ',r'\ ')
               yield(substitutions)
         except StopIteration:
            pass
         except csv.Error as e:
            errorMessage = "Error in parameter data file %s, line %d: %s" % (csvPath,csvReader.line_num,e.args[0])
            self.logger.log(logging.ERROR,getLogMessage(errorMessage))
         finally:
            fpCSV.close()
      except (IOError,OSError):
         self.logger.log(logging.ERROR,getLogMessage("%s could not be opened" % (csvPath)))


