def tolog(msg, tofile=True, label='INFO', essential=False):
    """ Write date+msg to pilot log and to stdout """

    try:
        import inspect

        MAXLENGTH = 12
        # getting the name of the module that is invoking tolog() and adjust the length
        try:
            module_name = os.path.basename(inspect.stack()[1][1])
        except Exception, e:
            module_name = "unknown"
            #print "Exception caught by tolog(): ", e,
        module_name_cut = module_name[0:MAXLENGTH].ljust(MAXLENGTH)
        msg = "%i|%s| %s" % (os.getpid(),module_name_cut, msg)

        t = timeStampUTC(format='%Y-%m-%d %H:%M:%S')
        if tofile:
            appendToLog("%s|%s\n" % (t, msg))

        # remove backquotes from the msg since they cause problems with batch submission of pilot
        # (might be present in error messages from the OS)
        msg = msg.replace("`","'")
        msg = msg.replace('"','\\"')
        print "%s| %s" % (t, msg)

        # write any FAILED messages to stderr
        if "!!FAILED!!" in msg:
            try:
                print >> sys.stderr, "%s| %s" % (t, msg)
            except:
                print "Failed to print to sys.stderr: %s" % (t, msg)
    except:
        print "!!WARNING!!4000!! %s" % traceback.format_exc() def readpar(parameter, alt=False, version=0, queuename=None):
    """ Read 'parameter' from queuedata via SiteInformation class """

    from SiteInformation import SiteInformation
    si = SiteInformation()

    return si.readpar(parameter, alt=alt, version=version, queuename=queuename) def isAnalysisJob(trf):
    """ Determine whether the job is an analysis job or not """

    if (trf.startswith('https://') or trf.startswith('http://')):
        analysisJob = True
    else:
        analysisJob = False

    return analysisJob def grep(patterns, file_name):
    """ Search for the patterns in the given list in a file """
    # Example:
    # grep(["St9bad_alloc", "FATAL"], "athena_stdout.txt")
    # -> [list containing the lines below]
    #   CaloTrkMuIdAlg2.sysExecute()             ERROR St9bad_alloc
    #   AthAlgSeq.sysExecute()                   FATAL  Standard std::exception is caught

    matched_lines = []
    p = []
    for pattern in patterns:
        p.append(re.compile(pattern))

    try:
        f = open(file_name, "r")
    except IOError, e:
        tolog("!!WARNING!!2999!! %s" % e)
    else:
        while True:
            # get the next line in the file
            line = f.readline()
            if not line:
                break

            # can the search pattern be found
            for cp in p:
                if re.search(cp, line):
                    matched_lines.append(line)
        f.close()
    return matched_lines def getCmtconfig(jobCmtconfig):
    """ Get the cmtconfig from the job def or schedconfig """

    # the job def should always contain the cmtconfig
    if jobCmtconfig != "" and jobCmtconfig != "None" and jobCmtconfig != "NULL":
        cmtconfig = jobCmtconfig
        tolog("Will try to use cmtconfig: %s (from job definition)" % (cmtconfig))
    else:
        cmtconfig = readpar('cmtconfig')
        tolog("Will try to use cmtconfig: %s (from schedconfig DB)" % (cmtconfig))

    return cmtconfig def verifyReleaseString(release):
    """ Verify that the release (or homepackage) string is set """

    if release == None:
        release = ""
    release = release.upper()
    if release == "":
        release = "NULL"
    if release == "NULL":
        tolog("Detected unset (NULL) release/homepackage string")
    return release def timedCommand(cmd, timeout=300):
    """ Protect cmd with timed_command """

    tolog("Executing command: %s (protected by timed_command, timeout: %d s)" % (cmd, timeout))
    t0 = os.times()
    try:
        from TimerCommand import TimerCommand
        timerCommand = TimerCommand(cmd)
        exitcode, output = timerCommand.run(timeout=timeout)
    except Exception, e:
        pilotErrorDiag = 'TimedCommand() threw an exception: %s' % e
        tolog("!!WARNING!!2220!! %s" % pilotErrorDiag)
        exitcode = 1
        output = str(e)
    else:
        if exitcode != 0:
            tolog("!!WARNING!!2220!! Timed command returned: %s" % (output))

    t1 = os.times()
    telapsed = int(round(t1[4] - t0[4]))
    tolog("Elapsed time: %d" % (telapsed))

    if telapsed >= timeout:
        tolog("!!WARNING!!2220!! Command timed out")
        output += " (timed out)"

    # timed_command adds a trailing \n, remove it
    if output.endswith('\n'):
        output = output[:-1]

    return exitcode, output def getSiteInformation(experiment):
    """ Return a reference to a site information class """

    # The SiteInformationFactory ensures that the returned object is a Singleton
    # Usage:
    #    _exp = getSiteInformation(readpar('experiment')) # or from pilot option
    #    if _exp:
    #        _exp.somemethod("Hello")
    #    else:
    #        tolog("!!WARNING!!1111!! Failed to instantiate experiment class")

    from SiteInformationFactory import SiteInformationFactory
    factory = SiteInformationFactory()
    _exp = None

    try:
        siteInformationClass = factory.newSiteInformation(experiment)
    except Exception, e:
        tolog("!!WARNING!!1114!! SiteInformation factory threw an exception: %s" % (e))
    else:
        _exp = siteInformationClass()
        tolog("getSiteInformation: got experiment=%s" % (_exp.getExperiment()))

    return _exp def isBuildJob(outFiles):
    """
    Check if the job is a build job
    (i.e. check if the job only has one output file that is a lib file)
    """

    isABuildJob = False
    # outFiles only contains a single file for build jobs, the lib file
    if len(outFiles) == 1:
        # e.g. outFiles[0] = user.paulnilsson.lxplus001.lib._001.log.tgz
        if outFiles[0].find(".lib.") > 0:
            isABuildJob = True

    return isABuildJob def remove(entries):
    """
    Remove files and directories
    entries should be a list
    """
    status = True

    # protect against wrong usage
    if type(entries) == list:
        if len(entries) > 0:
            for entry in entries:
                try:
                    os.system("rm -rf %s" %entry)
                except OSError:
                    tolog("Could not remove %s" % entry)
                    status = False
    else:
        tolog("Argument has wrong type, expected list: %s" % str(type(entries)))
        status = False
    return status def isAGreaterOrEqualToB(A, B):
    """ Is numbered string A >= B?
    "1.2.3" > "1.2"  -- more digits
    "1.2.3" > "1.2.2"  -- rank based comparison
    "1.3.2" > "1.2.3"  -- rank based comparison
    "1.2.N" > "1.2.2"  -- nightlies checker, they are always grater
    """
    # > a="1.2.3"
    # > b="2.2.2"
    # > e.isAGreaterOrEqualToB(a,b)

    return split_version(A) >= split_version(B) def convert_unicode_string(unicode_string):
    if unicode_string is not None:
        return str(unicode_string)
    return None class PilotErrors:
    """ Pilot error handling """

    # error codes
    ERR_UNKNOWNERROR = 0
    ERR_GENERALERROR = 1008
    ERR_DIRECTIOFILE = 1009 # harmless, just means that copy-to-scratch was skipped in favor or direct i/o access
    ERR_GETDATAEXC = 1097
    ERR_NOLOCALSPACE = 1098
    ERR_STAGEINFAILED = 1099
    ERR_REPNOTFOUND = 1100
    ERR_LRCREGCONNREF = 1101
    ERR_NOSUCHFILE = 1103
    ERR_USERDIRTOOLARGE = 1104
    ERR_LFCADDCSUMFAILED = 1105
    ERR_STDOUTTOOBIG = 1106
    ERR_MISSDBREL = 1107
    ERR_FAILEDLCGREG = 1108
    ERR_CMTCONFIG = 1109
    ERR_SETUPFAILURE = 1110
    ERR_RUNJOBEXC = 1111
    ERR_PILOTEXC = 1112
    ERR_GETLFCIMPORT = 1113
    ERR_PUTLFCIMPORT = 1114
    ERR_NFSSQLITE = 1115
    ERR_QUEUEDATA = 1116
    ERR_QUEUEDATANOTOK = 1117
    ERR_CURLSPACE = 1118
    ERR_DDMSPACE = 1119
    ERR_NOSTMATCHDEST = 1120 # not used
    ERR_NOLFCSFN = 1122
    ERR_MISSINGGUID = 1123
    ERR_OUTPUTFILETOOLARGE = 1124
    ERR_NOPFC = 1130
    ERR_PUTFUNCNOCALL = 1131
    ERR_LRCREG = 1132
    ERR_NOSTORAGE = 1133
    ERR_MKDIR = 1134
    ERR_FAILEDSIZELOCAL = 1135
    ERR_FAILEDMD5LOCAL = 1136
    ERR_STAGEOUTFAILED = 1137
    ERR_FAILEDSIZE = 1138
    ERR_PUTWRONGSIZE = 1139
    ERR_FAILEDMD5 = 1140
    ERR_PUTMD5MISMATCH = 1141
    ERR_CHMODTRF = 1143
    ERR_PANDAKILL = 1144
    ERR_GETMD5MISMATCH = 1145
    ERR_DYNTRFINST = 1146
    ERR_FAILEDRM = 1148
    ERR_TRFDOWNLOAD = 1149
    ERR_LOOPINGJOB = 1150
    ERR_GETTIMEOUT = 1151
    ERR_PUTTIMEOUT = 1152
    ERR_LOSTJOBNOTFINISHED = 1153
    ERR_LOSTJOBLOGREG = 1154
    ERR_LOSTJOBFILETRANSFER = 1155
    ERR_LOSTJOBRECOVERY = 1156
    ERR_LOSTJOBMAXEDOUT = 1158
    ERR_LOSTJOBPFC = 1159
    ERR_LRCREGSTRSIZE = 1160
    ERR_LOSTJOBXML = 1161
    ERR_LRCREGDUP = 1162
    ERR_NOPROXY = 1163
    ERR_MISSINGLOCALFILE = 1164
    ERR_MISSINGOUTPUTFILE = 1165
    ERR_SIGPIPE = 1166
    ERR_MISSFILEXML = 1167
    ERR_SIZETOOLARGE = 1168
    ERR_FAILEDLFCREG = 1169
    ERR_FAILEDADLOCAL = 1170
    ERR_GETADMISMATCH = 1171
    ERR_PUTADMISMATCH = 1172
    ERR_PANDAMOVERFILENOTCACHED = 1173
    ERR_PANDAMOVERTRANSFER = 1174
    ERR_GETWRONGSIZE = 1175
    ERR_NOCHILDPROCESSES = 1176
    ERR_NOVOMSPROXY = 1177
    ERR_NOSTAGEDFILES = 1178
    ERR_FAILEDLFCGETREPS = 1179
    ERR_GETGLOBUSSYSERR = 1180
    ERR_PUTGLOBUSSYSERR = 1181
    ERR_FAILEDLFCGETREP = 1182
    ERR_GUIDSEXISTSINLRC = 1183
    ERR_MISSINGPFC = 1184
    ERR_NOSOFTWAREDIR = 1186
    ERR_NOPAYLOADMETADATA = 1187
    ERR_LCGGETTURLS = 1188
    ERR_LCGGETTURLSTIMEOUT = 1189
    ERR_LFNTOOLONG = 1190
    ERR_ZEROFILESIZE = 1191
    ERR_DBRELNOTYETTRANSFERRED = 1192
    ERR_SEPROBLEM = 1193
    ERR_NOFILEVERIFICATION = 1194
    ERR_COMMANDTIMEOUT = 1195
    ERR_GETFAILEDTOMOUNTNFS4 = 1196
    ERR_GETPNFSSYSTEMERROR = 1197
    ERR_MKDIRWORKDIR = 1199
    ERR_KILLSIGNAL = 1200
    ERR_SIGTERM = 1201
    ERR_SIGQUIT = 1202
    ERR_SIGSEGV = 1203
    ERR_SIGXCPU = 1204
#    ERR_USERKILL = 1205 # not used by pilot
    ERR_SIGBUS = 1206
    ERR_SIGUSR1 = 1207
    ERR_NOPAYLOADOUTPUT = 1210
    ERR_MISSINGINSTALLATION = 1211
    ERR_PAYLOADOUTOFMEMORY = 1212
    ERR_REACHEDMAXTIME = 1213
    ERR_DAFSNOTALLOWED = 1214
    ERR_NOTCPCONNECTION = 1215
    ERR_NOPILOTTCPSERVER = 1216
    ERR_CORECOUNTMISMATCH = 1217
    ERR_RUNEVENTEXC = 1218
    ERR_UUIDGEN = 1219
    ERR_UNKNOWN = 1220
    ERR_FILEEXIST = 1221
    ERR_GETKEYPAIR = 1222
    ERR_BADALLOC = 1223
    ERR_ESRECOVERABLE = 1224
    ERR_ESMERGERECOVERABLE = 1225
    ERR_GLEXEC = 1226
    ERR_ESATHENAMPDIED = 1227
    ERR_ESFATAL = 1228
    ERR_TEFATAL = 1229
    ERR_TEBADURL = 1230
    ERR_TEINVALIDGUID = 1231
    ERR_TEWRONGGUID = 1232
    ERR_TEHOSTNAME = 1233
    ERR_EXECUTEDCLONEJOB = 1234
    ERR_PAYLOADEXCEEDMAXMEM = 1235
    ERR_FAILEDBYSERVER = 1236
    ERR_ESKILLEDBYSERVER = 1237
    ERR_NOEVENTS = 1238
    ERR_OVERSUBSCRIBEDEVENTS = 1239
    ERR_ESMESSAGESERVER = 1240
    ERR_ESOBJECTSTORESETUP = 1241
    ERR_CHKSUMNOTSUP = 1242
    ERR_ESPREFETCHERDIED = 1243
    ERR_NORELEASEFOUND = 1244
    ERR_TOOFEWEVENTS = 1245
    ERR_NOUSERTARBALL = 1246
    ERR_BADXML = 1247
    ERR_WRONGARCHITECTURE = 1248
    ERR_RUNGENFAILURE = 1249

    # internal error codes
    ERR_DDMREG = 1
    ERR_FILEONTAPE = 2

    pilotError = {
        ERR_UNKNOWNERROR : "",
        ERR_GENERALERROR : "General pilot error, consult batch log",
        ERR_GETDATAEXC : "Get function can not be called for staging input file",
        ERR_NOLOCALSPACE : "No space left on local disk",
        ERR_STAGEINFAILED : "Get error: Staging input file failed",
        ERR_REPNOTFOUND : "Get error: Replica not found",
        ERR_LRCREGCONNREF : "LRC registration error: Connection refused",
        # 1102 : "Expected output file does not exist", # not used, see ERR_MISSINGOUTPUTFILE below
        ERR_NOSUCHFILE : "No such file or directory",
        ERR_USERDIRTOOLARGE : "User work directory too large",
        ERR_LFCADDCSUMFAILED : "Put error: Failed to add file size and checksum to LFC",
        ERR_STDOUTTOOBIG : "Payload stdout file too big",
        ERR_MISSDBREL : "Get error: Missing DBRelease file",
        ERR_FAILEDLCGREG : "Put error: LCG registration failed",
        ERR_CMTCONFIG : "Required CMTCONFIG incompatible with WN",
        ERR_SETUPFAILURE : "Failed during setup",
        ERR_RUNJOBEXC : "Exception caught by RunJob*",
        ERR_PILOTEXC : "Exception caught by pilot",
        ERR_GETLFCIMPORT : "Get error: Failed to import LFC python module",
        ERR_PUTLFCIMPORT : "Put error: Failed to import LFC python module",
        ERR_NFSSQLITE : "NFS SQLite locking problems",
        ERR_QUEUEDATA : "Pilot could not download queuedata",
        ERR_QUEUEDATANOTOK : "Pilot found non-valid queuedata",
        ERR_CURLSPACE : "Pilot could not curl space report",
        ERR_DDMSPACE : "Pilot aborted due to DDM space shortage",
        ERR_NOSTMATCHDEST : "Space token descriptor does not match destination path",
        # 1121 : "Can not read the xml file for registering output files to dispatcher", # not used
        ERR_NOLFCSFN : "Bad replica entry returned by lfc_getreplicas(): SFN not set in LFC for this guid",
        ERR_MISSINGGUID : "Missing guid in output file list",
        ERR_OUTPUTFILETOOLARGE : "Output file too large",
        ERR_NOPFC : "Get error: Failed to get PoolFileCatalog",
        ERR_PUTFUNCNOCALL : "Put function can not be called for staging out",
        ERR_LRCREG : "LRC registration error (consult log file)",
        ERR_NOSTORAGE : "Put error: Fetching default storage URL failed",
        ERR_MKDIR : "Put error: Error in mkdir on localSE, not allowed or no available space",
        ERR_FAILEDSIZELOCAL : "Could not get file size in job workdir",
        ERR_FAILEDMD5LOCAL : "Error running md5sum on the file in job workdir",
        ERR_STAGEOUTFAILED : "Put error: Error in copying the file from job workdir to localSE",
        ERR_FAILEDSIZE : "Put error: could not get the file size on localSE",
        ERR_PUTWRONGSIZE : "Put error: Problem with copying from job workdir to local SE: size mismatch",
        ERR_FAILEDMD5 : "Put error: Error running md5sum on the file on local SE",
        ERR_PUTMD5MISMATCH : "Put error: Problem with copying from job workdir to local SE: md5sum mismatch",
        # 1142 : "Put error: failed to register the file on local SE", # not used
        ERR_CHMODTRF : "Failed to chmod trf",
        ERR_PANDAKILL : "This job was killed by panda server",
        ERR_GETMD5MISMATCH : "Get error: md5sum mismatch on input file",
        ERR_DYNTRFINST : "Trf installation dir does not exist and could not be installed",
        # 1147 : "Put error: dccp returned readOnly",  # not used
        ERR_FAILEDRM : "Put error: Failed to remove readOnly file in dCache",
        ERR_TRFDOWNLOAD : "wget command failed to download trf",
        ERR_LOOPINGJOB : "Looping job killed by pilot",
        ERR_GETTIMEOUT : "Get error: Input file staging timed out",
        ERR_PUTTIMEOUT : "Put error: File copy timed out",
        ERR_LOSTJOBNOTFINISHED : "Lost job was not finished",
        ERR_LOSTJOBLOGREG : "Failed to register log file",
        ERR_LOSTJOBFILETRANSFER : "Failed to move output files for lost job",
        ERR_LOSTJOBRECOVERY : "Pilot could not recover job",
        # 1157 : "Could not create log file", # not used
        ERR_LOSTJOBMAXEDOUT : "Reached maximum number of recovery attempts",
        ERR_LOSTJOBPFC : "Job recovery could not read PoolFileCatalog.xml file (guids lost)",
        ERR_LRCREGSTRSIZE : "LRC registration error: file name string size exceeded limit of 250",
        ERR_LOSTJOBXML : "Job recovery could not generate xml for remaining output files",
        ERR_LRCREGDUP : "LRC registration error: Non-unique LFN",
        ERR_NOPROXY : "Grid proxy not valid",
        ERR_MISSINGLOCALFILE : "Get error: Local input file missing",
        ERR_MISSINGOUTPUTFILE : "Put error: Local output file missing",
        ERR_SIGPIPE : "Put error: File copy broken by SIGPIPE",
        ERR_MISSFILEXML : "Get error: Input file missing in PoolFileCatalog.xml",
        ERR_SIZETOOLARGE : "Get error: Total file size too large",
        ERR_FAILEDLFCREG : "Put error: File registration failed",
        ERR_FAILEDADLOCAL : "Error running adler32 on the file in job workdir",
        ERR_GETADMISMATCH : "Get error: adler32 mismatch on input file",
        ERR_PUTADMISMATCH : "Put error: adler32 mismatch on output file",
        ERR_PANDAMOVERFILENOTCACHED : "PandaMover staging error: File is not cached",
        ERR_PANDAMOVERTRANSFER : "PandaMover transfer failure",
        ERR_GETWRONGSIZE : "Get error: Problem with copying from local SE to job workdir: size mismatch",
        ERR_NOCHILDPROCESSES : "Pilot has no child processes (job wrapper has either crashed or did not send final status)",
        ERR_NOVOMSPROXY : "Voms proxy not valid",
        ERR_NOSTAGEDFILES : "Get error: No input files are staged",
        ERR_FAILEDLFCGETREPS : "Get error: Failed to get replicas",
        ERR_GETGLOBUSSYSERR : "Get error: Globus system error",
        ERR_PUTGLOBUSSYSERR : "Put error: Globus system error",
        ERR_FAILEDLFCGETREP : "Get error: Failed to get replica",
        ERR_GUIDSEXISTSINLRC : "LRC registration error: Guid-metadata entry already exists",
        ERR_MISSINGPFC : "Put error: PoolFileCatalog could not be found in workdir",
        # 1185 : "Put error: Error running adler32 on the file in job workdir", # not used
        ERR_NOSOFTWAREDIR : "Software directory does not exist",
        ERR_NOPAYLOADMETADATA : "Payload metadata is not available",
        ERR_LCGGETTURLS : "lcg-getturls failed",
        ERR_LCGGETTURLSTIMEOUT : "lcg-getturls was timed-out",
        ERR_LFNTOOLONG : "LFN too long (exceeding limit of 150 characters)",
        ERR_ZEROFILESIZE : "Illegal zero file size",
        ERR_DBRELNOTYETTRANSFERRED : "DBRelease file has not been transferred yet",
        ERR_NOFILEVERIFICATION : "File verification failed",
        ERR_COMMANDTIMEOUT : "Command timed out",
        ERR_GETFAILEDTOMOUNTNFS4 : "Get error: Failed to mount NSF4",
        ERR_GETPNFSSYSTEMERROR : "Get error: PNFS system error",
        # 1198 : "Can not check the child process status from the heartbeat process", # not used
        ERR_MKDIRWORKDIR : "Could not create directory",
        ERR_KILLSIGNAL : "Job terminated by unknown kill signal",
        ERR_SIGTERM : "Job killed by signal: SIGTERM",
        ERR_SIGQUIT : "Job killed by signal: SIGQUIT",
        ERR_SIGSEGV : "Job killed by signal: SIGSEGV",
        ERR_SIGXCPU : "Job killed by signal: SIGXCPU",
        ERR_SIGUSR1 : "Job killed by signal: SIGUSR1",
        ERR_SIGBUS : "Job killed by signal: SIGBUS",
        ERR_NOPAYLOADOUTPUT : "No payload output",
        ERR_MISSINGINSTALLATION : "Missing installation",
        ERR_PAYLOADOUTOFMEMORY : "Payload ran out of memory",
        ERR_REACHEDMAXTIME : "Reached batch system time limit",
        ERR_DAFSNOTALLOWED : "Site does not allow requested direct access or file stager",
        ERR_NOTCPCONNECTION : "Failed to open TCP connection to localhost (worker node network problem)",
        ERR_NOPILOTTCPSERVER : "Pilot TCP server has died",
        ERR_CORECOUNTMISMATCH : "Mismatch between core count in job and queue definition",
        ERR_RUNEVENTEXC : "Exception caught by runEvent",
        ERR_UUIDGEN : "uuidgen failed to produce a guid",
        ERR_UNKNOWN : "Job failed due to unknown reason (consult log file)",
        ERR_FILEEXIST : "File already exist",
        ERR_GETKEYPAIR : "Failed to get security key pair",
        ERR_BADALLOC : "TRF failed due to bad_alloc",
        ERR_ESMERGERECOVERABLE : "Recoverable Event Service Merge error",
        ERR_ESRECOVERABLE: "Recoverable Event Service error",
        ERR_GLEXEC: "gLExec related error",
        ERR_ESATHENAMPDIED: "AthenaMP ended Event Service job prematurely",
        ERR_ESFATAL: "Fatal Event Service error",
        ERR_TEFATAL: "Fatal Token Extractor error",
        ERR_TEHOSTNAME: "Token Extractor error: Host name could not be resolved",
        ERR_TEBADURL: "Token Extractor error: Bad URL",
        ERR_TEINVALIDGUID: "Token Extractor error: Invalid GUID length",
        ERR_TEWRONGGUID: "Token Extractor error: No tokens for this GUID",
        ERR_EXECUTEDCLONEJOB: "Already executed clone job",
        ERR_PAYLOADEXCEEDMAXMEM: "Payload exceeded maximum allowed memory",
        ERR_FAILEDBYSERVER: "Failed by server",
        ERR_ESKILLEDBYSERVER: "Event Service job killed by server",
        ERR_NOEVENTS: "Event Service no available events",
        ERR_OVERSUBSCRIBEDEVENTS: "Event Service over subscribed events",
        ERR_ESMESSAGESERVER: "Event service message server error",
        ERR_ESOBJECTSTORESETUP: "Event service objectstore setup",
        ERR_CHKSUMNOTSUP: "Mover error: query checksum is not supported",
        ERR_ESPREFETCHERDIED: "Prefetcher ended Event Service job prematurely",
        ERR_NORELEASEFOUND: "No release candidates found",
        ERR_TOOFEWEVENTS: "Too few events, less events than minimal requirement",
        ERR_NOUSERTARBALL: "User tarball cannot be downloaded from PanDA server",
        ERR_BADXML: "Badly formed XML",
        ERR_WRONGARCHITECTURE: "Job built on wrong architecture",
        ERR_RUNGENFAILURE: "RunGen failure (consult log file)",

    }

    getErrorCodes = [1097, 1099, 1100, 1103, 1107, 1113, 1130, 1145, 1151, 1164, 1167, 1168, 1171, 1175, 1178, 1179, 1180, 1182]
    putErrorCodes = [1101, 1114, 1122, 1131, 1132, 1133, 1134, 1135, 1136, 1137, 1138, 1140, 1141, 1152, 1154, 1155, 1181]
    recoverableErrorCodes = [0] + putErrorCodes

    # Error codes that will issue a Pilot-controlled resubmission
    PilotResubmissionErrorCodes = [1008, 1098, 1099, 1110, 1113, 1114, 1115, 1116, 1117, 1137, 1139, 1151, 1152, 1171, 1172, 1177, 1179, 1180, 1181, 1182, 1188, 1189, 1195, 1196, 1197, 1219]

    # Error codes used with FAX fail-over (only an error code in this list will allow FAX fail-over)
    PilotFAXErrorCodes = [1103] + PilotResubmissionErrorCodes

    def getPilotErrorDiag(self, code=0):
        """ Return text corresponding to error code """

        pilotErrorDiag = ""
        if code in self.pilotError.keys():
            pilotErrorDiag = self.pilotError[code]
        else:
            pilotErrorDiag = "Unknown pilot error code"
        return pilotErrorDiag

    def isGetErrorCode(self, code=0):
        """ Determine whether code is in the put error list or not """

        state = False
        if code in self.getErrorCodes:
            state = True

        return state

    def isPutErrorCode(self, code=0):
        """ Determine whether code is in the put error list or not """

        state = False
        if code in self.putErrorCodes:
            state = True

        return state

    @classmethod
    def isRecoverableErrorCode(self, code=0):
        """ Determine whether code is a recoverable error code or not """

        return code in self.recoverableErrorCodes

    def isPilotResubmissionErrorCode(self, code=0):
        """ Determine whether code issues a Pilot-controlled resubmission """

        state = False
        if code in self.PilotResubmissionErrorCodes:
            state = True

        return state

    def isPilotFAXErrorCode(self, code=0):
        """ Determine whether code allows for a FAX fail-over """

        state = False
        if code in self.PilotFAXErrorCodes:
            state = True

        return state

    @classmethod
    def getErrorStr(self, code):
        """
        Avoids exception if an error is not in the dictionary.
        An empty string is returned if the error is not in the dictionary.
        """
        return self.pilotError.get(code, '')

    def getErrorName(self, code):
        """ From the error code to get the error name"""
        for k in self.__class__.__dict__.keys():
            if self.__class__.__dict__[k] == code:
                return k
        return None def readFile(filename):
    """ Read the contents of a file """

    contents = ""
    if os.path.exists(filename):
        try:
            f = open(filename, 'r')
        except IOError, e:
            tolog("!!WARNING!!2121!! Failed to open file %s: %s" % (filename, e))
        else:
            try:
                contents = f.read()
            except Exception, e:
                tolog("!!WARNING!!2122!! Failed to read file %s: %s" % (filename, e))
            f.close()
    else:
        tolog("!!WARNING!!2121!! File does not exist: %s" % (filename))

    return contents def writeFile(filename, contents, mode='w'):
    """ Write the contents to filename """

    status = False
    try:
        f = open(filename, mode)
    except IOError, e:
        tolog("!!WARNING!!2123!! Failed to open file %s: %s" % (filename, e))
    else:
        try:
            f.write(contents)
        except IOError, e:
            tolog("!!WARNING!!2123!! Failed to write to file %s: %s" % (filename, e))
        else:
            status = True
        f.close()

    return status def updatePilotErrorReport(pilotErrorCode, pilotErrorDiag, priority, jobID, workdir):
    """ Write pilot error info to file """
    # Report format:
    # { jobID1: { priority1: [{ pilotErrorCode1:<nr>, pilotErrorDiag1:<str> }, .. ], .. }, .. }
    # The pilot will report only the first of the highest priority error when it reports the error at the end of the job
    # Use the following priority convention:
    # "0": highest priority [e.g. errors that originate from the main pilot module (unless otherwise necessary)]
    # "1": high priority [e.g. errors that originate from the Monitor module (-||-)]
    # "2": normal priority [errors that originate from other modules (-||-)]
    # etc

    # Convert to string if integer is sent for priority
    if type(priority) != str:
        priority = str(priority)

    filename = getPilotErrorReportFilename(workdir)
    if os.path.exists(filename):
        # The file already exists, read it back (with unicode to utf-8 conversion)
        dictionary = getJSONDictionary(filename)
    else:
        dictionary = {}

    # Sort the new error
    if dictionary.has_key(jobID):
        jobID_dictionary = dictionary[jobID]

        # Update with the latest error info
        if not jobID_dictionary.has_key(priority):
            dictionary[jobID][priority] = []

        new_dictionary = { 'pilotErrorCode':pilotErrorCode, 'pilotErrorDiag':pilotErrorDiag }

        # Update the dictionary with the new info
        dictionary[jobID][priority].append(new_dictionary)
        print dictionary

    else:
        # Create a first entry into the error report
        dictionary[jobID] = {}
        dictionary[jobID][priority] = []
        dictionary[jobID][priority].append({})
        dictionary[jobID][priority][0] = { 'pilotErrorCode':pilotErrorCode, 'pilotErrorDiag':pilotErrorDiag }

    # Finally update the file
    status = writeJSON(filename, dictionary) def getJSONDictionary(filename):
    """ Read a dictionary with unicode to utf-8 conversion """

    dictionary = None
    from json import load
    f = openFile(filename, 'r')
    if f:
        try:
            dictionary = load(f)
        except Exception, e:
            tolog("!!WARNING!!2222!! Failed to load json dictionary: %s" % (e))
        else:
            f.close()

            # Try to convert the dictionary from unicode to utf-8
            if dictionary != {}:
                try:
                    dictionary = convert(dictionary)
                except Exception, e:
                    tolog("!!WARNING!!2996!! Failed to convert dictionary from unicode to utf-8: %s, %s" % (dictionary, e))
            else:
                tolog("!!WARNING!!2995!! Load function returned empty JSON dictionary: %s" % (filename))

    return dictionary def dumpOutput(filename):
    """ dump an extract from an ascii file """

    ret = ""
    if os.path.exists(filename):
        fsize = os.path.getsize(filename)
        tolog("Filename : %s" % (filename))
        tolog("File size: %s" % str(fsize))
        if fsize > 2**14: # 16k
            tolog("Begin output (first and last 8k)............................................")
            ret = commands.getoutput("head --bytes=8192 %s" % filename)
            if ret == "":
                tolog("[no output?]")
            else:
                # protect against corrupted files containing illegal chars
                try:
                    tolog(str(ret))
                except Exception, e:
                    tolog("!!WARNING!!3000!! Could not dump file: %s" % str(e))
            tolog("\n.... [snip] ....\n")
            ret = commands.getoutput("tail --bytes=8192 %s" % filename)
            if ret == "":
                tolog("[no output?]")
            else:
                # protect against corrupted files containing illegal chars
                try:
                    tolog(str(ret))
                except Exception, e:
                    tolog("!!WARNING!!3000!! Could not dump file: %s" % str(e))
        else:
            tolog("Begin output (all)..........................................................")
            ret = commands.getoutput("cat %s" % filename)
            if ret == "":
                tolog("[no output?]")
            else:
                # protect against corrupted files containing illegal chars
                try:
                    tolog(ret)
                except Exception, e:
                    tolog("!!WARNING!!3000!! Could not dump file: %s" % str(e))
        tolog("End output..................................................................")
    else:
        tolog("!!WARNING!!3000!! file %s does not exist" % (filename))

    return ret def getStdoutFilename(workdir, _stdout, current_job_number, number_of_jobs):
    """ Return a proper stdout filename """

    if number_of_jobs > 1:
        _stdout = _stdout.replace(".txt", "_%d.txt" % (current_job_number))
    return os.path.join(workdir, _stdout) def findVmPeaks(setup):
    """ Find the VmPeak values """

    vmPeakMax = 0
    vmPeakMean = 0
    RSSMean = 0

#    matched_lines = grep(["Py\:PerfMonSvc\s*INFO\s*VmPeak:\s*[0-9]"], stdout_filename)
#    pattern = "([0-9]+)"

#    # now extract the digits from the found lines
#    N = 0
#    vmPeaks = 0
#    for line in matched_lines:
#        _vmPeak = re.search(pattern, line)
#        if _vmPeak:
#            N += 1
#            vmPeak = _vmPeak.group(1)
#            if vmPeak > vmPeakMax:
#                vmPeakMax = vmPeak
#            vmPeaks += vmPeak

    # use the VmPeak script to get all values
    cmd = "%s python VmPeak.py >Pilot_VmPeak.txt" % (setup)
    try:
        ec, output = timedCommand(cmd, timeout=getProperTimeout(setup))
    except Exception, e:
        tolog("!!WARNING!!1111!! Failed to execute VmPeak script: %s" % (e))
    else:
        # now get the values from the deault file
        file_name = os.path.join(os.getcwd(), "VmPeak_values.txt")
        if ec == 0:
            if os.path.exists(file_name):
                try:
                    f = open(file_name, "r")
                except Exception, e:
                    tolog("!!WARNING!!1111!! Failed to open VmPeak values file: %s" % (e))
                else:
                    _values = f.read()
                    f.close()
                    values = _values.split(",")
                    try:
                        vmPeakMax = int(values[0])
                        vmPeakMean = int(values[1])
                        RSSMean = int(values[2])
                    except Exception, e:
                        tolog("!!WARNING!!1111!! VmPeak exception: %s" % (e))
            else:
                tolog("Note: File %s does not exist" % (file_name))
        else:
            tolog("!!WARNING!!1111!! VmPeak script returned: %d, %s" % (ec, output))

    tolog("[VmPeak] vmPeakMax=%d" % (vmPeakMax))
    tolog("[VmPeak] vmPeakMean=%d" % (vmPeakMean))
    tolog("[VmPeak] RSSMean=%d" % (RSSMean))

    return vmPeakMax, vmPeakMean, RSSMean def getSourceSetup(runCommand):
    """ Extract the source setup command from the run command """

    if type(runCommand) is dict:
            to_str = " ".join(runCommand['environment'])
            to_str = "%s %s %s %s" % (to_str,  runCommand["interpreter"], runCommand["payload"], runCommand["parameters"])
            runCommand = to_str

    setup = ""
    pattern = re.compile(r"(source /.+?;)")
    s = re.findall(pattern, runCommand)
    if s != []:
        setup = s[0]

    return setup
from Experiment import Experiment               # Main experiment class
from pUtil import tolog                         # Logging method that sends text to the pilot log
from pUtil import readpar                       # Used to read values from the schedconfig DB (queuedata)
from pUtil import isAnalysisJob                 # Is the current job a user analysis job or a production job?
from pUtil import grep                          # Grep function - reimplement using cli command
from pUtil import getCmtconfig                  # Get the cmtconfig from the job def or queuedata
from pUtil import verifyReleaseString           # To verify the release string (move to Experiment later)
from pUtil import timedCommand                  # Protect cmd with timed_command
from pUtil import getSiteInformation            # Get the SiteInformation object corresponding to the given experiment
from pUtil import isBuildJob                    # Is the current job a build job?
from pUtil import remove                        # Used to remove redundant file before log file creation
from pUtil import isAGreaterOrEqualToB          #
from pUtil import convert_unicode_string        # Needed to avoid unicode strings in the memory output text file
from PilotErrors import PilotErrors             # Error codes
from FileHandling import readFile, writeFile    # File handling methods
from FileHandling import updatePilotErrorReport # Used to set the priority of an error
from FileHandling import getJSONDictionary      # Used by getUtilityInfo()
from RunJobUtilities import dumpOutput          # ASCII dump
from RunJobUtilities import getStdoutFilename   #
from RunJobUtilities import findVmPeaks         #
from RunJobUtilities import getSourceSetup      #
from glob import glob
        from Singularity import singularityWrapper
        from RunJobUtilities import updateCopysetups
        from FileHandling import getNumberOfEvents
                from xml.dom import minidom
        from SiteMover import SiteMover
                from futil import check_syserr
                from futil import check_syserr
import re
import os
import time
import commands
        import fnmatch
                #    rbytes = -1
                #else:
                #    node['rbytes'] = rbytes
                #try:
                #    wbytes = summary_dictionary['Other']['wbytes']
                #except:
                #    wbytes = -1
                #else:
                #    node['wbytes'] = wbytes
            except Exception, e:
                tolog("!!WARNING!!54541! Exception caught while parsing memory monitor file: %s" % (e))
                tolog("!!WARNING!!5455!! Will add -1 values for the memory info")
                node['maxRSS'] = -1
                node['maxVMEM'] = -1
                node['maxSWAP'] = -1
                node['maxPSS'] = -1
                node['avgRSS'] = -1
                node['avgVMEM'] = -1
                node['avgSWAP'] = -1
                node['avgPSS'] = -1
                #node['rchar'] = -1
                #node['wchar'] = -1
                #node['rbytes'] = -1
                #node['wbytes'] = -1
            else:
                tolog("Extracted standard info from memory monitor json")
            try:
                node['totRCHAR'] = summary_dictionary['Max']['totRCHAR']
                node['totWCHAR'] = summary_dictionary['Max']['totWCHAR']
                node['totRBYTES'] = summary_dictionary['Max']['totRBYTES']
                node['totWBYTES'] = summary_dictionary['Max']['totWBYTES']
                node['rateRCHAR'] = summary_dictionary['Avg']['rateRCHAR']
                node['rateWCHAR'] = summary_dictionary['Avg']['rateWCHAR']
                node['rateRBYTES'] = summary_dictionary['Avg']['rateRBYTES']
                node['rateWBYTES'] = summary_dictionary['Avg']['rateWBYTES']
            except Exception, e:
                tolog("totRCHAR,totWCHAR,totRBYTES,totWBYTES,rateRCHAR,rateWCHAR,rateRBYTES,rateWBYTES were not found in memory monitor json (or json doesn't exist yet) - ignoring")
            else:
                tolog("totRCHAR,totWCHAR,totRBYTES,totWBYTES,rateRCHAR,rateWCHAR,rateRBYTES,rateWBYTES were extracted from memory monitor json")
        else:
            tolog("Memory summary dictionary not yet available")

        return node

    def getMaxUtilityValue(self, value, maxValue, totalValue):
        """ Return the max and total value (used by memory monitoring) """
        # Return an error code, 1, in case of value error

        ec = 0
        try:
            value_int = int(value)
        except Exception, e:
            tolog("!!WARNING!!4543!! Exception caught: %s" % (e))
            ec = 1
        else:
            totalValue += value_int
            if value_int > maxValue:
                maxValue = value_int

        return ec, maxValue, totalValue

    def getMemoryValues(self, workdir, pilot_initdir):
        """ Find the values in the utility output file """

        # In case the summary JSON file has not yet been produced, create a summary dictionary with the same format
        # using the output text file (produced by the memory monitor and which is updated once per minute)
        #
        # FORMAT:
        #   {"Max":{"maxVMEM":40058624,"maxPSS":10340177,"maxRSS":16342012,"maxSwap":16235568},
        #    "Avg":{"avgVMEM":19384236,"avgPSS":5023500,"avgRSS":6501489,"avgSwap":5964997},
        #    "Other":{"rchar":NN,"wchar":NN,"rbytes":NN,"wbytes":NN}}

        maxVMEM = -1
        maxRSS = -1
        maxPSS = -1
        maxSwap = -1
        totRCHAR = -1 # tot=max
        totWCHAR = -1
        totRBYTES = -1
        totWBYTES = -1
        avgVMEM = 0
        avgRSS = 0

        avgPSS = 0
        avgSwap = 0
        totalVMEM = 0
        totalRSS = 0
        totalPSS = 0
        totalSwap = 0
        totalRCHAR = 0
        totalWCHAR = 0
        totalRBYTES = 0
        totalWBYTES = 0
        N = 0
        summary_dictionary = {}

        rchar = None
        wchar = None
        rbytes = None
        wbytes = None

        # Get the path to the proper memory info file (priority ordered)
        path = self.getUtilityInfoPath(workdir, pilot_initdir, allowTxtFile=True)
        if os.path.exists(path):

            tolog("Using path: %s" % (path))

            # Does a JSON summary file exist? If so, there's no need to calculate maximums and averages in the pilot
            if path.lower().endswith('json'):
                # Read the dictionary from the JSON file
                summary_dictionary = getJSONDictionary(path)
            else:
                # Loop over the output file, line by line, and look for the maximum PSS value
                first = True
                with open(path) as f:
                    for line in f:
                        # Skip the first line
                        if first:
                            first = False
                            continue

                        line = convert_unicode_string(line)