class PilotErrors:
    """ Pilot error handling """

    # error codes
    ERR_UNKNOWNERROR = 0
    ERR_GENERALERROR = 1008
    ERR_DIRECTIOFILE = 1009 # harmless, just means that copy-to-scratch was skipped in favor or direct i/o access
    ERR_GETDATAEXC = 1097
    ERR_NOLOCALSPACE = 1098
    ERR_STAGEINFAILED = 1099
    ERR_REPNOTFOUND = 1100
    ERR_LRCREGCONNREF = 1101
    ERR_NOSUCHFILE = 1103
    ERR_USERDIRTOOLARGE = 1104
    ERR_LFCADDCSUMFAILED = 1105
    ERR_STDOUTTOOBIG = 1106
    ERR_MISSDBREL = 1107
    ERR_FAILEDLCGREG = 1108
    ERR_CMTCONFIG = 1109
    ERR_SETUPFAILURE = 1110
    ERR_RUNJOBEXC = 1111
    ERR_PILOTEXC = 1112
    ERR_GETLFCIMPORT = 1113
    ERR_PUTLFCIMPORT = 1114
    ERR_NFSSQLITE = 1115
    ERR_QUEUEDATA = 1116
    ERR_QUEUEDATANOTOK = 1117
    ERR_CURLSPACE = 1118
    ERR_DDMSPACE = 1119
    ERR_NOSTMATCHDEST = 1120 # not used
    ERR_NOLFCSFN = 1122
    ERR_MISSINGGUID = 1123
    ERR_OUTPUTFILETOOLARGE = 1124
    ERR_NOPFC = 1130
    ERR_PUTFUNCNOCALL = 1131
    ERR_LRCREG = 1132
    ERR_NOSTORAGE = 1133
    ERR_MKDIR = 1134
    ERR_FAILEDSIZELOCAL = 1135
    ERR_FAILEDMD5LOCAL = 1136
    ERR_STAGEOUTFAILED = 1137
    ERR_FAILEDSIZE = 1138
    ERR_PUTWRONGSIZE = 1139
    ERR_FAILEDMD5 = 1140
    ERR_PUTMD5MISMATCH = 1141
    ERR_CHMODTRF = 1143
    ERR_PANDAKILL = 1144
    ERR_GETMD5MISMATCH = 1145
    ERR_DYNTRFINST = 1146
    ERR_FAILEDRM = 1148
    ERR_TRFDOWNLOAD = 1149
    ERR_LOOPINGJOB = 1150
    ERR_GETTIMEOUT = 1151
    ERR_PUTTIMEOUT = 1152
    ERR_LOSTJOBNOTFINISHED = 1153
    ERR_LOSTJOBLOGREG = 1154
    ERR_LOSTJOBFILETRANSFER = 1155
    ERR_LOSTJOBRECOVERY = 1156
    ERR_LOSTJOBMAXEDOUT = 1158
    ERR_LOSTJOBPFC = 1159
    ERR_LRCREGSTRSIZE = 1160
    ERR_LOSTJOBXML = 1161
    ERR_LRCREGDUP = 1162
    ERR_NOPROXY = 1163
    ERR_MISSINGLOCALFILE = 1164
    ERR_MISSINGOUTPUTFILE = 1165
    ERR_SIGPIPE = 1166
    ERR_MISSFILEXML = 1167
    ERR_SIZETOOLARGE = 1168
    ERR_FAILEDLFCREG = 1169
    ERR_FAILEDADLOCAL = 1170
    ERR_GETADMISMATCH = 1171
    ERR_PUTADMISMATCH = 1172
    ERR_PANDAMOVERFILENOTCACHED = 1173
    ERR_PANDAMOVERTRANSFER = 1174
    ERR_GETWRONGSIZE = 1175
    ERR_NOCHILDPROCESSES = 1176
    ERR_NOVOMSPROXY = 1177
    ERR_NOSTAGEDFILES = 1178
    ERR_FAILEDLFCGETREPS = 1179
    ERR_GETGLOBUSSYSERR = 1180
    ERR_PUTGLOBUSSYSERR = 1181
    ERR_FAILEDLFCGETREP = 1182
    ERR_GUIDSEXISTSINLRC = 1183
    ERR_MISSINGPFC = 1184
    ERR_NOSOFTWAREDIR = 1186
    ERR_NOPAYLOADMETADATA = 1187
    ERR_LCGGETTURLS = 1188
    ERR_LCGGETTURLSTIMEOUT = 1189
    ERR_LFNTOOLONG = 1190
    ERR_ZEROFILESIZE = 1191
    ERR_DBRELNOTYETTRANSFERRED = 1192
    ERR_SEPROBLEM = 1193
    ERR_NOFILEVERIFICATION = 1194
    ERR_COMMANDTIMEOUT = 1195
    ERR_GETFAILEDTOMOUNTNFS4 = 1196
    ERR_GETPNFSSYSTEMERROR = 1197
    ERR_MKDIRWORKDIR = 1199
    ERR_KILLSIGNAL = 1200
    ERR_SIGTERM = 1201
    ERR_SIGQUIT = 1202
    ERR_SIGSEGV = 1203
    ERR_SIGXCPU = 1204
#    ERR_USERKILL = 1205 # not used by pilot
    ERR_SIGBUS = 1206
    ERR_SIGUSR1 = 1207
    ERR_NOPAYLOADOUTPUT = 1210
    ERR_MISSINGINSTALLATION = 1211
    ERR_PAYLOADOUTOFMEMORY = 1212
    ERR_REACHEDMAXTIME = 1213
    ERR_DAFSNOTALLOWED = 1214
    ERR_NOTCPCONNECTION = 1215
    ERR_NOPILOTTCPSERVER = 1216
    ERR_CORECOUNTMISMATCH = 1217
    ERR_RUNEVENTEXC = 1218
    ERR_UUIDGEN = 1219
    ERR_UNKNOWN = 1220
    ERR_FILEEXIST = 1221
    ERR_GETKEYPAIR = 1222
    ERR_BADALLOC = 1223
    ERR_ESRECOVERABLE = 1224
    ERR_ESMERGERECOVERABLE = 1225
    ERR_GLEXEC = 1226
    ERR_ESATHENAMPDIED = 1227
    ERR_ESFATAL = 1228
    ERR_TEFATAL = 1229
    ERR_TEBADURL = 1230
    ERR_TEINVALIDGUID = 1231
    ERR_TEWRONGGUID = 1232
    ERR_TEHOSTNAME = 1233
    ERR_EXECUTEDCLONEJOB = 1234
    ERR_PAYLOADEXCEEDMAXMEM = 1235
    ERR_FAILEDBYSERVER = 1236
    ERR_ESKILLEDBYSERVER = 1237
    ERR_NOEVENTS = 1238
    ERR_OVERSUBSCRIBEDEVENTS = 1239
    ERR_ESMESSAGESERVER = 1240
    ERR_ESOBJECTSTORESETUP = 1241
    ERR_CHKSUMNOTSUP = 1242
    ERR_ESPREFETCHERDIED = 1243
    ERR_NORELEASEFOUND = 1244
    ERR_TOOFEWEVENTS = 1245
    ERR_NOUSERTARBALL = 1246
    ERR_BADXML = 1247
    ERR_WRONGARCHITECTURE = 1248
    ERR_RUNGENFAILURE = 1249

    # internal error codes
    ERR_DDMREG = 1
    ERR_FILEONTAPE = 2

    pilotError = {
        ERR_UNKNOWNERROR : "",
        ERR_GENERALERROR : "General pilot error, consult batch log",
        ERR_GETDATAEXC : "Get function can not be called for staging input file",
        ERR_NOLOCALSPACE : "No space left on local disk",
        ERR_STAGEINFAILED : "Get error: Staging input file failed",
        ERR_REPNOTFOUND : "Get error: Replica not found",
        ERR_LRCREGCONNREF : "LRC registration error: Connection refused",
        # 1102 : "Expected output file does not exist", # not used, see ERR_MISSINGOUTPUTFILE below
        ERR_NOSUCHFILE : "No such file or directory",
        ERR_USERDIRTOOLARGE : "User work directory too large",
        ERR_LFCADDCSUMFAILED : "Put error: Failed to add file size and checksum to LFC",
        ERR_STDOUTTOOBIG : "Payload stdout file too big",
        ERR_MISSDBREL : "Get error: Missing DBRelease file",
        ERR_FAILEDLCGREG : "Put error: LCG registration failed",
        ERR_CMTCONFIG : "Required CMTCONFIG incompatible with WN",
        ERR_SETUPFAILURE : "Failed during setup",
        ERR_RUNJOBEXC : "Exception caught by RunJob*",
        ERR_PILOTEXC : "Exception caught by pilot",
        ERR_GETLFCIMPORT : "Get error: Failed to import LFC python module",
        ERR_PUTLFCIMPORT : "Put error: Failed to import LFC python module",
        ERR_NFSSQLITE : "NFS SQLite locking problems",
        ERR_QUEUEDATA : "Pilot could not download queuedata",
        ERR_QUEUEDATANOTOK : "Pilot found non-valid queuedata",
        ERR_CURLSPACE : "Pilot could not curl space report",
        ERR_DDMSPACE : "Pilot aborted due to DDM space shortage",
        ERR_NOSTMATCHDEST : "Space token descriptor does not match destination path",
        # 1121 : "Can not read the xml file for registering output files to dispatcher", # not used
        ERR_NOLFCSFN : "Bad replica entry returned by lfc_getreplicas(): SFN not set in LFC for this guid",
        ERR_MISSINGGUID : "Missing guid in output file list",
        ERR_OUTPUTFILETOOLARGE : "Output file too large",
        ERR_NOPFC : "Get error: Failed to get PoolFileCatalog",
        ERR_PUTFUNCNOCALL : "Put function can not be called for staging out",
        ERR_LRCREG : "LRC registration error (consult log file)",
        ERR_NOSTORAGE : "Put error: Fetching default storage URL failed",
        ERR_MKDIR : "Put error: Error in mkdir on localSE, not allowed or no available space",
        ERR_FAILEDSIZELOCAL : "Could not get file size in job workdir",
        ERR_FAILEDMD5LOCAL : "Error running md5sum on the file in job workdir",
        ERR_STAGEOUTFAILED : "Put error: Error in copying the file from job workdir to localSE",
        ERR_FAILEDSIZE : "Put error: could not get the file size on localSE",
        ERR_PUTWRONGSIZE : "Put error: Problem with copying from job workdir to local SE: size mismatch",
        ERR_FAILEDMD5 : "Put error: Error running md5sum on the file on local SE",
        ERR_PUTMD5MISMATCH : "Put error: Problem with copying from job workdir to local SE: md5sum mismatch",
        # 1142 : "Put error: failed to register the file on local SE", # not used
        ERR_CHMODTRF : "Failed to chmod trf",
        ERR_PANDAKILL : "This job was killed by panda server",
        ERR_GETMD5MISMATCH : "Get error: md5sum mismatch on input file",
        ERR_DYNTRFINST : "Trf installation dir does not exist and could not be installed",
        # 1147 : "Put error: dccp returned readOnly",  # not used
        ERR_FAILEDRM : "Put error: Failed to remove readOnly file in dCache",
        ERR_TRFDOWNLOAD : "wget command failed to download trf",
        ERR_LOOPINGJOB : "Looping job killed by pilot",
        ERR_GETTIMEOUT : "Get error: Input file staging timed out",
        ERR_PUTTIMEOUT : "Put error: File copy timed out",
        ERR_LOSTJOBNOTFINISHED : "Lost job was not finished",
        ERR_LOSTJOBLOGREG : "Failed to register log file",
        ERR_LOSTJOBFILETRANSFER : "Failed to move output files for lost job",
        ERR_LOSTJOBRECOVERY : "Pilot could not recover job",
        # 1157 : "Could not create log file", # not used
        ERR_LOSTJOBMAXEDOUT : "Reached maximum number of recovery attempts",
        ERR_LOSTJOBPFC : "Job recovery could not read PoolFileCatalog.xml file (guids lost)",
        ERR_LRCREGSTRSIZE : "LRC registration error: file name string size exceeded limit of 250",
        ERR_LOSTJOBXML : "Job recovery could not generate xml for remaining output files",
        ERR_LRCREGDUP : "LRC registration error: Non-unique LFN",
        ERR_NOPROXY : "Grid proxy not valid",
        ERR_MISSINGLOCALFILE : "Get error: Local input file missing",
        ERR_MISSINGOUTPUTFILE : "Put error: Local output file missing",
        ERR_SIGPIPE : "Put error: File copy broken by SIGPIPE",
        ERR_MISSFILEXML : "Get error: Input file missing in PoolFileCatalog.xml",
        ERR_SIZETOOLARGE : "Get error: Total file size too large",
        ERR_FAILEDLFCREG : "Put error: File registration failed",
        ERR_FAILEDADLOCAL : "Error running adler32 on the file in job workdir",
        ERR_GETADMISMATCH : "Get error: adler32 mismatch on input file",
        ERR_PUTADMISMATCH : "Put error: adler32 mismatch on output file",
        ERR_PANDAMOVERFILENOTCACHED : "PandaMover staging error: File is not cached",
        ERR_PANDAMOVERTRANSFER : "PandaMover transfer failure",
        ERR_GETWRONGSIZE : "Get error: Problem with copying from local SE to job workdir: size mismatch",
        ERR_NOCHILDPROCESSES : "Pilot has no child processes (job wrapper has either crashed or did not send final status)",
        ERR_NOVOMSPROXY : "Voms proxy not valid",
        ERR_NOSTAGEDFILES : "Get error: No input files are staged",
        ERR_FAILEDLFCGETREPS : "Get error: Failed to get replicas",
        ERR_GETGLOBUSSYSERR : "Get error: Globus system error",
        ERR_PUTGLOBUSSYSERR : "Put error: Globus system error",
        ERR_FAILEDLFCGETREP : "Get error: Failed to get replica",
        ERR_GUIDSEXISTSINLRC : "LRC registration error: Guid-metadata entry already exists",
        ERR_MISSINGPFC : "Put error: PoolFileCatalog could not be found in workdir",
        # 1185 : "Put error: Error running adler32 on the file in job workdir", # not used
        ERR_NOSOFTWAREDIR : "Software directory does not exist",
        ERR_NOPAYLOADMETADATA : "Payload metadata is not available",
        ERR_LCGGETTURLS : "lcg-getturls failed",
        ERR_LCGGETTURLSTIMEOUT : "lcg-getturls was timed-out",
        ERR_LFNTOOLONG : "LFN too long (exceeding limit of 150 characters)",
        ERR_ZEROFILESIZE : "Illegal zero file size",
        ERR_DBRELNOTYETTRANSFERRED : "DBRelease file has not been transferred yet",
        ERR_NOFILEVERIFICATION : "File verification failed",
        ERR_COMMANDTIMEOUT : "Command timed out",
        ERR_GETFAILEDTOMOUNTNFS4 : "Get error: Failed to mount NSF4",
        ERR_GETPNFSSYSTEMERROR : "Get error: PNFS system error",
        # 1198 : "Can not check the child process status from the heartbeat process", # not used
        ERR_MKDIRWORKDIR : "Could not create directory",
        ERR_KILLSIGNAL : "Job terminated by unknown kill signal",
        ERR_SIGTERM : "Job killed by signal: SIGTERM",
        ERR_SIGQUIT : "Job killed by signal: SIGQUIT",
        ERR_SIGSEGV : "Job killed by signal: SIGSEGV",
        ERR_SIGXCPU : "Job killed by signal: SIGXCPU",
        ERR_SIGUSR1 : "Job killed by signal: SIGUSR1",
        ERR_SIGBUS : "Job killed by signal: SIGBUS",
        ERR_NOPAYLOADOUTPUT : "No payload output",
        ERR_MISSINGINSTALLATION : "Missing installation",
        ERR_PAYLOADOUTOFMEMORY : "Payload ran out of memory",
        ERR_REACHEDMAXTIME : "Reached batch system time limit",
        ERR_DAFSNOTALLOWED : "Site does not allow requested direct access or file stager",
        ERR_NOTCPCONNECTION : "Failed to open TCP connection to localhost (worker node network problem)",
        ERR_NOPILOTTCPSERVER : "Pilot TCP server has died",
        ERR_CORECOUNTMISMATCH : "Mismatch between core count in job and queue definition",
        ERR_RUNEVENTEXC : "Exception caught by runEvent",
        ERR_UUIDGEN : "uuidgen failed to produce a guid",
        ERR_UNKNOWN : "Job failed due to unknown reason (consult log file)",
        ERR_FILEEXIST : "File already exist",
        ERR_GETKEYPAIR : "Failed to get security key pair",
        ERR_BADALLOC : "TRF failed due to bad_alloc",
        ERR_ESMERGERECOVERABLE : "Recoverable Event Service Merge error",
        ERR_ESRECOVERABLE: "Recoverable Event Service error",
        ERR_GLEXEC: "gLExec related error",
        ERR_ESATHENAMPDIED: "AthenaMP ended Event Service job prematurely",
        ERR_ESFATAL: "Fatal Event Service error",
        ERR_TEFATAL: "Fatal Token Extractor error",
        ERR_TEHOSTNAME: "Token Extractor error: Host name could not be resolved",
        ERR_TEBADURL: "Token Extractor error: Bad URL",
        ERR_TEINVALIDGUID: "Token Extractor error: Invalid GUID length",
        ERR_TEWRONGGUID: "Token Extractor error: No tokens for this GUID",
        ERR_EXECUTEDCLONEJOB: "Already executed clone job",
        ERR_PAYLOADEXCEEDMAXMEM: "Payload exceeded maximum allowed memory",
        ERR_FAILEDBYSERVER: "Failed by server",
        ERR_ESKILLEDBYSERVER: "Event Service job killed by server",
        ERR_NOEVENTS: "Event Service no available events",
        ERR_OVERSUBSCRIBEDEVENTS: "Event Service over subscribed events",
        ERR_ESMESSAGESERVER: "Event service message server error",
        ERR_ESOBJECTSTORESETUP: "Event service objectstore setup",
        ERR_CHKSUMNOTSUP: "Mover error: query checksum is not supported",
        ERR_ESPREFETCHERDIED: "Prefetcher ended Event Service job prematurely",
        ERR_NORELEASEFOUND: "No release candidates found",
        ERR_TOOFEWEVENTS: "Too few events, less events than minimal requirement",
        ERR_NOUSERTARBALL: "User tarball cannot be downloaded from PanDA server",
        ERR_BADXML: "Badly formed XML",
        ERR_WRONGARCHITECTURE: "Job built on wrong architecture",
        ERR_RUNGENFAILURE: "RunGen failure (consult log file)",

    }

    getErrorCodes = [1097, 1099, 1100, 1103, 1107, 1113, 1130, 1145, 1151, 1164, 1167, 1168, 1171, 1175, 1178, 1179, 1180, 1182]
    putErrorCodes = [1101, 1114, 1122, 1131, 1132, 1133, 1134, 1135, 1136, 1137, 1138, 1140, 1141, 1152, 1154, 1155, 1181]
    recoverableErrorCodes = [0] + putErrorCodes

    # Error codes that will issue a Pilot-controlled resubmission
    PilotResubmissionErrorCodes = [1008, 1098, 1099, 1110, 1113, 1114, 1115, 1116, 1117, 1137, 1139, 1151, 1152, 1171, 1172, 1177, 1179, 1180, 1181, 1182, 1188, 1189, 1195, 1196, 1197, 1219]

    # Error codes used with FAX fail-over (only an error code in this list will allow FAX fail-over)
    PilotFAXErrorCodes = [1103] + PilotResubmissionErrorCodes

    def getPilotErrorDiag(self, code=0):
        """ Return text corresponding to error code """

        pilotErrorDiag = ""
        if code in self.pilotError.keys():
            pilotErrorDiag = self.pilotError[code]
        else:
            pilotErrorDiag = "Unknown pilot error code"
        return pilotErrorDiag

    def isGetErrorCode(self, code=0):
        """ Determine whether code is in the put error list or not """

        state = False
        if code in self.getErrorCodes:
            state = True

        return state

    def isPutErrorCode(self, code=0):
        """ Determine whether code is in the put error list or not """

        state = False
        if code in self.putErrorCodes:
            state = True

        return state

    @classmethod
    def isRecoverableErrorCode(self, code=0):
        """ Determine whether code is a recoverable error code or not """

        return code in self.recoverableErrorCodes

    def isPilotResubmissionErrorCode(self, code=0):
        """ Determine whether code issues a Pilot-controlled resubmission """

        state = False
        if code in self.PilotResubmissionErrorCodes:
            state = True

        return state

    def isPilotFAXErrorCode(self, code=0):
        """ Determine whether code allows for a FAX fail-over """

        state = False
        if code in self.PilotFAXErrorCodes:
            state = True

        return state

    @classmethod
    def getErrorStr(self, code):
        """
        Avoids exception if an error is not in the dictionary.
        An empty string is returned if the error is not in the dictionary.
        """
        return self.pilotError.get(code, '')

    def getErrorName(self, code):
        """ From the error code to get the error name"""
        for k in self.__class__.__dict__.keys():
            if self.__class__.__dict__[k] == code:
                return k
        return None def tolog(msg, tofile=True, label='INFO', essential=False):
    """ Write date+msg to pilot log and to stdout """

    try:
        import inspect

        MAXLENGTH = 12
        # getting the name of the module that is invoking tolog() and adjust the length
        try:
            module_name = os.path.basename(inspect.stack()[1][1])
        except Exception, e:
            module_name = "unknown"
            #print "Exception caught by tolog(): ", e,
        module_name_cut = module_name[0:MAXLENGTH].ljust(MAXLENGTH)
        msg = "%i|%s| %s" % (os.getpid(),module_name_cut, msg)

        t = timeStampUTC(format='%Y-%m-%d %H:%M:%S')
        if tofile:
            appendToLog("%s|%s\n" % (t, msg))

        # remove backquotes from the msg since they cause problems with batch submission of pilot
        # (might be present in error messages from the OS)
        msg = msg.replace("`","'")
        msg = msg.replace('"','\\"')
        print "%s| %s" % (t, msg)

        # write any FAILED messages to stderr
        if "!!FAILED!!" in msg:
            try:
                print >> sys.stderr, "%s| %s" % (t, msg)
            except:
                print "Failed to print to sys.stderr: %s" % (t, msg)
    except:
        print "!!WARNING!!4000!! %s" % traceback.format_exc() def readpar(parameter, alt=False, version=0, queuename=None):
    """ Read 'parameter' from queuedata via SiteInformation class """

    from SiteInformation import SiteInformation
    si = SiteInformation()

    return si.readpar(parameter, alt=alt, version=version, queuename=queuename) def timeStamp():
    """ return ISO-8601 compliant date/time format """

    tmptz = time.timezone
    if tmptz > 0:
        signstr = '-'
    else:
        signstr = '+'
    tmptz_hours = int(tmptz/3600)

    return str("%s%s%02d%02d" % (time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime()), signstr, tmptz_hours, int(tmptz/60-tmptz_hours*60))) def getBatchSystemJobID():
    """ return the batch system job id (will be reported to the server) """

    # BQS (e.g. LYON)
    if os.environ.has_key("QSUB_REQNAME"):
        return "BQS", os.environ["QSUB_REQNAME"]
    # BQS alternative
    if os.environ.has_key("BQSCLUSTER"):
        return "BQS", os.environ["BQSCLUSTER"]
    # Torque
    if os.environ.has_key("PBS_JOBID"):
        return "Torque", os.environ["PBS_JOBID"]
    # LSF
    if os.environ.has_key("LSB_JOBID"):
        return "LSF", os.environ["LSB_JOBID"]
    # Sun's Grid Engine
    if os.environ.has_key("JOB_ID"):
        return "Grid Engine", os.environ["JOB_ID"]
    # Condor (variable sent through job submit file)
    if os.environ.has_key("clusterid"):
        return "Condor", os.environ["clusterid"]
    # Condor (get jobid from classad file)
    if os.environ.has_key("_CONDOR_JOB_AD"):
        return "Condor", commands.getoutput('sed -n "s/GlobalJobId.*\\"\\(.*\\)\\".*/\\1/p" %s' % os.environ["_CONDOR_JOB_AD"])
    # SLURM
    if os.environ.has_key("SLURM_JOB_ID"):
        return "SLURM", os.environ["SLURM_JOB_ID"]

#    # Condor (id unknown)
#    if os.environ.has_key("_CONDOR_SCRATCH_DIR"):
#        return "Condor", "(unknown clusterid)"

    return None, "" def getCPUmodel():
    """ Get cpu model and cache size from /proc/cpuinfo """
    # model name      : Intel(R) Xeon(TM) CPU 2.40GHz
    # cache size      : 512 KB
    # gives the return string "Intel(R) Xeon(TM) CPU 2.40GHz 512 KB"

    cpumodel = ''
    cpucache = ''
    modelstring = ''
    try:
        f = open('/proc/cpuinfo', 'r')
    except Exception, e:
        tolog("Could not open /proc/cpuinfo: %s" % e)
    else:
        re_model = re.compile('^model name\s+:\s+(\w.+)')
        re_cache = re.compile('^cache size\s+:\s+(\d+ KB)')

        # loop over all lines in cpuinfo
        for line in f.readlines():
            # try to grab cpumodel from current line
            model = re_model.search(line)
            if model:
                # found cpu model
                cpumodel = model.group(1)

            # try to grab cache size from current line
            cache = re_cache.search(line)
            if cache:
                # found cache size
                cpucache = cache.group(1)

            # stop after 1st pair found - can be multiple cpus
            if cpumodel and cpucache:
                # create return string
                modelstring = cpumodel + " " + cpucache
                break

        f.close()

    # default return string if no info was found
    if not modelstring:
        modelstring = "UNKNOWN"

    return modelstring def PFCxml(experiment, fname, fnlist=[], fguids=[], fntag=None, alog=None, alogguid=None, fsize=[], checksum=[], analJob=False, jr=False, additionalOutputFile=None, additionalOutputFileGuid=None, logToOS=False, archive=None):
    """ Create a PFC style XML file """

    # fnlist = output file list
    # fguids = output file guid list
    # fntag = pfn/lfn identifier
    # alog = name of log file
    # alogguid = guid of log file
    # fsize = file size list
    # checksum = checksum list
    # analJob = analysis job
    # jr = job recovery mode, guid generation by pilot not allowed

    # fntag = lfn is used for the metadata-<jobId>.xml that is sent to the server
    # fntag = pfn is used for OutPutFileCatalog.xml that is used by the mover for the stage out
    # The SURL will be added to the metadata file for fntag = lfn to allow for server side LFC registration

    status = True
    flist = []
    glist = []
    from SiteMover import SiteMover

    # get the experiment object
    thisExperiment = getExperiment(experiment)

    # for metadata.xml prepare the file for potential guid grabbing
    if "metadata" in fname and None in fguids:
        metadata_filename = prepareMetadata(fname + ".PAYLOAD")
    else:
        metadata_filename = fname

    # add log file
    if alog:
        flist.append(alog)
        if not alogguid:
            if not jr:
                alogguid = getGUID()
                tolog("Generated log guid: %s" % (alogguid))
            else:
                tolog("!!WARNING!!2999!! Log guid generation not allowed in recovery mode")
                alogguid = ''
                status = False
        glist.append(alogguid)

    # add additional output files (only for CERNVM, not NG or any other sites)
    if additionalOutputFile:
        flist.append(additionalOutputFile)
        if not additionalOutputFileGuid:
            additionalOutputFileGuid = getGUID()
        glist.append(additionalOutputFileGuid)

    if fnlist:
        flist = flist + fnlist
        tolog("fnlist = %s" % str(fnlist))
        tolog("fguids = %s" % str(fguids))
        for i in range(0, len(fnlist)):
            # check for guid
            try:
                _dummy = fguids[i]
                del _dummy
            except IndexError, e:
                guid = findGuid(analJob, metadata_filename, fnlist[i])
                if guid and guid != "":
                    tolog("Found guid for file: %s (%s)" % (fnlist[i], guid))
                else:
                    if not jr:
                        guid = getGUID()
                        tolog("Generated guid for file (%d): %s (%s)" % (i, fnlist[i], guid))
                    else:
                        tolog("!!WARNING!!2999!! Guid generation not allowed in recovery mode (file: %s)" % (fnlist[i]))
                        guid = ''
                        status = False
                fguids.insert(i, guid)
            else:
                if not fguids[i]:
                    guid = findGuid(analJob, metadata_filename, fnlist[i])
                    if guid and guid != "":
                        tolog("Found guid for file: %s (%s)" % (fnlist[i], guid))
                    else:
                        if not jr:
                            guid = getGUID()
                            tolog("Generated guid for file (%d): %s (%s)" % (i, fnlist[i], guid))
                        else:
                            tolog("!!WARNING!!2999!! Guid generation not allowed in recovery mode (file: %s)" % (fnlist[i]))
                            guid = ''
                            status = False
                    try:
                        fguids[i] = guid
                    except:
                        fguids.insert(i, guid)

            if fntag == "lfn":
                # check for file size
                try:
                    _dummy = fsize[i]
                    del _dummy
                except IndexError, e:
                    #print "This item doesn't exist"
                    fsize.insert(i, "")

                # check for checksum
                try:
                    _dummy = checksum[i]
                    del _dummy
                except IndexError, e:
                    #print "This item doesn't exist"
                    checksum.insert(i, "")

        glist = glist + fguids

    if fntag == "pfn":
        #create the PoolFileCatalog.xml-like file in the workdir
        fd = open(fname, "w")
        fd.write('<?xml version="1.0" encoding="UTF-8" standalone="no" ?>\n')
        fd.write("<!-- Edited By POOL -->\n")
        fd.write('<!DOCTYPE POOLFILECATALOG SYSTEM "InMemory">\n')
        fd.write("<POOLFILECATALOG>\n")
        for i in range(0, len(flist)): # there's only one file in flist if it is for the object store
            fd.write('  <File ID="%s">\n' % (glist[i]))
            fd.write("    <physical>\n")
            fd.write('      <pfn filetype="ROOT_All" name="%s"/>\n' % (flist[i]))
            fd.write("    </physical>\n")
            fd.write("  </File>\n")
        fd.write("</POOLFILECATALOG>\n")
        fd.close()
    elif fntag == "lfn":
        # create the metadata.xml-like file that's needed by dispatcher jobs
        fd=open(fname, "w")
        fd.write('<?xml version="1.0" encoding="UTF-8" standalone="no" ?>\n')
        fd.write("<!-- ATLAS file meta-data catalog -->\n")
        fd.write('<!DOCTYPE POOLFILECATALOG SYSTEM "InMemory">\n')
        fd.write("<POOLFILECATALOG>\n")
        for i in range(0, len(flist)):
            fd.write('  <File ID="%s">\n' % (glist[i]))
            fd.write("    <logical>\n")
            fd.write('      <lfn name="%s"/>\n' % (flist[i]))
            fd.write("    </logical>\n")

            # if the log is to be transferred to an OS, add an endpoint tag
            if logToOS and alog == flist[i]:
                fd.write('    <endpoint>%s-ddmendpoint_tobeset</endpoint>\n' % (alog))

            # add SURL metadata (not known yet) for server LFC registration
            # use the GUID as identifier (the string "<GUID>-surltobeset" will later be replaced with the SURL)
            if thisExperiment:
                special_xml = thisExperiment.getMetadataForRegistration(glist[i])
                if special_xml != "":
                    fd.write(special_xml)

            # add log file metadata later (not known yet)
            if flist[i] == alog:
                fd.write('    <metadata att_name="fsize" att_value=""/>\n')
                fd.write('    <metadata att_name="csumtypetobeset" att_value=""/>\n')
            elif (additionalOutputFile and flist[i] == additionalOutputFile):
                if ".xml" in additionalOutputFile:
                    fd.write('    <metadata att_name="fsizeXML" att_value=""/>\n')
                    fd.write('    <metadata att_name="csumtypetobesetXML" att_value=""/>\n')
                else:
                    fd.write('    <metadata att_name="fsizeAdditional" att_value=""/>\n')
                    fd.write('    <metadata att_name="csumtypetobesetAdditional" att_value=""/>\n')
            else:
                if len(fsize) != 0:
                    fd.write('    <metadata att_name="fsize" att_value="%s"/>\n' % (fsize[i]))
                if len(checksum) != 0:
                    fd.write('    <metadata att_name="%s" att_value="%s"/>\n' %\
                             (SiteMover.getChecksumType(checksum[i]), checksum[i]))
            fd.write("  </File>\n")
        fd.write("</POOLFILECATALOG>\n")
        fd.close()
    else:
        tolog("!!WARNING!!1234!! fntag is neither lfn nor pfn, did not manage to create the XML file for output files")

    # dump the file to the log
    dumpFile(fname, topilotlog=True)

    return status def updateMetadata(fname, fsize, checksum, format=None, fsizeXML=None, checksumXML=None, fsizeAdditional=None, checksumAdditional=None):
    """
    Add the fsize and checksum values for the log (left empty until this point)
    Return exit code and xml
    If format = NG, then the NorduGrid format of the metadata will be assumed
    fsizeXML and checksumXML are extra attributes needed for CERNVM xml file handling
    """
    ec = 0
    lines = ""

    try:
        f = open(fname, 'r')
    except Exception, e:
        tolog("Failed to open metadata file: %s" % e)
        ec = -1
    else:
        if format == 'NG':
            metadata1 = '<size></size>'
            new_metadata1 = '<size>%s</size>' % (fsize)
        else:
            metadata1 = '<metadata att_name="fsize" att_value=""/>'
            new_metadata1 = '<metadata att_name="fsize" att_value="%s"/>' % (fsize)

        # find out if checksum or adler32 should be added
        from SiteMover import SiteMover
        csumtype = SiteMover.getChecksumType(checksum)

        # special handling for CERNVM metadata
        if checksumXML:
            metadata4 = '<metadata att_name="csumtypetobesetXML" att_value=""/>'
        else:
            metadata4 = 'notused'
        if fsizeXML:
            metadata5 = '<metadata att_name="fsizeXML" att_value=""/>'
            new_metadata5 = '<metadata att_name="fsize" att_value="%s"/>' % (fsizeXML)
        else:
            metadata5 = 'notused'
            new_metadata5 = ''
        if checksumAdditional:
            metadata6 = '<metadata att_name="csumtypetobesetXML" att_value=""/>'
            if csumtype == "adler32":
                new_metadata6 = '<ad32>%s</ad32>' % (checksumAdditional)
            else:
                new_metadata6 = '<md5sum>%s</md5sum>' % (checksumAdditional)
        else:
            metadata6 = 'notused'
            new_metadata6 = ''
        if fsizeAdditional:
            metadata7 = '<metadata att_name="fsizeAdditional" att_value=""/>'
            new_metadata7 = '<metadata att_name="fsize" att_value="%s"/>' % (fsizeAdditional)
        else:
            metadata7 = 'notused'
            new_metadata7 = ''

        # for NG and CERNVM
        if format == 'NG':
            if csumtype == "adler32":
                metadata2 = '<ad32></ad32>'
                new_metadata2 = '<ad32>%s</ad32>' % (checksum)
            else:
                metadata2 = '<md5sum></md5sum>'
                new_metadata2 = '<md5sum>%s</md5sum>' % (checksum)
        else:
            if csumtype == "adler32":
                metadata2 = '<metadata att_name="adler32" att_value=""/>'
                new_metadata2 = '<metadata att_name="adler32" att_value="%s"/>' % (checksum)
            else:
                metadata2 = '<metadata att_name="md5sum" att_value=""/>'
                new_metadata2 = '<metadata att_name="md5sum" att_value="%s"/>' % (checksum)
        metadata3 = '<metadata att_name="csumtypetobeset" att_value=""/>'

        for line in f.readlines():
            newline = ""
            if line.find(metadata1) != -1:
                newline = line.replace(metadata1, new_metadata1)
                lines += newline
            elif line.find(metadata2) != -1:
                newline = line.replace(metadata2, new_metadata2)
                lines += newline
            elif line.find(metadata3) != -1:
                newline = line.replace(metadata3, new_metadata2)
                lines += newline
            elif line.find(metadata4) != -1:
                newline = line.replace(metadata4, new_metadata2)
                lines += newline
            elif line.find(metadata5) != -1:
                newline = line.replace(metadata5, new_metadata5)
                lines += newline
            elif line.find(metadata6) != -1:
                newline = line.replace(metadata6, new_metadata6)
                lines += newline
            elif line.find(metadata7) != -1:
                newline = line.replace(metadata7, new_metadata7)
                lines += newline
            elif line.find('csumtypetobeset') != -1:
                newline = line.replace()
            else:
                lines += line

        f.close()
        try:
            f = open(fname, 'w')
            f.write(lines)
            f.close()
        except Exception, e:
            tolog("Failed to write new metadata for log: %s" % e)
            ec = -1

    return ec, lines def addSkippedToPFC(fname, skippedfname):
    """ add skipped input file info to metadata.xml """

    ec = 0
    try:
        fd = open(skippedfname, "r")
    except Exception, e:
        tolog("!!WARNING!!2999!! %s" % e)
        ec = -1
    else:
        skippedXML = fd.read()
        fd.close()
        try:
            fdPFC = open(fname, "r")
        except Exception, e:
            tolog("!!WARNING!!2999!! %s" % e)
            ec = -1
        else:
            PFCXML = fdPFC.read()
            fdPFC.close()

    if ec == 0:
        # add the skipped file info to the end of the PFC
        PFCXML = PFCXML.replace("</POOLFILECATALOG>", skippedXML)
        PFCXML += "</POOLFILECATALOG>\n"

        # move the old PFC and create a new PFC
        try:
            os.system("mv %s %s.BAK2" % (fname, fname))
        except Exception, e:
            tolog("!!WARNING!!2999!! %s" % e)
            ec = -1
        else:
            try:
                fdNEW = open(fname, "w")
            except Exception, e:
                tolog("!!WARNING!!2999!! %s" % e)
                ec = -1
            else:
                fdNEW.write(PFCXML)
                fdNEW.close()
                tolog("Wrote updated XML with skipped file info:\n%s" % (PFCXML))
    return ec def makeHTTPUpdate(state, node, port, url='pandaserver.cern.ch', path=None):
    """ make http connection to jobdispatcher """

    if state == 'finished' or state == 'failed' or state == 'holding':
        tolog("Preparing for final Panda server update")
        trial = 1
        max_trials = 10
        delay = 2*60 # seconds
        tolog("Max number of trials: %d, separated by delays of %d seconds" % (max_trials, delay))
    else:
        # standard non final update
        trial = 1
        max_trials = 1
        delay = None

    # make http connection to jobdispatcher
    while trial <= max_trials:
        # draw a random server URL
        _url = '%s:%s/server/panda' % (url, port)
        tolog("HTTP connect using server: %s" % (_url))
        ret = httpConnect(node, _url, path=path)
        if ret[0] and trial == max_trials: # non-zero exit code
            if delay: # final update
                tolog("!!FAILED!!4000!! [Trial %d/%d] Could not update Panda server (putting job in holding state if possible)" %\
                      (trial, max_trials))
                # state change will take place in postJobTask
                # (the internal pilot state will not be holding but lostheartbeat)
            else:
                tolog("!!WARNING!!4000!! [Trial %d/%d] Could not update Panda server, EC = %d" %\
                      (trial, max_trials, ret[0]))
            break
        elif ret[0]: # non-zero exit code
            tolog("!!WARNING!!4000!! [Trial %d/%d] Could not update Panda server, EC = %d" %\
                  (trial, max_trials, ret[0]))
            if delay: # final update
                tolog("Can not miss the final update. Will take a nap for %d seconds and then try again.." % (delay))
                trial += 1
                time.sleep(delay)
            else: # try again later
                tolog("Panda server update postponed..")
                break
        else:
            break
    return ret def tailPilotErrorDiag(pilotErrorDiag, size=256):
    """ Return the last n characters of pilotErrorDiag """

    try:
        return pilotErrorDiag[-size:]
    except:
        return pilotErrorDiag def isLogfileCopied(workdir, jobId=None):
    """ check whether the log file has been copied or not """

    if jobId:
        if os.path.exists(workdir + '/LOGFILECOPIED_%s' % jobId):
            return True
        else:
            return False
    else:
        if os.path.exists(workdir + '/LOGFILECOPIED'):
            return True
        else:
            return False def updateJobState(job, site, workNode, recoveryAttempt=0):
    """ update the job state file """

    status = True

    # create a job state object and give it the current job state information
    from JobState import JobState
    JS = JobState()
    if JS.put(job, site, workNode, recoveryAttempt):
        if recoveryAttempt > 0:
            tolog("Successfully updated job state file (recovery attempt number: %d)" % (recoveryAttempt))
        else:
            tolog("Successfully updated job state file at: %s" % (JS.getCurrentFilename()))
    else:
        tolog("!!WARNING!!1000!! Failed to update job state file")
        status = False

    return status def updateXMLWithSURLs(experiment, node_xml, workDir, jobId, jobrec, format=''):
    """ update the XML with the SURLs """

    xml = ""

    # read back the SURL dictionary
    from SiteMover import SiteMover
    sitemover = SiteMover()
    surlDictionary = sitemover.getSURLDictionary(workDir, jobId)

    # get the experiment object
    thisExperiment = getExperiment(experiment)

    tolog("node_xml = %s" % (node_xml))

    node_xml_list = node_xml.split("\n")
    # loop over the xml and update where it is needed
    if surlDictionary != {}:
        if format == 'NG':
            re_tobeset = re.compile('\<surl\>([a-zA-Z0-9-]+)\-surltobeset')
        else:
            if thisExperiment:
                metadata_attr_name = thisExperiment.getAttrForRegistration()
            else:
                metadata_attr_name = "surl"
            re_tobeset = re.compile('\<metadata att\_name\=\"%s\" att\_value\=\"([a-zA-Z0-9-]+)\-surltobeset\"\/\>' % (metadata_attr_name))
        for line in node_xml_list:
            tobeset = re_tobeset.search(line)
            if tobeset:
                # extract the guid and surl
                guid = tobeset.group(1)
                # note: in case of an earlier transfer error, the corresponding guid will not be in the surl dictionary
                # since it is only written to the surl dictionary for successful transfers
                try:
                    surl = surlDictionary[guid]
                except Exception, e:
                    tolog("!!WARNING!!2996!! Encountered a missing guid in the surl dictionary - did the corresponding transfer fail? guid = %s, %s" % (guid, e))
                    # add the 'surltobeset' line when job recovery is used
                    if jobrec:
                        xml += line + "\n"
                else:
                    # replace the guid and the "surltobeset"-string with the surl
                    if surl and surl != "":
                        xml += line.replace(guid + "-surltobeset", surl) + "\n"
                    else:
                        tolog("!!WARNING!!2996!! Could not extract guid %s from xml line: %s" % (guid, line))
            # fail safe in case something went wrong above, remove the guid+surltobeset line
            elif "surltobeset" in line:
                tolog("Failed to remove surltobeset from line: %s" % (line))
            else:
                xml += line + "\n"

    else:
        tolog("!!WARNING!!2997!! Encountered an empty SURL dictionary")

        # remove the metadata for the SURL since it cannot be updated
        for line in node_xml_list:
            if not jobrec:
                if not "surltobeset" in line:
                    xml += line + "\n"
            else:
                xml += line + "\n"

    if xml == "\n":
        xml = ""
        tolog("Reset XML")

    return xml def getMetadata(workdir, jobId, athena=False, altpath=""):
    """ read metadata from file """

    strXML = None

    if altpath == "":
        BAK = ""
        if athena:
            BAK = ".PAYLOAD"

        # are we in recovery mode? then jobId is set
        if jobId:
            filename = "metadata-%s.xml%s" % (jobId, BAK)
        else:
            filename = "metadata.xml%s" % (BAK)

        fname = os.path.join(workdir, filename)
    else:
        fname = altpath

    tolog("Trying to read metadata from file: %s" % (fname))
    if os.path.exists(fname):
        try:
            f = open(fname)
        except Exception, e:
            tolog("!!WARNING!!1000!! Can not open the file %s, %s" % (fname, e))
        else:
            strXML = ""
            for line in f:
                strXML += line
            f.close()
            if len(strXML) > 0:
                tolog("Found metadata")
            else:
                tolog("!!WARNING!!1000!! Empty metadata")
    else:
        tolog("getMetadata: metadata does not seem to have been created (file %s does not exist)" % (fname))

    return strXML def toPandaLogger(data):
    try:
        tpre = datetime.datetime.utcnow()
    except:
        pass
    tolog("toPandaLogger: len(data) = %d" % len(data))
    tolog("data = %s" % str(data))
    response = None

    # instantiate curl
    curl = _Curl()
    url = 'http://pandamon.cern.ch/system/loghandler'

    curlstat, response = curl.get(url, data, os.getcwd())

    try:
        tpost = datetime.datetime.utcnow()
        tolog("Elapsed seconds: %d" % ((tpost-tpre).seconds))
    except:
        pass
    try:
        if curlstat == 0:
            # parse response message
            outtxt = response.lower()
            if outtxt.find('<html>') > 0:
                if outtxt.find('read timeout') > 0:
                    tolog("!!WARNING!!2999!! Timeout on dispatcher exchange")
                else:
                    tolog("!!WARNING!!2999!! HTTP error on dispatcher exchange")
                tolog("HTTP output: %s" % (response))
                return EC_Failed, None, None

            # create the parameter list from the dispatcher response
            data, response = parseDispatcherResponse(response)

            #status = int(data['StatusCode'])
            #if status != 0:
            #    # pilotErrorDiag = getDispatcherErrorDiag(status)
            #    tolog("Dumping %s/curl.config file:" % (path))
            #    dumpFile('%s/curl.config' % (path), topilotlog=True)
        else:
            tolog("!!WARNING!!2999!! Dispatcher message curl error: %d " % (curlstat))
            tolog("Response = %s" % (response))
            tolog("Dumping curl.config file:")
            dumpFile('%s/curl.config' % (path), topilotlog=True)
            return curlstat, None, None

        return 0, data, response

    except:
        _type, value, _traceback = sys.exc_info()
        tolog("ERROR : %s %s" % ( _type, traceback.format_exc()))
        return EC_Failed, None, None def getSiteInformation(experiment):
    """ Return a reference to a site information class """

    # The SiteInformationFactory ensures that the returned object is a Singleton
    # Usage:
    #    _exp = getSiteInformation(readpar('experiment')) # or from pilot option
    #    if _exp:
    #        _exp.somemethod("Hello")
    #    else:
    #        tolog("!!WARNING!!1111!! Failed to instantiate experiment class")

    from SiteInformationFactory import SiteInformationFactory
    factory = SiteInformationFactory()
    _exp = None

    try:
        siteInformationClass = factory.newSiteInformation(experiment)
    except Exception, e:
        tolog("!!WARNING!!1114!! SiteInformation factory threw an exception: %s" % (e))
    else:
        _exp = siteInformationClass()
        tolog("getSiteInformation: got experiment=%s" % (_exp.getExperiment()))

    return _exp def getExperiment(experiment):
    """ Return a reference to an experiment class """

    from ExperimentFactory import ExperimentFactory
    factory = ExperimentFactory()
    _exp = None

    try:
        experimentClass = factory.newExperiment(experiment)
    except Exception, e:
        tolog("!!WARNING!!1114!! Experiment factory threw an exception: %s" % (e))
    else:
        _exp = experimentClass()

    return _exp def readStringFromFile(filename):
    """ read exit code from file <workdir>/EXITCODE """

    s = ""
    if os.path.exists(filename):
        try:
            f = open(filename, "r")
        except Exception, e:
            tolog("Failed to open %s: %s" % (filename, e))
        else:
            s = f.read()
            tolog("Found string %s in file %s" % (s, filename))
            f.close()
    else:
        tolog("No string to report (file %s does not exist)" % (filename))
    return s def merge_dictionaries(*dict_args):
    """
    Given any number of dicts, shallow copy and merge into a new dict.
    precedence goes to key value pairs in latter dicts
    """

    result = {}
    for dictionary in dict_args:
        result.update(dictionary)

    return result def updateXMLWithEndpoints(xml, filenames, endpoints):
    """ Replace any 'tobeset' strings in the XML with final ddm endpoints """

    # Input:  xml (string)
    #         filenames (list of strings)
    #         endpoints (list of strings, 1-to-1 mapped to the filenames list)
    # Output: xml (string)
    # Description: The method looks for patterns '<filename>-ddmendpoint_tobeset' and
    #              replaces it with the corresponding 'endpoint'

    lines = []
    for line in xml.split('\n'):
        i = 0
        s = False
        for filename in filenames:
            p = filename + '-ddmendpoint_tobeset'
            if p in line:
                if endpoints[i] == None:
                    # remove this line without append
                    s = True
                else:
                    lines.append(line.replace(p, endpoints[i]))
                    s = True
                break
            i += 1
        if not s:
            lines.append(line)

    if lines == []:
        r = xml
    else:
        r = '\n'.join(lines)
    return r def isAnalysisJob(trf):
    """ Determine whether the job is an analysis job or not """

    if (trf.startswith('https://') or trf.startswith('http://')):
        analysisJob = True
    else:
        analysisJob = False

    return analysisJob class JobState:
    """
    This class is used to set the current job state.
    When the job is running, the file jobState-<JobID>.[pickle|json]
    is created which contains the state of the Site, Job and Node
    objects. The job state file is updated at every heartbeat.
    """
    def __init__(self):
        """ Default init """
        self.job = None            # Job class object
        self.site = None           # Site class object
        self.node = None           # Node class object
        self.filename = ""         # jobState-<jobId>.[pickle|json]
        self.recoveryAttempt = 0   # recovery attempt number
        self.objectDictionary = {} # file dictionary holding all objects

    def decode(self):
        """ Decode the job state file """

        if self.objectDictionary:
            try:
                # setup the internal structure (needed for cleanup)
                self.job = self.objectDictionary['job']
                self.site = self.objectDictionary['site']
                self.node = self.objectDictionary['node']
            except:
                tolog("JOBSTATE WARNING: Objects are missing in the recovery file (abort recovery)")
                pass
            try:
                self.recoveryAttempt = self.objectDictionary['recoveryAttempt']
            except:
                tolog("JOBSTATE WARNING: recoveryAttempt number is missing (set to 0)")
                self.recoveryAttempt = 0
                pass
        else:
            tolog("JOBSTATE FAILURE: Object dictionary not defined")

        return self.job, self.site, self.node, self.recoveryAttempt

    def get(self, filename):
        """ Read job state dictionary from file """
        self.filename = filename
        status = True

        # De-serialize the job state file
        try:
            fp = open(self.filename, "r")
        except:
            tolog("JOBSTATE FAILURE: JobState get function could not open file: %s" % self.filename)
            status = False
            pass
        else:
            tolog("Managed to open job state file")
            # is it a json or pickle file? import the relevant loader
            importedLoad = False
            if self.filename.endswith('json'):
                try:
                    from json import load
                except Exception, e:
                    tolog("JOBSTATE FAILURE: Could not import load function from json module (too old python version?): %s" % str(e))
                    status = False
                    pass
                else:
                    tolog("Imported json load")
                    importedLoad = True
            else:
                from pickle import load
                importedLoad = True
                tolog("Imported pickle load")

            if importedLoad:
                # load the dictionary from file
                try:
                    # load the dictionary from file
                    self.objectDictionary = load(fp)
                except:
                    tolog("JOBSTATE FAILURE: JobState could not deserialize file: %s" % self.filename)
                    status = False
                    pass
                else:
                    tolog("Managed to load object dictionary")
            else:
                tolog("Failed to import load function")
            fp.close()

        return status

    def getCurrentFilename(self):
        """ return the current file name """

        return self.filename

    def getFilename(self, workdir, jobId):
        """ get the name of the job state file """

        # force pickle format for the time being, there seems to be problems storing the node structure in json format
        extension = "pickle"
        return "%s/jobState-%s.%s" % (workdir, jobId, extension)
        # return "%s/jobState-%s.%s" % (workdir, jobId, getExtension())

    def put(self, job, site, node, recoveryAttempt=0, mode=""):
        """
        Create/Update the job state file
        """
        status = True

        self.job = job
        self.node = node
        self.site = site
        self.recoveryAttempt = recoveryAttempt
        if not (self.node and self.job and self.site):
            status = False
        else:
            # get the appropriate filename
            self.filename = self.getFilename(self.site.workdir, self.job.jobId)

            # add mode variable if needed
            if mode != "":
                if "json" in self.filename:
                    self.filename = self.filename.replace(".json", "-%s.json" % (mode))
                else:
                    self.filename = self.filename.replace(".pickle", "-%s.pickle" % (mode))

            # create object dictionary
            objectDictionary = {}
            objectDictionary['job'] = self.job
            objectDictionary['site'] = self.site
            objectDictionary['node'] = self.node
            objectDictionary['recoveryAttempt'] = self.recoveryAttempt

            # write the dictionary
            if "json" in self.filename:
                from json import dump
            else:
                from pickle import dump
            try:
                fp = open(self.filename, "w")
            except Exception, e:
                tolog("JOBSTATE FAILURE: Could not open job state file: %s, %s" % (self.filename, str(e)))
                _cmd = "whoami; ls -lF %s" % (self.filename)
                tolog("Executing command: %s" % (_cmd))
                ec, rs = commands.getstatusoutput(_cmd)
                tolog("%d, %s" % (ec, rs))
                status = False
            else:
                try:
                    # write the dictionary to file
                    dump(objectDictionary, fp)
                except Exception, e:
                    tolog("JOBSTATE FAILURE: Could not encode data to job state file: %s, %s" % (self.filename, str(e)))
                    status = False

                fp.close()
                                                                                                        
        return status

    def rename(self, site, job):
        """
        Rename the job state file. Should only be called for
        holding jobs that have passed the maximum number of recovery attempts.
        """
        status = True

        # # Experiencing problems with that code
        # get the file extension
        # extension = getExtension()
        #
        # fileNameOld = "%s/jobState-%s.%s" % (site.workdir, job.jobId, extension)
        # fileNameNew = "%s/jobState-%s.%s.MAXEDOUT" % (site.workdir, job.jobId, extension)

        fileNameOld = self.getFilename(site.workdir,job.jobId)
        fileNameNew = "%s.MAXEDOUT" % fileNameOld
        if os.path.isfile(fileNameOld):
            # rename the job state file
            try:
                os.system("mv %s %s" % (fileNameOld, fileNameNew))
            except OSError:
                tolog("JOBSTATE FAILURE: Failed to rename job state file: %s" % (fileNameOld))
                status = False
            else:
                tolog("Job state file renamed to: %s" % (fileNameNew))
        else:
            tolog("JOBSTATE FAILURE: Job state file does not exist: %s" % (fileNameOld))
            status = False
            
        return status

    def remove(self, site, job):
        """ Remove the job state file. Should only be called for
        finished jobs after the last server update. """
        status = True

#        # get the file extension
#        extension = getExtension()
#
#        # do not use self.filename in this case since this function is only
#        # used in pilot.cleanup() where self.filename has not been set
#        fileName = "%s/jobState-%s.%s" % (site.workdir, job.jobId, extension)

        # get the appropriate filename
        fileName = self.getFilename(site.workdir, job.jobId)

        if os.path.isfile(fileName):
            # remove the job state file
            try:
                os.system("rm -f %s" % fileName)
            except OSError:
                tolog("JOBSTATE FAILURE: Failed to remove job state file: %s" % fileName)
                status = False
        else:
            tolog("JOBSTATE FAILURE: Job state file does not exist: %s" % fileName)
            status = False
            
        return status
    
    def cleanup(self):
        """ Cleanup job state file as well as work and site directory (when needed) """
        status = True

        # remove the job state file
        ec = -1
        try:
            cmd = "rm -f %s" % (self.filename)
            tolog("Executing command: %s" % (cmd))
            ec, rs = commands.getstatusoutput(cmd)
        except Exception, e:
            tolog("JOBSTATE FAILURE: JobState cleanup failed to remove file: %s, %s" % (self.filename, str(e)))
#            status = False
        else:
            if ec == 0:
                tolog("Removed lost job state file: %s" % (self.filename))
            else:
                tolog("JOBSTATE FAILURE: Could not delete job state file: %d, %s" % (ec, rs))

        # remove the site directory only if there are no other job state files
        from glob import glob
        ec = -1
        files = glob("%s/jobState-*" % self.site.workdir)
        if len(files) == 0:
            # remove the site directory
            try:
                cmd = "rm -rf %s" % (self.site.workdir)
                tolog("Executing command: %s" % (cmd))
                ec, rs = commands.getstatusoutput(cmd)
            except Exception, e:
                tolog("JOBSTATE FAILURE: Could not delete site workdir: %s" % str(e))
                status = False
                pass
            else:
                if ec == 0:
                    tolog("Removed site dir: %s" % (self.site.workdir))
                else:
                    tolog("JOBSTATE FAILURE: Could not delete site workdir: %d, %s" % (ec, rs))
        else:
            # remove only the work directory (if it exists)
            if os.path.isdir(self.job.newDirNM):
                ec = -1
                try:
                    cmd = "rm -rf %s" % (self.job.newDirNM)
                    tolog("Executing command: %s" % (cmd))
                    ec, rs = commands.getstatusoutput(cmd)
                except Exception, e:
                    tolog("JOBSTATE FAILURE: Could not delete job workdir: %s" % str(e))
                    status = False
                    pass
                else:
                    if ec == 0:
                        tolog("Removed job workdir: %s" % (self.job.newDirNM))
                    else:
                        tolog("JOBSTATE FAILURE: Could not delete job workdir: %d, %s" % (ec, rs))
        return status def getOSTransferDictionaryFilename():
    """ Return the name of the objectstore transfer dictionary file """

    return "os_transfer_dictionary.json" def getOSTransferDictionary(filename):
    """ Get the dictionary of objectstore os_bucket_ids with populated buckets from the OS transfer dictionary """
    # Note: will return a dictionary of os_bucket_ids identifiers to which files were actually transferred, along
    # with the names of the buckets where the files were transferred to
    # This function is used to generate the jobMetrics OS message (only the OS and bucket endpoints are of interest)

    # os_bucket_ids_dictionary
    # FORMAT: { 'os_bucket_id': ['os_bucket_endpoint', ''], .. }
    # OS transfer dictionary
    # FORMAT: { 'os_bucket_id': { 'os_bucket_endpoint': <number of transferred files>, .. }, .. }

    os_bucket_ids_dictionary = {}

    if os.path.exists(filename):
        # Get the OS transfer dictionary
        dictionary = readJSON(filename)
        if dictionary != {}:
            tmp_os_bucket_ids_list = dictionary.keys()

            # Only report the os_bucket_id if there were files transferred to it
            for os_bucket_id in tmp_os_bucket_ids_list:

                # Get the os_bucket_endpoint list and populate the final dictionary
                os_bucket_endpoint_list = dictionary[os_bucket_id].keys()
                for os_bucket_endpoint in os_bucket_endpoint_list:
                    n = dictionary[os_bucket_id][os_bucket_endpoint]

                    tolog("OS bucket id %s: %d file(s) transferred to bucket %s" % (os_bucket_id, n, os_bucket_endpoint))
                    if n > 0:
                        if os_bucket_ids_dictionary.has_key(os_bucket_id):
                            os_bucket_ids_dictionary[os_bucket_id].append(os_bucket_endpoint)
                        else:
                            os_bucket_ids_dictionary[os_bucket_id] = [os_bucket_endpoint]
        else:
            tolog("!!WARNING!!3334!! OS transfer dictionary is empty")
    else:
        tolog("!!WARNING!!3333!! OS transfer dictionary does not exist at: %s" % (filename))

    return os_bucket_ids_dictionary def getHighestPriorityError(jobId, workdir):
    """ Return the highest priority error for jobId from the pilot error report file """
    # Return: {'pilotErrorCode': <nr>, 'pilotErrorDiag': '..'}
    # Note: only the first of the highest priority errors will be returned
    errorInfo = {}

    filename = getPilotErrorReportFilename(workdir)
    if os.path.exists(filename):
        # The file already exists, read it back (with unicode to utf-8 conversion)
        dictionary = getJSONDictionary(filename)
        if dictionary.has_key(jobId):

            # Find the highest priority error for this job
            highestPriority = 999
            for priority in dictionary[jobId].keys():
                try:
                    p = int(priority)
                except Exception, e:
                    tolog("!!WARNING!!2321!! Unexpected key in pilot error report: %s" % (e))
                else:
                    if p < highestPriority:
                        highestPriority = p

                if highestPriority < 999:
                    # Get the first reported error
                    errorInfo = dictionary[jobId][str(highestPriority)][0]
                else:
                    tolog("!!WARNING!!2322!! Could not locate the highest priority error")
        else:
            tolog("Pilot error report does not contain any error info for job %s" % (jobId))
    else:
        tolog("Pilot error report does not exist: %s (should only exist if there actually was an error)" % (filename))

    return errorInfo
import os
import traceback
from time import time, gmtime, strftime
from datetime import date
from commands import getstatusoutput, getoutput
from shutil import copy2
from PilotErrors import PilotErrors
from pUtil import tolog, readpar, timeStamp, getBatchSystemJobID, getCPUmodel, PFCxml, updateMetadata, addSkippedToPFC,\
    makeHTTPUpdate, tailPilotErrorDiag, isLogfileCopied, updateJobState, updateXMLWithSURLs, getMetadata, toPandaLogger,\
    getSiteInformation, getExperiment, readStringFromFile, merge_dictionaries, updateXMLWithEndpoints, isAnalysisJob
from JobState import JobState
from FileStateClient import getFilesOfState
from FileHandling import getOSTransferDictionaryFilename, getOSTransferDictionary, getHighestPriorityError
        from pickle import dump
                    from SiteMover import SiteMover
        from re import split
                from json import load
                    from SiteMover import SiteMover
            #tolog("Could have reported: %s" % (_jobMetrics))

            # Report which output files were moved to an alternative SE
            filenames = getFilesOfState(site.workdir, job.jobId, state="alt_transferred")
            if filenames != "":
                jobMetrics += self.jobMetric(key="altTransferred", value=filenames)

        # report on which OS bucket the log was written to, if any
        if job.logBucketID != -1:
            jobMetrics += self.jobMetric(key="logBucketID", value=job.logBucketID)

        # only add the JEM bit if explicitly set to YES, otherwise assumed to be NO
        if job.JEM == "YES":
            jobMetrics += self.jobMetric(key="JEM", value=1)
            # old format: jobMetrics += " JEM=%s" % (job.JEM)

        if job.dbTime != "":
            jobMetrics += self.jobMetric(key="dbTime", value=job.dbTime)
        if job.dbData != "":
            jobMetrics += self.jobMetric(key="dbData", value=job.dbData)

        # machine and job features, max disk space used by the payload
        jobMetrics += workerNode.addToJobMetrics(job.result[0], self.__pilot_initdir, job.jobId)

        si = getSiteInformation(job.experiment)

        _jobMetrics = ""

        # report any OS transfers
        #message = self.getOSJobMetrics()
        #if message != "":
        #    _jobMetrics = self.jobMetric(key="OS", value=message)
        #    tolog("Could have added: %s to job metrics" % (_jobMetrics))

        # correct for potential initial and trailing space
        jobMetrics = jobMetrics.lstrip().rstrip()

        if jobMetrics != "":
            tolog('Job metrics=\"%s\"' % (jobMetrics))
        else:
            tolog("No job metrics (all values are zero)")

        # is jobMetrics within allowed size?
        if len(jobMetrics) > 500:
            tolog("!!WARNING!!2223!! jobMetrics out of size (%d)" % (len(jobMetrics)))

            # try to reduce the field size and remove the last entry which might be cut
            jobMetrics = jobMetrics[:500]
            jobMetrics = " ".join(jobMetrics.split(" ")[:-1])

            tolog("jobMetrics has been reduced to: %s" % (jobMetrics))

        return jobMetrics

    # deprecated
    def getOSJobMetrics(self):
        """ Generate the objectstore jobMetrics message """
        # Message format:
        # OS=<os_name_0>:<os_bucket_endpoint_0>:<os_bucket_endpoint_1>: ..
        # Example:
        # os_name = BNL_OS_0, os_bucket_name = atlas_eventservice_F0 or atlas_logs_3D (where F0 and 3D are examples of file name hashes)
        # -> OS=BNL_OS_0;atlas_eventservice_F0:atlas_logs_3D
        # (note: at least one os_bucket_endpoint will be included in a message, but not necessarily both of them and order is random)

        message = ""

        # Locate the OS transfer dictionary
        filename = getOSTransferDictionaryFilename()
        path = os.path.join(self.__pilot_initdir, filename)
        if os.path.exists(path):
            # Which OS's were used?
            os_names_dictionary = getOSTransferDictionary(path)
            if os_names_dictionary != {}:
                message = ""

                os_names = os_names_dictionary.keys()
                # Note: the should only be one os_name
                if len(os_names) > 1:
                    tolog("!!WARNING!!2345!! Can only report one ddm endpoint (will use first only): %s" % (os_names_dictionary))

                # Which buckets were written to?
                for os_name in os_names_dictionary.keys():
                    message += os_name + ";"
                    bucket_list = os_names_dictionary[os_name]
                    for os_bucket_endpoint in bucket_list:
                        message += os_bucket_endpoint + ":"
                        # Remove the last ':'
                        message = message[:-1]

                    # Ignore any other os_names - there should one be one and we can only report one
                    break
            else:
                tolog("!!WARNING!!3335!! No OS transfers were found in: %s" % (filename))

        else:
            tolog("OS transfer dictionary does not exist, will not report OS transfers in jobMetrics (%s)" % (path))

        return message

    def getNodeStructure(self, job, site, workerNode, spaceReport=False, log=None):
        """ define the node structure expected by the server """

        node = {}

        node['node'] = workerNode.nodename
        node['workdir'] = job.workdir
        node['siteName'] = site.sitename
        node['jobId'] = job.jobId
        node['state'] = job.result[0]
        node['timestamp'] = timeStamp()
        if job.attemptNr > -1:
            node['attemptNr'] = job.attemptNr
        if self.__jobSchedulerId:
            node['schedulerID'] = self.__jobSchedulerId
        if self.__pilotId:
            use_newmover = str(readpar('use_newmover')).lower() in ["1", "true"]
            use_newmover_tag = 'NEWMOVER-%s' % ('ON' if use_newmover else 'OFF')
            tolog("Checking if new site movers workflow is enabled: use_newmover=%s" % use_newmover)

            # report the batch system job id, if available

            batchSystemType, _id = getBatchSystemJobID()