#manage hpc jobs
import sys, json, os, hashlib, time
from datetime import datetime
from collections import OrderedDict

#Q1: not sure if mananger can be run parallelly
#Q2: access hpc members directly, is it good?
class manager:
    def __init__(self, conf="magic", wait=105, **kwargs):
        self.id = "I am the manager"
        self.retries = 0
        self.wait = wait
        if conf == "magic":
            conf = os.path.join(os.path.dirname(__file__), "sys.config")
        with open(conf, "r") as f:
            self.sys = json.load(f, object_pairs_hook=OrderedDict)
        f.close()
        # print (self.sys)
        if "retries" in kwargs:
            self.retries = kwargs["retries"]

    def start(self, jobq, force=False, skip=False):
        if skip:
            return 0
        rjobq = []
        for j in jobq:
            j.set_retries(self.retries)
            if force or not self.check_job(j): #force or not run sucessfully before
                if force:
                    self.rm_tag(j)
                rjobq.append(j)
                if j.run():
                    j.set_retries(0)
                    j.set_rtn(1)
        while True:
            for j in rjobq: # can be problem if job has been finished?
                j.check_status()
                # j.wait() # or to keep the return values and check next?
                if j.rtn is not None: # better not access j.rtn directly ?  
                    if j.rtn:
                        print ("command {0} failed, return code: {1}".format(j.cmd if type(j.cmd) == str else " ".join(j.cmd), str(j.rtn)))
                        if j.retries:
                            tp = self.check_errt(j)
                            if tp == 1: # these codes are related to your system, need a config file
                                print ("extend memory and try again")
                                j.ext_mem()
                                if j.adjq(self.sys[j.platform]["queues"]):
                                    j.run()    
                                    j.decre_retries()
                                else:
                                    print ("fail to find a suitable queue")
                                    j.reset_retries()
                            elif tp == 2: # these codes are related to your system, need a config file
                                print ("change job queue and try again")
                                if j.chgq(self.sys[j.platform]["queues"]):
                                    j.run()
                                    j.decre_retries()
                                else:
                                    print ("fail to find a suitable queue")
                                    j.reset_retries()
                            else:
                                print ("unkown error, please check error log")
                                j.reset_retries()
                    elif not j.suc:
                        print ("command {0} run successfully".format(j.cmd if type(j.cmd) == str else " ".join(j.cmd), str(j.rtn)))
                        self.tag_job(j)
                        j.set_suc() # in case of tag again, better way?
                        j.reset_retries()
            if all(j.retries == 0 and j.rtn is not None for j in rjobq):
                break
            time.sleep(self.wait)    
        suc = all(j.rtn == 0 for j in rjobq)
        # for j in rjobq:
            # print ("command {0} return code: {1}".format(j.cmd if type(j.cmd) == str else " ".join(j.cmd), str(j.rtn)))
        if suc:
            return 0
        else:
            return 1
        # for j in rjobq:
            # if j.rtn:
                # return j.rtn
        # return 0
        
        # return artn

    
    def check_errt(self, j):
        rtn = j.rtn
        if j.platform in self.sys:
            if self.sys[j.platform]["errors"]["MEM"] == rtn:
                return 1 #mem error
            elif self.sys[j.platform]["errors"]["RUNTIME"] == rtn:
                return 2 # runtime error
            else:
                return 3 # unkown maybe will set later
        else:
            return 4 # unkown



    
    def rm_tag(self, j):
        #tag a job if run sucessfully
        cmd = j.cmd if type(j.cmd) == str else " ".join(j.cmd)
        hashmark =hashlib.sha256(cmd.encode('utf-8')).hexdigest()
        
        dirn = os.path.dirname(j.out)
        if not dirn:
            dirn = '.'
        tag_fn = '{0}/.{1}.done'.format(dirn, hashmark) 
        # print ("{} tag ".format(cmd))
        if os.path.isfile(tag_fn):
            os.remove(tag_fn) # what if failed to remove file?
        # open(tag_fn,'w').close()

    def tag_job(self, j):
        #tag a job if run sucessfully
        cmd = j.cmd if type(j.cmd) == str else " ".join(j.cmd)
        hashmark =hashlib.sha256(cmd.encode('utf-8')).hexdigest()
        
        dirn = os.path.dirname(j.out)
        if not dirn:
            dirn = '.'
        tag_fn = '{0}/.{1}.done'.format(dirn, hashmark) 
        # print ("{} tag ".format(cmd))
        with open(tag_fn,'w') as f:
            f.write("CMD : {0}\nTIME: {1}\n".format(cmd, datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
            f.close() 

    def check_job(self, j):
        # check if a job has run before 
        cmd = j.cmd if type(j.cmd) == str else " ".join(j.cmd)
        hashmark =hashlib.sha256(cmd.encode('utf-8')).hexdigest()
        
        dirn = os.path.dirname(j.out)
        if not dirn:
            dirn = '.'
        tag_fn = '{0}/.{1}.done'.format(dirn, hashmark) 
        if os.path.isfile(tag_fn):
            print ("{} ran before, skip".format(cmd))
            return True 
        else:
            return False 


# Notes on LSF Platform
# only take two types of error
#yesterday	High priority queue (as in "I needed this run yesterday"). You can only run 10 jobs at a time in this queue.
# small	Queue for short running jobs (< 1 minute). Has job-chunking enabled (see the LSF docs). Such short jobs are generally a bad idea; try to make your jobs run for at least 10 minutes, if possible, and use other queues.
# normal	The default queue. For jobs running for up to an hour each. There is a hard limit of 12 hours, after which your job will be killed.
# hugemem	Queue for jobs which require > 196 GB of memory. You must specify how much memory you expect your job to use.
# teramem	Queue for jobs which require > 745 GB of memory. You must specify how much memory you expect your job to use.
# long	Queue for long running jobs (> 1 hour). There is a hard limit of 48 hours, after which the job will be killed. [1, 48] 
# basement	Queue for jobs which run for more than a day. You should consider checkpointing long running jobs [0,24]
#Parallel Queue for multi-node, multi-cpu jobs (ie PVM/MPI jobs, not threaded jobs).

#manage hpc jobs
import sys, json, os, hashlib, time
from datetime import datetime
from collections import OrderedDict

#Q1: not sure if mananger can be run parallelly
#Q2: access hpc members directly, is it good?
class manager:
    def __init__(self, conf="magic", wait=105, **kwargs):
        self.id = "I am the manager"
        self.retries = 0
        self.wait = wait
        if conf == "magic":
            conf = os.path.join(os.path.dirname(__file__), "sys.config")
        with open(conf, "r") as f:
            self.sys = json.load(f, object_pairs_hook=OrderedDict)
        f.close()
        # print (self.sys)
        if "retries" in kwargs:
            self.retries = kwargs["retries"]

    def start(self, jobq, force=False, skip=False):
        if skip:
            return 0
        rjobq = []
        for j in jobq:
            j.set_retries(self.retries)
            if force or not self.check_job(j): #force or not run sucessfully before
                if force:
                    self.rm_tag(j)
                rjobq.append(j)
                if j.run():
                    j.set_retries(0)
                    j.set_rtn(1)
        while True:
            for j in rjobq: # can be problem if job has been finished?
                j.check_status()
                # j.wait() # or to keep the return values and check next?
                if j.rtn is not None: # better not access j.rtn directly ?  
                    if j.rtn:
                        print ("command {0} failed, return code: {1}".format(j.cmd if type(j.cmd) == str else " ".join(j.cmd), str(j.rtn)))
                        if j.retries:
                            tp = self.check_errt(j)
                            if tp == 1: # these codes are related to your system, need a config file
                                print ("extend memory and try again")
                                j.ext_mem()
                                if j.adjq(self.sys[j.platform]["queues"]):
                                    j.run()    
                                    j.decre_retries()
                                else:
                                    print ("fail to find a suitable queue")
                                    j.reset_retries()
                            elif tp == 2: # these codes are related to your system, need a config file
                                print ("change job queue and try again")
                                if j.chgq(self.sys[j.platform]["queues"]):
                                    j.run()
                                    j.decre_retries()
                                else:
                                    print ("fail to find a suitable queue")
                                    j.reset_retries()
                            else:
                                print ("unkown error, please check error log")
                                j.reset_retries()
                    elif not j.suc:
                        print ("command {0} run successfully".format(j.cmd if type(j.cmd) == str else " ".join(j.cmd), str(j.rtn)))
                        self.tag_job(j)
                        j.set_suc() # in case of tag again, better way?
                        j.reset_retries()
            if all(j.retries == 0 and j.rtn is not None for j in rjobq):
                break
            time.sleep(self.wait)    
        suc = all(j.rtn == 0 for j in rjobq)
        # for j in rjobq:
            # print ("command {0} return code: {1}".format(j.cmd if type(j.cmd) == str else " ".join(j.cmd), str(j.rtn)))
        if suc:
            return 0
        else:
            return 1
        # for j in rjobq:
            # if j.rtn:
                # return j.rtn
        # return 0
        
        # return artn

    
    def check_errt(self, j):
        rtn = j.rtn
        if j.platform in self.sys:
            if self.sys[j.platform]["errors"]["MEM"] == rtn:
                return 1 #mem error
            elif self.sys[j.platform]["errors"]["RUNTIME"] == rtn:
                return 2 # runtime error
            else:
                return 3 # unkown maybe will set later
        else:
            return 4 # unkown



    
    def rm_tag(self, j):
        #tag a job if run sucessfully
        cmd = j.cmd if type(j.cmd) == str else " ".join(j.cmd)
        hashmark =hashlib.sha256(cmd.encode('utf-8')).hexdigest()
        
        dirn = os.path.dirname(j.out)
        if not dirn:
            dirn = '.'
        tag_fn = '{0}/.{1}.done'.format(dirn, hashmark) 
        # print ("{} tag ".format(cmd))
        if os.path.isfile(tag_fn):
            os.remove(tag_fn) # what if failed to remove file?
        # open(tag_fn,'w').close()

    def tag_job(self, j):
        #tag a job if run sucessfully
        cmd = j.cmd if type(j.cmd) == str else " ".join(j.cmd)
        hashmark =hashlib.sha256(cmd.encode('utf-8')).hexdigest()
        
        dirn = os.path.dirname(j.out)
        if not dirn:
            dirn = '.'
        tag_fn = '{0}/.{1}.done'.format(dirn, hashmark) 
        # print ("{} tag ".format(cmd))
        with open(tag_fn,'w') as f:
            f.write("CMD : {0}\nTIME: {1}\n".format(cmd, datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
            f.close() 

    def check_job(self, j):
        # check if a job has run before 
        cmd = j.cmd if type(j.cmd) == str else " ".join(j.cmd)
        hashmark =hashlib.sha256(cmd.encode('utf-8')).hexdigest()
        
        dirn = os.path.dirname(j.out)
        if not dirn:
            dirn = '.'
        tag_fn = '{0}/.{1}.done'.format(dirn, hashmark) 
        if os.path.isfile(tag_fn):
            print ("{} ran before, skip".format(cmd))
            return True 
        else:
            return False 


# Notes on LSF Platform
# only take two types of error
#yesterday	High priority queue (as in "I needed this run yesterday"). You can only run 10 jobs at a time in this queue.
# small	Queue for short running jobs (< 1 minute). Has job-chunking enabled (see the LSF docs). Such short jobs are generally a bad idea; try to make your jobs run for at least 10 minutes, if possible, and use other queues.
# normal	The default queue. For jobs running for up to an hour each. There is a hard limit of 12 hours, after which your job will be killed.
# hugemem	Queue for jobs which require > 196 GB of memory. You must specify how much memory you expect your job to use.
# teramem	Queue for jobs which require > 745 GB of memory. You must specify how much memory you expect your job to use.
# long	Queue for long running jobs (> 1 hour). There is a hard limit of 48 hours, after which the job will be killed. [1, 48] 
# basement	Queue for jobs which run for more than a day. You should consider checkpointing long running jobs [0,24]
#Parallel Queue for multi-node, multi-cpu jobs (ie PVM/MPI jobs, not threaded jobs).

#manage hpc jobs #Q1: not sure if mananger can be run parallelly #Q2: access hpc members directly, is it good? # print (self.sys) #force or not run sucessfully before # can be problem if job has been finished? # j.wait() # or to keep the return values and check next? # better not access j.rtn directly ? # these codes are related to your system, need a config file # these codes are related to your system, need a config file # in case of tag again, better way? # for j in rjobq: # print ("command {0} return code: {1}".format(j.cmd if type(j.cmd) == str else " ".join(j.cmd), str(j.rtn))) # for j in rjobq: # if j.rtn: # return j.rtn # return 0 # return artn #mem error # runtime error # unkown maybe will set later # unkown #tag a job if run sucessfully # print ("{} tag ".format(cmd)) # what if failed to remove file? # open(tag_fn,'w').close() #tag a job if run sucessfully # print ("{} tag ".format(cmd)) # check if a job has run before # Notes on LSF Platform # only take two types of error #yesterday	High priority queue (as in "I needed this run yesterday"). You can only run 10 jobs at a time in this queue. # small	Queue for short running jobs (< 1 minute). Has job-chunking enabled (see the LSF docs). Such short jobs are generally a bad idea; try to make your jobs run for at least 10 minutes, if possible, and use other queues. # normal	The default queue. For jobs running for up to an hour each. There is a hard limit of 12 hours, after which your job will be killed. # hugemem	Queue for jobs which require > 196 GB of memory. You must specify how much memory you expect your job to use. # teramem	Queue for jobs which require > 745 GB of memory. You must specify how much memory you expect your job to use. # long	Queue for long running jobs (> 1 hour). There is a hard limit of 48 hours, after which the job will be killed. [1, 48] # basement	Queue for jobs which run for more than a day. You should consider checkpointing long running jobs [0,24] #Parallel Queue for multi-node, multi-cpu jobs (ie PVM/MPI jobs, not threaded jobs).