src/toil_lib/spark.py
def start(self, job):
        """
        Start spark and hdfs master containers

        :param job: The underlying job.
        """

        if self.hostname is None:
            self.hostname = subprocess.check_output(["hostname", "-f",])[:-1]

        _log.info("Started Spark master container.")
        self.sparkContainerID = dockerCheckOutput(job=job,
                                                  defer=STOP,
                                                  workDir=os.getcwd(),
                                                  tool="quay.io/ucsc_cgl/apache-spark-master:1.5.2",
                                                  dockerParameters=["--net=host",
                                                                    "-d",
                                                                    "-v", "/mnt/ephemeral/:/ephemeral/:rw",
                                                                    "-e", "SPARK_MASTER_IP=" + self.hostname,
                                                                    "-e", "SPARK_LOCAL_DIRS=/ephemeral/spark/local",
                                                                    "-e", "SPARK_WORKER_DIR=/ephemeral/spark/work"],
                                                  parameters=[self.hostname])[:-1]
        _log.info("Started HDFS Datanode.")
        self.hdfsContainerID = dockerCheckOutput(job=job,
                                           defer=STOP,
                                           workDir=os.getcwd(),
                                           tool="quay.io/ucsc_cgl/apache-hadoop-master:2.6.2",
                                           dockerParameters=["--net=host",
                                                              "-d"],
                                           parameters=[self.hostname])[:-1]

        return self.hostname
def start(self, job):
        if self.hostname is None:
            self.hostname = subprocess.check_output(["hostname", "-f",])[:-1]

        _log.info("Started Spark master container.")
        self.sparkContainerID = dockerCheckOutput(job=job,
                                                  defer=STOP,
                                                  workDir=os.getcwd(),
                                                  tool="quay.io/ucsc_cgl/apache-spark-master:1.5.2",
                                                  dockerParameters=["--net=host",
                                                                    "-d",
                                                                    "-v", "/mnt/ephemeral/:/ephemeral/:rw",
                                                                    "-e", "SPARK_MASTER_IP=" + self.hostname,
                                                                    "-e", "SPARK_LOCAL_DIRS=/ephemeral/spark/local",
                                                                    "-e", "SPARK_WORKER_DIR=/ephemeral/spark/work"],
                                                  parameters=[self.hostname])[:-1]
        _log.info("Started HDFS Datanode.")
        self.hdfsContainerID = dockerCheckOutput(job=job,
                                           defer=STOP,
                                           workDir=os.getcwd(),
                                           tool="quay.io/ucsc_cgl/apache-hadoop-master:2.6.2",
                                           dockerParameters=["--net=host",
                                                              "-d"],
                                           parameters=[self.hostname])[:-1]

        return self.hostname
def start ( self , job ) : if self . hostname is None : self . hostname = subprocess . check_output ( [ "hostname" , "-f" , ] ) [ : - 1 ] _log . info ( "Started Spark master container." ) self . sparkContainerID = dockerCheckOutput ( job = job , defer = STOP , workDir = os . getcwd ( ) , tool = "quay.io/ucsc_cgl/apache-spark-master:1.5.2" , dockerParameters = [ "--net=host" , "-d" , "-v" , "/mnt/ephemeral/:/ephemeral/:rw" , "-e" , "SPARK_MASTER_IP=" + self . hostname , "-e" , "SPARK_LOCAL_DIRS=/ephemeral/spark/local" , "-e" , "SPARK_WORKER_DIR=/ephemeral/spark/work" ] , parameters = [ self . hostname ] ) [ : - 1 ] _log . info ( "Started HDFS Datanode." ) self . hdfsContainerID = dockerCheckOutput ( job = job , defer = STOP , workDir = os . getcwd ( ) , tool = "quay.io/ucsc_cgl/apache-hadoop-master:2.6.2" , dockerParameters = [ "--net=host" , "-d" ] , parameters = [ self . hostname ] ) [ : - 1 ] return self . hostname
Start spark and hdfs master containers

        :param job: The underlying job.
Start spark and hdfs master containers