def picard_mark_duplicates(job, bam, bai, validation_stringency='LENIENT'):
    """
    Runs Picard MarkDuplicates on a BAM file. Requires that the BAM file be coordinate sorted.

    :param JobFunctionWrappingJob job: passed automatically by Toil
    :param str bam: FileStoreID for BAM file
    :param str bai: FileStoreID for BAM index file
    :param str validation_stringency: BAM file validation stringency, default is LENIENT
    :return: FileStoreIDs for BAM and BAI files
    :rtype: tuple
    """
    work_dir = job.fileStore.getLocalTempDir()

    # Retrieve file path
    job.fileStore.readGlobalFile(bam, os.path.join(work_dir, 'sorted.bam'))
    job.fileStore.readGlobalFile(bai, os.path.join(work_dir, 'sorted.bai'))

    # Call: picardtools
    command = ['MarkDuplicates',
               'INPUT=sorted.bam',
               'OUTPUT=mkdups.bam',
               'METRICS_FILE=metrics.txt',
               'ASSUME_SORTED=true',
               'CREATE_INDEX=true',
               'VALIDATION_STRINGENCY=%s' % validation_stringency.upper()]

    # picard-tools container doesn't have JAVA_OPTS variable
    # Set TMPDIR to /data to prevent writing temporary files to /tmp
    docker_parameters = ['--rm',
                         '--log-driver', 'none',
                         '-e', 'JAVA_OPTIONS=-Djava.io.tmpdir=/data/ -Xmx{}'.format(job.memory),
                         '-v', '{}:/data'.format(work_dir)]

    start_time = time.time()
    dockerCall(job=job, workDir=work_dir,
               parameters=command,
               tool='quay.io/ucsc_cgl/picardtools:1.95--dd5ac549b95eb3e5d166a5e310417ef13651994e',
               dockerParameters=docker_parameters)
    end_time = time.time()
    _log_runtime(job, start_time, end_time, "Picard MarkDuplicates")

    bam = job.fileStore.writeGlobalFile(os.path.join(work_dir, 'mkdups.bam'))
    bai = job.fileStore.writeGlobalFile(os.path.join(work_dir, 'mkdups.bai'))
    return bam, bai

Runs Picard MarkDuplicates on a BAM file . Requires that the BAM file be coordinate sorted .


def run_picard_sort(job, bam, sort_by_name=False):
    """
    Sorts BAM file using Picard SortSam

    :param JobFunctionWrappingJob job: passed automatically by Toil
    :param str bam: FileStoreID for BAM file
    :param boolean sort_by_name: If true, sorts by read name instead of coordinate.
    :return: FileStoreID for sorted BAM file
    :rtype: str
    """
    work_dir = job.fileStore.getLocalTempDir()
    job.fileStore.readGlobalFile(bam, os.path.join(work_dir, 'input.bam'))
    command = ['SortSam',
               'O=/data/output.bam',
               'I=/data/input.bam']

    # picard-tools container doesn't have JAVA_OPTS variable
    # Set TMPDIR to /data to prevent writing temporary files to /tmp
    docker_parameters = ['--rm',
                         '--log-driver', 'none',
                         '-e', 'JAVA_OPTIONS=-Djava.io.tmpdir=/data/ -Xmx{}'.format(job.memory),
                         '-v', '{}:/data'.format(work_dir)]

    if sort_by_name:
        command.append('SO=queryname')
    else:
        command.append('SO=coordinate')

    start_time = time.time()
    dockerCall(job=job, workDir=work_dir,
               parameters=command,
               tool='quay.io/ucsc_cgl/picardtools:1.95--dd5ac549b95eb3e5d166a5e310417ef13651994e',
               dockerParameters=docker_parameters)
    end_time = time.time()
    _log_runtime(job, start_time, end_time, "Picard SortSam")
    return job.fileStore.writeGlobalFile(os.path.join(work_dir, 'output.bam'))


def run_base_recalibration(job, bam, bai, ref, ref_dict, fai, dbsnp, mills, unsafe=False):
    """
    Creates recalibration table for Base Quality Score Recalibration

    :param JobFunctionWrappingJob job: passed automatically by Toil
    :param str bam: FileStoreID for BAM file
    :param str bai: FileStoreID for BAM index file
    :param str ref: FileStoreID for reference genome fasta file
    :param str ref_dict: FileStoreID for reference genome sequence dictionary file
    :param str fai: FileStoreID for reference genome fasta index file
    :param str dbsnp: FileStoreID for dbSNP VCF file
    :param str mills: FileStoreID for Mills VCF file
    :param bool unsafe: If True, runs GATK in UNSAFE mode: "-U ALLOW_SEQ_DICT_INCOMPATIBILITY"
    :return: FileStoreID for the recalibration table file
    :rtype: str
    """
    inputs = {'ref.fasta': ref,
              'ref.fasta.fai': fai,
              'ref.dict': ref_dict,
              'input.bam': bam,
              'input.bai': bai,
              'dbsnp.vcf': dbsnp,
              'mills.vcf': mills}

    work_dir = job.fileStore.getLocalTempDir()
    for name, file_store_id in inputs.iteritems():
        job.fileStore.readGlobalFile(file_store_id, os.path.join(work_dir, name))

    # Call: GATK -- BaseRecalibrator
    parameters = ['-T', 'BaseRecalibrator',
                  '-nct', str(int(job.cores)),
                  '-R', '/data/ref.fasta',
                  '-I', '/data/input.bam',
                  # Recommended known sites:
                  # https://software.broadinstitute.org/gatk/guide/article?id=1247
                  '-knownSites', '/data/dbsnp.vcf',
                  '-knownSites', '/data/mills.vcf',
                  '-o', '/data/recal_data.table']

    if unsafe:
        parameters.extend(['-U', 'ALLOW_SEQ_DICT_INCOMPATIBILITY'])

    # Set TMPDIR to /data to prevent writing temporary files to /tmp
    docker_parameters = ['--rm',
                         '--log-driver', 'none',
                         '-e', 'JAVA_OPTS=-Djava.io.tmpdir=/data/ -Xmx{}'.format(job.memory),
                         '-v', '{}:/data'.format(work_dir)]
    start_time = time.time()
    dockerCall(job=job, tool='quay.io/ucsc_cgl/gatk:3.5--dba6dae49156168a909c43330350c6161dc7ecc2',
               workDir=work_dir,
               parameters=parameters,
               dockerParameters=docker_parameters)
    end_time = time.time()
    _log_runtime(job, start_time, end_time, "GATK3 BaseRecalibrator")

    return job.fileStore.writeGlobalFile(os.path.join(work_dir, 'recal_data.table'))


def run_kallisto(job, r1_id, r2_id, kallisto_index_url):
    """
    RNA quantification via Kallisto

    :param JobFunctionWrappingJob job: passed automatically by Toil
    :param str r1_id: FileStoreID of fastq (pair 1)
    :param str r2_id: FileStoreID of fastq (pair 2 if applicable, otherwise pass None for single-end)
    :param str kallisto_index_url: FileStoreID for Kallisto index file
    :return: FileStoreID from Kallisto output
    :rtype: str
    """
    work_dir = job.fileStore.getLocalTempDir()
    download_url(job, url=kallisto_index_url, name='kallisto_hg38.idx', work_dir=work_dir)
    # Retrieve files
    parameters = ['quant',
                  '-i', '/data/kallisto_hg38.idx',
                  '-t', str(job.cores),
                  '-o', '/data/',
                  '-b', '100',
                  '--fusion']
    if r1_id and r2_id:
        job.fileStore.readGlobalFile(r1_id, os.path.join(work_dir, 'R1.fastq'))
        job.fileStore.readGlobalFile(r2_id, os.path.join(work_dir, 'R2.fastq'))
        parameters.extend(['/data/R1.fastq', '/data/R2.fastq'])
    else:
        job.fileStore.readGlobalFile(r1_id, os.path.join(work_dir, 'R1.fastq'))
        parameters.extend(['--single', '-l', '200', '-s', '15', '/data/R1.fastq'])

    # Call: Kallisto
    dockerCall(job=job, tool='quay.io/ucsc_cgl/kallisto:0.42.4--35ac87df5b21a8e8e8d159f26864ac1e1db8cf86',
               workDir=work_dir, parameters=parameters)

    # Tar output files together and store in fileStore
    output_files = [os.path.join(work_dir, x) for x in ['run_info.json', 'abundance.tsv', 'abundance.h5', 'fusion.txt']]
    tarball_files(tar_name='kallisto.tar.gz', file_paths=output_files, output_dir=work_dir)
    return job.fileStore.writeGlobalFile(os.path.join(work_dir, 'kallisto.tar.gz'))


def run_rsem(job, bam_id, rsem_ref_url, paired=True):
    """
    RNA quantification with RSEM

    :param JobFunctionWrappingJob job: Passed automatically by Toil
    :param str bam_id: FileStoreID of transcriptome bam for quantification
    :param str rsem_ref_url: URL of RSEM reference (tarball)
    :param bool paired: If True, uses parameters for paired end data
    :return: FileStoreIDs for RSEM's gene and isoform output
    :rtype: str
    """
    work_dir = job.fileStore.getLocalTempDir()
    download_url(job, url=rsem_ref_url, name='rsem_ref.tar.gz', work_dir=work_dir)
    subprocess.check_call(['tar', '-xvf', os.path.join(work_dir, 'rsem_ref.tar.gz'), '-C', work_dir])
    os.remove(os.path.join(work_dir, 'rsem_ref.tar.gz'))
    # Determine tarball structure - based on it, ascertain folder name and rsem reference prefix
    rsem_files = []
    for root, directories, files in os.walk(work_dir):
        rsem_files.extend([os.path.join(root, x) for x in files])
    # "grp" is a required RSEM extension that should exist in the RSEM reference
    ref_prefix = [os.path.basename(os.path.splitext(x)[0]) for x in rsem_files if 'grp' in x][0]
    ref_folder = os.path.join('/data', os.listdir(work_dir)[0]) if len(os.listdir(work_dir)) == 1 else '/data'
    # I/O
    job.fileStore.readGlobalFile(bam_id, os.path.join(work_dir, 'transcriptome.bam'))
    output_prefix = 'rsem'
    # Call: RSEM
    parameters = ['--quiet',
                  '--no-qualities',
                  '-p', str(job.cores),
                  '--forward-prob', '0.5',
                  '--seed-length', '25',
                  '--fragment-length-mean', '-1.0',
                  '--bam', '/data/transcriptome.bam',
                  os.path.join(ref_folder, ref_prefix),
                  output_prefix]
    if paired:
        parameters = ['--paired-end'] + parameters
    dockerCall(job=job, tool='quay.io/ucsc_cgl/rsem:1.2.25--d4275175cc8df36967db460b06337a14f40d2f21',
               parameters=parameters, workDir=work_dir)
    # Write to FileStore
    gene_id = job.fileStore.writeGlobalFile(os.path.join(work_dir, output_prefix + '.genes.results'))
    isoform_id = job.fileStore.writeGlobalFile(os.path.join(work_dir, output_prefix + '.isoforms.results'))
    return gene_id, isoform_id


def get_user_affinity(self, test):
        """Prepare test set for C++ SAR prediction code.
        Find all items the test users have seen in the past.

        Arguments:
            test (pySpark.DataFrame): input dataframe which contains test users.
        """
        test.createOrReplaceTempView(self.f("{prefix}df_test"))

        query = self.f(
            "SELECT DISTINCT {col_user} FROM {prefix}df_test CLUSTER BY {col_user}"
        )

        df_test_users = self.spark.sql(query)
        df_test_users.write.mode("overwrite").saveAsTable(
            self.f("{prefix}df_test_users")
        )

        query = self.f(
        """
          SELECT a.{col_user}, a.{col_item}, CAST(a.{col_rating} AS double) {col_rating}
          FROM {prefix}df_train a INNER JOIN {prefix}df_test_users b ON a.{col_user} = b.{col_user}
          DISTRIBUTE BY {col_user}
          SORT BY {col_user}, {col_item}
        """
        )

        return self.spark.sql(query)

Prepare test set for C ++ SAR prediction code . Find all items the test users have seen in the past .


def send(self, cmd):
        """Send the given command thru the websocket"""
        with self.ws_sendlock:
            self.ws.send(json.dumps(cmd))

def send ( self , cmd ) : with self . ws_sendlock : self . ws . send ( json . dumps ( cmd ) )


def subscribe(self, stream, callback, transform=""):
        """Given a stream, a callback and an optional transform, sets up the subscription"""
        if self.status == "disconnected" or self.status == "disconnecting" or self.status == "connecting":
            self.connect()
        if self.status is not "connected":
            return False
        logging.debug("Subscribing to %s", stream)

        self.send({"cmd": "subscribe", "arg": stream, "transform": transform})
        with self.subscription_lock:
            self.subscriptions[stream + ":" + transform] = callback
        return True

def
subscribe
(
self
,
stream
,
callback
,
transform
=
""
)
:
if
self
.
status
==
"disconnected"
or
self
.
status
==
"disconnecting"
or
self
.
status
==
"connecting"
:
self
.
connect
(
)
if
self
.
status
is
not
"connected"
:
return
False
logging
.
debug
(
"Subscribing to %s"
,
stream
)
self
.
send
(
{
"cmd"
:
"subscribe"
,
"arg"
:
stream
,
"transform"
:
transform
}
)
with
self
.
subscription_lock
:
self
.
subscriptions
[
stream
+
":"
+
transform
]
=
callback
return
True

Given a stream a callback and an optional transform sets up the subscription


def connect(self):
        """Attempt to connect to the websocket - and returns either True or False depending on if
        the connection was successful or not"""

        # Wait for the lock to be available (ie, the websocket is not being used (yet))
        self.ws_openlock.acquire()
        self.ws_openlock.release()

        if self.status == "connected":
            return True  # Already connected
        if self.status == "disconnecting":
            # If currently disconnecting, wait a moment, and retry connect
            time.sleep(0.1)
            return self.connect()
        if self.status == "disconnected" or self.status == "reconnecting":
            self.ws = websocket.WebSocketApp(self.ws_url,
                                             header=self.headers,
                                             on_message=self.__on_message,
                                             on_ping=self.__on_ping,
                                             on_open=self.__on_open,
                                             on_close=self.__on_close,
                                             on_error=self.__on_error)
            self.ws_thread = threading.Thread(target=self.ws.run_forever)
            self.ws_thread.daemon = True

            self.status = "connecting"
            self.ws_openlock.acquire()
            self.ws_thread.start()

        self.ws_openlock.acquire()
        self.ws_openlock.release()

        return self.status == "connected"


def __reconnect(self):
        """This is called when a connection is lost - it attempts to reconnect to the server"""
        self.status = "reconnecting"

        # Reset the disconnect time after 15 minutes
        if self.disconnected_time - self.connected_time > 15 * 60:
            self.reconnect_time = self.reconnect_time_starting_seconds
        else:
            self.reconnect_time *= self.reconnect_time_backoff_multiplier

        if self.reconnect_time > self.reconnect_time_max_seconds:
            self.reconnect_time = self.reconnect_time_max_seconds

        # We want to add some randomness to the reconnect rate - necessary so that we don't pound the server
        # if it goes down
        self.reconnect_time *= 1 + random.uniform(-0.2, 0.2)

        if self.reconnect_time < self.reconnect_time_starting_seconds:
            self.reconnect_time = self.reconnect_time_starting_seconds

        logging.warn("ConnectorDB:WS: Attempting to reconnect in %fs",
                     self.reconnect_time)

        self.reconnector = threading.Timer(self.reconnect_time,
                                           self.__reconnect_fnc)
        self.reconnector.daemon = True
        self.reconnector.start()


def __resubscribe(self):
        """Send subscribe command for all existing subscriptions. This allows to resume a connection
        that was closed"""
        with self.subscription_lock:
            for sub in self.subscriptions:
                logging.debug("Resubscribing to %s", sub)
                stream_transform = sub.split(":", 1)
                self.send({
                    "cmd": "subscribe",
                    "arg": stream_transform[0],
                    "transform": stream_transform[1]
                })

def
__resubscribe
(
self
)
:
with
self
.
subscription_lock
:
for
sub
in
self
.
subscriptions
:
logging
.
debug
(
"Resubscribing to %s"
,
sub
)
stream_transform
=
sub
.
split
(
":"
,
1
)
self
.
send
(
{
"cmd"
:
"subscribe"
,
"arg"
:
stream_transform
[
0
]
,
"transform"
:
stream_transform
[
1
]
}
)

Send subscribe command for all existing subscriptions . This allows to resume a connection that was closed


def __on_open(self, ws):
        """Called when the websocket is opened"""
        logging.debug("ConnectorDB: Websocket opened")

        # Connection success - decrease the wait time for next connection
        self.reconnect_time /= self.reconnect_time_backoff_multiplier

        self.status = "connected"

        self.lastpingtime = time.time()
        self.__ensure_ping()

        self.connected_time = time.time()

        # Release the lock that connect called
        self.ws_openlock.release()


def __on_close(self, ws):
        """Called when the websocket is closed"""
        if self.status == "disconnected":
            return  # This can be double-called on disconnect
        logging.debug("ConnectorDB:WS: Websocket closed")

        # Turn off the ping timer
        if self.pingtimer is not None:
            self.pingtimer.cancel()

        self.disconnected_time = time.time()
        if self.status == "disconnecting":
            self.status = "disconnected"
        elif self.status == "connected":
            self.__reconnect()


def __on_error(self, ws, err):
        """Called when there is an error in the websocket"""
        logging.debug("ConnectorDB:WS: Connection Error")

        if self.status == "connecting":
            self.status = "errored"
            self.ws_openlock.release()

def
__on_error
(
self
,
ws
,
err
)
:
logging
.
debug
(
"ConnectorDB:WS: Connection Error"
)
if
self
.
status
==
"connecting"
:
self
.
status
=
"errored"
self
.
ws_openlock
.
release
(
)


def __on_message(self, ws, msg):
        """This function is called whenever there is a message received from the server"""
        msg = json.loads(msg)
        logging.debug("ConnectorDB:WS: Msg '%s'", msg["stream"])

        # Build the subcription key
        stream_key = msg["stream"] + ":"
        if "transform" in msg:
            stream_key += msg["transform"]

        self.subscription_lock.acquire()
        if stream_key in self.subscriptions:
            subscription_function = self.subscriptions[stream_key]
            self.subscription_lock.release()

            fresult = subscription_function(msg["stream"], msg["data"])

            if fresult is True:
                # This is a special result - if the subscription function of a downlink returns True,
                # then the datapoint is acknowledged automatically (ie, reinserted in non-downlink stream)
                fresult = msg["data"]

            if fresult is not False and fresult is not None and msg["stream"].endswith(
                    "/downlink") and msg["stream"].count("/") == 3:
                # If the above conditions are true, it means that the datapoints were from a downlink,
                # and the subscriber function chooses to acknowledge them, so we reinsert them.
                self.insert(msg["stream"][:-9], fresult)
        else:
            self.subscription_lock.release()
            logging.warn(
                "ConnectorDB:WS: Msg '%s' not subscribed! Subscriptions: %s",
                msg["stream"], list(self.subscriptions.keys()))


def __ensure_ping(self):
        """Each time the server sends a ping message, we record the timestamp. If we haven't received a ping
        within the given interval, then we assume that the connection was lost, close the websocket and
        attempt to reconnect"""

        logging.debug("ConnectorDB:WS: pingcheck")
        if (time.time() - self.lastpingtime > self.connection_ping_timeout):
            logging.warn("ConnectorDB:WS: Websocket ping timed out!")
            if self.ws is not None:
                self.ws.close()
                self.__on_close(self.ws)
        else:
            # reset the ping timer
            self.pingtimer = threading.Timer(self.connection_ping_timeout,
                                             self.__ensure_ping)
            self.pingtimer.daemon = True
            self.pingtimer.start()

Each time the server sends a ping message we record the timestamp . If we haven t received a ping within the given interval then we assume that the connection was lost close the websocket and attempt to reconnect


def gatk_select_variants(job, mode, vcf_id, ref_fasta, ref_fai, ref_dict):
    """
    Isolates a particular variant type from a VCF file using GATK SelectVariants

    :param JobFunctionWrappingJob job: passed automatically by Toil
    :param str mode: variant type (i.e. SNP or INDEL)
    :param str vcf_id: FileStoreID for input VCF file
    :param str ref_fasta: FileStoreID for reference genome fasta
    :param str ref_fai: FileStoreID for reference genome index file
    :param str ref_dict: FileStoreID for reference genome sequence dictionary file
    :return: FileStoreID for filtered VCF
    :rtype: str
    """
    job.fileStore.logToMaster('Running GATK SelectVariants to select %ss' % mode)

    inputs = {'genome.fa': ref_fasta,
              'genome.fa.fai': ref_fai,
              'genome.dict': ref_dict,
              'input.vcf': vcf_id}

    work_dir = job.fileStore.getLocalTempDir()
    for name, file_store_id in inputs.iteritems():
        job.fileStore.readGlobalFile(file_store_id, os.path.join(work_dir, name))

    command = ['-T', 'SelectVariants',
               '-R', 'genome.fa',
               '-V', 'input.vcf',
               '-o', 'output.vcf',
               '-selectType', mode]

    docker_parameters = ['--rm', 'log-driver', 'none',
                         '-e', 'JAVA_OPTS=-Djava.io.tmpdir=/data/ -Xmx{}'.format(job.memory)]
    dockerCall(job=job, workDir=work_dir,
               parameters=command,
               tool='quay.io/ucsc_cgl/gatk:3.5--dba6dae49156168a909c43330350c6161dc7ecc2',
               dockerParameters=docker_parameters)

    return job.fileStore.writeGlobalFile(os.path.join(work_dir, 'output.vcf'))


def gatk_variant_filtration(job, vcf_id, filter_name, filter_expression, ref_fasta, ref_fai, ref_dict):
    """
    Filters VCF file using GATK VariantFiltration. Fixes extra pair of quotation marks in VCF header that
    may interfere with other VCF tools.

    :param JobFunctionWrappingJob job: passed automatically by Toil
    :param str vcf_id: FileStoreID for input VCF file
    :param str filter_name: Name of filter for VCF header
    :param str filter_expression: JEXL filter expression
    :param str ref_fasta: FileStoreID for reference genome fasta
    :param str ref_fai: FileStoreID for reference genome index file
    :param str ref_dict: FileStoreID for reference genome sequence dictionary file
    :return: FileStoreID for filtered VCF file
    :rtype: str
    """
    inputs = {'genome.fa': ref_fasta,
              'genome.fa.fai': ref_fai,
              'genome.dict': ref_dict,
              'input.vcf': vcf_id}

    work_dir = job.fileStore.getLocalTempDir()
    for name, file_store_id in inputs.iteritems():
        job.fileStore.readGlobalFile(file_store_id, os.path.join(work_dir, name))

    command = ['-T', 'VariantFiltration',
               '-R', 'genome.fa',
               '-V', 'input.vcf',
               '--filterName', filter_name,   # Documents filter name in header
               '--filterExpression', filter_expression,
               '-o', 'filtered_variants.vcf']

    job.fileStore.logToMaster('Running GATK VariantFiltration using {name}: '
                              '{expression}'.format(name=filter_name, expression=filter_expression))

    docker_parameters = ['--rm', 'log-driver', 'none',
                         '-e', 'JAVA_OPTS=-Djava.io.tmpdir=/data/ -Xmx{}'.format(job.memory)]
    dockerCall(job=job, workDir=work_dir,
               parameters=command,
               tool='quay.io/ucsc_cgl/gatk:3.5--dba6dae49156168a909c43330350c6161dc7ecc2',
               dockerParameters=docker_parameters)

    # Remove extra quotation marks around filter expression.
    malformed_header = os.path.join(work_dir, 'filtered_variants.vcf')
    fixed_header = os.path.join(work_dir, 'fixed_header.vcf')
    filter_regex = re.escape('"%s"' % filter_expression)
    with open(malformed_header, 'r') as f, open(fixed_header, 'w') as g:
        for line in f:
            g.write(re.sub(filter_regex, filter_expression, line))

    return job.fileStore.writeGlobalFile(fixed_header)

Filters VCF file using GATK VariantFiltration . Fixes extra pair of quotation marks in VCF header that may interfere with other VCF tools .


def gatk_variant_recalibrator(job,
                              mode,
                              vcf,
                              ref_fasta, ref_fai, ref_dict,
                              annotations,
                              hapmap=None, omni=None, phase=None, dbsnp=None, mills=None,
                              max_gaussians=4,
                              unsafe_mode=False):
    """
    Runs either SNP or INDEL variant quality score recalibration using GATK VariantRecalibrator. Because the VQSR method
    models SNPs and INDELs differently, VQSR must be run separately for these variant types.

    :param JobFunctionWrappingJob job: passed automatically by Toil
    :param str mode: Determines variant recalibration mode (SNP or INDEL)
    :param str vcf: FileStoreID for input VCF file
    :param str ref_fasta: FileStoreID for reference genome fasta
    :param str ref_fai: FileStoreID for reference genome index file
    :param str ref_dict: FileStoreID for reference genome sequence dictionary file
    :param list[str] annotations: List of GATK variant annotations to filter on
    :param str hapmap: FileStoreID for HapMap resource file, required for SNP VQSR
    :param str omni: FileStoreID for Omni resource file, required for SNP VQSR
    :param str phase: FileStoreID for 1000G resource file, required for SNP VQSR
    :param str dbsnp: FilesStoreID for dbSNP resource file, required for SNP and INDEL VQSR
    :param str mills: FileStoreID for Mills resource file, required for INDEL VQSR
    :param int max_gaussians: Number of Gaussians used during training, default is 4
    :param bool unsafe_mode: If True, runs gatk UNSAFE mode: "-U ALLOW_SEQ_DICT_INCOMPATIBILITY"
    :return: FileStoreID for the variant recalibration table, tranche file, and plots file
    :rtype: tuple
    """
    mode = mode.upper()

    inputs = {'genome.fa': ref_fasta,
              'genome.fa.fai': ref_fai,
              'genome.dict': ref_dict,
              'input.vcf': vcf}

    # Refer to GATK documentation for description of recommended parameters:
    # https://software.broadinstitute.org/gatk/documentation/article?id=1259
    # https://software.broadinstitute.org/gatk/documentation/article?id=2805

    # This base command includes parameters for both INDEL and SNP VQSR.
    command = ['-T', 'VariantRecalibrator',
               '-R', 'genome.fa',
               '-input', 'input.vcf',
               '-tranche', '100.0',
               '-tranche', '99.9',
               '-tranche', '99.0',
               '-tranche', '90.0',
               '--maxGaussians', str(max_gaussians),
               '-recalFile', 'output.recal',
               '-tranchesFile', 'output.tranches',
               '-rscriptFile', 'output.plots.R']

    # Parameters and resource files for SNP VQSR.
    if mode == 'SNP':
        command.extend(
            ['-resource:hapmap,known=false,training=true,truth=true,prior=15.0', 'hapmap.vcf',
             '-resource:omni,known=false,training=true,truth=true,prior=12.0', 'omni.vcf',
             '-resource:dbsnp,known=true,training=false,truth=false,prior=2.0', 'dbsnp.vcf',
             '-resource:1000G,known=false,training=true,truth=false,prior=10.0', '1000G.vcf',
             '-mode', 'SNP'])

        inputs['hapmap.vcf'] = hapmap
        inputs['omni.vcf'] = omni
        inputs['dbsnp.vcf'] = dbsnp
        inputs['1000G.vcf'] = phase

    # Parameters and resource files for INDEL VQSR
    elif mode == 'INDEL':
        command.extend(
            ['-resource:mills,known=false,training=true,truth=true,prior=12.0', 'mills.vcf',
             '-resource:dbsnp,known=true,training=false,truth=false,prior=2.0', 'dbsnp.vcf',
             '-mode', 'INDEL'])

        inputs['mills.vcf'] = mills
        inputs['dbsnp.vcf'] = dbsnp

    else:
        raise ValueError('Variant filter modes can be SNP or INDEL, got %s' % mode)

    for annotation in annotations:
        command.extend(['-an', annotation])

    if unsafe_mode:
        command.extend(['-U', 'ALLOW_SEQ_DICT_INCOMPATIBILITY'])

    # Delay reading in files until function is configured
    work_dir = job.fileStore.getLocalTempDir()
    for name, file_store_id in inputs.iteritems():
        job.fileStore.readGlobalFile(file_store_id, os.path.join(work_dir, name))

    job.fileStore.logToMaster('Running GATK VariantRecalibrator on {mode}s using the following annotations:\n'
                              '{annotations}'.format(mode=mode, annotations='\n'.join(annotations)))

    docker_parameters = ['--rm', 'log-driver', 'none',
                         '-e', 'JAVA_OPTS=-Djava.io.tmpdir=/data/ -Xmx{}'.format(job.memory)]
    dockerCall(job=job, workDir=work_dir,
               parameters=command,
               tool='quay.io/ucsc_cgl/gatk:3.5--dba6dae49156168a909c43330350c6161dc7ecc2',
               dockerParameters=docker_parameters)

    recal_id = job.fileStore.writeGlobalFile(os.path.join(work_dir, 'output.recal'))
    tranches_id = job.fileStore.writeGlobalFile(os.path.join(work_dir, 'output.tranches'))
    plots_id = job.fileStore.writeGlobalFile(os.path.join(work_dir, 'output.plots.R'))
    return recal_id, tranches_id, plots_id

Runs either SNP or INDEL variant quality score recalibration using GATK VariantRecalibrator . Because the VQSR method models SNPs and INDELs differently VQSR must be run separately for these variant types .


def gatk_apply_variant_recalibration(job,
                                     mode,
                                     vcf,
                                     recal_table, tranches,
                                     ref_fasta, ref_fai, ref_dict,
                                     ts_filter_level=99.0,
                                     unsafe_mode=False):
    """
    Applies variant quality score recalibration to VCF file using GATK ApplyRecalibration

    :param JobFunctionWrappingJob job: passed automatically by Toil
    :param str mode: Determines variant recalibration mode (SNP or INDEL)
    :param str vcf: FileStoreID for input VCF file
    :param str recal_table: FileStoreID for recalibration table file
    :param str tranches: FileStoreID for tranches file
    :param str ref_fasta: FileStoreID for reference genome fasta
    :param str ref_fai: FileStoreID for reference genome index file
    :param str ref_dict: FileStoreID for reference genome sequence dictionary file
    :param float ts_filter_level: Sensitivity expressed as a percentage, default is 99.0
    :param bool unsafe_mode: If True, runs gatk UNSAFE mode: "-U ALLOW_SEQ_DICT_INCOMPATIBILITY"
    :return: FileStoreID for recalibrated VCF file
    :rtype: str
    """
    inputs = {'genome.fa': ref_fasta,
              'genome.fa.fai': ref_fai,
              'genome.dict': ref_dict,
              'input.vcf': vcf,
              'recal': recal_table,
              'tranches': tranches}

    work_dir = job.fileStore.getLocalTempDir()
    for name, file_store_id in inputs.iteritems():
        job.fileStore.readGlobalFile(file_store_id, os.path.join(work_dir, name))

    mode = mode.upper()

    # GATK recommended parameters:
    # https://software.broadinstitute.org/gatk/documentation/article?id=2805
    command = ['-T', 'ApplyRecalibration',
               '-mode', mode,
               '-R', 'genome.fa',
               '-input', 'input.vcf',
               '-o', 'vqsr.vcf',
               '-ts_filter_level', str(ts_filter_level),
               '-recalFile', 'recal',
               '-tranchesFile', 'tranches']

    if unsafe_mode:
        command.extend(['-U', 'ALLOW_SEQ_DICT_INCOMPATIBILITY'])

    job.fileStore.logToMaster('Running GATK ApplyRecalibration on {mode}s '
                              'with a sensitivity of {sensitivity}%'.format(mode=mode,
                                                                            sensitivity=ts_filter_level))
    docker_parameters = ['--rm', 'log-driver', 'none',
                         '-e', 'JAVA_OPTS=-Djava.io.tmpdir=/data/ -Xmx{}'.format(job.memory)]
    dockerCall(job=job, workDir=work_dir,
               parameters=command,
               tool='quay.io/ucsc_cgl/gatk:3.5--dba6dae49156168a909c43330350c6161dc7ecc2',
               dockerParameters=docker_parameters)

    return job.fileStore.writeGlobalFile(os.path.join(work_dir, 'vqsr.vcf'))


def gatk_combine_variants(job, vcfs, ref_fasta, ref_fai, ref_dict, merge_option='UNIQUIFY'):
    """
    Merges VCF files using GATK CombineVariants

    :param JobFunctionWrappingJob job: Toil Job instance
    :param dict vcfs: Dictionary of VCF FileStoreIDs {sample identifier: FileStoreID}
    :param str ref_fasta: FileStoreID for reference genome fasta
    :param str ref_fai: FileStoreID for reference genome index file
    :param str ref_dict: FileStoreID for reference genome sequence dictionary file
    :param str merge_option: Value for --genotypemergeoption flag (Default: 'UNIQUIFY')
                            'UNIQUIFY': Multiple variants at a single site are merged into a
                                        single variant record.
                            'UNSORTED': Used to merge VCFs from the same sample
    :return: FileStoreID for merged VCF file
    :rtype: str
    """
    job.fileStore.logToMaster('Running GATK CombineVariants')

    inputs = {'genome.fa': ref_fasta,
              'genome.fa.fai': ref_fai,
              'genome.dict': ref_dict}
    inputs.update(vcfs)

    work_dir = job.fileStore.getLocalTempDir()
    for name, file_store_id in inputs.iteritems():
        job.fileStore.readGlobalFile(file_store_id, os.path.join(work_dir, name))

    command = ['-T', 'CombineVariants',
               '-R', '/data/genome.fa',
               '-o', '/data/merged.vcf',
               '--genotypemergeoption', merge_option]

    for uuid, vcf_id in vcfs.iteritems():
        command.extend(['--variant', os.path.join('/data', uuid)])

    docker_parameters = ['--rm', 'log-driver', 'none',
                         '-e', 'JAVA_OPTS=-Djava.io.tmpdir=/data/ -Xmx{}'.format(job.memory)]
    dockerCall(job=job, workDir=work_dir,
               parameters=command,
               tool='quay.io/ucsc_cgl/gatk:3.5--dba6dae49156168a909c43330350c6161dc7ecc2',
               dockerParameters=docker_parameters)

    return job.fileStore.writeGlobalFile(os.path.join(work_dir, 'merged.vcf'))


def bam_quickcheck(bam_path):
    """
    Perform a quick check on a BAM via `samtools quickcheck`.
    This will detect obvious BAM errors such as truncation.

    :param str bam_path: path to BAM file to checked

    :rtype: boolean
    :return: True if the BAM is valid, False is BAM is invalid or something related to the call went wrong
    """
    directory, bam_name = os.path.split(bam_path)
    exit_code = subprocess.call(['docker', 'run', '-v', directory + ':/data',
                                 'quay.io/ucsc_cgl/samtools:1.3--256539928ea162949d8a65ca5c79a72ef557ce7c',
                                 'quickcheck', '-vv', '/data/' + bam_name])
    if exit_code != 0:
        return False
    return True

Perform a quick check on a BAM via samtools quickcheck . This will detect obvious BAM errors such as truncation .


def load_handlers(handler_mapping):
    """
    Given a dictionary mapping which looks like the following, import the
    objects based on the dotted path and yield the packet type and handler as
    pairs.

    If the special string '*' is passed, don't process that, pass it on as it
    is a wildcard.

    If an non-string object is given for either packet or handler (key or
    value) assume these are the objects to use and yield them.

    ::
        {
        'rfxcom.protocol.Status': 'home.collect.logging_handler',
        'rfxcom.protocol.Elec': 'home.collect.elec_handler',
        'rfxcom.protocol.TempHumidity': 'home.collect.temp_humidity_handler',
        '*': 'home.collect.logging_handler'
        }
    """

    handlers = {}

    for packet_type, handler in handler_mapping.items():

        if packet_type == '*':
            Packet = packet_type
        elif isinstance(packet_type, str):
            Packet = importer(packet_type)
        else:
            Packet = packet_type

        if isinstance(handler, str):
            Handler = importer(handler)
        else:
            Handler = handler

        if Packet in handlers:
            raise HandlerConfigError(
                "Handler already provided for packet %s" % Packet)

        handlers[Packet] = Handler

    return handlers

Given a dictionary mapping which looks like the following import the objects based on the dotted path and yield the packet type and handler as pairs .


def write_config(configuration):
    """Helper to write the JSON configuration to a file"""
    with open(CONFIG_PATH, 'w') as f:
        json.dump(configuration, f, indent=2, sort_keys=True)

def write_config ( configuration ) : with open ( CONFIG_PATH , 'w' ) as f : json . dump ( configuration , f , indent = 2 , sort_keys = True )


def get_config():
    """Gets the configuration for this project from the default JSON file, or writes one if it doesn't exist

    :rtype: dict
    """
    if not os.path.exists(CONFIG_PATH):
        write_config({})

    with open(CONFIG_PATH) as f:
        return json.load(f)

def get_config ( ) : if not os . path . exists ( CONFIG_PATH ) : write_config ( { } ) with open ( CONFIG_PATH ) as f : return json . load ( f )

Gets the configuration for this project from the default JSON file or writes one if it doesn t exist


def get_term(self, ontology, iri):
        """Gets the data for a given term

        :param str ontology: The name of the ontology
        :param str iri: The IRI of a term
        :rtype: dict
        """
        url = self.ontology_term_fmt.format(ontology, iri)
        response = requests.get(url)

        return response.json()

def get_term ( self , ontology , iri ) : url = self . ontology_term_fmt . format ( ontology , iri ) response = requests . get ( url ) return response . json ( )


def search(self, name, query_fields=None):
        """Searches the OLS with the given term

        :param str name:
        :param list[str] query_fields: Fields to query
        :return: dict
        """
        params = {'q': name}
        if query_fields is not None:
            params['queryFields'] = '{{{}}}'.format(','.join(query_fields))
        response = requests.get(self.ontology_search, params=params)

        return response.json()

def search ( self , name , query_fields = None ) : params = { 'q' : name } if query_fields is not None : params [ 'queryFields' ] = '{{{}}}' . format ( ',' . join ( query_fields ) ) response = requests . get ( self . ontology_search , params = params ) return response . json ( )


def suggest(self, name, ontology=None):
        """Suggest terms from an optional list of ontologies

        :param str name:
        :param list[str] ontology:
        :rtype: dict

        .. seealso:: https://www.ebi.ac.uk/ols/docs/api#_suggest_term
        """
        params = {'q': name}
        if ontology:
            params['ontology'] = ','.join(ontology)
        response = requests.get(self.ontology_suggest, params=params)

        return response.json()

def suggest ( self , name , ontology = None ) : params = { 'q' : name } if ontology : params [ 'ontology' ] = ',' . join ( ontology ) response = requests . get ( self . ontology_suggest , params = params ) return response . json ( )


def iter_descendants(self, ontology, iri, size=None, sleep=None):
        """Iterates over the descendants of a given term

        :param str ontology: The name of the ontology
        :param str iri: The IRI of a term
        :param int size: The size of each page. Defaults to 500, which is the maximum allowed by the EBI.
        :param int sleep: The amount of time to sleep between pages. Defaults to 0 seconds.
        :rtype: iter[dict]
        """
        url = self.ontology_term_descendants_fmt.format(ontology=ontology, iri=iri)
        log.info('getting %s', url)
        for term in self._iter_terms_helper(url, size=size, sleep=sleep):
            yield term

def
iter_descendants
(
self
,
ontology
,
iri
,
size
=
,
sleep
=
)
:
url
=
self
.
ontology_term_descendants_fmt
.
format
(
ontology
=
ontology
,
iri
=
iri
)
log
.
info
(
'getting %s'
,
url
)
for
term
in
self
.
_iter_terms_helper
(
url
,
size
=
size
,
sleep
=
sleep
)
:
yield
term


def iter_descendants_labels(self, ontology, iri, size=None, sleep=None):
        """Iterates over the labels for the descendants of a given term

        :param str ontology: The name of the ontology
        :param str iri: The IRI of a term
        :param int size: The size of each page. Defaults to 500, which is the maximum allowed by the EBI.
        :param int sleep: The amount of time to sleep between pages. Defaults to 0 seconds.
        :rtype: iter[str]
        """
        for label in _help_iterate_labels(self.iter_descendants(ontology, iri, size=size, sleep=sleep)):
            yield label

def iter_descendants_labels ( self , ontology , iri , size = None , sleep = None ) : for label in _help_iterate_labels ( self . iter_descendants ( ontology , iri , size = size , sleep = sleep ) ) : yield label


def iter_labels(self, ontology, size=None, sleep=None):
        """Iterates over the labels of terms in the ontology. Automatically wraps the pager returned by the OLS.

        :param str ontology: The name of the ontology
        :param int size: The size of each page. Defaults to 500, which is the maximum allowed by the EBI.
        :param int sleep: The amount of time to sleep between pages. Defaults to 0 seconds.
        :rtype: iter[str]
        """
        for label in _help_iterate_labels(self.iter_terms(ontology=ontology, size=size, sleep=sleep)):
            yield label

def iter_labels ( self , ontology , size = None , sleep = None ) : for label in _help_iterate_labels ( self . iter_terms ( ontology = ontology , size = size , sleep = sleep ) ) : yield label

Iterates over the labels of terms in the ontology . Automatically wraps the pager returned by the OLS .


def iter_hierarchy(self, ontology, size=None, sleep=None):
        """Iterates over parent-child relations

        :param str ontology: The name of the ontology
        :param int size: The size of each page. Defaults to 500, which is the maximum allowed by the EBI.
        :param int sleep: The amount of time to sleep between pages. Defaults to 0 seconds.
        :rtype: iter[tuple[str,str]]
        """
        for term in self.iter_terms(ontology=ontology, size=size, sleep=sleep):
            try:
                hierarchy_children_link = term['_links'][HIERARCHICAL_CHILDREN]['href']
            except KeyError:  # there's no children for this one
                continue

            response = requests.get(hierarchy_children_link).json()

            for child_term in response['_embedded']['terms']:
                yield term['label'], child_term['label']

def
iter_hierarchy
(
self
,
ontology
,
size
=
,
sleep
=
)
:
for
term
in
self
.
iter_terms
(
ontology
=
ontology
,
size
=
size
,
sleep
=
sleep
)
:
try
:
hierarchy_children_link
=
term
[
'_links'
]
[
HIERARCHICAL_CHILDREN
]
[
'href'
]
except
KeyError
:
# there's no children for this one
continue
response
=
requests
.
get
(
hierarchy_children_link
)
.
json
(
)
for
child_term
in
response
[
'_embedded'
]
[
'terms'
]
:
yield
term
[
'label'
]
,
child_term
[
'label'
]


def run_fastqc(job, r1_id, r2_id):
    """
    Run Fastqc on the input reads

    :param JobFunctionWrappingJob job: passed automatically by Toil
    :param str r1_id: FileStoreID of fastq read 1
    :param str r2_id: FileStoreID of fastq read 2
    :return: FileStoreID of fastQC output (tarball)
    :rtype: str
    """
    work_dir = job.fileStore.getLocalTempDir()
    job.fileStore.readGlobalFile(r1_id, os.path.join(work_dir, 'R1.fastq'))
    parameters = ['/data/R1.fastq']
    output_names = ['R1_fastqc.html', 'R1_fastqc.zip']
    if r2_id:
        job.fileStore.readGlobalFile(r2_id, os.path.join(work_dir, 'R2.fastq'))
        parameters.extend(['-t', '2', '/data/R2.fastq'])
        output_names.extend(['R2_fastqc.html', 'R2_fastqc.zip'])
    dockerCall(job=job, tool='quay.io/ucsc_cgl/fastqc:0.11.5--be13567d00cd4c586edf8ae47d991815c8c72a49',
               workDir=work_dir, parameters=parameters)
    output_files = [os.path.join(work_dir, x) for x in output_names]
    tarball_files(tar_name='fastqc.tar.gz', file_paths=output_files, output_dir=work_dir)
    return job.fileStore.writeGlobalFile(os.path.join(work_dir, 'fastqc.tar.gz'))


def addStream(self, stream, t1=None, t2=None, limit=None, i1=None, i2=None, transform=None):
        """Adds the given stream to the query construction. The function supports both stream
        names and Stream objects."""
        params = query_maker(t1, t2, limit, i1, i2, transform)

        params["stream"] = get_stream(self.cdb, stream)

        # Now add the stream to the query parameters
        self.query.append(params)

Adds the given stream to the query construction . The function supports both stream names and Stream objects .


def create_app(config=None):
    """ This needs some tidying up. To avoid circular imports we import
    everything here but it makes this method a bit more gross.
    """

    # Initialise the app
    from home.config import TEMPLATE_FOLDER, STATIC_FOLDER
    app = Flask(__name__, static_folder=STATIC_FOLDER,
                template_folder=TEMPLATE_FOLDER)

    app.config['SECRET_KEY'] = 'ssh, its a secret.'

    # Load the default config, the specified config file and then any
    # overwrites that are manually passed in.
    app.config.from_object('home.config')

    if 'HOME_SETTINGS' in environ:
        app.config.from_envvar('HOME_SETTINGS')

    app.config.from_object(config)

    # Register the web front end and the API.
    from home.dash.web import web
    from home.dash.api import api
    app.register_blueprint(web)
    app.register_blueprint(api, url_prefix='/api')

    login_manager.init_app(app)
    login_manager.login_view = 'Dashboard Web.login'

    from home.dash.models import User

    @login_manager.user_loader
    def load_user(user_id):
        return User.query.get(int(user_id))

    # Initialise the migrations app, we want to store all migrations within
    # the project directory for easier packaging.
    Migrate(app, db, directory=app.config['MIGRATE_DIRECTORY'])

    admin = Admin(app)

    from home.dash.admin import setup_admin
    setup_admin(admin)

    # Wire up the database to the app so it gets the config.
    db.init_app(app)

    return app

This needs some tidying up . To avoid circular imports we import everything here but it makes this method a bit more gross .


def start(self, job):
        """
        Start spark and hdfs master containers

        :param job: The underlying job.
        """

        if self.hostname is None:
            self.hostname = subprocess.check_output(["hostname", "-f",])[:-1]

        _log.info("Started Spark master container.")
        self.sparkContainerID = dockerCheckOutput(job=job,
                                                  defer=STOP,
                                                  workDir=os.getcwd(),
                                                  tool="quay.io/ucsc_cgl/apache-spark-master:1.5.2",
                                                  dockerParameters=["--net=host",
                                                                    "-d",
                                                                    "-v", "/mnt/ephemeral/:/ephemeral/:rw",
                                                                    "-e", "SPARK_MASTER_IP=" + self.hostname,
                                                                    "-e", "SPARK_LOCAL_DIRS=/ephemeral/spark/local",
                                                                    "-e", "SPARK_WORKER_DIR=/ephemeral/spark/work"],
                                                  parameters=[self.hostname])[:-1]
        _log.info("Started HDFS Datanode.")
        self.hdfsContainerID = dockerCheckOutput(job=job,
                                           defer=STOP,
                                           workDir=os.getcwd(),
                                           tool="quay.io/ucsc_cgl/apache-hadoop-master:2.6.2",
                                           dockerParameters=["--net=host",
                                                              "-d"],
                                           parameters=[self.hostname])[:-1]

        return self.hostname


def start(self, job):
        """
        Start spark and hdfs worker containers

        :param job: The underlying job.
        """

        # start spark and our datanode
        self.sparkContainerID = dockerCheckOutput(job=job,
                                                  defer=STOP,
                                                  workDir=os.getcwd(),
                                                  tool="quay.io/ucsc_cgl/apache-spark-worker:1.5.2",
                                                  dockerParameters=["--net=host",
                                                                    "-d",
                                                                    "-v", "/mnt/ephemeral/:/ephemeral/:rw",
                                                                    "-e",
                                                                    "\"SPARK_MASTER_IP=" + self.masterIP + ":" + _SPARK_MASTER_PORT + "\"",
                                                                    "-e", "SPARK_LOCAL_DIRS=/ephemeral/spark/local",
                                                                    "-e", "SPARK_WORKER_DIR=/ephemeral/spark/work"],
                                                  parameters=[self.masterIP + ":" + _SPARK_MASTER_PORT])[:-1]
        self.__start_datanode(job)

        # fake do/while to check if HDFS is up
        hdfs_down = True
        retries = 0
        while hdfs_down and (retries < 5):

            _log.info("Sleeping 30 seconds before checking HDFS startup.")
            time.sleep(30)
            clusterID = ""
            try:
                clusterID = subprocess.check_output(["docker",
                                                     "exec",
                                                     self.hdfsContainerID,
                                                     "grep",
                                                     "clusterID",
                                                     "-R",
                                                     "/opt/apache-hadoop/logs"])
            except:
                # grep returns a non-zero exit code if the pattern is not found
                # we expect to not find the pattern, so a non-zero code is OK
                pass

            if "Incompatible" in clusterID:
                _log.warning("Hadoop Datanode failed to start with: %s", clusterID)
                _log.warning("Retrying container startup, retry #%d.", retries)
                retries += 1

                _log.warning("Removing ephemeral hdfs directory.")
                subprocess.check_call(["docker",
                                       "exec",
                                       self.hdfsContainerID,
                                       "rm",
                                       "-rf",
                                       "/ephemeral/hdfs"])

                _log.warning("Killing container %s.", self.hdfsContainerID)
                subprocess.check_call(["docker",
                                       "kill",
                                       self.hdfsContainerID])

                # todo: this is copied code. clean up!
                _log.info("Restarting datanode.")
                self.__start_datanode(job)

            else:
                _log.info("HDFS datanode started up OK!")
                hdfs_down = False

        if retries >= 5:
            raise RuntimeError("Failed %d times trying to start HDFS datanode." % retries)

        return


def __start_datanode(self, job):
        """
        Launches the Hadoop datanode.

        :param job: The underlying job.
        """
        self.hdfsContainerID = dockerCheckOutput(job=job,
                                                 defer=STOP,
                                                 workDir=os.getcwd(),
                                                 tool="quay.io/ucsc_cgl/apache-hadoop-worker:2.6.2",
                                                 dockerParameters=["--net=host",
                                                                    "-d",
                                                                    "-v", "/mnt/ephemeral/:/ephemeral/:rw"],
                                                 parameters=[self.masterIP])[:-1]


def stop(self, fileStore):
        """
        Stop spark and hdfs worker containers

        :param job: The underlying job.
        """

        subprocess.call(["docker", "exec", self.sparkContainerID, "rm", "-r", "/ephemeral/spark"])
        subprocess.call(["docker", "stop", self.sparkContainerID])
        subprocess.call(["docker", "rm", self.sparkContainerID])
        _log.info("Stopped Spark worker.")

        subprocess.call(["docker", "exec", self.hdfsContainerID, "rm", "-r", "/ephemeral/hdfs"])
        subprocess.call(["docker", "stop", self.hdfsContainerID])
        subprocess.call(["docker", "rm", self.hdfsContainerID])
        _log.info("Stopped HDFS datanode.")

        return

def
stop
(
self
,
fileStore
)
:
subprocess
.
call
(
[
"docker"
,
"exec"
,
self
.
sparkContainerID
,
"rm"
,
"-r"
,
"/ephemeral/spark"
]
)
subprocess
.
call
(
[
"docker"
,
"stop"
,
self
.
sparkContainerID
]
)
subprocess
.
call
(
[
"docker"
,
"rm"
,
self
.
sparkContainerID
]
)
_log
.
info
(
"Stopped Spark worker."
)
subprocess
.
call
(
[
"docker"
,
"exec"
,
self
.
hdfsContainerID
,
"rm"
,
"-r"
,
"/ephemeral/hdfs"
]
)
subprocess
.
call
(
[
"docker"
,
"stop"
,
self
.
hdfsContainerID
]
)
subprocess
.
call
(
[
"docker"
,
"rm"
,
self
.
hdfsContainerID
]
)
_log
.
info
(
"Stopped HDFS datanode."
)
return


def check(self):
        """
        Checks to see if Spark worker and HDFS datanode are still running.
        """

        status = _checkContainerStatus(self.sparkContainerID,
                                       self.hdfsContainerID,
                                       sparkNoun='worker',
                                       hdfsNoun='datanode')

        return status

def check ( self ) : status = _checkContainerStatus ( self . sparkContainerID , self . hdfsContainerID , sparkNoun = 'worker' , hdfsNoun = 'datanode' ) return status

Checks to see if Spark worker and HDFS datanode are still running .


def base_tokenizer(fp):
    'Tokenizer. Generates tokens stream from text'
    if isinstance(fp, StringIO):
        template_file = fp
        size = template_file.len
    else:
        #empty file check
        if os.fstat(fp.fileno()).st_size == 0:
            yield TOKEN_EOF, 'EOF', 0, 0
            return
        template_file = mmap.mmap(fp.fileno(), 0, access=mmap.ACCESS_READ)
        size = template_file.size()
    lineno = 0
    while 1:
        lineno += 1
        pos = 1

        # end of file
        if template_file.tell() == size:
            yield TOKEN_EOF, 'EOF', lineno, 0
            break

        # now we tokinize line by line
        line = template_file.readline().decode('utf-8')
        line = line.replace('\r\n', '')
        line = line.replace('\n', '')
        # ignoring non XML comments
        if re_comment.match(line):
            continue

        last_text = deque()
        while line:
            line_len = len(line)
            for token in tokens:
                m = token.regex.match(line)
                if m:
                    if last_text:
                        yield TOKEN_TEXT, ''.join(last_text), lineno, pos
                        pos += len(last_text)
                        last_text.clear()
                    offset, value = m.end(), m.group()
                    line = line[offset:]
                    yield token, value, lineno, pos
                    pos += offset
                    break

            # we did not get right in tokens list, so next char is text
            if line_len == len(line):
                last_text.append(line[0])
                line = line[1:]

        if last_text:
            yield TOKEN_TEXT, ''.join(last_text), lineno, pos
            pos += len(last_text)
            last_text.clear()
        yield TOKEN_NEWLINE, '\n', lineno, pos

    # all work is done
    template_file.close()

Tokenizer . Generates tokens stream from text


def lookup_zone(conn, zone):
  """Look up a zone ID for a zone string.

  Args: conn: boto.route53.Route53Connection
        zone: string eg. foursquare.com
  Returns: zone ID eg. ZE2DYFZDWGSL4.
  Raises: ZoneNotFoundError if zone not found."""
  all_zones = conn.get_all_hosted_zones()
  for resp in all_zones['ListHostedZonesResponse']['HostedZones']:
    if resp['Name'].rstrip('.') == zone.rstrip('.'):
      return resp['Id'].replace('/hostedzone/', '')
  raise ZoneNotFoundError('zone %s not found in response' % zone)

def
lookup_zone
(
conn
,
zone
)
:
all_zones
=
conn
.
get_all_hosted_zones
(
)
for
resp
in
all_zones
[
'ListHostedZonesResponse'
]
[
'HostedZones'
]
:
if
resp
[
'Name'
]
.
rstrip
(
'.'
)
==
zone
.
rstrip
(
'.'
)
:
return
resp
[
'Id'
]
.
replace
(
'/hostedzone/'
,
''
)
raise
ZoneNotFoundError
(
'zone %s not found in response'
%
zone
)


def fetch_config(zone, conn):
  """Fetch all pieces of a Route 53 config from Amazon.

  Args: zone: string, hosted zone id.
        conn: boto.route53.Route53Connection
  Returns: list of ElementTrees, one for each piece of config."""
  more_to_fetch = True
  cfg_chunks = []
  next_name = None
  next_type = None
  next_identifier = None
  while more_to_fetch == True:
    more_to_fetch = False
    getstr = '/%s/hostedzone/%s/rrset' % (R53_API_VERSION, zone)
    if next_name is not None:
      getstr += '?name=%s&type=%s' % (next_name, next_type)
      if next_identifier is not None:
        getstr += '&identifier=%s' % next_identifier
    log.debug('requesting %s' % getstr)
    resp = conn.make_request('GET', getstr)
    etree = lxml.etree.parse(resp)
    cfg_chunks.append(etree)
    root = etree.getroot()
    truncated = root.find('{%s}IsTruncated' % R53_XMLNS)
    if truncated is not None and truncated.text == 'true':
      more_to_fetch = True
      next_name = root.find('{%s}NextRecordName' % R53_XMLNS).text
      next_type = root.find('{%s}NextRecordType' % R53_XMLNS).text
      try:
        next_identifier = root.find('{%s}NextRecordIdentifier' % R53_XMLNS).text
      except AttributeError:  # may not have next_identifier
        next_identifier = None
  return cfg_chunks

def
fetch_config
(
zone
,
conn
)
:
more_to_fetch
=
True
cfg_chunks
=
[
]
next_name
=
next_type
=
next_identifier
=
while
more_to_fetch
==
True
:
more_to_fetch
=
False
getstr
=
'/%s/hostedzone/%s/rrset'
%
(
R53_API_VERSION
,
zone
)
if
next_name
is
not
:
getstr
+=
'?name=%s&type=%s'
%
(
next_name
,
next_type
)
if
next_identifier
is
not
:
getstr
+=
'&identifier=%s'
%
next_identifier
log
.
debug
(
'requesting %s'
%
getstr
)
resp
=
conn
.
make_request
(
'GET'
,
getstr
)
etree
=
lxml
.
etree
.
parse
(
resp
)
cfg_chunks
.
append
(
etree
)
root
=
etree
.
getroot
(
)
truncated
=
root
.
find
(
'{%s}IsTruncated'
%
R53_XMLNS
)
if
truncated
is
not
and
truncated
.
text
==
'true'
:
more_to_fetch
=
True
next_name
=
root
.
find
(
'{%s}NextRecordName'
%
R53_XMLNS
)
.
text
next_type
=
root
.
find
(
'{%s}NextRecordType'
%
R53_XMLNS
)
.
text
try
:
next_identifier
=
root
.
find
(
'{%s}NextRecordIdentifier'
%
R53_XMLNS
)
.
text
except
AttributeError
:
# may not have next_identifier
next_identifier
=
return
cfg_chunks

Fetch all pieces of a Route 53 config from Amazon .


def merge_config(cfg_chunks):
  """Merge a set of fetched Route 53 config Etrees into a canonical form.

  Args: cfg_chunks: [ lxml.etree.ETree ]
  Returns: lxml.etree.Element"""
  root = lxml.etree.XML('<ResourceRecordSets xmlns="%s"></ResourceRecordSets>' % R53_XMLNS, parser=XML_PARSER)
  for chunk in cfg_chunks:
    for rrset in chunk.iterfind('.//{%s}ResourceRecordSet' % R53_XMLNS):
      root.append(rrset)
  return root

Merge a set of fetched Route 53 config Etrees into a canonical form .


def validate_changeset(changeset):
  """Validate a changeset is compatible with Amazon's API spec.

  Args: changeset: lxml.etree.Element (<ChangeResourceRecordSetsRequest>)
  Returns: [ errors ] list of error strings or []."""
  errors = []
  changes = changeset.findall('.//{%s}Change' % R53_XMLNS)
  num_changes = len(changes)
  if num_changes == 0:
    errors.append('changeset must have at least one <Change> element')
  if num_changes > 100:
    errors.append('changeset has %d <Change> elements: max is 100' % num_changes)
  rrs = changeset.findall('.//{%s}ResourceRecord' % R53_XMLNS)
  num_rrs = len(rrs)
  if num_rrs > 1000:
    errors.append('changeset has %d ResourceRecord elements: max is 1000' % num_rrs)
  values = changeset.findall('.//{%s}Value' % R53_XMLNS)
  num_chars = 0
  for value in values:
    num_chars += len(value.text)
  if num_chars > 10000:
    errors.append('changeset has %d chars in <Value> text: max is 10000' % num_chars)
  return errors

Validate a changeset is compatible with Amazon s API spec .


def minimize_best_n(Members):
    '''
    Orders population members from lowest fitness to highest fitness

    Args:
        Members (list): list of PyGenetics Member objects

    Returns:
        lsit: ordered lsit of Members, from highest fitness to lowest fitness
    '''

    return(list(reversed(sorted(
        Members, key=lambda Member: Member.fitness_score
    ))))

def minimize_best_n ( Members ) : return ( list ( reversed ( sorted ( Members , key = lambda Member : Member . fitness_score ) ) ) )


def fitness(self):
        '''Population fitness == average member fitness score'''

        if len(self.__members) != 0:
            if self.__num_processes > 1:
                members = [m.get() for m in self.__members]
            else:
                members = self.__members
            return sum(m.fitness_score for m in members) / len(members)
        else:
            return None

def fitness ( self ) : if len ( self . __members ) != 0 : if self . __num_processes > 1 : members = [ m . get ( ) for m in self . __members ] else : members = self . __members return sum ( m . fitness_score for m in members ) / len ( members ) else : return None


def ave_cost_fn_val(self):
        '''Returns average cost function return value for all members'''

        if len(self.__members) != 0:
            if self.__num_processes > 1:
                members = [m.get() for m in self.__members]
            else:
                members = self.__members
            return sum(m.cost_fn_val for m in members) / len(members)
        else:
            return None

def ave_cost_fn_val ( self ) : if len ( self . __members ) != 0 : if self . __num_processes > 1 : members = [ m . get ( ) for m in self . __members ] else : members = self . __members return sum ( m . cost_fn_val for m in members ) / len ( members ) else : return None


def med_cost_fn_val(self):
        '''Returns median cost function return value for all members'''

        if len(self.__members) != 0:
            if self.__num_processes > 1:
                members = [m.get() for m in self.__members]
            else:
                members = self.__members
            return median([m.cost_fn_val for m in members])
        else:
            return None

def med_cost_fn_val ( self ) : if len ( self . __members ) != 0 : if self . __num_processes > 1 : members = [ m . get ( ) for m in self . __members ] else : members = self . __members return median ( [ m . cost_fn_val for m in members ] ) else : return None


def parameters(self):
        '''Population parameter vals == average member parameter vals'''

        if len(self.__members) != 0:
            if self.__num_processes > 1:
                members = [m.get() for m in self.__members]
            else:
                members = self.__members
            params = {}
            for p in self.__parameters:
                params[p.name] = sum(
                    m.parameters[p.name] for m in members
                ) / len(members)
            return params
        else:
            return None

def parameters ( self ) : if len ( self . __members ) != 0 : if self . __num_processes > 1 : members = [ m . get ( ) for m in self . __members ] else : members = self . __members params = { } for p in self . __parameters : params [ p . name ] = sum ( m . parameters [ p . name ] for m in members ) / len ( members ) return params else : return None


def members(self):
        '''Returns Member objects of population'''

        if self.__num_processes > 1:
            return [m.get() for m in self.__members]
        else:
            return self.__members

def members ( self ) : if self . __num_processes > 1 : return [ m . get ( ) for m in self . __members ] else : return self . __members


def add_parameter(self, name, min_val, max_val):
        '''Adds a paramber to the Population

        Args:
            name (str): name of the parameter
            min_val (int or float): minimum value for the parameter
            max_val (int or float): maximum value for the parameter
        '''

        self.__parameters.append(Parameter(name, min_val, max_val))

def add_parameter ( self , name , min_val , max_val ) : self . __parameters . append ( Parameter ( name , min_val , max_val ) )


def next_generation(self, mut_rate=0, max_mut_amt=0, log_base=10):
        '''Generates the next population from a previously evaluated generation

        Args:
            mut_rate (float): mutation rate for new members (0.0 - 1.0)
            max_mut_amt (float): how much the member is allowed to mutate
                (0.0 - 1.0, proportion change of mutated parameter)
            log_base (int): the higher this number, the more likely the first
                Members (chosen with supplied selection function) are chosen
                as parents for the next generation
        '''

        if self.__num_processes > 1:
            process_pool = Pool(processes=self.__num_processes)
            members = [m.get() for m in self.__members]
        else:
            members = self.__members

        if len(members) == 0:
            raise Exception(
                'Generation 0 not found: use generate_population() first'
            )

        selected_members = self.__select_fn(members)
        reproduction_probs = list(reversed(logspace(0.0, 1.0,
                                  num=len(selected_members), base=log_base)))
        reproduction_probs = reproduction_probs / sum(reproduction_probs)

        self.__members = []

        for _ in range(self.__pop_size):
            parent_1 = nrandom.choice(selected_members, p=reproduction_probs)
            parent_2 = nrandom.choice(selected_members, p=reproduction_probs)

            feed_dict = {}
            for param in self.__parameters:
                which_parent = uniform(0, 1)
                if which_parent < 0.5:
                    feed_dict[param.name] = parent_1.parameters[param.name]
                else:
                    feed_dict[param.name] = parent_2.parameters[param.name]
                feed_dict[param.name] = self.__mutate_parameter(
                    feed_dict[param.name], param, mut_rate, max_mut_amt
                )

            if self.__num_processes > 1:
                self.__members.append(process_pool.apply_async(
                    self._start_process,
                    [self.__cost_fn, feed_dict, self.__cost_fn_args])
                )
            else:
                self.__members.append(
                    Member(
                        feed_dict,
                        self.__cost_fn(feed_dict, self.__cost_fn_args)
                    )
                )

        if self.__num_processes > 1:
            process_pool.close()
            process_pool.join()

        self.__determine_best_member()


def normalize_keys(self, items):
        """Return a config dictionary with normalized keys regardless of
        whether the keys were specified in environment variables or in config
        files"""
        normalized = {}
        for key, val in items:
            key = key.replace('_', '-')
            if not key.startswith('--'):
                key = '--%s' % key  # only prefer long opts
            normalized[key] = val
        return normalized

def
normalize_keys
(
self
,
items
)
:
normalized
=
{
}
for
key
,
val
in
items
:
key
=
key
.
replace
(
'_'
,
'-'
)
if
not
key
.
startswith
(
'--'
)
:
key
=
'--%s'
%
key
# only prefer long opts
normalized
[
key
]
=
val
return
normalized


def get_environ_vars(self):
        """Returns a generator with all environmental vars with prefix PIP_"""
        for key, val in os.environ.items():
            if _environ_prefix_re.search(key):
                yield (_environ_prefix_re.sub("", key).lower(), val)

def get_environ_vars ( self ) : for key , val in os . environ . items ( ) : if _environ_prefix_re . search ( key ) : yield ( _environ_prefix_re . sub ( "" , key ) . lower ( ) , val )


def throws_exception(callable, *exceptions):
	"""
	Return True if the callable throws the specified exception

	>>> throws_exception(lambda: int('3'))
	False
	>>> throws_exception(lambda: int('a'))
	True
	>>> throws_exception(lambda: int('a'), KeyError)
	False
	"""
	with context.ExceptionTrap():
		with context.ExceptionTrap(*exceptions) as exc:
			callable()
	return bool(exc)

def throws_exception ( callable , * exceptions ) : with context . ExceptionTrap ( ) : with context . ExceptionTrap ( * exceptions ) as exc : callable ( ) return bool ( exc )


def transform_hits(hits):
    """
    The list from pypi is really a list of versions. We want a list of
    packages with the list of versions stored inline. This converts the
    list from pypi into one we can use.
    """
    packages = {}
    for hit in hits:
        name = hit['name']
        summary = hit['summary']
        version = hit['version']
        score = hit['_pypi_ordering']
        if score is None:
            score = 0

        if name not in packages.keys():
            packages[name] = {
                'name': name,
                'summary': summary,
                'versions': [version],
                'score': score,
            }
        else:
            packages[name]['versions'].append(version)

            # if this is the highest version, replace summary and score
            if version == highest_version(packages[name]['versions']):
                packages[name]['summary'] = summary
                packages[name]['score'] = score

    # each record has a unique name now, so we will convert the dict into a
    # list sorted by score
    package_list = sorted(
        packages.values(),
        key=lambda x: x['score'],
        reverse=True,
    )
    return package_list

The list from pypi is really a list of versions . We want a list of packages with the list of versions stored inline . This converts the list from pypi into one we can use .


def _transform_result(typ, result):
    """Convert the result back into the input type.
    """
    if issubclass(typ, bytes):
        return tostring(result, encoding='utf-8')
    elif issubclass(typ, unicode):
        return tostring(result, encoding='unicode')
    else:
        return result

def _transform_result ( typ , result ) : if issubclass ( typ , bytes ) : return tostring ( result , encoding = 'utf-8' ) elif issubclass ( typ , unicode ) : return tostring ( result , encoding = 'unicode' ) else : return result

Convert the result back into the input type .


def html_to_xhtml(html):
    """Convert all tags in an HTML tree to XHTML by moving them to the
    XHTML namespace.
    """
    try:
        html = html.getroot()
    except AttributeError:
        pass
    prefix = "{%s}" % XHTML_NAMESPACE
    for el in html.iter(etree.Element):
        tag = el.tag
        if tag[0] != '{':
            el.tag = prefix + tag

def html_to_xhtml ( html ) : try : html = html . getroot ( ) except AttributeError : pass prefix = "{%s}" % XHTML_NAMESPACE for el in html . iter ( etree . Element ) : tag = el . tag if tag [ 0 ] != '{' : el . tag = prefix + tag

Convert all tags in an HTML tree to XHTML by moving them to the XHTML namespace .


def xhtml_to_html(xhtml):
    """Convert all tags in an XHTML tree to HTML by removing their
    XHTML namespace.
    """
    try:
        xhtml = xhtml.getroot()
    except AttributeError:
        pass
    prefix = "{%s}" % XHTML_NAMESPACE
    prefix_len = len(prefix)
    for el in xhtml.iter(prefix + "*"):
        el.tag = el.tag[prefix_len:]

def xhtml_to_html ( xhtml ) : try : xhtml = xhtml . getroot ( ) except AttributeError : pass prefix = "{%s}" % XHTML_NAMESPACE prefix_len = len ( prefix ) for el in xhtml . iter ( prefix + "*" ) : el . tag = el . tag [ prefix_len : ]

Convert all tags in an XHTML tree to HTML by removing their XHTML namespace .


def tostring(doc, pretty_print=False, include_meta_content_type=False,
             encoding=None, method="html", with_tail=True, doctype=None):
    """Return an HTML string representation of the document.

    Note: if include_meta_content_type is true this will create a
    ``<meta http-equiv="Content-Type" ...>`` tag in the head;
    regardless of the value of include_meta_content_type any existing
    ``<meta http-equiv="Content-Type" ...>`` tag will be removed

    The ``encoding`` argument controls the output encoding (defauts to
    ASCII, with &#...; character references for any characters outside
    of ASCII).  Note that you can pass the name ``'unicode'`` as
    ``encoding`` argument to serialise to a Unicode string.

    The ``method`` argument defines the output method.  It defaults to
    'html', but can also be 'xml' for xhtml output, or 'text' to
    serialise to plain text without markup.

    To leave out the tail text of the top-level element that is being
    serialised, pass ``with_tail=False``.

    The ``doctype`` option allows passing in a plain string that will
    be serialised before the XML tree.  Note that passing in non
    well-formed content here will make the XML output non well-formed.
    Also, an existing doctype in the document tree will not be removed
    when serialising an ElementTree instance.

    Example::

        >>> from lxml import html
        >>> root = html.fragment_fromstring('
Hello
world!
')

        >>> html.tostring(root)
        b'
Hello
world!
'
        >>> html.tostring(root, method='html')
        b'
Hello
world!
'

        >>> html.tostring(root, method='xml')
        b'
Hello
world!
'

        >>> html.tostring(root, method='text')
        b'Helloworld!'

        >>> html.tostring(root, method='text', encoding='unicode')
        u'Helloworld!'

        >>> root = html.fragment_fromstring('

Hello
world!
TAIL
')
        >>> html.tostring(root[0], method='text', encoding='unicode')
        u'Helloworld!TAIL'

        >>> html.tostring(root[0], method='text', encoding='unicode', with_tail=False)
        u'Helloworld!'

        >>> doc = html.document_fromstring('
Hello
world!
')
        >>> html.tostring(doc, method='html', encoding='unicode')
        u'
Hello
world!
'

        >>> print(html.tostring(doc, method='html', encoding='unicode',
        ...          doctype='<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN"'
        ...                  ' "http://www.w3.org/TR/html4/strict.dtd">'))
        <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">

Hello
world!

    """
    html = etree.tostring(doc, method=method, pretty_print=pretty_print,
                          encoding=encoding, with_tail=with_tail,
                          doctype=doctype)
    if method == 'html' and not include_meta_content_type:
        if isinstance(html, str):
            html = __str_replace_meta_content_type('', html)
        else:
            html = __bytes_replace_meta_content_type(bytes(), html)
    return html

def tostring ( doc , pretty_print = False , include_meta_content_type = False , encoding = None , method = "html" , with_tail = True , doctype = None ) : html = etree . tostring ( doc , method = method , pretty_print = pretty_print , encoding = encoding , with_tail = with_tail , doctype = doctype ) if method == 'html' and not include_meta_content_type : if isinstance ( html , str ) : html = __str_replace_meta_content_type ( '' , html ) else : html = __bytes_replace_meta_content_type ( bytes ( ) , html ) return html

Return an HTML string representation of the document .


def open_in_browser(doc, encoding=None):
    """
    Open the HTML document in a web browser, saving it to a temporary
    file to open it.  Note that this does not delete the file after
    use.  This is mainly meant for debugging.
    """
    import os
    import webbrowser
    import tempfile
    if not isinstance(doc, etree._ElementTree):
        doc = etree.ElementTree(doc)
    handle, fn = tempfile.mkstemp(suffix='.html')
    f = os.fdopen(handle, 'wb')
    try:
        doc.write(f, method="html", encoding=encoding or doc.docinfo.encoding or "UTF-8")
    finally:
        # we leak the file itself here, but we should at least close it
        f.close()
    url = 'file://' + fn.replace(os.path.sep, '/')
    print(url)
    webbrowser.open(url)

Open the HTML document in a web browser saving it to a temporary file to open it . Note that this does not delete the file after use . This is mainly meant for debugging .


def drop_tree(self):
        """
        Removes this element from the tree, including its children and
        text.  The tail text is joined to the previous element or
        parent.
        """
        parent = self.getparent()
        assert parent is not None
        if self.tail:
            previous = self.getprevious()
            if previous is None:
                parent.text = (parent.text or '') + self.tail
            else:
                previous.tail = (previous.tail or '') + self.tail
        parent.remove(self)

def drop_tree ( self ) : parent = self . getparent ( ) assert parent is not None if self . tail : previous = self . getprevious ( ) if previous is None : parent . text = ( parent . text or '' ) + self . tail else : previous . tail = ( previous . tail or '' ) + self . tail parent . remove ( self )

Removes this element from the tree including its children and text . The tail text is joined to the previous element or parent .


def drop_tag(self):
        """
        Remove the tag, but not its children or text.  The children and text
        are merged into the parent.

        Example::

            >>> h = fragment_fromstring('
Hello World!
')
            >>> h.find('.//b').drop_tag()
            >>> print(tostring(h, encoding='unicode'))

Hello World!

        """
        parent = self.getparent()
        assert parent is not None
        previous = self.getprevious()
        if self.text and isinstance(self.tag, basestring):
            # not a Comment, etc.
            if previous is None:
                parent.text = (parent.text or '') + self.text
            else:
                previous.tail = (previous.tail or '') + self.text
        if self.tail:
            if len(self):
                last = self[-1]
                last.tail = (last.tail or '') + self.tail
            elif previous is None:
                parent.text = (parent.text or '') + self.tail
            else:
                previous.tail = (previous.tail or '') + self.tail
        index = parent.index(self)
        parent[index:index+1] = self[:]

def
drop_tag
(
self
)
:
parent
=
self
.
getparent
(
)
assert
parent
is
not
previous
=
self
.
getprevious
(
)
if
self
.
text
and
isinstance
(
self
.
tag
,
basestring
)
:
# not a Comment, etc.
if
previous
is
:
parent
.
text
=
(
parent
.
text
or
''
)
+
self
.
text
else
:
previous
.
tail
=
(
previous
.
tail
or
''
)
+
self
.
text
if
self
.
tail
:
if
len
(
self
)
:
last
=
self
[
1
]
last
.
tail
=
(
last
.
tail
or
''
)
+
self
.
tail
elif
previous
is
:
parent
.
text
=
(
parent
.
text
or
''
)
+
self
.
tail
else
:
previous
.
tail
=
(
previous
.
tail
or
''
)
+
self
.
tail
index
=
parent
.
index
(
self
)
parent
[
index
:
index
+
1
]
=
self
[
:
]

Remove the tag but not its children or text . The children and text are merged into the parent .


def get_element_by_id(self, id, *default):
        """
        Get the first element in a document with the given id.  If none is
        found, return the default argument if provided or raise KeyError
        otherwise.

        Note that there can be more than one element with the same id,
        and this isn't uncommon in HTML documents found in the wild.
        Browsers return only the first match, and this function does
        the same.
        """
        try:
            # FIXME: should this check for multiple matches?
            # browsers just return the first one
            return _id_xpath(self, id=id)[0]
        except IndexError:
            if default:
                return default[0]
            else:
                raise KeyError(id)

Get the first element in a document with the given id . If none is found return the default argument if provided or raise KeyError otherwise .


def cssselect(self, expr, translator='html'):
        """
        Run the CSS expression on this element and its children,
        returning a list of the results.

        Equivalent to lxml.cssselect.CSSSelect(expr, translator='html')(self)
        -- note that pre-compiling the expression can provide a substantial
        speedup.
        """
        # Do the import here to make the dependency optional.
        from lxml.cssselect import CSSSelector
        return CSSSelector(expr, translator=translator)(self)

Run the CSS expression on this element and its children returning a list of the results .


def loghandler_members():
    """iterate through the attributes of every logger's handler

    this is used to switch out stderr and stdout in tests when buffer is True

    :returns: generator of tuples, each tuple has (name, handler, member_name, member_val)
    """
    Members = namedtuple("Members", ["name", "handler", "member_name", "member"])
    log_manager = logging.Logger.manager
    loggers = []
    ignore = set([modname()])
    if log_manager.root:
        loggers = list(log_manager.loggerDict.items())
        loggers.append(("root", log_manager.root))

    for logger_name, logger in loggers:
        if logger_name in ignore: continue

        for handler in getattr(logger, "handlers", []):
            members = inspect.getmembers(handler)
            for member_name, member in members:
                yield Members(logger_name, handler, member_name, member)

def loghandler_members ( ) : Members = namedtuple ( "Members" , [ "name" , "handler" , "member_name" , "member" ] ) log_manager = logging . Logger . manager loggers = [ ] ignore = set ( [ modname ( ) ] ) if log_manager . root : loggers = list ( log_manager . loggerDict . items ( ) ) loggers . append ( ( "root" , log_manager . root ) ) for logger_name , logger in loggers : if logger_name in ignore : continue for handler in getattr ( logger , "handlers" , [ ] ) : members = inspect . getmembers ( handler ) for member_name , member in members : yield Members ( logger_name , handler , member_name , member )

iterate through the attributes of every logger s handler


def get_counts():
    """return test counts that are set via pyt environment variables when pyt
    runs the test

    :returns: dict, 3 keys (classes, tests, modules) and how many tests of each
        were found by pyt
    """
    counts = {}
    ks = [
        ('PYT_TEST_CLASS_COUNT', "classes"),
        ('PYT_TEST_COUNT', "tests"),
        ('PYT_TEST_MODULE_COUNT', "modules"),
    ]

    for ek, cn in ks:
        counts[cn] = int(os.environ.get(ek, 0))

    return counts

def get_counts ( ) : counts = { } ks = [ ( 'PYT_TEST_CLASS_COUNT' , "classes" ) , ( 'PYT_TEST_COUNT' , "tests" ) , ( 'PYT_TEST_MODULE_COUNT' , "modules" ) , ] for ek , cn in ks : counts [ cn ] = int ( os . environ . get ( ek , 0 ) ) return counts


def is_single_class():
    """Returns True if only a single class is being run or some tests within a single class"""
    ret = False
    counts = get_counts()
    if counts["classes"] < 1 and counts["modules"] < 1:
        ret = counts["tests"] > 0
    else:
        ret = counts["classes"] <= 1 and counts["modules"] <= 1
    return ret

def is_single_class ( ) : ret = False counts = get_counts ( ) if counts [ "classes" ] < 1 and counts [ "modules" ] < 1 : ret = counts [ "tests" ] > 0 else : ret = counts [ "classes" ] <= 1 and counts [ "modules" ] <= 1 return ret


def is_single_module():
    """Returns True if only a module is being run"""
    ret = False
    counts = get_counts()
    if counts["modules"] == 1:
        ret = True

    elif counts["modules"] < 1:
        ret = is_single_class()

    return ret

def is_single_module ( ) : ret = False counts = get_counts ( ) if counts [ "modules" ] == 1 : ret = True elif counts [ "modules" ] < 1 : ret = is_single_class ( ) return ret


def validate_params(request):
    """Validate request params."""

    if 'params' in request:
        correct_params = isinstance(request['params'], (list, dict))
        error = 'Incorrect parameter values'
        assert correct_params, error

def
validate_params
(
request
)
:
if
'params'
in
request
:
correct_params
=
isinstance
(
request
[
'params'
]
,
(
list
,
dict
)
)
error
=
'Incorrect parameter values'
assert
correct_params
,
error


def validate_id(request):
    """Validate request id."""

    if 'id' in request:
        correct_id = isinstance(
            request['id'],
            (string_types, int, None),
        )
        error = 'Incorrect identifier'
        assert correct_id, error

def
validate_id
(
request
)
:
if
'id'
in
request
:
correct_id
=
isinstance
(
request
[
'id'
]
,
(
string_types
,
int
,
)
,
)
error
=
'Incorrect identifier'
assert
correct_id
,
error


def filesys_decode(path):
    """
    Ensure that the given path is decoded,
    NONE when no expected encoding works
    """

    fs_enc = sys.getfilesystemencoding()
    if isinstance(path, decoded_string):
        return path

    for enc in (fs_enc, "utf-8"):
        try:
            return path.decode(enc)
        except UnicodeDecodeError:
            continue

def filesys_decode ( path ) : fs_enc = sys . getfilesystemencoding ( ) if isinstance ( path , decoded_string ) : return path for enc in ( fs_enc , "utf-8" ) : try : return path . decode ( enc ) except UnicodeDecodeError : continue

Ensure that the given path is decoded NONE when no expected encoding works


def _escape_argspec(obj, iterable, escape):
    """Helper for various string-wrapped functions."""
    for key, value in iterable:
        if hasattr(value, '__html__') or isinstance(value, string_types):
            obj[key] = escape(value)
    return obj

def _escape_argspec ( obj , iterable , escape ) : for key , value in iterable : if hasattr ( value , '__html__' ) or isinstance ( value , string_types ) : obj [ key ] = escape ( value ) return obj

Helper for various string - wrapped functions .


def codecName(encoding):
    """Return the python codec name corresponding to an encoding or None if the
    string doesn't correspond to a valid encoding."""
    if isinstance(encoding, bytes):
        try:
            encoding = encoding.decode("ascii")
        except UnicodeDecodeError:
            return None
    if encoding:
        canonicalName = ascii_punctuation_re.sub("", encoding).lower()
        return encodings.get(canonicalName, None)
    else:
        return None

def codecName ( encoding ) : if isinstance ( encoding , bytes ) : try : encoding = encoding . decode ( "ascii" ) except UnicodeDecodeError : return None if encoding : canonicalName = ascii_punctuation_re . sub ( "" , encoding ) . lower ( ) return encodings . get ( canonicalName , None ) else : return None

Return the python codec name corresponding to an encoding or None if the string doesn t correspond to a valid encoding .


def detectBOM(self):
        """Attempts to detect at BOM at the start of the stream. If
        an encoding can be determined from the BOM return the name of the
        encoding otherwise return None"""
        bomDict = {
            codecs.BOM_UTF8: 'utf-8',
            codecs.BOM_UTF16_LE: 'utf-16-le', codecs.BOM_UTF16_BE: 'utf-16-be',
            codecs.BOM_UTF32_LE: 'utf-32-le', codecs.BOM_UTF32_BE: 'utf-32-be'
        }

        # Go to beginning of file and read in 4 bytes
        string = self.rawStream.read(4)
        assert isinstance(string, bytes)

        # Try detecting the BOM using bytes from the string
        encoding = bomDict.get(string[:3])         # UTF-8
        seek = 3
        if not encoding:
            # Need to detect UTF-32 before UTF-16
            encoding = bomDict.get(string)         # UTF-32
            seek = 4
            if not encoding:
                encoding = bomDict.get(string[:2])  # UTF-16
                seek = 2

        # Set the read position past the BOM if one was found, otherwise
        # set it to the start of the stream
        self.rawStream.seek(encoding and seek or 0)

        return encoding

Attempts to detect at BOM at the start of the stream . If an encoding can be determined from the BOM return the name of the encoding otherwise return None


def get_remote_addr(self, forwarded_for):
        """Selects the new remote addr from the given list of ips in
        X-Forwarded-For.  By default it picks the one that the `num_proxies`
        proxy server provides.  Before 0.9 it would always pick the first.

        .. versionadded:: 0.8
        """
        if len(forwarded_for) >= self.num_proxies:
            return forwarded_for[-1 * self.num_proxies]

def get_remote_addr ( self , forwarded_for ) : if len ( forwarded_for ) >= self . num_proxies : return forwarded_for [ - 1 * self . num_proxies ]

Selects the new remote addr from the given list of ips in X - Forwarded - For . By default it picks the one that the num_proxies proxy server provides . Before 0 . 9 it would always pick the first .


def amount_converter(obj):
    """Converts amount value from several types into Decimal."""
    if isinstance(obj, Decimal):
        return obj
    elif isinstance(obj, (str, int, float)):
        return Decimal(str(obj))
    else:
        raise ValueError('do not know how to convert: {}'.format(type(obj)))

def
amount_converter
(
obj
)
:
if
isinstance
(
obj
,
Decimal
)
:
return
obj
elif
isinstance
(
obj
,
(
str
,
int
,
float
)
)
:
return
Decimal
(
str
(
obj
)
)
else
:
raise
ValueError
(
'do not know how to convert: {}'
.
format
(
type
(
obj
)
)
)

Converts amount value from several types into Decimal .


def fromstring(data, beautifulsoup=None, makeelement=None, **bsargs):
    """Parse a string of HTML data into an Element tree using the
    BeautifulSoup parser.

    Returns the root ``<html>`` Element of the tree.

    You can pass a different BeautifulSoup parser through the
    `beautifulsoup` keyword, and a diffent Element factory function
    through the `makeelement` keyword.  By default, the standard
    ``BeautifulSoup`` class and the default factory of `lxml.html` are
    used.
    """
    return _parse(data, beautifulsoup, makeelement, **bsargs)

def fromstring ( data , beautifulsoup = None , makeelement = None , * * bsargs ) : return _parse ( data , beautifulsoup , makeelement , * * bsargs )

Parse a string of HTML data into an Element tree using the BeautifulSoup parser .


def parse(file, beautifulsoup=None, makeelement=None, **bsargs):
    """Parse a file into an ElemenTree using the BeautifulSoup parser.

    You can pass a different BeautifulSoup parser through the
    `beautifulsoup` keyword, and a diffent Element factory function
    through the `makeelement` keyword.  By default, the standard
    ``BeautifulSoup`` class and the default factory of `lxml.html` are
    used.
    """
    if not hasattr(file, 'read'):
        file = open(file)
    root = _parse(file, beautifulsoup, makeelement, **bsargs)
    return etree.ElementTree(root)

def parse ( file , beautifulsoup = None , makeelement = None , * * bsargs ) : if not hasattr ( file , 'read' ) : file = open ( file ) root = _parse ( file , beautifulsoup , makeelement , * * bsargs ) return etree . ElementTree ( root )

Parse a file into an ElemenTree using the BeautifulSoup parser .


def convert_tree(beautiful_soup_tree, makeelement=None):
    """Convert a BeautifulSoup tree to a list of Element trees.

    Returns a list instead of a single root Element to support
    HTML-like soup with more than one root element.

    You can pass a different Element factory through the `makeelement`
    keyword.
    """
    if makeelement is None:
        makeelement = html.html_parser.makeelement
    root = _convert_tree(beautiful_soup_tree, makeelement)
    children = root.getchildren()
    for child in children:
        root.remove(child)
    return children

def convert_tree ( beautiful_soup_tree , makeelement = None ) : if makeelement is None : makeelement = html . html_parser . makeelement root = _convert_tree ( beautiful_soup_tree , makeelement ) children = root . getchildren ( ) for child in children : root . remove ( child ) return children

Convert a BeautifulSoup tree to a list of Element trees .


def get_current_traceback(ignore_system_exceptions=False,
                          show_hidden_frames=False, skip=0):
    """Get the current exception info as `Traceback` object.  Per default
    calling this method will reraise system exceptions such as generator exit,
    system exit or others.  This behavior can be disabled by passing `False`
    to the function as first parameter.
    """
    exc_type, exc_value, tb = sys.exc_info()
    if ignore_system_exceptions and exc_type in system_exceptions:
        raise
    for x in range_type(skip):
        if tb.tb_next is None:
            break
        tb = tb.tb_next
    tb = Traceback(exc_type, exc_value, tb)
    if not show_hidden_frames:
        tb.filter_hidden_frames()
    return tb

def get_current_traceback ( ignore_system_exceptions = False , show_hidden_frames = False , skip = 0 ) : exc_type , exc_value , tb = sys . exc_info ( ) if ignore_system_exceptions and exc_type in system_exceptions : raise for x in range_type ( skip ) : if tb . tb_next is None : break tb = tb . tb_next tb = Traceback ( exc_type , exc_value , tb ) if not show_hidden_frames : tb . filter_hidden_frames ( ) return tb

Get the current exception info as Traceback object . Per default calling this method will reraise system exceptions such as generator exit system exit or others . This behavior can be disabled by passing False to the function as first parameter .


def exception(self):
        """String representation of the exception."""
        buf = traceback.format_exception_only(self.exc_type, self.exc_value)
        rv = ''.join(buf).strip()
        return rv.decode('utf-8', 'replace') if PY2 else rv

def exception ( self ) : buf = traceback . format_exception_only ( self . exc_type , self . exc_value ) rv = '' . join ( buf ) . strip ( ) return rv . decode ( 'utf-8' , 'replace' ) if PY2 else rv

String representation of the exception .


def render_summary(self, include_title=True):
        """Render the traceback for the interactive console."""
        title = ''
        frames = []
        classes = ['traceback']
        if not self.frames:
            classes.append('noframe-traceback')

        if include_title:
            if self.is_syntax_error:
                title = u'Syntax Error'
            else:
                title = u'Traceback (most recent call last):'

        for frame in self.frames:
            frames.append(u'
%s' % (
                frame.info and u' title="%s"' % escape(frame.info) or u'',
                frame.render()
            ))

        if self.is_syntax_error:
            description_wrapper = u'
%s
'
        else:
            description_wrapper = u'
%s
'

        return SUMMARY_HTML % {
            'classes':      u' '.join(classes),
            'title':        title and u'
%s
' % title or u'',
            'frames':       u'\n'.join(frames),
            'description':  description_wrapper % escape(self.exception)
        }

u'Traceback <em>(most recent call last)</em>:'

Render the traceback for the interactive console .


def generate_plaintext_traceback(self):
        """Like the plaintext attribute but returns a generator"""
        yield u'Traceback (most recent call last):'
        for frame in self.frames:
            yield u'  File "%s", line %s, in %s' % (
                frame.filename,
                frame.lineno,
                frame.function_name
            )
            yield u'    ' + frame.current_line.strip()
        yield self.exception

def
generate_plaintext_traceback
(
self
)
:
yield
u'Traceback (most recent call last):'
for
frame
in
self
.
frames
:
yield
u'  File "%s", line %s, in %s'
%
(
frame
.
filename
,
frame
.
lineno
,
frame
.
function_name
)
yield
u'    '
+
frame
.
current_line
.
strip
(
)
yield
self
.
exception


def get_annotated_lines(self):
        """Helper function that returns lines with extra information."""
        lines = [Line(idx + 1, x) for idx, x in enumerate(self.sourcelines)]

        # find function definition and mark lines
        if hasattr(self.code, 'co_firstlineno'):
            lineno = self.code.co_firstlineno - 1
            while lineno > 0:
                if _funcdef_re.match(lines[lineno].code):
                    break
                lineno -= 1
            try:
                offset = len(inspect.getblock([x.code + '\n' for x
                                               in lines[lineno:]]))
            except TokenError:
                offset = 0
            for line in lines[lineno:lineno + offset]:
                line.in_frame = True

        # mark current line
        try:
            lines[self.lineno - 1].current = True
        except IndexError:
            pass

        return lines

Helper function that returns lines with extra information .


def render_source(self):
        """Render the sourcecode."""
        return SOURCE_TABLE_HTML % u'\n'.join(line.render() for line in
                                              self.get_annotated_lines())

def render_source ( self ) : return SOURCE_TABLE_HTML % u'\n' . join ( line . render ( ) for line in self . get_annotated_lines ( ) )