def multiqc(store, institute_id, case_name):
    """Find MultiQC report for the case."""
    institute_obj, case_obj = institute_and_case(store, institute_id, case_name)
    return dict(
        institute=institute_obj,
        case=case_obj,
    )

def multiqc ( store , institute_id , case_name ) : institute_obj , case_obj = institute_and_case ( store , institute_id , case_name ) return dict ( institute = institute_obj , case = case_obj , )


def get_sanger_unevaluated(store, institute_id, user_id):
    """Get all variants for an institute having Sanger validations ordered but still not evaluated

        Args:
            store(scout.adapter.MongoAdapter)
            institute_id(str)

        Returns:
            unevaluated: a list that looks like this: [ {'case1': [varID_1, varID_2, .., varID_n]}, {'case2' : [varID_1, varID_2, .., varID_n]} ],
                         where the keys are case_ids and the values are lists of variants with Sanger ordered but not yet validated

    """

    # Retrieve a list of ids for variants with Sanger ordered grouped by case from the 'event' collection
    # This way is much faster than querying over all variants in all cases of an institute
    sanger_ordered_by_case = store.sanger_ordered(institute_id, user_id)
    unevaluated = []

    # for each object where key==case and value==[variant_id with Sanger ordered]
    for item in sanger_ordered_by_case:
        case_id = item['_id']
        # Get the case to collect display name
        case_obj = store.case(case_id=case_id)

        if not case_obj: # the case might have been removed
            continue

        case_display_name = case_obj.get('display_name')

        # List of variant document ids
        varid_list = item['vars']

        unevaluated_by_case = {}
        unevaluated_by_case[case_display_name] = []

        for var_id in varid_list:
            # For each variant with sanger validation ordered
            variant_obj = store.variant(document_id=var_id, case_id=case_id)

            # Double check that Sanger was ordered (and not canceled) for the variant
            if variant_obj is None or variant_obj.get('sanger_ordered') is None or variant_obj.get('sanger_ordered') is False:
                continue

            validation = variant_obj.get('validation', 'not_evaluated')

            # Check that the variant is not evaluated
            if validation in ['True positive', 'False positive']:
                continue

            unevaluated_by_case[case_display_name].append(variant_obj['_id'])

        # If for a case there is at least one Sanger validation to evaluate add the object to the unevaluated objects list
        if len(unevaluated_by_case[case_display_name]) > 0:
            unevaluated.append(unevaluated_by_case)

    return unevaluated


def mme_add(store, user_obj, case_obj, add_gender, add_features, add_disorders, genes_only,
    mme_base_url, mme_accepts, mme_token):
    """Add a patient to MatchMaker server

    Args:
        store(adapter.MongoAdapter)
        user_obj(dict) a scout user object (to be added as matchmaker contact)
        case_obj(dict) a scout case object
        add_gender(bool) if True case gender will be included in matchmaker
        add_features(bool) if True HPO features will be included in matchmaker
        add_disorders(bool) if True OMIM diagnoses will be included in matchmaker
        genes_only(bool) if True only genes and not variants will be shared
        mme_base_url(str) base url of the MME server
        mme_accepts(str) request content accepted by MME server
        mme_token(str) auth token of the MME server

    Returns:
        submitted_info(dict) info submitted to MatchMaker and its responses
    """

    if not mme_base_url or not mme_accepts or not mme_token:
        return 'Please check that Matchmaker connection parameters are valid'
    url = ''.join([mme_base_url, '/patient/add'])
    features = []   # this is the list of HPO terms
    disorders = []  # this is the list of OMIM diagnoses
    g_features = []

     # create contact dictionary
    contact_info = {
        'name' : user_obj['name'],
        'href' : ''.join( ['mailto:',user_obj['email']] ),
        'institution' : 'Scout software user, Science For Life Laboratory, Stockholm, Sweden'
    }
    if add_features: # create features dictionaries
        features = hpo_terms(case_obj)

    if add_disorders: # create OMIM disorders dictionaries
        disorders = omim_terms(case_obj)

    # send a POST request and collect response for each affected individual in case
    server_responses = []

    submitted_info = {
        'contact' : contact_info,
        'sex' : add_gender,
        'features' : features,
        'disorders' : disorders,
        'genes_only' : genes_only,
        'patient_id' : []
    }
    for individual in case_obj.get('individuals'):
        if not individual['phenotype'] in  [2, 'affected']: # include only affected individuals
            continue

        patient = {
            'contact' : contact_info,
            'id' : '.'.join([case_obj['_id'], individual.get('individual_id')]), # This is a required field form MME
            'label' : '.'.join([case_obj['display_name'], individual.get('display_name')]),
            'features' : features,
            'disorders' : disorders
        }
        if add_gender:
            if individual['sex'] == '1':
                patient['sex'] = 'MALE'
            else:
                patient['sex'] = 'FEMALE'

        if case_obj.get('suspects'):
            g_features = genomic_features(store, case_obj, individual.get('display_name'), genes_only)
            patient['genomicFeatures'] = g_features

        # send add request to server and capture response
        resp = matchmaker_request(url=url, token=mme_token, method='POST', content_type=mme_accepts,
            accept='application/json', data={'patient':patient})

        server_responses.append({
                'patient': patient,
                'message': resp.get('message'),
                'status_code' : resp.get('status_code')
            })
    submitted_info['server_responses'] = server_responses
    return submitted_info


def mme_delete(case_obj, mme_base_url, mme_token):
    """Delete all affected samples for a case from MatchMaker

    Args:
        case_obj(dict) a scout case object
        mme_base_url(str) base url of the MME server
        mme_token(str) auth token of the MME server

    Returns:
         server_responses(list): a list of object of this type:
                    {
                        'patient_id': patient_id
                        'message': server_message,
                        'status_code': server_status_code
                    }
    """
    server_responses = []

    if not mme_base_url or not mme_token:
        return 'Please check that Matchmaker connection parameters are valid'

    # for each patient of the case in matchmaker
    for patient in case_obj['mme_submission']['patients']:

        # send delete request to server and capture server's response
        patient_id = patient['id']
        url = ''.join([mme_base_url, '/patient/delete/', patient_id])
        resp = matchmaker_request(url=url, token=mme_token, method='DELETE', )

        server_responses.append({
            'patient_id': patient_id,
            'message': resp.get('message'),
            'status_code': resp.get('status_code')
        })

    return server_responses


def mme_matches(case_obj, institute_obj, mme_base_url, mme_token):
    """Show Matchmaker submission data for a sample and eventual matches.

    Args:
        case_obj(dict): a scout case object
        institute_obj(dict): an institute object
        mme_base_url(str) base url of the MME server
        mme_token(str) auth token of the MME server

    Returns:
        data(dict): data to display in the html template
    """
    data = {
        'institute' : institute_obj,
        'case' : case_obj,
        'server_errors' : []
    }
    matches = {}
    # loop over the submitted samples and get matches from the MatchMaker server
    if not case_obj.get('mme_submission'):
        return None

    for patient in case_obj['mme_submission']['patients']:
        patient_id = patient['id']
        matches[patient_id] = None
        url = ''.join([ mme_base_url, '/matches/', patient_id])
        server_resp = matchmaker_request(url=url, token=mme_token, method='GET')
        if 'status_code' in server_resp: # the server returned a valid response
            # and this will be a list of match objects sorted by desc date
            pat_matches = []
            if server_resp.get('matches'):
                pat_matches = parse_matches(patient_id, server_resp['matches'])
            matches[patient_id] = pat_matches
        else:
            LOG.warning('Server returned error message: {}'.format(server_resp['message']))
            data['server_errors'].append(server_resp['message'])

    data['matches'] = matches

    return data

Show Matchmaker submission data for a sample and eventual matches .


def mme_match(case_obj, match_type, mme_base_url, mme_token, nodes=None, mme_accepts=None):
    """Initiate a MatchMaker match against either other Scout patients or external nodes

    Args:
        case_obj(dict): a scout case object already submitted to MME
        match_type(str): 'internal' or 'external'
        mme_base_url(str): base url of the MME server
        mme_token(str): auth token of the MME server
        mme_accepts(str): request content accepted by MME server (only for internal matches)

    Returns:
        matches(list): a list of eventual matches
    """
    query_patients = []
    server_responses = []
    url = None
    # list of patient dictionaries is required for internal matching
    query_patients = case_obj['mme_submission']['patients']
    if match_type=='internal':
        url = ''.join([mme_base_url,'/match'])
        for patient in query_patients:
            json_resp = matchmaker_request(url=url, token=mme_token, method='POST',
                content_type=mme_accepts, accept=mme_accepts, data={'patient':patient})
            resp_obj = {
                'server' : 'Local MatchMaker node',
                'patient_id' : patient['id'],
                'results' : json_resp.get('results'),
                'status_code' : json_resp.get('status_code'),
                'message' : json_resp.get('message') # None if request was successful
            }
            server_responses.append(resp_obj)
    else: # external matching
        # external matching requires only patient ID
        query_patients = [ patient['id'] for patient in query_patients]
        node_ids = [ node['id'] for node in nodes ]
        if match_type in node_ids: # match is against a specific external node
            node_ids = [match_type]

        # Match every affected patient
        for patient in query_patients:
            # Against every node
            for node in node_ids:
                url = ''.join([mme_base_url,'/match/external/', patient, '?node=', node])
                json_resp = matchmaker_request(url=url, token=mme_token, method='POST')
                resp_obj = {
                    'server' : node,
                    'patient_id' : patient,
                    'results' : json_resp.get('results'),
                    'status_code' : json_resp.get('status_code'),
                    'message' : json_resp.get('message') # None if request was successful
                }
                server_responses.append(resp_obj)
    return server_responses


def genes(context, build, api_key):
    """
    Load the hgnc aliases to the mongo database.
    """
    LOG.info("Running scout update genes")
    adapter = context.obj['adapter']

    # Fetch the omim information
    api_key = api_key or context.obj.get('omim_api_key')
    if not api_key:
        LOG.warning("Please provide a omim api key to load the omim gene panel")
        context.abort()

    try:
        mim_files = fetch_mim_files(api_key, mim2genes=True, morbidmap=True, genemap2=True)
    except Exception as err:
        LOG.warning(err)
        context.abort()

    LOG.warning("Dropping all gene information")
    adapter.drop_genes(build)
    LOG.info("Genes dropped")
    LOG.warning("Dropping all transcript information")
    adapter.drop_transcripts(build)
    LOG.info("transcripts dropped")

    hpo_genes = fetch_hpo_genes()

    if build:
        builds = [build]
    else:
        builds = ['37', '38']

    hgnc_lines = fetch_hgnc()
    exac_lines = fetch_exac_constraint()

    for build in builds:
        ensembl_genes = fetch_ensembl_genes(build=build)

        # load the genes
        hgnc_genes = load_hgnc_genes(
            adapter=adapter,
            ensembl_lines=ensembl_genes,
            hgnc_lines=hgnc_lines,
            exac_lines=exac_lines,
            mim2gene_lines=mim_files['mim2genes'],
            genemap_lines=mim_files['genemap2'],
            hpo_lines=hpo_genes,
            build=build,
        )

        ensembl_genes = {}
        for gene_obj in hgnc_genes:
            ensembl_id = gene_obj['ensembl_id']
            ensembl_genes[ensembl_id] = gene_obj

        # Fetch the transcripts from ensembl
        ensembl_transcripts = fetch_ensembl_transcripts(build=build)

        transcripts = load_transcripts(adapter, ensembl_transcripts, build, ensembl_genes)

    adapter.update_indexes()

    LOG.info("Genes, transcripts and Exons loaded")

Load the hgnc aliases to the mongo database .


def parse_callers(variant, category='snv'):
    """Parse how the different variant callers have performed

        Args:
            variant (cyvcf2.Variant): A variant object

        Returns:
            callers (dict): A dictionary on the format
            {'gatk': <filter>,'freebayes': <filter>,'samtools': <filter>}
    """
    relevant_callers = CALLERS[category]
    callers = {caller['id']: None for caller in relevant_callers}
    raw_info = variant.INFO.get('set')
    if raw_info:
        info = raw_info.split('-')
        for call in info:
            if call == 'FilteredInAll':
                for caller in callers:
                    callers[caller] = 'Filtered'
            elif call == 'Intersection':
                for caller in callers:
                    callers[caller] = 'Pass'
            elif 'filterIn' in call:
                for caller in callers:
                    if caller in call:
                        callers[caller] = 'Filtered'

            elif call in set(callers.keys()):
                callers[call] = 'Pass'
    # The following is parsing of a custom made merge
    other_info = variant.INFO.get('FOUND_IN')
    if other_info:
        for info in other_info.split(','):
            called_by = info.split('|')[0]
            callers[called_by] = 'Pass'

    return callers


def build_transcript(transcript_info, build='37'):
    """Build a hgnc_transcript object

        Args:
            transcript_info(dict): Transcript information

        Returns:
            transcript_obj(HgncTranscript)
            {
                transcript_id: str, required
                hgnc_id: int, required
                build: str, required
                refseq_id: str,
                chrom: str, required
                start: int, required
                end: int, required
                is_primary: bool
            }
    """
    try:
        transcript_id = transcript_info['ensembl_transcript_id']
    except KeyError:
        raise KeyError("Transcript has to have ensembl id")

    build = build
    is_primary = transcript_info.get('is_primary', False)

    refseq_id = transcript_info.get('refseq_id')
    refseq_identifiers = transcript_info.get('refseq_identifiers')

    try:
        chrom = transcript_info['chrom']
    except KeyError:
        raise KeyError("Transcript has to have a chromosome")

    try:
        start = int(transcript_info['transcript_start'])
    except KeyError:
        raise KeyError("Transcript has to have start")
    except TypeError:
        raise TypeError("Transcript start has to be integer")

    try:
        end = int(transcript_info['transcript_end'])
    except KeyError:
        raise KeyError("Transcript has to have end")
    except TypeError:
        raise TypeError("Transcript end has to be integer")

    try:
        hgnc_id = int(transcript_info['hgnc_id'])
    except KeyError:
        raise KeyError("Transcript has to have a hgnc id")
    except TypeError:
        raise TypeError("hgnc id has to be integer")

    transcript_obj = HgncTranscript(
        transcript_id=transcript_id,
        hgnc_id=hgnc_id,
        chrom=chrom,
        start=start,
        end=end,
        is_primary=is_primary,
        refseq_id=refseq_id,
        refseq_identifiers=refseq_identifiers,
        build=build
    )
    # Remove unnessesary keys
    for key in list(transcript_obj):
        if transcript_obj[key] is None:
            transcript_obj.pop(key)

    return transcript_obj

def
build_transcript
(
transcript_info
,
build
=
'37'
)
:
try
:
transcript_id
=
transcript_info
[
'ensembl_transcript_id'
]
except
KeyError
:
raise
KeyError
(
"Transcript has to have ensembl id"
)
build
=
build
is_primary
=
transcript_info
.
get
(
'is_primary'
,
False
)
refseq_id
=
transcript_info
.
get
(
'refseq_id'
)
refseq_identifiers
=
transcript_info
.
get
(
'refseq_identifiers'
)
try
:
chrom
=
transcript_info
[
'chrom'
]
except
KeyError
:
raise
KeyError
(
"Transcript has to have a chromosome"
)
try
:
start
=
int
(
transcript_info
[
'transcript_start'
]
)
except
KeyError
:
raise
KeyError
(
"Transcript has to have start"
)
except
TypeError
:
raise
TypeError
(
"Transcript start has to be integer"
)
try
:
end
=
int
(
transcript_info
[
'transcript_end'
]
)
except
KeyError
:
raise
KeyError
(
"Transcript has to have end"
)
except
TypeError
:
raise
TypeError
(
"Transcript end has to be integer"
)
try
:
hgnc_id
=
int
(
transcript_info
[
'hgnc_id'
]
)
except
KeyError
:
raise
KeyError
(
"Transcript has to have a hgnc id"
)
except
TypeError
:
raise
TypeError
(
"hgnc id has to be integer"
)
transcript_obj
=
HgncTranscript
(
transcript_id
=
transcript_id
,
hgnc_id
=
hgnc_id
,
chrom
=
chrom
,
start
=
start
,
end
=
end
,
is_primary
=
is_primary
,
refseq_id
=
refseq_id
,
refseq_identifiers
=
refseq_identifiers
,
build
=
build
)
# Remove unnessesary keys
for
key
in
list
(
transcript_obj
)
:
if
transcript_obj
[
key
]
is
:
transcript_obj
.
pop
(
key
)
return
transcript_obj


def load_institute(adapter, internal_id, display_name, sanger_recipients=None):
    """Load a institute into the database

        Args:
            adapter(MongoAdapter)
            internal_id(str)
            display_name(str)
            sanger_recipients(list(email))
    """

    institute_obj = build_institute(
        internal_id=internal_id,
        display_name=display_name,
        sanger_recipients=sanger_recipients
    )
    log.info("Loading institute {0} with display name {1}" \
             " into database".format(internal_id, display_name))

    adapter.add_institute(institute_obj)


def parse_cadd(variant, transcripts):
    """Check if the cadd phred score is annotated"""
    cadd = 0
    cadd_keys = ['CADD', 'CADD_PHRED']
    for key in cadd_keys:
        cadd = variant.INFO.get(key, 0)
        if cadd:
            return float(cadd)

    for transcript in transcripts:
        cadd_entry = transcript.get('cadd')
        if (cadd_entry and cadd_entry > cadd):
            cadd = cadd_entry

    return cadd

def parse_cadd ( variant , transcripts ) : cadd = 0 cadd_keys = [ 'CADD' , 'CADD_PHRED' ] for key in cadd_keys : cadd = variant . INFO . get ( key , 0 ) if cadd : return float ( cadd ) for transcript in transcripts : cadd_entry = transcript . get ( 'cadd' ) if ( cadd_entry and cadd_entry > cadd ) : cadd = cadd_entry return cadd


def case(context, vcf, vcf_sv, vcf_cancer, vcf_str, owner, ped, update, config,
         no_variants, peddy_ped, peddy_sex, peddy_check):
    """Load a case into the database.

    A case can be loaded without specifying vcf files and/or bam files
    """
    adapter = context.obj['adapter']

    if config is None and ped is None:
        LOG.warning("Please provide either scout config or ped file")
        context.abort()

    # Scout needs a config object with the neccessary information
    # If no config is used create a dictionary
    config_raw = yaml.load(config) if config else {}

    try:
        config_data = parse_case_data(
            config=config_raw,
            ped=ped,
            owner=owner,
            vcf_snv=vcf,
            vcf_sv=vcf_sv,
            vcf_str=vcf_str,
            vcf_cancer=vcf_cancer,
            peddy_ped=peddy_ped,
            peddy_sex=peddy_sex,
            peddy_check=peddy_check
        )
    except SyntaxError as err:
        LOG.warning(err)
        context.abort()

    LOG.info("Use family %s" % config_data['family'])

    try:
        case_obj = adapter.load_case(config_data, update)
    except Exception as err:
        LOG.error("Something went wrong during loading")
        LOG.warning(err)
        context.abort()


def update_variant(self, variant_obj):
        """Update one variant document in the database.

        This means that the variant in the database will be replaced by variant_obj.

        Args:
            variant_obj(dict)

        Returns:
            new_variant(dict)
        """
        LOG.debug('Updating variant %s', variant_obj.get('simple_id'))

        new_variant = self.variant_collection.find_one_and_replace(
            {'_id': variant_obj['_id']},
            variant_obj,
            return_document=pymongo.ReturnDocument.AFTER
        )
        return new_variant

def
update_variant
(
self
,
variant_obj
)
:
LOG
.
debug
(
'Updating variant %s'
,
variant_obj
.
get
(
'simple_id'
)
)
new_variant
=
self
.
variant_collection
.
find_one_and_replace
(
{
'_id'
:
variant_obj
[
'_id'
]
}
,
variant_obj
,
return_document
=
pymongo
.
ReturnDocument
.
AFTER
)
return
new_variant

Update one variant document in the database .


def update_variant_rank(self, case_obj, variant_type='clinical', category='snv'):
        """Updates the manual rank for all variants in a case

        Add a variant rank based on the rank score
        Whenever variants are added or removed from a case we need to update the variant rank

        Args:
            case_obj(Case)
            variant_type(str)
        """
        # Get all variants sorted by rank score
        variants = self.variant_collection.find({
            'case_id': case_obj['_id'],
            'category': category,
            'variant_type': variant_type,
        }).sort('rank_score', pymongo.DESCENDING)

        LOG.info("Updating variant_rank for all variants")

        requests = []

        for index, var_obj in enumerate(variants):
            if len(requests) > 5000:
                try:
                    self.variant_collection.bulk_write(requests, ordered=False)
                    requests = []
                except BulkWriteError as err:
                    LOG.warning("Updating variant rank failed")
                    raise err

            operation = pymongo.UpdateOne(
                {'_id': var_obj['_id']},
                {
                    '$set': {
                        'variant_rank': index + 1,
                    }
                })
            requests.append(operation)

        #Update the final bulk
        try:
            self.variant_collection.bulk_write(requests, ordered=False)
        except BulkWriteError as err:
            LOG.warning("Updating variant rank failed")
            raise err

        LOG.info("Updating variant_rank done")


def update_variant_compounds(self, variant, variant_objs = None):
        """Update compounds for a variant.

        This will add all the necessary information of a variant on a compound object.

        Args:
            variant(scout.models.Variant)
            variant_objs(dict): A dictionary with _ids as keys and variant objs as values.

        Returns:
            compound_objs(list(dict)): A dictionary with updated compound objects.

        """
        compound_objs = []
        for compound in variant.get('compounds', []):
            not_loaded = True
            gene_objs = []
            # Check if the compound variant exists
            if variant_objs:
                variant_obj = variant_objs.get(compound['variant'])
            else:
                variant_obj = self.variant_collection.find_one({'_id': compound['variant']})
            if variant_obj:
                # If the variant exosts we try to collect as much info as possible
                not_loaded = False
                compound['rank_score'] = variant_obj['rank_score']
                for gene in variant_obj.get('genes', []):
                    gene_obj = {
                        'hgnc_id': gene['hgnc_id'],
                        'hgnc_symbol': gene.get('hgnc_symbol'),
                        'region_annotation': gene.get('region_annotation'),
                        'functional_annotation': gene.get('functional_annotation'),
                    }
                    gene_objs.append(gene_obj)
                    compound['genes'] = gene_objs

            compound['not_loaded'] = not_loaded
            compound_objs.append(compound)

        return compound_objs


def update_compounds(self, variants):
        """Update the compounds for a set of variants.

        Args:
            variants(dict): A dictionary with _ids as keys and variant objs as values

        """
        LOG.debug("Updating compound objects")

        for var_id in variants:
            variant_obj = variants[var_id]
            if not variant_obj.get('compounds'):
                continue

            updated_compounds = self.update_variant_compounds(variant_obj, variants)
            variant_obj['compounds'] = updated_compounds

        LOG.debug("Compounds updated")

        return variants

def
update_compounds
(
self
,
variants
)
:
LOG
.
debug
(
"Updating compound objects"
)
for
var_id
in
variants
:
variant_obj
=
variants
[
var_id
]
if
not
variant_obj
.
get
(
'compounds'
)
:
continue
updated_compounds
=
self
.
update_variant_compounds
(
variant_obj
,
variants
)
variant_obj
[
'compounds'
]
=
updated_compounds
LOG
.
debug
(
"Compounds updated"
)
return
variants

Update the compounds for a set of variants .


def update_mongo_compound_variants(self, bulk):
        """Update the compound information for a bulk of variants in the database

            Args:
                bulk(dict): {'_id': scout.models.Variant}

        """
        requests = []
        for var_id in bulk:
            var_obj = bulk[var_id]
            if not var_obj.get('compounds'):
                continue
            # Add a request to update compounds
            operation = pymongo.UpdateOne(
                {'_id': var_obj['_id']},
                {
                    '$set': {
                        'compounds': var_obj['compounds']
                    }
                })
            requests.append(operation)

        if not requests:
            return

        try:
            self.variant_collection.bulk_write(requests, ordered=False)
        except BulkWriteError as err:
            LOG.warning("Updating compounds failed")
            raise err

def
update_mongo_compound_variants
(
self
,
bulk
)
:
requests
=
[
]
for
var_id
in
bulk
:
var_obj
=
bulk
[
var_id
]
if
not
var_obj
.
get
(
'compounds'
)
:
continue
# Add a request to update compounds
operation
=
pymongo
.
UpdateOne
(
{
'_id'
:
var_obj
[
'_id'
]
}
,
{
'$set'
:
{
'compounds'
:
var_obj
[
'compounds'
]
}
}
)
requests
.
append
(
operation
)
if
not
requests
:
return
try
:
self
.
variant_collection
.
bulk_write
(
requests
,
ordered
=
False
)
except
BulkWriteError
as
err
:
LOG
.
warning
(
"Updating compounds failed"
)
raise
err


def update_case_compounds(self, case_obj, build='37'):
        """Update the compounds for a case

        Loop over all coding intervals to get coordinates for all potential compound positions.
        Update all variants within a gene with a bulk operation.
        """

        case_id = case_obj['_id']
        # Possible categories 'snv', 'sv', 'str', 'cancer':
        categories = set()
        # Possible variant types 'clinical', 'research':
        variant_types = set()

        for file_type in FILE_TYPE_MAP:
            if case_obj.get('vcf_files',{}).get(file_type):
                categories.add(FILE_TYPE_MAP[file_type]['category'])
                variant_types.add(FILE_TYPE_MAP[file_type]['variant_type'])

        coding_intervals = self.get_coding_intervals(build=build)
        # Loop over all intervals
        for chrom in CHROMOSOMES:
            intervals = coding_intervals.get(chrom, IntervalTree())
            for var_type in variant_types:
                for category in categories:
                    LOG.info("Updating compounds on chromosome:{0}, type:{1}, category:{2} for case:{3}".format(
                             chrom, var_type, category, case_id))

                    # Fetch all variants from a chromosome
                    query  = {
                        'variant_type': var_type,
                        'chrom': chrom,
                    }

                    # Get all variants from the database of the specific type
                    variant_objs = self.variants(
                        case_id=case_id,
                        query=query,
                        category=category,
                        nr_of_variants=-1,
                        sort_key='position'
                    )

                    # Initiate a bulk
                    bulk = {}
                    current_region = None
                    special = False

                    # Loop over the variants and check if they are in a coding region
                    for var_obj in variant_objs:
                        var_id = var_obj['_id']
                        var_chrom = var_obj['chromosome']
                        var_start = var_obj['position']
                        var_end = var_obj['end'] + 1

                        update_bulk = True
                        new_region = None

                        # Check if the variant is in a coding region
                        genomic_regions = coding_intervals.get(var_chrom, IntervalTree()).search(var_start, var_end)

                        # If the variant is in a coding region
                        if genomic_regions:
                            # We know there is data here so get the interval id
                            new_region = genomic_regions.pop().data

                        if new_region and (new_region == current_region):
                            # If the variant is in the same region as previous
                            # we add it to the same bulk
                            update_bulk = False

                        current_region = new_region

                        # If the variant is not in a current region we update the compounds
                        # from the previous region, if any. Otherwise continue
                        if update_bulk and bulk:
                            self.update_compounds(bulk)
                            self.update_mongo_compound_variants(bulk)
                            bulk = {}

                        if new_region:
                            bulk[var_id] = var_obj

                    if not bulk:
                        continue

                    self.update_compounds(bulk)
                    self.update_mongo_compound_variants(bulk)

        LOG.info("All compounds updated")
        return


def load_variant(self, variant_obj):
        """Load a variant object

        Args:
            variant_obj(dict)

        Returns:
            inserted_id
        """
        # LOG.debug("Loading variant %s", variant_obj['_id'])
        try:
            result = self.variant_collection.insert_one(variant_obj)
        except DuplicateKeyError as err:
            raise IntegrityError("Variant %s already exists in database", variant_obj['_id'])
        return result


def upsert_variant(self, variant_obj):
        """Load a variant object, if the object already exists update compounds.

        Args:
            variant_obj(dict)

        Returns:
            result
        """
        LOG.debug("Upserting variant %s", variant_obj['_id'])
        try:
            result = self.variant_collection.insert_one(variant_obj)
        except DuplicateKeyError as err:
            LOG.debug("Variant %s already exists in database", variant_obj['_id'])
            result = self.variant_collection.find_one_and_update(
                {'_id': variant_obj['_id']},
                {
                    '$set': {
                        'compounds': variant_obj.get('compounds',[])
                    }
                }
            )
            variant = self.variant_collection.find_one({'_id': variant_obj['_id']})
        return result

def
upsert_variant
(
self
,
variant_obj
)
:
LOG
.
debug
(
"Upserting variant %s"
,
variant_obj
[
'_id'
]
)
try
:
result
=
self
.
variant_collection
.
insert_one
(
variant_obj
)
except
DuplicateKeyError
as
err
:
LOG
.
debug
(
"Variant %s already exists in database"
,
variant_obj
[
'_id'
]
)
result
=
self
.
variant_collection
.
find_one_and_update
(
{
'_id'
:
variant_obj
[
'_id'
]
}
,
{
'$set'
:
{
'compounds'
:
variant_obj
.
get
(
'compounds'
,
[
]
)
}
}
)
variant
=
self
.
variant_collection
.
find_one
(
{
'_id'
:
variant_obj
[
'_id'
]
}
)
return
result

Load a variant object if the object already exists update compounds .


def load_variant_bulk(self, variants):
        """Load a bulk of variants

        Args:
            variants(iterable(scout.models.Variant))

        Returns:
            object_ids
        """
        if not len(variants) > 0:
            return

        LOG.debug("Loading variant bulk")
        try:
            result = self.variant_collection.insert_many(variants)
        except (DuplicateKeyError, BulkWriteError) as err:
            # If the bulk write is wrong there are probably some variants already existing
            # In the database. So insert each variant
            for var_obj in variants:
                try:
                    self.upsert_variant(var_obj)
                except IntegrityError as err:
                    pass

        return


def assign(self, institute, case, user, link):
        """Assign a user to a case.

        This function will create an Event to log that a person has been assigned
        to a case. Also the user will be added to case "assignees".

        Arguments:
            institute (dict): A institute
            case (dict): A case
            user (dict): A User object
            link (str): The url to be used in the event

        Returns:
            updated_case(dict)
        """
        LOG.info("Creating event for assigning {0} to {1}"
                    .format(user['name'].encode('utf-8'), case['display_name']))

        self.create_event(
            institute=institute,
            case=case,
            user=user,
            link=link,
            category='case',
            verb='assign',
            subject=case['display_name']
        )
        LOG.info("Updating {0} to be assigned with {1}"
                    .format(case['display_name'], user['name']))

        updated_case = self.case_collection.find_one_and_update(
            {'_id': case['_id']},
            {'$addToSet': {'assignees': user['_id']}},
            return_document=pymongo.ReturnDocument.AFTER
        )
        return updated_case


def share(self, institute, case, collaborator_id, user, link):
        """Share a case with a new institute.

        Arguments:
            institute (dict): A Institute object
            case (dict): Case object
            collaborator_id (str): A instute id
            user (dict): A User object
            link (str): The url to be used in the event

        Return:
            updated_case
        """
        if collaborator_id in case.get('collaborators', []):
            raise ValueError('new customer is already a collaborator')

        self.create_event(
            institute=institute,
            case=case,
            user=user,
            link=link,
            category='case',
            verb='share',
            subject=collaborator_id
        )

        updated_case = self.case_collection.find_one_and_update(
            {'_id': case['_id']},
            {
                '$push': {'collaborators': collaborator_id}
            },
            return_document=pymongo.ReturnDocument.AFTER
        )
        LOG.debug("Case updated")
        return updated_case


def diagnose(self, institute, case, user, link, level, omim_id, remove=False):
        """Diagnose a case using OMIM ids.

        Arguments:
            institute (dict): A Institute object
            case (dict): Case object
            user (dict): A User object
            link (str): The url to be used in the event
            level (str): choices=('phenotype','gene')

        Return:
            updated_case

        """
        if level == 'phenotype':
            case_key = 'diagnosis_phenotypes'
        elif level == 'gene':
            case_key = 'diagnosis_genes'
        else:
            raise TypeError('wrong level')

        diagnosis_list = case.get(case_key, [])
        omim_number = int(omim_id.split(':')[-1])

        updated_case = None
        if remove and omim_number in diagnosis_list:
            updated_case = self.case_collection.find_one_and_update(
                {'_id': case['_id']},
                {'$pull': {case_key: omim_number}},
                return_document=pymongo.ReturnDocument.AFTER
            )

        elif omim_number not in diagnosis_list:
            updated_case = self.case_collection.find_one_and_update(
                {'_id': case['_id']},
                {'$push': {case_key: omim_number}},
                return_document=pymongo.ReturnDocument.AFTER
            )

        if updated_case:
            self.create_event(
                institute=institute,
                case=case,
                user=user,
                link=link,
                category='case',
                verb='update_diagnosis',
                subject=case['display_name'],
                content=omim_id
            )

        return updated_case

def
diagnose
(
self
,
institute
,
case
,
user
,
link
,
level
,
omim_id
,
remove
=
False
)
:
if
level
==
'phenotype'
:
case_key
=
'diagnosis_phenotypes'
elif
level
==
'gene'
:
case_key
=
'diagnosis_genes'
else
:
raise
TypeError
(
'wrong level'
)
diagnosis_list
=
case
.
get
(
case_key
,
[
]
)
omim_number
=
int
(
omim_id
.
split
(
':'
)
[
1
]
)
updated_case
=
if
remove
and
omim_number
in
diagnosis_list
:
updated_case
=
self
.
case_collection
.
find_one_and_update
(
{
'_id'
:
case
[
'_id'
]
}
,
{
'$pull'
:
{
case_key
:
omim_number
}
}
,
return_document
=
pymongo
.
ReturnDocument
.
AFTER
)
elif
omim_number
not
in
diagnosis_list
:
updated_case
=
self
.
case_collection
.
find_one_and_update
(
{
'_id'
:
case
[
'_id'
]
}
,
{
'$push'
:
{
case_key
:
omim_number
}
}
,
return_document
=
pymongo
.
ReturnDocument
.
AFTER
)
if
updated_case
:
self
.
create_event
(
institute
=
institute
,
case
=
case
,
user
=
user
,
link
=
link
,
category
=
'case'
,
verb
=
'update_diagnosis'
,
subject
=
case
[
'display_name'
]
,
content
=
omim_id
)
return
updated_case


def mark_checked(self, institute, case, user, link,
                     unmark=False):
        """Mark a case as checked from an analysis point of view.

        Arguments:
            institute (dict): A Institute object
            case (dict): Case object
            user (dict): A User object
            link (str): The url to be used in the event
            unmark (bool): If case should ve unmarked

        Return:
            updated_case
        """

        LOG.info("Updating checked status of {}"
                    .format(case['display_name']))

        status = 'not checked' if unmark else 'checked'
        self.create_event(
            institute=institute,
            case=case,
            user=user,
            link=link,
            category='case',
            verb='check_case',
            subject=status
        )

        LOG.info("Updating {0}'s checked status {1}"
                    .format(case['display_name'], status))
        analysis_checked = False if unmark else True
        updated_case = self.case_collection.find_one_and_update(
            {'_id': case['_id']},
            {
                '$set': {'analysis_checked': analysis_checked}
            },
            return_document=pymongo.ReturnDocument.AFTER
        )
        LOG.debug("Case updated")
        return updated_case

def
mark_checked
(
self
,
institute
,
case
,
user
,
link
,
unmark
=
False
)
:
LOG
.
info
(
"Updating checked status of {}"
.
format
(
case
[
'display_name'
]
)
)
status
=
'not checked'
if
unmark
else
'checked'
self
.
create_event
(
institute
=
institute
,
case
=
case
,
user
=
user
,
link
=
link
,
category
=
'case'
,
verb
=
'check_case'
,
subject
=
status
)
LOG
.
info
(
"Updating {0}'s checked status {1}"
.
format
(
case
[
'display_name'
]
,
status
)
)
analysis_checked
=
False
if
unmark
else
True
updated_case
=
self
.
case_collection
.
find_one_and_update
(
{
'_id'
:
case
[
'_id'
]
}
,
{
'$set'
:
{
'analysis_checked'
:
analysis_checked
}
}
,
return_document
=
pymongo
.
ReturnDocument
.
AFTER
)
LOG
.
debug
(
"Case updated"
)
return
updated_case

Mark a case as checked from an analysis point of view .


def order_verification(self, institute, case, user, link, variant):
        """Create an event for a variant verification for a variant
        and an event for a variant verification for a case

        Arguments:
            institute (dict): A Institute object
            case (dict): Case object
            user (dict): A User object
            link (str): The url to be used in the event
            variant (dict): A variant object

        Returns:
            updated_variant(dict)
        """
        LOG.info("Creating event for ordering validation for variant" \
                    " {0}".format(variant['display_name']))

        updated_variant = self.variant_collection.find_one_and_update(
            {'_id': variant['_id']},
            {'$set': {'sanger_ordered': True}},
            return_document=pymongo.ReturnDocument.AFTER
        )

        self.create_event(
            institute=institute,
            case=case,
            user=user,
            link=link,
            category='variant',
            verb='sanger',
            variant=variant,
            subject=variant['display_name'],
        )

        LOG.info("Creating event for ordering sanger for case" \
                    " {0}".format(case['display_name']))

        self.create_event(
            institute=institute,
            case=case,
            user=user,
            link=link,
            category='case',
            verb='sanger',
            variant=variant,
            subject=variant['display_name'],
        )
        return updated_variant


def sanger_ordered(self, institute_id=None, user_id=None):
        """Get all variants with validations ever ordered.

        Args:
            institute_id(str) : The id of an institute
            user_id(str) : The id of an user

        Returns:
            sanger_ordered(list) : a list of dictionaries, each with "case_id" as keys and list of variant ids as values
        """
        query = {'$match': {
                '$and': [
                    {'verb': 'sanger'},
                ],
            }}

        if institute_id:
            query['$match']['$and'].append({'institute': institute_id})
        if user_id:
            query['$match']['$and'].append({'user_id': user_id})

        # Get all sanger ordered variants grouped by case_id
        results = self.event_collection.aggregate([
            query,
            {'$group': {
                '_id': "$case",
                'vars': {'$addToSet' : '$variant_id'}
            }}
        ])

        sanger_ordered =  [item for item in results]
        return sanger_ordered

Get all variants with validations ever ordered .


def validate(self, institute, case, user, link, variant, validate_type):
        """Mark validation status for a variant.

        Arguments:
            institute (dict): A Institute object
            case (dict): Case object
            user (dict): A User object
            link (str): The url to be used in the event
            variant (dict): A variant object
            validate_type(str): The outcome of validation.
                                choices=('True positive', 'False positive')

        Returns:
            updated_variant(dict)
        """
        if not validate_type in SANGER_OPTIONS:
            LOG.warning("Invalid validation string: %s", validate_type)
            LOG.info("Validation options: %s", ', '.join(SANGER_OPTIONS))
            return

        updated_variant = self.variant_collection.find_one_and_update(
            {'_id': variant['_id']},
            {'$set': {'validation': validate_type}},
            return_document=pymongo.ReturnDocument.AFTER
        )

        self.create_event(
            institute=institute,
            case=case,
            user=user,
            link=link,
            category='variant',
            verb='validate',
            variant=variant,
            subject=variant['display_name'],
        )
        return updated_variant

def
validate
(
self
,
institute
,
case
,
user
,
link
,
variant
,
validate_type
)
:
if
not
validate_type
in
SANGER_OPTIONS
:
LOG
.
warning
(
"Invalid validation string: %s"
,
validate_type
)
LOG
.
info
(
"Validation options: %s"
,
', '
.
join
(
SANGER_OPTIONS
)
)
return
updated_variant
=
self
.
variant_collection
.
find_one_and_update
(
{
'_id'
:
variant
[
'_id'
]
}
,
{
'$set'
:
{
'validation'
:
validate_type
}
}
,
return_document
=
pymongo
.
ReturnDocument
.
AFTER
)
self
.
create_event
(
institute
=
institute
,
case
=
case
,
user
=
user
,
link
=
link
,
category
=
'variant'
,
verb
=
'validate'
,
variant
=
variant
,
subject
=
variant
[
'display_name'
]
,
)
return
updated_variant


def mark_causative(self, institute, case, user, link, variant):
        """Create an event for marking a variant causative.

        Arguments:
          institute (dict): A Institute object
          case (dict): Case object
          user (dict): A User object
          link (str): The url to be used in the event
          variant (variant): A variant object

        Returns:
            updated_case(dict)
        """
        display_name = variant['display_name']
        LOG.info("Mark variant {0} as causative in the case {1}".format(
            display_name, case['display_name']))

        LOG.info("Adding variant to causatives in case {0}".format(
            case['display_name']))

        LOG.info("Marking case {0} as solved".format(
            case['display_name']))

        updated_case = self.case_collection.find_one_and_update(
            {'_id': case['_id']},
            {
                '$push': {'causatives': variant['_id']},
                '$set': {'status': 'solved'}
            },
            return_document=pymongo.ReturnDocument.AFTER
        )

        LOG.info("Creating case event for marking {0}" \
                    " causative".format(variant['display_name']))

        self.create_event(
            institute=institute,
            case=case,
            user=user,
            link=link,
            category='case',
            verb='mark_causative',
            variant=variant,
            subject=variant['display_name'],
        )

        LOG.info("Creating variant event for marking {0}" \
                    " causative".format(case['display_name']))

        self.create_event(
            institute=institute,
            case=case,
            user=user,
            link=link,
            category='variant',
            verb='mark_causative',
            variant=variant,
            subject=variant['display_name'],
        )
        return updated_case

Create an event for marking a variant causative .


def update_dismiss_variant(self, institute, case, user, link, variant,
                               dismiss_variant):
        """Create an event for updating the manual dismiss variant entry

          This function will create a event and update the dismiss variant
          field of the variant.

        Arguments:
            institute (dict): A Institute object
            case (dict): Case object
            user (dict): A User object
            link (str): The url to be used in the event
            variant (dict): A variant object
            dismiss_variant (list): The new dismiss variant list

        Return:
            updated_variant

        """
        LOG.info("Creating event for updating dismiss variant for "
                    "variant {0}".format(variant['display_name']))

        self.create_event(
            institute=institute,
            case=case,
            user=user,
            link=link,
            category='variant',
            verb='dismiss_variant',
            variant=variant,
            subject=variant['display_name'],
        )

        if dismiss_variant:
            LOG.info("Setting dismiss variant to {0} for variant {1}"
                        .format(dismiss_variant, variant['display_name']))
            action = '$set'
        else:
            LOG.info("Reset dismiss variant from {0} for variant {1}"
                        .format(variant['dismiss_variant'], variant['display_name']))
            action = '$unset'

        updated_variant = self.variant_collection.find_one_and_update(
            {'_id': variant['_id']},
            {action: {'dismiss_variant': dismiss_variant}},
            return_document=pymongo.ReturnDocument.AFTER
        )
        LOG.debug("Variant updated")
        return updated_variant


def update_acmg(self, institute_obj, case_obj, user_obj, link, variant_obj, acmg_str):
        """Create an event for updating the ACMG classification of a variant.

        Arguments:
            institute_obj (dict): A Institute object
            case_obj (dict): Case object
            user_obj (dict): A User object
            link (str): The url to be used in the event
            variant_obj (dict): A variant object
            acmg_str (str): The new ACMG classification string

        Returns:
            updated_variant
        """
        self.create_event(
            institute=institute_obj,
            case=case_obj,
            user=user_obj,
            link=link,
            category='variant',
            verb='acmg',
            variant=variant_obj,
            subject=variant_obj['display_name'],
        )
        LOG.info("Setting ACMG to {} for: {}".format(acmg_str, variant_obj['display_name']))

        if acmg_str is None:
            updated_variant = self.variant_collection.find_one_and_update(
                {'_id': variant_obj['_id']},
                {'$unset': {'acmg_classification': 1}},
                return_document=pymongo.ReturnDocument.AFTER
            )
        else:
            updated_variant = self.variant_collection.find_one_and_update(
                {'_id': variant_obj['_id']},
                {'$set': {'acmg_classification': REV_ACMG_MAP[acmg_str]}},
                return_document=pymongo.ReturnDocument.AFTER
            )

        LOG.debug("Variant updated")
        return updated_variant

def
update_acmg
(
self
,
institute_obj
,
case_obj
,
user_obj
,
link
,
variant_obj
,
acmg_str
)
:
self
.
create_event
(
institute
=
institute_obj
,
case
=
case_obj
,
user
=
user_obj
,
link
=
link
,
category
=
'variant'
,
verb
=
'acmg'
,
variant
=
variant_obj
,
subject
=
variant_obj
[
'display_name'
]
,
)
LOG
.
info
(
"Setting ACMG to {} for: {}"
.
format
(
acmg_str
,
variant_obj
[
'display_name'
]
)
)
if
acmg_str
is
:
updated_variant
=
self
.
variant_collection
.
find_one_and_update
(
{
'_id'
:
variant_obj
[
'_id'
]
}
,
{
'$unset'
:
{
'acmg_classification'
:
1
}
}
,
return_document
=
pymongo
.
ReturnDocument
.
AFTER
)
else
:
updated_variant
=
self
.
variant_collection
.
find_one_and_update
(
{
'_id'
:
variant_obj
[
'_id'
]
}
,
{
'$set'
:
{
'acmg_classification'
:
REV_ACMG_MAP
[
acmg_str
]
}
}
,
return_document
=
pymongo
.
ReturnDocument
.
AFTER
)
LOG
.
debug
(
"Variant updated"
)
return
updated_variant

Create an event for updating the ACMG classification of a variant .


def parse_ids(chrom, pos, ref, alt, case_id, variant_type):
    """Construct the necessary ids for a variant

    Args:
        chrom(str): Variant chromosome
        pos(int): Variant position
        ref(str): Variant reference
        alt(str): Variant alternative
        case_id(str): Unique case id
        variant_type(str): 'clinical' or 'research'

    Returns:
        ids(dict): Dictionary with the relevant ids
    """
    ids = {}
    pos = str(pos)

    ids['simple_id'] = parse_simple_id(chrom, pos, ref, alt)
    ids['variant_id'] = parse_variant_id(chrom, pos, ref, alt, variant_type)
    ids['display_name'] = parse_display_name(chrom, pos, ref, alt, variant_type)
    ids['document_id'] = parse_document_id(chrom, pos, ref, alt, variant_type, case_id)

    return ids

def parse_ids ( chrom , pos , ref , alt , case_id , variant_type ) : ids = { } pos = str ( pos ) ids [ 'simple_id' ] = parse_simple_id ( chrom , pos , ref , alt ) ids [ 'variant_id' ] = parse_variant_id ( chrom , pos , ref , alt , variant_type ) ids [ 'display_name' ] = parse_display_name ( chrom , pos , ref , alt , variant_type ) ids [ 'document_id' ] = parse_document_id ( chrom , pos , ref , alt , variant_type , case_id ) return ids


def parse_simple_id(chrom, pos, ref, alt):
    """Parse the simple id for a variant

    Simple id is used as a human readable reference for a position, it is
    in no way unique.

    Args:
        chrom(str)
        pos(str)
        ref(str)
        alt(str)

    Returns:
        simple_id(str): The simple human readable variant id
    """
    return '_'.join([chrom, pos, ref, alt])

def parse_simple_id ( chrom , pos , ref , alt ) : return '_' . join ( [ chrom , pos , ref , alt ] )


def parse_document_id(chrom, pos, ref, alt, variant_type, case_id):
    """Parse the unique document id for a variant.

    This will always be unique in the database.

    Args:
        chrom(str)
        pos(str)
        ref(str)
        alt(str)
        variant_type(str): 'clinical' or 'research'
        case_id(str): unqiue family id

    Returns:
        document_id(str): The unique document id in an md5 string
    """
    return generate_md5_key([chrom, pos, ref, alt, variant_type, case_id])

def parse_document_id ( chrom , pos , ref , alt , variant_type , case_id ) : return generate_md5_key ( [ chrom , pos , ref , alt , variant_type , case_id ] )

Parse the unique document id for a variant .


def convert(context, panel):
    """Convert a gene panel with hgnc symbols to a new one with hgnc ids."""
    adapter = context.obj['adapter']
    new_header = ["hgnc_id","hgnc_symbol","disease_associated_transcripts",
                 "reduced_penetrance", "genetic_disease_models", "mosaicism",
                 "database_entry_version"]

    genes = parse_genes(panel)

    adapter.add_hgnc_id(genes)

    click.echo("#{0}".format('\t'.join(new_header)))
    for gene in genes:
        if gene.get('hgnc_id'):
            print_info = []
            for head in new_header:
                print_info.append(str(gene[head]) if gene.get(head) else '')

            click.echo('\t'.join(print_info))

def convert ( context , panel ) : adapter = context . obj [ 'adapter' ] new_header = [ "hgnc_id" , "hgnc_symbol" , "disease_associated_transcripts" , "reduced_penetrance" , "genetic_disease_models" , "mosaicism" , "database_entry_version" ] genes = parse_genes ( panel ) adapter . add_hgnc_id ( genes ) click . echo ( "#{0}" . format ( '\t' . join ( new_header ) ) ) for gene in genes : if gene . get ( 'hgnc_id' ) : print_info = [ ] for head in new_header : print_info . append ( str ( gene [ head ] ) if gene . get ( head ) else '' ) click . echo ( '\t' . join ( print_info ) )

Convert a gene panel with hgnc symbols to a new one with hgnc ids .


def get_variantid(variant_obj, family_id):
    """Create a new variant id.

    Args:
        variant_obj(dict)
        family_id(str)

    Returns:
        new_id(str): The new variant id
    """
    new_id = parse_document_id(
        chrom=variant_obj['chromosome'],
        pos=str(variant_obj['position']),
        ref=variant_obj['reference'],
        alt=variant_obj['alternative'],
        variant_type=variant_obj['variant_type'],
        case_id=family_id,
    )
    return new_id

def get_variantid ( variant_obj , family_id ) : new_id = parse_document_id ( chrom = variant_obj [ 'chromosome' ] , pos = str ( variant_obj [ 'position' ] ) , ref = variant_obj [ 'reference' ] , alt = variant_obj [ 'alternative' ] , variant_type = variant_obj [ 'variant_type' ] , case_id = family_id , ) return new_id


def nr_cases(self, institute_id=None):
        """Return the number of cases

        This function will change when we migrate to 3.7.1

        Args:
            collaborator(str): Institute id

        Returns:
            nr_cases(int)
        """
        query = {}

        if institute_id:
            query['collaborators'] = institute_id

        LOG.debug("Fetch all cases with query {0}".format(query))
        nr_cases = self.case_collection.find(query).count()

        return nr_cases

def
nr_cases
(
self
,
institute_id
=
)
:
query
=
{
}
if
institute_id
:
query
[
'collaborators'
]
=
institute_id
LOG
.
debug
(
"Fetch all cases with query {0}"
.
format
(
query
)
)
nr_cases
=
self
.
case_collection
.
find
(
query
)
.
count
(
)
return
nr_cases


def update_dynamic_gene_list(self, case, hgnc_symbols=None, hgnc_ids=None,
                                 phenotype_ids=None, build='37'):
        """Update the dynamic gene list for a case

        Adds a list of dictionaries to case['dynamic_gene_list'] that looks like

        {
            hgnc_symbol: str,
            hgnc_id: int,
            description: str
        }

        Arguments:
            case (dict): The case that should be updated
            hgnc_symbols (iterable): A list of hgnc_symbols
            hgnc_ids (iterable): A list of hgnc_ids

        Returns:
            updated_case(dict)
        """
        dynamic_gene_list = []
        res = []
        if hgnc_ids:
            LOG.info("Fetching genes by hgnc id")
            res = self.hgnc_collection.find({'hgnc_id': {'$in': hgnc_ids}, 'build': build})
        elif hgnc_symbols:
            LOG.info("Fetching genes by hgnc symbols")
            res = []
            for symbol in hgnc_symbols:
                for gene_obj in self.gene_by_alias(symbol=symbol, build=build):
                    res.append(gene_obj)

        for gene_obj in res:
            dynamic_gene_list.append(
                {
                    'hgnc_symbol': gene_obj['hgnc_symbol'],
                    'hgnc_id': gene_obj['hgnc_id'],
                    'description': gene_obj['description'],
                }
            )

        LOG.info("Update dynamic gene panel for: %s", case['display_name'])
        updated_case = self.case_collection.find_one_and_update(
            {'_id': case['_id']},
            {'$set': {'dynamic_gene_list': dynamic_gene_list,
                      'dynamic_panel_phenotypes': phenotype_ids or []}},
            return_document=pymongo.ReturnDocument.AFTER
        )
        LOG.debug("Case updated")
        return updated_case

def
update_dynamic_gene_list
(
self
,
case
,
hgnc_symbols
=
,
hgnc_ids
=
,
phenotype_ids
=
,
build
=
'37'
)
:
dynamic_gene_list
=
[
]
res
=
[
]
if
hgnc_ids
:
LOG
.
info
(
"Fetching genes by hgnc id"
)
res
=
self
.
hgnc_collection
.
find
(
{
'hgnc_id'
:
{
'$in'
:
hgnc_ids
}
,
'build'
:
build
}
)
elif
hgnc_symbols
:
LOG
.
info
(
"Fetching genes by hgnc symbols"
)
res
=
[
]
for
symbol
in
hgnc_symbols
:
for
gene_obj
in
self
.
gene_by_alias
(
symbol
=
symbol
,
build
=
build
)
:
res
.
append
(
gene_obj
)
for
gene_obj
in
res
:
dynamic_gene_list
.
append
(
{
'hgnc_symbol'
:
gene_obj
[
'hgnc_symbol'
]
,
'hgnc_id'
:
gene_obj
[
'hgnc_id'
]
,
'description'
:
gene_obj
[
'description'
]
,
}
)
LOG
.
info
(
"Update dynamic gene panel for: %s"
,
case
[
'display_name'
]
)
updated_case
=
self
.
case_collection
.
find_one_and_update
(
{
'_id'
:
case
[
'_id'
]
}
,
{
'$set'
:
{
'dynamic_gene_list'
:
dynamic_gene_list
,
'dynamic_panel_phenotypes'
:
phenotype_ids
or
[
]
}
}
,
return_document
=
pymongo
.
ReturnDocument
.
AFTER
)
LOG
.
debug
(
"Case updated"
)
return
updated_case


def case(self, case_id=None, institute_id=None, display_name=None):
        """Fetches a single case from database

        Use either the _id or combination of institute_id and display_name

        Args:
            case_id(str): _id for a caes
            institute_id(str):
            display_name(str)

        Yields:
            A single Case
        """
        query = {}
        if case_id:
            query['_id'] = case_id
            LOG.info("Fetching case %s", case_id)
        else:
            if not (institute_id and display_name):
                raise ValueError("Have to provide both institute_id and display_name")
            LOG.info("Fetching case %s institute %s", display_name, institute_id)
            query['owner'] = institute_id
            query['display_name'] = display_name

        return self.case_collection.find_one(query)


def delete_case(self, case_id=None, institute_id=None, display_name=None):
        """Delete a single case from database

        Args:
            institute_id(str)
            case_id(str)

        Returns:
            case_obj(dict): The case that was deleted
        """
        query = {}
        if case_id:
            query['_id'] = case_id
            LOG.info("Deleting case %s", case_id)
        else:
            if not (institute_id and display_name):
                raise ValueError("Have to provide both institute_id and display_name")
            LOG.info("Deleting case %s institute %s", display_name, institute_id)
            query['owner'] = institute_id
            query['display_name'] = display_name

        result = self.case_collection.delete_one(query)
        return result


def _add_case(self, case_obj):
        """Add a case to the database
           If the case already exists exception is raised

            Args:
                case_obj(Case)
        """
        if self.case(case_obj['_id']):
            raise IntegrityError("Case %s already exists in database" % case_obj['_id'])

        return self.case_collection.insert_one(case_obj)

def
_add_case
(
self
,
case_obj
)
:
if
self
.
case
(
case_obj
[
'_id'
]
)
:
raise
IntegrityError
(
"Case %s already exists in database"
%
case_obj
[
'_id'
]
)
return
self
.
case_collection
.
insert_one
(
case_obj
)


def replace_case(self, case_obj):
        """Replace a existing case with a new one

        Keeps the object id

        Args:
            case_obj(dict)

        Returns:
            updated_case(dict)
        """
        # Todo: Figure out and describe when this method destroys a case if invoked instead of
        # update_case
        LOG.info("Saving case %s", case_obj['_id'])
        # update updated_at of case to "today"

        case_obj['updated_at'] = datetime.datetime.now(),

        updated_case = self.case_collection.find_one_and_replace(
            {'_id': case_obj['_id']},
            case_obj,
            return_document=pymongo.ReturnDocument.AFTER
        )

        return updated_case


def update_caseid(self, case_obj, family_id):
        """Update case id for a case across the database.

        This function is used when a case is a rerun or updated for another reason.

        Args:
            case_obj(dict)
            family_id(str): The new family id

        Returns:
            new_case(dict): The updated case object

        """
        new_case = deepcopy(case_obj)
        new_case['_id'] = family_id

        # update suspects and causatives
        for case_variants in ['suspects', 'causatives']:
            new_variantids = []
            for variant_id in case_obj.get(case_variants, []):
                case_variant = self.variant(variant_id)
                if not case_variant:
                    continue
                new_variantid = get_variantid(case_variant, family_id)
                new_variantids.append(new_variantid)
            new_case[case_variants] = new_variantids

        # update ACMG
        for acmg_obj in self.acmg_collection.find({'case_id': case_obj['_id']}):
            LOG.info("update ACMG classification: %s", acmg_obj['classification'])
            acmg_variant = self.variant(acmg_obj['variant_specific'])
            new_specific_id = get_variantid(acmg_variant, family_id)
            self.acmg_collection.find_one_and_update(
                {'_id': acmg_obj['_id']},
                {'$set': {'case_id': family_id, 'variant_specific': new_specific_id}},
            )

        # update events
        institute_obj = self.institute(case_obj['owner'])
        for event_obj in self.events(institute_obj, case=case_obj):
            LOG.info("update event: %s", event_obj['verb'])
            self.event_collection.find_one_and_update(
                {'_id': event_obj['_id']},
                {'$set': {'case': family_id}},
            )

        # insert the updated case
        self.case_collection.insert_one(new_case)
        # delete the old case
        self.case_collection.find_one_and_delete({'_id': case_obj['_id']})
        return new_case

def
update_caseid
(
self
,
case_obj
,
family_id
)
:
new_case
=
deepcopy
(
case_obj
)
new_case
[
'_id'
]
=
family_id
# update suspects and causatives
for
case_variants
in
[
'suspects'
,
'causatives'
]
:
new_variantids
=
[
]
for
variant_id
in
case_obj
.
get
(
case_variants
,
[
]
)
:
case_variant
=
self
.
variant
(
variant_id
)
if
not
case_variant
:
continue
new_variantid
=
get_variantid
(
case_variant
,
family_id
)
new_variantids
.
append
(
new_variantid
)
new_case
[
case_variants
]
=
new_variantids
# update ACMG
for
acmg_obj
in
self
.
acmg_collection
.
find
(
{
'case_id'
:
case_obj
[
'_id'
]
}
)
:
LOG
.
info
(
"update ACMG classification: %s"
,
acmg_obj
[
'classification'
]
)
acmg_variant
=
self
.
variant
(
acmg_obj
[
'variant_specific'
]
)
new_specific_id
=
get_variantid
(
acmg_variant
,
family_id
)
self
.
acmg_collection
.
find_one_and_update
(
{
'_id'
:
acmg_obj
[
'_id'
]
}
,
{
'$set'
:
{
'case_id'
:
family_id
,
'variant_specific'
:
new_specific_id
}
}
,
)
# update events
institute_obj
=
self
.
institute
(
case_obj
[
'owner'
]
)
for
event_obj
in
self
.
events
(
institute_obj
,
case
=
case_obj
)
:
LOG
.
info
(
"update event: %s"
,
event_obj
[
'verb'
]
)
self
.
event_collection
.
find_one_and_update
(
{
'_id'
:
event_obj
[
'_id'
]
}
,
{
'$set'
:
{
'case'
:
family_id
}
}
,
)
# insert the updated case
self
.
case_collection
.
insert_one
(
new_case
)
# delete the old case
self
.
case_collection
.
find_one_and_delete
(
{
'_id'
:
case_obj
[
'_id'
]
}
)
return
new_case

Update case id for a case across the database .


def submit_evaluation(self, variant_obj, user_obj, institute_obj, case_obj, link, criteria):
        """Submit an evaluation to the database

        Get all the relevant information, build a evaluation_obj

        Args:
            variant_obj(dict)
            user_obj(dict)
            institute_obj(dict)
            case_obj(dict)
            link(str): variant url
            criteria(list(dict)):

                [
            {
            'term': str,
            'comment': str,
            'links': list(str)
            },
            .
            .
        ]
        """
        variant_specific = variant_obj['_id']
        variant_id = variant_obj['variant_id']
        user_id = user_obj['_id']
        user_name = user_obj.get('name', user_obj['_id'])
        institute_id = institute_obj['_id']
        case_id = case_obj['_id']

        evaluation_terms = [evluation_info['term'] for evluation_info in criteria]

        classification = get_acmg(evaluation_terms)

        evaluation_obj = build_evaluation(
            variant_specific=variant_specific,
            variant_id=variant_id,
            user_id=user_id,
            user_name=user_name,
            institute_id=institute_id,
            case_id=case_id,
            classification=classification,
            criteria=criteria
        )

        self._load_evaluation(evaluation_obj)

        # Update the acmg classification for the variant:
        self.update_acmg(institute_obj, case_obj, user_obj, link, variant_obj, classification)
        return classification


def get_evaluations(self, variant_obj):
        """Return all evaluations for a certain variant.

        Args:
            variant_obj (dict): variant dict from the database

        Returns:
            pymongo.cursor: database cursor
        """
        query = dict(variant_id=variant_obj['variant_id'])
        res = self.acmg_collection.find(query).sort([('created_at', pymongo.DESCENDING)])
        return res

def get_evaluations ( self , variant_obj ) : query = dict ( variant_id = variant_obj [ 'variant_id' ] ) res = self . acmg_collection . find ( query ) . sort ( [ ( 'created_at' , pymongo . DESCENDING ) ] ) return res

Return all evaluations for a certain variant .


def parse_transcripts(transcript_lines):
    """Parse and massage the transcript information

    There could be multiple lines with information about the same transcript.
    This is why it is necessary to parse the transcripts first and then return a dictionary
    where all information has been merged.

    Args:
        transcript_lines(): This could be an iterable with strings or a pandas.DataFrame

    Returns:
        parsed_transcripts(dict): Map from enstid -> transcript info
    """
    LOG.info("Parsing transcripts")
    # Parse the transcripts, we need to check if it is a request or a file handle
    if isinstance(transcript_lines, DataFrame):
        transcripts = parse_ensembl_transcript_request(transcript_lines)
    else:
        transcripts = parse_ensembl_transcripts(transcript_lines)

    # Since there can be multiple lines with information about the same transcript
    # we store transcript information in a dictionary for now
    parsed_transcripts = {}
    # Loop over the parsed transcripts
    for tx in transcripts:
        tx_id = tx['ensembl_transcript_id']
        ens_gene_id = tx['ensembl_gene_id']

        # Check if the transcript has been added
        # If not, create a new transcript
        if not tx_id in parsed_transcripts:
            tx_info = {
                'chrom': tx['chrom'],
                'transcript_start': tx['transcript_start'],
                'transcript_end': tx['transcript_end'],
                'mrna': set(),
                'mrna_predicted': set(),
                'nc_rna': set(),
                'ensembl_gene_id': ens_gene_id,
                'ensembl_transcript_id': tx_id,
            }
            parsed_transcripts[tx_id] = tx_info

        tx_info = parsed_transcripts[tx_id]
        # Add the ref seq information
        if tx.get('refseq_mrna_predicted'):
            tx_info['mrna_predicted'].add(tx['refseq_mrna_predicted'])
        if tx.get('refseq_mrna'):
            tx_info['mrna'].add(tx['refseq_mrna'])
        if tx.get('refseq_ncrna'):
            tx_info['nc_rna'].add(tx['refseq_ncrna'])

    return parsed_transcripts


def parse_ensembl_gene_request(result):
    """Parse a dataframe with ensembl gene information

    Args:
        res(pandas.DataFrame)

    Yields:
        gene_info(dict)
    """
    LOG.info("Parsing genes from request")

    for index, row in result.iterrows():
        # print(index, row)
        ensembl_info = {}

        # Pandas represents missing data with nan which is a float
        if type(row['hgnc_symbol']) is float:
            # Skip genes without hgnc information
            continue

        ensembl_info['chrom'] = row['chromosome_name']
        ensembl_info['gene_start'] = int(row['start_position'])
        ensembl_info['gene_end'] = int(row['end_position'])
        ensembl_info['ensembl_gene_id'] = row['ensembl_gene_id']
        ensembl_info['hgnc_symbol'] = row['hgnc_symbol']

        hgnc_id = row['hgnc_id']

        if type(hgnc_id) is float:
            hgnc_id = int(hgnc_id)
        else:
            hgnc_id = int(hgnc_id.split(':')[-1])

        ensembl_info['hgnc_id'] = hgnc_id

        yield ensembl_info


def parse_ensembl_transcript_request(result):
    """Parse a dataframe with ensembl transcript information

    Args:
        res(pandas.DataFrame)

    Yields:
        transcript_info(dict)
    """
    LOG.info("Parsing transcripts from request")

    keys = [
        'chrom',
        'ensembl_gene_id',
        'ensembl_transcript_id',
        'transcript_start',
        'transcript_end',
        'refseq_mrna',
        'refseq_mrna_predicted',
        'refseq_ncrna',
    ]
    # for res in result.itertuples():
    for index, row in result.iterrows():
        ensembl_info = {}

        ensembl_info['chrom'] = str(row['chromosome_name'])
        ensembl_info['ensembl_gene_id'] = row['ensembl_gene_id']
        ensembl_info['ensembl_transcript_id'] = row['ensembl_transcript_id']

        ensembl_info['transcript_start'] = int(row['transcript_start'])
        ensembl_info['transcript_end'] = int(row['transcript_end'])

        # Check if refseq data is annotated
        # Pandas represent missing data with nan
        for key in keys[-3:]:
            if type(row[key]) is float:
                ensembl_info[key] = None
            else:
                ensembl_info[key] = row[key]
        yield ensembl_info


def parse_ensembl_line(line, header):
    """Parse an ensembl formated line

        Args:
            line(list): A list with ensembl gene info
            header(list): A list with the header info

        Returns:
            ensembl_info(dict): A dictionary with the relevant info
    """
    line = line.rstrip().split('\t')
    header = [head.lower() for head in header]
    raw_info = dict(zip(header, line))

    ensembl_info = {}

    for word in raw_info:
        value = raw_info[word]

        if not value:
            continue

        if 'chromosome' in word:
            ensembl_info['chrom'] = value

        if 'gene' in word:
            if 'id' in word:
                ensembl_info['ensembl_gene_id'] = value
            elif 'start' in word:
                ensembl_info['gene_start'] = int(value)
            elif 'end' in word:
                ensembl_info['gene_end'] = int(value)

        if 'hgnc symbol' in word:
            ensembl_info['hgnc_symbol'] = value
        if "gene name" in word:
            ensembl_info['hgnc_symbol'] = value

        if 'hgnc id' in word:
            ensembl_info['hgnc_id'] = int(value.split(':')[-1])

        if 'transcript' in word:
            if 'id' in word:
                ensembl_info['ensembl_transcript_id'] = value
            elif 'start' in word:
                ensembl_info['transcript_start'] = int(value)
            elif 'end' in word:
                ensembl_info['transcript_end'] = int(value)

        if 'exon' in word:
            if 'start' in word:
                ensembl_info['exon_start'] = int(value)
            elif 'end' in word:
                ensembl_info['exon_end'] = int(value)
            elif 'rank' in word:
                ensembl_info['exon_rank'] = int(value)

        if 'utr' in word:

            if 'start' in word:
                if '5' in word:
                    ensembl_info['utr_5_start'] = int(value)
                elif '3' in word:
                    ensembl_info['utr_3_start'] = int(value)
            elif 'end' in word:
                if '5' in word:
                    ensembl_info['utr_5_end'] = int(value)
                elif '3' in word:
                    ensembl_info['utr_3_end'] = int(value)

        if 'strand' in word:
            ensembl_info['strand'] = int(value)

        if 'refseq' in word:
            if 'mrna' in word:
                if 'predicted' in word:
                    ensembl_info['refseq_mrna_predicted'] = value
                else:
                    ensembl_info['refseq_mrna'] = value

            if 'ncrna' in word:
                ensembl_info['refseq_ncrna'] = value

    return ensembl_info

def
parse_ensembl_line
(
line
,
header
)
:
line
=
line
.
rstrip
(
)
.
split
(
'\t'
)
header
=
[
head
.
lower
(
)
for
head
in
header
]
raw_info
=
dict
(
zip
(
header
,
line
)
)
ensembl_info
=
{
}
for
word
in
raw_info
:
value
=
raw_info
[
word
]
if
not
value
:
continue
if
'chromosome'
in
word
:
ensembl_info
[
'chrom'
]
=
value
if
'gene'
in
word
:
if
'id'
in
word
:
ensembl_info
[
'ensembl_gene_id'
]
=
value
elif
'start'
in
word
:
ensembl_info
[
'gene_start'
]
=
int
(
value
)
elif
'end'
in
word
:
ensembl_info
[
'gene_end'
]
=
int
(
value
)
if
'hgnc symbol'
in
word
:
ensembl_info
[
'hgnc_symbol'
]
=
value
if
"gene name"
in
word
:
ensembl_info
[
'hgnc_symbol'
]
=
value
if
'hgnc id'
in
word
:
ensembl_info
[
'hgnc_id'
]
=
int
(
value
.
split
(
':'
)
[
1
]
)
if
'transcript'
in
word
:
if
'id'
in
word
:
ensembl_info
[
'ensembl_transcript_id'
]
=
value
elif
'start'
in
word
:
ensembl_info
[
'transcript_start'
]
=
int
(
value
)
elif
'end'
in
word
:
ensembl_info
[
'transcript_end'
]
=
int
(
value
)
if
'exon'
in
word
:
if
'start'
in
word
:
ensembl_info
[
'exon_start'
]
=
int
(
value
)
elif
'end'
in
word
:
ensembl_info
[
'exon_end'
]
=
int
(
value
)
elif
'rank'
in
word
:
ensembl_info
[
'exon_rank'
]
=
int
(
value
)
if
'utr'
in
word
:
if
'start'
in
word
:
if
'5'
in
word
:
ensembl_info
[
'utr_5_start'
]
=
int
(
value
)
elif
'3'
in
word
:
ensembl_info
[
'utr_3_start'
]
=
int
(
value
)
elif
'end'
in
word
:
if
'5'
in
word
:
ensembl_info
[
'utr_5_end'
]
=
int
(
value
)
elif
'3'
in
word
:
ensembl_info
[
'utr_3_end'
]
=
int
(
value
)
if
'strand'
in
word
:
ensembl_info
[
'strand'
]
=
int
(
value
)
if
'refseq'
in
word
:
if
'mrna'
in
word
:
if
'predicted'
in
word
:
ensembl_info
[
'refseq_mrna_predicted'
]
=
value
else
:
ensembl_info
[
'refseq_mrna'
]
=
value
if
'ncrna'
in
word
:
ensembl_info
[
'refseq_ncrna'
]
=
value
return
ensembl_info


def parse_ensembl_genes(lines):
    """Parse lines with ensembl formated genes

        This is designed to take a biomart dump with genes from ensembl.
        Mandatory columns are:
        'Gene ID' 'Chromosome' 'Gene Start' 'Gene End' 'HGNC symbol

        Args:
            lines(iterable(str)): An iterable with ensembl formated genes
        Yields:
            ensembl_gene(dict): A dictionary with the relevant information
    """
    LOG.info("Parsing ensembl genes from file")
    header = []
    for index, line in enumerate(lines):

        # File allways start with a header line
        if index == 0:
            header = line.rstrip().split('\t')
            continue
        # After that each line represents a gene

        yield parse_ensembl_line(line, header)


def parse_ensembl_exons(lines):
    """Parse lines with ensembl formated exons

        This is designed to take a biomart dump with exons from ensembl.
        Check documentation for spec for download

        Args:
            lines(iterable(str)): An iterable with ensembl formated exons
        Yields:
            ensembl_gene(dict): A dictionary with the relevant information
    """
    header = []
    LOG.debug("Parsing ensembl exons...")
    for index, line in enumerate(lines):

        # File allways start with a header line
        if index == 0:
            header = line.rstrip().split('\t')
            continue

        exon_info = parse_ensembl_line(line, header)
        chrom = exon_info['chrom']
        start = exon_info['exon_start']
        end = exon_info['exon_end']
        transcript = exon_info['ensembl_transcript_id']
        gene = exon_info['ensembl_gene_id']

        rank = exon_info['exon_rank']
        strand = exon_info['strand']

        # Recalculate start and stop (taking UTR regions into account for end exons)
        if strand == 1:
            # highest position: start of exon or end of 5' UTR
            # If no 5' UTR make sure exon_start is allways choosen
            start = max(start, exon_info.get('utr_5_end') or -1)
            # lowest position: end of exon or start of 3' UTR
            end = min(end, exon_info.get('utr_3_start') or float('inf'))
        elif strand == -1:
            # highest position: start of exon or end of 3' UTR
            start = max(start, exon_info.get('utr_3_end') or -1)
            # lowest position: end of exon or start of 5' UTR
            end = min(end, exon_info.get('utr_5_start') or float('inf'))

        exon_id = "-".join([chrom, str(start), str(end)])

        if start > end:
            raise ValueError("ERROR: %s" % exon_id)
        data = {
            "exon_id": exon_id,
            "chrom": chrom,
            "start": start,
            "end": end,
            "transcript": transcript,
            "gene": gene,
            "rank": rank,
        }

        yield data


def parse_ensembl_exon_request(result):
    """Parse a dataframe with ensembl exon information

    Args:
        res(pandas.DataFrame)

    Yields:
        gene_info(dict)
    """
    keys = [
        'chrom',
        'gene',
        'transcript',
        'exon_id',
        'exon_chrom_start',
        'exon_chrom_end',
        '5_utr_start',
        '5_utr_end',
        '3_utr_start',
        '3_utr_end',
        'strand',
        'rank'
    ]

    # for res in result.itertuples():
    for res in zip(result['Chromosome/scaffold name'],
                   result['Gene stable ID'],
                   result['Transcript stable ID'],
                   result['Exon stable ID'],
                   result['Exon region start (bp)'],
                   result['Exon region end (bp)'],
                   result["5' UTR start"],
                   result["5' UTR end"],
                   result["3' UTR start"],
                   result["3' UTR end"],
                   result["Strand"],
                   result["Exon rank in transcript"]):
        ensembl_info = dict(zip(keys, res))

        # Recalculate start and stop (taking UTR regions into account for end exons)
        if ensembl_info['strand'] == 1:
            # highest position: start of exon or end of 5' UTR
            # If no 5' UTR make sure exon_start is allways choosen
            start = max(ensembl_info['exon_chrom_start'], ensembl_info['5_utr_end'] or -1)
            # lowest position: end of exon or start of 3' UTR
            end = min(ensembl_info['exon_chrom_end'], ensembl_info['3_utr_start'] or float('inf'))
        elif ensembl_info['strand'] == -1:
            # highest position: start of exon or end of 3' UTR
            start = max(ensembl_info['exon_chrom_start'], ensembl_info['3_utr_end'] or -1)
            # lowest position: end of exon or start of 5' UTR
            end = min(ensembl_info['exon_chrom_end'], ensembl_info['5_utr_start'] or float('inf'))

        ensembl_info['start'] = start
        ensembl_info['end'] = end

        yield ensembl_info


def init_log(logger, filename=None, loglevel=None):
    """
    Initializes the log file in the proper format.

    Arguments:

        filename (str): Path to a file. Or None if logging is to
                         be disabled.
        loglevel (str): Determines the level of the log output.
    """
    template = '[%(asctime)s] %(levelname)-8s: %(name)-25s: %(message)s'
    formatter = logging.Formatter(template)

    if loglevel:
        logger.setLevel(getattr(logging, loglevel))

    # We will always print warnings and higher to stderr
    console = logging.StreamHandler()
    console.setLevel('WARNING')
    console.setFormatter(formatter)

    if filename:
        file_handler = logging.FileHandler(filename, encoding='utf-8')
        if loglevel:
            file_handler.setLevel(getattr(logging, loglevel))
        file_handler.setFormatter(formatter)
        logger.addHandler(file_handler)
    # If no logfile is provided we print all log messages that the user has
    # defined to stderr
    else:
        if loglevel:
            console.setLevel(getattr(logging, loglevel))

    logger.addHandler(console)

Initializes the log file in the proper format .


def parse_omim_line(line, header):
    """docstring for parse_omim_2_line"""
    omim_info = dict(zip(header, line.split('\t')))
    return omim_info

def parse_omim_line ( line , header ) : omim_info = dict ( zip ( header , line . split ( '\t' ) ) ) return omim_info


def parse_omim_morbid(lines):
    """docstring for parse_omim_morbid"""
    header = []
    for i,line in enumerate(lines):
        line = line.rstrip()
        if line.startswith('#'):
            if i < 10:
                if line.startswith('# Phenotype'):
                    header = line[2:].split('\t')
        else:
            yield parse_omim_line(line, header)

def
parse_omim_morbid
(
lines
)
:
header
=
[
]
for
i
,
line
in
enumerate
(
lines
)
:
line
=
line
.
rstrip
(
)
if
line
.
startswith
(
'#'
)
:
if
i
<
10
:
if
line
.
startswith
(
'# Phenotype'
)
:
header
=
line
[
2
:
]
.
split
(
'\t'
)
else
:
yield
parse_omim_line
(
line
,
header
)


def get_mim_phenotypes(genemap_lines):
    """Get a dictionary with phenotypes

    Use the mim numbers for phenotypes as keys and phenotype information as
    values.

    Args:
        genemap_lines(iterable(str))

    Returns:
        phenotypes_found(dict): A dictionary with mim_numbers as keys and
        dictionaries with phenotype information as values.

        {
             'description': str, # Description of the phenotype
             'hgnc_symbols': set(), # Associated hgnc symbols
             'inheritance': set(),  # Associated phenotypes
             'mim_number': int, # mim number of phenotype
        }
    """
    # Set with all omim numbers that are phenotypes
    # Parsed from mim2gene.txt
    phenotype_mims = set()

    phenotypes_found = {}

    # Genemap is a file with one entry per gene.
    # Each line hold a lot of information and in specific it
    # has information about the phenotypes that a gene is associated with
    # From this source we collect inheritane patterns and what hgnc symbols
    # a phenotype is associated with
    for entry in parse_genemap2(genemap_lines):
        hgnc_symbol = entry['hgnc_symbol']
        for phenotype in entry['phenotypes']:
            mim_nr = phenotype['mim_number']
            if mim_nr in phenotypes_found:
                phenotype_entry = phenotypes_found[mim_nr]
                phenotype_entry['inheritance'] = phenotype_entry['inheritance'].union(phenotype['inheritance'])
                phenotype_entry['hgnc_symbols'].add(hgnc_symbol)
            else:
                phenotype['hgnc_symbols'] = set([hgnc_symbol])
                phenotypes_found[mim_nr] = phenotype

    return phenotypes_found

Get a dictionary with phenotypes Use the mim numbers for phenotypes as keys and phenotype information as values .


def cli(context, morbid, genemap, mim2gene, mim_titles, phenotypes):
    """Parse the omim files"""
    # if not (morbid and genemap and mim2gene, mim_titles):
    #     print("Please provide all files")
    #     context.abort()

    from scout.utils.handle import get_file_handle
    from pprint import pprint as pp

    print("Morbid file: %s" % morbid)
    print("Genemap file: %s" % genemap)
    print("mim2gene file: %s" % mim2gene)
    print("MimTitles file: %s" % mim_titles)

    if morbid:
        morbid_handle = get_file_handle(morbid)
    if genemap:
        genemap_handle = get_file_handle(genemap)
    if mim2gene:
        mim2gene_handle = get_file_handle(mim2gene)
    if mim_titles:
        mimtitles_handle = get_file_handle(mim_titles)

    mim_genes = get_mim_genes(genemap_handle, mim2gene_handle)
    for entry in mim_genes:
        if entry == 'C10orf11':
            pp(mim_genes[entry])

    context.abort()
    if phenotypes:
        if not genemap:
            click.echo("Please provide the genemap file")
            context.abort()
        phenotypes = get_mim_phenotypes(genemap_handle)
        for i,mim_term in enumerate(phenotypes):
            # pp(phenotypes[mim_term])
            pass

    print("Number of phenotypes found: %s" % i)

    context.abort()
    # hgnc_genes = get_mim_genes(genemap_handle, mim2gene_handle)
    # for hgnc_symbol in hgnc_genes:
    #     pp(hgnc_genes[hgnc_symbol])
    # phenotypes = get_mim_phenotypes(genemap_handle, mim2gene_handle, mimtitles_handle)
    # for mim_nr in phenotypes:
    #     pp(phenotypes[mim_nr])

    genes = get_mim_genes(genemap_handle, mim2gene_handle)
    for hgnc_symbol in genes:
        if hgnc_symbol == 'OPA1':
            print(genes[hgnc_symbol])


def convert_number(string):
    """Convert a string to number
    If int convert to int otherwise float

    If not possible return None
    """
    res = None
    if isint(string):
        res = int(string)
    elif isfloat(string):
        res = float(string)
    return res

def convert_number ( string ) : res = None if isint ( string ) : res = int ( string ) elif isfloat ( string ) : res = float ( string ) return res


def formatmonth(self, theyear, themonth, withyear=True, net=None, qs=None, template='happenings/partials/calendar/month_table.html'):
        """Return a formatted month as a table."""
        context = self.get_context()
        context['month_start_date'] = date(self.yr, self.mo, 1)
        context['week_rows'] = []
        for week in self.monthdays2calendar(theyear, themonth):
            week_row = []
            for day, weekday in week:
                week_row.append(self.formatday(day, weekday))
            context['week_rows'].append(week_row)

        nxt, prev = get_next_and_prev(net)
        extra_qs = ('&' + '&'.join(qs)) if qs else ''
        context['prev_qs'] = mark_safe('?cal_prev=%d%s' % (prev, extra_qs))
        context['next_qs'] = mark_safe('?cal_next=%d%s' % (nxt, extra_qs))
        context['withyear'] = withyear
        return render_to_string(template, context)


def formatday(self, day, weekday):
        """Set some commonly used variables."""
        self.wkday_not_today = '
' % (
            self.cssclasses[weekday])

        self.wkday_today = (
            '
' % (
                self.cssclasses[weekday])
        )
        if URLS_NAMESPACE:
            url_name = '%s:day_list' % (URLS_NAMESPACE)
        else:
            url_name = 'day_list'

        self.day_url = reverse(url_name, args=(self.yr, self.mo, day))
        self.day = day

        self.anch = '%d' % (
            self.day_url, day
        )

        self.end = '
	'


def formatmonthname(self, theyear, themonth, withyear=True):
        """
        Change colspan to "5", add "today" button, and return a month
        name as a table row.
        """
        display_month = month_name[themonth]

        if isinstance(display_month, six.binary_type) and self.encoding:
            display_month = display_month.decode(self.encoding)

        if withyear:
            s = u'%s %s' % (display_month, theyear)
        else:
            s = u'%s' % display_month
        return ('
'
                '<button id="cal-today-btn" class="btn btn-small">'
                'Today</button> %s
' % s)

Change colspan to 5 add today button and return a month name as a table row .


def popover_helper(self):
        """Populate variables used to build popovers."""
        # when
        display_month = month_name[self.mo]

        if isinstance(display_month, six.binary_type) and self.encoding:
            display_month = display_month.decode('utf-8')

        self.when = ('
When: ' + display_month + ' ' +
                     str(self.day) + ', ' + self.event.l_start_date.strftime(
                         LEGACY_CALENDAR_TIME_FORMAT).lstrip('0') + ' - ' +
                     self.event.l_end_date.strftime(LEGACY_CALENDAR_TIME_FORMAT).lstrip('0') +
                     '
')

        if self.event.location.exists():  # where
            self.where = '
Where: '
            for l in self.event.location.all():
                self.where += l.name
            self.where += '
'
        else:
            self.where = ''

        # description
        self.desc = '
Description: ' + self.event.description[:100]
        self.desc += ('...
' if len(self.event.description) > 100
                      else '
')

        self.event_url = self.event.get_absolute_url()  # url
        t = LEGACY_CALENDAR_TIME_FORMAT if self.event.l_start_date.minute else LEGACY_CALENDAR_HOUR_FORMAT
        self.title2 = (self.event.l_start_date.strftime(t).lstrip('0') +
                       ' ' + self.title)

def
popover_helper
(
self
)
:
# when
display_month
=
month_name
[
self
.
mo
]
if
isinstance
(
display_month
,
six
.
binary_type
)
and
self
.
encoding
:
display_month
=
display_month
.
decode
(
'utf-8'
)
self
.
when
=
(
'
When: '
+
display_month
+
' '
+
str
(
self
.
day
)
+
', '
+
self
.
event
.
l_start_date
.
strftime
(
LEGACY_CALENDAR_TIME_FORMAT
)
.
lstrip
(
'0'
)
+
' - '
+
self
.
event
.
l_end_date
.
strftime
(
LEGACY_CALENDAR_TIME_FORMAT
)
.
lstrip
(
'0'
)
+
'
'
)
if
self
.
event
.
location
.
exists
(
)
:
# where
self
.
where
=
'
Where: '
for
l
in
self
.
event
.
location
.
all
(
)
:
self
.
where
+=
l
.
name
self
.
where
+=
'
'
else
:
self
.
where
=
''
# description
self
.
desc
=
'
Description: '
+
self
.
event
.
description
[
:
100
]
self
.
desc
+=
(
'...
'
if
len
(
self
.
event
.
description
)
>
100
else
'
'
)
self
.
event_url
=
self
.
event
.
get_absolute_url
(
)
# url
t
=
LEGACY_CALENDAR_TIME_FORMAT
if
self
.
event
.
l_start_date
.
minute
else
LEGACY_CALENDAR_HOUR_FORMAT
self
.
title2
=
(
self
.
event
.
l_start_date
.
strftime
(
t
)
.
lstrip
(
'0'
)
+
' '
+
self
.
title
)

Populate variables used to build popovers .


def get_panel_info(panel_lines=None, panel_id=None, institute=None, version=None, date=None,
                   display_name=None):
    """Parse metadata for a gene panel

    For historical reasons it is possible to include all information about a gene panel in the
    header of a panel file. This function parses the header.

    Args:
        panel_lines(iterable(str))

    Returns:
        panel_info(dict): Dictionary with panel information
    """
    panel_info = {
        'panel_id': panel_id,
        'institute': institute,
        'version': version,
        'date': date,
        'display_name': display_name,
    }

    if panel_lines:
        for line in panel_lines:
            line = line.rstrip()
            if not line.startswith('##'):
                break

            info = line[2:].split('=')
            field = info[0]
            value = info[1]

            if not panel_info.get(field):
                panel_info[field] = value

    panel_info['date'] = get_date(panel_info['date'])

    return panel_info

def get_panel_info ( panel_lines = None , panel_id = None , institute = None , version = None , date = None , display_name = None ) : panel_info = { 'panel_id' : panel_id , 'institute' : institute , 'version' : version , 'date' : date , 'display_name' : display_name , } if panel_lines : for line in panel_lines : line = line . rstrip ( ) if not line . startswith ( '##' ) : break info = line [ 2 : ] . split ( '=' ) field = info [ 0 ] value = info [ 1 ] if not panel_info . get ( field ) : panel_info [ field ] = value panel_info [ 'date' ] = get_date ( panel_info [ 'date' ] ) return panel_info


def parse_gene(gene_info):
    """Parse a gene line with information from a panel file

        Args:
            gene_info(dict): dictionary with gene info

        Returns:
            gene(dict): A dictionary with the gene information
                {
                'hgnc_id': int,
                'hgnc_symbol': str,
                'disease_associated_transcripts': list(str),
                'inheritance_models': list(str),
                'mosaicism': bool,
                'reduced_penetrance': bool,
                'database_entry_version': str,
                }

    """
    gene = {}
    # This is either hgnc id or hgnc symbol
    identifier = None

    hgnc_id = None
    try:
        if 'hgnc_id' in gene_info:
            hgnc_id = int(gene_info['hgnc_id'])
        elif 'hgnc_idnumber' in gene_info:
            hgnc_id = int(gene_info['hgnc_idnumber'])
        elif 'hgncid' in gene_info:
            hgnc_id = int(gene_info['hgncid'])
    except ValueError as e:
        raise SyntaxError("Invalid hgnc id: {0}".format(hgnc_id))

    gene['hgnc_id'] = hgnc_id
    identifier = hgnc_id

    hgnc_symbol = None
    if 'hgnc_symbol' in gene_info:
        hgnc_symbol = gene_info['hgnc_symbol']
    elif 'hgncsymbol' in gene_info:
        hgnc_symbol = gene_info['hgncsymbol']
    elif 'symbol' in gene_info:
        hgnc_symbol = gene_info['symbol']

    gene['hgnc_symbol'] = hgnc_symbol

    if not identifier:
        if hgnc_symbol:
            identifier = hgnc_symbol
        else:
            raise SyntaxError("No gene identifier could be found")
    gene['identifier'] = identifier
    # Disease associated transcripts is a ','-separated list of
    # manually curated transcripts
    transcripts = ""
    if 'disease_associated_transcripts' in gene_info:
        transcripts = gene_info['disease_associated_transcripts']
    elif 'disease_associated_transcript' in gene_info:
        transcripts = gene_info['disease_associated_transcript']
    elif 'transcripts' in gene_info:
        transcripts = gene_info['transcripts']

    gene['transcripts'] = [
            transcript.strip() for transcript in
            transcripts.split(',') if transcript
        ]

    # Genetic disease models is a ','-separated list of manually curated
    # inheritance patterns that are followed for a gene
    models = ""
    if 'genetic_disease_models' in gene_info:
        models = gene_info['genetic_disease_models']
    elif 'genetic_disease_model' in gene_info:
        models = gene_info['genetic_disease_model']
    elif 'inheritance_models' in gene_info:
        models = gene_info['inheritance_models']
    elif 'genetic_inheritance_models' in gene_info:
        models = gene_info['genetic_inheritance_models']

    gene['inheritance_models'] = [
        model.strip() for model in models.split(',')
        if model.strip() in VALID_MODELS
    ]

    # If a gene is known to be associated with mosaicism this is annotated
    gene['mosaicism'] = True if gene_info.get('mosaicism') else False

    # If a gene is known to have reduced penetrance this is annotated
    gene['reduced_penetrance'] = True if gene_info.get('reduced_penetrance') else False

    # The database entry version is a way to track when a a gene was added or
    # modified, optional
    gene['database_entry_version'] = gene_info.get('database_entry_version')

    return gene