def _process_features(self, limit):
        """
        These are all of the genomic features genes, variations,
        transgenes, etc
        :param limit:
        :return:

        """

        if self.test_mode:
            graph = self.testgraph
        else:
            graph = self.graph
        model = Model(graph)
        raw = '/'.join((self.rawdir, 'feature'))
        LOG.info("building labels for features")

        line_counter = 0
        with open(raw, 'r') as f:
            filereader = csv.reader(f, delimiter='\t', quotechar='\"')
            f.readline()  # read the header row; skip
            for line in filereader:
                (feature_id, dbxref_id, organism_id, name, uniquename,
                 residues, seqlen, md5checksum, type_id, is_analysis,
                 timeaccessioned, timelastmodified) = line

                feature_key = feature_id
                if re.search(r'[\|\s\[\]\{\}\\<\>]', uniquename):
                    # some uniquenames have pipes or other nasty chars!
                    # for example: FB||||FBrf0133242|Hugh-u1
                    feature_id = self._makeInternalIdentifier('feature', feature_key)
                else:
                    feature_id = 'FlyBase:' + uniquename
                self.idhash['feature'][feature_key] = feature_id
                self.feature_types[feature_key] = type_id
                self.label_hash[feature_id] = name

                if feature_key not in self.feature_to_organism_hash:
                    self.feature_to_organism_hash[feature_key] = set()
                self.feature_to_organism_hash[feature_key].add(organism_id)

                # HACK - FBgn are genes, and therefore classes,
                # all else be individuals
                is_gene = False
                if re.search(r'(FBgn|FBog)', feature_id):
                    self.idhash['gene'][feature_key] = feature_id
                    is_gene = True
                elif re.search(r'FBa[lb]', feature_id):
                    self.idhash['allele'][feature_key] = feature_id
                elif re.search(r'FBt[ip]', feature_id):
                    self.idhash['feature'][feature_key] = feature_id

                if self.test_mode and \
                        feature_key not in self.test_keys['gene'] and \
                        feature_key not in self.test_keys['allele'] and \
                        feature_key not in self.test_keys['feature']:
                    continue

                # now do something with it!
                # switch on type_id
                if name.strip() == '':
                    name = uniquename

                type_key = type_id
                type_id = self.idhash['cvterm'][type_key]

                # skip some features by type
                types_to_skip = [
                    'SO:0000316',  # CDS
                    'SO:0000696',  # oligos
                    'SO:0000358',  # polypeptide
                    'SO:0000234',  # transcripts
                    ]

                type_keys_to_skip = [
                    596,    # pcr_product
                    57096,  # mature peptide
                    57097,  # signal_peptide
                    57270,  # repeat masker
                    58210,  # alignment
                    59643,  # cDNA_clone
                    60006,  # uncharacterized_change_in_nucleotide_sequence
                    61351,  # oligo
                    61467,  # polypeptide_domain
                    257,    # exon
                    286,    # intron
                ]

                organisms_to_skip = [
                    2  # computational result
                ]

                if type_id in types_to_skip \
                        or int(type_key) in type_keys_to_skip\
                        or int(organism_id) in organisms_to_skip:
                    continue

                line_counter += 1

                if int(type_key) == 604:  # RNAi_reagent
                    # TODO add other reagents?
                    self.idhash['reagent'][feature_key] = feature_id

                # deal with the taxonomy
                # only get taxa for features that are actually used in our set
                tax_internal_id = self._makeInternalIdentifier(
                    'organism', organism_id)
                if organism_id not in self.checked_organisms:
                    # will get the NCBITax if necessary
                    tax_id = self._get_organism_id(organism_id)
                    self.checked_organisms.add(organism_id)
                else:
                    tax_id = self.idhash['organism'][organism_id]

                tax_label = self.label_hash.get(tax_id)
                if not re.search(r'FBog', feature_id) and \
                        re.search(r'Drosophila', tax_label):
                    # make only fly things leaders
                    model.makeLeader(feature_id)

                if not self.test_mode and limit is not None and line_counter > limit:
                    pass
                else:
                    if is_gene:
                        model.addClassToGraph(feature_id, name, type_id)
                        graph.addTriple(feature_id, self.globaltt['in taxon'], tax_id)
                    else:
                        if re.search('FBa[lb]', feature_id):
                            type_id = self.globaltt['allele']
                        model.addIndividualToGraph(feature_id, name, type_id)

                    # stop adding what we do not appreciate
                    # if is_obsolete == 't':
                    #    if is_gene:
                    #        model.addDeprecatedClass(feature_id)
                    #    else:
                    #        model.addDeprecatedIndividual(feature_id)
                    #    self.deprecated_features.add(feature_key)

                    model.addClassToGraph(tax_id)
                    if tax_id != tax_internal_id:
                        model.addEquivalentClass(tax_id, tax_internal_id)

                    model.addComment(
                        feature_id,
                        self._makeInternalIdentifier('feature', feature_key))

            # TODO save checked_organisms fbid to ncbitax mapping to
            # a local file to speed up subsequent searches

        return
def _process_features ( self , limit ) : if self . test_mode : graph = self . testgraph else : graph = self . graph model = Model ( graph ) raw = '/' . join ( ( self . rawdir , 'feature' ) ) LOG . info ( "building labels for features" ) line_counter = 0 with open ( raw , 'r' ) as f : filereader = csv . reader ( f , delimiter = '\t' , quotechar = '\"' ) f . readline ( ) for line in filereader : ( feature_id , dbxref_id , organism_id , name , uniquename , residues , seqlen , md5checksum , type_id , is_analysis , timeaccessioned , timelastmodified ) = line feature_key = feature_id if re . search ( r'[\|\s\[\]\{\}\\<\>]' , uniquename ) : feature_id = self . _makeInternalIdentifier ( 'feature' , feature_key ) else : feature_id = 'FlyBase:' + uniquename self . idhash [ 'feature' ] [ feature_key ] = feature_id self . feature_types [ feature_key ] = type_id self . label_hash [ feature_id ] = name if feature_key not in self . feature_to_organism_hash : self . feature_to_organism_hash [ feature_key ] = set ( ) self . feature_to_organism_hash [ feature_key ] . add ( organism_id ) is_gene = False if re . search ( r'(FBgn|FBog)' , feature_id ) : self . idhash [ 'gene' ] [ feature_key ] = feature_id is_gene = True elif re . search ( r'FBa[lb]' , feature_id ) : self . idhash [ 'allele' ] [ feature_key ] = feature_id elif re . search ( r'FBt[ip]' , feature_id ) : self . idhash [ 'feature' ] [ feature_key ] = feature_id if self . test_mode and feature_key not in self . test_keys [ 'gene' ] and feature_key not in self . test_keys [ 'allele' ] and feature_key not in self . test_keys [ 'feature' ] : continue if name . strip ( ) == '' : name = uniquename type_key = type_id type_id = self . idhash [ 'cvterm' ] [ type_key ] types_to_skip = [ 'SO:0000316' , 'SO:0000696' , 'SO:0000358' , 'SO:0000234' , ] type_keys_to_skip = [ 596 , 57096 , 57097 , 57270 , 58210 , 59643 , 60006 , 61351 , 61467 , 257 , 286 , ] organisms_to_skip = [ 2 ] if type_id in types_to_skip or int ( type_key ) in type_keys_to_skip or int ( organism_id ) in organisms_to_skip : continue line_counter += 1 if int ( type_key ) == 604 : self . idhash [ 'reagent' ] [ feature_key ] = feature_id tax_internal_id = self . _makeInternalIdentifier ( 'organism' , organism_id ) if organism_id not in self . checked_organisms : tax_id = self . _get_organism_id ( organism_id ) self . checked_organisms . add ( organism_id ) else : tax_id = self . idhash [ 'organism' ] [ organism_id ] tax_label = self . label_hash . get ( tax_id ) if not re . search ( r'FBog' , feature_id ) and re . search ( r'Drosophila' , tax_label ) : model . makeLeader ( feature_id ) if not self . test_mode and limit is not None and line_counter > limit : pass else : if is_gene : model . addClassToGraph ( feature_id , name , type_id ) graph . addTriple ( feature_id , self . globaltt [ 'in taxon' ] , tax_id ) else : if re . search ( 'FBa[lb]' , feature_id ) : type_id = self . globaltt [ 'allele' ] model . addIndividualToGraph ( feature_id , name , type_id ) model . addClassToGraph ( tax_id ) if tax_id != tax_internal_id : model . addEquivalentClass ( tax_id , tax_internal_id ) model . addComment ( feature_id , self . _makeInternalIdentifier ( 'feature' , feature_key ) ) return
These are all of the genomic features genes, variations,
        transgenes, etc
        :param limit:
        :return: