def rnn_train(dataset,config_params,vocab,umls_vocab):
    global params,setup_NN

    global X,U,Y,Z,Mask,i2t,t2i,w2i,i2w,splits,numTags,emb_w,umls_v
    params=config_params
    umls_v=umls_vocab
    if 'CRF_MODEL_ON' in params and params['CRF_MODEL_ON']:
        sl.info('CRF IS ON. CRF_MODELS WILL BE USED')
        if params['mode']==1:
            from bionlp.taggers.rnn_feature.networks.approx_network import setup_NN
            sl.info('MODE :Using the Approximate Message Passing framework')
        elif params['mode']==-1:
            from bionlp.taggers.rnn_feature.networks.network import setup_NN
            sl.info('MODE : Modeling only the unary potentials')
        else:
            sl.info('MODE : Modeling both unary and binary potentials')
            from bionlp.taggers.rnn_feature.networks.dual_network import setup_NN
    else:
        sl.info('CRF IS NOT ON. This tagger only supports CRF models. A default CRF_MODEL will be used.')
        params['mode']=1
        params['CRF_MODEL_ON']=True
        from bionlp.taggers.rnn_feature.networks.approx_network import setup_NN
        sl.info('MODE :Using the Approximate Message Passing framework')


    sl.info('Using the parameters:\n {0}'.format(json.dumps(params,indent=2)))

    # Preparing Dataset

    sl.info('Preparing entire dataset for Neural Net computation ...')
    (X,U,Z,Y) , numTags, emb_w , t2i,w2i =preprocess.load_data(dataset,params,entire_note=params['document'],vocab=vocab)


    X,U,Y,Z,Mask=preprocess.pad_and_mask(X,U,Y,Z,params['maxlen'])
    sl.info('Total non zero entries in the Mask Inputs are {0}. This number should be equal to total number of tokens in the entire dataset'.format(sum(sum(_) for _ in Mask)))
    if params['shuffle']==1:
        X,U,Y,Z,Mask=sk_shuffle(X,U,Y,Z,Mask,random_state=0)
    i2t = {v: k for k, v in t2i.items()}
    i2w = {v: k for k, v in w2i.items()}
    splits = data_utils.make_cross_validation_sets(len(Y),params['folds'],training_percent=params['training-percent'])
    try:
        if params['trainable'] is False:
            (o,l,p)=evaluate_run()
        elif params['deploy']==1:
            (o,l,p)=deploy_run(splits[0],params)
        elif params['cross-validation']==0:
            (o,l,p)=single_run()
        else:
            (o,l,p)=cross_validation_run()
    except IOError, e:
        if e.errno!=errno.EINTR:
            raise
        else:
            print " EINTR ERROR CAUGHT. YET AGAIN "
    sl.info('Using the parameters:\n {0}'.format(json.dumps(params,indent=2)))
    return (o,l,p) def encode_data_format(documents,raw_text,umls_params):
    logger.info('Encoding dataset into data format')
    document_dict={did_:(dtext_,metamap_) for (did_,dtext_,metamap_) in raw_text}
    documentList=[]
    for did,document in tqdm(documents):
        sentenceList=[]
        for sid,sent in enumerate(document):
            tid=0
            tokenList=[]
            for token in sent:
                newToken=Token(token[0],tid)
                newToken.attr['Annotation']=token[2]
                newToken.attr['offset']=token[1]
                newToken.attr['document']=did
                tokenList.append(newToken)
                tid+=1
            newSentence=Sentence(tokenList,sid)
            sentenceList.append(newSentence)
        newDocument=Document(sentenceList,did)
        newDocument.attr['raw_text']=document_dict[did][0]
        if umls_params!=0:
            newDocument.attr['metamap_anns']=document_dict[did][1]
        documentList.append(newDocument)
    dataset =Dataset(documentList)
    dataset.passive.append('Annotation')
    return dataset def decode_training_data(dataset):
    documentList=[]
    for document in dataset.value:
        sentenceList=[]
        for sent in document.value:
            sentenceList.append([((token.value,token),token.attr['Annotation']) for token in sent.value])
        documentList.append(sentenceList)
    logger.info('Number of Records decoded into training data format {0}'.format(documentList.__len__()))
    return documentList def file_extractor(input_list_file,umls_param):
    # Needs an input file with list of text file locations
    if umls_param!=0:
        logger.info('UMLS parameter is on. I will extract all umls annotations from *.umls.json . If I dont find any such file, I will populate empty umls features')
    raw_text=[]
    notes =[]
    list_file =[ filename for filename in open(input_list_file,'r').readlines() if filename.strip()!='' ]
    for filename in tqdm(list_file):
        metamap_anns=[]
        file_text= open(filename.strip(),'r').read()
        #Checking for existing UMLS files
        if umls_param!=0:
            if os.path.isfile('{0}.umls.json'.format(filename.strip())):
                metamap_anns=json.load(open('{0}.umls.json'.format(filename.strip()),'r'))
            else:
                logger.warning('UMLS file not found for {0}. Populating with empty umls annotations'.format(filename.strip()))
        raw_text.append((filename.strip(),file_text,metamap_anns))
        notes.append((filename.strip(),build_char_annotations([],file_text)))
    return raw_text,notes def annotated_file_extractor(input_list_file,umls_param):
    '''
    Needs an input file with list of text file locations.
    Each input text file 'filepath/filename' should have a json file at 'filepath/filename.json'.
    The json file should contain a list of annotation objects.
    Each annotation object is itself a list of the following format [start char offset, end char offset, annotated text, annotation type, annotation id]
    '''
    if umls_param!=0:
        logger.info('UMLS parameter is on. I will extract all umls annotations from *.umls.json . If I dont find any such file, I will populate empty umls features')

    raw_text=[]
    notes =[]
    list_file =[ filename for filename in open(input_list_file,'r').readlines() if filename.strip()!='' ]
    for filename in tqdm(list_file):
        metamap_anns=[]
        assert (os.path.isfile(filename.strip()) and os.path.isfile('{0}.json'.format(filename.strip()))), 'The provided filename {0} either does not exist or does not have an annotation file of format .json'.format(filename.strip())
        file_text= open(filename.strip(),'r').read()
        anns=json.load(open('{0}.json'.format(filename.strip()),'r'))
        #Checking for existing UMLS files
        if umls_param!=0:
            if os.path.isfile('{0}.umls.json'.format(filename.strip())):
                metamap_anns=json.load(open('{0}.umls.json'.format(filename.strip()),'r'))
            else:
                logger.warning('UMLS file not found for {0}. Populating with empty umls annotations'.format(filename.strip()))
        valid_anns=verify_positions(anns,file_text)
        notes.append((filename.strip(),build_char_annotations(valid_anns,file_text)))
        raw_text.append((filename.strip(),file_text,metamap_anns))
    return raw_text,notes class Token:
    def __init__(self,value,id=-1,document=-1,offset=-1):
        self.id=id
        self.attr={}
        self.attr['offset']=offset
        self.attr['length']=value.__len__()
        self.attr['document']=document
        self.value=value

    def __str__(self):
        s ="id: {0}, value: {1}, attributes: {2}".format(self.id,self.value,self.attr)
        return s class Sentence:
    def __init__(self,TokenList,id=-1):
        self.id=id
        self.attr={}
        self.attr['document']=TokenList[0].attr['document']
        self.value=TokenList

    def get_text(self):
        return ' '.join(token.value for token in self.value)

    def get_list(self):
        return [token.value for token in self.value]


    def __str__(self):
        s ="Sentence id: {0}, value: {1}, attributes: {2}".format(self.id,self.get_text(),self.attr)
        return s class Document:
    def __init__(self,SentenceList,id=-1):
        self.id=id
        self.value=SentenceList
        self.attr={}

    def __str__(self):
        s ="Document id: {0}, value: {1}, attributes: {2}".format(self.id,self.value,self.attr)
        return s class Dataset:
    def __init__(self,DocumentList,desc="None"):
        self.description=desc
        self.value=DocumentList
        self.active=[]          #set of active attributes used for Machine Learning
        self.passive=[]       #set of passive attributes not used for trainig. e.g. gold-label, char offsets.
        self.delayed=[]       #set of active attributes which require too much memory. So they are only computed in the tagger section.
        self.attr={}

    def __str__(self):
        s="Dataset description = {0}\n active Attributes {1}, passive Attributes {2}, delayed attributes {3} , Number of Documents {4}".format(self.description,self.active,self.passive,self.delayed,self.value.__len__())
        return s def add_BIO(dataset):
    exempt_list=['ADE'] # BIO tags are not applied to these labels, because they are already too few. 
    logger.info('Exempt list is {0}'.format(exempt_list))
    dataset.passive.append('BIO')
    for document in tqdm(dataset.value):
        for sent in document.value:
            sentence= deepcopy(sent.value)
            for word_id,token in enumerate(sentence):
                if (word_id==0 or sentence[word_id-1].attr['Annotation']!=sentence[word_id].attr['Annotation']) and sentence[word_id].attr['Annotation'] not in exempt_list:
                    sent.value[word_id].attr['Annotation']='B-'+token.attr['Annotation']
    return dataset def add_surface_feature_list(dataset):
    dataset.active.append('SURFACE')
    logger.info('Adding Surface features to dataset')
    for document in tqdm(dataset.value):
        for sent in document.value:
            for word_id,token in enumerate(sent.value):
                feature_list=[]
                if token.value in punct_list:
                    feature_list.append(1)
                else:
                    feature_list.append(0)
                if token.value[0].isupper():
                    feature_list.append(1)
                else:
                    feature_list.append(0)

                if all([x.isupper() for x in token.value]):
                    feature_list.append(1)
                else:
                    feature_list.append(0)

                if any([x.isupper() for x in token.value]):
                    feature_list.append(1)
                else:
                    feature_list.append(0)

                if any([x.isdigit() for x in token.value]):
                    feature_list.append(1)
                else:
                    feature_list.append(0)
                token.attr['SURFACE']=feature_list

    return dataset def add_umls_type(dataset,cache_file=None):
    dataset.active.append('UMLS_TYPE')
    logger.info('Adding UMLS Semantic Type information')
    for current_doc in tqdm(dataset.value):
        text_i=current_doc.attr['raw_text']
        tag_positions=[[] for idxs in xrange(len(text_i))]
        umls_objects=current_doc.attr['metamap_anns']
        sorted_umls_objects=sorted(umls_objects,key=itemgetter('begin'))
        for umls_obj in umls_objects:
            for idx in xrange(umls_obj['begin'],umls_obj['end']):
                tag_positions[idx].append(umls_obj['sem_type'].split('+'))

        for sent in current_doc.value:
            for word in sent.value:
                t_list=list(itertools.chain.from_iterable(tag_positions[word.attr['offset']:word.attr['offset']+len(word.value)]))
                t_list=list(itertools.chain.from_iterable(t_list))
                type_list=set(t_list)
                word.attr['umls_type']=list(type_list)

    return dataset def construct_umls_rnn_features(dataset,umls_type_vocab=None):
    # For now this modifier should always be called after add_surface_feature_list()
    assert 'SURFACE' in dataset.value[0].value[0].value[0].attr,'add_surface_feature_list() should be called before construct_umls_rnn_features()'
    if umls_type_vocab is None:
        logging.info('Vocabulary for semantic types not provided. Scanning for all UMLS Semantic Types')
        type_vocab=set()
        for document in tqdm(dataset.value):
            for sent in document.value:
                for word_id,token in enumerate(sent.value):
                    type_vocab =type_vocab.union(set(token.attr['umls_type']))
        type_vocab={umty:idxs for idxs,umty in enumerate(list(type_vocab))}
    else:
        logging.info('Using vocabulary for UMLS semantic type that was provided in dataset object')
        type_vocab=umls_type_vocab

    logger.info('Constructing multi-hot vectors for UMLS features')
    logger.info('UMLS Vocab {0}'.format(type_vocab))
    vocab_size=len(type_vocab)
    logger.info('UMLS type vocab size {0}'.format(vocab_size))
    for document in tqdm(dataset.value):
        for sent in document.value:
            for word_id,token in enumerate(sent.value):
                feature_list=[0]*vocab_size
                for umty in token.attr['umls_type']:
                    feature_list[type_vocab[umty]]=1
                token.attr['SURFACE'].extend(feature_list)
    return dataset,type_vocab def deploy_arguments():

    parser = argparse.ArgumentParser()

    parser.add_argument('-model','--model',dest='model',type= str, default='None',help ='location of pickled model file. default None')
    parser.add_argument('-i','--input',dest='input',type= str, default='None',help ='location of input file list.')
    parser.add_argument('-o','--output',dest='output',type= str, default='None',help ='location of output file. default None')
    parser.add_argument('-d','--outputdir',dest='outputdir',type= str, default='None',help ='location of output directory for annotations. default None')
    parser.add_argument('-n','--noeval',dest='noeval',type= int, default=1 ,help ='Do not run evaluation on the deployed dataset. This is useful when you do not have gold standard labels for the input text.0:Evaluation, 1:No Evaluation. Default 1')

    args = parser.parse_args()
    if args.input=='None':
        parser.error("Input is a required argument")
    elif args.model=='None':
        parser.error("Model is a required argument")
    elif args.output=='None' and  args.outputdir=='None' and  args.noeval:
        parser.error("Please provide either an output or set noeval to 0")
    deploy_params = vars(args)
    deploy_params['noeval']=bool(deploy_params['noeval'])
    return deploy_params def get_Approx_Metrics(y_true,y_pred,verbose =True,preMsg='',flat_list=False):

    if verbose:
        print('------------------------ Approx Metrics---------------------------')
    if flat_list:
        z_true=y_true
        z_pred=y_pred
    else:
        z_true=list(itertools.chain.from_iterable(y_true))
        z_pred=list(itertools.chain.from_iterable(y_pred))
    z_true=[token[2:] if token[:2]=='B-' else token for token in z_true]
    z_pred=[token[2:] if token[:2]=='B-' else token for token in z_pred]
    label_dict={x:i for i,x in enumerate(list(set(z_true) | set(z_pred)))}
    freq_dict=collections.Counter(z_true)
    z_true=[ label_dict[x] for x in z_true]
    z_pred=[ label_dict[x] for x in z_pred]
    f1s= f1_score(z_true,z_pred, average=None)
    rs= recall_score(z_true,z_pred, average=None)
    ps= precision_score(z_true,z_pred, average=None)
    results =[]
    f1_none=[]
    avg_recall=avg_precision=avg_f1=0.0
    for i in label_dict:
            if verbose:
                print("{5} The tag \'{0}\' has {1} elements and recall,precision,f1 ={3},{4}, {2}".format(i,freq_dict[i],f1s[label_dict[i]],rs[label_dict[i]],ps[label_dict[i]],preMsg))
            if i!='None' and i!='|O':
                f1_none=f1_none+[(f1s[label_dict[i]],freq_dict[i]),]
                avg_recall+=float(rs[label_dict[i]])*float(freq_dict[i])
                avg_precision+=float(ps[label_dict[i]])*float(freq_dict[i])
    intermediate_sum=sum([float(z[1]) for z in f1_none])
    if intermediate_sum ==0:
        intermediate_sum +=1
    avg_recall =float(avg_recall)/float(intermediate_sum)
    avg_precision =float(avg_precision)/float(intermediate_sum)
    if (float(avg_recall)+float(avg_precision)) !=0.0:
        avg_f1=2.0*float(avg_precision)*float(avg_recall)/(float(avg_recall)+float(avg_precision))
    else:
        avg_f1=0.0
    if verbose:
        print("All medical tags collectively have {0} elements and recall,precision,f1 ={1},{2}, {3}".format(intermediate_sum,avg_recall,avg_precision,avg_f1))
    return avg_f1 def get_Exact_Metrics(true,predicted,verbose=True):
    true,predicted=strip_BIO(true,predicted)
    if verbose:
        print('------------------------ Exact Metrics---------------------------')
        get_ConfusionMatrix(true,predicted)
    labels=get_labels(true,predicted)
    true_positive={label:0 for label in labels}
    trues={label:0 for label in labels}
    positives={label:0 for label in labels}
    for i,sent in enumerate(true):
        if sent.__len__() ==0:
            continue
        label_tags=[]
        predicted_tags=[]
        j=0
        tag='Nothing'
        pos=[]
        while j<len(sent):
            if tag!=sent[j]:
                if tag != 'Nothing':
                    label_tags.append((tag,tuple(pos)))
                pos=[]
                pos.append(j)
                tag=sent[j]
            else:
                pos.append(j)
            j+=1
        label_tags.append((tag,tuple(pos)))

        j=0
        tag='Nothing'
        pos=[]
        psent=predicted[i]
        while j<len(psent):
            if tag!=psent[j]:
                if tag != 'Nothing':
                    predicted_tags.append((tag,tuple(pos)))
                pos=[]
                pos.append(j)
                tag=psent[j]
            else:
                pos.append(j)
            j+=1
        predicted_tags.append((tag,tuple(pos)))
        for z in predicted_tags:
            positives[z[0]]+=1
        for z in label_tags:
            trues[z[0]]+=1
        for z in list(set(label_tags)&set(predicted_tags)):
            true_positive[z[0]]+=1
    avg_recall = 0.0
    avg_precision =0.0
    num_candidates=0


    #print positives,trues,true_positive
    for l in labels:
        if trues[l] ==0:
            recall =0
        else:
            recall=float(true_positive[l])/float(trues[l])
        if positives[l] ==0:
            precision =0
        else:
            precision=float(true_positive[l])/float(positives[l])
        if (recall+precision) ==0:
            f1 =0
        else:
            f1=2.0*recall*precision/(recall+precision)
        if l != IGNORE_TAG:
            avg_recall +=float(trues[l])*float(recall)
            avg_precision+=float(trues[l])*float(precision)
            num_candidates+=trues[l]
        if verbose:
            print("The tag \'{0}\' has {1} elements and recall,precision,f1 ={2},{3}, {4}".format(l,trues[l],recall,precision,f1))
    if num_candidates >0:
        avg_recall =float(avg_recall)/float(num_candidates)
        avg_precision =float(avg_precision)/float(num_candidates)
    avg_f1 =0.0
    if (avg_recall+avg_precision) >0:
        avg_f1=2.0*float(avg_precision)*float(avg_recall)/(float(avg_recall)+float(avg_precision))
    if verbose:
        print("All medical tags collectively have {0} elements and recall,precision,f1 ={1},{2}, {3}".format(num_candidates,avg_recall,avg_precision,avg_f1))
    return avg_f1 def evaluator(l,p,metric_func=get_Exact_Metrics):
    metric_func(l,p) def prepare_document_report(o,l,p,encoded_documents,output_dir):
    logger.info('Preparing the documents reports with the IGNORE_TAG = \'{0}\' ( This is case sensitive)'.format(IGNORE_TAG))
    untouched_tokens=sum([len(sent_token) for sent_token in l])
    logger.info('Total Tokens {0}'.format(untouched_tokens))
    doc_list={}
    for s_id,(sentence,sent_token) in enumerate(o):
        if len(p[s_id]) ==0:
            continue
        for t_id,token in enumerate(sent_token):
            tem=p[s_id]
            if t_id >= len(p[s_id]):
                print t_id, len(p[s_id])
            token.attr['predicted']=tem[t_id]
        produced_sentence=[(tk.value,(tk.attr['offset'],tk.attr['offset']+tk.attr['length']),tk.attr['predicted']) for tk in sent_token]
        if sent_token[0].attr['document'] in doc_list:
            doc_list[sent_token[0].attr['document']].append(label_aggregator(produced_sentence))
        else:
            doc_list[sent_token[0].attr['document']]=[label_aggregator(produced_sentence)]
    logger.info('Writing the predicted annotations to files in {0}'.format(output_dir))
    for idxs,document in enumerate(encoded_documents.value):
        doc_id=document.id
        if doc_id not in doc_list:
            logger.warning('Could not find the Document {0} in the processed output. Please verify'.format(doc_id))
            continue
        doc_text=document.attr['raw_text']
        filename='-'.join(subname for subname in str(doc_id).split('/') if subname.strip()!='')
        with open(os.path.join(output_dir,'{0}.json'.format(filename)),'w') as fout:
            ann_id=0
            json_list=[]
            for res in doc_list[doc_id]:
                for tok in res:
                    if tok[2]!=IGNORE_TAG:
                        json_list.append({'id':ann_id,'type':tok[2],'begin':tok[1][0],'end':tok[1][1],'tokenized_text':tok[0],'text':doc_text[tok[1][0]:tok[1][1]]})
                        ann_id+=1
            json_obj={}
            json_obj['file_id']=doc_id
            json_obj['predictions']=json_list
            json.dump(json_list,fout)
        with open(os.path.join(output_dir,'{0}.txt'.format(filename)),'w') as fout:
            fout.write(doc_text)
            fout.write('\n')
import sys,os,json,pickle
import logging
from bionlp.taggers.rnn_feature.tagger import rnn_train
from bionlp.preprocess.dataset_preprocess import encode_data_format,decode_training_data
from bionlp.preprocess.extract_data import file_extractor,annotated_file_extractor
from bionlp.data.token import Token as Token
from bionlp.data.sentence import Sentence as Sentence
from bionlp.data.document import Document as Document
from bionlp.data.dataset import Dataset as Dataset
from bionlp.modifiers.crf_modifiers import add_BIO
from bionlp.modifiers.rnn_modifiers import add_surface_feature_list,add_umls_type,construct_umls_rnn_features
from bionlp.utils.crf_arguments import deploy_arguments
from bionlp.evaluate.evaluation import get_Approx_Metrics,get_Exact_Metrics,evaluator
from bionlp.evaluate.postprocess import prepare_document_report
full_path = os.path.realpath(__file__)
sys.path.insert(0,full_path)

###### code above this line is used to treat bioNLP as a local python package. #####

logging.basicConfig(level=logging.INFO)
logger=logging.getLogger(__name__)



def trainer(config_params):
    logger.info('Loading new input dataset from  {0}'.format(deploy_params['input']))
    if not config_params['noeval']:
        logger.info('Evaluation on the input dataset is on ( -noeval 0). I will search for gold standard annotation in .json files ')
        raw_text,documents=annotated_file_extractor(deploy_params['input'],config_params['umls'])
    else:
        raw_text,documents=file_extractor(deploy_params['input'],config_params['umls'])
    encoded_documents=encode_data_format(documents,raw_text,config_params['umls'])

    if config_params['umls']!=0 and 'UMLS_TYPE' not in encoded_documents.active:
        logger.info('UMLS is on, but the dataset does not have umls tags. Adding and caching ..')
        encoded_documents=add_umls_type(encoded_documents)
    del raw_text
    del documents
    logger.info('Gathering stored vocabularies')
    model_file= pickle.load(open(config_params['model'],'rb'))
    w2i=model_file['w2i']
    umls_vocab_dict=model_file['umls_vocab']


    encoded_documents=add_surface_feature_list(encoded_documents)
    if config_params['umls']!=0:
        encoded_documents,umls_vocab_dict=construct_umls_rnn_features(encoded_documents,umls_vocab_dict)
    logger.info('Dataset Tags \nActive {0}\nPassive {1}\nDelayed {2}'.format(encoded_documents.active,encoded_documents.passive,encoded_documents.delayed))

    #logger.info('Sample of encoded data {0}'.format(encoded_documents.value[0].value[1].value[0]))
    if config_params['bio']==1:
        encoded_documents=add_BIO(encoded_documents)

    ##### Decoding Data into training format #####
    dataset=decode_training_data(encoded_documents)
    #logger.info('Sample of decoded data {0}'.format(dataset[0][2]))

    texts,label,pred=rnn_train(dataset,config_params,w2i,umls_vocab_dict)