def get_epi_set():
    # Split the development set in two
    dev_it, test_it = _evenly_split_it(
            _get_documents(BIONLP_2011_EPI_DEV_DIR), seed=0x22612817)
    return (_get_documents(BIONLP_2011_EPI_TRAIN_DIR), dev_it, test_it) def get_genia_set():
    # Split the development set in two
    dev_it, test_it = _evenly_split_it(
            _get_documents(BIONLP_2011_GENIA_DEV_DIR), seed=0xd81979e3)
    return (_get_documents(BIONLP_2011_GENIA_TRAIN_DIR), dev_it, test_it) def get_id_set():
    # Split the development set in two
    dev_it, test_it = _evenly_split_it(
            _get_documents(BIONLP_2011_ID_DEV_DIR), seed=0xb2f32d56)

    return (_get_documents(BIONLP_2011_ID_TRAIN_DIR), dev_it, test_it) def get_bionlp_2009_set():
    # Split the development set in two
    dev_it, test_it = _evenly_split_it(
            _get_documents(BIONLP_2009_DEV_DIR), seed=0xb2f32d56)

    return (_get_documents(BIONLP_2009_TRAIN_DIR), dev_it, test_it) def get_grec_set():
    random_seed(0x763f059c)
    paths = [d.path for d in _get_documents(GREC_DIR)]
    train_paths = set(sample(paths, len(paths) / 2))
    paths = [p for p in paths if p not in train_paths]
    dev_paths = set(sample(paths, len(paths) / 2))
    test_paths = set([p for p in paths if p not in dev_paths])
    del paths

    #print 'train', train_paths
    #print 'dev', dev_paths
    #print 'test', test_paths

    return [_filter_generic_grec_terms(it) for it in (
            (d for d in _get_documents(GREC_DIR)
                if d.path in train_paths),
            (d for d in _get_documents(GREC_DIR)
                if d.path in dev_paths),
            (d for d in _get_documents(GREC_DIR)
                if d.path in test_paths),
            )] def get_super_grec_set():
    return [_alter_terms(it, _collapse_grec_term) for it in get_grec_set()] def get_calbc_cii_set():
    random_seed('0xc78e13c3')
    #XXX: We down-sample to 250, 125, 125
    paths = sample([d.path for d in _get_documents(CALBC_CII_DIR)], 500)
    train_paths = set(sample(paths, len(paths) / 2))
    paths = [p for p in paths if p not in train_paths]
    dev_paths = set(sample(paths, len(paths) / 2))
    test_paths = set([p for p in paths if p not in dev_paths])
    del paths

    #print 'train', train_paths
    #print 'dev', dev_paths
    #print 'test', test_paths

    return (
            (d for d in _get_documents(CALBC_CII_DIR)
                if d.path in train_paths),
            (d for d in _get_documents(CALBC_CII_DIR)
                if d.path in dev_paths),
            (d for d in _get_documents(CALBC_CII_DIR)
                if d.path in test_paths),
            ) def get_nlpba_set():
    random_seed(0x25e453f9)
    paths = [d.path for d in _get_documents(NLPBA_DIR)]
    train_paths = set(sample(paths, len(paths) / 2))
    paths = [p for p in paths if p not in train_paths]
    dev_paths = set(sample(paths, len(paths) / 2))
    test_paths = set([p for p in paths if p not in dev_paths])
    del paths

    #print 'train', train_paths
    #print 'dev', dev_paths
    #print 'test', test_paths

    return (
            (d for d in _get_documents(NLPBA_DIR)
                if d.path in train_paths),
            (d for d in _get_documents(NLPBA_DIR)
                if d.path in dev_paths),
            (d for d in _get_documents(NLPBA_DIR)
                if d.path in test_paths),
            ) def get_nlpba_down_set():
    random_seed(0x8a3403ed)
    
    paths = [d.path for d in _get_documents(NLPBA_DOWN_DIR)]
    train_paths = set(sample(paths, len(paths) / 2))
    paths = [p for p in paths if p not in train_paths]
    dev_paths = set(sample(paths, len(paths) / 2))
    test_paths = set([p for p in paths if p not in dev_paths])
    del paths

    return (
            (d for d in _get_documents(NLPBA_DOWN_DIR)
                if d.path in train_paths),
            (d for d in _get_documents(NLPBA_DOWN_DIR)
                if d.path in dev_paths),
            (d for d in _get_documents(NLPBA_DOWN_DIR)
                if d.path in test_paths),
            ) class NaiveClassifier(Classifier):
    def __init__(self):
        # We will not use defaultdicts for the sake of pickling
        self.prob_by_class = {}
        self.seen_by_class = {}

    def _classify(self, vec):
        ann_text = vec[0]

        # Try to find a previous observation in the training data
        alternatives = [(_class, self.prob_by_class[_class])
                for _class, seen in self.seen_by_class.iteritems()
                if ann_text in seen]
            
        # Do we have a single clear alternative? If not, me need to rank them
        if len(alternatives) == 1:
            return alternatives[0][0]
        else:
            # Get the maximum probabilites for the alternatives or all classes
            max_prob = max((prob for _, prob
                    in (alternatives or self.prob_by_class.iteritems())))

            most_likely = [_class for _class, prob
                    in (alternatives or self.prob_by_class.iteritems())
                    if prob == max_prob]
           
            # If there were multiple we will pick one at random
            return choice(most_likely)


    def classify(self, document, sentence, annotation):
        return self._classify((sentence.annotation_text(annotation), ))
        '''
        ann_text = sentence.annotation_text(annotation)

        # Try to find a previous observation in the training data
        alternatives = [(_class, self.prob_by_class[_class])
                for _class, seen in self.seen_by_class.iteritems()
                if ann_text in seen]
            
        # Do we have a single clear alternative? If not, me need to rank them
        if len(alternatives) == 1:
            return alternatives[0][0]
        else:
            # Get the maximum probabilites for the alternatives or all classes
            max_prob = max((prob for _, prob
                    in (alternatives or self.prob_by_class.iteritems())))

            most_likely = [_class for _class, prob
                    in (alternatives or self.prob_by_class.iteritems())
                    if prob == max_prob]
           
            # If there were multiple we will pick one at random
            return choice(most_likely)
        '''

    def _gen_lbls_vecs(self, documents):
        lbls = []
        vecs = []

        # Iterate over all documents, sentences and annotations
        for document in documents:
            for sentence in document:
                for annotation in sentence:
                    lbls.append(annotation.type)
                    ann_text = sentence.annotation_text(annotation)
                    vecs.append((ann_text, ))
        return lbls, vecs

    def _train(self, lbls, vecs):
        self.prob_by_class = {}
        self.seen_by_class = {}
        
        from itertools import izip
       
        for lbl, vec in izip(lbls, vecs):
            try:
                self.seen_by_class[lbl]
            except KeyError:
                self.seen_by_class[lbl] = {}

            ann_text = vec[0]
            try:
                self.seen_by_class[lbl][ann_text] += 1
            except KeyError:
                self.seen_by_class[lbl][ann_text] = 1

        # Calculate the maximum likelihood for each class based on occurences
        total_seen = float(sum((sum(seen.itervalues())
            for seen in self.seen_by_class.itervalues())))
        for _class, seen in self.seen_by_class.iteritems():
            self.prob_by_class[_class] = sum(seen.itervalues()) / total_seen

    def train(self, documents):
        lbls, vecs = self._gen_lbls_vecs(documents)
        self._train(lbls, vecs)
        '''
        return
        # Iterate over all documents, sentences and annotations
        for document in documents:
            for sentence in document:
                for annotation in sentence:
                    try:
                        self.seen_by_class[annotation.type]
                    except KeyError:
                        self.seen_by_class[annotation.type] = {}

                    ann_text = sentence.annotation_text(annotation)
                    try:
                        self.seen_by_class[annotation.type][ann_text] += 1
                    except KeyError:
                        self.seen_by_class[annotation.type][ann_text] = 1

        # Calculate the maximum likelihood for each class based on occurences
        total_seen = float(sum((sum(seen.itervalues())
            for seen in self.seen_by_class.itervalues())))
        if self.prob_by_class is None:
            self.prob_by_class = {}
        for _class, seen in self.seen_by_class.iteritems():
            self.prob_by_class[_class] = sum(seen.itervalues()) / total_seen
        #print self.prob_by_class
        ''' class MaximumClassifier(Classifier):
    def __init__(self):
        self.count_by_class = None

    def classify(self, document, sentence, annotation):
        if self.count_by_class is None:
            raise ClassifierNotTrainedError
       
        # 
        max_count = max((count for count in self.count_by_class.itervalues()))

        most_likely = [_class for _class, count
                    in self.count_by_class.iteritems()
                    if count == max_count]
           
        # If there were multiple we will pick one at random
        return choice(most_likely)
        
    def train(self, documents):
        if self.count_by_class is None:
            self.count_by_class = {}

        # Iterate over all documents, sentences and annotations
        for document in documents:
            for sentence in document:
                for annotation in sentence:
                    try:
                        self.count_by_class[annotation.type] += 1
                    except KeyError:
                        self.count_by_class[annotation.type] = 0 def writeable_dir(string):
    #XXX: TODO: Dir!
    from os.path import exists, isdir
    from os import access, W_OK
    from argparse import ArgumentTypeError
    
    if not isdir(string):
        if not exists(string):
            raise ArgumentTypeError(
                    '{}: no such file or directory'.format(string))
        else:
            raise ArgumentTypeError('{}: is not a directory'.format(string))
    elif not access(string, W_OK):
        raise ArgumentTypeError(
                '{}: is not a writeable directory'.format(string))
    else:
        return string class SimStringEnsembleClassifier(LibLinearClassifier):
    def __init__(self):
        LibLinearClassifier.__init__(self)
        self.feature_class = SimStringEnsembleFeature class SimStringGazetterEnsembleClassifier(LibLinearClassifier):
    def __init__(self):
        LibLinearClassifier.__init__(self)
        self.feature_class = SimStringGazetterEnsembleFeature  def query_simstring_db(query, db_path, reader_arg=None):
    from simstring import reader as simstring_reader
    from simstring import cosine as simstring_cosine

    global SIMSTRING_QUERY_CACHE
    global MODIFIED_SIMSTRING_QUERY_CACHE
    if SIMSTRING_QUERY_CACHE is None:
        _load_simstring_cache()

    try:
        cache = SIMSTRING_QUERY_CACHE[db_path]
    except KeyError:
        cache = {}
        SIMSTRING_QUERY_CACHE[db_path] = cache
        MODIFIED_SIMSTRING_QUERY_CACHE = True

    try:
        return cache[query]
    except KeyError:
        MODIFIED_SIMSTRING_QUERY_CACHE = True

    # We have to query this...
    #assert False, 'NOT ALLOWED TO QUERY!'

    if reader_arg is None:
        reader = None
    try:
        if reader_arg is None:
            reader = simstring_reader(db_path)
        else:
            reader = reader_arg
        
        reader.measure = simstring_cosine
        for threshold in (v / 10.0 for v in xrange(10, QUERY_CUT_OFF - 1, -1)):
            reader.threshold = threshold

            # The reader will choke on unicode objects, so encode it
            query_utf8 = query.encode('utf-8')
            response = reader.retrieve(query_utf8)

            if not TSURUOKA_DIST:
                # Only save whether we got a response or not
                if response:
                    response = True
                else:
                    response = False
                tsuruoka_dist = None
            else:
                # Okay, now we are in a pickle, SimString has returned
                # everything sorted by length... Although it had it internally
                # by n-gram. *sigh* We need it by n-gram.

                if response:
                    # Sort the response to prepare a cut-off
                    from lib.ngram import n_gram_ref_cos_cmp, n_gram_gen
                    ref_grams = set(g for g in n_gram_gen(query, n=3,
                        guards=TSURUOKA_GUARDED))

                    # We need Unicode internally at this point
                    response = [s.decode('utf-8') for s in response]
                    response = sorted(response,
                            cmp=lambda a, b: -n_gram_ref_cos_cmp(
                                a, b, ref_grams, guards=TSURUOKA_GUARDED))
                    # Cut-off time!
                    response = response[:RESPONSE_CUT_OFF]

                    if TSURUOKA_NORMALISED:
                        tsuruoka_dist = max(bucket_norm_tsuruoka(query, resp_str)
                                for resp_str in response)
                    else:
                        tsuruoka_dist = min(bucket_tsuruoka(query, resp_str)
                                for resp_str in response)
            if response:
                cache[query] = (threshold, tsuruoka_dist)
                # We can and should bail at this point
                break
        else:
            # We found no results for any threshold
            cache[query] = (None, None)
    finally:
        # Only close if we were not passed the reader
        if reader_arg is None and reader is not None:
            reader.close()
    
    #print cache
    #print SIMSTRING_CACHE_BY_DB[db_path]
    #print SIMSTRING_CACHE_BY_DB
    #if len(cache) > 100:
    #    exit(-1)
    return cache[query] SIMSTRING_DB_PATHS = [path_join(SIMSTRING_DB_DIR, p)
        for p in listdir(SIMSTRING_DB_DIR) if p.endswith('.db')] def score_classifier(classifier, test_set):
    # (TP, FP, FN) # Leaving out TN
    results_by_class = {} #XXX: THIS HAS TO BE A CLASS!

    for document in test_set:
        for sentence in document:
            for annotation in sentence:
                # TODO: Cast annotation into span! It needs to be censored
                predicted = classifier.classify(document, sentence, annotation)
                
                try:
                    results_by_class[annotation.type]
                except KeyError:
                    results_by_class[annotation.type] = (0, 0, 0)

                try:
                    results_by_class[predicted]
                except KeyError:
                    results_by_class[predicted] = (0, 0, 0)

                a_tp, a_fp, a_fn = results_by_class[annotation.type]
                p_tp, p_fp, p_fn = results_by_class[predicted]

                if predicted == annotation.type:
                    results_by_class[annotation.type] = (a_tp + 1, a_fp, a_fn)
                if predicted != annotation.type:
                    results_by_class[annotation.type] = (a_tp, a_fp, a_fn + 1)
                    results_by_class[predicted] =  (p_tp, p_fp + 1, p_fn)

    # Extend the results to:
    # macro, micro, {RESULTS_BY_CLASS}
    tp_sum = sum([tp for tp, _, _ in results_by_class.itervalues()])
    fn_sum = sum([fn for _, _, fn in results_by_class.itervalues()])
    macro_score = tp_sum / float(tp_sum + fn_sum)
    
    micro_scores = []
    for res_tup in results_by_class.itervalues():
        m_tp, _, m_fn = res_tup
        m_tot = float(m_tp + m_fn)
        if m_tot <= 0:
            micro_scores.append(1.0)
        else:
            micro_scores.append(m_tp / float(m_tp + m_fn))
    micro_score = mean(micro_scores)

    return (macro_score, micro_score, tp_sum, fn_sum, results_by_class) def score_classifier_by_tup(classifier, test_tups):
    # (TP, FP, FN) # Leaving out TN
    results_by_class = {} #XXX: THIS HAS TO BE A CLASS!

    for test_lbl, test_vec in izip(*test_tups):
        if not isinstance(test_lbl, str):
            test_lbl_type = classifier.name_by_lbl_id[test_lbl]
        else:
            test_lbl_type = test_lbl

        # TODO: Cast annotation into span! It needs to be censored
        predicted = classifier._classify(test_vec)
        
        try:
            results_by_class[test_lbl_type]
        except KeyError:
            results_by_class[test_lbl_type] = (0, 0, 0)

        try:
            results_by_class[predicted]
        except KeyError:
            results_by_class[predicted] = (0, 0, 0)

        a_tp, a_fp, a_fn = results_by_class[test_lbl_type]
        p_tp, p_fp, p_fn = results_by_class[predicted]

        if predicted == test_lbl_type:
            results_by_class[test_lbl_type] = (a_tp + 1, a_fp, a_fn)
        if predicted != test_lbl_type:
            results_by_class[test_lbl_type] = (a_tp, a_fp, a_fn + 1)
            results_by_class[predicted] =  (p_tp, p_fp + 1, p_fn)

    # Extend the results to:
    # macro, micro, {RESULTS_BY_CLASS}
    tp_sum = sum([tp for tp, _, _ in results_by_class.itervalues()])
    fn_sum = sum([fn for _, _, fn in results_by_class.itervalues()])
    macro_score = tp_sum / float(tp_sum + fn_sum)
    
    micro_scores = []
    for res_tup in results_by_class.itervalues():
        m_tp, _, m_fn = res_tup
        m_tot = float(m_tp + m_fn)
        if m_tot <= 0:
            micro_scores.append(1.0)
        else:
            micro_scores.append(m_tp / float(m_tp + m_fn))
    micro_score = mean(micro_scores)

    return (macro_score, micro_score, tp_sum, fn_sum, results_by_class) def score_classifier_by_tup_ranked(classifier, test_tups,
        conf_threshold=0.995, unseen=False):
    results_by_class = {}
    ambd_by_class = defaultdict(list)
    not_in_range_by_class = defaultdict(int)
    for test_lbl, test_vec in izip(*test_tups):
        if not isinstance(test_lbl, str):
            test_lbl_type = classifier.name_by_lbl_id[test_lbl]
        else:
            test_lbl_type = test_lbl

        # TODO: Cast annotation into span! It needs to be censored
        # XXX: Ranked will fail for some classifiers since not all are prob.
        predicted = classifier._classify(test_vec, ranked=True)

        # Find where the correct answer was ranked
        for i, pred in enumerate((p for p, _ in predicted), start=1):
            if pred == test_lbl_type:
                rank = i
                unseen_sample = False
                break
        else:
            # If there are unseen categories our assumption can not hold
            if not unseen:
                assert False, "'{}' not in {}".format(test_lbl_type,
                        predicted) # Should not happen
            else:
                rank = i
                unseen_sample = True

        # Find at which index our confidence threshold would cut
        conf_sum = 0.0
        for i, prob in enumerate((p for _, p in predicted), start=1):
            conf_sum += prob
            if conf_sum >= conf_threshold:
                conf_threshold_cutoff = i
                break
        else:
            # The threshold contained all possible alternatives
            conf_threshold_cutoff = len(predicted)
        ambd_by_class[test_lbl_type].append(conf_threshold_cutoff)

        # Determine if the threshold cut away the correct answer, or if
        # we could not have gotten it correct since we had not seen any
        # training data for this annotation
        if unseen_sample or rank > conf_threshold_cutoff:
            not_in_range_by_class[test_lbl_type] += 1

        try:
            results_by_class[test_lbl_type].append(rank)
        except KeyError:
            results_by_class[test_lbl_type] = [rank, ]

    ## Going from class-specific (micro) to global (macro)
    # All rank entries in a single collection
    ranks = [e for e in chain(*results_by_class.itervalues())]
    # All ambiguities in a single collection
    ambiguities = [e for e in chain(*ambd_by_class.itervalues())]
    assert len(ranks) == len(ambiguities)
    assert len(ranks) == len(test_tups[0])
    assert len(ranks) == len(test_tups[1])
    # Recall over all classes
    recall = 1.0 - sum(not_in_range_by_class.itervalues()) / float(len(ranks))

    return (ranks, ambiguities, recall) SIMSTRING_LIB_PATH = join(dirname(__file__),
        'external/simstring-1.0/swig/python') class SimpleInternalEnsembleClassifier(LibLinearClassifier):
    def __init__(self):
        LibLinearClassifier.__init__(self)
        self.feature_class = SimpleInternalEnsembleFeature class CompetitiveEnsembleClassifier(LibLinearClassifier):
    def __init__(self):
        LibLinearClassifier.__init__(self)
        self.feature_class = CompetitiveEnsembleFeature class SimStringCompetitiveEnsembleClassifier(LibLinearClassifier):
    def __init__(self):
        LibLinearClassifier.__init__(self)
        self.feature_class = SimStringCompetitiveEnsembleFeature class SimStringInternalClassifier(LibLinearClassifier):
    def __init__(self):
        LibLinearClassifier.__init__(self)
        self.feature_class = SimStringInternalFeature class GazetterInternalClassifier(LibLinearClassifier):
    def __init__(self):
        LibLinearClassifier.__init__(self)
        self.feature_class = GazetterInternalFeature class InternalClassifier(LibLinearClassifier):
    def __init__(self):
        LibLinearClassifier.__init__(self)
        self.feature_class = InternalFeature class NPInternalClassifier(LibLinearClassifier):
    def __init__(self):
        LibLinearClassifier.__init__(self)
        self.feature_class = NPInternalFeature class SimStringNPInternalClassifier(LibLinearClassifier):
    def __init__(self):
        LibLinearClassifier.__init__(self)
        self.feature_class = SimStringNPInternalFeature class TsuruokaInternalClassifier(LibLinearClassifier):
    def __init__(self):
        LibLinearClassifier.__init__(self)
        self.feature_class = TsuruokaInternalFeature class TsuruokaClassifier(LibLinearClassifier):
    def __init__(self):
        LibLinearClassifier.__init__(self)
        self.feature_class = TsuruokaEnsembleFeature class SimStringTsuruokaInternalClassifier(LibLinearClassifier):
    def __init__(self):
        LibLinearClassifier.__init__(self)
        self.feature_class = SimStringTsuruokaInternalFeature class SimStringTsuruokaClassifier(LibLinearClassifier):
    def __init__(self):
        LibLinearClassifier.__init__(self)
        self.feature_class = SimStringTsuruokaFeature def _k_folds(k, coll):
    to_see = set(coll)
    coll_size = len(to_see)
    for i in xrange(k):
        fold_size = int(coll_size / k)
        if i == (k - 1) and coll_size % 2 != 0:
            fold_size -= 1
            
        fold = set(sample(to_see, fold_size))
        to_see = to_see - fold
        yield fold class hashabledict(dict):
    def __hash__(self):
        try:
            return self.cached_hash
        except AttributeError:
            self.cached_hash = hash(tuple(sorted(self.items())))
        return self.cached_hash def simstring_caching(classifiers, document_sets, verbose=False):
    if any((True for k in classifiers
        # NOTE: Keep this check up to date
        if 'SIMSTRING' in k or 'GAZETTER' in k or 'TSURUOKA' in k)):
        if verbose:
            print >> stderr, 'Caching queries for SimString:'

        cache_simstring(document_sets, verbose=verbose)
    else:
        if verbose:
            print >> stderr, 'No caching necessary for the given classifier' def learning_curve_avg(classifiers, datasets, outdir, pickle_name='learning'):

    with open(_get_learning_pickle_path(outdir, name=pickle_name), 'r') as results_file:
        results = pickle_load(results_file)

    for dataset in datasets:
        print 'Dataset:', dataset
        for classifier in classifiers:
            print 'Classifier:', classifier
            macro_avg = mean([res_tup[0] for res_tup
                in results[dataset][classifier].itervalues()]) * 100
            macro_tip = sorted((size, res_tup[0]) for size, res_tup
                    in results[dataset][classifier].iteritems())[-1][1] * 100
            amb_avg = mean([res_tup[11] for res_tup
                in results[dataset][classifier].itervalues()])
            amb_tip = sorted((size, res_tup[11]) for size, res_tup
                    in results[dataset][classifier].iteritems())[-1][1]
            rec_avg = mean([res_tup[13] for res_tup
                in results[dataset][classifier].itervalues()]) * 100
            rec_tip = sorted((size, res_tup[13]) for size, res_tup
                    in results[dataset][classifier].iteritems())[-1][1] * 100

            print ('{:.2f}/{:.2f}/{:.2f}/{:.2f}/{:.2f}/{:.2f} '
                    'MACROAVG/MACROTIP/AMBAVG/AMBTIP/RECAVG/RECTIP').format(
                    macro_avg, macro_tip, amb_avg, amb_tip, rec_avg, rec_tip) def learning_curve_test(classifiers, datasets, outdir,
        verbose=False, no_simstring_cache=False, folds=10, worker_pool=None,
        min_perc=5, max_perc=100, step_perc=5, it_factor=1,
        pickle_name='learning', use_test_set=False
        ):
    ### This part is really generic
    # TODO: We could keep old results... But dangerous, mix-up
    results_file_path = _get_learning_pickle_path(outdir, pickle_name)
    #XXX: RESUME GOES HERE!
    results_by_dataset = {}
    
    for dataset_id, dataset_getter in datasets.iteritems():
        
        if verbose:
            print >> stderr, 'Data set:', dataset_id
            
        if verbose:
            print >> stderr, 'Caching vectorised data...',

        train_set, dev_set, test_set = dataset_getter()
        if use_test_set:
            train, test = list(chain(train_set, dev_set)), list(test_set)
        else:
            train, test = list(train_set), list(dev_set)
        del train_set, dev_set, test_set

        if verbose:
            print >> stderr, 'Done!'

        results_by_dataset[dataset_id] = _learning_curve_test_data_set(
                classifiers, train, test, worker_pool,
                verbose=verbose, no_simstring_cache=no_simstring_cache,
                use_test_set=use_test_set, folds=folds, min_perc=min_perc,
                max_perc=max_perc, step_perc=step_perc, it_factor=it_factor)

        ### HACK TO GET INTERMEDIATE!
        with open(results_file_path, 'w') as results_file:
            pickle_dump(results_by_dataset, results_file)

        if verbose:
            print >> stderr, 'Results written to:', results_file_path def plot_learning_curve_results(classifiers, datasets, outdir,
        pickle_name='learning', verbose=False):
    from matplotlib import pyplot as plt

    if not classifiers and not datasets:
        print >> stderr, 'No classifiers or datasets specified, exiting'
        return

    # Load the results to be plotted
    if verbose:
        print >> stderr, 'Loading results...',
    with open(_get_learning_pickle_path(outdir, name=pickle_name), 'r'
            ) as results_file:
        results = pickle_load(results_file)
    if verbose:
        print >> stderr, 'Done!'

    image_formats = ('svg', )

    for fig_name, fig in _ambiguity_plot_gen(results, classifiers, datasets):
        for image_format in image_formats:
            plt.savefig(path_join(outdir, fig_name + '.' + image_format),
                    format=image_format, figure=fig) 

    # Over all classifiers for one dataset
    # Over all datasets for one classifier
    # Same as the above for ambiguity and recall

    #_plot_curve(outdir, results, pickle_name)
    #_plot_curve(outdir, results, pickle_name + '_ambiguity', new_metric=True)
    #_plot_curve(outdir, results, pickle_name + '_recall', new_metric=True, recall=True) def mean(l):
    return sum(l) / float(len(l)) def median(l):
    return sorted(l)[len(l)/2] def truncated_mean(l, truncation=0.05):
    offset = int(len(l) * truncation)
    return sum(sorted(l)[offset:-offset]) / float(len(l) - 2 * offset)
from argparse import ArgumentParser
from collections import defaultdict, namedtuple
from copy import deepcopy
from itertools import chain, izip, tee
from json import dumps as json_dumps
from math import sqrt
from multiprocessing import Pool
from operator import itemgetter
from os import listdir, makedirs
from os.path import abspath, basename, dirname, join as join_path
from random import sample, seed, random
from shutil import copy
from sys import path as sys_path, stderr
    from cPickle import dump as pickle_dump, load as pickle_load
    from pickle import dump as pickle_dump, load as pickle_load
    from collections import OrderedDict
    from ordereddict import OrderedDict
from reader.bionlp import (get_epi_set, get_genia_set, get_id_set,
        get_bionlp_2009_set, get_grec_set, get_super_grec_set,
        get_calbc_cii_set, get_nlpba_set, get_nlpba_down_set)
from classifier.naive import NaiveClassifier, MaximumClassifier 
from misc import writeable_dir
from classifier.simstring.classifier import (SimStringEnsembleClassifier,
        SimStringGazetterEnsembleClassifier)
from classifier.simstring.query import query_simstring_db
from classifier.simstring.config import SIMSTRING_DB_PATHS
from experiment.scoring import (score_classifier, score_classifier_by_tup,
        score_classifier_by_tup_ranked)
from toolsconf import SIMSTRING_LIB_PATH
from classifier.competitive import SimpleInternalEnsembleClassifier
from classifier.competitive import CompetitiveEnsembleClassifier
from classifier.competitive import SimStringCompetitiveEnsembleClassifier
from classifier.competitive import SimStringInternalClassifier
from classifier.competitive import GazetterInternalClassifier
from classifier.competitive import InternalClassifier
from classifier.competitive import NPInternalClassifier
from classifier.competitive import SimStringNPInternalClassifier
from classifier.competitive import TsuruokaInternalClassifier
from classifier.competitive import TsuruokaClassifier
from classifier.competitive import SimStringTsuruokaInternalClassifier
from classifier.competitive import SimStringTsuruokaClassifier
from classifier.liblinear import _k_folds, hashabledict
from experiment.common import simstring_caching
from experiment.learning import (learning_curve_avg, learning_curve_test,
        plot_learning_curve_results)
from maths import mean, median, truncated_mean
            from classifier.simstring.features import SIMSTRING_FEATURES
                    from md import dict_to_table
    from sys import argv

                test_censored_vecs = [d for d in _censor_sparse_vectors_gen(
                    test_vecs, idxs_to_censor)]
                curr_macro_score = score_classifier_by_tup(classifier,
                        (test_lbls, test_censored_vecs))[0]

                print 'Current state on test is: {}'.format(curr_macro_score)
                if last_macro_score is not None:
                    print 'Last state was: {} (diff: {})'.format(last_macro_score,
                        curr_macro_score - last_macro_score)
                last_macro_score = curr_macro_score

                # Prepare to go parallel
                f_args = ((f_id, classifier, train_data, train_folds,
                    to_censor) for f_id, to_censor
                    in vec_idxs_by_feat_id.iteritems() if f_id in to_evaluate)
                # Also cram in our non-censored one in there
                f_args = chain(((None, classifier, train_data, train_folds,
                    set()), ), f_args)

                score_by_knockout = {}
                print 'Evaluating knockouts ({} in total)'.format(
                        len(to_evaluate) + 1)
                # TODO: A bit reduntant, prettify!
                if worker_pool is not None:
                    i = 1
                    for f_id, mean in worker_pool.imap_unordered(
                            __knockout_pass, f_args):
                        score_by_knockout[f_id] = mean
                        print 'it: {} k: {} res: {} {}'.format(
                                iteration, i, f_id, mean)
                        i += 1
                else:
                    for i, args in enumerate(f_args, start=1):
                        f_id, mean = _knockout_pass(*args)
                        score_by_knockout[f_id] = mean
                        print 'it: {} k: {} res: {} {}'.format(
                                iteration, i, f_id, mean)

                # Set the result dictionary
                for f_id, mean in score_by_knockout.iteritems():
                    res_dic[str(iteration)][f_id] = mean
                # And write the results incrementally for each round
                with open(join_path(outdir, 'descent_{}_{}.md'.format(
                    classifier_name, dataset_name)), 'w') as md_file:
                    md_file.write(dict_to_table(res_dic, total=False, perc=False))
                    md_file.write('\n')
                
                # Find the best scoring one...
                scores = [(s, f_id)
                        for f_id, s in score_by_knockout.iteritems()]
                scores.sort()
                scores.reverse()

                best_score, best_f_id = scores[0]

                print 'Round winner: {} with {}'.format(best_f_id, best_score)

                if best_f_id is None:
                    # We are done, no removal gave a better score
                    break

                removed.add(best_f_id)
                to_evaluate.remove(best_f_id)
                
                iteration += 1

            if removed:
                # TODO: Could do more metrics here?

                print 'Training and evaluating a model of our previous state...',
                classifier._liblinear_train(train_lbls, train_uncensored_vecs)
                before_macro_score = score_classifier_by_tup(classifier,
                        (test_lbls, test_vecs))[0]
                print 'Done!'

                print 'Training and evaluating a model of our current state...',
                train_censored_vecs = [d for d in _censor_sparse_vectors_gen(train_vecs,
                    set(i for i in chain(*(vec_idxs_by_feat_id[f_id] for f_id in removed))))]
                classifier._liblinear_train(train_lbls, train_censored_vecs)
                print 'Done!'

                test_censored_vecs = [d for d in _censor_sparse_vectors_gen(test_vecs,
                    set(i for i in chain(*(vec_idxs_by_feat_id[f_id] for f_id in removed))))]
                after_macro_score = score_classifier_by_tup(classifier,
                        (test_lbls, test_censored_vecs))[0]

                res_str = 'Before: {} After: {}'.format(before_macro_score,
                        after_macro_score)
                print res_str
                print 'Happy?'
            else:
                res_str = 'Unable to remove any lexical resource to make improvements...'
                print res_str

            # Ugly but saves the final result safely
            with open(join_path(outdir, 'descent_{}_{}.txt'.format(
                classifier_name, dataset_name)), 'w') as res_file:
                res_file.write(res_str)

def __knockout_pass(args):
    return _knockout_pass(*args)

def _knockout_pass(f_id, classifier, train_data, folds, to_censor):
    macro_scores = []
    for fold_num, fold in enumerate(folds, start=1):
        train_set = train_data - fold
        test_set = fold

        assert len(train_set) + len(test_set) == len(train_data)

        train_vecs = [d for d in _censor_sparse_vectors_gen(
            (v for _, v in train_set), to_censor)]
        train_lbls = [l for l, _ in train_set]

        classifier._liblinear_train(train_lbls, train_vecs)

        test_vecs = [d for d in _censor_sparse_vectors_gen(
            (v for _, v in test_set), to_censor)]
        test_lbls = (l for l, _ in test_set)

        res_tup =score_classifier_by_tup(classifier, (test_lbls, test_vecs))