Transforms a folder structure into positive and negative examples
    to feed to ``linker.model.extract``.  This transforms SortingDesk's
    foldering structure into *supervision* data for the extractor.

    This works best if folder name (``fid``) is the ``name`` of an entity
    in question or, more generally, a query that a user might have
    issued to a search engine.  In particular, this approach makes
    sense for the annotated version of this task, which is what
    SortingDesk enables.

    This returns five queries with the original_query name:

      0. if `include_original`, then original name; the model will
         eliminate if bad but it seems like it's a mistake to omit

      1. plus the most predictive keyword

      2. minus the least predictive keyword

      3. minus the most predictive keyword for the negative class

      4. plus the least predictive keyword for the negative class

    Additionally, if any of these words are the name, we skip to the
    next keyword in the list.

    Returns a three tuple of ([unicode], [unicode], bool) where the
    first list is query strings to send to a search engine, and the
    second list is feature strings to put in a StringCounter.

def extract_keyword_queries(store, label_store, folders, fid, sid, include_original=False):
    '''Transforms a folder structure into positive and negative examples
    to feed to ``linker.model.extract``.  This transforms SortingDesk's
    foldering structure into *supervision* data for the extractor.

    This works best if folder name (``fid``) is the ``name`` of an entity
    in question or, more generally, a query that a user might have
    issued to a search engine.  In particular, this approach makes
    sense for the annotated version of this task, which is what
    SortingDesk enables.

    This returns five queries with the original_query name:

      0. if `include_original`, then original name; the model will
         eliminate if bad but it seems like it's a mistake to omit

      1. plus the most predictive keyword

      2. minus the least predictive keyword

      3. minus the most predictive keyword for the negative class

      4. plus the least predictive keyword for the negative class

    Additionally, if any of these words are the name, we skip to the
    next keyword in the list.

    Returns a three tuple of ([unicode], [unicode], bool) where the
    first list is query strings to send to a search engine, and the
    second list is feature strings to put in a StringCounter.

    '''
    keyword_feature_keys = []

    query_names = fid.split('_')
    ## quotes added so that google treats the name as one token
    name1 = ' '.join(query_names)
    #keyword_feature_keys.append(name1)
    original_query = '\"' + name1  + '\"'
    logger.info('the original query was %s', original_query)

    queries = []

    ## 0. original name
    if include_original:
        logger.info('query 0: including the original: %r', original_query)
        queries.append(original_query)

    if sid:
        name2 = ' '.join(sid.split('_'))
        keyword_feature_keys.append(name2)
        queries.append( '\"' + name2 + '\"' )

    ## generate positive and negative examples by traversing folders
    try:
        ids = map(itemgetter(0), folders.items(fid, sid))
    except KeyError:
        logger.info('Folder traversal failed to find ids, so no '
                    'training data; giving up on model-based queries')
        # third return value of `False` means no observations
        return queries, keyword_feature_keys, False

    positive_fcs = map(itemgetter(1), store.get_many(ids))
    negative_ids = imap(itemgetter(0),
                        negative_subfolder_ids(label_store, folders, fid, sid))
    negative_fcs = map(itemgetter(1), store.get_many(negative_ids))

    ## These features were selected by manual inspection of current
    ## FOSS NER output.
    pos_words, neg_words = extract(positive_fcs, negative_fcs,
                                   features=['GPE', 'PERSON', 'ORGANIZATION'])

    ## 1. plus the most predictive keyword
    query_plus_pred = original_query + ' ' + \
                                name_filter(pos_words, query_names)
    logger.info('query 1: + most predictive: %r', query_plus_pred)
    queries.append(query_plus_pred)

    ## 2. minus the least predictive keyword
    query_min_least = original_query + ' -' + \
                                name_filter(reversed(pos_words), query_names)
    logger.info('query 2: - least predictive: %r', query_min_least)
    queries.append(query_min_least)

    ## 3. minus the most predictive keyword for the negative class
    query_min_most_neg = original_query + ' -' + \
                                name_filter(neg_words, query_names)
    logger.info('query 3: - most predictive for neg: %r', query_min_most_neg)
    queries.append(query_min_most_neg)

    ## 4. plus the least predictive keyword for the negative class
    query_plus_least_neg = original_query + ' ' + \
                                name_filter(reversed(neg_words), query_names)
    logger.info('query 4: + least predictive for neg: %r', query_plus_least_neg)
    queries.append(query_plus_least_neg)

    ## for debugging
    # logger.info('length %d', len(positive_fcs))

    # for fc in positive_fcs:
    #     logger.info('pos fc %r', fc['title'])


    # logger.info('pos fc %r', positive_fcs[3]['GPE'])

    # logger.info('pos fc %r', positive_fcs[3].keys())
    # logger.info('pos fc %r', positive_fcs[3]['PERSON'])

    # logger.info('positive keywords: %r', pos_words)
    # logger.info('negative keywords: %r', neg_words)

    # logger.info('most positive keyword: %r', pos_words[0])

    return queries, keyword_feature_keys, True