Given a config and a
    `wukey=cbor.dumps((folder_name,subfolder_name))`, traverse the
    folders to generate queries, issue them to Google, fetch the
    results, and ingest them.

def traverse_extract_fetch(config, wukey, stop_after_extraction=False):
    '''Given a config and a
    `wukey=cbor.dumps((folder_name,subfolder_name))`, traverse the
    folders to generate queries, issue them to Google, fetch the
    results, and ingest them.

    '''

    config.kvlclient.setup_namespace({'openquery': (str,)})
    try:
        data = list(config.kvlclient.get('openquery', (wukey,)))
        if data:
            if data[0][1]:
                logger.info('found existing query results: %r', data)
                return
            else:
                config.kvlclient.delete('openquery', (wukey,))
    except:
        logger.error('failed to get data from existing table', exc_info=True)

    fid, sid = cbor.loads(wukey)
    tfidf = config.tfidf
    folders = Folders(config.kvlclient)
    fetcher = Fetcher()

    ## To disable the keyword extractor model, you can uncomment out
    ## the next three lines (`get_subfolder_queries`) and comment out
    ## the following two lines (`extract_keyword_queries`).
    #keyword_feature_keys = []
    #queries = get_subfolder_queries(
    #    config.store, config.label_store, folders, fid, sid)

    queries, keyword_feature_keys, has_observations = extract_keyword_queries(
        config.store, config.label_store, folders, fid, sid)

    logger.info('Model found %d queries: %r', len(queries), queries)

    if stop_after_extraction:
        return

    keywords = set()
    for key in keyword_feature_keys:
        ckey = cleanse(key.decode('utf8'))
        keywords.add(ckey)
        for part in ckey.split():
            keywords.add(part)

    #link2queries = defaultdict(set)
    links = set()
    logger.info('searching google for: %r', queries)
    for q in queries:
        for result in config.google.web_search_with_paging(q, limit=10):
            links.add(result['link'])
            #map(link2queries[result['link']].add, cleanse(q.decode('utf8')).split())
            logger.info('discovered %r', result['link'])

    result = None

    #logger.info('got %d URLs from %d queries', len(link2queries), len(queries))
    logger.info('got %d URLs from %d queries', len(links), len(queries))

    # content_ids gets modified within the 'callback' closure
    content_ids = []
    #for link, queries in link2queries.items():

    def callback(si, link):
        if si is None: return
        cid_url = hashlib.md5(str(link)).hexdigest()
        cid = etl.interface.mk_content_id(cid_url)
        content_ids.append(cid)

        # hack alert!
        # We currently use FCs to store subtopic text data, which
        # means we cannot overwrite existing FCs with reckless
        # abandon. So we adopt a heuristic: check if an FC already
        # exists, and if it does, check if it is being used to store
        # user data. If so, don't overwrite it and move on.
        fc = config.store.get(cid)
        if fc is not None and any(k.startswith('subtopic|')
                                  for k in fc.iterkeys()):
            logger.info('skipping ingest for %r (abs url: %r) because '
                        'an FC with user data already exists.',
                        cid, link)
            return

        other_features = {
            u'keywords': StringCounter(keywords), #list(queries)),
        }

        try:
            fc = etl.create_fc_from_html(
                link, si.body.raw,
                encoding=si.body.encoding or 'utf-8', tfidf=tfidf,
                other_features=other_features,
            )
            if not fc:
                logger.info('failed to get an FC, moving on')
                return
            logger.info('created FC for %r (abs url: %r)',
                        cid, link)
            config.store.put([(cid, fc)])
        except Exception:
            logger.info('trapped ingest failure on %r (abs url: %r)',
                        cid, link, exc_info=True)

    logger.info('FETCHING using ASYNC')
    fetcher.get_async(islice(links, None), callback)

    data = json.dumps({'content_ids': content_ids})
    logger.info('saving %d content_ids in %d bytes on wukey %r',
                len(content_ids), len(data), wukey)
    config.kvlclient.put('openquery', ((wukey,), data))
    logger.info('done saving for %r', wukey)