Train an LSI model on a "corpus" of documnets, like tweets

    Arguments:
      tokenize (callable): Function that returns a sequence of tokens (str) when passed a document (str)
      ram (float):         Portion of RAM to limit the document generator from consuming before StopIteration
      num_docs (int):      Number of documents to limit the document generator to before StopIteration
      num_topics (int):    See gensim.models.LsiModel, default: 200
      chunksize (int):     See gensim.models.LsiModel, default: 10000
      power_iters (int):   See gensim.models.LsiModel, default: None
      extra_samples (int): See gensim.models.LsiModel, default: None

    My process the document generator 3 times (internal tees maintained to "reset" the generator):

    1. "train" Dictionary (Vocab): find set of unique tokens and assigned int IDs)
    2. "train" TfidfModel transform: need global frequencies (by tokenizing docs from document generator) for DF (document freuency) part
    3. "train" LsiModel transform: need to iterate through the word vectors which are generated afresh by retokenizing documents

    TODO: GET RID OF UNUSED ARGS!
          Does the order of documents affect the resulting PCA topics?
          Are topics limited to 10 tokens in gensim?
          Should matches/pairs follow one another?
          Should nonmatching vocab be used at all?

    >>> lsi(ram=0.1, num_docs=2, verbose=0)
    <gensim.models.lsimodel.LsiModel object at ...>

def lsi(tokenize=None, iter_dics=None, ram=0.4, num_docs=5000, num_topics=200, num_tokens=1000000,
        num_plots=4, local_path=DATA_PATH, suffix='', ngrams=2,
        verbose=1, show=True, save=True, **kwargs):
    """Train an LSI model on a "corpus" of documnets, like tweets

    Arguments:
      tokenize (callable): Function that returns a sequence of tokens (str) when passed a document (str)
      ram (float):         Portion of RAM to limit the document generator from consuming before StopIteration
      num_docs (int):      Number of documents to limit the document generator to before StopIteration
      num_topics (int):    See gensim.models.LsiModel, default: 200
      chunksize (int):     See gensim.models.LsiModel, default: 10000
      power_iters (int):   See gensim.models.LsiModel, default: None
      extra_samples (int): See gensim.models.LsiModel, default: None

    My process the document generator 3 times (internal tees maintained to "reset" the generator):

    1. "train" Dictionary (Vocab): find set of unique tokens and assigned int IDs)
    2. "train" TfidfModel transform: need global frequencies (by tokenizing docs from document generator) for DF (document freuency) part
    3. "train" LsiModel transform: need to iterate through the word vectors which are generated afresh by retokenizing documents

    TODO: GET RID OF UNUSED ARGS!
          Does the order of documents affect the resulting PCA topics?
          Are topics limited to 10 tokens in gensim?
          Should matches/pairs follow one another?
          Should nonmatching vocab be used at all?

    >>> lsi(ram=0.1, num_docs=2, verbose=0)
    <gensim.models.lsimodel.LsiModel object at ...>
    """
    lsi_kwargs = {'num_topics': num_topics, 'onepass': True}
    lsi_kwargs.update(kwargs)
    if lsi_kwargs.get('onepass', True) and (lsi_kwargs.get('power_iters', None) or lsi_kwargs.get('extra_samples', None)):
        lsi_kwargs['onepass'] = False
    mem0 = psutil.virtual_memory()
    tokenize = tokenize or Tokenizer(ngrams=ngrams)  # , stem='WordNet', lower=True)
    tokenize('Dummy text to get tokenizer to load any data it needs.')
    mem1_tokenizer = psutil.virtual_memory()
    if verbose:
        print('{} tokenizer used {}B'.format(tokenize, friendly(mem1_tokenizer.used - mem0.used)))
    suffix = suffix or 'auto'
    suffix = ('_' * (bool(suffix) and not suffix.startswith('_'))) + suffix
    docgen_orig = (stringify(d) for d in iter_docs)
    docgen_orig, docgen = tee_gen_limit(docgen_orig, limit=num_docs, ram=ram, interval=1000, verbose=verbose)  # 1st generator copy
    mem2_gen = psutil.virtual_memory()
    vocab = Vocab((tokenize(s) for s in docgen), prune_at=num_tokens)
    mem3_vocab = psutil.virtual_memory()
    if verbose:
        print('Built vocabulary of {} unique words and used {}B of RAM. Total RAM used = {}B or {:.3%}'.format(
              friendly(len(vocab)), friendly(mem3_vocab.used - mem2_gen.used), friendly(mem3_vocab.used - mem0.used),
              1. * (mem3_vocab.total - mem3_vocab.available) / mem3_vocab.available))
    if suffix == '_auto':
        suffix = '_' + str(len(vocab)) + '_{}grams'.format(ngrams)
    full_name = 'lsi' + suffix
    if verbose and save:
        print('Will save LsiModel under the name {}'.format(full_name))
    tfidf = TfidfModel(dictionary=vocab)
    mem4_tfidf = psutil.virtual_memory()
    if verbose:
        print('Built tfidf of {} docs and used {}B of RAM. Total RAM used = {}B or {:.4%}'.format(
              friendly(tfidf.num_docs), friendly(mem4_tfidf.used - mem3_vocab.used), friendly(mem4_tfidf.used - mem0.used),
              1. * (mem4_tfidf.total - mem4_tfidf.available) / mem4_tfidf.available))
    docgen_orig, docgen = tee_gen_limit(docgen_orig, limit=num_docs, ram=ram, interval=1000, verbose=verbose)  # 2nd generator copy
    lsi = LsiModel(tfidf[(vocab.doc2bow(tokenize(s)) for s in docgen_orig)], id2word=vocab, **lsi_kwargs)
    mem5_lsi = psutil.virtual_memory()
    if verbose:
        print('Built LSI model: {} docs, {} terms, {} topics. '.format(
            friendly(lsi.docs_processed), friendly(lsi.num_terms), friendly(lsi.num_topics)))
        print('Used {}B of RAM. Total RAM: {}B or {:.4%}'.format(
            friendly(mem5_lsi.used - mem4_tfidf.used), friendly(mem5_lsi.used - mem0.used),
            1. * (mem5_lsi.total - mem5_lsi.available) / mem5_lsi.available))
    # TODO: Consider marshaling the tokenizer and any other user-specified functions
    #   Marshaling stores the bytecode for the tokenizer rather than just its fully-qualified name
    #   This would prevent changes to the tokenizer code from causing previously pickled objects
    #   to behave differently than when they were pickled.
    if save:
        try_preserve(lsi, name=full_name, local_path=local_path, ext='', verbose=verbose)
        # TODO: set lsi.tfidf.vocab=None # already saved in lsi.id2word
        try_preserve(tokenize, name=full_name + '_tokenizer', local_path=local_path, ext='.pickle', verbose=verbose)
    return lsi