<gh_stars>0
import gensim, nltk, logging, multiprocessing, os, json, re, logging
from gensim.models import Word2Vec
from gensim.parsing.preprocessing import stem_text, split_alphanum
from nltk.tokenize import RegexpTokenizer
from gensim.models.word2vec import LineSentence
from gensim import corpora

tokenizer = RegexpTokenizer(r'\w+')
stop_words = nltk.corpus.stopwords.words('english')


logging.basicConfig(format='%(asctime)s: %(levelname)s: %(message)s')
logging.root.setLevel(level=logging.INFO)
log = logging.getLogger("my-logger")



tokenizer = RegexpTokenizer(r'\w+')
stop_words = nltk.corpus.stopwords.words('english')

pt_stop = []
for stop_word in stop_words:
    pt_stop.append(stop_word.encode('utf-8'))

def funcao_limpa_tudo(artigo):
    tokenizer = RegexpTokenizer(r'\w+')
    lista_nova = []
    #logger.info('Setting it to 0, do not use it in your scoring function.')
    #print(docs)
    artigo = stem_text(artigo)
    artigo = split_alphanum(artigo)
    artigo = tokenizer.tokenize(artigo)

    list_artigo = list(artigo)
    try:
        for palavra in list_artigo:
            palavra = palavra.encode('utf-8')
            if re.match('^\d+$', palavra):
                #artigo.remove(palavra)
                pass
            elif palavra in pt_stop:
                #artigo.remove(palavra)
                pass

            elif len(palavra) < 3:
                #artigo.remove(palavra)
                pass
            else:
                lista_nova.append(palavra)
    except Exception as erro:
        print(erro)
        #numero_palavra = artigo.index(palavra)
        #artigo.pop(numero_palavra) #achou lixo
        pass
    del list_artigo
    del artigo
    #log.info("Lista de Tokens: %s", docs)
    if len(lista_nova) < 5:
        return None
    else:
        return lista_nova

maximo = 50
ja_passou = False

fname = 'enwiki-latest-completo.json'

class MySentences(object):


    def __init__(self, dirname, lista_final):
        self.dirname = dirname
        self.lista_final = lista_final

    def __iter__(self):
        cont = 0
        for fname in os.listdir(self.dirname):
            if cont >= maximo:
                break
            for line in open(os.path.join(self.dirname, fname)):
                line = json.loads(line)
                if cont >= maximo:
                    break
                for paragrafo_artigo in line[u'section_texts']:
                    funct = funcao_limpa_tudo(paragrafo_artigo)
                    if funct != None:
                        self.lista_final.append(funct)
                        if (cont % 1000) == 0:
                            log.info(funct)

                if len(self.lista_final) != 0:
                    dictionary = corpora.Dictionary(self.lista_final)
                    corpus = [dictionary.doc2bow(text) for text in self.lista_final]
                    cont = cont + 1
                    yield corpus, dictionary


for corpus, dictionary in MySentences('/home/marco/projetos/nlp/wikipedia/', []):
    #if ja_passou == False:
    lda = gensim.models.ldamulticore.LdaMulticore(corpus=corpus, id2word=dictionary, num_topics=5, chunksize=500, passes=50,  workers=3)
    '''
    topicos = lda.print_topics(num_topics=1, num_words=15)
    topicos = re.findall('([0-9]\.[0-9]{3})\*"([^"]*)"', topicos[0][1])
    top_topics = lda.top_topics(corpus, topn=20)
    
    3 linhas acima eu extraio os topicos usando expressao regular, pro Turing era util.
    '''
    # Average topic coherence is the sum of topic coherences of all topics, divided by the number of topics.
    avg_topic_coherence = sum([t[1] for t in top_topics]) / 5
    print('Average topic coherence: %.4f.' % avg_topic_coherence)
    from pprint import pprint
    pprint(top_topics)



import gensim, nltk, logging, multiprocessing, os, json, re, logging
from gensim.models import Word2Vec
from gensim.parsing.preprocessing import stem_text, split_alphanum
from nltk.tokenize import RegexpTokenizer
from gensim.models.word2vec import LineSentence
from gensim import corpora

tokenizer = RegexpTokenizer(r'\w+')
stop_words = nltk.corpus.stopwords.words('english')


logging.basicConfig(format='%(asctime)s: %(levelname)s: %(message)s')
logging.root.setLevel(level=logging.INFO)
log = logging.getLogger("my-logger")



tokenizer = RegexpTokenizer(r'\w+')
stop_words = nltk.corpus.stopwords.words('english')

pt_stop = []
for stop_word in stop_words:
    pt_stop.append(stop_word.encode('utf-8'))

def funcao_limpa_tudo(artigo):
    tokenizer = RegexpTokenizer(r'\w+')
    lista_nova = []
    #logger.info('Setting it to 0, do not use it in your scoring function.')
    #print(docs)
    artigo = stem_text(artigo)
    artigo = split_alphanum(artigo)
    artigo = tokenizer.tokenize(artigo)

    list_artigo = list(artigo)
    try:
        for palavra in list_artigo:
            palavra = palavra.encode('utf-8')
            if re.match('^\d+$', palavra):
                #artigo.remove(palavra)
                pass
            elif palavra in pt_stop:
                #artigo.remove(palavra)
                pass

            elif len(palavra) < 3:
                #artigo.remove(palavra)
                pass
            else:
                lista_nova.append(palavra)
    except Exception as erro:
        print(erro)
        #numero_palavra = artigo.index(palavra)
        #artigo.pop(numero_palavra) #achou lixo
        pass
    del list_artigo
    del artigo
    #log.info("Lista de Tokens: %s", docs)
    if len(lista_nova) < 5:
        return None
    else:
        return lista_nova

maximo = 50
ja_passou = False

fname = 'enwiki-latest-completo.json'

class MySentences(object):


    def __init__(self, dirname, lista_final):
        self.dirname = dirname
        self.lista_final = lista_final

    def __iter__(self):
        cont = 0
        for fname in os.listdir(self.dirname):
            if cont >= maximo:
                break
            for line in open(os.path.join(self.dirname, fname)):
                line = json.loads(line)
                if cont >= maximo:
                    break
                for paragrafo_artigo in line[u'section_texts']:
                    funct = funcao_limpa_tudo(paragrafo_artigo)
                    if funct != None:
                        self.lista_final.append(funct)
                        if (cont % 1000) == 0:
                            log.info(funct)

                if len(self.lista_final) != 0:
                    dictionary = corpora.Dictionary(self.lista_final)
                    corpus = [dictionary.doc2bow(text) for text in self.lista_final]
                    cont = cont + 1
                    yield corpus, dictionary


for corpus, dictionary in MySentences('/home/marco/projetos/nlp/wikipedia/', []):
    #if ja_passou == False:
    lda = gensim.models.ldamulticore.LdaMulticore(corpus=corpus, id2word=dictionary, num_topics=5, chunksize=500, passes=50,  workers=3)
    '''
    topicos = lda.print_topics(num_topics=1, num_words=15)
    topicos = re.findall('([0-9]\.[0-9]{3})\*"([^"]*)"', topicos[0][1])
    top_topics = lda.top_topics(corpus, topn=20)
    
    3 linhas acima eu extraio os topicos usando expressao regular, pro Turing era util.
    '''
    # Average topic coherence is the sum of topic coherences of all topics, divided by the number of topics.
    avg_topic_coherence = sum([t[1] for t in top_topics]) / 5
    print('Average topic coherence: %.4f.' % avg_topic_coherence)
    from pprint import pprint
    pprint(top_topics)
#logger.info('Setting it to 0, do not use it in your scoring function.') #print(docs) #artigo.remove(palavra) #artigo.remove(palavra) #artigo.remove(palavra) #numero_palavra = artigo.index(palavra) #artigo.pop(numero_palavra) #achou lixo #log.info("Lista de Tokens: %s", docs) #if ja_passou == False: topicos = lda.print_topics(num_topics=1, num_words=15)     topicos = re.findall('([0-9]\.[0-9]{3})\*"([^"]*)"', topicos[0][1])     top_topics = lda.top_topics(corpus, topn=20)          3 linhas acima eu extraio os topicos usando expressao regular, pro Turing era util. # Average topic coherence is the sum of topic coherences of all topics, divided by the number of topics.