# -*- encoding: utf-8 -*-
from __future__ import unicode_literals

import cPickle
import codecs
import re
from gensim import corpora
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords

multilingual_stop_words = set(stopwords.words('english')) | set(stopwords.words('french')) | set(['http'])

REBUILD_PROAIXY_DICT = False

rootclass_re = re.compile(r'^((SHS\.|)[A-Z\-]*)')
def get_root_class(label):
    m = rootclass_re.match(label)
    if m:
        return m.group(0)

class HALCorpus(corpora.TextCorpus):
    def __init__(self, input=None, dct=None):
        super(HALCorpus, self).__init__(input)
        self.docId = []
        self.classes = []
        self.all_labels = set()

        self.char_re = re.compile('.*[a-zA-Z]')

        if dct is None:
            self.dictionary = corpora.Dictionary(self.read())
        else:
            self.dictionary = dct
        # self.dictionary.filter_extremes()  # remove stopwords etc

    def __iter__(self):
        for tokens in self.read():
            yield self.dictionary.doc2bow(tokens)

    def iter_labelled(self):
        for i, tokens in enumerate(self.read()):
            if len(self.classes[i]):
                label = str(self.classes[i][0])
                # TODO more accurate label ?
                doc = self.dictionary.doc2bow(tokens)
                yield (doc, label)


    def not_a_stopword(self, w):
        return (self.char_re.match(w) is not None) and not w.startswith('//') and w not in multilingual_stop_words

    def setDocId(self, lineno, docId):
        if lineno >= len(self.docId):
            self.docId.append(docId)
        else:
            self.docId[lineno] = docId

    def setClasses(self, lineno, classes):
        if lineno >= len(self.classes):
            self.classes.append(classes)
            self.all_labels |= set(classes[:1])
        else:
            self.classes[lineno] = classes

    def read(self):
        with self.getstream() as f:
            buffer = ''
            for lineno, line in enumerate(f):
                buffer += line.decode('utf-8')[0:-1]
                if line[-1] != '\n':
                    continue
                fields = buffer.strip().split('\t')
                buffer = ""
                if len(fields) != 3:
                    print "Skipping line "+str(lineno)+" with "+str(len(fields))+" fields"
                    print "Fields:"
                    print fields
                    print "Line:"
                    print line
                    print "################"
                    continue
                topics = fields[1].split(',')
                topics = filter(lambda x: len(x) > 1, topics)
                topics = map(get_root_class, topics)
                abstract = self.tokenize_abstract(fields[2])
                self.setDocId(lineno, fields[0])
                self.setClasses(lineno, topics)
                yield abstract

    def tokenize_abstract(self, string):
        words = word_tokenize(string)
        words = map(unicode.lower, words)
        words = filter(self.not_a_stopword, words)
        return words



# -*- encoding: utf-8 -*-
from __future__ import unicode_literals

import cPickle
import codecs
import re
from gensim import corpora
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords

multilingual_stop_words = set(stopwords.words('english')) | set(stopwords.words('french')) | set(['http'])

REBUILD_PROAIXY_DICT = False

rootclass_re = re.compile(r'^((SHS\.|)[A-Z\-]*)')
def get_root_class(label):
    m = rootclass_re.match(label)
    if m:
        return m.group(0)

class HALCorpus(corpora.TextCorpus):
    def __init__(self, input=None, dct=None):
        super(HALCorpus, self).__init__(input)
        self.docId = []
        self.classes = []
        self.all_labels = set()

        self.char_re = re.compile('.*[a-zA-Z]')

        if dct is None:
            self.dictionary = corpora.Dictionary(self.read())
        else:
            self.dictionary = dct
        # self.dictionary.filter_extremes()  # remove stopwords etc

    def __iter__(self):
        for tokens in self.read():
            yield self.dictionary.doc2bow(tokens)

    def iter_labelled(self):
        for i, tokens in enumerate(self.read()):
            if len(self.classes[i]):
                label = str(self.classes[i][0])
                # TODO more accurate label ?
                doc = self.dictionary.doc2bow(tokens)
                yield (doc, label)


    def not_a_stopword(self, w):
        return (self.char_re.match(w) is not None) and not w.startswith('//') and w not in multilingual_stop_words

    def setDocId(self, lineno, docId):
        if lineno >= len(self.docId):
            self.docId.append(docId)
        else:
            self.docId[lineno] = docId

    def setClasses(self, lineno, classes):
        if lineno >= len(self.classes):
            self.classes.append(classes)
            self.all_labels |= set(classes[:1])
        else:
            self.classes[lineno] = classes

    def read(self):
        with self.getstream() as f:
            buffer = ''
            for lineno, line in enumerate(f):
                buffer += line.decode('utf-8')[0:-1]
                if line[-1] != '\n':
                    continue
                fields = buffer.strip().split('\t')
                buffer = ""
                if len(fields) != 3:
                    print "Skipping line "+str(lineno)+" with "+str(len(fields))+" fields"
                    print "Fields:"
                    print fields
                    print "Line:"
                    print line
                    print "################"
                    continue
                topics = fields[1].split(',')
                topics = filter(lambda x: len(x) > 1, topics)
                topics = map(get_root_class, topics)
                abstract = self.tokenize_abstract(fields[2])
                self.setDocId(lineno, fields[0])
                self.setClasses(lineno, topics)
                yield abstract

    def tokenize_abstract(self, string):
        words = word_tokenize(string)
        words = map(unicode.lower, words)
        words = filter(self.not_a_stopword, words)
        return words



# -*- encoding: utf-8 -*- # self.dictionary.filter_extremes()  # remove stopwords etc # TODO more accurate label ? ###############"