__author__ = 'juliewe'

import re
import numpy
import scipy
import scipy.sparse as sparse
from scipy.stats import binom
import math
from scipy.stats import norm as normal
from db import blessDB, untag


def update_params(dim,a,b):
    WordVector.dim=dim
    WordVector.adj_constant=dim
    WordVector.adja=a
    WordVector.adjb=b
#    WordVector.adj_constant=math.log(dim/2.0,2)

def widthfunction(w1,w2):

    w1=math.log(float(w1))
    w2=math.log(float(w2))
    #return (w1+w2)/2
    return w1

class WordVector:

    windows=False
    windowPATT=re.compile('T:.*')
    filteredPATT=re.compile('___FILTERED___')
    posPATT=re.compile('(.*)[_|/].*')
    beta=0
    gamma=0
    dim = -1 #dimensionality will be set once all vectors read in
    adj_constant = -1
    adja=1 #linadj parameter a
    adjb=1 # linadj parameter b

    def __init__(self, wordpos):

        self.word=wordpos[0]
        self.pos=wordpos[1]
        self.vector={}
        self.length2=0
        self.width=0
        self.length=0
        self.array=""
        self.allsims={}
        self.nearestneighbour=''
        self.topsim=-1.0
        self.avgsim=-1.0
        self.sd=-1.0
        self.nosims=0
        self.totalsim=0
        self.square=0
        self.debug=False
        self.tuplelist=[] #sorted (descending sim) list of words and sim scores, generated by topk
        self.analysed=False


    def addfeature(self,feature,score):
        if float(score) <=0:
            #print "Rejecting negative PMI score "+self.word+"/"+self.pos +" : "+feature+" : "+score
            return False
        else:
            if feature in self.vector:
                print "Error "+self.word+" already has feature "+feature
                exit(1)
            else:
                matchobj=WordVector.windowPATT.match(feature)
                if matchobj:
                    if WordVector.windows:
                        self.vector[feature]=float(score)
                        self.width+=1
                        self.length2+=float(score)*float(score)
                        return True
                    else:
                        return False
                else:
                    matchobj=WordVector.filteredPATT.match(feature)
                    if matchobj:
                        #ignore
                        return False
                    else:
                        if WordVector.windows==False:
                            self.vector[feature]=float(score)
                            self.width+=1
                            self.length2+=float(score)*float(score)
                            return True
                        else:
                            return False


#    def update(self,featurelist):
#        #print "Updating "+self.word+"/"+self.pos
#        while(len(featurelist)>0):
#
#            self.addfeature(featurelist.pop(),featurelist.pop())
#        self.length=pow(self.length2,0.5)
#        #self.display()

    def add(self,avector):
        #replaced by add_array
        print "Warning - using deprecated method WordVector.add()"
        if len(avector.vector)>0:
            #print"adding features"
            #avector.display()
            for feature in avector.vector:
                score=avector.vector[feature]
                if feature in self.vector:
                    oldscore=self.vector[feature]
                    self.length2-=oldscore
                    newscore=oldscore+score
                    self.vector[feature]=newscore
                    self.length2+=newscore*newscore
                else:
                    self.vector[feature]=score
                    self.width+=1
                    self.length2+=score*score
            self.length=pow(self.length2,0.5)

    def add_array(self,avector): #unnecessary wrapper method for add if initialised properly
        #replacement for add working with sparse arrays rather than dictionaries
        if self.array=="":
            self.array=avector.array
        else:
            self.array = self.array + avector.array

    def mult_array(self,avector):  #unnecessary wrapper method for multiply if initialised properly
        if self.array=="":
            self.array=avector.array
        else:
            self.array = self.array.multiply(avector.array)


    def getfeature(self,feature):
        if feature in self.vector.keys():
            return self.vector[feature]
        else:
            return 0

    def display(self):
        print self.word+"/"+self.pos
        if self.pos =="S":
            print self.array
            print self.length
        else:
            print self.vector
            print self.width,self.length2,self.length

    def findsim(self,avector,metric):
        if metric =="cosine":
            return self.cossim_array(avector)
        elif metric =="lin":
            return self.linsim(avector)
        elif metric =="cr":
            return self.cr(avector,WordVector.beta,WordVector.gamma)
        elif metric =="binprob":
            return self.simprob(avector)
        elif metric =="linadj":
            return self.linadj(avector)
        else:
            print "Unknown similarity metric "+metric

    def cossim(self,avector):
        #replaced by cossim
        print "Warning - using deprecated method WordVector.cossim"
        if self.length*avector.length == 0:
            sim = 0
            print"Warning: 0 length vectors"
        else:
            dotprod=0
            print self.width,avector.width
            if self.width<avector.width:
                for feature in self.vector.keys():
                    dotprod+=avector.getfeature(feature)*self.getfeature(feature)
            else:
                for feature in avector.vector.keys():
                    dotprod+=self.getfeature(feature)*avector.getfeature(feature)

            sim = dotprod/(self.length*avector.length)
#        print "cossim: "+str(sim)
        return sim

    def cossim_array(self,avector):
        #replaces cossim using the arrays rather than dictionaries
        if self.length==0:
            self.length = pow(self.array.multiply(self.array).sum(),0.5)
        if avector.length==0:
            avector.length = pow(avector.array.multiply(avector.array).sum(),0.5)

        if self.length*avector.length == 0:
            sim =0
            print "Warning: 0 length vectors"
        else:
            dotprod = self.array.multiply(avector.array).sum()
            sim = dotprod/(self.length*avector.length)
        #print sim
        return sim

    def linsim(self,avector):

        intersect = set(self.vector.keys()).intersection(set(avector.vector.keys()))

        den=0
        num=0
        for feature in self.vector.keys():
            den = den+self.vector[feature]
        for feature in avector.vector.keys():
            den = den+avector.vector[feature]
        #print "Intersect is ...."
        for feature in intersect:
            num = num + self.vector[feature]+avector.vector[feature]
        #    print feature, self.vector[feature],avector.vector[feature]
        sim = (num*1.0)/(den*1.0)
        return sim

    def precision(self,avector):

        intersect = set(self.vector.keys()).intersection(set(avector.vector.keys()))

        den = 0
        num =0
        for feature in self.vector.keys():
            den = den +self.vector[feature]
            #den = den + 1
        for feature in intersect:
            num = num + min(self.vector[feature],avector.vector[feature])
            #num = num + 1
        sim = (num*1.0)/(den*1.0)
        return sim

    def cr(self,avector,beta,gamma):

        pre = float(self.precision(avector))
        rec = float(avector.precision(self))
        if pre*rec == 0:
            hm =0
        else:
            hm = pre*rec/(pre+rec)
        am = beta *pre + (1-beta)*rec
        sim = gamma *hm + (1-gamma)*am
        return sim


    def linsim_array(self,avector):
        #computes lin similarities over arrays
        #doesn't work because can't find ceil() function?
        den = self.array + avector.array
        denominator = den.sum()
        a = self.array.copy()
        b = avector.array.copy()
        a = a/1000
        a.ceil()
        b=b/1000
        b.ceil()
        intersection = a * b
        num = intersection * den
        numerator = num.sum()
        sim = numerator/denominator
        return sim

    def simprob(self,avector):
        #function to calculate binomial probability of getting sim or less by chance given widths of vectors

        mywidth=self.width
        awidth=avector.width
        #width = (mywidth+awidth)*0.5
        n=awidth
        p=mywidth/WordVector.dim
        sim = self.linsim(avector)
        r= math.floor(sim*awidth)
        prob = binom.cdf(r,n,p)
        return prob

    def linadj(self,avector):
        # function to adjust lin similarity given widths of distributions

        mywidth = self.width
        awidth=avector.width

        adj1=mywidth*awidth*1.0/(mywidth+awidth)
#        adj=math.log(adj1,2)
        adj= math.pow(WordVector.adj_constant/adj1,(1.0/WordVector.adja))
#        adj = adj1
        sim = self.linsim(avector)
        if self.debug:
            print mywidth,awidth, adj1, sim, WordVector.adj_constant, WordVector.adja, WordVector.adjb
        sim = 1-1/(math.pow(sim,WordVector.adjb) * adj+1)
        if self.debug:
            print sim
        self.debug=False
        return sim

    def makecache(self,outstream):
        outstream.write(self.word+"/"+self.pos)
        for feature in self.vector.keys():
            outstream.write("\t"+feature+"\t"+str(self.vector[feature]))
        outstream.write("\n")

    def equals(self,avector):
        #check if it is the same word/pos
        if self.word == avector.word:
            if self.pos == avector.pos:
                return True
        return False

    def analyse(self):
        if not self.analysed:
            total =0.0
            count=0
            max=0.0
            squares=0.0
            for neigh in self.allsims.keys():
                sim =self.allsims[neigh]
                if sim > max:
                    max = sim
                    nn=neigh

                total+=float(sim)
                count+=1
                squares+=sim*sim

            self.topsim=max
            self.nearestneighbour=nn
            self.avgsim=total/count
            self.totalsim=total
            self.squaretotal=squares
            self.nosims=count
            self.sd = pow(squares*1.0/count - self.avgsim*self.avgsim,0.5)
            self.analysed=True
    def outputsims(self,outstream):
        outstream.write(self.word+"/"+self.pos+"\t"+str(self.width)+"\t"+str(self.length))
        #for word in self.allsims.keys():
        #    outstream.write("\t"+word+"\t"+str(self.allsims[word]))
        for(sim,word) in self.tuplelist:
            outstream.write("\t"+word+"\t"+str(sim))
        outstream.write("\n")

    def displaysims(self):
        res=self.word+"/"+self.pos
        #for word in self.allsims.keys():
        #    print("\t"+word+"\t"+str(self.allsims[word]))
        done=0
        for (sim,word) in self.tuplelist:
            res+="\t"+word+"\t"+str(sim)
            done+=1
           # if done%7==0:
            #    res+="\n\t"
        res+="\n"
        print res

    def topk(self,k):
        #only retain top k neighbours
        if len(self.tuplelist)==0:
            self.tuplelist=[]
            for item in self.allsims.keys():
                self.tuplelist.append((float(self.allsims[item]),item))
            self.tuplelist.sort()
        else:
            self.tuplelist.reverse()
        self.allsims={}
        tuplelist=self.tuplelist
        self.tuplelist=[]
        done=0
        while done < k:
            if len(tuplelist)>0:
                (sim,word)=tuplelist.pop()
                self.allsims[word]=float(sim)
                self.tuplelist.append((sim,word))
            else:
                done = k
            done+=1

    def keeptopsim(self,sim):
        #only retain neighbours above sim threshold
        tuplelist=[]
        for item in self.allsims.keys():
            tuplelist.append((float(self.allsims[item]),item))
        tuplelist.sort()
        self.allsims={}
        if len(tuplelist)>0:
          (thissim,word)=tuplelist.pop()
        else:
            thissim=-1
        while thissim > sim:

            self.allsims[word]=float(thissim)
            if len(tuplelist)==0:
                thissim=-1
            else:
                (thissim,word)=tuplelist.pop()


    def outputtopk(self,outstream,k):

        self.topk(k)
        self.outputsims(outstream)

#        outstream.write(self.word+"/"+self.pos+"\t"+str(self.width)+"\t"+str(self.length))
#        tuplelist=[]
#        for item in self.allsims.keys():
#            tuplelist.append((float(self.allsims[item]),item))
#        tuplelist.sort()
#        self.allsims={}
#        done=0
#        while done < k:
#            (sim,word)=tuplelist.pop()
#            self.allsims[word]=float(sim)
#            outstream.write("\t"+word+"\t"+str(sim))
#            done+=1
#        outstream.write("\n")

    def evaluaterecall(self,gs):
        num=0
        den=0
        for word in gs:
            den+=1
        #    print word, self.allsims.keys()
            for key in self.allsims.keys():
                if self.match(word,key):
                    num+=1
        if den>0:
            return num*1.0/den
        else:
            return 1.0

    def evaluateprecision(self,gs):
        num=0
        den=0
        for key in self.allsims.keys():
            den+=1
            for word in gs:
                if self.match(word,key):
                    num+=1
        if den>0:
            return num*1.0/den
        else:
            return 1.0

    def evaluaterecall2(self,gs):
        num=0
        den=0
        for word in gs:
            den+=1
            #    print word, self.allsims.keys()
            for key in self.allsims.keys():
                if self.match(word,key):
                    num+=1
        return (num,den)

    def evaluateprecision2(self,gs):
        num=0
        den=0
        for key in self.allsims.keys():
            den+=1
            for word in gs:
                if self.match(word,key):
                    num+=1
        return (num,den)

    def getk(self):
        return len(self.allsims.keys())

    def match(self,word,key):

        thisword=self.extractword(word)
        thiskey=self.extractword(key)
        return thisword==thiskey

    def extractword(self,word):

        #print word
        matchobj = WordVector.posPATT.match(word)
        if matchobj:
            res = matchobj.group(1)
        else:
            print "Argh"
        #print res
        return res

    def znorm(self):
        #estimate normal dist params and transform into normal probs
        self.analyse()
        #print self.word, self.avgsim,self.sd

        for (sim,neigh) in self.tuplelist:
            p = normal(self.avgsim,self.sd).cdf(sim)
            self.allsims[neigh]=p

        self.analyse()
        #print self.word,self.avgsim,self.sd

        self.tuplelist=[]
        self.topk(self.getk())

    def znorm_fixed(self,mean,sd):
        self.analyse()
        #print self.word,self.avgsim,self.sd
        for(sim,neigh) in self.tuplelist:
            p=normal(mean,sd).cdf(sim)
            self.allsims[neigh]=p
        self.analyse()
        #print self.word,self.avgsim,self.sd
        self.tuplelist=[]
        self.topk(self.getk())

    def adjustsims(self,myBless,meanpoly,sdpoly):

        (_,w1)=myBless.countdict.get(self.word,(0,0))
  #      print self.word,w1
        for(sim,neigh) in self.tuplelist:
            (_,w2)=myBless.countdict.get(untag(neigh,'/'),(0,0))
            #print self.word,w1,neigh,w2
            jointwidth=widthfunction(w1,w2)
            mean=meanpoly(jointwidth)
            sd=sdpoly(jointwidth)
            p=normal(mean,sd).cdf(sim)
            self.allsims[neigh]=p
        self.tuplelist=[]
        self.topk(self.getk())