import numpy as np
import random

from q1_softmax import softmax
from q2_gradcheck import gradcheck_naive
from q2_sigmoid import sigmoid, sigmoid_grad

def normalizeRows(x):
    """ Row normalization function """
    l2norm = np.sqrt((x**2).sum(axis=1, keepdims=True))
    x /= l2norm
    return x

def test_normalize_rows():
    print "Testing normalizeRows..."
    x = normalizeRows(np.array([[3.0,4.0],[1, 2]]))
    # the result should be [[0.6, 0.8], [0.4472, 0.8944]]
    print x
    assert (x.all() == np.array([[0.6, 0.8], [0.4472, 0.8944]]).all())
    print ""

def softmaxCost(uo, vc, outputVectors):
    """ This is softmax(o,c) or y^o or p(o|c) """

    return np.exp(uo*vc) / np.sum(np.exp(outputVectors * vc), \
                                    axis=len(vc.shape)-1, keepdims=True)

# vc, u0, uws
def softmaxCostAndGradient(predicted, target, outputVectors, dataset):
    """ Softmax cost function for word2vec models """

    # Implement the cost and gradients for one predicted word vector
    # and one target word vector as a building block for word2vec
    # models, assuming the softmax prediction function and cross
    # entropy loss.

    # Inputs:
    # - predicted: numpy ndarray, predicted word vector (\hat{v} in
    #   the written component or \hat{r} in an earlier version)
    # - target: integer, the index of the target word
    # - outputVectors: "output" vectors (as rows) for all tokens
    # - dataset: needed for negative sampling, unused here.

    # Outputs:
    # - cost: cross entropy cost for the softmax word prediction
    # - gradPred: the gradient with respect to the predicted word
    #        vector
    # - grad: the gradient with respect to all the other word
    #        vectors
    v_c = predicted
    uws = outputVectors
    u_0 = target
    # this is from https://github.com/Khabermas/NLP_ps1
    probabilities = softmax(predicted.dot(outputVectors.T))
    cost = -np.log(probabilities[target])
    delta = probabilities
    delta[target] -= 1
    N = delta.shape[0]
    D = predicted.shape[0]
    grad = delta.reshape((N,1)) * predicted.reshape((1,D))
    gradPred = (delta.reshape((1,N)).dot(outputVectors)).flatten()

    #y_0 = softmaxCost(u_0, v_c, uws) ##np.exp(u_0 * v_c ) / np.sum(u_ws, axis=len(x.shape)-1, keepdims=True)
    ## TODO: hack np.summing here, but should be done in
    ## softmaxCost
    ##cost = np.sum(y_0)

    ## this is the full cost formula
    ##cost = -u_0 * v_c + np.log(np.sum(np.exp(uws * v_c)))
    #cost = np.sum(np.exp(uws * v_c))

    #grad_vc = -u_0 + np.sum(softmaxCost(uws, v_c, uws))*uws
    #gradPred = grad_vc
    #dce_duw = -v_c*np.sum(softmaxCost(uws, v_c, uws))
    #dce_duo = -v_c
    #grad = [dce_duw, dce_duo]

    return cost, gradPred, grad

def negSamplingCostAndGradient(predicted, target, outputVectors, dataset,
    K=10):
    """ Negative sampling cost function for word2vec models """

    # Implement the cost and gradients for one predicted word vector
    # and one target word vector as a building block for word2vec
    # models, using the negative sampling technique. K is the sample
    # size. You might want to use dataset.sampleTokenIdx() to sample
    # a random word index.
    #
    # Note: See test_word2vec below for dataset's initialization.
    #
    # Input/Output Specifications: same as softmaxCostAndGradient
    # We will not provide starter code for this function, but feel
    # free to reference the code you previously wrote for this
    # assignment!

    v_c = predicted
    uws = outputVectors
    u_0 = target

    u_k = np.zeros((K, v_c.shape[0]))
    for i in xrange(K):
        u_k[i] = outputVectors[dataset.sampleTokenIdx()]

    #cost = np.log(sigmoid(u_0 * v_c)) - np.sum(np.log(sigmoid(u_k * v_c)))
    cost = np.log(sigmoid(u_0 * v_c)) - np.sum(np.log(sigmoid(u_k * v_c)))
    cost = np.sum(cost)
    #dce_vc = - (1-sigmoid(u_0 * v_c)) - np.sum(1-sigmoid(-u_k * v_c))

    gradPred = - (1-sigmoid(u_0 * v_c)) - np.sum(1-sigmoid(u_k * v_c))

    dce_uo = -(1-sigmoid(u_0 * v_c)) * v_c

    dce_uk = - (np.sum(-v_c + v_c * sigmoid(-u_k * v_c)))
    #grad = np.sum(v_c - v_c * sigmoid(-u_k * v_c))
    grad = [dce_uo, dce_uk]
    return cost, gradPred, grad

def skipgram(currentWord, C, contextWords, tokens, inputVectors, outputVectors,
    dataset, word2vecCostAndGradient = softmaxCostAndGradient):
    """ Skip-gram model in word2vec """

    #params provided centerword, C1, context, tokens, inputVectors, outputVectors, dataset, word2vecCostAndGradient

    #print skipgram("c", 3, ["a", "b", "e", "d", "b", "c"], dummy_tokens, dummy_vectors[:5,:], dummy_vectors[5:,:], dataset)


    # Implement the skip-gram model in this function.

    # Inputs:
    # - currrentWord: a string of the current center word
    # - C: integer, context size
    # - contextWords: list of no more than 2*C strings, the context words
    # - tokens: a dictionary that maps words to their indices in
    #      the word vector list
    # - inputVectors: "input" word vectors (as rows) for all tokens
    # - outputVectors: "output" word vectors (as rows) for all tokens
    # - word2vecCostAndGradient: the cost and gradient function for
    #      a prediction vector given the target word vectors,
    #      could be one of the two cost functions you
    #      implemented above

    # Outputs:
    # - cost: the cost function value for the skip-gram model
    # - grad: the gradient with respect to the word vectors
    # We will not provide starter code for this function, but feel
    # free to reference the code you previously wrote for this
    # assignment!
    uws = outputVectors # rows
    vc = inputVectors[tokens[currentWord]]
    gradIn = np.zeros(inputVectors.shape)
    gradOut = np.zeros(outputVectors.shape)
    cost = 0.0
    #           vc,        u0,     uws
    # softmax(predicted, target, outputVectors, dataset)
    # return cost, gradPred (grad_vc), grad (grad uw, uo)
    for cwd in contextWords:
        #uo = inputVectors[i]
        uo = tokens[cwd]
        single_cost, single_gin, single_gout = word2vecCostAndGradient(vc, uo, uws, dataset)
        cost += single_cost
        try:
            gradIn[tokens[currentWord],:] += single_gin
            gradOut += single_gout[0]
        except TypeError:
            import pdb; pdb.set_trace()

    return cost, gradIn, gradOut

def cbow(currentWord, C, contextWords, tokens, inputVectors, outputVectors,
    dataset, word2vecCostAndGradient = softmaxCostAndGradient):
    """ CBOW model in word2vec """

    # Implement the continuous bag-of-words model in this function.
    # Input/Output specifications: same as the skip-gram model
    # We will not provide starter code for this function, but feel
    # free to reference the code you previously wrote for this
    # assignment!

    #################################################################
    # IMPLEMENTING CBOW IS EXTRA CREDIT, DERIVATIONS IN THE WRIITEN #
    # ASSIGNMENT ARE NOT!                                           #
    #################################################################

    cost = 0
    gradIn = np.zeros(inputVectors.shape)
    gradOut = np.zeros(outputVectors.shape)

    ### YOUR CODE HERE
    #raise NotImplementedError
    ### END YOUR CODE

    return cost, gradIn, gradOut

#############################################
# Testing functions below. DO NOT MODIFY!   #
#############################################

def word2vec_sgd_wrapper(word2vecModel, tokens, wordVectors, dataset, C, word2vecCostAndGradient = softmaxCostAndGradient):
    batchsize = 50
    cost = 0.0
    grad = np.zeros(wordVectors.shape)
    N = wordVectors.shape[0]
    inputVectors = wordVectors[:N/2,:]
    outputVectors = wordVectors[N/2:,:]
    for i in xrange(batchsize):
        C1 = random.randint(1,C)
        centerword, context = dataset.getRandomContext(C1)

        if word2vecModel == skipgram:
            denom = 1
        else:
            denom = 1

        c, gin, gout = word2vecModel(centerword, C1, context, tokens, inputVectors, outputVectors, dataset, word2vecCostAndGradient)
        cost += c / batchsize / denom
        # HACK, I am returning multiple gradients because currently dce/duw and
        # dce / duo are returned separately, this might be incorrect
        #gout = gout[1]
        grad[:N/2, :] += gin / batchsize / denom
        grad[N/2:, :] += gout / batchsize / denom

    return cost, grad

def test_word2vec():
    # Interface to the dataset for negative sampling
    dataset = type('dummy', (), {})()
    def dummySampleTokenIdx():
        return random.randint(0, 4)

    def getRandomContext(C):
        tokens = ["a", "b", "c", "d", "e"]
        return tokens[random.randint(0,4)], [tokens[random.randint(0,4)] \
           for i in xrange(2*C)]
    dataset.sampleTokenIdx = dummySampleTokenIdx
    dataset.getRandomContext = getRandomContext

    random.seed(31415)
    np.random.seed(9265)
    dummy_vectors = normalizeRows(np.random.randn(10,3))
    dummy_tokens = dict([("a",0), ("b",1), ("c",2),("d",3),("e",4)])
    print "==== Gradient check for skip-gram ===="

    gradcheck_naive(lambda vec: word2vec_sgd_wrapper(skipgram, dummy_tokens, vec, dataset, 5, softmaxCostAndGradient), dummy_vectors)
    gradcheck_naive(lambda vec: word2vec_sgd_wrapper(skipgram, dummy_tokens, vec, dataset, 5, negSamplingCostAndGradient), dummy_vectors)
    #print "\n==== Gradient check for CBOW      ===="
    #gradcheck_naive(lambda vec: word2vec_sgd_wrapper(cbow, dummy_tokens, vec, dataset, 5), dummy_vectors)
    #gradcheck_naive(lambda vec: word2vec_sgd_wrapper(cbow, dummy_tokens, vec, dataset, 5, negSamplingCostAndGradient), dummy_vectors)

    print "\n=== Results ==="
    print "\n=== Skipgram ==="
    print "\n=== Softmax ==="
    print skipgram("c", 3, ["a", "b", "e", "d", "b", "c"], dummy_tokens, dummy_vectors[:5,:], dummy_vectors[5:,:], dataset)
    print "\n=== Negative Sampling ===="
    print skipgram("a", 3, ["a", "b", "c", "d", "b", "e"], dummy_tokens, dummy_vectors[:5,:], dummy_vectors[5:,:], dataset, negSamplingCostAndGradient)

    print skipgram("c", 1, ["a", "b"], dummy_tokens, dummy_vectors[:5,:], dummy_vectors[5:,:], dataset, negSamplingCostAndGradient)
    #print cbow("a", 2, ["a", "b", "c", "a"], dummy_tokens, dummy_vectors[:5,:], dummy_vectors[5:,:], dataset)
    #print cbow("a", 2, ["a", "b", "a", "c"], dummy_tokens, dummy_vectors[:5,:], dummy_vectors[5:,:], dataset, negSamplingCostAndGradient)

if __name__ == "__main__":
    test_normalize_rows()
    test_word2vec()

import numpy as np
import random

from q1_softmax import softmax
from q2_gradcheck import gradcheck_naive
from q2_sigmoid import sigmoid, sigmoid_grad

def normalizeRows(x):
    """ Row normalization function """
    l2norm = np.sqrt((x**2).sum(axis=1, keepdims=True))
    x /= l2norm
    return x

def test_normalize_rows():
    print "Testing normalizeRows..."
    x = normalizeRows(np.array([[3.0,4.0],[1, 2]]))
    # the result should be [[0.6, 0.8], [0.4472, 0.8944]]
    print x
    assert (x.all() == np.array([[0.6, 0.8], [0.4472, 0.8944]]).all())
    print ""

def softmaxCost(uo, vc, outputVectors):
    """ This is softmax(o,c) or y^o or p(o|c) """

    return np.exp(uo*vc) / np.sum(np.exp(outputVectors * vc), \
                                    axis=len(vc.shape)-1, keepdims=True)

# vc, u0, uws
def softmaxCostAndGradient(predicted, target, outputVectors, dataset):
    """ Softmax cost function for word2vec models """

    # Implement the cost and gradients for one predicted word vector
    # and one target word vector as a building block for word2vec
    # models, assuming the softmax prediction function and cross
    # entropy loss.

    # Inputs:
    # - predicted: numpy ndarray, predicted word vector (\hat{v} in
    #   the written component or \hat{r} in an earlier version)
    # - target: integer, the index of the target word
    # - outputVectors: "output" vectors (as rows) for all tokens
    # - dataset: needed for negative sampling, unused here.

    # Outputs:
    # - cost: cross entropy cost for the softmax word prediction
    # - gradPred: the gradient with respect to the predicted word
    #        vector
    # - grad: the gradient with respect to all the other word
    #        vectors
    v_c = predicted
    uws = outputVectors
    u_0 = target
    # this is from https://github.com/Khabermas/NLP_ps1
    probabilities = softmax(predicted.dot(outputVectors.T))
    cost = -np.log(probabilities[target])
    delta = probabilities
    delta[target] -= 1
    N = delta.shape[0]
    D = predicted.shape[0]
    grad = delta.reshape((N,1)) * predicted.reshape((1,D))
    gradPred = (delta.reshape((1,N)).dot(outputVectors)).flatten()

    #y_0 = softmaxCost(u_0, v_c, uws) ##np.exp(u_0 * v_c ) / np.sum(u_ws, axis=len(x.shape)-1, keepdims=True)
    ## TODO: hack np.summing here, but should be done in
    ## softmaxCost
    ##cost = np.sum(y_0)

    ## this is the full cost formula
    ##cost = -u_0 * v_c + np.log(np.sum(np.exp(uws * v_c)))
    #cost = np.sum(np.exp(uws * v_c))

    #grad_vc = -u_0 + np.sum(softmaxCost(uws, v_c, uws))*uws
    #gradPred = grad_vc
    #dce_duw = -v_c*np.sum(softmaxCost(uws, v_c, uws))
    #dce_duo = -v_c
    #grad = [dce_duw, dce_duo]

    return cost, gradPred, grad

def negSamplingCostAndGradient(predicted, target, outputVectors, dataset,
    K=10):
    """ Negative sampling cost function for word2vec models """

    # Implement the cost and gradients for one predicted word vector
    # and one target word vector as a building block for word2vec
    # models, using the negative sampling technique. K is the sample
    # size. You might want to use dataset.sampleTokenIdx() to sample
    # a random word index.
    #
    # Note: See test_word2vec below for dataset's initialization.
    #
    # Input/Output Specifications: same as softmaxCostAndGradient
    # We will not provide starter code for this function, but feel
    # free to reference the code you previously wrote for this
    # assignment!

    v_c = predicted
    uws = outputVectors
    u_0 = target

    u_k = np.zeros((K, v_c.shape[0]))
    for i in xrange(K):
        u_k[i] = outputVectors[dataset.sampleTokenIdx()]

    #cost = np.log(sigmoid(u_0 * v_c)) - np.sum(np.log(sigmoid(u_k * v_c)))
    cost = np.log(sigmoid(u_0 * v_c)) - np.sum(np.log(sigmoid(u_k * v_c)))
    cost = np.sum(cost)
    #dce_vc = - (1-sigmoid(u_0 * v_c)) - np.sum(1-sigmoid(-u_k * v_c))

    gradPred = - (1-sigmoid(u_0 * v_c)) - np.sum(1-sigmoid(u_k * v_c))

    dce_uo = -(1-sigmoid(u_0 * v_c)) * v_c

    dce_uk = - (np.sum(-v_c + v_c * sigmoid(-u_k * v_c)))
    #grad = np.sum(v_c - v_c * sigmoid(-u_k * v_c))
    grad = [dce_uo, dce_uk]
    return cost, gradPred, grad

def skipgram(currentWord, C, contextWords, tokens, inputVectors, outputVectors,
    dataset, word2vecCostAndGradient = softmaxCostAndGradient):
    """ Skip-gram model in word2vec """

    #params provided centerword, C1, context, tokens, inputVectors, outputVectors, dataset, word2vecCostAndGradient

    #print skipgram("c", 3, ["a", "b", "e", "d", "b", "c"], dummy_tokens, dummy_vectors[:5,:], dummy_vectors[5:,:], dataset)


    # Implement the skip-gram model in this function.

    # Inputs:
    # - currrentWord: a string of the current center word
    # - C: integer, context size
    # - contextWords: list of no more than 2*C strings, the context words
    # - tokens: a dictionary that maps words to their indices in
    #      the word vector list
    # - inputVectors: "input" word vectors (as rows) for all tokens
    # - outputVectors: "output" word vectors (as rows) for all tokens
    # - word2vecCostAndGradient: the cost and gradient function for
    #      a prediction vector given the target word vectors,
    #      could be one of the two cost functions you
    #      implemented above

    # Outputs:
    # - cost: the cost function value for the skip-gram model
    # - grad: the gradient with respect to the word vectors
    # We will not provide starter code for this function, but feel
    # free to reference the code you previously wrote for this
    # assignment!
    uws = outputVectors # rows
    vc = inputVectors[tokens[currentWord]]
    gradIn = np.zeros(inputVectors.shape)
    gradOut = np.zeros(outputVectors.shape)
    cost = 0.0
    #           vc,        u0,     uws
    # softmax(predicted, target, outputVectors, dataset)
    # return cost, gradPred (grad_vc), grad (grad uw, uo)
    for cwd in contextWords:
        #uo = inputVectors[i]
        uo = tokens[cwd]
        single_cost, single_gin, single_gout = word2vecCostAndGradient(vc, uo, uws, dataset)
        cost += single_cost
        try:
            gradIn[tokens[currentWord],:] += single_gin
            gradOut += single_gout[0]
        except TypeError:
            import pdb; pdb.set_trace()

    return cost, gradIn, gradOut

def cbow(currentWord, C, contextWords, tokens, inputVectors, outputVectors,
    dataset, word2vecCostAndGradient = softmaxCostAndGradient):
    """ CBOW model in word2vec """

    # Implement the continuous bag-of-words model in this function.
    # Input/Output specifications: same as the skip-gram model
    # We will not provide starter code for this function, but feel
    # free to reference the code you previously wrote for this
    # assignment!

    #################################################################
    # IMPLEMENTING CBOW IS EXTRA CREDIT, DERIVATIONS IN THE WRIITEN #
    # ASSIGNMENT ARE NOT!                                           #
    #################################################################

    cost = 0
    gradIn = np.zeros(inputVectors.shape)
    gradOut = np.zeros(outputVectors.shape)

    ### YOUR CODE HERE
    #raise NotImplementedError
    ### END YOUR CODE

    return cost, gradIn, gradOut

#############################################
# Testing functions below. DO NOT MODIFY!   #
#############################################

def word2vec_sgd_wrapper(word2vecModel, tokens, wordVectors, dataset, C, word2vecCostAndGradient = softmaxCostAndGradient):
    batchsize = 50
    cost = 0.0
    grad = np.zeros(wordVectors.shape)
    N = wordVectors.shape[0]
    inputVectors = wordVectors[:N/2,:]
    outputVectors = wordVectors[N/2:,:]
    for i in xrange(batchsize):
        C1 = random.randint(1,C)
        centerword, context = dataset.getRandomContext(C1)

        if word2vecModel == skipgram:
            denom = 1
        else:
            denom = 1

        c, gin, gout = word2vecModel(centerword, C1, context, tokens, inputVectors, outputVectors, dataset, word2vecCostAndGradient)
        cost += c / batchsize / denom
        # HACK, I am returning multiple gradients because currently dce/duw and
        # dce / duo are returned separately, this might be incorrect
        #gout = gout[1]
        grad[:N/2, :] += gin / batchsize / denom
        grad[N/2:, :] += gout / batchsize / denom

    return cost, grad

def test_word2vec():
    # Interface to the dataset for negative sampling
    dataset = type('dummy', (), {})()
    def dummySampleTokenIdx():
        return random.randint(0, 4)

    def getRandomContext(C):
        tokens = ["a", "b", "c", "d", "e"]
        return tokens[random.randint(0,4)], [tokens[random.randint(0,4)] \
           for i in xrange(2*C)]
    dataset.sampleTokenIdx = dummySampleTokenIdx
    dataset.getRandomContext = getRandomContext

    random.seed(31415)
    np.random.seed(9265)
    dummy_vectors = normalizeRows(np.random.randn(10,3))
    dummy_tokens = dict([("a",0), ("b",1), ("c",2),("d",3),("e",4)])
    print "==== Gradient check for skip-gram ===="

    gradcheck_naive(lambda vec: word2vec_sgd_wrapper(skipgram, dummy_tokens, vec, dataset, 5, softmaxCostAndGradient), dummy_vectors)
    gradcheck_naive(lambda vec: word2vec_sgd_wrapper(skipgram, dummy_tokens, vec, dataset, 5, negSamplingCostAndGradient), dummy_vectors)
    #print "\n==== Gradient check for CBOW      ===="
    #gradcheck_naive(lambda vec: word2vec_sgd_wrapper(cbow, dummy_tokens, vec, dataset, 5), dummy_vectors)
    #gradcheck_naive(lambda vec: word2vec_sgd_wrapper(cbow, dummy_tokens, vec, dataset, 5, negSamplingCostAndGradient), dummy_vectors)

    print "\n=== Results ==="
    print "\n=== Skipgram ==="
    print "\n=== Softmax ==="
    print skipgram("c", 3, ["a", "b", "e", "d", "b", "c"], dummy_tokens, dummy_vectors[:5,:], dummy_vectors[5:,:], dataset)
    print "\n=== Negative Sampling ===="
    print skipgram("a", 3, ["a", "b", "c", "d", "b", "e"], dummy_tokens, dummy_vectors[:5,:], dummy_vectors[5:,:], dataset, negSamplingCostAndGradient)

    print skipgram("c", 1, ["a", "b"], dummy_tokens, dummy_vectors[:5,:], dummy_vectors[5:,:], dataset, negSamplingCostAndGradient)
    #print cbow("a", 2, ["a", "b", "c", "a"], dummy_tokens, dummy_vectors[:5,:], dummy_vectors[5:,:], dataset)
    #print cbow("a", 2, ["a", "b", "a", "c"], dummy_tokens, dummy_vectors[:5,:], dummy_vectors[5:,:], dataset, negSamplingCostAndGradient)

if __name__ == "__main__":
    test_normalize_rows()
    test_word2vec()

Row normalization function # the result should be [[0.6, 0.8], [0.4472, 0.8944]] This is softmax(o,c) or y^o or p(o|c) # vc, u0, uws Softmax cost function for word2vec models # Implement the cost and gradients for one predicted word vector # and one target word vector as a building block for word2vec # models, assuming the softmax prediction function and cross # entropy loss. # Inputs: # - predicted: numpy ndarray, predicted word vector (\hat{v} in #   the written component or \hat{r} in an earlier version) # - target: integer, the index of the target word # - outputVectors: "output" vectors (as rows) for all tokens # - dataset: needed for negative sampling, unused here. # Outputs: # - cost: cross entropy cost for the softmax word prediction # - gradPred: the gradient with respect to the predicted word #        vector # - grad: the gradient with respect to all the other word #        vectors # this is from https://github.com/Khabermas/NLP_ps1 #y_0 = softmaxCost(u_0, v_c, uws) ##np.exp(u_0 * v_c ) / np.sum(u_ws, axis=len(x.shape)-1, keepdims=True) ## TODO: hack np.summing here, but should be done in ## softmaxCost ##cost = np.sum(y_0) ## this is the full cost formula ##cost = -u_0 * v_c + np.log(np.sum(np.exp(uws * v_c))) #cost = np.sum(np.exp(uws * v_c)) #grad_vc = -u_0 + np.sum(softmaxCost(uws, v_c, uws))*uws #gradPred = grad_vc #dce_duw = -v_c*np.sum(softmaxCost(uws, v_c, uws)) #dce_duo = -v_c #grad = [dce_duw, dce_duo] Negative sampling cost function for word2vec models # Implement the cost and gradients for one predicted word vector # and one target word vector as a building block for word2vec # models, using the negative sampling technique. K is the sample # size. You might want to use dataset.sampleTokenIdx() to sample # a random word index. # # Note: See test_word2vec below for dataset's initialization. # # Input/Output Specifications: same as softmaxCostAndGradient # We will not provide starter code for this function, but feel # free to reference the code you previously wrote for this # assignment! #cost = np.log(sigmoid(u_0 * v_c)) - np.sum(np.log(sigmoid(u_k * v_c))) #dce_vc = - (1-sigmoid(u_0 * v_c)) - np.sum(1-sigmoid(-u_k * v_c)) #grad = np.sum(v_c - v_c * sigmoid(-u_k * v_c)) Skip-gram model in word2vec #params provided centerword, C1, context, tokens, inputVectors, outputVectors, dataset, word2vecCostAndGradient #print skipgram("c", 3, ["a", "b", "e", "d", "b", "c"], dummy_tokens, dummy_vectors[:5,:], dummy_vectors[5:,:], dataset) # Implement the skip-gram model in this function. # Inputs: # - currrentWord: a string of the current center word # - C: integer, context size # - contextWords: list of no more than 2*C strings, the context words # - tokens: a dictionary that maps words to their indices in #      the word vector list # - inputVectors: "input" word vectors (as rows) for all tokens # - outputVectors: "output" word vectors (as rows) for all tokens # - word2vecCostAndGradient: the cost and gradient function for #      a prediction vector given the target word vectors, #      could be one of the two cost functions you #      implemented above # Outputs: # - cost: the cost function value for the skip-gram model # - grad: the gradient with respect to the word vectors # We will not provide starter code for this function, but feel # free to reference the code you previously wrote for this # assignment! # rows #           vc,        u0,     uws # softmax(predicted, target, outputVectors, dataset) # return cost, gradPred (grad_vc), grad (grad uw, uo) #uo = inputVectors[i] CBOW model in word2vec # Implement the continuous bag-of-words model in this function. # Input/Output specifications: same as the skip-gram model # We will not provide starter code for this function, but feel # free to reference the code you previously wrote for this # assignment! ################################################################# # IMPLEMENTING CBOW IS EXTRA CREDIT, DERIVATIONS IN THE WRIITEN # # ASSIGNMENT ARE NOT!                                           # ################################################################# ### YOUR CODE HERE #raise NotImplementedError ### END YOUR CODE ############################################# # Testing functions below. DO NOT MODIFY!   # ############################################# # HACK, I am returning multiple gradients because currently dce/duw and # dce / duo are returned separately, this might be incorrect #gout = gout[1] # Interface to the dataset for negative sampling #print "\n==== Gradient check for CBOW      ====" #gradcheck_naive(lambda vec: word2vec_sgd_wrapper(cbow, dummy_tokens, vec, dataset, 5), dummy_vectors) #gradcheck_naive(lambda vec: word2vec_sgd_wrapper(cbow, dummy_tokens, vec, dataset, 5, negSamplingCostAndGradient), dummy_vectors) #print cbow("a", 2, ["a", "b", "c", "a"], dummy_tokens, dummy_vectors[:5,:], dummy_vectors[5:,:], dataset) #print cbow("a", 2, ["a", "b", "a", "c"], dummy_tokens, dummy_vectors[:5,:], dummy_vectors[5:,:], dataset, negSamplingCostAndGradient)