from __future__ import print_function
import collections
import math
import numpy as np
import random
import tensorflow as tf
from six.moves import range

from assignment5_word2vec.dataset import vocabulary_size, loadDataset

data, count, dictionary, reverse_dictionary = loadDataset()

class BatchGenerator():
    def __init__(self, array, windowSize, rndSeed=21344):
        self.array = array
        self.N = len(array)
        self.windowSize = windowSize
        # indexes of elements in array that have at least windowSize elements to each size
        self.sampledIndexes = np.arange(windowSize, self.N - windowSize)
        np.random.seed(rndSeed)
        self.iter = 0

    def __call__(self, batchSize):
        #indexes = np.random.choice(self.sampledIndexes, batchSize, replace=False)
        indexes = np.empty(batchSize, dtype='int64')
        for i in range(batchSize):
            indexes[i] = self.sampledIndexes[self.iter]
            self.iter = (self.iter + 1) % len(self.sampledIndexes)
        windows = np.empty(batchSize*self.windowSize*2, dtype='int64')
        wi = 0
        # write filling of the array more efficiently, with numpy
        for i in indexes:
            for j in range(1, self.windowSize+1):
                windows[wi] = self.array[i+j]; wi += 1
                windows[wi] = self.array[i-j]; wi += 1
        centers = np.empty((batchSize, 1), dtype='int64')
        for i in range(batchSize): centers[i,0] = self.array[indexes[i]]
        return centers, windows

def test_batch():
    bg = BatchGenerator(data, 1)
    ind, win = bg(3)
    print(ind)
    print(win)

def train_cbow(learnRate=1.0, num_sampled = 64, train_steps = 100001, window_size = 1):
    batch_size = 128
    embedding_size = 128  # Dimension of the embedding vector.
    # We pick a random validation set to sample nearest neighbors. here we limit the
    # validation samples to the words that have a low numeric ID, which by
    # construction are also the most frequent.
    valid_size = 16  # Random set of words to evaluate similarity on.
    valid_window = 100  # Only pick dev samples in the head of the distribution.
    valid_examples = np.array(random.sample(range(valid_window), valid_size))
     # Number of negative examples to sample.

    graph = tf.Graph()

    with graph.as_default(): #, tf.device('/cpu:0'):
        # Input data.
        train_dataset = tf.placeholder(tf.int32, shape=[batch_size * window_size * 2])
        train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])
        valid_dataset = tf.constant(valid_examples, dtype=tf.int32)

        # Variables.
        embeddings = tf.Variable(
            tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))
        softmax_weights = tf.Variable(
            tf.truncated_normal([vocabulary_size, embedding_size],
                                stddev=1.0 / math.sqrt(embedding_size)))
        softmax_biases = tf.Variable(tf.zeros([vocabulary_size]))

        # Model.
        # Look up embeddings for inputs.
        embed = tf.nn.embedding_lookup(embeddings, train_dataset)
        embed = tf.reduce_sum(tf.reshape(embed, (batch_size, window_size * 2, embedding_size)), 1)
        # Compute the softmax loss, using a sample of the negative labels each time.
        loss = tf.reduce_mean(
            tf.nn.sampled_softmax_loss(weights=softmax_weights, biases=softmax_biases, inputs=embed,
                                       labels=train_labels, num_sampled=num_sampled, num_classes=vocabulary_size))

        # Optimizer.
        # Note: The optimizer will optimize the softmax_weights AND the embeddings.
        # This is because the embeddings are defined as a variable quantity and the
        # optimizer's `minimize` method will by default modify all variable quantities
        # that contribute to the tensor it is passed.
        # See docs on `tf.train.Optimizer.minimize()` for more details.
        optimizer = tf.train.AdagradOptimizer(learning_rate=learnRate).minimize(loss)

        # Compute the similarity between minibatch examples and all embeddings.
        # We use the cosine distance:
        norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
        normalized_embeddings = embeddings / norm
        valid_embeddings = tf.nn.embedding_lookup(
            normalized_embeddings, valid_dataset)
        similarity = tf.matmul(valid_embeddings, tf.transpose(normalized_embeddings))

    with tf.Session(graph=graph) as session:
      tf.global_variables_initializer().run()
      batchGen = BatchGenerator(data, window_size)
      print('Initialized')
      average_loss = 0
      for step in range(train_steps):
        batch_labels, batch_data = batchGen(batch_size)
        feed_dict = {train_dataset : batch_data, train_labels : batch_labels}
        _, l = session.run([optimizer, loss], feed_dict=feed_dict)
        average_loss += l
        if step % 2000 == 0:
          if step > 0:
            average_loss = average_loss / 2000
          # The average loss is an estimate of the loss over the last 2000 batches.
          print('Average loss at step %d: %f' % (step, average_loss))
          average_loss = 0
        # note that this is expensive (~20% slowdown if computed every 500 steps)
        if step % 10000 == 0:
          sim = similarity.eval()
          for i in range(valid_size):
            valid_word = reverse_dictionary[valid_examples[i]]
            top_k = 8 # number of nearest neighbors
            nearest = (-sim[i, :]).argsort()[1:top_k+1]
            log = 'Nearest to %s:' % valid_word
            for k in range(top_k):
              close_word = reverse_dictionary[nearest[k]]
              log = '%s %s,' % (log, close_word)
            print(log)
      final_embeddings = normalized_embeddings.eval()
    return final_embeddings

if __name__ == '__main__':
    #test_batch()
    train_cbow()
    #train_skipgram(learnRate=0.1)
    #train_skipgram(num_sampled=128)
from __future__ import print_function
import collections
import math
import numpy as np
import random
import tensorflow as tf
from six.moves import range

from assignment5_word2vec.dataset import vocabulary_size, loadDataset

data, count, dictionary, reverse_dictionary = loadDataset()

class BatchGenerator():
    def __init__(self, array, windowSize, rndSeed=21344):
        self.array = array
        self.N = len(array)
        self.windowSize = windowSize
        # indexes of elements in array that have at least windowSize elements to each size
        self.sampledIndexes = np.arange(windowSize, self.N - windowSize)
        np.random.seed(rndSeed)
        self.iter = 0

    def __call__(self, batchSize):
        #indexes = np.random.choice(self.sampledIndexes, batchSize, replace=False)
        indexes = np.empty(batchSize, dtype='int64')
        for i in range(batchSize):
            indexes[i] = self.sampledIndexes[self.iter]
            self.iter = (self.iter + 1) % len(self.sampledIndexes)
        windows = np.empty(batchSize*self.windowSize*2, dtype='int64')
        wi = 0
        # write filling of the array more efficiently, with numpy
        for i in indexes:
            for j in range(1, self.windowSize+1):
                windows[wi] = self.array[i+j]; wi += 1
                windows[wi] = self.array[i-j]; wi += 1
        centers = np.empty((batchSize, 1), dtype='int64')
        for i in range(batchSize): centers[i,0] = self.array[indexes[i]]
        return centers, windows

def test_batch():
    bg = BatchGenerator(data, 1)
    ind, win = bg(3)
    print(ind)
    print(win)

def train_cbow(learnRate=1.0, num_sampled = 64, train_steps = 100001, window_size = 1):
    batch_size = 128
    embedding_size = 128  # Dimension of the embedding vector.
    # We pick a random validation set to sample nearest neighbors. here we limit the
    # validation samples to the words that have a low numeric ID, which by
    # construction are also the most frequent.
    valid_size = 16  # Random set of words to evaluate similarity on.
    valid_window = 100  # Only pick dev samples in the head of the distribution.
    valid_examples = np.array(random.sample(range(valid_window), valid_size))
     # Number of negative examples to sample.

    graph = tf.Graph()

    with graph.as_default(): #, tf.device('/cpu:0'):
        # Input data.
        train_dataset = tf.placeholder(tf.int32, shape=[batch_size * window_size * 2])
        train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])
        valid_dataset = tf.constant(valid_examples, dtype=tf.int32)

        # Variables.
        embeddings = tf.Variable(
            tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))
        softmax_weights = tf.Variable(
            tf.truncated_normal([vocabulary_size, embedding_size],
                                stddev=1.0 / math.sqrt(embedding_size)))
        softmax_biases = tf.Variable(tf.zeros([vocabulary_size]))

        # Model.
        # Look up embeddings for inputs.
        embed = tf.nn.embedding_lookup(embeddings, train_dataset)
        embed = tf.reduce_sum(tf.reshape(embed, (batch_size, window_size * 2, embedding_size)), 1)
        # Compute the softmax loss, using a sample of the negative labels each time.
        loss = tf.reduce_mean(
            tf.nn.sampled_softmax_loss(weights=softmax_weights, biases=softmax_biases, inputs=embed,
                                       labels=train_labels, num_sampled=num_sampled, num_classes=vocabulary_size))

        # Optimizer.
        # Note: The optimizer will optimize the softmax_weights AND the embeddings.
        # This is because the embeddings are defined as a variable quantity and the
        # optimizer's `minimize` method will by default modify all variable quantities
        # that contribute to the tensor it is passed.
        # See docs on `tf.train.Optimizer.minimize()` for more details.
        optimizer = tf.train.AdagradOptimizer(learning_rate=learnRate).minimize(loss)

        # Compute the similarity between minibatch examples and all embeddings.
        # We use the cosine distance:
        norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
        normalized_embeddings = embeddings / norm
        valid_embeddings = tf.nn.embedding_lookup(
            normalized_embeddings, valid_dataset)
        similarity = tf.matmul(valid_embeddings, tf.transpose(normalized_embeddings))

    with tf.Session(graph=graph) as session:
      tf.global_variables_initializer().run()
      batchGen = BatchGenerator(data, window_size)
      print('Initialized')
      average_loss = 0
      for step in range(train_steps):
        batch_labels, batch_data = batchGen(batch_size)
        feed_dict = {train_dataset : batch_data, train_labels : batch_labels}
        _, l = session.run([optimizer, loss], feed_dict=feed_dict)
        average_loss += l
        if step % 2000 == 0:
          if step > 0:
            average_loss = average_loss / 2000
          # The average loss is an estimate of the loss over the last 2000 batches.
          print('Average loss at step %d: %f' % (step, average_loss))
          average_loss = 0
        # note that this is expensive (~20% slowdown if computed every 500 steps)
        if step % 10000 == 0:
          sim = similarity.eval()
          for i in range(valid_size):
            valid_word = reverse_dictionary[valid_examples[i]]
            top_k = 8 # number of nearest neighbors
            nearest = (-sim[i, :]).argsort()[1:top_k+1]
            log = 'Nearest to %s:' % valid_word
            for k in range(top_k):
              close_word = reverse_dictionary[nearest[k]]
              log = '%s %s,' % (log, close_word)
            print(log)
      final_embeddings = normalized_embeddings.eval()
    return final_embeddings

if __name__ == '__main__':
    #test_batch()
    train_cbow()
    #train_skipgram(learnRate=0.1)
    #train_skipgram(num_sampled=128)
# indexes of elements in array that have at least windowSize elements to each size #indexes = np.random.choice(self.sampledIndexes, batchSize, replace=False) # write filling of the array more efficiently, with numpy # Dimension of the embedding vector. # We pick a random validation set to sample nearest neighbors. here we limit the # validation samples to the words that have a low numeric ID, which by # construction are also the most frequent. # Random set of words to evaluate similarity on. # Only pick dev samples in the head of the distribution. # Number of negative examples to sample. #, tf.device('/cpu:0'): # Input data. # Variables. # Model. # Look up embeddings for inputs. # Compute the softmax loss, using a sample of the negative labels each time. # Optimizer. # Note: The optimizer will optimize the softmax_weights AND the embeddings. # This is because the embeddings are defined as a variable quantity and the # optimizer's `minimize` method will by default modify all variable quantities # that contribute to the tensor it is passed. # See docs on `tf.train.Optimizer.minimize()` for more details. # Compute the similarity between minibatch examples and all embeddings. # We use the cosine distance: # The average loss is an estimate of the loss over the last 2000 batches. # note that this is expensive (~20% slowdown if computed every 500 steps) # number of nearest neighbors #test_batch() #train_skipgram(learnRate=0.1) #train_skipgram(num_sampled=128)