import argparse
import csv
import cPickle as pickle
import collections
from HTMLParser import HTMLParser
import operator
import re
import json

import sklearn.preprocessing

# Taken from
# https://github.com/tensorflow/tensorflow/blob/16254e75e2fe4bb0f879b45fbad0c4b62c028011/tensorflow/models/rnn/translate/data_utils.py#L43
_WORD_SPLIT = re.compile(b"([.,!?\"':;)(])")


def basic_tokenizer(split_sentence):
  """Very basic tokenizer: split the sentence into a list of tokens."""
  words = []
  for space_separated_fragment in split_sentence:
    words.extend(re.split(_WORD_SPLIT, space_separated_fragment))
  return [w for w in words if w]

def parse_recipes(input_data):
  # Example entry:
  #"title": "Tweet New RSS Feed Item", 
  #"description": "Automatically tweet new RSS feed items.", 
  #"action_channel": "TwitterV2API", 
  #"event_channel": "RSSAPI", 
  #"action": "tweet",
  #"event": "new_feed", 
  #"rule": "{u'message': u'{{title}}: {{link}}'}"
  recipes = []
  raw_data = json.load(input_data)
  for item in raw_data:
      if item["event_channel"] == None:
        item["event_channel"]="None"
      if item["action_channel"] == None:
        item["action_channel"]="None"
      if item["event"] == None:
        item["event"]="None"
      if item["action"] == None:
        item["action"]="None"
      recipes.append({
          'recipe': item["title"],
          'trigger_chan': item["event_channel"],
          'trigger_func': item["event_channel"] + '.' + item["event"],
          'trigger_func_pure': item["event"],
          'action_chan': item["action_channel"],
          'action_func': item["action_channel"] + '.' + item["action"],
          'action_func_pure': item["action"],
          'rule': item["rule"]
      })
  return recipes


def parse_Zapier(input_train, input_dev, input_test):
  training_recipes = parse_recipes(input_train)
  dev_recipes = parse_recipes(input_dev)
  test_recipes = parse_recipes(input_test)
  data = {}
  for section_name, section_data in zip(
      ['train', 'dev','test'], [training_recipes, dev_recipes, test_recipes]):
    data[section_name] = {}
    for m in ['trigger', 'action']:
      data[section_name][m] = []
      for item in section_data:
        data[section_name][m].append({'recipe': basic_tokenizer(item['recipe'].lower().split()),
                                      'chan': item[m + '_chan'],
                                      'func': item[m + '_func'],
                                     })
  return data


if __name__ == '__main__':
  parser = argparse.ArgumentParser()
  parser.add_argument('--output', required=True)
  parser.add_argument('--input-train', help='Path to Zapier training data(json file)')
  parser.add_argument('--input-dev', help='Path to Zapier dev data(json file)')
  parser.add_argument('--input-test', help='Path to Zapier test data(json file)')
  parser.add_argument('--word-list', help='Path to word list(json file)')
  args = parser.parse_args()

  with open(args.input_train) as input_train, open(args.input_dev) as input_dev, open(args.input_test) as input_test:
      data = parse_Zapier(input_train, input_dev, input_test)

  # Assign IDs to each word and label in the training data
  vocab = collections.Counter()
  for item in data['train']['action']:
    for word in item['recipe']:
      vocab[word] += 1
  words_dec_freq = sorted(vocab.iteritems(),
                          key=operator.itemgetter(1),
                          reverse=True)

  fout = open('Zapier_words.json','w')

  word_list = []
  for item in words_dec_freq:
    new_item = {}
    new_item["word"] = item[0]
    new_item["freq"] = item[1]
    word_list.append(new_item)
  fout.write(json.dumps(word_list, indent = 1))

  if args.word_list:
    with open(args.word_list) as f:
      word_list = json.load(f)
    word_ids = {}
    for item in word_list:
      word_ids[item["word"]] = len(word_ids)
  else:
    word_ids = {k: i for i, (k, count) in enumerate(words_dec_freq)}

  # Make {train,test}_{trigger,action}_{channels,functions}.
  all_labels = collections.defaultdict()
  labelers = {}
  for m in ['trigger', 'action']:
    for n in ['chans', 'funcs']:
      label_type = m + '_' + n
      labels = {}

      for section in ['train', 'dev', 'test']:
        if section not in data: continue
        labels[section] = map(operator.itemgetter(n[:-1]), data[section][m])

      labeler = sklearn.preprocessing.LabelEncoder()
      labeler.fit(labels['train'] + labels['test'])

      for section in ['train', 'dev', 'test']:
        if section not in data: continue
        labels[section] = labeler.transform(labels[section])

      all_labels[label_type] = labels
      labelers[label_type] = labeler

  label_types = ('trigger_chans', 'trigger_funcs', 'action_chans',
                 'action_funcs')

  outputs = {}
  for section in ['train', 'dev', 'test']:
    output = []
    for i in xrange(len(data[section]['action'])):
      words = data[section]['action'][i]['recipe']
      item = {'ids': [word_ids.get(word, len(word_ids)) for word in words],
              'labels': [all_labels[t][section][i] for t in label_types],
              'label_names': [labelers[t].classes_[all_labels[t][section][i]]
                              for t in label_types],
              'words': words}
      output.append(item)
      #output.append({'words': words, 'ids': ids, 'labels': labels})
    outputs[section] = output
  outputs['label_types'] = label_types
  outputs['labelers'] = labelers
  outputs['word_ids'] = word_ids
  outputs['num_labels'] = [len(labelers[t].classes_) for t in label_types]

  pickle.dump(outputs, open(args.output, 'w'), pickle.HIGHEST_PROTOCOL)

import argparse
import csv
import cPickle as pickle
import collections
from HTMLParser import HTMLParser
import operator
import re
import json

import sklearn.preprocessing

# Taken from
# https://github.com/tensorflow/tensorflow/blob/16254e75e2fe4bb0f879b45fbad0c4b62c028011/tensorflow/models/rnn/translate/data_utils.py#L43
_WORD_SPLIT = re.compile(b"([.,!?\"':;)(])")


def basic_tokenizer(split_sentence):
  """Very basic tokenizer: split the sentence into a list of tokens."""
  words = []
  for space_separated_fragment in split_sentence:
    words.extend(re.split(_WORD_SPLIT, space_separated_fragment))
  return [w for w in words if w]

def parse_recipes(input_data):
  # Example entry:
  #"title": "Tweet New RSS Feed Item", 
  #"description": "Automatically tweet new RSS feed items.", 
  #"action_channel": "TwitterV2API", 
  #"event_channel": "RSSAPI", 
  #"action": "tweet",
  #"event": "new_feed", 
  #"rule": "{u'message': u'{{title}}: {{link}}'}"
  recipes = []
  raw_data = json.load(input_data)
  for item in raw_data:
      if item["event_channel"] == None:
        item["event_channel"]="None"
      if item["action_channel"] == None:
        item["action_channel"]="None"
      if item["event"] == None:
        item["event"]="None"
      if item["action"] == None:
        item["action"]="None"
      recipes.append({
          'recipe': item["title"],
          'trigger_chan': item["event_channel"],
          'trigger_func': item["event_channel"] + '.' + item["event"],
          'trigger_func_pure': item["event"],
          'action_chan': item["action_channel"],
          'action_func': item["action_channel"] + '.' + item["action"],
          'action_func_pure': item["action"],
          'rule': item["rule"]
      })
  return recipes


def parse_Zapier(input_train, input_dev, input_test):
  training_recipes = parse_recipes(input_train)
  dev_recipes = parse_recipes(input_dev)
  test_recipes = parse_recipes(input_test)
  data = {}
  for section_name, section_data in zip(
      ['train', 'dev','test'], [training_recipes, dev_recipes, test_recipes]):
    data[section_name] = {}
    for m in ['trigger', 'action']:
      data[section_name][m] = []
      for item in section_data:
        data[section_name][m].append({'recipe': basic_tokenizer(item['recipe'].lower().split()),
                                      'chan': item[m + '_chan'],
                                      'func': item[m + '_func'],
                                     })
  return data


if __name__ == '__main__':
  parser = argparse.ArgumentParser()
  parser.add_argument('--output', required=True)
  parser.add_argument('--input-train', help='Path to Zapier training data(json file)')
  parser.add_argument('--input-dev', help='Path to Zapier dev data(json file)')
  parser.add_argument('--input-test', help='Path to Zapier test data(json file)')
  parser.add_argument('--word-list', help='Path to word list(json file)')
  args = parser.parse_args()

  with open(args.input_train) as input_train, open(args.input_dev) as input_dev, open(args.input_test) as input_test:
      data = parse_Zapier(input_train, input_dev, input_test)

  # Assign IDs to each word and label in the training data
  vocab = collections.Counter()
  for item in data['train']['action']:
    for word in item['recipe']:
      vocab[word] += 1
  words_dec_freq = sorted(vocab.iteritems(),
                          key=operator.itemgetter(1),
                          reverse=True)

  fout = open('Zapier_words.json','w')

  word_list = []
  for item in words_dec_freq:
    new_item = {}
    new_item["word"] = item[0]
    new_item["freq"] = item[1]
    word_list.append(new_item)
  fout.write(json.dumps(word_list, indent = 1))

  if args.word_list:
    with open(args.word_list) as f:
      word_list = json.load(f)
    word_ids = {}
    for item in word_list:
      word_ids[item["word"]] = len(word_ids)
  else:
    word_ids = {k: i for i, (k, count) in enumerate(words_dec_freq)}

  # Make {train,test}_{trigger,action}_{channels,functions}.
  all_labels = collections.defaultdict()
  labelers = {}
  for m in ['trigger', 'action']:
    for n in ['chans', 'funcs']:
      label_type = m + '_' + n
      labels = {}

      for section in ['train', 'dev', 'test']:
        if section not in data: continue
        labels[section] = map(operator.itemgetter(n[:-1]), data[section][m])

      labeler = sklearn.preprocessing.LabelEncoder()
      labeler.fit(labels['train'] + labels['test'])

      for section in ['train', 'dev', 'test']:
        if section not in data: continue
        labels[section] = labeler.transform(labels[section])

      all_labels[label_type] = labels
      labelers[label_type] = labeler

  label_types = ('trigger_chans', 'trigger_funcs', 'action_chans',
                 'action_funcs')

  outputs = {}
  for section in ['train', 'dev', 'test']:
    output = []
    for i in xrange(len(data[section]['action'])):
      words = data[section]['action'][i]['recipe']
      item = {'ids': [word_ids.get(word, len(word_ids)) for word in words],
              'labels': [all_labels[t][section][i] for t in label_types],
              'label_names': [labelers[t].classes_[all_labels[t][section][i]]
                              for t in label_types],
              'words': words}
      output.append(item)
      #output.append({'words': words, 'ids': ids, 'labels': labels})
    outputs[section] = output
  outputs['label_types'] = label_types
  outputs['labelers'] = labelers
  outputs['word_ids'] = word_ids
  outputs['num_labels'] = [len(labelers[t].classes_) for t in label_types]

  pickle.dump(outputs, open(args.output, 'w'), pickle.HIGHEST_PROTOCOL)

# Taken from # https://github.com/tensorflow/tensorflow/blob/16254e75e2fe4bb0f879b45fbad0c4b62c028011/tensorflow/models/rnn/translate/data_utils.py#L43 Very basic tokenizer: split the sentence into a list of tokens. # Example entry: #"title": "Tweet New RSS Feed Item", #"description": "Automatically tweet new RSS feed items.", #"action_channel": "TwitterV2API", #"event_channel": "RSSAPI", #"action": "tweet", #"event": "new_feed", #"rule": "{u'message': u'{{title}}: {{link}}'}" # Assign IDs to each word and label in the training data # Make {train,test}_{trigger,action}_{channels,functions}. #output.append({'words': words, 'ids': ids, 'labels': labels})