<filename>Archived/labs_21/gz/ingr_scratchpad.py
# -*- coding: utf-8 -*-
"""ingr_scratchpad.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1LpU5z1NpyFJFJR5g3Vabd4MQ66LJJ0Tj

# Really tho'... how many ingredients could there possibly be???
"""


# imports.  you're prolly gonna use all these, right?
import pdb

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

from tensorflow.keras.preprocessing import sequence
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Embedding
from tensorflow.keras.layers import LSTM

from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.neighbors import NearestNeighbors

import re
import string

import spacy
nlp = spacy.load("en_core_web_sm")

# Pulling in recipe lists
df = pd.read_csv('../data/clean_recipes.csv', sep=';')
kgl1_df = pd.read_json('../data/kgl_ingredient_train.json')
kgl2_df = pd.read_json('../data/kgl_ingredient_test.json')

df = df.rename(columns={'Ingredients':'ingredients'})
df.head(3)

# df ingredients is a single string, splitting into a list of strings
df['ingredients'] = df['ingredients'].apply(lambda x: x.split(','))

l1 = df['ingredients']
l2 = kgl1_df['ingredients']
l3 = kgl2_df['ingredients']

ingr_series = l1.append([l2, l3], ignore_index=True)

ingr_series[:10]

def scrub_ingredients(ingr_list):

    chtble = str.maketrans('', '', string.punctuation + '0123456789')

    # drop special characters and multiple spaces, then convert to lowercase
    clean_list = []
    for ingr_string in ingr_list:
        ingr_string = re.sub(r'\W\s+', ' ', ingr_string.strip().translate(chtble))
        clean_list.append(ingr_string.lower())

    # convert to lowercase & drop known measurement words
    dropwords = ['ounce', 'ounces', 'oz', 'lb', ' pound', 'the', 'a', 'an'
                 'package', 'packages', 'inches', 'and', 'for', 'as', 'is']

    outlist = []

    for i in clean_list:
        words = i.split()
        words = [word for word in words if word not in dropwords]
        i = ' '.join(words)
        outlist.append(i)

    return outlist

ingr_s = ingr_series.apply(scrub_ingredients)
ingr_s[:10]

def ingredient_count(dfcolumn):

    # Create dictionary with word count
    d = dict()

    for l in dfcolumn:
        for i in l:
            if i in d:
                d[i] = d[i] + 1
            else:
                d[i] = 1

    return pd.DataFrame(d.items(), columns=['ingredient', 'count'])

# create df of unique ingredients and their number of occurances in recipe dfs
ingr_count_df = ingredient_count(ingr_s).sort_values('count', ascending=False).reset_index(drop=True)
ingr_count_df.head(15)

modeling_df = ingr_s.apply(pd.Series)
modeling_df = modeling_df.rename(columns = lambda x: 'ingr_' + str(x))
modeling_df.head(10)

(modeling_df.isnull().mean() * 100).head(25)

modeling_df = modeling_df.replace(np.nan, '', regex=True)
modeling_df.head()

ingr_int = {ingr:i for i, ingr in enumerate(ingr_count_df['ingredient'])}
int_ingr = {i:ingr for i, ingr in enumerate(ingr_count_df['ingredient'])}

# convert ingredients to numeric representations
encoded_df = modeling_df.applymap(ingr_int.get)
print(encoded_df.shape)
encoded_df.head()

# drop all recipes that have fewer than 5 ingredients
encoded_df = encoded_df[encoded_df['ingr_5'] < 4232]
encoded_df.reset_index(drop=True)
print(encoded_df.shape)
encoded_df.head()

derp_df = encoded_df.iloc[0:5]
derp_df

encoded_df = encoded_df[encoded_df['ingr_59'] == 4232]
print(encoded_df.shape)

def create_rec_sequences(recipe_row):

    # take length of row & create length 5 lists that can be added to sequences[]

    counter = 0

    r_list = []
    next_ingr = []

    for counter in range(0, len(recipe_row), 3):
        if recipe_row[counter + 5] == 4232:
            return r_list, next_ingr
        r_list.append(recipe_row[counter:counter+5])
        next_ingr.append(recipe_row[counter + 5])

#  create list of sequences for 5 item lists
#  i.e. sequences[0] = [352, 1, 10, 0, 15]
#       sequences[1] = [1, 10, 0, 15, 3]

# testing grounds!
sequences = []
next_ingr = []

for row in range(0, len(derp_df)):
    derpen = derp_df.iloc[row]

    seq_to_add, next_to_add = create_rec_sequences(derpen)

    for seq in seq_to_add:
        sequences.append(seq)

    for nxt in next_to_add:
        next_ingr.append(nxt)


print(sequences, next_ingr)

sequences = []
next_ingr = []

for row in range(0, len(encoded_df)):

    ingr_list = encoded_df.iloc[row]
    seq_to_add, next_to_add = create_rec_sequences(ingr_list)

    for seq in seq_to_add:
        sequences.append(seq)
    for nxt in next_to_add:
        next_ingr.append(nxt)

for i in range(0, 10):
    print(sequences[i], '\n', next_ingr[i])

print('\n sequences:', len(sequences), '\n', 'next_ingredients:', len(next_ingr))

x = np.zeros((len(sequences), 5, len(ingr_count_df)), dtype='uint8')
y = np.zeros((len(sequences), len(ingr_count_df)), dtype='uint8')

for i, sequence in enumerate(sequences):
    for t, ingr in enumerate(sequence):
        x[i,t,ingr] = 1

    y[i, next_ingr[i]] = 1

print(x.shape)
print(y.shape)

model = Sequential()
model.add(LSTM(128, input_shape=(5, len(ingr_count_df))))
model.add(Dense(len(ingr_count_df), activation='softmax'))

model.compile(loss='categorical_crossentropy', optimizer='nadam')

model.fit(x, y,
          batch_size=128,
          epochs=2)

ingr_names = [ingr_int.keys()]

def sample(preds, temperature=1.0):
    # helper function to sample an index from a probability array
    preds = np.asarray(preds).astype('float64')
    preds = np.log(preds) / temperature
    exp_preds = np.exp(preds)
    preds = exp_preds / np.sum(exp_preds)
    probas = np.random.multinomial(1, preds, 1)
    return np.argmax(probas)

five_ingr = ['peanut butter', 'flour', 'sugar', 'eggs', 'chocolate chips']

pred_next = []

for diversity in [0.2, 0.5, 1.0, 1.2]:

    start_ingr = [ingr_int[x] for x in five_ingr]

    for i in range(10):
        x_pred = np.zeros((1, 5, len(ingr_count_df)))
        for t, ingr in enumerate(start_ingr):
            x_pred[0, t, ingr] = 1

        preds = model.predict(x_pred, verbose=0)[0]
        next_index = sample(preds, diversity)
        next_ingredient = int_ingr[next_index]

        pred_next.append(next_ingredient)


print(set(pred_next))




<filename>Archived/labs_21/gz/ingr_scratchpad.py
# -*- coding: utf-8 -*-
"""ingr_scratchpad.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1LpU5z1NpyFJFJR5g3Vabd4MQ66LJJ0Tj

# Really tho'... how many ingredients could there possibly be???
"""


# imports.  you're prolly gonna use all these, right?
import pdb

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

from tensorflow.keras.preprocessing import sequence
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Embedding
from tensorflow.keras.layers import LSTM

from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.neighbors import NearestNeighbors

import re
import string

import spacy
nlp = spacy.load("en_core_web_sm")

# Pulling in recipe lists
df = pd.read_csv('../data/clean_recipes.csv', sep=';')
kgl1_df = pd.read_json('../data/kgl_ingredient_train.json')
kgl2_df = pd.read_json('../data/kgl_ingredient_test.json')

df = df.rename(columns={'Ingredients':'ingredients'})
df.head(3)

# df ingredients is a single string, splitting into a list of strings
df['ingredients'] = df['ingredients'].apply(lambda x: x.split(','))

l1 = df['ingredients']
l2 = kgl1_df['ingredients']
l3 = kgl2_df['ingredients']

ingr_series = l1.append([l2, l3], ignore_index=True)

ingr_series[:10]

def scrub_ingredients(ingr_list):

    chtble = str.maketrans('', '', string.punctuation + '0123456789')

    # drop special characters and multiple spaces, then convert to lowercase
    clean_list = []
    for ingr_string in ingr_list:
        ingr_string = re.sub(r'\W\s+', ' ', ingr_string.strip().translate(chtble))
        clean_list.append(ingr_string.lower())

    # convert to lowercase & drop known measurement words
    dropwords = ['ounce', 'ounces', 'oz', 'lb', ' pound', 'the', 'a', 'an'
                 'package', 'packages', 'inches', 'and', 'for', 'as', 'is']

    outlist = []

    for i in clean_list:
        words = i.split()
        words = [word for word in words if word not in dropwords]
        i = ' '.join(words)
        outlist.append(i)

    return outlist

ingr_s = ingr_series.apply(scrub_ingredients)
ingr_s[:10]

def ingredient_count(dfcolumn):

    # Create dictionary with word count
    d = dict()

    for l in dfcolumn:
        for i in l:
            if i in d:
                d[i] = d[i] + 1
            else:
                d[i] = 1

    return pd.DataFrame(d.items(), columns=['ingredient', 'count'])

# create df of unique ingredients and their number of occurances in recipe dfs
ingr_count_df = ingredient_count(ingr_s).sort_values('count', ascending=False).reset_index(drop=True)
ingr_count_df.head(15)

modeling_df = ingr_s.apply(pd.Series)
modeling_df = modeling_df.rename(columns = lambda x: 'ingr_' + str(x))
modeling_df.head(10)

(modeling_df.isnull().mean() * 100).head(25)

modeling_df = modeling_df.replace(np.nan, '', regex=True)
modeling_df.head()

ingr_int = {ingr:i for i, ingr in enumerate(ingr_count_df['ingredient'])}
int_ingr = {i:ingr for i, ingr in enumerate(ingr_count_df['ingredient'])}

# convert ingredients to numeric representations
encoded_df = modeling_df.applymap(ingr_int.get)
print(encoded_df.shape)
encoded_df.head()

# drop all recipes that have fewer than 5 ingredients
encoded_df = encoded_df[encoded_df['ingr_5'] < 4232]
encoded_df.reset_index(drop=True)
print(encoded_df.shape)
encoded_df.head()

derp_df = encoded_df.iloc[0:5]
derp_df

encoded_df = encoded_df[encoded_df['ingr_59'] == 4232]
print(encoded_df.shape)

def create_rec_sequences(recipe_row):

    # take length of row & create length 5 lists that can be added to sequences[]

    counter = 0

    r_list = []
    next_ingr = []

    for counter in range(0, len(recipe_row), 3):
        if recipe_row[counter + 5] == 4232:
            return r_list, next_ingr
        r_list.append(recipe_row[counter:counter+5])
        next_ingr.append(recipe_row[counter + 5])

#  create list of sequences for 5 item lists
#  i.e. sequences[0] = [352, 1, 10, 0, 15]
#       sequences[1] = [1, 10, 0, 15, 3]

# testing grounds!
sequences = []
next_ingr = []

for row in range(0, len(derp_df)):
    derpen = derp_df.iloc[row]

    seq_to_add, next_to_add = create_rec_sequences(derpen)

    for seq in seq_to_add:
        sequences.append(seq)

    for nxt in next_to_add:
        next_ingr.append(nxt)


print(sequences, next_ingr)

sequences = []
next_ingr = []

for row in range(0, len(encoded_df)):

    ingr_list = encoded_df.iloc[row]
    seq_to_add, next_to_add = create_rec_sequences(ingr_list)

    for seq in seq_to_add:
        sequences.append(seq)
    for nxt in next_to_add:
        next_ingr.append(nxt)

for i in range(0, 10):
    print(sequences[i], '\n', next_ingr[i])

print('\n sequences:', len(sequences), '\n', 'next_ingredients:', len(next_ingr))

x = np.zeros((len(sequences), 5, len(ingr_count_df)), dtype='uint8')
y = np.zeros((len(sequences), len(ingr_count_df)), dtype='uint8')

for i, sequence in enumerate(sequences):
    for t, ingr in enumerate(sequence):
        x[i,t,ingr] = 1

    y[i, next_ingr[i]] = 1

print(x.shape)
print(y.shape)

model = Sequential()
model.add(LSTM(128, input_shape=(5, len(ingr_count_df))))
model.add(Dense(len(ingr_count_df), activation='softmax'))

model.compile(loss='categorical_crossentropy', optimizer='nadam')

model.fit(x, y,
          batch_size=128,
          epochs=2)

ingr_names = [ingr_int.keys()]

def sample(preds, temperature=1.0):
    # helper function to sample an index from a probability array
    preds = np.asarray(preds).astype('float64')
    preds = np.log(preds) / temperature
    exp_preds = np.exp(preds)
    preds = exp_preds / np.sum(exp_preds)
    probas = np.random.multinomial(1, preds, 1)
    return np.argmax(probas)

five_ingr = ['peanut butter', 'flour', 'sugar', 'eggs', 'chocolate chips']

pred_next = []

for diversity in [0.2, 0.5, 1.0, 1.2]:

    start_ingr = [ingr_int[x] for x in five_ingr]

    for i in range(10):
        x_pred = np.zeros((1, 5, len(ingr_count_df)))
        for t, ingr in enumerate(start_ingr):
            x_pred[0, t, ingr] = 1

        preds = model.predict(x_pred, verbose=0)[0]
        next_index = sample(preds, diversity)
        next_ingredient = int_ingr[next_index]

        pred_next.append(next_ingredient)


print(set(pred_next))




# -*- coding: utf-8 -*- ingr_scratchpad.ipynb  Automatically generated by Colaboratory.  Original file is located at     https://colab.research.google.com/drive/1LpU5z1NpyFJFJR5g3Vabd4MQ66LJJ0Tj  # Really tho'... how many ingredients could there possibly be??? # imports.  you're prolly gonna use all these, right? # Pulling in recipe lists # df ingredients is a single string, splitting into a list of strings # drop special characters and multiple spaces, then convert to lowercase # convert to lowercase & drop known measurement words # Create dictionary with word count # create df of unique ingredients and their number of occurances in recipe dfs # convert ingredients to numeric representations # drop all recipes that have fewer than 5 ingredients # take length of row & create length 5 lists that can be added to sequences[] #  create list of sequences for 5 item lists #  i.e. sequences[0] = [352, 1, 10, 0, 15] #       sequences[1] = [1, 10, 0, 15, 3] # testing grounds! # helper function to sample an index from a probability array