# %%
"""
pip install konlpy
pip install customized_konlpy
"""

import numpy as np
import pandas as pd
import io
import re
import gensim
from konlpy.tag import Hannanum, Kkma, Komoran, Mecab, Okt
from gensim.models import Word2Vec
from ckonlpy.tag import Twitter
import os.path

FILE_PATHS = ['lol_stories.csv', 'mbti_output.csv']
ADDITIONAL_WORDS_PATHS = ['lol_additional_words.csv', 'mbti_additional_words.csv']
OUTPUT_MODEL_PATH = './../models/lol_model.model'
TEXT_INDEXS = ['text', 'text']
EXCLUDE_WORDS = ['앤서니', '레이놀즈',
                '스크롤', '그', '살펴보기', '스크롤하기', 
                '그녀', '루오', '마이클', '굴딩',
                '데이나', '루어리', '그레이엄', '맥닐',
                '제러드', '로슨', '로완', '윌리엄스',
                '오브라이언', '마커스', '테럴', '스미스'
                '이안', '마틴', '마이클', '하우겐', '위스크',
                '매튜', '필립', '바르가스', '로라', 
                '미셋', '대니얼', '코츠', '아만다', '제프리',
                '아비게일', '하비', '로완', '노엘', '윌리엄스',
                '케이티', '치로니스', '이차오',
                '데이비드', '슬래글', '오딘', '오스틴', '샤퍼']
WORD2VEC_WINDOW = 4
WORD2VEC_MIN_COUNT = 2

tokenized_text = pd.DataFrame([])
count = 0
for path in FILE_PATHS:
  df = pd.read_csv(path, encoding='utf-8')
  tokenized_text = pd.concat([tokenized_text, df[TEXT_INDEXS[count]]], axis=0, ignore_index=True)
  count += 1
tokenized_text = tokenized_text[0]

# 토큰화 함수
hannanum = Hannanum() # 대상 text 길이 너무 길면 오류나는듯(대략 한글 12800자 정도)
kkma = Kkma()
komoran = Komoran()
twitter = Twitter()

# 커스텀 단어 추가
for path in ADDITIONAL_WORDS_PATHS:
  additional_words = pd.read_csv(path, encoding='cp949', header=None)
  for index, row in additional_words.iterrows():
    twitter.add_dictionary(row[0], row[1])

def tokenizer(row):
  pos_list = twitter.pos(row)
  r = []
  for word in pos_list:
    if not word[1] in ["Josa", "Punctuation", "Foreign",
    "Suffix", "Eomi", "Determiner", "PreEomi", "Adverb",
    "Conjunction", "Exclamation"]:
      w = word[0]
      if w not in EXCLUDE_WORDS:
        r.append(w)
  rl = (" ".join(r)).strip()
  return twitter.morphs(rl, stem=True)

tokenized_text = tokenized_text.apply(tokenizer)
print(tokenized_text.head())

print("최대 토큰 개수 :", (max(len(i) for i in tokenized_text)))
print("최소 토큰 개수 :", (min(len(i) for i in tokenized_text)))
print("평균 토큰 개수 :", sum(len(i) for i in tokenized_text)/len(tokenized_text))

new_model = Word2Vec(size = 200, window = WORD2VEC_WINDOW, min_count = WORD2VEC_MIN_COUNT, workers = 4, sg = 1)
new_model.build_vocab(tokenized_text)
total_examples = len(tokenized_text)
new_model.train(tokenized_text, total_examples=total_examples, epochs=new_model.iter)
print(new_model.wv.vocab.keys())
print("new vocab count:", len(new_model.wv.vocab.keys()))
model_result = new_model.wv.most_similar('빌지워터')
print(model_result)
new_model.save(OUTPUT_MODEL_PATH)

# %%
"""
pip install konlpy
pip install customized_konlpy
"""

import numpy as np
import pandas as pd
import io
import re
import gensim
from konlpy.tag import Hannanum, Kkma, Komoran, Mecab, Okt
from gensim.models import Word2Vec
from ckonlpy.tag import Twitter
import os.path

FILE_PATHS = ['lol_stories.csv', 'mbti_output.csv']
ADDITIONAL_WORDS_PATHS = ['lol_additional_words.csv', 'mbti_additional_words.csv']
OUTPUT_MODEL_PATH = './../models/lol_model.model'
TEXT_INDEXS = ['text', 'text']
EXCLUDE_WORDS = ['앤서니', '레이놀즈',
                '스크롤', '그', '살펴보기', '스크롤하기', 
                '그녀', '루오', '마이클', '굴딩',
                '데이나', '루어리', '그레이엄', '맥닐',
                '제러드', '로슨', '로완', '윌리엄스',
                '오브라이언', '마커스', '테럴', '스미스'
                '이안', '마틴', '마이클', '하우겐', '위스크',
                '매튜', '필립', '바르가스', '로라', 
                '미셋', '대니얼', '코츠', '아만다', '제프리',
                '아비게일', '하비', '로완', '노엘', '윌리엄스',
                '케이티', '치로니스', '이차오',
                '데이비드', '슬래글', '오딘', '오스틴', '샤퍼']
WORD2VEC_WINDOW = 4
WORD2VEC_MIN_COUNT = 2

tokenized_text = pd.DataFrame([])
count = 0
for path in FILE_PATHS:
  df = pd.read_csv(path, encoding='utf-8')
  tokenized_text = pd.concat([tokenized_text, df[TEXT_INDEXS[count]]], axis=0, ignore_index=True)
  count += 1
tokenized_text = tokenized_text[0]

# 토큰화 함수
hannanum = Hannanum() # 대상 text 길이 너무 길면 오류나는듯(대략 한글 12800자 정도)
kkma = Kkma()
komoran = Komoran()
twitter = Twitter()

# 커스텀 단어 추가
for path in ADDITIONAL_WORDS_PATHS:
  additional_words = pd.read_csv(path, encoding='cp949', header=None)
  for index, row in additional_words.iterrows():
    twitter.add_dictionary(row[0], row[1])

def tokenizer(row):
  pos_list = twitter.pos(row)
  r = []
  for word in pos_list:
    if not word[1] in ["Josa", "Punctuation", "Foreign",
    "Suffix", "Eomi", "Determiner", "PreEomi", "Adverb",
    "Conjunction", "Exclamation"]:
      w = word[0]
      if w not in EXCLUDE_WORDS:
        r.append(w)
  rl = (" ".join(r)).strip()
  return twitter.morphs(rl, stem=True)

tokenized_text = tokenized_text.apply(tokenizer)
print(tokenized_text.head())

print("최대 토큰 개수 :", (max(len(i) for i in tokenized_text)))
print("최소 토큰 개수 :", (min(len(i) for i in tokenized_text)))
print("평균 토큰 개수 :", sum(len(i) for i in tokenized_text)/len(tokenized_text))

new_model = Word2Vec(size = 200, window = WORD2VEC_WINDOW, min_count = WORD2VEC_MIN_COUNT, workers = 4, sg = 1)
new_model.build_vocab(tokenized_text)
total_examples = len(tokenized_text)
new_model.train(tokenized_text, total_examples=total_examples, epochs=new_model.iter)
print(new_model.wv.vocab.keys())
print("new vocab count:", len(new_model.wv.vocab.keys()))
model_result = new_model.wv.most_similar('빌지워터')
print(model_result)
new_model.save(OUTPUT_MODEL_PATH)

# %% pip install konlpy pip install customized_konlpy # 토큰화 함수 # 대상 text 길이 너무 길면 오류나는듯(대략 한글 12800자 정도) # 커스텀 단어 추가