# -*- coding: utf-8 -*-
"""Time Series Bitcoin Historical Data.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1u7_GLjW25mLW3eZXcOXi3GbXUJIoizWb

# Time Series Bitcoin Historical Data

Bitcoin - USD data with 1-day intervals from Yahoo Finance (https://finance.yahoo.com/quote/BTC-USD/).

**Training Accuracy (%)**

- loss: 0.41
- mae: 4.48

**Validation Accuracy (%)**

- loss: 0.41
- mae: 4.75

The training process takes ***7m 25s*** using ***500 epochs***
"""

# install library needed
!pip install yfinance

# import library needed
from tensorflow import keras
from tensorflow import data as td
from tensorflow import expand_dims
from keras.optimizers import Adam, SGD
from keras.models import Sequential, Model, load_model
from keras.layers import Input, LSTM, CuDNNLSTM, Bidirectional, SpatialDropout1D, Conv1D, Dropout, Dense, Activation, Lambda
from keras.callbacks import ModelCheckpoint, EarlyStopping, CSVLogger
from keras.losses import Huber
from keras.utils.vis_utils import plot_model

import yfinance as yf
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split
from google.colab import drive
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import seaborn as sn
import numpy as np
import pandas as pd

import os
import io
import uuid
import time
import datetime
import requests
import warnings
from IPython.display import Markdown
from IPython.display import display
from slugify import slugify

# ignore the unwanted warning messages
warnings.filterwarnings('ignore')

# Define base path for worksheet
BASE_PATH = '/content/drive/MyDrive/Playground/BitcoinPrice'
# No. of number of samples that will be propagated through the network
BATCH_SIZE=100
# Window Size
WINDOW_SIZE=50
# Learning rate
LEARNING_RATE=1e-8
# Momentum
MOMENTUM=0.9
# Number of iterations
EPOCH=500
# Set the train dropout
DROPOUT=0.2

# mount our google drive
drive.mount('/content/drive')

"""## Preprocessing

### Data Gatering
"""

def yf_downloader(ticker, period="max", interval="1d"):
  '''
  ticker: ticker code
  period: 1d,5d,1mo,3mo,6mo,1y,2y,5y,10y,ytd,max
  interval: 1m,2m,5m,15m,30m,60m,90m,1h,1d,5d,1wk,1mo,3mo
  '''
  # download dataframe
  try:
    data = pdr.get_data_yahoo("BTC-USD", period=period, interval=interval)
  except Exception as e:
    print("Downloading data from yahoo finance error: {}".format(e))

  # dataset save path and filename
  dataset_path = os.path.join(BASE_PATH, ticker + '-' + period + '-' + interval + '.csv')
  # save data into csv for future usage
  data.to_csv(dataset_path)

  print("Successfully download dataset and saved at: {}".format(dataset_path))

# download the dataset
yf_downloader(ticker="BTC-USD")

"""### Load Dataset"""

# load dataset
dataset = os.path.join(BASE_PATH, 'BTC-USD-max-1d.csv')
df = pd.read_csv(dataset, sep=',')
print('Dataset loaded ...')

"""### Data Exploration"""

# sort data frame by date
df = df.sort_values('Date')
# check the result
df.head()

# get total data
df.count()

# simple metric study
df['Close'].describe()

# check null value
df.isnull().sum()

"""For price I will only use the High and Low price. So, unused data will be droped."""

df['Price'] = (df['Low']+df['High'])/2.0
df.drop(['Open','High','Low','Close','Adj Close','Volume'],axis=1, inplace=True)

df['Date'] = pd.to_datetime(df['Date'], format='%Y-%m-%d',errors='coerce')
df = df.set_index('Date')
df = df.dropna()

df.head()

"""### Data Visualization"""

plt.figure(figsize=(20,8))
plt.plot(df)
plt.title('Bitcoin price',fontsize=20)
plt.xlabel('year',fontsize=15)
plt.ylabel('price',fontsize=15)
plt.show()

"""### Normalization

By using MinMaxScaler, The price range is (0 ~ 1) now.
"""

scaler = MinMaxScaler()
price = scaler.fit_transform(np.array(df['Price']).reshape(-1,1))
df['Price'] = price

"""### Sliding Window and Data Spliting"""

# sliding window
# predict 1 data by using 50 datas.
X_l = []
y_l = []
N = len(df)
D = 50
for i in range(N-D-1):
    X_l.append(df.iloc[i:i+D])
    y_l.append(df.iloc[i+D])
    
X = np.array(X_l)
y = np.array(y_l)

print(X.shape, y.shape)

X_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.2, random_state=100)

"""## Modeling"""

def ModelLSTM():
  model = Sequential()
  model.add(Conv1D(filters=128, kernel_size=5, strides=1, padding="causal", activation="relu", input_shape=(50, 1)))
  model.add(Dropout(rate=DROPOUT))
  model.add(LSTM(64, return_sequences=True))
  model.add(Dropout(rate=DROPOUT))
  model.add(LSTM(64, return_sequences=True))
  model.add(Dense(units=32, activation="relu"))
  model.add(Dense(units=16, activation="relu"))
  model.add(Dense(units=1))
  model.add(Lambda(lambda x: x * 400))

  optimizer = SGD(lr=LEARNING_RATE, momentum=MOMENTUM)
  losses = Huber()
  model.compile(optimizer=optimizer,
                loss=losses,
                metrics=['mae'])

  return model

# Visualize LSTM model
print(ModelLSTM().summary())

"""## Training"""

# Print in markdown format
def printmd(string):
    display(Markdown(string))

# Plot training loss and mae
def plot_training(history, title, save=True, save_path=os.path.join(BASE_PATH, 'visualization')):
    printmd("### Training Loss and MAE")
    
    acc = history.history['mae']
    val_acc = history.history['val_mae']
    loss = history.history['loss']
    val_loss = history.history['val_loss']
    epoch_list = range(1, len(acc) + 1)
    
    plt.plot(loss, label='Training Loss')
    plt.plot(val_loss, label='Validation Loss')
    plt.legend(loc=0, fontsize='large')
    plt.xlabel("Epoch #")
    plt.ylabel("Loss")
    plt.show()
    
    if save is True:
        filename = slugify(title) + '_loss_model.png'
        plt.savefig(save_path + '/' + filename)

    plt.plot(acc, label='Training MAE')
    plt.plot(val_acc, label='Validation MAE')
    plt.legend(loc=4, fontsize='large')
    plt.xlabel("Epoch #")
    plt.ylabel("MAE")
    plt.show()
    
    if save is True:
        filename = slugify(title) + '_acc_model.png'
        plt.savefig(save_path + '/' + filename)

# Evaluate model, confusion matrix and classification report
def model_evaluation(title, model, x_train, x_test, y_train, y_test, save=True, save_path=os.path.join(BASE_PATH, 'visualization')):
    printmd('### Training Accuracy')
    loss, mae = model.evaluate(x_train, y_train, verbose=1)

    print("loss: %.2f" % (loss*100))
    print("mae: %.2f" % (mae*100))
    
    printmd('### Validation Accuracy')
    loss, mae = model.evaluate(x_test, y_test, verbose=1)

    print("loss: %.2f" % (loss*100))
    print("mae: %.2f" % (mae*100))

# Adding callbacks
train_id = 'LSTM-' + str(uuid.uuid1())
best_model = os.path.join(BASE_PATH, 'model/best_model-' + train_id + '.h5')
last_model = os.path.join(BASE_PATH, 'model/last_model-' + train_id + '.h5')
csv_log = os.path.join(BASE_PATH, 'monitor/logs-' + train_id + '.h5')

ES = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=10)
MC = ModelCheckpoint(best_model, monitor='val_mae', mode='min', save_best_only=True, verbose=1)
CSVL = CSVLogger(csv_log, separator=",", append=True)

callbacks = [MC, ES, CSVL]

# Training start time
start_time = time.time()

print("Training strated with Train ID: {}".format(train_id))

model = ModelLSTM()
history = model.fit(X_train, y_train,
                  validation_data=(X_test, y_test),
                  epochs=EPOCH,
                  verbose=1,
                  callbacks=callbacks)

# Training complete
time_elapsed = time.time() - start_time
# Save last model
model.save(last_model)

print("=" * 30)
print("Training Finished, took {:.0f}m {:.0f}s".format(time_elapsed // 60, time_elapsed % 60))
print("Best model:", best_model)
print("Last model:", last_model)

# Plot training and loss accuracy
plot_training(history, train_id)

# Load best model and evaluate
model = load_model(best_model)
model_evaluation(train_id, model, X_train, X_test, y_train, y_test)