<reponame>felixJets/CoinMarketCal
import selenium
from selenium import webdriver
import time
import re
import pickle
import datetime
import os
import argparse
from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC


### Get chromedriver on  the fly via webdriver_manager
### Or download the respective chromedriver from https://sites.google.com/a/chromium.org/chromedriver/downloads

#driver = webdriver.Chrome("path_to_chromedriver")
driver = webdriver.Chrome(ChromeDriverManager().install())


### Get website
driver.get("https://coinmarketcal.com/en/")

### Collect Data for every listeted coin and use the coin-name as key
COIN_DATA = dict()

### Starting Page Number
page_number = 1

### RE-pattern for extracting floats and ints
float_pattern = re.compile(r"[+-]?\d*\.\d*")
int_pattern = re.compile(r"\d+")

### Go over all events on one page
next_page = True

### DEBUG
### Click the ">>"-Button -> Test-Purpose
# element = driver.find_element_by_link_text(">>")
# element.click()

start_time = time.time()

### While there is a next page -> Go to the next page
while(next_page):

    ### CoinMarketCal-Structure
    ### o o o o
    ### Searchbar
    ### X X X X
    ### X X X X
    ### X X X X
    ### X X X X

    ### o == Trending Coins at the top of the page
    ### X == Coin with respective event

    ### Get coin-html elements
    ### via css-selector
    cards = driver.find_element_by_css_selector("div.row.list-card")
    ### via xpath
    articles = cards.find_elements_by_xpath("//article[contains(@class, 'col')]")

    ### Normally 16 coins per page
    ### Can be less on the last page
    number_coins_on_page = len(articles)

    ### Select listed coins
    for i in range(number_coins_on_page):

        ### cards has to be extracted every time after a coin-path was visited otherwise is is not working.
        ### class-name: "row list-card" --> write as "row.list-card"
        ### via css
        cards = driver.find_element_by_css_selector("div.row.list-card")
        ### via xpath
        # cards = driver.find_elements_by_xpath("//div[contains(@class, 'row list-card')]")[0]

        ### Get <article> ... </article> --> contains information about the coin_name/coin_event
        articles = cards.find_elements_by_xpath("//article[contains(@class, 'col')]")


        ### Get coin_name, coin_date, coin_event
        coin_name_element = articles[i].find_element_by_xpath(".//a[contains(@href, '/en/coin')]")
        coin_date_element = articles[i].find_element_by_xpath(".//h5[contains(@class, 'card__date')]")
        coin_event_element = articles[i].find_element_by_xpath(".//h5[contains(@class, 'card__title')]")

        ### Get the html-text element
        coin_name = coin_name_element.text
        coin_date = coin_date_element.text
        coin_event = coin_event_element.text


        ### check if coin was already added to the COIN_DATA dict
        ### if so: Only add the new event...
        if coin_name in COIN_DATA:
            COIN_DATA[coin_name]['Events'].append((coin_date, coin_event))

        ### ... otherwise create new dict for the coin and jump to https://coinmarketcal.com/en/coin/coin_name to collect more information
        else:
            COIN_DATA[coin_name] = dict()
            COIN_DATA[coin_name]["Events"] = [(coin_date, coin_event)]

            ### Jump to: https://coinmarketcal.com/en/coin/coin_name --> Get further information (price, market-cap)

            ### TODO: fix coin_name_element.click()
            ###WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, coin_name_element))).click()
            driver.execute_script("arguments[0].click();", coin_name_element)
            ###coin_name_element.click()


            ### Get information about the coin's supply
            ### Watch out if the data is N/A

            ### Max supply
            try:
                data_supply_max = driver.find_element_by_xpath("//div[contains(text(), '(Max)')]")
                maxSupply = data_supply_max.text.replace(' ', '')
                maxSupply = int(re.findall(int_pattern, maxSupply)[0])
                maxSupply = f"{maxSupply:,}"
            except:
                maxSupply = "N/A"

            ### Circulating Supply
            try:
                data_supply_available = driver.find_element_by_xpath("//div[contains(text(), '(Available)')]")
                circulatingSupply = data_supply_available.text.replace(' ', '')
                circulatingSupply = int(re.findall(int_pattern, circulatingSupply)[0])
                circulatingSupply = f"{circulatingSupply:,}"
            except:
                circulatingSupply = "N/A"

            ### Get price and price-change
            try:
                price_and_change = driver.find_elements_by_xpath("//*[@id = 'coin-detail']/div[2]/div[1]/div[2]/div[2]")
                price = price_and_change[0].text
                price_and_change = price.split()
                price = ''.join(price_and_change[:-2])
                percent_change = price_and_change[-2]
                price = float(re.findall(float_pattern, price)[0])
                percent_change = float(re.findall(float_pattern, percent_change)[0])
                price = f"{price:,}"
                percent_change = f"{percent_change:,}"
            except:
                price = "N/A"
                percent_change = "N/A"

            ### Market Cap
            try:
                market_cap = driver.find_elements_by_xpath("//*[@id = 'coin-detail']/div[2]/div[1]/div[3]/div[2]")
                market_cap = market_cap[0].text.replace(' ', '')
                market_cap = int(re.findall(int_pattern, market_cap)[0])
                market_cap= f"{market_cap:,}"
            except:
                market_cap = "N/A"

            ### Trading volume
            try:
                trading_colume_24h = driver.find_elements_by_xpath("//*[@id = 'coin-detail']/div[2]/div[1]/div[4]/div[2]")
                trading_colume_24h = trading_colume_24h[0].text.replace(' ', '')
                trading_colume_24h = int(re.findall(int_pattern, trading_colume_24h)[0])
                trading_colume_24h = f"{trading_colume_24h:,}"
            except:
                trading_colume_24h = "N/A"

            ### Get img src reference
            ### css-selector: tag_name.class_name
            img_src = driver.find_element_by_css_selector("img.rounded").get_attribute("src")
            COIN_DATA[coin_name]["src"] = img_src


            ####################################################################################

            ### Add all values to the Coin-dict
            COIN_DATA[coin_name]['Price'] = price
            COIN_DATA[coin_name]['Change'] = percent_change
            COIN_DATA[coin_name]['MarketCap'] = market_cap
            COIN_DATA[coin_name]['TradingVolume'] = trading_colume_24h
            COIN_DATA[coin_name]['MaxSupply'] = maxSupply
            COIN_DATA[coin_name]['CirculatingSupply'] = circulatingSupply

            """
            COIN_DATA['Upcoming'] = upcoming
            COIN_DATA['Trending'] = trending
            COIN_DATA['Significant'] = significant
            COIN_DATA['Hot'] = hot
            """

            ### Go back to the Overview article page
            driver.execute_script("window.history.go(-1)")
            ### TODO: driver.back() sometime not correctly working, why?
            #driver.back()


    page_number = page_number + 1
    print("Page: ", page_number)


    ### try clicking the next site
    try:
        next_site = driver.find_element_by_link_text(str(page_number))
        next_site.click()
        #time.sleep(1)

    except:
        print("Last Page: ", page_number-1)
        break


### Finished web scraping
### Quit driver

end_time = time.time()

print(f"Web-scraping took {end_time-start_time} seconds")

time.sleep(3)
driver.quit()


### Dump data into pkl-file
filename = "coin-data-" + str(datetime.date.today()) + ".pkl"

with open("data" + os.sep + filename, "wb") as file_coin:
     pickle.dump(COIN_DATA, file_coin)



print("FINISHED")






