<reponame>BartQuaink/Programmeerproject<filename>Code/data/total3pmscraper.py
# scrapes the total 3 pointers made per season for the top 3 point shooters, needed for the line graphs

import requests
from lxml import html
import json
import re
from pattern.web import URL, DOM, plaintext
from collections import defaultdict

playerlist = dict()

players = ["3975/stephen-curry", "9/ray-allen", "552/reggie-miller", "841/jason-terry", "662/paul-pierce", "429/jason-kidd", "136/vince-carter", "165/jamal-crawford", "63/chauncey-billups", "2011/kyle-korver", "469/rashard-lewis", "813/peja-stojakovic", "1007/joe-johnson", "110/kobe-bryant"]

htmllink = "http://espn.go.com/nba/player/stats/_/id/"

output_file = open('new3pointers.json', 'w')

# get data for all players
for player in players:
    TARGET_URL = URL(htmllink + player)
    dom = DOM(TARGET_URL.download(cached=True))
    dataofyear = list()
    tempdata= dict()

    #  loop over the html table
    for e in dom.by_tag("div.mod-container mod-table mod-player-stats"):
        for a in e.by_tag("div.mod-content")[1:2]:
            for tablehead in a.by_class("tablehead"):
                year = -1
                for oddrow in tablehead.by_class("oddrow"):
                    madeshots = oddrow[4].content[:3]
                    madeshots = int(madeshots.replace("-", ""))

                    year += 2

                    percentage = float(oddrow[5].content)

                    tempdata["year"] = year
                    tempdata["tot3fg"] = madeshots
                    tempdata["percentage"] = percentage

                    dataofyear.append(tempdata)
                    tempdata = {}

                year = 0
                for evenrow in tablehead.by_class("evenrow"):
                    madeshots = evenrow[4].content[:3]
                    madeshots = int(madeshots.replace("-", ""))

                    year += 2

                    percentage = float(evenrow[5].content)

                    tempdata["year"] = year
                    tempdata["tot3fg"] = madeshots
                    tempdata["percentage"] = percentage

                    dataofyear.append(tempdata)
                    tempdata = {}

    # re.sub removes every non alphabetic character
    playerlist[re.sub("[^a-zA-Z]+", "", player)] = dataofyear
    dataofyear = []


json.dump(playerlist, output_file, indent = 4)
output_file.write('\n')

# scrapes the total 3 pointers made per season for the top 3 point shooters, needed for the line graphs

import requests
from lxml import html
import json
import re
from pattern.web import URL, DOM, plaintext
from collections import defaultdict

playerlist = dict()

players = ["3975/stephen-curry", "9/ray-allen", "552/reggie-miller", "841/jason-terry", "662/paul-pierce", "429/jason-kidd", "136/vince-carter", "165/jamal-crawford", "63/chauncey-billups", "2011/kyle-korver", "469/rashard-lewis", "813/peja-stojakovic", "1007/joe-johnson", "110/kobe-bryant"]

htmllink = "http://espn.go.com/nba/player/stats/_/id/"

output_file = open('new3pointers.json', 'w')

# get data for all players
for player in players:
    TARGET_URL = URL(htmllink + player)
    dom = DOM(TARGET_URL.download(cached=True))
    dataofyear = list()
    tempdata= dict()

    #  loop over the html table
    for e in dom.by_tag("div.mod-container mod-table mod-player-stats"):
        for a in e.by_tag("div.mod-content")[1:2]:
            for tablehead in a.by_class("tablehead"):
                year = -1
                for oddrow in tablehead.by_class("oddrow"):
                    madeshots = oddrow[4].content[:3]
                    madeshots = int(madeshots.replace("-", ""))

                    year += 2

                    percentage = float(oddrow[5].content)

                    tempdata["year"] = year
                    tempdata["tot3fg"] = madeshots
                    tempdata["percentage"] = percentage

                    dataofyear.append(tempdata)
                    tempdata = {}

                year = 0
                for evenrow in tablehead.by_class("evenrow"):
                    madeshots = evenrow[4].content[:3]
                    madeshots = int(madeshots.replace("-", ""))

                    year += 2

                    percentage = float(evenrow[5].content)

                    tempdata["year"] = year
                    tempdata["tot3fg"] = madeshots
                    tempdata["percentage"] = percentage

                    dataofyear.append(tempdata)
                    tempdata = {}

    # re.sub removes every non alphabetic character
    playerlist[re.sub("[^a-zA-Z]+", "", player)] = dataofyear
    dataofyear = []


json.dump(playerlist, output_file, indent = 4)
output_file.write('\n')
# scrapes the total 3 pointers made per season for the top 3 point shooters, needed for the line graphs # get data for all players #  loop over the html table # re.sub removes every non alphabetic character