from builtins import map
from past.builtins import basestring
import copy
import re

import numpy as np
import pandas as pd

import sportsref

RUSH_OPTS = {
    'left end': 'LE', 'left tackle': 'LT', 'left guard': 'LG',
    'up the middle': 'M', 'middle': 'M',
    'right end': 'RE', 'right tackle': 'RT', 'right guard': 'RG',
}
PASS_OPTS = {
    'short left': 'SL', 'short middle': 'SM', 'short right': 'SR',
    'deep left': 'DL', 'deep middle': 'DM', 'deep right': 'DR',
}

def expand_details(df, detailCol='detail'):
    """Expands the details column of the given dataframe and returns the
    resulting DataFrame.

    :df: The input DataFrame.
    :detailCol: The detail column name.
    :returns: Returns DataFrame with new columns from pbp parsing.
    """
    df = copy.deepcopy(df)
    df['detail'] = df[detailCol]
    dicts = [sportsref.nfl.pbp.parse_play_details(detail) for detail in df['detail'].values]
    # clean up unmatched details
    cols = {c for d in dicts if d for c in d.keys()}
    blankEntry = {c: np.nan for c in cols}
    newDicts = [d if d else blankEntry for d in dicts]
    # get details DataFrame and merge it with original to create main DataFrame
    details = pd.DataFrame(newDicts)
    df = pd.merge(df, details, left_index=True, right_index=True)
    # add isError column
    errors = [i for i, d in enumerate(dicts) if d is None]
    df['isError'] = False
    df.loc[errors, 'isError'] = True
    # fill in some NaN's necessary for _clean_features
    df.loc[0, 'qtr_time_remain'] = '15:00'
    df.qtr_time_remain.fillna(method='bfill', inplace=True)
    df.qtr_time_remain.fillna(
        pd.Series(np.where(df.quarter == 4, '0:00', '15:00')), inplace=True
    )
    # use _clean_features to clean up and add columns
    new_df = df.apply(_clean_features, axis=1)
    return new_df

@sportsref.decorators.memoize
def parse_play_details(details):
    """Parses play details from play-by-play string and returns structured
    data.

    :details: detail string for play
    :returns: dictionary of play attributes
    """

    # if input isn't a string, return None
    if not isinstance(details, basestring):
        return None

    rushOptRE = r'(?P<rushDir>{})'.format(
        r'|'.join(RUSH_OPTS.keys())
    )
    passOptRE = r'(?P<passLoc>{})'.format(
        r'|'.join(PASS_OPTS.keys())
    )

    playerRE = r"\S{6,8}\d{2}"

    # initialize return dictionary - struct
    struct = {}

    # handle challenges
    # TODO: record the play both before & after an overturned challenge
    challengeRE = re.compile(
        r'.+\. (?P<challenger>.+?) challenged.*? the play was '
        '(?P<callUpheld>upheld|overturned)\.',
        re.IGNORECASE
    )
    match = challengeRE.search(details)
    if match:
        struct['isChallenge'] = True
        struct.update(match.groupdict())
        # if overturned, only record updated play
        if 'overturned' in details:
            overturnedIdx = details.index('overturned.')
            newStart = overturnedIdx + len('overturned.')
            details = details[newStart:].strip()
    else:
        struct['isChallenge'] = False

    # TODO: expand on laterals
    struct['isLateral'] = details.find('lateral') != -1

    # create rushing regex
    rusherRE = r"(?P<rusher>{0})".format(playerRE)
    rushOptRE = r"(?: {})?".format(rushOptRE)
    rushYardsRE = r"(?:(?:(?P<rushYds>\-?\d+) yards?)|(?:no gain))"
    # cases: tackle, fumble, td, penalty
    tackleRE = (r"(?: \(tackle by (?P<tackler1>{0})"
                r"(?: and (?P<tackler2>{0}))?\))?"
                .format(playerRE))
    # currently, plays with multiple fumbles record the original fumbler
    # and the final fumble recoverer
    fumbleRE = (
        r"(?:"
        r"\.? ?(?P<fumbler>{0}) fumbles"
        r"(?: \(forced by (?P<fumbForcer>{0})\))?"
        r"(?:.*, recovered by (?P<fumbRecoverer>{0}) at )?"
        r"(?:, ball out of bounds at )?"
        r"(?:(?P<fumbRecFieldSide>[a-z]+)?\-?(?P<fumbRecYdLine>\-?\d+))?"
        r"(?: and returned for (?P<fumbRetYds>\-?\d*) yards)?"
        r")?"
        .format(playerRE))
    tdSafetyRE = r"(?:(?P<isTD>, touchdown)|(?P<isSafety>, safety))?"
    # TODO: offsetting penalties
    penaltyRE = (r"(?:.*?"
                 r"\. Penalty on (?P<penOn>{0}|): "
                 r"(?P<penalty>[^\(,]+)"
                 r"(?: \((?P<penDeclined>Declined)\)|"
                 r", (?P<penYds>\d*) yards?)"
                 r"(?: \(no play\))?"
                 r")?"
                 .format(playerRE))

    rushREstr = (
        r"{}{}(?: for {}{}{}{}{})?"
    ).format(rusherRE, rushOptRE, rushYardsRE, tackleRE, fumbleRE, tdSafetyRE,
             penaltyRE)
    rushRE = re.compile(rushREstr, re.IGNORECASE)

    # create passing regex
    # TODO: capture "defended by X" for defensive stats
    passerRE = r"(?P<passer>{0})".format(playerRE)
    sackRE = (r"(?:sacked (?:by (?P<sacker1>{0})(?: and (?P<sacker2>{0}))? )?"
              r"for (?P<sackYds>\-?\d+) yards?)"
              .format(playerRE))
    # create throw RE
    completeRE = r"pass (?P<isComplete>(?:in)?complete)"
    passOptRE = r"(?: {})?".format(passOptRE)
    targetedRE = r"(?: (?:to |intended for )?(?P<target>{0}))?".format(
        playerRE)
    passYardsRE = r"(?: for (?:(?P<passYds>\-?\d+) yards?|no gain))"
    intRE = (r'(?: is intercepted by (?P<interceptor>{0}) at '.format(playerRE)
             + r'(?:(?P<intFieldSide>[a-z]*)?\-?(?P<intYdLine>\-?\d*))?'
             + r'(?: and returned for (?P<intRetYds>\-?\d+) yards?\.?)?)?')
    throwRE = r'(?:{}{}{}(?:(?:{}|{}){})?)'.format(
        completeRE, passOptRE, targetedRE, passYardsRE, intRE, tackleRE
    )
    passREstr = (
        r"{} (?:{}|{})(?:{}{}{})?"
    ).format(passerRE, sackRE, throwRE, fumbleRE, tdSafetyRE, penaltyRE)
    passRE = re.compile(passREstr, re.IGNORECASE)

    # create kickoff regex
    koKickerRE = r'(?P<koKicker>{0})'.format(playerRE)
    koYardsRE = (r' kicks (?:off|(?P<isOnside>onside))'
                 r' (?:(?P<koYds>\d+) yards?|no gain)')
    nextREs = []
    nextREs.append(
        (r', (?:returned|recovered) by (?P<koReturner>{0})(?: for '
         r'(?:(?P<koRetYds>\-?\d+) yards?|no gain))?').format(playerRE)
    )
    nextREs.append(
        (r'(?P<isMuffedCatch>, muffed catch by )(?P<muffedBy>{0}),'
         r'(?: recovered by (?P<muffRecoverer>{0}))?').format(playerRE) +
        r'(?: and returned for (?:(?P<muffRetYds>\-?\d+) yards|no gain))?'
    )
    nextREs.append(
        r', recovered by (?P<onsideRecoverer>{0})'.format(playerRE)
    )
    nextREs.append(r'(?P<oob>, out of bounds)')
    nextREs.append(r'(?P<isTouchback>, touchback)')
    # TODO: test the following line to fix a small subset of cases
    # (ex: muff -> oob)
    nextRE = ''.join(r'(?:{})?'.format(nre) for nre in nextREs)
    kickoffREstr = r'{}{}{}{}{}{}{}'.format(
        koKickerRE, koYardsRE, nextRE,
        tackleRE, fumbleRE, tdSafetyRE, penaltyRE
    )
    kickoffRE = re.compile(kickoffREstr, re.IGNORECASE)

    # create timeout regex
    timeoutREstr = r'Timeout #(?P<timeoutNum>\d) by (?P<timeoutTeam>.+)'
    timeoutRE = re.compile(timeoutREstr, re.IGNORECASE)

    # create FG regex
    fgKickerRE = r'(?P<fgKicker>{0})'.format(playerRE)
    fgBaseRE = (r' (?P<fgDist>\d+) yard field goal'
                r' (?P<fgGood>good|no good)')
    fgBlockRE = (
        r'(?:, (?P<isBlocked>blocked) by '
        r'(?P<fgBlocker>{0}))?'.format(playerRE) +
        r'(?:, recovered by (?P<fgBlockRecoverer>{0}))?'.format(playerRE) +
        r'(?: and returned for (?:(?P<fgBlockRetYds>\-?\d+) yards?|no gain))?'
    )
    fgREstr = r'{}{}{}{}{}'.format(fgKickerRE, fgBaseRE,
                                   fgBlockRE, tdSafetyRE, penaltyRE)
    fgRE = re.compile(fgREstr, re.IGNORECASE)

    # create punt regex
    punterRE = r'.*?(?P<punter>{0})'.format(playerRE)
    puntBlockRE = (
        (r' punts, (?P<isBlocked>blocked) by (?P<puntBlocker>{0})'
         r'(?:, recovered by (?P<puntBlockRecoverer>{0})').format(playerRE) +
        r'(?: and returned (?:(?P<puntBlockRetYds>\-?\d+) yards|no gain))?)?'
    )
    puntYdsRE = r' punts (?P<puntYds>\d+) yards?'
    nextREs = []
    nextREs.append(r', (?P<isFairCatch>fair catch) by (?P<fairCatcher>{0})'
                   .format(playerRE))
    nextREs.append(r', (?P<oob>out of bounds)')
    nextREs.append(
        (r'(?P<isMuffedCatch>, muffed catch by )(?P<muffedBy>{0}),'
         r' recovered by (?P<muffRecoverer>{0})').format(playerRE) +
        r' and returned for ' +
        r'(?:(?P<muffRetYds>\d+) yards|no gain)'
    )
    nextREs.append(
        r', returned by (?P<puntReturner>{0}) for '.format(playerRE) +
        r'(?:(?P<puntRetYds>\-?\d+) yards?|no gain)'
    )
    nextRE = r'(?:{})?'.format('|'.join(nextREs))
    puntREstr = r'{}(?:{}|{}){}{}{}{}{}'.format(
        punterRE, puntBlockRE, puntYdsRE, nextRE,
        tackleRE, fumbleRE, tdSafetyRE, penaltyRE
    )
    puntRE = re.compile(puntREstr, re.IGNORECASE)

    # create kneel regex
    kneelREstr = (r'(?P<kneelQB>{0}) kneels for '.format(playerRE) +
                  r'(?:(?P<kneelYds>\-?\d+) yards?|no gain)')
    kneelRE = re.compile(kneelREstr, re.IGNORECASE)

    # create spike regex
    spikeREstr = r'(?P<spikeQB>{0}) spiked the ball'.format(playerRE)
    spikeRE = re.compile(spikeREstr, re.IGNORECASE)

    # create XP regex
    extraPointREstr = (r'(?:(?P<xpKicker>{0}) kicks)? ?extra point '
                       r'(?P<xpGood>good|no good)').format(playerRE)
    extraPointRE = re.compile(extraPointREstr, re.IGNORECASE)

    # create 2pt conversion regex
    twoPointREstr = (
        r'Two Point Attempt: (?P<twoPoint>.*?),?\s+conversion\s+'
        r'(?P<twoPointSuccess>succeeds|fails)'
    )
    twoPointRE = re.compile(twoPointREstr, re.IGNORECASE)

    # create penalty regex
    psPenaltyREstr = (
        r'^Penalty on (?P<penOn>{0}|'.format(playerRE) + r'\w{3}): ' +
        r'(?P<penalty>[^\(,]+)(?: \((?P<penDeclined>Declined)\)|' +
        r', (?P<penYds>\d*) yards?|' +
        r'.*?(?: \(no play\)))')
    psPenaltyRE = re.compile(psPenaltyREstr, re.IGNORECASE)

    # try parsing as a kickoff
    match = kickoffRE.search(details)
    if match:
        # parse as a kickoff
        struct['isKickoff'] = True
        struct.update(match.groupdict())
        return struct

    # try parsing as a timeout
    match = timeoutRE.search(details)
    if match:
        # parse as timeout
        struct['isTimeout'] = True
        struct.update(match.groupdict())
        return struct

    # try parsing as a field goal
    match = fgRE.search(details)
    if match:
        # parse as a field goal
        struct['isFieldGoal'] = True
        struct.update(match.groupdict())
        return struct

    # try parsing as a punt
    match = puntRE.search(details)
    if match:
        # parse as a punt
        struct['isPunt'] = True
        struct.update(match.groupdict())
        return struct

    # try parsing as a kneel
    match = kneelRE.search(details)
    if match:
        # parse as a kneel
        struct['isKneel'] = True
        struct.update(match.groupdict())
        return struct

    # try parsing as a spike
    match = spikeRE.search(details)
    if match:
        # parse as a spike
        struct['isSpike'] = True
        struct.update(match.groupdict())
        return struct

    # try parsing as an XP
    match = extraPointRE.search(details)
    if match:
        # parse as an XP
        struct['isXP'] = True
        struct.update(match.groupdict())
        return struct

    # try parsing as a 2-point conversion
    match = twoPointRE.search(details)
    if match:
        # parse as a 2-point conversion
        struct['isTwoPoint'] = True
        struct['twoPointSuccess'] = match.group('twoPointSuccess')
        realPlay = sportsref.nfl.pbp.parse_play_details(
            match.group('twoPoint'))
        if realPlay:
            struct.update(realPlay)
        return struct

    # try parsing as a pass
    match = passRE.search(details)
    if match:
        # parse as a pass
        struct['isPass'] = True
        struct.update(match.groupdict())
        return struct

    # try parsing as a pre-snap penalty
    match = psPenaltyRE.search(details)
    if match:
        # parse as a pre-snap penalty
        struct['isPresnapPenalty'] = True
        struct.update(match.groupdict())
        return struct

    # try parsing as a run
    match = rushRE.search(details)
    if match:
        # parse as a run
        struct['isRun'] = True
        struct.update(match.groupdict())
        return struct

    return None

def _clean_features(struct):
    """Cleans up the features collected in parse_play_details.

    :struct: Pandas Series of features parsed from details string.
    :returns: the same dict, but with cleaner features (e.g., convert bools,
    ints, etc.)
    """
    struct = dict(struct)
    # First, clean up play type bools
    ptypes = ['isKickoff', 'isTimeout', 'isFieldGoal', 'isPunt', 'isKneel',
              'isSpike', 'isXP', 'isTwoPoint', 'isPresnapPenalty', 'isPass',
              'isRun']
    for pt in ptypes:
        struct[pt] = struct[pt] if pd.notnull(struct.get(pt)) else False
    # Second, clean up other existing variables on a one-off basis
    struct['callUpheld'] = struct.get('callUpheld') == 'upheld'
    struct['fgGood'] = struct.get('fgGood') == 'good'
    struct['isBlocked'] = struct.get('isBlocked') == 'blocked'
    struct['isComplete'] = struct.get('isComplete') == 'complete'
    struct['isFairCatch'] = struct.get('isFairCatch') == 'fair catch'
    struct['isMuffedCatch'] = pd.notnull(struct.get('isMuffedCatch'))
    struct['isNoPlay'] = (
        ' (no play)' in struct['detail'] and
        'penalty enforced in end zone' not in struct['detail']
        if struct.get('detail') else False)
    struct['isOnside'] = struct.get('isOnside') == 'onside'
    struct['isSack'] = pd.notnull(struct.get('sackYds'))
    struct['isSafety'] = (struct.get('isSafety') == ', safety' or
                          (struct.get('detail') and
                           'enforced in end zone, safety' in struct['detail']))
    struct['isTD'] = struct.get('isTD') == ', touchdown'
    struct['isTouchback'] = struct.get('isTouchback') == ', touchback'
    struct['oob'] = pd.notnull(struct.get('oob'))
    struct['passLoc'] = PASS_OPTS.get(struct.get('passLoc'), np.nan)
    if struct['isPass']:
        pyds = struct['passYds']
        struct['passYds'] = pyds if pd.notnull(pyds) else 0
    if pd.notnull(struct['penalty']):
        struct['penalty'] = struct['penalty'].strip()
    struct['penDeclined'] = struct.get('penDeclined') == 'Declined'
    if struct['quarter'] == 'OT':
        struct['quarter'] = 5
    struct['rushDir'] = RUSH_OPTS.get(struct.get('rushDir'), np.nan)
    if struct['isRun']:
        ryds = struct['rushYds']
        struct['rushYds'] = ryds if pd.notnull(ryds) else 0
    year = struct.get('season', np.nan)
    struct['timeoutTeam'] = sportsref.nfl.teams.team_ids(year).get(
        struct.get('timeoutTeam'), np.nan
    )
    struct['twoPointSuccess'] = struct.get('twoPointSuccess') == 'succeeds'
    struct['xpGood'] = struct.get('xpGood') == 'good'

    # Third, ensure types are correct
    bool_vars = [
        'fgGood', 'isBlocked', 'isChallenge', 'isComplete', 'isFairCatch',
        'isFieldGoal', 'isKickoff', 'isKneel', 'isLateral', 'isNoPlay',
        'isPass', 'isPresnapPenalty', 'isPunt', 'isRun', 'isSack', 'isSafety',
        'isSpike', 'isTD', 'isTimeout', 'isTouchback', 'isTwoPoint', 'isXP',
        'isMuffedCatch', 'oob', 'penDeclined', 'twoPointSuccess', 'xpGood'
    ]
    int_vars = [
        'down', 'fgBlockRetYds', 'fgDist', 'fumbRecYdLine', 'fumbRetYds',
        'intRetYds', 'intYdLine', 'koRetYds', 'koYds', 'muffRetYds',
        'pbp_score_aw', 'pbp_score_hm', 'passYds', 'penYds', 'puntBlockRetYds',
        'puntRetYds', 'puntYds', 'quarter', 'rushYds', 'sackYds', 'timeoutNum',
        'ydLine', 'yds_to_go'
    ]
    float_vars = [
        'exp_pts_after', 'exp_pts_before', 'home_wp'
    ]
    string_vars = [
        'challenger', 'detail', 'fairCatcher', 'fgBlockRecoverer',
        'fgBlocker', 'fgKicker', 'fieldSide', 'fumbForcer',
        'fumbRecFieldSide', 'fumbRecoverer', 'fumbler', 'intFieldSide',
        'interceptor', 'kneelQB', 'koKicker', 'koReturner', 'muffRecoverer',
        'muffedBy', 'passLoc', 'passer', 'penOn', 'penalty',
        'puntBlockRecoverer', 'puntBlocker', 'puntReturner', 'punter',
        'qtr_time_remain', 'rushDir', 'rusher', 'sacker1', 'sacker2',
        'spikeQB', 'tackler1', 'tackler2', 'target', 'timeoutTeam',
        'xpKicker'
    ]
    for var in bool_vars:
        struct[var] = struct.get(var) is True
    for var in int_vars:
        try:
            struct[var] = int(struct.get(var))
        except (ValueError, TypeError):
            struct[var] = np.nan
    for var in float_vars:
        try:
            struct[var] = float(struct.get(var))
        except (ValueError, TypeError):
            struct[var] = np.nan
    for var in string_vars:
        if var not in struct or pd.isnull(struct[var]) or var == '':
            struct[var] = np.nan

    # Fourth, create new helper variables based on parsed variables
    # creating fieldSide and ydline from location
    if struct['isXP']:
        struct['fieldSide'] = struct['ydLine'] = np.nan
    else:
        fieldSide, ydline = _loc_to_features(struct.get('location'))
        struct['fieldSide'] = fieldSide
        struct['ydLine'] = ydline
    # creating secsElapsed (in entire game) from qtr_time_remain and quarter
    if pd.notnull(struct.get('qtr_time_remain')):
        qtr = struct['quarter']
        mins, secs = map(int, struct['qtr_time_remain'].split(':'))
        struct['secsElapsed'] = qtr * 900 - mins * 60 - secs
    # creating columns for turnovers
    struct['isInt'] = pd.notnull(struct.get('interceptor'))
    struct['isFumble'] = pd.notnull(struct.get('fumbler'))
    # create column for isPenalty
    struct['isPenalty'] = pd.notnull(struct.get('penalty'))
    # create columns for EPA
    struct['team_epa'] = struct['exp_pts_after'] - struct['exp_pts_before']
    struct['opp_epa'] = struct['exp_pts_before'] - struct['exp_pts_after']
    return pd.Series(struct)

def _loc_to_features(loc):
    """Converts a location string "{Half}, {YardLine}" into a tuple of those
    values, the second being an int.

    :l: The string from the play by play table representing location.
    :returns: A tuple that separates out the values, making them missing
    (np.nan) when necessary.

    """
    if loc:
        if isinstance(loc, basestring):
            loc = loc.strip()
            if ' ' in loc:
                r = loc.split()
                r[0] = r[0].lower()
                r[1] = int(r[1])
            else:
                r = (np.nan, int(loc))
        elif isinstance(loc, float):
            return (np.nan, 50)
    else:
        r = (np.nan, np.nan)
    return r

def _add_team_columns(features):
    """Function that adds 'team' and 'opp' columns to the features by iterating
    through the rows in order. A precondition is that the features dicts are in
    order in a continuous game sense and that all rows are from the same game.

    :features: A DataFrame with each row representing each play (in order).
    :returns: A similar DataFrame but with 'team' and 'opp' columns added.
    """
    features = features.to_dict('records')
    curTm = curOpp = None
    playAfterKickoff = False
    # fill in team and opp columns
    for row in features:
        # if it's a kickoff or the play after a kickoff,
        # figure out who has possession manually
        if row['isKickoff'] or playAfterKickoff:
            curTm, curOpp = _team_and_opp(row)
        else:
            curTm, curOpp = _team_and_opp(row, curTm, curOpp)
        row['team'], row['opp'] = curTm, curOpp
        # set playAfterKickoff
        playAfterKickoff = row['isKickoff']

    features = pd.DataFrame(features)
    features.team.fillna(method='bfill', inplace=True)
    features.opp.fillna(method='bfill', inplace=True)
    # ffill for last row
    features.team.fillna(method='ffill', inplace=True)
    features.opp.fillna(method='ffill', inplace=True)
    return features

def _team_and_opp(struct, curTm=None, curOpp=None):
    """Given a dict representing a play and the current team with the ball,
    returns (team, opp) where team is the team with the ball and opp is the
    team without the ball at the end of the play.

    :struct: A Series/dict representing the play.
    :curTm: The current team with the ball; None means it's the first play of
    the game or the offensive team on the previous play's offensive team was
    somehow undetermined.
    :curOpp: The current team on defense; None means same as curTm.
    :returns: (team, opp) tuple where team and opp are the 3-character team IDs
    or the offensive and defensive teams respectively.
    """
    # if we don't know the current team, figure it out
    if pd.isnull(curTm):
        if struct['isRun']:
            pID = struct['rusher']
        elif struct['isPass']:
            pID = struct['passer']
        elif struct['isFieldGoal']:
            pID = struct['fgKicker']
        elif struct['isPunt']:
            pID = struct['punter']
        elif struct['isXP']:
            pID = struct['xpKicker']
        elif struct['isKickoff']:
            pID = struct['koKicker']
        elif struct['isSpike']:
            pID = struct['spikeQB']
        elif struct['isKneel']:
            pID = struct['kneelQB']
        else:
            pID = None
        curTm = curOpp = np.nan
        bs = sportsref.nfl.boxscores.BoxScore(struct['boxscore_id'])
        if pID and len(pID) == 3:
            curTm = pID
            curOpp = bs.away() if bs.home() == curTm else bs.home()
        elif pID:
            player = sportsref.nfl.Player(pID)
            gamelog = player.gamelog(kind='B')
            curTm = gamelog.loc[
                gamelog.boxscore_id == struct['boxscore_id'], 'team_id'
            ].item()
            curOpp = bs.home() if bs.home() != curTm else bs.away()

        return curTm, curOpp

    # use row's class to determine when possession changes
    if struct['has_class_divider']:
        return curOpp, curTm
    else:
        return curTm, curOpp

def _add_team_features(df):
    """Adds extra convenience features based on teams with and without
    possession, with the precondition that the there are 'team' and 'opp'
    specified in row.

    :df: A DataFrame representing a game's play-by-play data after
        _clean_features has been called and 'team' and 'opp' have been added by
        _add_team_columns.
    :returns: A dict with new features in addition to previous features.
    """
    assert df.team.notnull().all()

    homeOnOff = df['team'] == df['home']
    # create column for distToGoal
    df['distToGoal'] = np.where(df['team'] != df['fieldSide'],
                                df['ydLine'], 100 - df['ydLine'])
    df['distToGoal'] = np.where(df['isXP'] | df['isTwoPoint'],
                                2, df['distToGoal'])
    # create column for each team's WP
    df['team_wp'] = np.where(homeOnOff, df['home_wp'], 100. - df['home_wp'])
    df['opp_wp'] = 100. - df['team_wp']
    # create columns for each team's WPA
    df['team_wpa'] = np.where(homeOnOff, df['home_wpa'], -df['home_wpa'])
    df['opp_wpa'] = -df['team_wpa']
    # create column for offense and defense scores if not already there
    assert df['boxscore_id'].nunique() == 1
    bs_id = df['boxscore_id'].values[0]
    bs = sportsref.nfl.boxscores.BoxScore(bs_id)
    df['team_score'] = np.where(df['team'] == bs.home(),
                                df['pbp_score_hm'], df['pbp_score_aw'])
    df['opp_score'] = np.where(df['team'] == bs.home(),
                               df['pbp_score_aw'], df['pbp_score_hm'])

    return df


import future
import future.utils

import pandas as pd
from pyquery import PyQuery as pq

import sportsref

__all__ = ['Season']

class Season(future.utils.with_metaclass(sportsref.decorators.Cached, object)):

    """Object representing a given NFL season."""

    def __init__(self, year):
        """Initializes a Season object for an NFL season.

        :year: The year of the season we want.
        """
        self.yr = int(year)

    def __eq__(self, other):
        return (self.yr == other.yr)

    def __hash__(self):
        return hash(self.yr)

    def __repr__(self):
        return 'Season({})'.format(self.yr)

    def _subpage_url(self, page):
        return (sportsref.nfl.BASE_URL +
                '/years/{}/{}.htm'.format(self.yr, page))

    @sportsref.decorators.memoize
    def get_main_doc(self):
        """Returns PyQuery object for the main season URL.
        :returns: PyQuery object.
        """
        url = sportsref.nfl.BASE_URL + '/years/{}/'.format(self.yr)
        return pq(sportsref.utils.get_html(url))

    @sportsref.decorators.memoize
    def get_sub_doc(self, subpage):
        """Returns PyQuery object for a given subpage URL.
        :subpage: The subpage of the season, e.g. 'per_game'.
        :returns: PyQuery object.
        """
        html = sportsref.utils.get_html(self._subpage_url(subpage))
        return pq(html)

    @sportsref.decorators.memoize
    def get_team_ids(self):
        """Returns a list of the team IDs for the given year.
        :returns: List of team IDs.
        """
        return sportsref.nfl.teams.list_teams(self.yr)

    @sportsref.decorators.memoize
    def team_ids_to_names(self):
        """Mapping from 3-letter team IDs to full team names.

        :returns: Dictionary with team IDs as keys and full team strings as
            values.
        """
        return sportsref.nfl.teams.team_names(self.yr)

    @sportsref.decorators.memoize
    def team_names_to_ids(self):
        """Mapping from full team names to 3-letter team IDs.
        :returns: Dictionary with tean names as keys and team IDs as values.
        """
        return sportsref.nfl.teams.team_ids(self.yr)

    @sportsref.decorators.memoize
    def _get_player_stats_table(self, subpage, table_id):
        """Helper function for player season stats.

        :identifier: string identifying the type of stat, e.g. 'passing'.
        :returns: A DataFrame of stats.
        """
        doc = self.get_sub_doc(subpage)
        table = doc('table#{}'.format(table_id))
        df = sportsref.utils.parse_table(table)
        return df

    def player_stats_passing(self):
        """Returns a DataFrame of passing player stats for a season."""
        return self._get_player_stats_table('passing', 'passing')

    def player_stats_rushing(self):
        """Returns a DataFrame of rushing player stats for a season."""
        return self._get_player_stats_table('rushing', 'rushing_and_receiving')

    def player_stats_receiving(self):
        """Returns a DataFrame of receiving player stats for a season."""
        return self._get_player_stats_table('receiving', 'receiving')


from __future__ import division
import numpy as np
from scipy.stats import norm

def initialWinProb(line):
    """Gets the initial win probability of a game given its Vegas line.

    :line: The Vegas line from the home team's perspective (negative means
    home team is favored).
    :returns: A float in [0., 100.] that represents the win probability.
    """
    line = float(line)
    probWin = 1. - norm.cdf(0.5, -line, 13.86)
    probTie = norm.cdf(0.5, -line, 13.86) - norm.cdf(-0.5, -line, 13.86)
    return 100. * (probWin + 0.5 * probTie)

def winProb(line, margin, secsElapsed, expPts):
    line = float(line)
    margin = float(margin)
    expPts = float(expPts)
    baseMean = -line
    baseStd = 13.46
    expMargin = margin + expPts
    minRemain = 60 - secsElapsed / 60 + 0.00001
    adjMean = baseMean * minRemain / 60
    adjStd = baseStd / np.sqrt(60 / minRemain)
    probWin = 1. - norm.cdf(-expMargin + 0.5, adjMean, adjStd)
    probTie = (norm.cdf(-expMargin + 0.5, adjMean, adjStd) -
               norm.cdf(-expMargin - 0.5, adjMean, adjStd))
    return 100. * (probWin + 0.5 * probTie)


from __future__ import division
from future import standard_library
standard_library.install_aliases()
from builtins import map
import future
import future.utils

import datetime
import re
import urllib.parse

import numpy as np
from pyquery import PyQuery as pq

import sportsref

__all__ = [
    'Player',
]

class Player(future.utils.with_metaclass(sportsref.decorators.Cached, object)):

    def __init__(self, player_id):
        self.player_id = player_id
        self.mainURL = (sportsref.nfl.BASE_URL +
                        '/players/{0[0]}/{0}.htm').format(self.player_id)

    def __eq__(self, other):
        return self.player_id == other.player_id

    def __hash__(self):
        return hash(self.player_id)

    def __repr__(self):
        return 'Player({})'.format(self.player_id)

    def __str__(self):
        return self.name()

    def __reduce__(self):
        return Player, (self.player_id,)

    def _subpage_url(self, page, year=None):
        # if no year, return career version
        if year is None:
            return urllib.parse.urljoin(
                self.mainURL, '{}/{}/'.format(self.player_id, page)
            )
        # otherwise, return URL for a given year
        else:
            return urllib.parse.urljoin(
                self.mainURL, '{}/{}/{}/'.format(self.player_id, page, year)
            )

    @sportsref.decorators.memoize
    def get_doc(self):
        doc = pq(sportsref.utils.get_html(self.mainURL))
        return doc

    @sportsref.decorators.memoize
    def name(self):
        doc = self.get_doc()
        name = doc('div#meta h1:first').text()
        return name

    @sportsref.decorators.memoize
    def age(self, year, month=9, day=1):
        doc = self.get_doc()
        span = doc('div#meta span#necro-birth')
        birthstring = span.attr('data-birth')
        try:
            dateargs = re.match(r'(\d{4})\-(\d{2})\-(\d{2})',
                                birthstring).groups()
            dateargs = map(int, dateargs)
            birthDate = datetime.date(*dateargs)
            delta = datetime.date(year=year, month=month, day=day) - birthDate
            age = delta.days / 365
            return age
        except Exception:
            return None

    @sportsref.decorators.memoize
    def position(self):
        doc = self.get_doc()
        rawText = (doc('div#meta p')
                   .filter(lambda i, e: 'Position' in e.text_content())
                   .text())
        rawPos = re.search(r'Position\W*(\S+)', rawText, re.I).group(1)
        allPositions = rawPos.split('-')
        # right now, returning just the primary position for those with
        # multiple positions
        return allPositions[0]

    @sportsref.decorators.memoize
    def height(self):
        doc = self.get_doc()
        rawText = doc('div#meta p span[itemprop="height"]').text()
        try:
            feet, inches = map(int, rawText.split('-'))
            return feet * 12 + inches
        except ValueError:
            return None

    @sportsref.decorators.memoize
    def weight(self):
        doc = self.get_doc()
        rawText = doc('div#meta p span[itemprop="weight"]').text()
        try:
            weight = re.match(r'(\d+)lb', rawText, re.I).group(1)
            return int(weight)
        except AttributeError:
            return None

    @sportsref.decorators.memoize
    def hand(self):
        doc = self.get_doc()
        try:
            rawText = (doc('div#meta p')
                       .filter(lambda i, e: 'Throws' in e.text_content())
                       .text())
            rawHand = re.search(r'Throws\W+(\S+)', rawText, re.I).group(1)
        except AttributeError:
            return None
        return rawHand[0]  # 'L' or 'R'

    @sportsref.decorators.memoize
    def current_team(self):
        doc = self.get_doc()
        team = (doc('div#meta p')
                .filter(lambda i, e: 'Team' in e.text_content()))
        text = sportsref.utils.flatten_links(team)
        try:
            m = re.match(r'Team: (\w{3})', text)
            return m.group(1)
        except Exception:
            return None

    @sportsref.decorators.memoize
    def draft_pick(self):
        doc = self.get_doc()
        rawDraft = (doc('div#meta p')
                    .filter(lambda i, e: 'Draft' in e.text_content())
                    .text())
        m = re.search(r'Draft.*? round \((\d+).*?overall\)', rawDraft, re.I)
        # if not drafted or taken in supplemental draft, return NaN
        if m is None or 'Supplemental' in rawDraft:
            return None
        else:
            return int(m.group(1))

    @sportsref.decorators.memoize
    def draft_class(self):
        doc = self.get_doc()
        rawDraft = (doc('div#meta p')
                    .filter(lambda i, e: 'Draft' in e.text_content())
                    .text())
        m = re.search(r'Draft.*?of the (\d{4}) NFL', rawDraft, re.I)
        if not m:
            return None
        else:
            return int(m.group(1))

    @sportsref.decorators.memoize
    def draft_team(self):
        doc = self.get_doc()
        rawDraft = (doc('div#meta p')
                    .filter(lambda i, e: 'Draft' in e.text_content()))
        try:
            draftStr = sportsref.utils.flatten_links(rawDraft)
            m = re.search(r'Draft\W+(\w+)', draftStr)
            return m.group(1)
        except Exception:
            return None

    @sportsref.decorators.memoize
    def college(self):
        doc = self.get_doc()
        rawText = (doc('div#meta p')
                   .filter(lambda i, e: 'College' in e.text_content()))
        cleanedText = sportsref.utils.flatten_links(rawText)
        college = re.search(r'College:\s*(\S+)', cleanedText).group(1)
        return college

    @sportsref.decorators.memoize
    def high_school(self):
        doc = self.get_doc()
        rawText = (doc('div#meta p')
                   .filter(lambda i, e: 'High School' in e.text_content()))
        cleanedText = sportsref.utils.flatten_links(rawText)
        hs = re.search(r'High School:\s*(\S+)', cleanedText).group(1)
        return hs

    @sportsref.decorators.memoize
    @sportsref.decorators.kind_rpb(include_type=True)
    def gamelog(self, year=None, kind='R'):
        """Gets the career gamelog of the given player.
        :kind: One of 'R', 'P', or 'B' (for regular season, playoffs, or both).
        Case-insensitive; defaults to 'R'.
        :year: The year for which the gamelog should be returned; if None,
        return entire career gamelog. Defaults to None.
        :returns: A DataFrame with the player's career gamelog.
        """
        url = self._subpage_url('gamelog', None)  # year is filtered later
        doc = pq(sportsref.utils.get_html(url))
        table = (doc('table#stats') if kind == 'R' else
                 doc('table#stats_playoffs'))
        df = sportsref.utils.parse_table(table)
        if year is not None:
            df = df.query('year == @year').reset_index(drop=True)
        return df

    @sportsref.decorators.memoize
    @sportsref.decorators.kind_rpb(include_type=True)
    def passing(self, kind='R'):
        """Gets yearly passing stats for the player.

        :kind: One of 'R', 'P', or 'B'. Case-insensitive; defaults to 'R'.
        :returns: Pandas DataFrame with passing stats.
        """
        doc = self.get_doc()
        table = (doc('table#passing') if kind == 'R' else
                 doc('table#passing_playoffs'))
        df = sportsref.utils.parse_table(table)
        return df

    @sportsref.decorators.memoize
    @sportsref.decorators.kind_rpb(include_type=True)
    def rushing_and_receiving(self, kind='R'):
        """Gets yearly rushing/receiving stats for the player.

        :kind: One of 'R', 'P', or 'B'. Case-insensitive; defaults to 'R'.
        :returns: Pandas DataFrame with rushing/receiving stats.
        """
        doc = self.get_doc()
        table = (doc('table#rushing_and_receiving') if kind == 'R'
                 else doc('table#rushing_and_receiving_playoffs'))
        if not table:
            table = (doc('table#receiving_and_rushing') if kind == 'R'
                     else doc('table#receiving_and_rushing_playoffs'))
        df = sportsref.utils.parse_table(table)
        return df

    @sportsref.decorators.memoize
    @sportsref.decorators.kind_rpb(include_type=True)
    def defense(self, kind='R'):
        """Gets yearly defense stats for the player (also has AV stats for OL).

        :kind: One of 'R', 'P', or 'B'. Case-insensitive; defaults to 'R'.
        :returns: Pandas DataFrame with rushing/receiving stats.
        """
        doc = self.get_doc()
        table = (doc('table#defense') if kind == 'R' else
                 doc('table#defense_playoffs'))
        df = sportsref.utils.parse_table(table)
        return df

    def _plays(self, year, play_type, expand_details):
        """Returns a DataFrame of plays for a given year for a given play type
        (like rushing, receiving, or passing).

        :year: The year for the season.
        :play_type: A type of play for which there are plays (as of this
        writing, either "passing", "rushing", or "receiving")
        :expand_details: Bool for whether PBP should be parsed.
        :returns: A DataFrame of plays, each row is a play. Returns None if
        there were no such plays in that year.
        """
        url = self._subpage_url('{}-plays'.format(play_type), year)
        doc = pq(sportsref.utils.get_html(url))
        table = doc('table#all_plays')
        if table:
            if expand_details:
                plays = sportsref.nfl.pbp.expand_details(
                    sportsref.utils.parse_table(table), detailCol='description'
                )
                return plays
            else:
                return sportsref.utils.parse_table(table)
        else:
            return None

    @sportsref.decorators.memoize
    def passing_plays(self, year, expand_details=True):
        """Returns a pbp DataFrame of a player's passing plays in a season.

        :year: The year for the season.
        :expand_details: bool for whether PBP should be parsed.
        :returns: A DataFrame of stats, each row is a play.
        """
        return self._plays(year, 'passing', expand_details)

    @sportsref.decorators.memoize
    def rushing_plays(self, year, expand_details=True):
        """Returns a pbp DataFrame of a player's rushing plays in a season.

        :year: The year for the season.
        :expand_details: bool for whether PBP should be parsed.
        :returns: A DataFrame of stats, each row is a play.
        """
        return self._plays(year, 'rushing', expand_details)

    @sportsref.decorators.memoize
    def receiving_plays(self, year, expand_details=True):
        """Returns a pbp DataFrame of a player's receiving plays in a season.

        :year: The year for the season.
        :expand_details: bool for whether PBP should be parsed.
        :returns: A DataFrame of stats, each row is a play.
        """
        return self._plays(year, 'receiving', expand_details)

    @sportsref.decorators.memoize
    def splits(self, year=None):
        """Returns a DataFrame of splits data for a player-year.

        :year: The year for the season in question. If None, returns career
        splits.
        :returns: A DataFrame of splits data.
        """
        # get the table
        url = self._subpage_url('splits', year)
        doc = pq(sportsref.utils.get_html(url))
        table = doc('table#stats')
        df = sportsref.utils.parse_table(table)
        # cleaning the data
        if not df.empty:
            df.split_id.fillna(method='ffill', inplace=True)
        return df

    @sportsref.decorators.memoize
    def advanced_splits(self, year=None):
        """Returns a DataFrame of advanced splits data for a player-year. Note:
            only go back to 2012.

        :year: The year for the season in question. If None, returns career
        advanced splits.
        :returns: A DataFrame of advanced splits data.
        """
        # get the table
        url = self._subpage_url('splits', year)
        doc = pq(sportsref.utils.get_html(url))
        table = doc('table#advanced_splits')
        df = sportsref.utils.parse_table(table)
        # cleaning the data
        if not df.empty:
            df.split_type.fillna(method='ffill', inplace=True)
        return df

    @sportsref.decorators.memoize
    def _simple_year_award(self, award_id):
        """Template for simple award functions that simply list years, such as
        pro bowls and first-team all pro.

        :award_id: The div ID that is appended to "leaderboard_" in selecting
        the table's div.
        :returns: List of years for the award.
        """
        doc = self.get_doc()
        table = doc('div#leaderboard_{} table'.format(award_id))
        return list(map(int, sportsref.utils.parse_awards_table(table)))

    def pro_bowls(self):
        """Returns a list of years in which the player made the Pro Bowl."""
        return self._simple_year_award('pro_bowls')

    def first_team_all_pros(self):
        """Returns a list of years in which the player made 1st-Tm All Pro."""
        return self._simple_year_award('all_pro')

    # TODO: other awards like MVP, OPOY, DPOY, NFL Top 100, etc.


from builtins import next, range, zip
import future
import future.utils

import re

import numpy as np
import pandas as pd
from pyquery import PyQuery as pq

import sportsref

__all__ = [
    'team_names',
    'team_ids',
    'list_teams',
    'Team',
]

@sportsref.decorators.memoize
def team_names(year):
    """Returns a mapping from team ID to full team name for a given season.
    Example of a full team name: "New England Patriots"

    :year: The year of the season in question (as an int).
    :returns: A dictionary with teamID keys and full team name values.
    """
    doc = pq(sportsref.utils.get_html(sportsref.nfl.BASE_URL + '/teams/'))
    active_table = doc('table#teams_active')
    active_df = sportsref.utils.parse_table(active_table)
    inactive_table = doc('table#teams_inactive')
    inactive_df = sportsref.utils.parse_table(inactive_table)
    df = pd.concat((active_df, inactive_df))
    df = df.loc[~df['has_class_partial_table']]
    ids = df.team_id.str[:3].values
    names = [tr('th a') for tr in active_table('tr').items()]
    names.extend(tr('th a') for tr in inactive_table('tr').items())
    names = [_f for _f in names if _f]
    names = [lst[0].text_content() for lst in names]
    # combine IDs and team names into pandas series
    series = pd.Series(names, index=ids)
    # create a mask to filter to teams from the given year
    mask = ((df.year_min <= year) & (year <= df.year_max)).values
    # filter, convert to a dict, and return
    return series[mask].to_dict()

@sportsref.decorators.memoize
def team_ids(year):
    """Returns a mapping from team name to team ID for a given season. Inverse
    mapping of team_names. Example of a full team name: "New England Patriots"

    :year: The year of the season in question (as an int).
    :returns: A dictionary with full team name keys and teamID values.
    """
    names = team_names(year)
    return {v: k for k, v in names.items()}

@sportsref.decorators.memoize
def list_teams(year):
    """Returns a list of team IDs for a given season.

    :year: The year of the season in question (as an int).
    :returns: A list of team IDs.
    """
    return list(team_names(year).keys())

class Team(future.utils.with_metaclass(sportsref.decorators.Cached, object)):

    def __init__(self, teamID):
        self.teamID = teamID

    def __eq__(self, other):
        return (self.teamID == other.teamID)

    def __hash__(self):
        return hash(self.teamID)

    def __repr__(self):
        return 'Team({})'.format(self.teamID)

    def __str__(self):
        return self.name()

    def __reduce__(self):
        return Team, (self.teamID,)

    @sportsref.decorators.memoize
    def team_year_url(self, yr_str):
        return (sportsref.nfl.BASE_URL +
                '/teams/{}/{}.htm'.format(self.teamID, yr_str))

    @sportsref.decorators.memoize
    def get_main_doc(self):
        relURL = '/teams/{}'.format(self.teamID)
        teamURL = sportsref.nfl.BASE_URL + relURL
        mainDoc = pq(sportsref.utils.get_html(teamURL))
        return mainDoc

    @sportsref.decorators.memoize
    def get_year_doc(self, yr_str):
        return pq(sportsref.utils.get_html(self.team_year_url(yr_str)))

    @sportsref.decorators.memoize
    def name(self):
        """Returns the real name of the franchise given the team ID.

        Examples:
        'nwe' -> 'New England Patriots'
        'sea' -> 'Seattle Seahawks'

        :returns: A string corresponding to the team's full name.
        """
        doc = self.get_main_doc()
        headerwords = doc('div#meta h1')[0].text_content().split()
        lastIdx = headerwords.index('Franchise')
        teamwords = headerwords[:lastIdx]
        return ' '.join(teamwords)

    @sportsref.decorators.memoize
    def roster(self, year):
        """Returns the roster table for the given year.

        :year: The year for which we want the roster; defaults to current year.
        :returns: A DataFrame containing roster information for that year.
        """
        doc = self.get_year_doc('{}_roster'.format(year))
        roster_table = doc('table#games_played_team')
        df = sportsref.utils.parse_table(roster_table)
        starter_table = doc('table#starters')
        if not starter_table.empty:
            start_df = sportsref.utils.parse_table(starter_table)
            start_df = start_df.dropna(axis=0, subset=['position'])
            starters = start_df.set_index('position').player_id
            df['is_starter'] = df.player_id.isin(starters)
            df['starting_pos'] = df.player_id.map(
                lambda pid: (starters[starters == pid].index[0]
                             if pid in starters.values else None)
            )
        return df

    @sportsref.decorators.memoize
    def boxscores(self, year):
        """Gets list of BoxScore objects corresponding to the box scores from
        that year.

        :year: The year for which we want the boxscores; defaults to current
        year.
        :returns: np.array of strings representing boxscore IDs.
        """
        doc = self.get_year_doc(year)
        table = doc('table#games')
        df = sportsref.utils.parse_table(table)
        if df.empty:
            return np.array([])
        return df.boxscore_id.values

    @sportsref.decorators.memoize
    def _year_info_pq(self, year, keyword):
        """Returns a PyQuery object containing the info from the meta div at
        the top of the team year page with the given keyword.

        :year: Int representing the season.
        :keyword: A keyword to filter to a single p tag in the meta div.
        :returns: A PyQuery object for the selected p element.
        """
        doc = self.get_year_doc(year)
        p_tags = doc('div#meta div:not(.logo) p')
        texts = [p_tag.text_content().strip() for p_tag in p_tags]
        try:
            return next(
                pq(p_tag) for p_tag, text in zip(p_tags, texts)
                if keyword.lower() in text.lower()
            )
        except StopIteration:
            if len(texts):
                raise ValueError('Keyword not found in any p tag.')
            else:
                raise ValueError('No meta div p tags found.')

    # TODO: add functions for OC, DC, PF, PA, W-L, etc.
    # TODO: Also give a function at BoxScore.homeCoach and BoxScore.awayCoach
    # TODO: BoxScore needs a gameNum function to do this?

    @sportsref.decorators.memoize
    def head_coaches_by_game(self, year):
        """Returns head coach data by game.

        :year: An int representing the season in question.
        :returns: An array with an entry per game of the season that the team
        played (including playoffs). Each entry is the head coach's ID for that
        game in the season.
        """
        coach_str = self._year_info_pq(year, 'Coach').text()
        regex = r'(\S+?) \((\d+)-(\d+)-(\d+)\)'
        coachAndTenure = []
        m = True
        while m:
            m = re.search(regex, coach_str)
            coachID, wins, losses, ties = m.groups()
            nextIndex = m.end(4) + 1
            coachStr = coachStr[nextIndex:]
            tenure = int(wins) + int(losses) + int(ties)
            coachAndTenure.append((coachID, tenure))

        coachIDs = [
            cID for cID, games in coachAndTenure for _ in range(games)
        ]
        return np.array(coachIDs[::-1])

    @sportsref.decorators.memoize
    def wins(self, year):
        """Returns the # of regular season wins a team in a year.

        :year: The year for the season in question.
        :returns: The number of regular season wins.
        """
        schedule = self.schedule(year)
        if schedule.empty:
            return np.nan
        return schedule.query('week_num <= 17').is_win.sum()

    @sportsref.decorators.memoize
    def schedule(self, year):
        """Returns a DataFrame with schedule information for the given year.

        :year: The year for the season in question.
        :returns: Pandas DataFrame with schedule information.
        """
        doc = self.get_year_doc(year)
        table = doc('table#games')
        df = sportsref.utils.parse_table(table)
        if df.empty:
            return pd.DataFrame()
        df = df.loc[df['week_num'].notnull()]
        df['week_num'] = np.arange(len(df)) + 1
        df['is_win'] = df['game_outcome'] == 'W'
        df['is_loss'] = df['game_outcome'] == 'L'
        df['is_tie'] = df['game_outcome'] == 'T'
        df['is_bye'] = df['game_outcome'].isnull()
        df['is_ot'] = df['overtime'].notnull()
        return df

    @sportsref.decorators.memoize
    def srs(self, year):
        """Returns the SRS (Simple Rating System) for a team in a year.

        :year: The year for the season in question.
        :returns: A float of SRS.
        """
        try:
            srs_text = self._year_info_pq(year, 'SRS').text()
        except ValueError:
            return None
        m = re.match(r'SRS\s*?:\s*?(\S+)', srs_text)
        if m:
            return float(m.group(1))
        else:
            return None

    @sportsref.decorators.memoize
    def sos(self, year):
        """Returns the SOS (Strength of Schedule) for a team in a year, based
        on SRS.

        :year: The year for the season in question.
        :returns: A float of SOS.
        """
        try:
            sos_text = self._year_info_pq(year, 'SOS').text()
        except ValueError:
            return None
        m = re.search(r'SOS\s*:\s*(\S+)', sos_text)
        if m:
            return float(m.group(1))
        else:
            return None

    @sportsref.decorators.memoize
    def off_coordinator(self, year):
        """Returns the coach ID for the team's OC in a given year.

        :year: An int representing the year.
        :returns: A string containing the coach ID of the OC.
        """
        try:
            oc_anchor = self._year_info_pq(year, 'Offensive Coordinator')('a')
            if oc_anchor:
                return oc_anchor.attr['href']
        except ValueError:
            return None

    @sportsref.decorators.memoize
    def def_coordinator(self, year):
        """Returns the coach ID for the team's DC in a given year.

        :year: An int representing the year.
        :returns: A string containing the coach ID of the DC.
        """
        try:
            dc_anchor = self._year_info_pq(year, 'Defensive Coordinator')('a')
            if dc_anchor:
                return dc_anchor.attr['href']
        except ValueError:
            return None

    @sportsref.decorators.memoize
    def stadium(self, year):
        """Returns the ID for the stadium in which the team played in a given
        year.

        :year: The year in question.
        :returns: A string representing the stadium ID.
        """
        anchor = self._year_info_pq(year, 'Stadium')('a')
        return sportsref.utils.rel_url_to_id(anchor.attr['href'])

    @sportsref.decorators.memoize
    def off_scheme(self, year):
        """Returns the name of the offensive scheme the team ran in the given
        year.

        :year: Int representing the season year.
        :returns: A string representing the offensive scheme.
        """
        scheme_text = self._year_info_pq(year, 'Offensive Scheme').text()
        m = re.search(r'Offensive Scheme[:\s]*(.+)\s*', scheme_text, re.I)
        if m:
            return m.group(1)
        else:
            return None

    @sportsref.decorators.memoize
    def def_alignment(self, year):
        """Returns the name of the defensive alignment the team ran in the
        given year.

        :year: Int representing the season year.
        :returns: A string representing the defensive alignment.
        """
        scheme_text = self._year_info_pq(year, 'Defensive Alignment').text()
        m = re.search(r'Defensive Alignment[:\s]*(.+)\s*', scheme_text, re.I)
        if m:
            return m.group(1)
        else:
            return None

    @sportsref.decorators.memoize
    def team_stats(self, year):
        """Returns a Series (dict-like) of team stats from the team-season
        page.

        :year: Int representing the season.
        :returns: A Series of team stats.
        """
        doc = self.get_year_doc(year)
        table = doc('table#team_stats')
        df = sportsref.utils.parse_table(table)
        if df.empty:
            return pd.Series()
        return df.loc[df.player_id == 'Team Stats'].iloc[0]

    @sportsref.decorators.memoize
    def opp_stats(self, year):
        """Returns a Series (dict-like) of the team's opponent's stats from the
        team-season page.

        :year: Int representing the season.
        :returns: A Series of team stats.
        """
        doc = self.get_year_doc(year)
        table = doc('table#team_stats')
        df = sportsref.utils.parse_table(table)
        return df.loc[df.player_id == 'Opp. Stats'].iloc[0]

    @sportsref.decorators.memoize
    def passing(self, year):
        doc = self.get_year_doc(year)
        table = doc('table#passing')
        df = sportsref.utils.parse_table(table)
        return df

    @sportsref.decorators.memoize
    def rushing_and_receiving(self, year):
        doc = self.get_year_doc(year)
        table = doc('#rushing_and_receiving')
        df = sportsref.utils.parse_table(table)
        return df

    @sportsref.decorators.memoize
    def off_splits(self, year):
        """Returns a DataFrame of offensive team splits for a season.

        :year: int representing the season.
        :returns: Pandas DataFrame of split data.
        """
        doc = self.get_year_doc('{}_splits'.format(year))
        tables = doc('table.stats_table')
        dfs = [sportsref.utils.parse_table(table) for table in tables.items()]
        dfs = [
            df.assign(split=df.columns[0])
            .rename(columns={df.columns[0]: 'split_value'})
            for df in dfs
        ]
        if not dfs:
            return pd.DataFrame()
        return pd.concat(dfs).reset_index(drop=True)

    @sportsref.decorators.memoize
    def def_splits(self, year):
        """Returns a DataFrame of defensive team splits (i.e. opponent splits)
        for a season.

        :year: int representing the season.
        :returns: Pandas DataFrame of split data.
        """
        doc = self.get_year_doc('{}_opp_splits'.format(year))
        tables = doc('table.stats_table')
        dfs = [sportsref.utils.parse_table(table) for table in tables.items()]
        dfs = [
            df.assign(split=df.columns[0])
            .rename(columns={df.columns[0]: 'split_value'})
            for df in dfs
        ]
        if not dfs:
            return pd.DataFrame()
        return pd.concat(dfs).reset_index(drop=True)


from __future__ import division, print_function, unicode_literals
from builtins import object, str
from past.builtins import basestring
import ctypes
import threading
import multiprocessing
import re
import time

import pandas as pd
from pyquery import PyQuery as pq
import requests

import sportsref

# time between requests, in seconds
THROTTLE_DELAY = 0.5

# variables used to throttle requests across processes
throttle_thread_lock = threading.Lock()
throttle_process_lock = multiprocessing.Lock()
last_request_time = multiprocessing.Value(ctypes.c_longdouble, time.time() - 10 * THROTTLE_DELAY)

@sportsref.decorators.cache
def get_html(url):
    """Gets the HTML for the given URL using a GET request.

    :url: the absolute URL of the desired page.
    :returns: a string of HTML.
    """
    global last_request_time
    with throttle_process_lock:
        with throttle_thread_lock:
            # sleep until THROTTLE_DELAY secs have passed since last request
            wait_left = THROTTLE_DELAY - (time.time() - last_request_time.value)
            if wait_left > 0:
                time.sleep(wait_left)

            # make request
            response = requests.get(url)

            # update last request time for throttling
            last_request_time.value = time.time()

    # raise ValueError on 4xx status code, get rid of comments, and return
    if 400 <= response.status_code < 500:
        raise ValueError(
            'Status Code {} received fetching URL "{}"'
            .format(response.status_code, url)
        )
    html = response.text
    html = html.replace('<!--', '').replace('-->', '')

    return html

def parse_table(table, flatten=True, footer=False):
    """Parses a table from sports-reference sites into a pandas dataframe.

    :param table: the PyQuery object representing the HTML table
    :param flatten: if True, flattens relative URLs to IDs. otherwise, leaves
        all fields as text without cleaning.
    :param footer: If True, returns the summary/footer of the page. Recommended
        to use this with flatten=False. Defaults to False.
    :returns: pd.DataFrame
    """
    if not len(table):
        return pd.DataFrame()

    # get columns
    columns = [c.attrib['data-stat']
               for c in table('thead tr:not([class]) th[data-stat]')]

    # get data
    rows = list(table('tbody tr' if not footer else 'tfoot tr')
                .not_('.thead, .stat_total, .stat_average').items())
    data = [
        [flatten_links(td) if flatten else td.text()
         for td in row.items('th,td')]
        for row in rows
    ]

    # make DataFrame
    df = pd.DataFrame(data, columns=columns, dtype='float')

    # add has_class columns
    allClasses = set(
        cls
        for row in rows
        if row.attr['class']
        for cls in row.attr['class'].split()
    )
    for cls in allClasses:
        df['has_class_' + cls] = [
            bool(row.attr['class'] and
                 cls in row.attr['class'].split())
            for row in rows
        ]

    # cleaning the DataFrame

    df.drop(['ranker', 'Xxx', 'Yyy', 'Zzz'],
            axis=1, inplace=True, errors='ignore')

    # year_id -> year (as int)
    if 'year_id' in df.columns:
        df.rename(columns={'year_id': 'year'}, inplace=True)
        if flatten:
            df.year = df.year.fillna(method='ffill')
            df['year'] = df.year.map(lambda s: str(s)[:4]).astype(int)

    # pos -> position
    if 'pos' in df.columns:
        df.rename(columns={'pos': 'position'}, inplace=True)

    # boxscore_word, game_date -> boxscore_id and separate into Y, M, D columns
    for bs_id_col in ('boxscore_word', 'game_date', 'box_score_text'):
        if bs_id_col in df.columns:
            df.rename(columns={bs_id_col: 'boxscore_id'}, inplace=True)
            break

    # ignore *, +, and other characters used to note things
    df.replace(re.compile(r'[\*\+\u2605]', re.U), '', inplace=True)
    for col in df.columns:
        if hasattr(df[col], 'str'):
            df[col] = df[col].str.strip()

    # player -> player_id and/or player_name
    if 'player' in df.columns:
        if flatten:
            df.rename(columns={'player': 'player_id'}, inplace=True)
            # when flattening, keep a column for names
            player_names = parse_table(table, flatten=False)['player_name']
            df['player_name'] = player_names
        else:
            df.rename(columns={'player': 'player_name'}, inplace=True)

    # team, team_name -> team_id
    for team_col in ('team', 'team_name'):
        if team_col in df.columns:
            # first, get rid of faulty rows
            df = df.loc[~df[team_col].isin(['XXX'])]
            if flatten:
                df.rename(columns={team_col: 'team_id'}, inplace=True)

    # season -> int
    if 'season' in df.columns and flatten:
        df['season'] = df['season'].astype(int)

    # handle date_game columns (different types)
    if 'date_game' in df.columns and flatten:
        date_re = r'month=(?P<month>\d+)&day=(?P<day>\d+)&year=(?P<year>\d+)'
        date_df = df['date_game'].str.extract(date_re, expand=True)
        if date_df.notnull().all(axis=1).any():
            df = pd.concat((df, date_df), axis=1)
        else:
            df.rename(columns={'date_game': 'boxscore_id'}, inplace=True)

    # game_location -> is_home
    if 'game_location' in df.columns and flatten:
        df['game_location'] = df['game_location'].isnull()
        df.rename(columns={'game_location': 'is_home'}, inplace=True)

    # mp: (min:sec) -> float(min + sec / 60), notes -> NaN, new column
    if 'mp' in df.columns and df.dtypes['mp'] == object and flatten:
        mp_df = df['mp'].str.extract(
            r'(?P<m>\d+):(?P<s>\d+)', expand=True).astype(float)
        no_match = mp_df.isnull().all(axis=1)
        if no_match.any():
            df.loc[no_match, 'note'] = df.loc[no_match, 'mp']
        df['mp'] = mp_df['m'] + mp_df['s'] / 60

    # converts number-y things to floats
    def convert_to_float(val):
        # percentages: (number%) -> float(number * 0.01)
        m = re.search(r'([-\.\d]+)\%',
                      val if isinstance(val, basestring) else str(val), re.U)
        try:
            if m:
                return float(m.group(1)) / 100 if m else val
            if m:
                return int(m.group(1)) + int(m.group(2)) / 60
        except ValueError:
            return val
        # salaries: $ABC,DEF,GHI -> float(ABCDEFGHI)
        m = re.search(r'\$[\d,]+',
                      val if isinstance(val, basestring) else str(val), re.U)
        try:
            if m:
                return float(re.sub(r'\$|,', '', val))
        except Exception:
            return val
        # generally try to coerce to float, unless it's an int or bool
        try:
            if isinstance(val, (int, bool)):
                return val
            else:
                return float(val)
        except Exception:
            return val

    if flatten:
        df = df.applymap(convert_to_float)

    df = df.loc[df.astype(bool).any(axis=1)]

    return df

def parse_info_table(table):
    """Parses an info table, like the "Game Info" table or the "Officials"
    table on the PFR Boxscore page. Keys are lower case and have spaces/special
    characters converted to underscores.

    :table: PyQuery object representing the HTML table.
    :returns: A dictionary representing the information.
    """
    ret = {}
    for tr in list(table('tr').not_('.thead').items()):
        th, td = list(tr('th, td').items())
        key = th.text().lower()
        key = re.sub(r'\W', '_', key)
        val = sportsref.utils.flatten_links(td)
        ret[key] = val
    return ret

def parse_awards_table(table):
    """Parses an awards table, like the "Pro Bowls" table on a PFR player page.

    :table: PyQuery object representing the HTML table.
    :returns: A list of the entries in the table, with flattened links.
    """
    return [flatten_links(tr) for tr in list(table('tr').items())]

def flatten_links(td, _recurse=False):
    """Flattens relative URLs within text of a table cell to IDs and returns
    the result.

    :td: the PyQuery object for the HTML to convert
    :returns: the string with the links flattened to IDs
    """

    # helper function to flatten individual strings/links
    def _flatten_node(c):
        if isinstance(c, basestring):
            return c.strip()
        elif 'href' in c.attrib:
            c_id = rel_url_to_id(c.attrib['href'])
            return c_id if c_id else c.text_content().strip()
        else:
            return flatten_links(pq(c), _recurse=True)

    # if there's no text, just return None
    if td is None or not td.text():
        return '' if _recurse else None

    td.remove('span.note')
    return ''.join(_flatten_node(c) for c in td.contents())

@sportsref.decorators.memoize
def rel_url_to_id(url):
    """Converts a relative URL to a unique ID.

    Here, 'ID' refers generally to the unique ID for a given 'type' that a
    given datum has. For example, 'BradTo00' is Tom Brady's player ID - this
    corresponds to his relative URL, '/players/B/BradTo00.htm'. Similarly,
    '201409070dal' refers to the boxscore of the SF @ DAL game on 09/07/14.

    Supported types:
    * player/...
    * boxscores/...
    * teams/...
    * years/...
    * leagues/...
    * awards/...
    * coaches/...
    * officials/...
    * schools/...
    * schools/high_schools.cgi?id=...

    :returns: ID associated with the given relative URL.
    """
    yearRegex = r'.*/years/(\d{4}).*|.*/gamelog/(\d{4}).*'
    playerRegex = r'.*/players/(?:\w/)?(.+?)(?:/|\.html?)'
    boxscoresRegex = r'.*/boxscores/(.+?)\.html?'
    teamRegex = r'.*/teams/(\w{3})/.*'
    coachRegex = r'.*/coaches/(.+?)\.html?'
    stadiumRegex = r'.*/stadiums/(.+?)\.html?'
    refRegex = r'.*/officials/(.+?r)\.html?'
    collegeRegex = r'.*/schools/(\S+?)/.*|.*college=([^&]+)'
    hsRegex = r'.*/schools/high_schools\.cgi\?id=([^\&]{8})'
    bsDateRegex = r'.*/boxscores/index\.f?cgi\?(month=\d+&day=\d+&year=\d+)'
    leagueRegex = r'.*/leagues/(.*_\d{4}).*'
    awardRegex = r'.*/awards/(.+)\.htm'

    regexes = [
        yearRegex,
        playerRegex,
        boxscoresRegex,
        teamRegex,
        coachRegex,
        stadiumRegex,
        refRegex,
        collegeRegex,
        hsRegex,
        bsDateRegex,
        leagueRegex,
        awardRegex,
    ]

    for regex in regexes:
        match = re.match(regex, url, re.I)
        if match:
            return [_f for _f in match.groups() if _f][0]

    # things we don't want to match but don't want to print a WARNING
    if any(
        url.startswith(s) for s in
        (
            '/play-index/',
        )
    ):
        return url

    print('WARNING. NO MATCH WAS FOUND FOR "{}"'.format(url))
    return url


from __future__ import print_function
from future import standard_library
standard_library.install_aliases()
from builtins import map, zip
from past.builtins import basestring
import collections
import json
import os
import time
import urllib.parse

from pyquery import PyQuery as pq

from ... import decorators, utils

PSF_URL = ('http://www.pro-football-reference.com/'
           'play-index/psl_finder.cgi')

PSF_CONSTANTS_FILENAME = 'PSFConstants.json'

def PlayerSeasonFinder(**kwargs):
    """ Docstring will be filled in by __init__.py """

    if 'offset' not in kwargs:
        kwargs['offset'] = 0

    playerSeasons = []
    while True:
        querystring = _kwargs_to_qs(**kwargs)
        url = '{}?{}'.format(PSF_URL, querystring)
        if kwargs.get('verbose', False):
            print(url)
        html = utils.get_html(url)
        doc = pq(html)
        table = doc('table#results')
        df = utils.parse_table(table)
        if df.empty:
            break

        thisSeason = list(zip(df.player_id, df.year))
        playerSeasons.extend(thisSeason)

        if doc('*:contains("Next Page")'):
            kwargs['offset'] += 100
        else:
            break

    return playerSeasons

def _kwargs_to_qs(**kwargs):
    """Converts kwargs given to PSF to a querystring.

    :returns: the querystring.
    """
    # start with defaults
    inpOptDef = inputs_options_defaults()
    opts = {
        name: dct['value']
        for name, dct in inpOptDef.items()
    }

    # clean up keys and values
    for k, v in kwargs.items():
        del kwargs[k]
        # bool => 'Y'|'N'
        if isinstance(v, bool):
            kwargs[k] = 'Y' if v else 'N'
        # tm, team => team_id
        elif k.lower() in ('tm', 'team'):
            kwargs['team_id'] = v
        # yr, year, yrs, years => year_min, year_max
        elif k.lower() in ('yr', 'year', 'yrs', 'years'):
            if isinstance(v, collections.Iterable):
                lst = list(v)
                kwargs['year_min'] = min(lst)
                kwargs['year_max'] = max(lst)
            elif isinstance(v, basestring):
                v = list(map(int, v.split(',')))
                kwargs['year_min'] = min(v)
                kwargs['year_max'] = max(v)
            else:
                kwargs['year_min'] = v
                kwargs['year_max'] = v
        # pos, position, positions => pos[]
        elif k.lower() in ('pos', 'position', 'positions'):
            if isinstance(v, basestring):
                v = v.split(',')
            elif not isinstance(v, collections.Iterable):
                v = [v]
            kwargs['pos[]'] = v
        # draft_pos, ... => draft_pos[]
        elif k.lower() in (
            'draft_pos', 'draftpos', 'draftposition', 'draftpositions',
            'draft_position', 'draft_positions'
        ):
            if isinstance(v, basestring):
                v = v.split(',')
            elif not isinstance(v, collections.Iterable):
                v = [v]
            kwargs['draft_pos[]'] = v
        # if not one of these cases, put it back in kwargs
        else:
            kwargs[k] = v

    # update based on kwargs
    for k, v in kwargs.items():
        # if overwriting a default, overwrite it (with a list so the
        # opts -> querystring list comp works)
        if k in opts or k in ('pos[]', 'draft_pos[]'):
            # if multiple values separated by commas, split em
            if isinstance(v, basestring):
                v = v.split(',')
            # otherwise, make sure it's a list
            elif not isinstance(v, collections.Iterable):
                v = [v]
            # then, add list of values to the querystring dict *opts*
            opts[k] = v
        if 'draft' in k:
            opts['draft'] = [1]

    opts['request'] = [1]
    opts['offset'] = [kwargs.get('offset', 0)]

    qs = '&'.join(
        '{}={}'.format(urllib.parse.quote_plus(name), val)
        for name, vals in sorted(opts.items()) for val in vals
    )

    return qs

@decorators.switch_to_dir(os.path.dirname(os.path.realpath(__file__)))
def inputs_options_defaults():
    """Handles scraping options for player-season finder form.

    :returns: {'name1': {'value': val, 'options': [opt1, ...] }, ... }
    """
    # set time variables
    if os.path.isfile(PSF_CONSTANTS_FILENAME):
        modtime = int(os.path.getmtime(PSF_CONSTANTS_FILENAME))
        curtime = int(time.time())
    # if file found and it's been <= a week
    if (os.path.isfile(PSF_CONSTANTS_FILENAME)
            and curtime - modtime <= 7 * 24 * 60 * 60):

        # just read the dict from cached file
        with open(PSF_CONSTANTS_FILENAME, 'r') as const_f:
            def_dict = json.load(const_f)

    # otherwise, we must regenerate the dict and rewrite it
    else:

        print('Regenerating PSFConstants file')

        html = utils.get_html(PSF_URL)
        doc = pq(html)

        def_dict = {}
        # start with input elements
        for inp in doc('form#psl_finder input[name]'):
            name = inp.attrib['name']
            # add blank dict if not present
            if name not in def_dict:
                def_dict[name] = {
                    'value': set(),
                    'options': set(),
                    'type': inp.attrib['type']
                }

            # handle checkboxes and radio buttons
            if inp.attrib['type'] in ('checkbox', 'radio'):
                # deal with default value
                if 'checked' in inp.attrib:
                    def_dict[name]['value'].add(inp.attrib['value'])
                # add to options
                def_dict[name]['options'].add(inp.attrib['value'])
            # handle other types of inputs (only other type is hidden?)
            else:
                def_dict[name]['value'].add(inp.attrib.get('value', ''))

        # deal with dropdowns (select elements)
        for sel in doc.items('form#psl_finder select[name]'):
            name = sel.attr['name']
            # add blank dict if not present
            if name not in def_dict:
                def_dict[name] = {
                    'value': set(),
                    'options': set(),
                    'type': 'select'
                }

            # deal with default value
            defaultOpt = sel('option[selected]')
            if len(defaultOpt):
                defaultOpt = defaultOpt[0]
                def_dict[name]['value'].add(defaultOpt.attrib.get('value', ''))
            else:
                def_dict[name]['value'].add(
                    sel('option')[0].attrib.get('value', '')
                )

            # deal with options
            def_dict[name]['options'] = {
                opt.attrib['value'] for opt in sel('option')
                if opt.attrib.get('value')
            }

        def_dict.pop('request', None)
        def_dict.pop('use_favorites', None)

        with open(PSF_CONSTANTS_FILENAME, 'w+') as f:
            for k in def_dict:
                try:
                    def_dict[k]['value'] = sorted(
                        list(def_dict[k]['value']), key=int
                    )
                    def_dict[k]['options'] = sorted(
                        list(def_dict[k]['options']), key=int
                    )
                except Exception:
                    def_dict[k]['value'] = sorted(list(def_dict[k]['value']))
                    def_dict[k]['options'] = sorted(
                        list(def_dict[k]['options'])
                    )
            json.dump(def_dict, f)

    return def_dict


# Copyright (C) 2016 G. H. Collin (ghcollin)
#
# This software may be modified and distributed under the terms
# of the MIT license.  See the LICENSE.txt file for details.

import numpy as np
import multiprocessing
import threading

__author__ = "G. H. Collin"
__version__ = "1.1.1"

class QueueClosed:
    """Signals that the queue has closed and no more elements will be added."""
    pass

class BarrierImpl:
    """Actual implementation of a barrier."""

    def __init__(self, n_procs):
        """
        Create a barrier that waits for n_procs processes.

        :param n_procs: The number of processes to wait for.
        """
        self.n_procs = n_procs
        self.count = multiprocessing.Value('i', 0, lock=False)
        self.cvar = multiprocessing.Condition()

    def wait(self):
        """Wait until all processes have reached the barrier."""
        with self.cvar:
            self.count.value += 1
            self.cvar.notify_all()
            while self.count.value < self.n_procs:
                self.cvar.wait()

    def reset(self):
        """Re-set the barrier so that it can be used again."""
        with self.cvar:
            self.count.value = 0

class Barrier:
    """Self re-setting synchronisation barrier. Resets after all processes have passed."""

    def __init__(self, n_procs):
        """
        Create a self re-setting barrier that waits for n_procs processes.

        :param n_procs: The number of processes to wait for.
        """
        self.barrier_A, self.barrier_B = BarrierImpl(n_procs), BarrierImpl(n_procs)

    def wait(self):
        """Wait until all processes have reached the barrier."""
        self.barrier_A.wait()
        # The current barrier (barrier_A) is switched with the reserve barrier.
        # This is because the current barrier cannot be safely reset until the reserve barrier has been passed.
        self.barrier_A, self.barrier_B = self.barrier_B, self.barrier_A
        self.barrier_A.reset()

class OrderedBarrier:
    """Synchronization barrier that allows processes through one at a time using an index based ordering."""

    class Guard:
        """with statement guard for ordered barrier"""
        def __init__(self, sync, index, next_index):
            """
            The guard waits on entry until the barrier is ready to let the process identified by index to pass.
            Once the guard exits, it increments the shared counter by the skip value and notifies waiting processes.
            :param sync: The parent OrderedBarrier object.
            :param index: The order to wait for.
            :param next_index: After exiting, release the process waiting on next_index.
            """
            self.sync = sync
            self.index = index
            self.next_index = next_index

        def __enter__(self):
            with self.sync.cvar:
                # Wait until index equals the internal shared counter.
                while self.sync.sval.value != self.index:
                    self.sync.cvar.wait()

        def __exit__(self, exc_type, exc_val, exc_tb):
            with self.sync.cvar:
                # Increase the shared counter by the skip value.
                self.sync.sval.value = self.next_index
                # Notifying the remaining processes allows the process with index = (this index + skip) to pass
                self.sync.cvar.notify_all()

    def __init__(self):
        """
        Create an ordered barrier. When processes wait on this barrier, they are let through one at a time based
        on the provided index. The first process to be let through should provide an index of zero. Each subsequent
        process to be let through should provide an index equal to the current value of the internal counter.
        """
        import multiprocessing.sharedctypes
        self.cvar = multiprocessing.Condition()
        self.sval = multiprocessing.sharedctypes.RawValue('L')
        self.sval.value = 0

    def wait(self, index, next_index=None):
        """
        Block until it is the turn indicated by index.
        :param index:
        :param next_index: Set the index to this value after finishing. Releases the process waiting on next_index.
            Defaults to incrementing index by 1.
        :return:
        """
        return OrderedBarrier.Guard(self, index, index+1 if next_index is None else next_index)

class GuardSynchronizer:
    """Synchronises access to the entering an exiting of a resource using ordered barriers."""

    class Guard:
        """Compositional guard between the provided resource and the internal ordered barriers."""
        def __init__(self, sync, inner_guard, index, next_index):
            """
            Creates a guard that will wait on the first barrier to enter the resource. Then wait on the second
            barrier to exit the resource.
            :param sync: The parent GuardSynchronizer object.
            :param inner_guard: The resource to control access to.
            :param index: The order to wait for.
            """
            self.sync = sync
            self.index = index, next_index
            self.inner_guard = inner_guard

        def __enter__(self):
            with self.sync.barrier_in.wait(*self.index):
                rval = self.inner_guard.__enter__()
            return rval

        def __exit__(self, exc_type, exc_val, exc_tb):
            with self.sync.barrier_out.wait(*self.index):
                self.inner_guard.__exit__(exc_type, exc_val, exc_tb)

    def __init__(self):
        """
        Create a synchronizer that consists of two ordered barriers.
        """
        self.barrier_in = OrderedBarrier()
        self.barrier_out = OrderedBarrier()

    def do(self, guard, index, next_index):
        """
        Create a guard that requires the resource guard to be entered and exited based on the order provided by index.
        :param guard: The context manager for the resource.
        :param index: The order to wait for.
        :param next_index: The next index to release.
        :return:
        """
        return GuardSynchronizer.Guard(self, guard, index, next_index)

class SafeQueue:
    """multiprocessing.Queue serialises python objects and stuffs them into a Pipe object.
    This serialisation happens in a background thread, and it not tied to the put() call.
    As such, no guarantee can be made about the order in which objects are put in the queue during threaded access.
    This poses a problem for the ordered mode, as it requires this guarantee.
    This class implements a very simple bounded queue that can guarantee ordering of inserted items."""
    def __init__(self, size):
        # The size of the queue is increased by one to give space for a QueueClosed signal.
        size += 1
        import multiprocessing.sharedctypes
        # The condition variable is used to both lock access to the internal resources and signal new items are ready.
        self.cvar = multiprocessing.Condition()
        # A shared array is used to store items in the queue
        sary = multiprocessing.sharedctypes.RawArray('b', 8*size)
        self.vals = np.frombuffer(sary, dtype=np.int64, count=size)
        self.vals[:] = -1
        # tail is the next item to be read from the queue
        self.tail = multiprocessing.sharedctypes.RawValue('l', 0)
        # size is the current number of items in the queue. head = tail + size
        self.size = multiprocessing.sharedctypes.RawValue('l', 0)

    def put(self, v):
        """
        Put an unsigned integer into the queue. This method always assumes that there is space in the queue.
        ( In the circular buffer, this is guaranteed by the implementation )
        :param v: The item to insert. Must be >= 0, as -2 is used to signal a queue close.
        :return:
        """
        if v is QueueClosed:
            v = -2
        else:
            assert(v >= 0)
        with self.cvar:
            assert(self.size.value < len(self.vals))

            head = (self.tail.value + self.size.value) % len(self.vals)
            self.vals[head] = v
            self.size.value += 1
            self.cvar.notify()

    def get(self):
        """
        Fetch the next item in the queue. Blocks until an item is ready.
        :return: The next unsigned integer in the queue.
        """
        with self.cvar:
            while True:
                if self.size.value > 0:
                    rval = self.vals[self.tail.value]
                    self.tail.value = (self.tail.value + 1) % len(self.vals)
                    self.size.value -= 1
                    if rval == -2:
                        return QueueClosed
                    assert(rval >= 0)
                    return rval
                self.cvar.wait()

class SharedCircBuf:
    """A circular buffer for numpy arrays that uses shared memory for inter-process communication."""
    def __init__(self, queue_size, ary_template):
        """
        Create the circular buffer. An array template must be passed to determine the size of the buffer elements.

        :param queue_size: Number of arrays to use as buffer elements.
        :param ary_template: Buffer elements match this array in shape and data-type.
        """
        import multiprocessing.sharedctypes
        # The buffer uses two queues to synchonise access to the buffer.
        # Element indices are put and fetched from these queues.
        # Elements that are ready to be written to go into the write_queue.
        # Elements that are ready to be read go into the read_queue.
        # This is essentially a token passing process. Tokens are taken out of queues and are not put back until
        # operations are complete.
        self.read_queue = SafeQueue(queue_size)
        self.write_queue = SafeQueue(queue_size)

        elem_n_bytes = ary_template.nbytes
        elem_dtype = ary_template.dtype
        elem_size = ary_template.size
        elem_shape = ary_template.shape
        self.arys = []
        for i in range(queue_size):
            sarray = multiprocessing.sharedctypes.RawArray('b', elem_n_bytes)
            # Elements are numpy arrays that point into allocated shared memory.
            self.arys.append(np.frombuffer(sarray, dtype=elem_dtype, count=elem_size).reshape(elem_shape))
            # The queue of elements ready to be written to is initially populated with all elements.
            self.write_queue.put(i)

    class Guard:
        """with statement guard object for synchronisation of access to the buffer."""
        def __init__(self, out_queue, arys, idx_op):
            """
            The guard object returns ary, and once the guard ends the value of idx is put into queue.
            Used to put an element index representing a token back into the buffer queues once operations on the
            element are complete.

            :param queue: The queue to be populated when the guard ends.
            :param idx: The value to put into the queue.
            :param ary: Value to return in the with statement.
            """
            self.out_queue = out_queue
            self.arys = arys
            self.idx_op = idx_op

        def __enter__(self):
            self.idx = self.idx_op()

            return self.arys[self.idx]

        def __exit__(self, *args):
            self.out_queue.put(self.idx)

    class Closed(Exception):
        pass

    def put(self, in_ary):
        """
        Convenience method to put in_ary into the buffer.
        Blocks until there is room to write into the buffer.

        :param in_ary: The array to place into the buffer.
        :return:
        """
        with self.put_direct() as ary:
            ary[:] = in_ary

    def __put_idx(self):
        write_idx = self.write_queue.get()

        if write_idx is QueueClosed:
            self.write_queue.put(QueueClosed)
            raise self.Closed("Queue closed")

        return write_idx

    def put_direct(self):
        """
        Allows direct access to the buffer element.
        Blocks until there is room to write into the buffer.

        :return: A guard object that returns the buffer element.
        """

        # Once the guard is released, write_idx will be placed into read_queue.
        return self.Guard(self.read_queue, self.arys, self.__put_idx)

    def get(self):
        """
        Convenience method to get a copy of an array in the buffer.
        Blocks until there is data to be read.

        :return: A copy of the next available array.
        """
        with self.get_direct() as ary:
            result = ary.copy()
        return result

    def __get_idx(self):
        read_idx = self.read_queue.get()

        if read_idx is QueueClosed:
            self.read_queue.put(QueueClosed)
            return QueueClosed

        return read_idx

    def get_direct(self):
        """
        Allows direct access to the buffer element.
        Blocks until there is data that can be read.

        :return: A guard object that returns the buffer element.
        """

        read_idx = self.__get_idx()

        if read_idx is QueueClosed:
            return QueueClosed

        # Once the guard is released, read_idx will be placed into write_queue.
        return self.Guard(self.write_queue, self.arys, lambda: read_idx)

    def close(self):
        """Close the queue, signalling that no more data can be put into the queue."""
        self.read_queue.put(QueueClosed)
        self.write_queue.put(QueueClosed)

def _Streamer__read_process(self, path, read_size, cbuf, stop, barrier, cyclic, offset, read_skip, sync):
    """
    Main function for the processes that read from the HDF5 file.

    :param self: A reference to the streamer object that created these processes.
    :param path: The HDF5 path to the node to be read from.
    :param read_size: The length of the block along the outer dimension to read.
    :param cbuf: The circular buffer to place read elements into.
    :param stop: The Event that signals the process to stop reading.
    :param barrier: The Barrier that synchonises read cycles.
    :param cyclic: True if the process should read cyclically.
    :param offset: Offset into the dataset that this process should start reading at.
    :param read_skip: How many element to skip on each iteration.
    :param sync: GuardSynchonizer to order writes to the buffer.
    :return: Nothing
    """
    # Multi-process access to HDF5 seems to behave better there are no top level imports of PyTables.
    import tables as tb
    h5_file = tb.open_file(self.filename, 'r', **self.h5_kw_args)
    ary = h5_file.get_node(path)

    i = offset
    while not stop.is_set():
        vals = ary[i:i + read_size]
        # If the read goes off the end of the dataset, then wrap to the start.
        if i + read_size > len(ary):
            vals = np.concatenate([vals, ary[0:read_size - len(vals)]])

        if sync is None:
            # If no ordering is requested, then just write to the next available space in the buffer.
            with cbuf.put_direct() as put_ary:
                put_ary[:] = vals
        else:
            # Otherwise, use the sync object to ensure that writes occur in the order provided by i.
            # So i = 0 will write first, then i = block_size, then i = 2*block_size, etc...
            # The sync object has two ordered barriers so that acquisition and release of the buffer spaces
            # are synchronized in order, but the actual writing to the buffer can happen simultaneously.
            # If only one barrier were used, writing to the buffer would be linearised.
            with sync.do(cbuf.put_direct(), i, (i+read_size) % len(ary)) as put_ary:
                put_ary[:] = vals

        i += read_skip
        if cyclic:
            # If the next iteration is past the end of the dataset, wrap it around.
            if i >= len(ary):
                i %= len(ary)
                barrier.wait()
        else:
            # But if cyclic mode is disabled, break the loop as the work is now done.
            if i + read_size > len(ary):
                break

class Streamer:
    """Provides methods for streaming data out of HDF5 files."""

    def __init__(self, filename, **kw_args):
        """
        An object for streaming data from filename. Additional key words arguments can be passed to the constructor.
        These arguments will be passed on to the open_file function from PyTables.

        :param filename: The HDF5 file to read from.
        :param kw_args: Additional options for opening the HDF5 file.
        """
        self.filename = filename
        self.h5_kw_args = kw_args

    def __get_batch(self, path, length, last=False):
        """
        Get a block of data from the node at path.

        :param path: The path to the node to read from.
        :param length: The length along the outer dimension to read.
        :param last: True if the remainder elements should be read.
        :return: A copy of the requested block of data as a numpy array.
        """
        import tables
        h5_file = tables.open_file(self.filename, 'r')
        h5_node = h5_file.get_node(path)

        if len(h5_node) == 0:
            raise Exception("Cannot read from empty dataset.")

        # If the length isn't specified, then fall back to default values.
        if length is None:
            chunkshape = h5_node.chunkshape
            # If the array isn't chunked, then try to make the block close to 128KB.
            if chunkshape is None:
                default_length = 128*2**10//h5_node[0].nbytes  # Divides by one row of the dataset.
                length = min(h5_node.shape[0], default_length)
            # If it is chunked, then use the chunkshape for best performance.
            else:
                length = chunkshape[0]

        if last:
            example = h5_node[length*(len(h5_node)//length):].copy()
        else:
            example = h5_node[:length].copy()

        h5_file.close()
        return example

    def get_remainder(self, path, block_size):
        """
        Get the remainder elements. These elements will not be read in the direct queue access cyclic=False mode.

        :param path: The HDF5 path to the dataset to be read.
        :param block_size: The block size is used to calculate which elements will remain.
        :return: A copy of the remainder elements as a numpy array.
        """
        return self.__get_batch(path, length=block_size, last=True)

    class Queue:
        """Abstract queue that is backed by the internal circular buffer."""
        def __init__(self, cbuf, stop, block_size):
            """
            A queue to provide a clean access API to the user.
            :param cbuf: The internal circular buffer.
            :param stop: The stop Event.
            :param block_size: The block size.
            """
            self.cbuf = cbuf
            self.stop = stop
            self.block_size = block_size

        def get(self):
            """
            Get the next element from the queue of data. This method returns a guard object that shnchonises access
            to the underlying buffer. The guard, when placed in a with statement, returns a reference to the next
            available element in the buffer.
            This method blocks until data is available.

            :return: A guard object that returns a reference to the element.
            """
            return self.cbuf.get_direct()

        def iter(self):
            """
            Convenience method for easy iteration over elements in the queue.
            Each iteration of the iterator will block until an element is available to be read.

            :return: An iterator for the queue.
            """
            while True:
                guard = self.get()
                if guard is QueueClosed:
                    break
                else:
                    yield guard

        def close(self):
            """Signals to the background processes to stop, and closes the queue."""
            self.stop.set()

        def __del__(self):
            self.close()

    def get_queue(self, path, n_procs=4, read_ahead=None, cyclic=False, block_size=None, ordered=False):
        """
        Get a queue that allows direct access to the internal buffer. If the dataset to be read is chunked, the
        block_size should be a multiple of the chunk size to maximise performance. In this case it is best to leave it
        to the default. When cyclic=False, and block_size does not divide the dataset evenly, the remainder elements
        will not be returned by the queue. When cyclic=True, the remainder elements will be part of a block that wraps
        around the end and includes element from the beginning of the dataset. By default, blocks are returned in the
        order in which they become available. The ordered option will force blocks to be returned in on-disk order.

        :param path: The HDF5 path to the dataset that should be read.
        :param n_procs: The number of background processes used to read the datset in parallel.
        :param read_ahead: The number of blocks to allocate in the internal buffer.
        :param cyclic: True if the queue should wrap at the end of the dataset.
        :param block_size: The size along the outer dimension of the blocks to be read. Defaults to a multiple of
            the chunk size, or to a 128KB sized block if the dataset is not chunked.
        :param ordered: Force the reader return data in on-disk order. May result in performance penalty.
        :return: A queue object that allows access to the internal buffer.
        """
        # Get a block_size length of elements from the dataset to serve as a template for creating the buffer.
        # If block_size=None, then get_batch calculates an appropriate block size.
        example = self.__get_batch(path, block_size)
        block_size = example.shape[0]

        if read_ahead is None:
            # 2x No. of processes for writing, 1 extra for reading.
            read_ahead = 2*n_procs + 1

        cbuf = SharedCircBuf(read_ahead, example)
        stop = multiprocessing.Event()
        barrier = Barrier(n_procs)

        # If ordering has been requested, create a synchronizer.
        sync = GuardSynchronizer() if ordered else None

        procs = []
        for i in range(n_procs):
            # Each process is offset in the dataset by i*block_size
            # The skip length is set to n_procs*block_size so that no block is read by 2 processes.
            process = multiprocessing.Process(target=_Streamer__read_process, args=(
                self, path, block_size, cbuf, stop, barrier, cyclic,
                i * block_size, n_procs * block_size, sync
            ))
            process.daemon = True
            process.start()
            procs.append(process)

        # If the queue is not cyclic, then the cessation of reading data needs to be monitored.
        if not cyclic:

            # This closure defines a background thread that waits until all processes have finished.
            # At this point, all data from the dataset has been read, and the buffer is closed.
            def monitor():
                for p in procs:
                    p.join()
                cbuf.close()

            monitor_thread = threading.Thread(target=monitor)
            monitor_thread.daemon = True
            monitor_thread.start()

        return Streamer.Queue(cbuf, stop, block_size)

    def get_generator(self, path, *args, **kw_args):
        """
        Get a generator that allows convenient access to the streamed data.
        Elements from the dataset are returned from the generator one row at a time.
        Unlike the direct access queue, this generator also returns the remainder elements.
        Additional arguments are forwarded to get_queue.
        See the get_queue method for documentation of these parameters.

        :param path:
        :return: A generator that iterates over the rows in the dataset.
        """
        q = self.get_queue(path=path, *args, **kw_args)

        try:
            # This generator just implements a standard access pattern for the direct access queue.
            for guard in q.iter():
                with guard as batch:
                    batch_copy = batch.copy()

                for row in batch_copy:
                    yield row

            last_batch = self.get_remainder(path, q.block_size)
            for row in last_batch:
                yield row

        finally:
            q.close()