def schedule(self, kind='R'):
        """Returns a list of BoxScore IDs for every game in the season.
        Only needs to handle 'R' or 'P' options because decorator handles 'B'.

        :param kind: 'R' for regular season, 'P' for playoffs, 'B' for both.
            Defaults to 'R'.
        :returns: DataFrame of schedule information.
        :rtype: pd.DataFrame
        """
        kind = kind.upper()[0]
        dfs = []

        # get games from each month
        for month in ('october', 'november', 'december', 'january', 'february',
                      'march', 'april', 'may', 'june'):
            try:
                doc = self.get_sub_doc('games-{}'.format(month))
            except ValueError:
                continue
            table = doc('table#schedule')
            df = sportsref.utils.parse_table(table)
            dfs.append(df)
        df = pd.concat(dfs).reset_index(drop=True)

        # figure out how many regular season games
        try:
            sportsref.utils.get_html('{}/playoffs/NBA_{}.html'.format(
                sportsref.nba.BASE_URL, self.yr)
            )
            is_past_season = True
        except ValueError:
            is_past_season = False

        if is_past_season:
            team_per_game = self.team_stats_per_game()
            n_reg_games = int(team_per_game.g.sum() // 2)
        else:
            n_reg_games = len(df)

        # subset appropriately based on `kind`
        if kind == 'P':
            return df.iloc[n_reg_games:]
        else:
            return df.iloc[:n_reg_games]

Returns a list of BoxScore IDs for every game in the season . Only needs to handle R or P options because decorator handles B .


def standings(self):
        """Returns a DataFrame containing standings information."""
        doc = self.get_sub_doc('standings')

        east_table = doc('table#divs_standings_E')
        east_df = pd.DataFrame(sportsref.utils.parse_table(east_table))
        east_df.sort_values('wins', ascending=False, inplace=True)
        east_df['seed'] = range(1, len(east_df) + 1)
        east_df['conference'] = 'E'

        west_table = doc('table#divs_standings_W')
        west_df = sportsref.utils.parse_table(west_table)
        west_df.sort_values('wins', ascending=False, inplace=True)
        west_df['seed'] = range(1, len(west_df) + 1)
        west_df['conference'] = 'W'

        full_df = pd.concat([east_df, west_df], axis=0).reset_index(drop=True)
        full_df['team_id'] = full_df.team_id.str.extract(r'(\w+)\W*\(\d+\)', expand=False)
        full_df['gb'] = [gb if isinstance(gb, int) or isinstance(gb, float) else 0
                         for gb in full_df['gb']]
        full_df = full_df.drop('has_class_full_table', axis=1)

        expanded_table = doc('table#expanded_standings')
        expanded_df = sportsref.utils.parse_table(expanded_table)

        full_df = pd.merge(full_df, expanded_df, on='team_id')
        return full_df

def standings ( self ) : doc = self . get_sub_doc ( 'standings' ) east_table = doc ( 'table#divs_standings_E' ) east_df = pd . DataFrame ( sportsref . utils . parse_table ( east_table ) ) east_df . sort_values ( 'wins' , ascending = False , inplace = True ) east_df [ 'seed' ] = range ( 1 , len ( east_df ) + 1 ) east_df [ 'conference' ] = 'E' west_table = doc ( 'table#divs_standings_W' ) west_df = sportsref . utils . parse_table ( west_table ) west_df . sort_values ( 'wins' , ascending = False , inplace = True ) west_df [ 'seed' ] = range ( 1 , len ( west_df ) + 1 ) west_df [ 'conference' ] = 'W' full_df = pd . concat ( [ east_df , west_df ] , axis = 0 ) . reset_index ( drop = True ) full_df [ 'team_id' ] = full_df . team_id . str . extract ( r'(\w+)\W*\(\d+\)' , expand = False ) full_df [ 'gb' ] = [ gb if isinstance ( gb , int ) or isinstance ( gb , float ) else 0 for gb in full_df [ 'gb' ] ] full_df = full_df . drop ( 'has_class_full_table' , axis = 1 ) expanded_table = doc ( 'table#expanded_standings' ) expanded_df = sportsref . utils . parse_table ( expanded_table ) full_df = pd . merge ( full_df , expanded_df , on = 'team_id' ) return full_df

Returns a DataFrame containing standings information .


def _get_team_stats_table(self, selector):
        """Helper function for stats tables on season pages. Returns a
        DataFrame."""
        doc = self.get_main_doc()
        table = doc(selector)
        df = sportsref.utils.parse_table(table)
        df.set_index('team_id', inplace=True)
        return df

def _get_team_stats_table ( self , selector ) : doc = self . get_main_doc ( ) table = doc ( selector ) df = sportsref . utils . parse_table ( table ) df . set_index ( 'team_id' , inplace = True ) return df

Helper function for stats tables on season pages . Returns a DataFrame .


def roy_voting(self):
        """Returns a DataFrame containing information about ROY voting."""
        url = '{}/awards/awards_{}.html'.format(sportsref.nba.BASE_URL, self.yr)
        doc = pq(sportsref.utils.get_html(url))
        table = doc('table#roy')
        df = sportsref.utils.parse_table(table)
        return df

def roy_voting ( self ) : url = '{}/awards/awards_{}.html' . format ( sportsref . nba . BASE_URL , self . yr ) doc = pq ( sportsref . utils . get_html ( url ) ) table = doc ( 'table#roy' ) df = sportsref . utils . parse_table ( table ) return df

Returns a DataFrame containing information about ROY voting .


def linescore(self):
        """Returns the linescore for the game as a DataFrame."""
        doc = self.get_main_doc()
        table = doc('table#line_score')

        columns = [th.text() for th in table('tr.thead').items('th')]
        columns[0] = 'team_id'

        data = [
            [sportsref.utils.flatten_links(td) for td in tr('td').items()]
            for tr in table('tr.thead').next_all('tr').items()
        ]

        return pd.DataFrame(data, index=['away', 'home'],
                            columns=columns, dtype='float')

def linescore ( self ) : doc = self . get_main_doc ( ) table = doc ( 'table#line_score' ) columns = [ th . text ( ) for th in table ( 'tr.thead' ) . items ( 'th' ) ] columns [ 0 ] = 'team_id' data = [ [ sportsref . utils . flatten_links ( td ) for td in tr ( 'td' ) . items ( ) ] for tr in table ( 'tr.thead' ) . next_all ( 'tr' ) . items ( ) ] return pd . DataFrame ( data , index = [ 'away' , 'home' ] , columns = columns , dtype = 'float' )

Returns the linescore for the game as a DataFrame .


def season(self):
        """
        Returns the year ID of the season in which this game took place.

        :returns: An int representing the year of the season.
        """
        d = self.date()
        if d.month >= 9:
            return d.year + 1
        else:
            return d.year

def season ( self ) : d = self . date ( ) if d . month >= 9 : return d . year + 1 else : return d . year

Returns the year ID of the season in which this game took place .


def _get_player_stats(self, table_id_fmt):
        """Returns a DataFrame of player stats from the game (either basic or
        advanced, depending on the argument.

        :param table_id_fmt: Format string for str.format with a placeholder
            for the team ID (e.g. 'box_{}_basic')
        :returns: DataFrame of player stats
        """

        # get data
        doc = self.get_main_doc()
        tms = self.away(), self.home()
        tm_ids = [table_id_fmt.format(tm) for tm in tms]
        tables = [doc('table#{}'.format(tm_id).lower()) for tm_id in tm_ids]
        dfs = [sportsref.utils.parse_table(table) for table in tables]

        # clean data and add features
        for i, (tm, df) in enumerate(zip(tms, dfs)):
            no_time = df['mp'] == 0
            stat_cols = [col for col, dtype in df.dtypes.items()
                         if dtype != 'object']
            df.loc[no_time, stat_cols] = 0
            df['team_id'] = tm
            df['is_home'] = i == 1
            df['is_starter'] = [p < 5 for p in range(df.shape[0])]
            df.drop_duplicates(subset='player_id', keep='first', inplace=True)

        return pd.concat(dfs)

def
_get_player_stats
(
self
,
table_id_fmt
)
:
# get data
doc
=
self
.
get_main_doc
(
)
tms
=
self
.
away
(
)
,
self
.
home
(
)
tm_ids
=
[
table_id_fmt
.
format
(
tm
)
for
tm
in
tms
]
tables
=
[
doc
(
'table#{}'
.
format
(
tm_id
)
.
lower
(
)
)
for
tm_id
in
tm_ids
]
dfs
=
[
sportsref
.
utils
.
parse_table
(
table
)
for
table
in
tables
]
# clean data and add features
for
i
,
(
tm
,
df
)
in
enumerate
(
zip
(
tms
,
dfs
)
)
:
no_time
=
df
[
'mp'
]
==
0
stat_cols
=
[
col
for
col
,
dtype
in
df
.
dtypes
.
items
(
)
if
dtype
!=
'object'
]
df
.
loc
[
no_time
,
stat_cols
]
=
0
df
[
'team_id'
]
=
tm
df
[
'is_home'
]
=
i
==
1
df
[
'is_starter'
]
=
[
p
<
5
for
p
in
range
(
df
.
shape
[
0
]
)
]
df
.
drop_duplicates
(
subset
=
'player_id'
,
keep
=
'first'
,
inplace
=
True
)
return
pd
.
concat
(
dfs
)

Returns a DataFrame of player stats from the game ( either basic or advanced depending on the argument .


def switch_to_dir(dirPath):
    """
    Decorator that switches to given directory before executing function, and
    then returning to orignal directory.
    """

    def decorator(func):
        @funcutils.wraps(func)
        def wrapper(*args, **kwargs):
            orig_cwd = os.getcwd()
            os.chdir(dirPath)
            ret = func(*args, **kwargs)
            os.chdir(orig_cwd)
            return ret
        return wrapper

    return decorator

def switch_to_dir ( dirPath ) : def decorator ( func ) : @ funcutils . wraps ( func ) def wrapper ( * args , * * kwargs ) : orig_cwd = os . getcwd ( ) os . chdir ( dirPath ) ret = func ( * args , * * kwargs ) os . chdir ( orig_cwd ) return ret return wrapper return decorator

Decorator that switches to given directory before executing function and then returning to orignal directory .


def cache(func):
    """Caches the HTML returned by the specified function `func`. Caches it in
    the user cache determined by the appdirs package.
    """

    CACHE_DIR = appdirs.user_cache_dir('sportsref', getpass.getuser())
    if not os.path.isdir(CACHE_DIR):
        os.makedirs(CACHE_DIR)

    @funcutils.wraps(func)
    def wrapper(url):
        # hash based on the URL
        file_hash = hashlib.md5()
        encoded_url = url.encode(errors='replace')
        file_hash.update(encoded_url)
        file_hash = file_hash.hexdigest()
        filename = '{}/{}'.format(CACHE_DIR, file_hash)

        sport_id = None
        for a_base_url, a_sport_id in sportsref.SITE_ABBREV.items():
            if url.startswith(a_base_url):
                sport_id = a_sport_id
                break
        else:
            print('No sport ID found for {}, not able to check cache'.format(url))

        # check whether cache is valid or stale
        file_exists = os.path.isfile(filename)
        if sport_id and file_exists:
            cur_time = int(time.time())
            mod_time = int(os.path.getmtime(filename))
            days_since_mod = datetime.timedelta(seconds=(cur_time - mod_time)).days
            days_cache_valid = globals()['_days_valid_{}'.format(sport_id)](url)
            cache_is_valid = days_since_mod < days_cache_valid
        else:
            cache_is_valid = False

        # if file found and cache is valid, read from file
        allow_caching = sportsref.get_option('cache')
        if file_exists and cache_is_valid and allow_caching:
            with codecs.open(filename, 'r', encoding='utf-8', errors='replace') as f:
                text = f.read()
        # otherwise, execute function and cache results
        else:
            text = func(url)
            with codecs.open(filename, 'w+', encoding='utf-8') as f:
                f.write(text)
        return text

    return wrapper

Caches the HTML returned by the specified function func . Caches it in the user cache determined by the appdirs package .


def get_class_instance_key(cls, args, kwargs):
    """
    Returns a unique identifier for a class instantiation.
    """
    l = [id(cls)]
    for arg in args:
        l.append(id(arg))
    l.extend((k, id(v)) for k, v in kwargs.items())
    return tuple(sorted(l))

def get_class_instance_key ( cls , args , kwargs ) : l = [ id ( cls ) ] for arg in args : l . append ( id ( arg ) ) l . extend ( ( k , id ( v ) ) for k , v in kwargs . items ( ) ) return tuple ( sorted ( l ) )

Returns a unique identifier for a class instantiation .


def memoize(fun):
    """A decorator for memoizing functions.

    Only works on functions that take simple arguments - arguments that take
    list-like or dict-like arguments will not be memoized, and this function
    will raise a TypeError.
    """
    @funcutils.wraps(fun)
    def wrapper(*args, **kwargs):

        do_memoization = sportsref.get_option('memoize')
        if not do_memoization:
            return fun(*args, **kwargs)

        hash_args = tuple(args)
        hash_kwargs = frozenset(sorted(kwargs.items()))
        key = (hash_args, hash_kwargs)

        def _copy(v):
            if isinstance(v, pq):
                return v.clone()
            else:
                return copy.deepcopy(v)

        try:
            ret = _copy(cache[key])
            return ret
        except KeyError:
            cache[key] = fun(*args, **kwargs)
            ret = _copy(cache[key])
            return ret
        except TypeError:
            print('memoization type error in function {} for arguments {}'
                  .format(fun.__name__, key))
            raise

    cache = {}
    return wrapper


def age(self, year, month=2, day=1):
        """Returns the age of the player on a given date.

        :year: int representing the year.
        :month: int representing the month (1-12).
        :day: int representing the day within the month (1-31).
        :returns: Age in years as a float.
        """
        doc = self.get_main_doc()
        date_string = doc('span[itemprop="birthDate"]').attr('data-birth')
        regex = r'(\d{4})\-(\d{2})\-(\d{2})'
        date_args = map(int, re.match(regex, date_string).groups())
        birth_date = datetime.date(*date_args)
        age_date = datetime.date(year=year, month=month, day=day)
        delta = age_date - birth_date
        age = delta.days / 365.
        return age

def age ( self , year , month = 2 , day = 1 ) : doc = self . get_main_doc ( ) date_string = doc ( 'span[itemprop="birthDate"]' ) . attr ( 'data-birth' ) regex = r'(\d{4})\-(\d{2})\-(\d{2})' date_args = map ( int , re . match ( regex , date_string ) . groups ( ) ) birth_date = datetime . date ( * date_args ) age_date = datetime . date ( year = year , month = month , day = day ) delta = age_date - birth_date age = delta . days / 365. return age

Returns the age of the player on a given date .


def _get_stats_table(self, table_id, kind='R', summary=False):
        """Gets a stats table from the player page; helper function that does
        the work for per-game, per-100-poss, etc. stats.

        :table_id: the ID of the HTML table.
        :kind: specifies regular season, playoffs, or both. One of 'R', 'P',
            'B'. Defaults to 'R'.
        :returns: A DataFrame of stats.
        """
        doc = self.get_main_doc()
        table_id = 'table#{}{}'.format(
            'playoffs_' if kind == 'P' else '', table_id)
        table = doc(table_id)
        df = sportsref.utils.parse_table(table, flatten=(not summary),
                                         footer=summary)
        return df

def _get_stats_table ( self , table_id , kind = 'R' , summary = False ) : doc = self . get_main_doc ( ) table_id = 'table#{}{}' . format ( 'playoffs_' if kind == 'P' else '' , table_id ) table = doc ( table_id ) df = sportsref . utils . parse_table ( table , flatten = ( not summary ) , footer = summary ) return df

Gets a stats table from the player page ; helper function that does the work for per - game per - 100 - poss etc . stats .


def stats_per_game(self, kind='R', summary=False):
        """Returns a DataFrame of per-game box score stats."""
        return self._get_stats_table('per_game', kind=kind, summary=summary)

def stats_per_game ( self , kind = 'R' , summary = False ) : return self . _get_stats_table ( 'per_game' , kind = kind , summary = summary )

Returns a DataFrame of per - game box score stats .


def stats_totals(self, kind='R', summary=False):
        """Returns a DataFrame of total box score statistics by season."""
        return self._get_stats_table('totals', kind=kind, summary=summary)

def stats_totals ( self , kind = 'R' , summary = False ) : return self . _get_stats_table ( 'totals' , kind = kind , summary = summary )

Returns a DataFrame of total box score statistics by season .


def stats_per36(self, kind='R', summary=False):
        """Returns a DataFrame of per-36-minutes stats."""
        return self._get_stats_table('per_minute', kind=kind, summary=summary)

def stats_per36 ( self , kind = 'R' , summary = False ) : return self . _get_stats_table ( 'per_minute' , kind = kind , summary = summary )

Returns a DataFrame of per - 36 - minutes stats .


def stats_per100(self, kind='R', summary=False):
        """Returns a DataFrame of per-100-possession stats."""
        return self._get_stats_table('per_poss', kind=kind, summary=summary)

def stats_per100 ( self , kind = 'R' , summary = False ) : return self . _get_stats_table ( 'per_poss' , kind = kind , summary = summary )

Returns a DataFrame of per - 100 - possession stats .


def stats_advanced(self, kind='R', summary=False):
        """Returns a DataFrame of advanced stats."""
        return self._get_stats_table('advanced', kind=kind, summary=summary)

def stats_advanced ( self , kind = 'R' , summary = False ) : return self . _get_stats_table ( 'advanced' , kind = kind , summary = summary )


def stats_shooting(self, kind='R', summary=False):
        """Returns a DataFrame of shooting stats."""
        return self._get_stats_table('shooting', kind=kind, summary=summary)

def stats_shooting ( self , kind = 'R' , summary = False ) : return self . _get_stats_table ( 'shooting' , kind = kind , summary = summary )


def stats_pbp(self, kind='R', summary=False):
        """Returns a DataFrame of play-by-play stats."""
        return self._get_stats_table('advanced_pbp', kind=kind,
                                     summary=summary)

def stats_pbp ( self , kind = 'R' , summary = False ) : return self . _get_stats_table ( 'advanced_pbp' , kind = kind , summary = summary )

Returns a DataFrame of play - by - play stats .


def gamelog_basic(self, year, kind='R'):
        """Returns a table of a player's basic game-by-game stats for a season.

        :param year: The year representing the desired season.
        :param kind: specifies regular season, playoffs, or both. One of 'R',
            'P', 'B'. Defaults to 'R'.
        :returns: A DataFrame of the player's standard boxscore stats from each
            game of the season.
        :rtype: pd.DataFrame
        """
        doc = self.get_sub_doc('gamelog/{}'.format(year))
        table = (doc('table#pgl_basic_playoffs')
                 if kind == 'P' else doc('table#pgl_basic'))
        df = sportsref.utils.parse_table(table)
        return df

def gamelog_basic ( self , year , kind = 'R' ) : doc = self . get_sub_doc ( 'gamelog/{}' . format ( year ) ) table = ( doc ( 'table#pgl_basic_playoffs' ) if kind == 'P' else doc ( 'table#pgl_basic' ) ) df = sportsref . utils . parse_table ( table ) return df

Returns a table of a player s basic game - by - game stats for a season .


def get(self):
        '''
        Please don't do this in production environments.
        '''
        self.write("Memory Session Object Demo:")
        if "sv" in self.session:
            current_value = self.session["sv"]
            self.write("current sv value is %s, and system will delete this value.<br/>" % self.session["sv"])
            self.session.delete("sv")
            if "sv" not in self.session:
                self.write("current sv value is empty")
        else:
            self.write("Session data not found")

Please don t do this in production environments .


def expand_details(df, detailCol='detail'):
    """Expands the details column of the given dataframe and returns the
    resulting DataFrame.

    :df: The input DataFrame.
    :detailCol: The detail column name.
    :returns: Returns DataFrame with new columns from pbp parsing.
    """
    df = copy.deepcopy(df)
    df['detail'] = df[detailCol]
    dicts = [sportsref.nfl.pbp.parse_play_details(detail) for detail in df['detail'].values]
    # clean up unmatched details
    cols = {c for d in dicts if d for c in d.keys()}
    blankEntry = {c: np.nan for c in cols}
    newDicts = [d if d else blankEntry for d in dicts]
    # get details DataFrame and merge it with original to create main DataFrame
    details = pd.DataFrame(newDicts)
    df = pd.merge(df, details, left_index=True, right_index=True)
    # add isError column
    errors = [i for i, d in enumerate(dicts) if d is None]
    df['isError'] = False
    df.loc[errors, 'isError'] = True
    # fill in some NaN's necessary for _clean_features
    df.loc[0, 'qtr_time_remain'] = '15:00'
    df.qtr_time_remain.fillna(method='bfill', inplace=True)
    df.qtr_time_remain.fillna(
        pd.Series(np.where(df.quarter == 4, '0:00', '15:00')), inplace=True
    )
    # use _clean_features to clean up and add columns
    new_df = df.apply(_clean_features, axis=1)
    return new_df

Expands the details column of the given dataframe and returns the resulting DataFrame .


def _add_team_columns(features):
    """Function that adds 'team' and 'opp' columns to the features by iterating
    through the rows in order. A precondition is that the features dicts are in
    order in a continuous game sense and that all rows are from the same game.

    :features: A DataFrame with each row representing each play (in order).
    :returns: A similar DataFrame but with 'team' and 'opp' columns added.
    """
    features = features.to_dict('records')
    curTm = curOpp = None
    playAfterKickoff = False
    # fill in team and opp columns
    for row in features:
        # if it's a kickoff or the play after a kickoff,
        # figure out who has possession manually
        if row['isKickoff'] or playAfterKickoff:
            curTm, curOpp = _team_and_opp(row)
        else:
            curTm, curOpp = _team_and_opp(row, curTm, curOpp)
        row['team'], row['opp'] = curTm, curOpp
        # set playAfterKickoff
        playAfterKickoff = row['isKickoff']

    features = pd.DataFrame(features)
    features.team.fillna(method='bfill', inplace=True)
    features.opp.fillna(method='bfill', inplace=True)
    # ffill for last row
    features.team.fillna(method='ffill', inplace=True)
    features.opp.fillna(method='ffill', inplace=True)
    return features

Function that adds team and opp columns to the features by iterating through the rows in order . A precondition is that the features dicts are in order in a continuous game sense and that all rows are from the same game .


def _add_team_features(df):
    """Adds extra convenience features based on teams with and without
    possession, with the precondition that the there are 'team' and 'opp'
    specified in row.

    :df: A DataFrame representing a game's play-by-play data after
        _clean_features has been called and 'team' and 'opp' have been added by
        _add_team_columns.
    :returns: A dict with new features in addition to previous features.
    """
    assert df.team.notnull().all()

    homeOnOff = df['team'] == df['home']
    # create column for distToGoal
    df['distToGoal'] = np.where(df['team'] != df['fieldSide'],
                                df['ydLine'], 100 - df['ydLine'])
    df['distToGoal'] = np.where(df['isXP'] | df['isTwoPoint'],
                                2, df['distToGoal'])
    # create column for each team's WP
    df['team_wp'] = np.where(homeOnOff, df['home_wp'], 100. - df['home_wp'])
    df['opp_wp'] = 100. - df['team_wp']
    # create columns for each team's WPA
    df['team_wpa'] = np.where(homeOnOff, df['home_wpa'], -df['home_wpa'])
    df['opp_wpa'] = -df['team_wpa']
    # create column for offense and defense scores if not already there
    assert df['boxscore_id'].nunique() == 1
    bs_id = df['boxscore_id'].values[0]
    bs = sportsref.nfl.boxscores.BoxScore(bs_id)
    df['team_score'] = np.where(df['team'] == bs.home(),
                                df['pbp_score_hm'], df['pbp_score_aw'])
    df['opp_score'] = np.where(df['team'] == bs.home(),
                               df['pbp_score_aw'], df['pbp_score_hm'])

    return df

Adds extra convenience features based on teams with and without possession with the precondition that the there are team and opp specified in row .


def initialWinProb(line):
    """Gets the initial win probability of a game given its Vegas line.

    :line: The Vegas line from the home team's perspective (negative means
    home team is favored).
    :returns: A float in [0., 100.] that represents the win probability.
    """
    line = float(line)
    probWin = 1. - norm.cdf(0.5, -line, 13.86)
    probTie = norm.cdf(0.5, -line, 13.86) - norm.cdf(-0.5, -line, 13.86)
    return 100. * (probWin + 0.5 * probTie)

def initialWinProb ( line ) : line = float ( line ) probWin = 1. - norm . cdf ( 0.5 , - line , 13.86 ) probTie = norm . cdf ( 0.5 , - line , 13.86 ) - norm . cdf ( - 0.5 , - line , 13.86 ) return 100. * ( probWin + 0.5 * probTie )

Gets the initial win probability of a game given its Vegas line .


def passing(self, kind='R'):
        """Gets yearly passing stats for the player.

        :kind: One of 'R', 'P', or 'B'. Case-insensitive; defaults to 'R'.
        :returns: Pandas DataFrame with passing stats.
        """
        doc = self.get_doc()
        table = (doc('table#passing') if kind == 'R' else
                 doc('table#passing_playoffs'))
        df = sportsref.utils.parse_table(table)
        return df

def passing ( self , kind = 'R' ) : doc = self . get_doc ( ) table = ( doc ( 'table#passing' ) if kind == 'R' else doc ( 'table#passing_playoffs' ) ) df = sportsref . utils . parse_table ( table ) return df

Gets yearly passing stats for the player .


def _simple_year_award(self, award_id):
        """Template for simple award functions that simply list years, such as
        pro bowls and first-team all pro.

        :award_id: The div ID that is appended to "leaderboard_" in selecting
        the table's div.
        :returns: List of years for the award.
        """
        doc = self.get_doc()
        table = doc('div#leaderboard_{} table'.format(award_id))
        return list(map(int, sportsref.utils.parse_awards_table(table)))

def
_simple_year_award
(
self
,
award_id
)
:
doc
=
self
.
get_doc
(
)
table
=
doc
(
'div#leaderboard_{} table'
.
format
(
award_id
)
)
return
list
(
map
(
int
,
sportsref
.
utils
.
parse_awards_table
(
table
)
)
)

Template for simple award functions that simply list years such as pro bowls and first - team all pro .


def name(self):
        """Returns the real name of the franchise given the team ID.

        Examples:
        'nwe' -> 'New England Patriots'
        'sea' -> 'Seattle Seahawks'

        :returns: A string corresponding to the team's full name.
        """
        doc = self.get_main_doc()
        headerwords = doc('div#meta h1')[0].text_content().split()
        lastIdx = headerwords.index('Franchise')
        teamwords = headerwords[:lastIdx]
        return ' '.join(teamwords)

def
name
(
self
)
:
doc
=
self
.
get_main_doc
(
)
headerwords
=
doc
(
'div#meta h1'
)
[
0
]
.
text_content
(
)
.
split
(
)
lastIdx
=
headerwords
.
index
(
'Franchise'
)
teamwords
=
headerwords
[
:
lastIdx
]
return
' '
.
join
(
teamwords
)

Returns the real name of the franchise given the team ID .


def boxscores(self, year):
        """Gets list of BoxScore objects corresponding to the box scores from
        that year.

        :year: The year for which we want the boxscores; defaults to current
        year.
        :returns: np.array of strings representing boxscore IDs.
        """
        doc = self.get_year_doc(year)
        table = doc('table#games')
        df = sportsref.utils.parse_table(table)
        if df.empty:
            return np.array([])
        return df.boxscore_id.values

def boxscores ( self , year ) : doc = self . get_year_doc ( year ) table = doc ( 'table#games' ) df = sportsref . utils . parse_table ( table ) if df . empty : return np . array ( [ ] ) return df . boxscore_id . values

Gets list of BoxScore objects corresponding to the box scores from that year .


def _year_info_pq(self, year, keyword):
        """Returns a PyQuery object containing the info from the meta div at
        the top of the team year page with the given keyword.

        :year: Int representing the season.
        :keyword: A keyword to filter to a single p tag in the meta div.
        :returns: A PyQuery object for the selected p element.
        """
        doc = self.get_year_doc(year)
        p_tags = doc('div#meta div:not(.logo) p')
        texts = [p_tag.text_content().strip() for p_tag in p_tags]
        try:
            return next(
                pq(p_tag) for p_tag, text in zip(p_tags, texts)
                if keyword.lower() in text.lower()
            )
        except StopIteration:
            if len(texts):
                raise ValueError('Keyword not found in any p tag.')
            else:
                raise ValueError('No meta div p tags found.')

def
_year_info_pq
(
self
,
year
,
keyword
)
:
doc
=
self
.
get_year_doc
(
year
)
p_tags
=
doc
(
'div#meta div:not(.logo) p'
)
texts
=
[
p_tag
.
text_content
(
)
.
strip
(
)
for
p_tag
in
p_tags
]
try
:
return
next
(
pq
(
p_tag
)
for
p_tag
,
text
in
zip
(
p_tags
,
texts
)
if
keyword
.
lower
(
)
in
text
.
lower
(
)
)
except
StopIteration
:
if
len
(
texts
)
:
raise
ValueError
(
'Keyword not found in any p tag.'
)
else
:
raise
ValueError
(
'No meta div p tags found.'
)

Returns a PyQuery object containing the info from the meta div at the top of the team year page with the given keyword .


def head_coaches_by_game(self, year):
        """Returns head coach data by game.

        :year: An int representing the season in question.
        :returns: An array with an entry per game of the season that the team
        played (including playoffs). Each entry is the head coach's ID for that
        game in the season.
        """
        coach_str = self._year_info_pq(year, 'Coach').text()
        regex = r'(\S+?) \((\d+)-(\d+)-(\d+)\)'
        coachAndTenure = []
        m = True
        while m:
            m = re.search(regex, coach_str)
            coachID, wins, losses, ties = m.groups()
            nextIndex = m.end(4) + 1
            coachStr = coachStr[nextIndex:]
            tenure = int(wins) + int(losses) + int(ties)
            coachAndTenure.append((coachID, tenure))

        coachIDs = [
            cID for cID, games in coachAndTenure for _ in range(games)
        ]
        return np.array(coachIDs[::-1])

def
head_coaches_by_game
(
self
,
year
)
:
coach_str
=
self
.
_year_info_pq
(
year
,
'Coach'
)
.
text
(
)
regex
=
r'(\S+?) \((\d+)-(\d+)-(\d+)\)'
coachAndTenure
=
[
]
m
=
True
while
m
:
m
=
re
.
search
(
regex
,
coach_str
)
coachID
,
wins
,
losses
,
ties
=
m
.
groups
(
)
nextIndex
=
m
.
end
(
4
)
+
1
coachStr
=
coachStr
[
nextIndex
:
]
tenure
=
int
(
wins
)
+
int
(
losses
)
+
int
(
ties
)
coachAndTenure
.
append
(
(
coachID
,
tenure
)
)
coachIDs
=
[
cID
for
cID
,
games
in
coachAndTenure
for
_
in
range
(
games
)
]
return
np
.
array
(
coachIDs
[
:
:
1
]
)


def schedule(self, year):
        """Returns a DataFrame with schedule information for the given year.

        :year: The year for the season in question.
        :returns: Pandas DataFrame with schedule information.
        """
        doc = self.get_year_doc(year)
        table = doc('table#games')
        df = sportsref.utils.parse_table(table)
        if df.empty:
            return pd.DataFrame()
        df = df.loc[df['week_num'].notnull()]
        df['week_num'] = np.arange(len(df)) + 1
        df['is_win'] = df['game_outcome'] == 'W'
        df['is_loss'] = df['game_outcome'] == 'L'
        df['is_tie'] = df['game_outcome'] == 'T'
        df['is_bye'] = df['game_outcome'].isnull()
        df['is_ot'] = df['overtime'].notnull()
        return df

def schedule ( self , year ) : doc = self . get_year_doc ( year ) table = doc ( 'table#games' ) df = sportsref . utils . parse_table ( table ) if df . empty : return pd . DataFrame ( ) df = df . loc [ df [ 'week_num' ] . notnull ( ) ] df [ 'week_num' ] = np . arange ( len ( df ) ) + 1 df [ 'is_win' ] = df [ 'game_outcome' ] == 'W' df [ 'is_loss' ] = df [ 'game_outcome' ] == 'L' df [ 'is_tie' ] = df [ 'game_outcome' ] == 'T' df [ 'is_bye' ] = df [ 'game_outcome' ] . isnull ( ) df [ 'is_ot' ] = df [ 'overtime' ] . notnull ( ) return df

Returns a DataFrame with schedule information for the given year .


def off_coordinator(self, year):
        """Returns the coach ID for the team's OC in a given year.

        :year: An int representing the year.
        :returns: A string containing the coach ID of the OC.
        """
        try:
            oc_anchor = self._year_info_pq(year, 'Offensive Coordinator')('a')
            if oc_anchor:
                return oc_anchor.attr['href']
        except ValueError:
            return None

def
off_coordinator
(
self
,
year
)
:
try
:
oc_anchor
=
self
.
_year_info_pq
(
year
,
'Offensive Coordinator'
)
(
'a'
)
if
oc_anchor
:
return
oc_anchor
.
attr
[
'href'
]
except
ValueError
:
return

Returns the coach ID for the team s OC in a given year .


def def_coordinator(self, year):
        """Returns the coach ID for the team's DC in a given year.

        :year: An int representing the year.
        :returns: A string containing the coach ID of the DC.
        """
        try:
            dc_anchor = self._year_info_pq(year, 'Defensive Coordinator')('a')
            if dc_anchor:
                return dc_anchor.attr['href']
        except ValueError:
            return None

def
def_coordinator
(
self
,
year
)
:
try
:
dc_anchor
=
self
.
_year_info_pq
(
year
,
'Defensive Coordinator'
)
(
'a'
)
if
dc_anchor
:
return
dc_anchor
.
attr
[
'href'
]
except
ValueError
:
return

Returns the coach ID for the team s DC in a given year .


def stadium(self, year):
        """Returns the ID for the stadium in which the team played in a given
        year.

        :year: The year in question.
        :returns: A string representing the stadium ID.
        """
        anchor = self._year_info_pq(year, 'Stadium')('a')
        return sportsref.utils.rel_url_to_id(anchor.attr['href'])

def stadium ( self , year ) : anchor = self . _year_info_pq ( year , 'Stadium' ) ( 'a' ) return sportsref . utils . rel_url_to_id ( anchor . attr [ 'href' ] )

Returns the ID for the stadium in which the team played in a given year .


def off_scheme(self, year):
        """Returns the name of the offensive scheme the team ran in the given
        year.

        :year: Int representing the season year.
        :returns: A string representing the offensive scheme.
        """
        scheme_text = self._year_info_pq(year, 'Offensive Scheme').text()
        m = re.search(r'Offensive Scheme[:\s]*(.+)\s*', scheme_text, re.I)
        if m:
            return m.group(1)
        else:
            return None

def
off_scheme
(
self
,
year
)
:
scheme_text
=
self
.
_year_info_pq
(
year
,
'Offensive Scheme'
)
.
text
(
)
m
=
re
.
search
(
r'Offensive Scheme[:\s]*(.+)\s*'
,
scheme_text
,
re
.
I
)
if
m
:
return
m
.
group
(
1
)
else
:
return

Returns the name of the offensive scheme the team ran in the given year .


def def_alignment(self, year):
        """Returns the name of the defensive alignment the team ran in the
        given year.

        :year: Int representing the season year.
        :returns: A string representing the defensive alignment.
        """
        scheme_text = self._year_info_pq(year, 'Defensive Alignment').text()
        m = re.search(r'Defensive Alignment[:\s]*(.+)\s*', scheme_text, re.I)
        if m:
            return m.group(1)
        else:
            return None

def
def_alignment
(
self
,
year
)
:
scheme_text
=
self
.
_year_info_pq
(
year
,
'Defensive Alignment'
)
.
text
(
)
m
=
re
.
search
(
r'Defensive Alignment[:\s]*(.+)\s*'
,
scheme_text
,
re
.
I
)
if
m
:
return
m
.
group
(
1
)
else
:
return

Returns the name of the defensive alignment the team ran in the given year .


def off_splits(self, year):
        """Returns a DataFrame of offensive team splits for a season.

        :year: int representing the season.
        :returns: Pandas DataFrame of split data.
        """
        doc = self.get_year_doc('{}_splits'.format(year))
        tables = doc('table.stats_table')
        dfs = [sportsref.utils.parse_table(table) for table in tables.items()]
        dfs = [
            df.assign(split=df.columns[0])
            .rename(columns={df.columns[0]: 'split_value'})
            for df in dfs
        ]
        if not dfs:
            return pd.DataFrame()
        return pd.concat(dfs).reset_index(drop=True)

def off_splits ( self , year ) : doc = self . get_year_doc ( '{}_splits' . format ( year ) ) tables = doc ( 'table.stats_table' ) dfs = [ sportsref . utils . parse_table ( table ) for table in tables . items ( ) ] dfs = [ df . assign ( split = df . columns [ 0 ] ) . rename ( columns = { df . columns [ 0 ] : 'split_value' } ) for df in dfs ] if not dfs : return pd . DataFrame ( ) return pd . concat ( dfs ) . reset_index ( drop = True )

Returns a DataFrame of offensive team splits for a season .


def get_html(url):
    """Gets the HTML for the given URL using a GET request.

    :url: the absolute URL of the desired page.
    :returns: a string of HTML.
    """
    global last_request_time
    with throttle_process_lock:
        with throttle_thread_lock:
            # sleep until THROTTLE_DELAY secs have passed since last request
            wait_left = THROTTLE_DELAY - (time.time() - last_request_time.value)
            if wait_left > 0:
                time.sleep(wait_left)

            # make request
            response = requests.get(url)

            # update last request time for throttling
            last_request_time.value = time.time()

    # raise ValueError on 4xx status code, get rid of comments, and return
    if 400 <= response.status_code < 500:
        raise ValueError(
            'Status Code {} received fetching URL "{}"'
            .format(response.status_code, url)
        )
    html = response.text
    html = html.replace('<!--', '').replace('-->', '')

    return html

Gets the HTML for the given URL using a GET request .


def flatten_links(td, _recurse=False):
    """Flattens relative URLs within text of a table cell to IDs and returns
    the result.

    :td: the PyQuery object for the HTML to convert
    :returns: the string with the links flattened to IDs
    """

    # helper function to flatten individual strings/links
    def _flatten_node(c):
        if isinstance(c, basestring):
            return c.strip()
        elif 'href' in c.attrib:
            c_id = rel_url_to_id(c.attrib['href'])
            return c_id if c_id else c.text_content().strip()
        else:
            return flatten_links(pq(c), _recurse=True)

    # if there's no text, just return None
    if td is None or not td.text():
        return '' if _recurse else None

    td.remove('span.note')
    return ''.join(_flatten_node(c) for c in td.contents())

Flattens relative URLs within text of a table cell to IDs and returns the result .


def rel_url_to_id(url):
    """Converts a relative URL to a unique ID.

    Here, 'ID' refers generally to the unique ID for a given 'type' that a
    given datum has. For example, 'BradTo00' is Tom Brady's player ID - this
    corresponds to his relative URL, '/players/B/BradTo00.htm'. Similarly,
    '201409070dal' refers to the boxscore of the SF @ DAL game on 09/07/14.

    Supported types:
    * player/...
    * boxscores/...
    * teams/...
    * years/...
    * leagues/...
    * awards/...
    * coaches/...
    * officials/...
    * schools/...
    * schools/high_schools.cgi?id=...

    :returns: ID associated with the given relative URL.
    """
    yearRegex = r'.*/years/(\d{4}).*|.*/gamelog/(\d{4}).*'
    playerRegex = r'.*/players/(?:\w/)?(.+?)(?:/|\.html?)'
    boxscoresRegex = r'.*/boxscores/(.+?)\.html?'
    teamRegex = r'.*/teams/(\w{3})/.*'
    coachRegex = r'.*/coaches/(.+?)\.html?'
    stadiumRegex = r'.*/stadiums/(.+?)\.html?'
    refRegex = r'.*/officials/(.+?r)\.html?'
    collegeRegex = r'.*/schools/(\S+?)/.*|.*college=([^&]+)'
    hsRegex = r'.*/schools/high_schools\.cgi\?id=([^\&]{8})'
    bsDateRegex = r'.*/boxscores/index\.f?cgi\?(month=\d+&day=\d+&year=\d+)'
    leagueRegex = r'.*/leagues/(.*_\d{4}).*'
    awardRegex = r'.*/awards/(.+)\.htm'

    regexes = [
        yearRegex,
        playerRegex,
        boxscoresRegex,
        teamRegex,
        coachRegex,
        stadiumRegex,
        refRegex,
        collegeRegex,
        hsRegex,
        bsDateRegex,
        leagueRegex,
        awardRegex,
    ]

    for regex in regexes:
        match = re.match(regex, url, re.I)
        if match:
            return [_f for _f in match.groups() if _f][0]

    # things we don't want to match but don't want to print a WARNING
    if any(
        url.startswith(s) for s in
        (
            '/play-index/',
        )
    ):
        return url

    print('WARNING. NO MATCH WAS FOUND FOR "{}"'.format(url))
    return url

Converts a relative URL to a unique ID .


def _kwargs_to_qs(**kwargs):
    """Converts kwargs given to PSF to a querystring.

    :returns: the querystring.
    """
    # start with defaults
    inpOptDef = inputs_options_defaults()
    opts = {
        name: dct['value']
        for name, dct in inpOptDef.items()
    }

    # clean up keys and values
    for k, v in kwargs.items():
        del kwargs[k]
        # bool => 'Y'|'N'
        if isinstance(v, bool):
            kwargs[k] = 'Y' if v else 'N'
        # tm, team => team_id
        elif k.lower() in ('tm', 'team'):
            kwargs['team_id'] = v
        # yr, year, yrs, years => year_min, year_max
        elif k.lower() in ('yr', 'year', 'yrs', 'years'):
            if isinstance(v, collections.Iterable):
                lst = list(v)
                kwargs['year_min'] = min(lst)
                kwargs['year_max'] = max(lst)
            elif isinstance(v, basestring):
                v = list(map(int, v.split(',')))
                kwargs['year_min'] = min(v)
                kwargs['year_max'] = max(v)
            else:
                kwargs['year_min'] = v
                kwargs['year_max'] = v
        # pos, position, positions => pos[]
        elif k.lower() in ('pos', 'position', 'positions'):
            if isinstance(v, basestring):
                v = v.split(',')
            elif not isinstance(v, collections.Iterable):
                v = [v]
            kwargs['pos[]'] = v
        # draft_pos, ... => draft_pos[]
        elif k.lower() in (
            'draft_pos', 'draftpos', 'draftposition', 'draftpositions',
            'draft_position', 'draft_positions'
        ):
            if isinstance(v, basestring):
                v = v.split(',')
            elif not isinstance(v, collections.Iterable):
                v = [v]
            kwargs['draft_pos[]'] = v
        # if not one of these cases, put it back in kwargs
        else:
            kwargs[k] = v

    # update based on kwargs
    for k, v in kwargs.items():
        # if overwriting a default, overwrite it (with a list so the
        # opts -> querystring list comp works)
        if k in opts or k in ('pos[]', 'draft_pos[]'):
            # if multiple values separated by commas, split em
            if isinstance(v, basestring):
                v = v.split(',')
            # otherwise, make sure it's a list
            elif not isinstance(v, collections.Iterable):
                v = [v]
            # then, add list of values to the querystring dict *opts*
            opts[k] = v
        if 'draft' in k:
            opts['draft'] = [1]

    opts['request'] = [1]
    opts['offset'] = [kwargs.get('offset', 0)]

    qs = '&'.join(
        '{}={}'.format(urllib.parse.quote_plus(name), val)
        for name, vals in sorted(opts.items()) for val in vals
    )

    return qs

Converts kwargs given to PSF to a querystring .


def _Streamer__read_process(self, path, read_size, cbuf, stop, barrier, cyclic, offset, read_skip, sync):
    """
    Main function for the processes that read from the HDF5 file.

    :param self: A reference to the streamer object that created these processes.
    :param path: The HDF5 path to the node to be read from.
    :param read_size: The length of the block along the outer dimension to read.
    :param cbuf: The circular buffer to place read elements into.
    :param stop: The Event that signals the process to stop reading.
    :param barrier: The Barrier that synchonises read cycles.
    :param cyclic: True if the process should read cyclically.
    :param offset: Offset into the dataset that this process should start reading at.
    :param read_skip: How many element to skip on each iteration.
    :param sync: GuardSynchonizer to order writes to the buffer.
    :return: Nothing
    """
    # Multi-process access to HDF5 seems to behave better there are no top level imports of PyTables.
    import tables as tb
    h5_file = tb.open_file(self.filename, 'r', **self.h5_kw_args)
    ary = h5_file.get_node(path)

    i = offset
    while not stop.is_set():
        vals = ary[i:i + read_size]
        # If the read goes off the end of the dataset, then wrap to the start.
        if i + read_size > len(ary):
            vals = np.concatenate([vals, ary[0:read_size - len(vals)]])

        if sync is None:
            # If no ordering is requested, then just write to the next available space in the buffer.
            with cbuf.put_direct() as put_ary:
                put_ary[:] = vals
        else:
            # Otherwise, use the sync object to ensure that writes occur in the order provided by i.
            # So i = 0 will write first, then i = block_size, then i = 2*block_size, etc...
            # The sync object has two ordered barriers so that acquisition and release of the buffer spaces
            # are synchronized in order, but the actual writing to the buffer can happen simultaneously.
            # If only one barrier were used, writing to the buffer would be linearised.
            with sync.do(cbuf.put_direct(), i, (i+read_size) % len(ary)) as put_ary:
                put_ary[:] = vals

        i += read_skip
        if cyclic:
            # If the next iteration is past the end of the dataset, wrap it around.
            if i >= len(ary):
                i %= len(ary)
                barrier.wait()
        else:
            # But if cyclic mode is disabled, break the loop as the work is now done.
            if i + read_size > len(ary):
                break

Main function for the processes that read from the HDF5 file .


def put_direct(self):
        """
        Allows direct access to the buffer element.
        Blocks until there is room to write into the buffer.

        :return: A guard object that returns the buffer element.
        """

        # Once the guard is released, write_idx will be placed into read_queue.
        return self.Guard(self.read_queue, self.arys, self.__put_idx)

Allows direct access to the buffer element . Blocks until there is room to write into the buffer .


def get_direct(self):
        """
        Allows direct access to the buffer element.
        Blocks until there is data that can be read.

        :return: A guard object that returns the buffer element.
        """

        read_idx = self.__get_idx()

        if read_idx is QueueClosed:
            return QueueClosed

        # Once the guard is released, read_idx will be placed into write_queue.
        return self.Guard(self.write_queue, self.arys, lambda: read_idx)

Allows direct access to the buffer element . Blocks until there is data that can be read .


def close(self):
        """Close the queue, signalling that no more data can be put into the queue."""
        self.read_queue.put(QueueClosed)
        self.write_queue.put(QueueClosed)

def close ( self ) : self . read_queue . put ( QueueClosed ) self . write_queue . put ( QueueClosed )

Close the queue signalling that no more data can be put into the queue .


def __get_batch(self, path, length, last=False):
        """
        Get a block of data from the node at path.

        :param path: The path to the node to read from.
        :param length: The length along the outer dimension to read.
        :param last: True if the remainder elements should be read.
        :return: A copy of the requested block of data as a numpy array.
        """
        import tables
        h5_file = tables.open_file(self.filename, 'r')
        h5_node = h5_file.get_node(path)

        if len(h5_node) == 0:
            raise Exception("Cannot read from empty dataset.")

        # If the length isn't specified, then fall back to default values.
        if length is None:
            chunkshape = h5_node.chunkshape
            # If the array isn't chunked, then try to make the block close to 128KB.
            if chunkshape is None:
                default_length = 128*2**10//h5_node[0].nbytes  # Divides by one row of the dataset.
                length = min(h5_node.shape[0], default_length)
            # If it is chunked, then use the chunkshape for best performance.
            else:
                length = chunkshape[0]

        if last:
            example = h5_node[length*(len(h5_node)//length):].copy()
        else:
            example = h5_node[:length].copy()

        h5_file.close()
        return example

Get a block of data from the node at path .


def get_remainder(self, path, block_size):
        """
        Get the remainder elements. These elements will not be read in the direct queue access cyclic=False mode.

        :param path: The HDF5 path to the dataset to be read.
        :param block_size: The block size is used to calculate which elements will remain.
        :return: A copy of the remainder elements as a numpy array.
        """
        return self.__get_batch(path, length=block_size, last=True)

def get_remainder ( self , path , block_size ) : return self . __get_batch ( path , length = block_size , last = True )

Get the remainder elements . These elements will not be read in the direct queue access cyclic = False mode .


def get_queue(self, path, n_procs=4, read_ahead=None, cyclic=False, block_size=None, ordered=False):
        """
        Get a queue that allows direct access to the internal buffer. If the dataset to be read is chunked, the
        block_size should be a multiple of the chunk size to maximise performance. In this case it is best to leave it
        to the default. When cyclic=False, and block_size does not divide the dataset evenly, the remainder elements
        will not be returned by the queue. When cyclic=True, the remainder elements will be part of a block that wraps
        around the end and includes element from the beginning of the dataset. By default, blocks are returned in the
        order in which they become available. The ordered option will force blocks to be returned in on-disk order.

        :param path: The HDF5 path to the dataset that should be read.
        :param n_procs: The number of background processes used to read the datset in parallel.
        :param read_ahead: The number of blocks to allocate in the internal buffer.
        :param cyclic: True if the queue should wrap at the end of the dataset.
        :param block_size: The size along the outer dimension of the blocks to be read. Defaults to a multiple of
            the chunk size, or to a 128KB sized block if the dataset is not chunked.
        :param ordered: Force the reader return data in on-disk order. May result in performance penalty.
        :return: A queue object that allows access to the internal buffer.
        """
        # Get a block_size length of elements from the dataset to serve as a template for creating the buffer.
        # If block_size=None, then get_batch calculates an appropriate block size.
        example = self.__get_batch(path, block_size)
        block_size = example.shape[0]

        if read_ahead is None:
            # 2x No. of processes for writing, 1 extra for reading.
            read_ahead = 2*n_procs + 1

        cbuf = SharedCircBuf(read_ahead, example)
        stop = multiprocessing.Event()
        barrier = Barrier(n_procs)

        # If ordering has been requested, create a synchronizer.
        sync = GuardSynchronizer() if ordered else None

        procs = []
        for i in range(n_procs):
            # Each process is offset in the dataset by i*block_size
            # The skip length is set to n_procs*block_size so that no block is read by 2 processes.
            process = multiprocessing.Process(target=_Streamer__read_process, args=(
                self, path, block_size, cbuf, stop, barrier, cyclic,
                i * block_size, n_procs * block_size, sync
            ))
            process.daemon = True
            process.start()
            procs.append(process)

        # If the queue is not cyclic, then the cessation of reading data needs to be monitored.
        if not cyclic:

            # This closure defines a background thread that waits until all processes have finished.
            # At this point, all data from the dataset has been read, and the buffer is closed.
            def monitor():
                for p in procs:
                    p.join()
                cbuf.close()

            monitor_thread = threading.Thread(target=monitor)
            monitor_thread.daemon = True
            monitor_thread.start()

        return Streamer.Queue(cbuf, stop, block_size)

Get a queue that allows direct access to the internal buffer . If the dataset to be read is chunked the block_size should be a multiple of the chunk size to maximise performance . In this case it is best to leave it to the default . When cyclic = False and block_size does not divide the dataset evenly the remainder elements will not be returned by the queue . When cyclic = True the remainder elements will be part of a block that wraps around the end and includes element from the beginning of the dataset . By default blocks are returned in the order in which they become available . The ordered option will force blocks to be returned in on - disk order .


def get_generator(self, path, *args, **kw_args):
        """
        Get a generator that allows convenient access to the streamed data.
        Elements from the dataset are returned from the generator one row at a time.
        Unlike the direct access queue, this generator also returns the remainder elements.
        Additional arguments are forwarded to get_queue.
        See the get_queue method for documentation of these parameters.

        :param path:
        :return: A generator that iterates over the rows in the dataset.
        """
        q = self.get_queue(path=path, *args, **kw_args)

        try:
            # This generator just implements a standard access pattern for the direct access queue.
            for guard in q.iter():
                with guard as batch:
                    batch_copy = batch.copy()

                for row in batch_copy:
                    yield row

            last_batch = self.get_remainder(path, q.block_size)
            for row in last_batch:
                yield row

        finally:
            q.close()

Get a generator that allows convenient access to the streamed data . Elements from the dataset are returned from the generator one row at a time . Unlike the direct access queue this generator also returns the remainder elements . Additional arguments are forwarded to get_queue . See the get_queue method for documentation of these parameters .


def parse(ifp, pb_cls, **kwargs):
    """Parse a stream.

    Args:
        ifp (string or file-like object): input stream.
        pb_cls (protobuf.message.Message.__class__): The class object of
            the protobuf message type encoded in the stream.
    """
    mode = 'rb'
    if isinstance(ifp, str):
        istream = open(ifp, mode=mode, **kwargs)
    else:
        istream = open(fileobj=ifp, mode=mode, **kwargs)
    with istream:
        for data in istream:
            pb_obj = pb_cls()
            pb_obj.ParseFromString(data)
            yield pb_obj

def parse ( ifp , pb_cls , * * kwargs ) : mode = 'rb' if isinstance ( ifp , str ) : istream = open ( ifp , mode = mode , * * kwargs ) else : istream = open ( fileobj = ifp , mode = mode , * * kwargs ) with istream : for data in istream : pb_obj = pb_cls ( ) pb_obj . ParseFromString ( data ) yield pb_obj


def dump(ofp, *pb_objs, **kwargs):
    """Write to a stream.

    Args:
        ofp (string or file-like object): output stream.
        pb_objs (*protobuf.message.Message): list of protobuf message objects
            to be written.
    """
    mode = 'wb'
    if isinstance(ofp, str):
        ostream = open(ofp, mode=mode, **kwargs)
    else:
        ostream = open(fileobj=ofp, mode=mode, **kwargs)
    with ostream:
        ostream.write(*pb_objs)

def dump ( ofp , * pb_objs , * * kwargs ) : mode = 'wb' if isinstance ( ofp , str ) : ostream = open ( ofp , mode = mode , * * kwargs ) else : ostream = open ( fileobj = ofp , mode = mode , * * kwargs ) with ostream : ostream . write ( * pb_objs )


def _read_varint(self):
        """Read a varint from file, parse it, and return the decoded integer.
        """
        buff = self._fd.read(1)
        if buff == b'':
            return 0

        while (bytearray(buff)[-1] & 0x80) >> 7 == 1:  # while the MSB is 1
            new_byte = self._fd.read(1)
            if new_byte == b'':
                raise EOFError('unexpected EOF.')
            buff += new_byte

        varint, _ = decodeVarint(buff, 0)

        return varint

def
_read_varint
(
self
)
:
buff
=
self
.
_fd
.
read
(
1
)
if
buff
==
b''
:
return
0
while
(
bytearray
(
buff
)
[
1
]
&
0x80
)
>>
7
==
1
:
# while the MSB is 1
new_byte
=
self
.
_fd
.
read
(
1
)
if
new_byte
==
b''
:
raise
EOFError
(
'unexpected EOF.'
)
buff
+=
new_byte
varint
,
_
=
decodeVarint
(
buff
,
0
)
return
varint

Read a varint from file parse it and return the decoded integer .


def _get_objs(self):
        """A generator yielding all protobuf object data in the file. It is the
        main parser of the stream encoding.
        """
        while True:
            count = self._read_varint()
            if count == 0:
                break
            # Read a group containing `count` number of objects.
            for _ in range(count):
                size = self._read_varint()
                if size == 0:
                    raise EOFError('unexpected EOF.')
                # Read an object from the object group.
                yield self._fd.read(size)

            if self._group_delim:
                yield self._delimiter() if self._delimiter is not None \
                                        else None

A generator yielding all protobuf object data in the file . It is the main parser of the stream encoding .


def close(self):
        """Close the stream."""
        self.flush()
        if self._myfd is not None:
            self._myfd.close()
            self._myfd = None

def close ( self ) : self . flush ( ) if self . _myfd is not None : self . _myfd . close ( ) self . _myfd = None


def write(self, *pb2_obj):
        """Write a group of one or more protobuf objects to the file. Multiple
        object groups can be written by calling this method several times
        before closing stream or exiting the runtime context.

        The input protobuf objects get buffered and will be written down when
        the number of buffered objects exceed the `self._buffer_size`.

        Args:
            pb2_obj (*protobuf.message.Message): list of protobuf messages.
        """
        base = len(self._write_buff)

        for idx, obj in enumerate(pb2_obj):
            if self._buffer_size > 0 and \
                    (idx + base) != 0 and \
                    (idx + base) % self._buffer_size == 0:
                self.flush()
            self._write_buff.append(obj)

        if self._buffer_size == 0:
            self.flush()

def write ( self , * pb2_obj ) : base = len ( self . _write_buff ) for idx , obj in enumerate ( pb2_obj ) : if self . _buffer_size > 0 and ( idx + base ) != 0 and ( idx + base ) % self . _buffer_size == 0 : self . flush ( ) self . _write_buff . append ( obj ) if self . _buffer_size == 0 : self . flush ( )

Write a group of one or more protobuf objects to the file . Multiple object groups can be written by calling this method several times before closing stream or exiting the runtime context .


def flush(self):
        """Write down buffer to the file."""
        if not self.is_output():
            return

        count = len(self._write_buff)
        if count == 0:
            return

        encodeVarint(self._fd.write, count, True)

        for obj in self._write_buff:
            obj_str = obj.SerializeToString()
            encodeVarint(self._fd.write, len(obj_str), True)
            self._fd.write(obj_str)

        self._write_buff = []

def flush ( self ) : if not self . is_output ( ) : return count = len ( self . _write_buff ) if count == 0 : return encodeVarint ( self . _fd . write , count , True ) for obj in self . _write_buff : obj_str = obj . SerializeToString ( ) encodeVarint ( self . _fd . write , len ( obj_str ) , True ) self . _fd . write ( obj_str ) self . _write_buff = [ ]


def get_game_dir(self, username=False):
        """Returns joined game directory path relative to Steamapps"""
        if not self.common and not username:
            raise RuntimeError("Can't determine this game's directory without username")
        if self.common:
            subdir = "common"
        else:
            subdir = "username"
        subsubdir = self.dir
        if WIN32 or CYGWIN:
            subsubdir = subsubdir.lower()
        return os.path.join(subdir, subsubdir)


def input_text_with_keyboard_emulation(self, text):
        """
            Works around the problem of emulating user interactions with text inputs.
            Emulates a key-down action on the first char of the input. This way, implementations which
            require key-down event to trigger auto-suggest are testable.
            Then the chains sends the rest of the text and releases the key.
        """
        ActionChains(self.driver).key_down(text).key_up(Keys.CONTROL).perform()

def input_text_with_keyboard_emulation ( self , text ) : ActionChains ( self . driver ) . key_down ( text ) . key_up ( Keys . CONTROL ) . perform ( )

Works around the problem of emulating user interactions with text inputs . Emulates a key - down action on the first char of the input . This way implementations which require key - down event to trigger auto - suggest are testable . Then the chains sends the rest of the text and releases the key .


def make_fake_movie(nframes, mask_shape=(64, 64), mask_center=None,
                    bg_intensity=0.1, mask_sigma=10, dt=0.02, rate=1.0,
                    tau=1., sigma=0.001, seed=None):
    """
    Generate 2D fake fluorescence movie

    Arguments:
    ---------------------------------------------------------------------------
        nframes:        number of timebins to simulate
        mask_shape:     tuple (nrows, ncols), shape of a single movie frame
        mask_center:    tuple (x, y), pixel coords of cell center
        bg_intensity:   scalar, amplitude of (static) baseline fluorescence
        mask_sigma:     scalar, standard deviation of Gaussian mask
        dt:             timestep (s)
        rate:           mean spike rate (Hz)
        tau:            time constant of decay in calcium concentration (s)
        sigma:          SD of additive noise on fluorescence
        seed:           Seed for RNG

    Returns:
    ---------------------------------------------------------------------------
        F:          fluorescence [npixels, nframes]
        c:          calcium concentration [nframes,]
        n:          spike train [nframes,]
        theta:      tuple of true model parameters:
                    (sigma, alpha, beta, lambda, gamma)

    """

    gen = np.random.RandomState(seed)

    # poisson spikes
    n = gen.poisson(rate * dt, size=nframes)

    # internal calcium dynamics
    gamma = np.exp(-dt / tau)
    c = signal.lfilter(np.r_[1], np.r_[1, -gamma], n, axis=0)

    # pixel weights (sum == 1)
    nr, nc = mask_shape
    npix = nr * nc
    if mask_center is None:
        mask_center = (nc // 2., nr // 2.)
    a, b = mask_center
    y, x = np.ogrid[:nr, :nc]
    xs = (x - a) ** 2.
    ys = (y - b) ** 2.
    twoss = 2. * mask_sigma ** 2.
    alpha = np.exp(-1 * ((xs / twoss) + (ys / twoss))).ravel()
    alpha /= alpha.sum()

    # background fluorescence
    beta = gen.randn(npix) * bg_intensity

    # firing rate (spike probability per sec)
    lamb = rate

    # spatially & temporally white noise
    epsilon = gen.randn(npix, nframes) * sigma

    # simulated fluorescence
    F = c[None, :] * alpha[:, None] + beta[:, None] + epsilon

    theta = (sigma, alpha, beta, lamb, gamma)

    return F, c, n, theta


def evaluate_traits(self):
        """
        Evaluates traits and returns a list containing the description of traits which are not true.
        Notice that if LAZY_EVALUATION is set to False all traits are evaluated before returning. Use this option
        only for debugging purposes.
        """
        return_value = []
        for trait in self.traits:
            if not trait.condition():
                if not self.traits_eager_evaluation:
                    return [trait.description]
                else:
                    return_value.append(trait.description)
        return return_value

def evaluate_traits ( self ) : return_value = [ ] for trait in self . traits : if not trait . condition ( ) : if not self . traits_eager_evaluation : return [ trait . description ] else : return_value . append ( trait . description ) return return_value

Evaluates traits and returns a list containing the description of traits which are not true . Notice that if LAZY_EVALUATION is set to False all traits are evaluated before returning . Use this option only for debugging purposes .


def until_condition(self, condition, condition_description):
        """
        Waits until conditions is True or returns a non-None value.
        If any of the trait is still not present after timeout, raises a TimeoutException.
        """
        end_time = time.time() + self._timeout
        count = 1
        while True:
            try:
                if not hasattr(condition, '__call__'):
                    raise TypeError("condition is not callable")
                value = condition()
                if type(value) is bool and value is not False:
                    return value
                elif type(value) is not bool and value is not None:
                    return value
                else:
                    logger.debug("#" + str(count) + " - wait until " + condition_description)  # pragma: no cover
            except self._ignored_exceptions as ex:
                logger.debug("Captured {0} : {1}".format(str(ex.__class__).replace("<type '", "").replace("'>", ""),
                                                         str(ex)))  # pragma: no cover
            time.sleep(self._poll)
            count += 1
            if time.time() > end_time:  # pragma: no cover
                break
        raise TimeoutException(
            msg="condition <" + condition_description + "> was not true after " + str(self._timeout) + " seconds.")

def
until_condition
(
self
,
condition
,
condition_description
)
:
end_time
=
time
.
time
(
)
+
self
.
_timeout
count
=
1
while
True
:
try
:
if
not
hasattr
(
condition
,
'__call__'
)
:
raise
TypeError
(
"condition is not callable"
)
value
=
condition
(
)
if
type
(
value
)
is
bool
and
value
is
not
False
:
return
value
elif
type
(
value
)
is
not
bool
and
value
is
not
:
return
value
else
:
logger
.
debug
(
"#"
+
str
(
count
)
+
" - wait until "
+
condition_description
)
# pragma: no cover
except
self
.
_ignored_exceptions
as
ex
:
logger
.
debug
(
"Captured {0} : {1}"
.
format
(
str
(
ex
.
__class__
)
.
replace
(
"<type '"
,
""
)
.
replace
(
"'>"
,
""
)
,
str
(
ex
)
)
)
# pragma: no cover
time
.
sleep
(
self
.
_poll
)
count
+=
1
if
time
.
time
(
)
>
end_time
:
# pragma: no cover
break
raise
TimeoutException
(
msg
=
"condition <"
+
condition_description
+
"> was not true after "
+
str
(
self
.
_timeout
)
+
" seconds."
)

Waits until conditions is True or returns a non - None value . If any of the trait is still not present after timeout raises a TimeoutException .


def until_traits_are_present(self, element_with_traits):
        """
        Waits until all traits are present.
        If any of the traits is still not present after timeout, raises a TimeoutException.
        """
        end_time = time.time() + self._timeout
        count = 1
        missing_traits_descriptions = None
        while True:
            missing_traits_descriptions = []
            try:
                missing_traits_descriptions = element_with_traits.evaluate_traits()
                if len(missing_traits_descriptions) == 0:
                    return True
                else:
                    logger.debug("#{0} - wait until all traits are present: <{1}>".format(str(count), '> <'.join(
                        missing_traits_descriptions)))
            except self._ignored_exceptions as ex:  # pragma: no cover
                logger.debug("Captured {0}: {1}".format(str(ex.__class__).replace("<type '", "").replace("'>", ""),
                                                        str(ex)))  # pragma: no cover
                pass  # pragma: no cover
            time.sleep(self._poll)
            count += 1
            if time.time() > end_time:
                break
        raise TimeoutException(
            msg="conditions " + '<' + '> <'.join(missing_traits_descriptions) + '>' + " not true after " + str(
                self._timeout) + " seconds.")

Waits until all traits are present . If any of the traits is still not present after timeout raises a TimeoutException .


def with_ignored_exceptions(self, *ignored_exceptions):
        """
        Set a list of exceptions that should be ignored inside the wait loop.
        """
        for exception in ignored_exceptions:
            self._ignored_exceptions = self._ignored_exceptions + (exception,)
        return self

def with_ignored_exceptions ( self , * ignored_exceptions ) : for exception in ignored_exceptions : self . _ignored_exceptions = self . _ignored_exceptions + ( exception , ) return self

Set a list of exceptions that should be ignored inside the wait loop .


def main_volume(self, operator, value=None):
        """
        Execute Main.Volume.

        Returns int
        """
        try:
            res = int(self.exec_command('main', 'volume', operator, value))
            return res

        except (ValueError, TypeError):
            pass

        return None

def main_volume ( self , operator , value = None ) : try : res = int ( self . exec_command ( 'main' , 'volume' , operator , value ) ) return res except ( ValueError , TypeError ) : pass return None

Execute Main.Volume.

        Returns int


def main_source(self, operator, value=None):
        """
        Execute Main.Source.

        Returns int
        """
        try:
            source = int(self.exec_command('main', 'source', operator, value))
            return source
        except (ValueError, TypeError):
            pass

        return None

def main_source ( self , operator , value = None ) : try : source = int ( self . exec_command ( 'main' , 'source' , operator , value ) ) return source except ( ValueError , TypeError ) : pass return None

Execute Main.Source.

        Returns int


def _send(self, message, read_reply=False):
        """Send a command string to the amplifier."""
        sock = None
        for tries in range(0, 3):
            try:
                sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                sock.connect((self._host, self.PORT))
                break
            except (ConnectionError, BrokenPipeError):
                if tries == 3:
                    print("socket connect failed.")
                    return
                sleep(0.1)
        sock.send(codecs.decode(message, 'hex_codec'))
        if read_reply:
            sleep(0.1)
            reply = ''
            tries = 0
            max_tries = 20
            while len(reply) < len(message) and tries < max_tries:
                try:
                    reply += codecs.encode(sock.recv(self.BUFFERSIZE), 'hex')\
                        .decode("utf-8")
                except (ConnectionError, BrokenPipeError):
                    pass
                tries += 1
            sock.close()
            if tries >= max_tries:
                return
            return reply
        sock.close()

def
_send
(
self
,
message
,
read_reply
=
False
)
:
sock
=
for
tries
in
range
(
0
,
3
)
:
try
:
sock
=
socket
.
socket
(
socket
.
AF_INET
,
socket
.
SOCK_STREAM
)
sock
.
connect
(
(
self
.
_host
,
self
.
PORT
)
)
break
except
(
ConnectionError
,
BrokenPipeError
)
:
if
tries
==
3
:
print
(
"socket connect failed."
)
return
sleep
(
0.1
)
sock
.
send
(
codecs
.
decode
(
message
,
'hex_codec'
)
)
if
read_reply
:
sleep
(
0.1
)
reply
=
''
tries
=
0
max_tries
=
20
while
len
(
reply
)
<
len
(
message
)
and
tries
<
max_tries
:
try
:
reply
+=
codecs
.
encode
(
sock
.
recv
(
self
.
BUFFERSIZE
)
,
'hex'
)
.
decode
(
"utf-8"
)
except
(
ConnectionError
,
BrokenPipeError
)
:
pass
tries
+=
1
sock
.
close
(
)
if
tries
>=
max_tries
:
return
return
reply
sock
.
close
(
)

Send a command string to the amplifier .


def status(self):
        """
        Return the status of the device.

        Returns a dictionary with keys 'volume' (int 0-200) , 'power' (bool),
         'muted' (bool) and 'source' (str).
        """
        nad_reply = self._send(self.POLL_VOLUME +
                               self.POLL_POWER +
                               self.POLL_MUTED +
                               self.POLL_SOURCE, read_reply=True)
        if nad_reply is None:
            return

        # split reply into parts of 10 characters
        num_chars = 10
        nad_status = [nad_reply[i:i + num_chars]
                      for i in range(0, len(nad_reply), num_chars)]

        return {'volume': int(nad_status[0][-2:], 16),
                'power': nad_status[1][-2:] == '01',
                'muted': nad_status[2][-2:] == '01',
                'source': self.SOURCES_REVERSED[nad_status[3][-2:]]}


def power_off(self):
        """Power the device off."""
        status = self.status()
        if status['power']:  # Setting power off when it is already off can cause hangs
            self._send(self.CMD_POWERSAVE + self.CMD_OFF)


def power_on(self):
        """Power the device on."""
        status = self.status()
        if not status['power']:
            self._send(self.CMD_ON, read_reply=True)
            sleep(0.5)

def power_on ( self ) : status = self . status ( ) if not status [ 'power' ] : self . _send ( self . CMD_ON , read_reply = True ) sleep ( 0.5 )


def set_volume(self, volume):
        """Set volume level of the device. Accepts integer values 0-200."""
        if 0 <= volume <= 200:
            volume = format(volume, "02x")  # Convert to hex
            self._send(self.CMD_VOLUME + volume)

def
set_volume
(
self
,
volume
)
:
if
0
<=
volume
<=
200
:
volume
=
format
(
volume
,
"02x"
)
# Convert to hex
self
.
_send
(
self
.
CMD_VOLUME
+
volume
)

Set volume level of the device . Accepts integer values 0 - 200 .


def select_source(self, source):
        """Select a source from the list of sources."""
        status = self.status()
        if status['power']:  # Changing source when off may hang NAD7050
            if status['source'] != source:  # Setting the source to the current source will hang the NAD7050
                if source in self.SOURCES:
                    self._send(self.CMD_SOURCE + self.SOURCES[source], read_reply=True)

Select a source from the list of sources .


def deobfuscate(request, key, juice=None):
    """
    Deobfuscates the URL and returns HttpResponse from source view.
    SEO juice is mostly ignored as it is intended for display purposes only.
    """
    try:
        url = decrypt(str(key),
                      settings.UNFRIENDLY_SECRET,
                      settings.UNFRIENDLY_IV,
                      checksum=settings.UNFRIENDLY_ENFORCE_CHECKSUM)
    except (CheckSumError, InvalidKeyError):
        return HttpResponseNotFound()

    try:
        url = url.decode('utf-8')
    except UnicodeDecodeError:
        return HttpResponseNotFound()

    url_parts = urlparse(unquote(url))
    path = url_parts.path
    query = url_parts.query

    try:
        view, args, kwargs = resolve(path)
    except Resolver404:
        return HttpResponseNotFound()

    # fix-up the environ object
    environ = request.environ.copy()
    environ['PATH_INFO'] = path[len(environ['SCRIPT_NAME']):]
    environ['QUERY_STRING'] = query

    # init a new request
    patched_request = request.__class__(environ)

    # copy over any missing request attributes - this feels hackish
    missing_items = set(dir(request)) - set(dir(patched_request))
    while missing_items:
        missing_item = missing_items.pop()
        patched_request.__setattr__(missing_item,
                                    request.__getattribute__(missing_item))

    # mark this request as obfuscated
    patched_request.META['obfuscated'] = True

    response = view(patched_request, *args, **kwargs)

    # offer up a friendlier juice-powered filename if downloaded
    if juice and not response.has_header('Content-Disposition'):
        response['Content-Disposition'] = 'inline; filename=%s' % juice

    return response

Deobfuscates the URL and returns HttpResponse from source view . SEO juice is mostly ignored as it is intended for display purposes only .


def _crc(plaintext):
    """Generates crc32. Modulo keep the value within int range."""
    if not isinstance(plaintext, six.binary_type):
        plaintext = six.b(plaintext)
    return (zlib.crc32(plaintext) % 2147483647) & 0xffffffff

def _crc ( plaintext ) : if not isinstance ( plaintext , six . binary_type ) : plaintext = six . b ( plaintext ) return ( zlib . crc32 ( plaintext ) % 2147483647 ) & 0xffffffff

Generates crc32 . Modulo keep the value within int range .


def obfuscate(value, juice=None):
    """
    Template filter that obfuscates whatever text it is applied to. The text is
    supposed to be a URL, but it will obfuscate anything.

    Usage:
        Extremely unfriendly URL:
        {{ "/my-secret-path/"|obfuscate }}

        Include some SEO juice:
        {{ "/my-secret-path/"|obfuscate:"some SEO friendly text" }}
    """
    if not settings.UNFRIENDLY_ENABLE_FILTER:
        return value
    kwargs = {
        'key': encrypt(value,
                       settings.UNFRIENDLY_SECRET,
                       settings.UNFRIENDLY_IV,
                       checksum=settings.UNFRIENDLY_ENFORCE_CHECKSUM),
    }
    if juice:
        kwargs['juice'] = slugify(juice)
    return reverse('unfriendly-deobfuscate', kwargs=kwargs)

def obfuscate ( value , juice = None ) : if not settings . UNFRIENDLY_ENABLE_FILTER : return value kwargs = { 'key' : encrypt ( value , settings . UNFRIENDLY_SECRET , settings . UNFRIENDLY_IV , checksum = settings . UNFRIENDLY_ENFORCE_CHECKSUM ) , } if juice : kwargs [ 'juice' ] = slugify ( juice ) return reverse ( 'unfriendly-deobfuscate' , kwargs = kwargs )

Template filter that obfuscates whatever text it is applied to . The text is supposed to be a URL but it will obfuscate anything .


def missing_schema(self,html,song_name):
		'''
		It will print the list of songs that can be downloaded
		'''
		#html=self.get_html_response(url)
		soup=BeautifulSoup(html)
		name=' '.join(song_name)
		print '%s not found'%name
		print "But you can download any of the following songs :"
		a_list=soup.findAll('a','touch')
		for x in xrange(len(a_list)-1):
			r=a_list[x]
			p=str(r)
			q=re.sub(r'||||||','',p)
			print q


def list_of_all_href(self,html):
		'''
		It will return all hyper links found in the mr-jatt page for download
		'''
		soup=BeautifulSoup(html)
		links=[]
		a_list=soup.findAll('a','touch')
		for x in xrange(len(a_list)-1):
			link = a_list[x].get('href')
			name = a_list[x]
			name = str(name)
			name=re.sub(r'||||||','',name)
			name=re.sub(r'^[0-9]+\.','',name)
			links.append([link,name])

		#quit()
		return links

It will return all hyper links found in the mr - jatt page for download


def check_if_song_name(self,html):
		'''
		Returns true if user entered artist or movie name
		'''
		soup=BeautifulSoup(html)
		a_list=soup.findAll('a','touch')
		#print a_list
		text=[str(x) for x in a_list]
		text=''.join(text)
		text=text.lower()
		string1='download in 48 kbps'
		string2='download in 128 kbps'
		string3='download in 320 kbps'

		href=''
		if string3 in text:
			#print 'Downloading in 320 kbps'
			href=a_list[2].get('href')

		elif string2 in text:
			#print 'Downloading in 128 kbps'
			href=a_list[1].get('href')

		elif string1 in text:
			#print 'Downloading in 48 kbps'
			href=a_list[0].get('href')
		else:
			return (True,'nothing')

		return (False,href)

def
check_if_song_name
(
self
,
html
)
:
soup
=
BeautifulSoup
(
html
)
a_list
=
soup
.
findAll
(
'a'
,
'touch'
)
#print a_list
text
=
[
str
(
x
)
for
x
in
a_list
]
text
=
''
.
join
(
text
)
text
=
text
.
lower
(
)
string1
=
'download in 48 kbps'
string2
=
'download in 128 kbps'
string3
=
'download in 320 kbps'
href
=
''
if
string3
in
text
:
#print 'Downloading in 320 kbps'
href
=
a_list
[
2
]
.
get
(
'href'
)
elif
string2
in
text
:
#print 'Downloading in 128 kbps'
href
=
a_list
[
1
]
.
get
(
'href'
)
elif
string1
in
text
:
#print 'Downloading in 48 kbps'
href
=
a_list
[
0
]
.
get
(
'href'
)
else
:
return
(
True
,
'nothing'
)
return
(
False
,
href
)


def Parse(self,url,song_name,flag):
		'''
		It will the resource URL if song is found,
		Otherwise it will return the list of songs that can be downloaded
		'''
		file_download=FileDownload()
		html=file_download.get_html_response(url)
		if flag == False:
			soup=BeautifulSoup(html)
			a_list=soup.findAll('a','touch')
			#print a_list
			text=[str(x) for x in a_list]
			text=''.join(text)
			text=text.lower()
			string1='download in 48 kbps'
			string2='download in 128 kbps'
			string3='download in 320 kbps'

			href=''
			if string3 in text:
				print 'Downloading in 320 kbps'
				href=a_list[2].get('href')

			elif string2 in text:
				print 'Downloading in 128 kbps'
				href=a_list[1].get('href')

			elif string1 in text:
				print 'Downloading in 48 kbps'
				href=a_list[0].get('href')
			else:
				self.missing_schema(html,song_name)
				quit()

			return href
		else:
			x,href=self.check_if_song_name(html)
			links = []
			if x==True:
				links=self.list_of_all_href(html)
			else:
				file_download=FileDownload()
				file_download.file_download_cross_platform(href)
				quit()

			return links

def
Parse
(
self
,
url
,
song_name
,
flag
)
:
file_download
=
FileDownload
(
)
html
=
file_download
.
get_html_response
(
url
)
if
flag
==
False
:
soup
=
BeautifulSoup
(
html
)
a_list
=
soup
.
findAll
(
'a'
,
'touch'
)
#print a_list
text
=
[
str
(
x
)
for
x
in
a_list
]
text
=
''
.
join
(
text
)
text
=
text
.
lower
(
)
string1
=
'download in 48 kbps'
string2
=
'download in 128 kbps'
string3
=
'download in 320 kbps'
href
=
''
if
string3
in
text
:
print
'Downloading in 320 kbps'
href
=
a_list
[
2
]
.
get
(
'href'
)
elif
string2
in
text
:
print
'Downloading in 128 kbps'
href
=
a_list
[
1
]
.
get
(
'href'
)
elif
string1
in
text
:
print
'Downloading in 48 kbps'
href
=
a_list
[
0
]
.
get
(
'href'
)
else
:
self
.
missing_schema
(
html
,
song_name
)
quit
(
)
return
href
else
:
x
,
href
=
self
.
check_if_song_name
(
html
)
links
=
[
]
if
x
==
True
:
links
=
self
.
list_of_all_href
(
html
)
else
:
file_download
=
FileDownload
(
)
file_download
.
file_download_cross_platform
(
href
)
quit
(
)
return
links

It will the resource URL if song is found Otherwise it will return the list of songs that can be downloaded


def google_url(self,song_name,website):
		''' It will return the google url to be searched'''
		name='+'.join(song_name)
		prefix='https://www.google.co.in/search?q='
		website=website.split(" ")
		suffix='+'.join(website)
		url=prefix+name+suffix
		#print url
		return url

def
google_url
(
self
,
song_name
,
website
)
:
name
=
'+'
.
join
(
song_name
)
prefix
=
'https://www.google.co.in/search?q='
website
=
website
.
split
(
" "
)
suffix
=
'+'
.
join
(
website
)
url
=
prefix
+
name
+
suffix
#print url
return
url


def parse_google(self,html):
		'''It will parse google html response
			and return the first url
		 '''
		soup = BeautifulSoup(html)
		href=soup.find('div','g').find('a').get('href')
		href_list=href.split('&')
		download_url=href_list[0]
		download_url=download_url.strip()
		download_url=download_url.replace('/url?q=','')
		return download_url

def parse_google ( self , html ) : soup = BeautifulSoup ( html ) href = soup . find ( 'div' , 'g' ) . find ( 'a' ) . get ( 'href' ) href_list = href . split ( '&' ) download_url = href_list [ 0 ] download_url = download_url . strip ( ) download_url = download_url . replace ( '/url?q=' , '' ) return download_url


def Parse(self,song_name,website):
		'''
		song_name is a list of strings
		website is a string
		It will return the url from where music file needs to be downloaded
		'''
		url_to_be_parsed=self.google_url(song_name,website)
		file_download=FileDownload()
		html=file_download.get_html_response(url_to_be_parsed)
		website_url=self.parse_google(html)
		return website_url

def Parse ( self , song_name , website ) : url_to_be_parsed = self . google_url ( song_name , website ) file_download = FileDownload ( ) html = file_download . get_html_response ( url_to_be_parsed ) website_url = self . parse_google ( html ) return website_url


def get_html_response(self,url):
		'''It will download the html page specified by url and return the html response '''
		print "Downloading page %s .."%url
		try:
			response=requests.get(url,timeout=50)
		except requests.exceptions.SSLError:
			try:
				response=requests.get(url,verify=False,timeout=50)
			except requests.exceptions.RequestException as e:
				print e
				quit()
		except requests.exceptions.RequestException as e:
			print e
			quit()

		return response.content

def
get_html_response
(
self
,
url
)
:
print
"Downloading page %s .."
%
url
try
:
response
=
requests
.
get
(
url
,
timeout
=
50
)
except
requests
.
exceptions
.
SSLError
:
try
:
response
=
requests
.
get
(
url
,
verify
=
False
,
timeout
=
50
)
except
requests
.
exceptions
.
RequestException
as
e
:
print
e
quit
(
)
except
requests
.
exceptions
.
RequestException
as
e
:
print
e
quit
(
)
return
response
.
content


def file_download_using_requests(self,url):
		'''It will download file specified by url using requests module'''
		file_name=url.split('/')[-1]

		if os.path.exists(os.path.join(os.getcwd(),file_name)):
			print 'File already exists'
			return
		#print 'Downloading file %s '%file_name
		#print 'Downloading from %s'%url

		try:
			r=requests.get(url,stream=True,timeout=200)
		except requests.exceptions.SSLError:
			try:
				response=requests.get(url,stream=True,verify=False,timeout=200)
			except requests.exceptions.RequestException as e:
				print e
				quit()
		except requests.exceptions.RequestException as e:
				print e
				quit()
		chunk_size = 1024
		total_size = int(r.headers['Content-Length'])
		total_chunks = total_size/chunk_size

		file_iterable = r.iter_content(chunk_size = chunk_size)
		tqdm_iter = tqdm(iterable = file_iterable,total = total_chunks,unit = 'KB',
			leave = False
			)
		with open(file_name,'wb') as f:
			for data in tqdm_iter:
				f.write(data)

		#total_size=float(r.headers['Content-Length'])/(1024*1024)
		'''print 'Total size of file to be downloaded %.2f MB '%total_size
		total_downloaded_size=0.0
		with open(file_name,'wb') as f:
			for chunk in r.iter_content(chunk_size=1*1024*1024):
				if chunk:
					size_of_chunk=float(len(chunk))/(1024*1024)
					total_downloaded_size+=size_of_chunk
					print '{0:.0%} Downloaded'.format(total_downloaded_size/total_size)
					f.write(chunk)'''

		print 'Downloaded file %s '%file_name


def file_download_using_wget(self,url):
		'''It will download file specified by url using wget utility of linux '''
		file_name=url.split('/')[-1]
		print 'Downloading file %s '%file_name
		command='wget -c --read-timeout=50 --tries=3 -q --show-progress --no-check-certificate '
		url='"'+url+'"'
		command=command+url
		os.system(command)


def findStationCodesByCity(city_name, token):
    """Lookup AQI database for station codes in a given city."""
    req = requests.get(
        API_ENDPOINT_SEARCH,
        params={
            'token': token,
            'keyword': city_name
        })

    if req.status_code == 200 and req.json()["status"] == "ok":
        return [result["uid"] for result in req.json()["data"]]
    else:
        return []

def findStationCodesByCity ( city_name , token ) : req = requests . get ( API_ENDPOINT_SEARCH , params = { 'token' : token , 'keyword' : city_name } ) if req . status_code == 200 and req . json ( ) [ "status" ] == "ok" : return [ result [ "uid" ] for result in req . json ( ) [ "data" ] ] else : return [ ]

Lookup AQI database for station codes in a given city .


def get_location_observation(lat, lng, token):
    """Lookup observations by geo coordinates."""
    req = requests.get(
        API_ENDPOINT_GEO % (lat, lng),
        params={
            'token': token
        })

    if req.status_code == 200 and req.json()["status"] == "ok":
        return parse_observation_response(req.json()["data"])
    return {}

def get_location_observation ( lat , lng , token ) : req = requests . get ( API_ENDPOINT_GEO % ( lat , lng ) , params = { 'token' : token } ) if req . status_code == 200 and req . json ( ) [ "status" ] == "ok" : return parse_observation_response ( req . json ( ) [ "data" ] ) return { }

Lookup observations by geo coordinates .


def parse_observation_response(json):
    """Decode AQICN observation response JSON into python object."""
    logging.debug(json)

    iaqi = json['iaqi']
    result = {
        'idx': json['idx'],
        'city': json.get('city', ''),
        'aqi': json['aqi'],
        'dominentpol': json.get("dominentpol", ''),
        'time': json['time']['s'],
        'iaqi': [{'p': item, 'v': iaqi[item]['v']} for item in iaqi]
    }

    return result

def parse_observation_response ( json ) : logging . debug ( json ) iaqi = json [ 'iaqi' ] result = { 'idx' : json [ 'idx' ] , 'city' : json . get ( 'city' , '' ) , 'aqi' : json [ 'aqi' ] , 'dominentpol' : json . get ( "dominentpol" , '' ) , 'time' : json [ 'time' ] [ 's' ] , 'iaqi' : [ { 'p' : item , 'v' : iaqi [ item ] [ 'v' ] } for item in iaqi ] } return result

Decode AQICN observation response JSON into python object .


def get_station_observation(station_code, token):
    """Request station data for a specific station identified by code.

    A language parameter can also be specified to translate location
    information (default: "en")
    """
    req = requests.get(
        API_ENDPOINT_OBS % (station_code),
        params={
            'token': token
        })

    if req.status_code == 200 and req.json()['status'] == "ok":
        return parse_observation_response(req.json()['data'])
    else:
        return {}

def get_station_observation ( station_code , token ) : req = requests . get ( API_ENDPOINT_OBS % ( station_code ) , params = { 'token' : token } ) if req . status_code == 200 and req . json ( ) [ 'status' ] == "ok" : return parse_observation_response ( req . json ( ) [ 'data' ] ) else : return { }

Request station data for a specific station identified by code .


def search_paths(self):
        """The list of logical paths which are used to search for an asset.
        This property makes sense only if the attributes was created with
        logical path.

        It is assumed that the logical path can be a directory containing a
        file named ``index`` with the same suffix.

        Example::

            >>> attrs = AssetAttributes(environment, 'js/app.js')
            >>> attrs.search_paths
            ['js/app.js', 'js/app/index.js']

            >>> attrs = AssetAttributes(environment, 'js/app/index.js')
            >>> attrs.search_paths
            ['js/models/index.js']
        """
        paths = [self.path]
        if os.path.basename(self.path_without_suffix) != 'index':
            path = os.path.join(self.path_without_suffix, 'index')
            paths.append(path + ''.join(self.suffix))
        return paths

def search_paths ( self ) : paths = [ self . path ] if os . path . basename ( self . path_without_suffix ) != 'index' : path = os . path . join ( self . path_without_suffix , 'index' ) paths . append ( path + '' . join ( self . suffix ) ) return paths

The list of logical paths which are used to search for an asset . This property makes sense only if the attributes was created with logical path .


def compilers(self):
        """The list of compilers used to build asset."""
        return [self.environment.compilers.get(e) for e in self.compiler_extensions]

def compilers ( self ) : return [ self . environment . compilers . get ( e ) for e in self . compiler_extensions ]

The list of compilers used to build asset .


def mimetype(self):
        """MIME type of the asset."""
        return (self.environment.mimetypes.get(self.format_extension) or
                self.compiler_mimetype or 'application/octet-stream')

def mimetype ( self ) : return ( self . environment . mimetypes . get ( self . format_extension ) or self . compiler_mimetype or 'application/octet-stream' )


def compiler_mimetype(self):
        """Implicit MIME type of the asset by its compilers."""
        for compiler in reversed(self.compilers):
            if compiler.result_mimetype:
                return compiler.result_mimetype
        return None

def compiler_mimetype ( self ) : for compiler in reversed ( self . compilers ) : if compiler . result_mimetype : return compiler . result_mimetype return None

Implicit MIME type of the asset by its compilers .


def compiler_format_extension(self):
        """Implicit format extension on the asset by its compilers."""
        for extension, mimetype in self.environment.mimetypes.items():
            if mimetype == self.compiler_mimetype:
                return extension
        return None

def compiler_format_extension ( self ) : for extension , mimetype in self . environment . mimetypes . items ( ) : if mimetype == self . compiler_mimetype : return extension return None

Implicit format extension on the asset by its compilers .


def register(self, mimetype, processor):
        """Register passed `processor` for passed `mimetype`."""
        if mimetype not in self or processor not in self[mimetype]:
            self.setdefault(mimetype, []).append(processor)

def register ( self , mimetype , processor ) : if mimetype not in self or processor not in self [ mimetype ] : self . setdefault ( mimetype , [ ] ) . append ( processor )

Register passed processor for passed mimetype .


def unregister(self, mimetype, processor):
        """Remove passed `processor` for passed `mimetype`. If processor for
        this MIME type does not found in the registry, nothing happens.
        """
        if mimetype in self and processor in self[mimetype]:
            self[mimetype].remove(processor)

def unregister ( self , mimetype , processor ) : if mimetype in self and processor in self [ mimetype ] : self [ mimetype ] . remove ( processor )

Remove passed processor for passed mimetype . If processor for this MIME type does not found in the registry nothing happens .


def paths(self):
        """The list of search paths. It is built from registered finders, which
        has ``paths`` property. Can be useful for compilers to resolve internal
        dependencies.
        """
        if not hasattr(self, '_paths'):
            paths = []
            for finder in self.finders:
                if hasattr(finder, 'paths'):
                    paths.extend(finder.paths)
            self._paths = paths
        return self._paths

def paths ( self ) : if not hasattr ( self , '_paths' ) : paths = [ ] for finder in self . finders : if hasattr ( finder , 'paths' ) : paths . extend ( finder . paths ) self . _paths = paths return self . _paths

The list of search paths . It is built from registered finders which has paths property . Can be useful for compilers to resolve internal dependencies .


def register_defaults(self):
        """Register default compilers, preprocessors and MIME types."""
        self.mimetypes.register_defaults()
        self.preprocessors.register_defaults()
        self.postprocessors.register_defaults()

def register_defaults ( self ) : self . mimetypes . register_defaults ( ) self . preprocessors . register_defaults ( ) self . postprocessors . register_defaults ( )

Register default compilers preprocessors and MIME types .