def perf_stats_bootstrap(returns, factor_returns=None, return_stats=True,
                         **kwargs):
    """Calculates various bootstrapped performance metrics of a strategy.

    Parameters
    ----------
    returns : pd.Series
        Daily returns of the strategy, noncumulative.
         - See full explanation in tears.create_full_tear_sheet.
    factor_returns : pd.Series, optional
        Daily noncumulative returns of the benchmark factor to which betas are
        computed. Usually a benchmark such as market returns.
         - This is in the same style as returns.
         - If None, do not compute alpha, beta, and information ratio.
    return_stats : boolean (optional)
        If True, returns a DataFrame of mean, median, 5 and 95 percentiles
        for each perf metric.
        If False, returns a DataFrame with the bootstrap samples for
        each perf metric.

    Returns
    -------
    pd.DataFrame
        if return_stats is True:
        - Distributional statistics of bootstrapped sampling
        distribution of performance metrics.
        if return_stats is False:
        - Bootstrap samples for each performance metric.
    """

    bootstrap_values = OrderedDict()

    for stat_func in SIMPLE_STAT_FUNCS:
        stat_name = STAT_FUNC_NAMES[stat_func.__name__]
        bootstrap_values[stat_name] = calc_bootstrap(stat_func,
                                                     returns)

    if factor_returns is not None:
        for stat_func in FACTOR_STAT_FUNCS:
            stat_name = STAT_FUNC_NAMES[stat_func.__name__]
            bootstrap_values[stat_name] = calc_bootstrap(
                stat_func,
                returns,
                factor_returns=factor_returns)

    bootstrap_values = pd.DataFrame(bootstrap_values)

    if return_stats:
        stats = bootstrap_values.apply(calc_distribution_stats)
        return stats.T[['mean', 'median', '5%', '95%']]
    else:
        return bootstrap_values

def perf_stats_bootstrap ( returns , factor_returns = None , return_stats = True , * * kwargs ) : bootstrap_values = OrderedDict ( ) for stat_func in SIMPLE_STAT_FUNCS : stat_name = STAT_FUNC_NAMES [ stat_func . __name__ ] bootstrap_values [ stat_name ] = calc_bootstrap ( stat_func , returns ) if factor_returns is not None : for stat_func in FACTOR_STAT_FUNCS : stat_name = STAT_FUNC_NAMES [ stat_func . __name__ ] bootstrap_values [ stat_name ] = calc_bootstrap ( stat_func , returns , factor_returns = factor_returns ) bootstrap_values = pd . DataFrame ( bootstrap_values ) if return_stats : stats = bootstrap_values . apply ( calc_distribution_stats ) return stats . T [ [ 'mean' , 'median' , '5%' , '95%' ] ] else : return bootstrap_values

Calculates various bootstrapped performance metrics of a strategy .


def calc_bootstrap(func, returns, *args, **kwargs):
    """Performs a bootstrap analysis on a user-defined function returning
    a summary statistic.

    Parameters
    ----------
    func : function
        Function that either takes a single array (commonly returns)
        or two arrays (commonly returns and factor returns) and
        returns a single value (commonly a summary
        statistic). Additional args and kwargs are passed as well.
    returns : pd.Series
        Daily returns of the strategy, noncumulative.
         - See full explanation in tears.create_full_tear_sheet.
    factor_returns : pd.Series, optional
        Daily noncumulative returns of the benchmark factor to which betas are
        computed. Usually a benchmark such as market returns.
         - This is in the same style as returns.
    n_samples : int, optional
        Number of bootstrap samples to draw. Default is 1000.
        Increasing this will lead to more stable / accurate estimates.

    Returns
    -------
    numpy.ndarray
        Bootstrapped sampling distribution of passed in func.
    """

    n_samples = kwargs.pop('n_samples', 1000)
    out = np.empty(n_samples)

    factor_returns = kwargs.pop('factor_returns', None)

    for i in range(n_samples):
        idx = np.random.randint(len(returns), size=len(returns))
        returns_i = returns.iloc[idx].reset_index(drop=True)
        if factor_returns is not None:
            factor_returns_i = factor_returns.iloc[idx].reset_index(drop=True)
            out[i] = func(returns_i, factor_returns_i,
                          *args, **kwargs)
        else:
            out[i] = func(returns_i,
                          *args, **kwargs)

    return out

def calc_bootstrap ( func , returns , * args , * * kwargs ) : n_samples = kwargs . pop ( 'n_samples' , 1000 ) out = np . empty ( n_samples ) factor_returns = kwargs . pop ( 'factor_returns' , None ) for i in range ( n_samples ) : idx = np . random . randint ( len ( returns ) , size = len ( returns ) ) returns_i = returns . iloc [ idx ] . reset_index ( drop = True ) if factor_returns is not None : factor_returns_i = factor_returns . iloc [ idx ] . reset_index ( drop = True ) out [ i ] = func ( returns_i , factor_returns_i , * args , * * kwargs ) else : out [ i ] = func ( returns_i , * args , * * kwargs ) return out

Performs a bootstrap analysis on a user - defined function returning a summary statistic .


def calc_distribution_stats(x):
    """Calculate various summary statistics of data.

    Parameters
    ----------
    x : numpy.ndarray or pandas.Series
        Array to compute summary statistics for.

    Returns
    -------
    pandas.Series
        Series containing mean, median, std, as well as 5, 25, 75 and
        95 percentiles of passed in values.
    """

    return pd.Series({'mean': np.mean(x),
                      'median': np.median(x),
                      'std': np.std(x),
                      '5%': np.percentile(x, 5),
                      '25%': np.percentile(x, 25),
                      '75%': np.percentile(x, 75),
                      '95%': np.percentile(x, 95),
                      'IQR': np.subtract.reduce(
                          np.percentile(x, [75, 25])),
                      })

def calc_distribution_stats ( x ) : return pd . Series ( { 'mean' : np . mean ( x ) , 'median' : np . median ( x ) , 'std' : np . std ( x ) , '5%' : np . percentile ( x , 5 ) , '25%' : np . percentile ( x , 25 ) , '75%' : np . percentile ( x , 75 ) , '95%' : np . percentile ( x , 95 ) , 'IQR' : np . subtract . reduce ( np . percentile ( x , [ 75 , 25 ] ) ) , } )

Calculate various summary statistics of data .


def get_max_drawdown_underwater(underwater):
    """
    Determines peak, valley, and recovery dates given an 'underwater'
    DataFrame.

    An underwater DataFrame is a DataFrame that has precomputed
    rolling drawdown.

    Parameters
    ----------
    underwater : pd.Series
       Underwater returns (rolling drawdown) of a strategy.

    Returns
    -------
    peak : datetime
        The maximum drawdown's peak.
    valley : datetime
        The maximum drawdown's valley.
    recovery : datetime
        The maximum drawdown's recovery.
    """

    valley = np.argmin(underwater)  # end of the period
    # Find first 0
    peak = underwater[:valley][underwater[:valley] == 0].index[-1]
    # Find last 0
    try:
        recovery = underwater[valley:][underwater[valley:] == 0].index[0]
    except IndexError:
        recovery = np.nan  # drawdown not recovered
    return peak, valley, recovery

def
get_max_drawdown_underwater
(
underwater
)
:
valley
=
np
.
argmin
(
underwater
)
# end of the period
# Find first 0
peak
=
underwater
[
:
valley
]
[
underwater
[
:
valley
]
==
0
]
.
index
[
1
]
# Find last 0
try
:
recovery
=
underwater
[
valley
:
]
[
underwater
[
valley
:
]
==
0
]
.
index
[
0
]
except
IndexError
:
recovery
=
np
.
# drawdown not recovered
return
peak
,
valley
,
recovery

Determines peak valley and recovery dates given an underwater DataFrame .


def get_max_drawdown(returns):
    """
    Determines the maximum drawdown of a strategy.

    Parameters
    ----------
    returns : pd.Series
        Daily returns of the strategy, noncumulative.
        - See full explanation in :func:`~pyfolio.timeseries.cum_returns`.

    Returns
    -------
    float
        Maximum drawdown.

    Note
    -----
    See https://en.wikipedia.org/wiki/Drawdown_(economics) for more details.
    """

    returns = returns.copy()
    df_cum = cum_returns(returns, 1.0)
    running_max = np.maximum.accumulate(df_cum)
    underwater = df_cum / running_max - 1
    return get_max_drawdown_underwater(underwater)

def get_max_drawdown ( returns ) : returns = returns . copy ( ) df_cum = cum_returns ( returns , 1.0 ) running_max = np . maximum . accumulate ( df_cum ) underwater = df_cum / running_max - 1 return get_max_drawdown_underwater ( underwater )

Determines the maximum drawdown of a strategy .


def get_top_drawdowns(returns, top=10):
    """
    Finds top drawdowns, sorted by drawdown amount.

    Parameters
    ----------
    returns : pd.Series
        Daily returns of the strategy, noncumulative.
         - See full explanation in tears.create_full_tear_sheet.
    top : int, optional
        The amount of top drawdowns to find (default 10).

    Returns
    -------
    drawdowns : list
        List of drawdown peaks, valleys, and recoveries. See get_max_drawdown.
    """

    returns = returns.copy()
    df_cum = ep.cum_returns(returns, 1.0)
    running_max = np.maximum.accumulate(df_cum)
    underwater = df_cum / running_max - 1

    drawdowns = []
    for t in range(top):
        peak, valley, recovery = get_max_drawdown_underwater(underwater)
        # Slice out draw-down period
        if not pd.isnull(recovery):
            underwater.drop(underwater[peak: recovery].index[1:-1],
                            inplace=True)
        else:
            # drawdown has not ended yet
            underwater = underwater.loc[:peak]

        drawdowns.append((peak, valley, recovery))
        if (len(returns) == 0) or (len(underwater) == 0):
            break

    return drawdowns

def
get_top_drawdowns
(
returns
,
top
=
10
)
:
returns
=
returns
.
copy
(
)
df_cum
=
ep
.
cum_returns
(
returns
,
1.0
)
running_max
=
np
.
maximum
.
accumulate
(
df_cum
)
underwater
=
df_cum
/
running_max
1
drawdowns
=
[
]
for
t
in
range
(
top
)
:
peak
,
valley
,
recovery
=
get_max_drawdown_underwater
(
underwater
)
# Slice out draw-down period
if
not
pd
.
isnull
(
recovery
)
:
underwater
.
drop
(
underwater
[
peak
:
recovery
]
.
index
[
1
:
1
]
,
inplace
=
True
)
else
:
# drawdown has not ended yet
underwater
=
underwater
.
loc
[
:
peak
]
drawdowns
.
append
(
(
peak
,
valley
,
recovery
)
)
if
(
len
(
returns
)
==
0
)
or
(
len
(
underwater
)
==
0
)
:
break
return
drawdowns

Finds top drawdowns sorted by drawdown amount .


def gen_drawdown_table(returns, top=10):
    """
    Places top drawdowns in a table.

    Parameters
    ----------
    returns : pd.Series
        Daily returns of the strategy, noncumulative.
         - See full explanation in tears.create_full_tear_sheet.
    top : int, optional
        The amount of top drawdowns to find (default 10).

    Returns
    -------
    df_drawdowns : pd.DataFrame
        Information about top drawdowns.
    """

    df_cum = ep.cum_returns(returns, 1.0)
    drawdown_periods = get_top_drawdowns(returns, top=top)
    df_drawdowns = pd.DataFrame(index=list(range(top)),
                                columns=['Net drawdown in %',
                                         'Peak date',
                                         'Valley date',
                                         'Recovery date',
                                         'Duration'])

    for i, (peak, valley, recovery) in enumerate(drawdown_periods):
        if pd.isnull(recovery):
            df_drawdowns.loc[i, 'Duration'] = np.nan
        else:
            df_drawdowns.loc[i, 'Duration'] = len(pd.date_range(peak,
                                                                recovery,
                                                                freq='B'))
        df_drawdowns.loc[i, 'Peak date'] = (peak.to_pydatetime()
                                            .strftime('%Y-%m-%d'))
        df_drawdowns.loc[i, 'Valley date'] = (valley.to_pydatetime()
                                              .strftime('%Y-%m-%d'))
        if isinstance(recovery, float):
            df_drawdowns.loc[i, 'Recovery date'] = recovery
        else:
            df_drawdowns.loc[i, 'Recovery date'] = (recovery.to_pydatetime()
                                                    .strftime('%Y-%m-%d'))
        df_drawdowns.loc[i, 'Net drawdown in %'] = (
            (df_cum.loc[peak] - df_cum.loc[valley]) / df_cum.loc[peak]) * 100

    df_drawdowns['Peak date'] = pd.to_datetime(df_drawdowns['Peak date'])
    df_drawdowns['Valley date'] = pd.to_datetime(df_drawdowns['Valley date'])
    df_drawdowns['Recovery date'] = pd.to_datetime(
        df_drawdowns['Recovery date'])

    return df_drawdowns

def
gen_drawdown_table
(
returns
,
top
=
10
)
:
df_cum
=
ep
.
cum_returns
(
returns
,
1.0
)
drawdown_periods
=
get_top_drawdowns
(
returns
,
top
=
top
)
df_drawdowns
=
pd
.
DataFrame
(
index
=
list
(
range
(
top
)
)
,
columns
=
[
'Net drawdown in %'
,
'Peak date'
,
'Valley date'
,
'Recovery date'
,
'Duration'
]
)
for
i
,
(
peak
,
valley
,
recovery
)
in
enumerate
(
drawdown_periods
)
:
if
pd
.
isnull
(
recovery
)
:
df_drawdowns
.
loc
[
i
,
'Duration'
]
=
np
.
else
:
df_drawdowns
.
loc
[
i
,
'Duration'
]
=
len
(
pd
.
date_range
(
peak
,
recovery
,
freq
=
'B'
)
)
df_drawdowns
.
loc
[
i
,
'Peak date'
]
=
(
peak
.
to_pydatetime
(
)
.
strftime
(
'%Y-%m-%d'
)
)
df_drawdowns
.
loc
[
i
,
'Valley date'
]
=
(
valley
.
to_pydatetime
(
)
.
strftime
(
'%Y-%m-%d'
)
)
if
isinstance
(
recovery
,
float
)
:
df_drawdowns
.
loc
[
i
,
'Recovery date'
]
=
recovery
else
:
df_drawdowns
.
loc
[
i
,
'Recovery date'
]
=
(
recovery
.
to_pydatetime
(
)
.
strftime
(
'%Y-%m-%d'
)
)
df_drawdowns
.
loc
[
i
,
'Net drawdown in %'
]
=
(
(
df_cum
.
loc
[
peak
]
df_cum
.
loc
[
valley
]
)
/
df_cum
.
loc
[
peak
]
)
*
100
df_drawdowns
[
'Peak date'
]
=
pd
.
to_datetime
(
df_drawdowns
[
'Peak date'
]
)
df_drawdowns
[
'Valley date'
]
=
pd
.
to_datetime
(
df_drawdowns
[
'Valley date'
]
)
df_drawdowns
[
'Recovery date'
]
=
pd
.
to_datetime
(
df_drawdowns
[
'Recovery date'
]
)
return
df_drawdowns


def rolling_volatility(returns, rolling_vol_window):
    """
    Determines the rolling volatility of a strategy.

    Parameters
    ----------
    returns : pd.Series
        Daily returns of the strategy, noncumulative.
         - See full explanation in tears.create_full_tear_sheet.
    rolling_vol_window : int
        Length of rolling window, in days, over which to compute.

    Returns
    -------
    pd.Series
        Rolling volatility.
    """

    return returns.rolling(rolling_vol_window).std() \
        * np.sqrt(APPROX_BDAYS_PER_YEAR)

def rolling_volatility ( returns , rolling_vol_window ) : return returns . rolling ( rolling_vol_window ) . std ( ) * np . sqrt ( APPROX_BDAYS_PER_YEAR )

Determines the rolling volatility of a strategy .


def rolling_sharpe(returns, rolling_sharpe_window):
    """
    Determines the rolling Sharpe ratio of a strategy.

    Parameters
    ----------
    returns : pd.Series
        Daily returns of the strategy, noncumulative.
         - See full explanation in tears.create_full_tear_sheet.
    rolling_sharpe_window : int
        Length of rolling window, in days, over which to compute.

    Returns
    -------
    pd.Series
        Rolling Sharpe ratio.

    Note
    -----
    See https://en.wikipedia.org/wiki/Sharpe_ratio for more details.
    """

    return returns.rolling(rolling_sharpe_window).mean() \
        / returns.rolling(rolling_sharpe_window).std() \
        * np.sqrt(APPROX_BDAYS_PER_YEAR)

def rolling_sharpe ( returns , rolling_sharpe_window ) : return returns . rolling ( rolling_sharpe_window ) . mean ( ) / returns . rolling ( rolling_sharpe_window ) . std ( ) * np . sqrt ( APPROX_BDAYS_PER_YEAR )

Determines the rolling Sharpe ratio of a strategy .


def simulate_paths(is_returns, num_days,
                   starting_value=1, num_samples=1000, random_seed=None):
    """
    Gnerate alternate paths using available values from in-sample returns.

    Parameters
    ----------
    is_returns : pandas.core.frame.DataFrame
        Non-cumulative in-sample returns.
    num_days : int
        Number of days to project the probability cone forward.
    starting_value : int or float
        Starting value of the out of sample period.
    num_samples : int
        Number of samples to draw from the in-sample daily returns.
        Each sample will be an array with length num_days.
        A higher number of samples will generate a more accurate
        bootstrap cone.
    random_seed : int
        Seed for the pseudorandom number generator used by the pandas
        sample method.

    Returns
    -------
    samples : numpy.ndarray
    """

    samples = np.empty((num_samples, num_days))
    seed = np.random.RandomState(seed=random_seed)
    for i in range(num_samples):
        samples[i, :] = is_returns.sample(num_days, replace=True,
                                          random_state=seed)

    return samples

def simulate_paths ( is_returns , num_days , starting_value = 1 , num_samples = 1000 , random_seed = None ) : samples = np . empty ( ( num_samples , num_days ) ) seed = np . random . RandomState ( seed = random_seed ) for i in range ( num_samples ) : samples [ i , : ] = is_returns . sample ( num_days , replace = True , random_state = seed ) return samples

Gnerate alternate paths using available values from in - sample returns .


def summarize_paths(samples, cone_std=(1., 1.5, 2.), starting_value=1.):
    """
    Gnerate the upper and lower bounds of an n standard deviation
    cone of forecasted cumulative returns.

    Parameters
    ----------
    samples : numpy.ndarray
        Alternative paths, or series of possible outcomes.
    cone_std : list of int/float
        Number of standard devations to use in the boundaries of
        the cone. If multiple values are passed, cone bounds will
        be generated for each value.

    Returns
    -------
    samples : pandas.core.frame.DataFrame
    """

    cum_samples = ep.cum_returns(samples.T,
                                 starting_value=starting_value).T

    cum_mean = cum_samples.mean(axis=0)
    cum_std = cum_samples.std(axis=0)

    if isinstance(cone_std, (float, int)):
        cone_std = [cone_std]

    cone_bounds = pd.DataFrame(columns=pd.Float64Index([]))
    for num_std in cone_std:
        cone_bounds.loc[:, float(num_std)] = cum_mean + cum_std * num_std
        cone_bounds.loc[:, float(-num_std)] = cum_mean - cum_std * num_std

    return cone_bounds

def summarize_paths ( samples , cone_std = ( 1. , 1.5 , 2. ) , starting_value = 1. ) : cum_samples = ep . cum_returns ( samples . T , starting_value = starting_value ) . T cum_mean = cum_samples . mean ( axis = 0 ) cum_std = cum_samples . std ( axis = 0 ) if isinstance ( cone_std , ( float , int ) ) : cone_std = [ cone_std ] cone_bounds = pd . DataFrame ( columns = pd . Float64Index ( [ ] ) ) for num_std in cone_std : cone_bounds . loc [ : , float ( num_std ) ] = cum_mean + cum_std * num_std cone_bounds . loc [ : , float ( - num_std ) ] = cum_mean - cum_std * num_std return cone_bounds

Gnerate the upper and lower bounds of an n standard deviation cone of forecasted cumulative returns .


def extract_interesting_date_ranges(returns):
    """
    Extracts returns based on interesting events. See
    gen_date_range_interesting.

    Parameters
    ----------
    returns : pd.Series
        Daily returns of the strategy, noncumulative.
         - See full explanation in tears.create_full_tear_sheet.

    Returns
    -------
    ranges : OrderedDict
        Date ranges, with returns, of all valid events.
    """

    returns_dupe = returns.copy()
    returns_dupe.index = returns_dupe.index.map(pd.Timestamp)
    ranges = OrderedDict()
    for name, (start, end) in PERIODS.items():
        try:
            period = returns_dupe.loc[start:end]
            if len(period) == 0:
                continue
            ranges[name] = period
        except BaseException:
            continue

    return ranges

def extract_interesting_date_ranges ( returns ) : returns_dupe = returns . copy ( ) returns_dupe . index = returns_dupe . index . map ( pd . Timestamp ) ranges = OrderedDict ( ) for name , ( start , end ) in PERIODS . items ( ) : try : period = returns_dupe . loc [ start : end ] if len ( period ) == 0 : continue ranges [ name ] = period except BaseException : continue return ranges

Extracts returns based on interesting events . See gen_date_range_interesting .


def model_returns_t_alpha_beta(data, bmark, samples=2000, progressbar=True):
    """
    Run Bayesian alpha-beta-model with T distributed returns.

    This model estimates intercept (alpha) and slope (beta) of two
    return sets. Usually, these will be algorithm returns and
    benchmark returns (e.g. S&P500). The data is assumed to be T
    distributed and thus is robust to outliers and takes tail events
    into account.  If a pandas.DataFrame is passed as a benchmark, then
    multiple linear regression is used to estimate alpha and beta.

    Parameters
    ----------
    returns : pandas.Series
        Series of simple returns of an algorithm or stock.
    bmark : pandas.DataFrame
        DataFrame of benchmark returns (e.g., S&P500) or risk factors (e.g.,
        Fama-French SMB, HML, and UMD).
        If bmark has more recent returns than returns_train, these dates
        will be treated as missing values and predictions will be
        generated for them taking market correlations into account.
    samples : int (optional)
        Number of posterior samples to draw.

    Returns
    -------
    model : pymc.Model object
        PyMC3 model containing all random variables.
    trace : pymc3.sampling.BaseTrace object
        A PyMC3 trace object that contains samples for each parameter
        of the posterior.
    """

    data_bmark = pd.concat([data, bmark], axis=1).dropna()

    with pm.Model() as model:
        sigma = pm.HalfCauchy(
            'sigma',
            beta=1)
        nu = pm.Exponential('nu_minus_two', 1. / 10.)

        # alpha and beta
        X = data_bmark.iloc[:, 1]
        y = data_bmark.iloc[:, 0]

        alpha_reg = pm.Normal('alpha', mu=0, sd=.1)
        beta_reg = pm.Normal('beta', mu=0, sd=1)

        mu_reg = alpha_reg + beta_reg * X
        pm.StudentT('returns',
                    nu=nu + 2,
                    mu=mu_reg,
                    sd=sigma,
                    observed=y)
        trace = pm.sample(samples, progressbar=progressbar)

    return model, trace

def
model_returns_t_alpha_beta
(
data
,
bmark
,
samples
=
2000
,
progressbar
=
True
)
:
data_bmark
=
pd
.
concat
(
[
data
,
bmark
]
,
axis
=
1
)
.
dropna
(
)
with
pm
.
Model
(
)
as
model
:
sigma
=
pm
.
HalfCauchy
(
'sigma'
,
beta
=
1
)
nu
=
pm
.
Exponential
(
'nu_minus_two'
,
1.
/
10.
)
# alpha and beta
X
=
data_bmark
.
iloc
[
:
,
1
]
y
=
data_bmark
.
iloc
[
:
,
0
]
alpha_reg
=
pm
.
Normal
(
'alpha'
,
mu
=
0
,
sd
=
.1
)
beta_reg
=
pm
.
Normal
(
'beta'
,
mu
=
0
,
sd
=
1
)
mu_reg
=
alpha_reg
+
beta_reg
*
X
pm
.
StudentT
(
'returns'
,
nu
=
nu
+
2
,
mu
=
mu_reg
,
sd
=
sigma
,
observed
=
y
)
trace
=
pm
.
sample
(
samples
,
progressbar
=
progressbar
)
return
model
,
trace

Run Bayesian alpha - beta - model with T distributed returns .


def model_returns_normal(data, samples=500, progressbar=True):
    """
    Run Bayesian model assuming returns are normally distributed.

    Parameters
    ----------
    returns : pandas.Series
        Series of simple returns of an algorithm or stock.
    samples : int (optional)
        Number of posterior samples to draw.

    Returns
    -------
    model : pymc.Model object
        PyMC3 model containing all random variables.
    trace : pymc3.sampling.BaseTrace object
        A PyMC3 trace object that contains samples for each parameter
        of the posterior.
    """

    with pm.Model() as model:
        mu = pm.Normal('mean returns', mu=0, sd=.01, testval=data.mean())
        sigma = pm.HalfCauchy('volatility', beta=1, testval=data.std())
        returns = pm.Normal('returns', mu=mu, sd=sigma, observed=data)
        pm.Deterministic(
            'annual volatility',
            returns.distribution.variance**.5 *
            np.sqrt(252))
        pm.Deterministic(
            'sharpe',
            returns.distribution.mean /
            returns.distribution.variance**.5 *
            np.sqrt(252))

        trace = pm.sample(samples, progressbar=progressbar)
    return model, trace

def
model_returns_normal
(
data
,
samples
=
500
,
progressbar
=
True
)
:
with
pm
.
Model
(
)
as
model
:
mu
=
pm
.
Normal
(
'mean returns'
,
mu
=
0
,
sd
=
.01
,
testval
=
data
.
mean
(
)
)
sigma
=
pm
.
HalfCauchy
(
'volatility'
,
beta
=
1
,
testval
=
data
.
std
(
)
)
returns
=
pm
.
Normal
(
'returns'
,
mu
=
mu
,
sd
=
sigma
,
observed
=
data
)
pm
.
Deterministic
(
'annual volatility'
,
returns
.
distribution
.
variance
**
.5
*
np
.
sqrt
(
252
)
)
pm
.
Deterministic
(
'sharpe'
,
returns
.
distribution
.
mean
/
returns
.
distribution
.
variance
**
.5
*
np
.
sqrt
(
252
)
)
trace
=
pm
.
sample
(
samples
,
progressbar
=
progressbar
)
return
model
,
trace

Run Bayesian model assuming returns are normally distributed .


def model_best(y1, y2, samples=1000, progressbar=True):
    """
    Bayesian Estimation Supersedes the T-Test

    This model runs a Bayesian hypothesis comparing if y1 and y2 come
    from the same distribution. Returns are assumed to be T-distributed.

    In addition, computes annual volatility and Sharpe of in and
    out-of-sample periods.

    This model replicates the example used in:
    Kruschke, John. (2012) Bayesian estimation supersedes the t
    test. Journal of Experimental Psychology: General.

    Parameters
    ----------
    y1 : array-like
        Array of returns (e.g. in-sample)
    y2 : array-like
        Array of returns (e.g. out-of-sample)
    samples : int, optional
        Number of posterior samples to draw.

    Returns
    -------
    model : pymc.Model object
        PyMC3 model containing all random variables.
    trace : pymc3.sampling.BaseTrace object
        A PyMC3 trace object that contains samples for each parameter
        of the posterior.

    See Also
    --------
    plot_stoch_vol : plotting of tochastic volatility model
    """

    y = np.concatenate((y1, y2))

    mu_m = np.mean(y)
    mu_p = 0.000001 * 1 / np.std(y)**2

    sigma_low = np.std(y) / 1000
    sigma_high = np.std(y) * 1000
    with pm.Model() as model:
        group1_mean = pm.Normal('group1_mean', mu=mu_m, tau=mu_p,
                                testval=y1.mean())
        group2_mean = pm.Normal('group2_mean', mu=mu_m, tau=mu_p,
                                testval=y2.mean())
        group1_std = pm.Uniform('group1_std', lower=sigma_low,
                                upper=sigma_high, testval=y1.std())
        group2_std = pm.Uniform('group2_std', lower=sigma_low,
                                upper=sigma_high, testval=y2.std())
        nu = pm.Exponential('nu_minus_two', 1 / 29., testval=4.) + 2.

        returns_group1 = pm.StudentT('group1', nu=nu, mu=group1_mean,
                                     lam=group1_std**-2, observed=y1)
        returns_group2 = pm.StudentT('group2', nu=nu, mu=group2_mean,
                                     lam=group2_std**-2, observed=y2)

        diff_of_means = pm.Deterministic('difference of means',
                                         group2_mean - group1_mean)
        pm.Deterministic('difference of stds',
                         group2_std - group1_std)
        pm.Deterministic('effect size', diff_of_means /
                         pm.math.sqrt((group1_std**2 +
                                       group2_std**2) / 2))

        pm.Deterministic('group1_annual_volatility',
                         returns_group1.distribution.variance**.5 *
                         np.sqrt(252))
        pm.Deterministic('group2_annual_volatility',
                         returns_group2.distribution.variance**.5 *
                         np.sqrt(252))

        pm.Deterministic('group1_sharpe', returns_group1.distribution.mean /
                         returns_group1.distribution.variance**.5 *
                         np.sqrt(252))
        pm.Deterministic('group2_sharpe', returns_group2.distribution.mean /
                         returns_group2.distribution.variance**.5 *
                         np.sqrt(252))

        trace = pm.sample(samples, progressbar=progressbar)
    return model, trace

def
model_best
(
y1
,
y2
,
samples
=
1000
,
progressbar
=
True
)
:
y
=
np
.
concatenate
(
(
y1
,
y2
)
)
mu_m
=
np
.
mean
(
y
)
mu_p
=
0.000001
*
1
/
np
.
std
(
y
)
**
2
sigma_low
=
np
.
std
(
y
)
/
1000
sigma_high
=
np
.
std
(
y
)
*
1000
with
pm
.
Model
(
)
as
model
:
group1_mean
=
pm
.
Normal
(
'group1_mean'
,
mu
=
mu_m
,
tau
=
mu_p
,
testval
=
y1
.
mean
(
)
)
group2_mean
=
pm
.
Normal
(
'group2_mean'
,
mu
=
mu_m
,
tau
=
mu_p
,
testval
=
y2
.
mean
(
)
)
group1_std
=
pm
.
Uniform
(
'group1_std'
,
lower
=
sigma_low
,
upper
=
sigma_high
,
testval
=
y1
.
std
(
)
)
group2_std
=
pm
.
Uniform
(
'group2_std'
,
lower
=
sigma_low
,
upper
=
sigma_high
,
testval
=
y2
.
std
(
)
)
nu
=
pm
.
Exponential
(
'nu_minus_two'
,
1
/
29.
,
testval
=
4.
)
+
2.
returns_group1
=
pm
.
StudentT
(
'group1'
,
nu
=
nu
,
mu
=
group1_mean
,
lam
=
group1_std
**
2
,
observed
=
y1
)
returns_group2
=
pm
.
StudentT
(
'group2'
,
nu
=
nu
,
mu
=
group2_mean
,
lam
=
group2_std
**
2
,
observed
=
y2
)
diff_of_means
=
pm
.
Deterministic
(
'difference of means'
,
group2_mean
group1_mean
)
pm
.
Deterministic
(
'difference of stds'
,
group2_std
group1_std
)
pm
.
Deterministic
(
'effect size'
,
diff_of_means
/
pm
.
math
.
sqrt
(
(
group1_std
**
2
+
group2_std
**
2
)
/
2
)
)
pm
.
Deterministic
(
'group1_annual_volatility'
,
returns_group1
.
distribution
.
variance
**
.5
*
np
.
sqrt
(
252
)
)
pm
.
Deterministic
(
'group2_annual_volatility'
,
returns_group2
.
distribution
.
variance
**
.5
*
np
.
sqrt
(
252
)
)
pm
.
Deterministic
(
'group1_sharpe'
,
returns_group1
.
distribution
.
mean
/
returns_group1
.
distribution
.
variance
**
.5
*
np
.
sqrt
(
252
)
)
pm
.
Deterministic
(
'group2_sharpe'
,
returns_group2
.
distribution
.
mean
/
returns_group2
.
distribution
.
variance
**
.5
*
np
.
sqrt
(
252
)
)
trace
=
pm
.
sample
(
samples
,
progressbar
=
progressbar
)
return
model
,
trace

Bayesian Estimation Supersedes the T - Test


def model_stoch_vol(data, samples=2000, progressbar=True):
    """
    Run stochastic volatility model.

    This model estimates the volatility of a returns series over time.
    Returns are assumed to be T-distributed. lambda (width of
    T-distributed) is assumed to follow a random-walk.

    Parameters
    ----------
    data : pandas.Series
        Return series to model.
    samples : int, optional
        Posterior samples to draw.

    Returns
    -------
    model : pymc.Model object
        PyMC3 model containing all random variables.
    trace : pymc3.sampling.BaseTrace object
        A PyMC3 trace object that contains samples for each parameter
        of the posterior.

    See Also
    --------
    plot_stoch_vol : plotting of tochastic volatility model
    """

    from pymc3.distributions.timeseries import GaussianRandomWalk

    with pm.Model() as model:
        nu = pm.Exponential('nu', 1. / 10, testval=5.)
        sigma = pm.Exponential('sigma', 1. / .02, testval=.1)
        s = GaussianRandomWalk('s', sigma**-2, shape=len(data))
        volatility_process = pm.Deterministic('volatility_process',
                                              pm.math.exp(-2 * s))
        pm.StudentT('r', nu, lam=volatility_process, observed=data)

        trace = pm.sample(samples, progressbar=progressbar)

    return model, trace

def model_stoch_vol ( data , samples = 2000 , progressbar = True ) : from pymc3 . distributions . timeseries import GaussianRandomWalk with pm . Model ( ) as model : nu = pm . Exponential ( 'nu' , 1. / 10 , testval = 5. ) sigma = pm . Exponential ( 'sigma' , 1. / .02 , testval = .1 ) s = GaussianRandomWalk ( 's' , sigma ** - 2 , shape = len ( data ) ) volatility_process = pm . Deterministic ( 'volatility_process' , pm . math . exp ( - 2 * s ) ) pm . StudentT ( 'r' , nu , lam = volatility_process , observed = data ) trace = pm . sample ( samples , progressbar = progressbar ) return model , trace


def plot_stoch_vol(data, trace=None, ax=None):
    """
    Generate plot for stochastic volatility model.

    Parameters
    ----------
    data : pandas.Series
        Returns to model.
    trace : pymc3.sampling.BaseTrace object, optional
        trace as returned by model_stoch_vol
        If not passed, sample from model.
    ax : matplotlib.axes object, optional
        Plot into axes object

    Returns
    -------
    ax object

    See Also
    --------
    model_stoch_vol : run stochastic volatility model
    """

    if trace is None:
        trace = model_stoch_vol(data)

    if ax is None:
        fig, ax = plt.subplots(figsize=(15, 8))

    data.abs().plot(ax=ax)
    ax.plot(data.index, np.exp(trace['s', ::30].T), 'r', alpha=.03)
    ax.set(title='Stochastic volatility', xlabel='Time', ylabel='Volatility')
    ax.legend(['Abs returns', 'Stochastic volatility process'],
              frameon=True, framealpha=0.5)

    return ax

def
plot_stoch_vol
(
data
,
trace
=
,
ax
=
)
:
if
trace
is
:
trace
=
model_stoch_vol
(
data
)
if
ax
is
:
fig
,
ax
=
plt
.
subplots
(
figsize
=
(
15
,
8
)
)
data
.
abs
(
)
.
plot
(
ax
=
ax
)
ax
.
plot
(
data
.
index
,
np
.
exp
(
trace
[
's'
,
:
:
30
]
.
T
)
,
'r'
,
alpha
=
.03
)
ax
.
set
(
title
=
'Stochastic volatility'
,
xlabel
=
'Time'
,
ylabel
=
'Volatility'
)
ax
.
legend
(
[
'Abs returns'
,
'Stochastic volatility process'
]
,
frameon
=
True
,
framealpha
=
0.5
)
return
ax

Generate plot for stochastic volatility model .


def compute_bayes_cone(preds, starting_value=1.):
    """
    Compute 5, 25, 75 and 95 percentiles of cumulative returns, used
    for the Bayesian cone.

    Parameters
    ----------
    preds : numpy.array
        Multiple (simulated) cumulative returns.
    starting_value : int (optional)
        Have cumulative returns start around this value.
        Default = 1.

    Returns
    -------
    dict of percentiles over time
        Dictionary mapping percentiles (5, 25, 75, 95) to a
        timeseries.
    """

    def scoreatpercentile(cum_preds, p):
        return [stats.scoreatpercentile(
            c, p) for c in cum_preds.T]

    cum_preds = np.cumprod(preds + 1, 1) * starting_value
    perc = {p: scoreatpercentile(cum_preds, p) for p in (5, 25, 75, 95)}

    return perc

def compute_bayes_cone ( preds , starting_value = 1. ) : def scoreatpercentile ( cum_preds , p ) : return [ stats . scoreatpercentile ( c , p ) for c in cum_preds . T ] cum_preds = np . cumprod ( preds + 1 , 1 ) * starting_value perc = { p : scoreatpercentile ( cum_preds , p ) for p in ( 5 , 25 , 75 , 95 ) } return perc

Compute 5 25 75 and 95 percentiles of cumulative returns used for the Bayesian cone .


def compute_consistency_score(returns_test, preds):
    """
    Compute Bayesian consistency score.

    Parameters
    ----------
    returns_test : pd.Series
        Observed cumulative returns.
    preds : numpy.array
        Multiple (simulated) cumulative returns.

    Returns
    -------
    Consistency score
        Score from 100 (returns_test perfectly on the median line of the
        Bayesian cone spanned by preds) to 0 (returns_test completely
        outside of Bayesian cone.)
    """

    returns_test_cum = cum_returns(returns_test, starting_value=1.)
    cum_preds = np.cumprod(preds + 1, 1)

    q = [sp.stats.percentileofscore(cum_preds[:, i],
                                    returns_test_cum.iloc[i],
                                    kind='weak')
         for i in range(len(returns_test_cum))]
    # normalize to be from 100 (perfect median line) to 0 (completely outside
    # of cone)
    return 100 - np.abs(50 - np.mean(q)) / .5


def run_model(model, returns_train, returns_test=None,
              bmark=None, samples=500, ppc=False, progressbar=True):
    """
    Run one of the Bayesian models.

    Parameters
    ----------
    model : {'alpha_beta', 't', 'normal', 'best'}
        Which model to run
    returns_train : pd.Series
        Timeseries of simple returns
    returns_test : pd.Series (optional)
        Out-of-sample returns. Datetimes in returns_test will be added to
        returns_train as missing values and predictions will be generated
        for them.
    bmark : pd.Series or pd.DataFrame (optional)
        Only used for alpha_beta to estimate regression coefficients.
        If bmark has more recent returns than returns_train, these dates
        will be treated as missing values and predictions will be
        generated for them taking market correlations into account.
    samples : int (optional)
        Number of posterior samples to draw.
    ppc : boolean (optional)
        Whether to run a posterior predictive check. Will generate
        samples of length returns_test.  Returns a second argument
        that contains the PPC of shape samples x len(returns_test).

    Returns
    -------
    trace : pymc3.sampling.BaseTrace object
        A PyMC3 trace object that contains samples for each parameter
        of the posterior.

    ppc : numpy.array (if ppc==True)
       PPC of shape samples x len(returns_test).
    """

    if model == 'alpha_beta':
        model, trace = model_returns_t_alpha_beta(returns_train,
                                                  bmark, samples,
                                                  progressbar=progressbar)
    elif model == 't':
        model, trace = model_returns_t(returns_train, samples,
                                       progressbar=progressbar)
    elif model == 'normal':
        model, trace = model_returns_normal(returns_train, samples,
                                            progressbar=progressbar)
    elif model == 'best':
        model, trace = model_best(returns_train, returns_test,
                                  samples=samples,
                                  progressbar=progressbar)
    else:
        raise NotImplementedError(
            'Model {} not found.'
            'Use alpha_beta, t, normal, or best.'.format(model))

    if ppc:
        ppc_samples = pm.sample_ppc(trace, samples=samples,
                                    model=model, size=len(returns_test),
                                    progressbar=progressbar)
        return trace, ppc_samples['returns']

    return trace

def
run_model
(
model
,
returns_train
,
returns_test
=
,
bmark
=
,
samples
=
500
,
ppc
=
False
,
progressbar
=
True
)
:
if
model
==
'alpha_beta'
:
model
,
trace
=
model_returns_t_alpha_beta
(
returns_train
,
bmark
,
samples
,
progressbar
=
progressbar
)
elif
model
==
't'
:
model
,
trace
=
model_returns_t
(
returns_train
,
samples
,
progressbar
=
progressbar
)
elif
model
==
'normal'
:
model
,
trace
=
model_returns_normal
(
returns_train
,
samples
,
progressbar
=
progressbar
)
elif
model
==
'best'
:
model
,
trace
=
model_best
(
returns_train
,
returns_test
,
samples
=
samples
,
progressbar
=
progressbar
)
else
:
raise
NotImplementedError
(
'Model {} not found.'
'Use alpha_beta, t, normal, or best.'
.
format
(
model
)
)
if
ppc
:
ppc_samples
=
pm
.
sample_ppc
(
trace
,
samples
=
samples
,
model
=
model
,
size
=
len
(
returns_test
)
,
progressbar
=
progressbar
)
return
trace
,
ppc_samples
[
'returns'
]
return
trace


def plot_bayes_cone(returns_train, returns_test, ppc,
                    plot_train_len=50, ax=None):
    """
    Generate cumulative returns plot with Bayesian cone.

    Parameters
    ----------
    returns_train : pd.Series
        Timeseries of simple returns
    returns_test : pd.Series
        Out-of-sample returns. Datetimes in returns_test will be added to
        returns_train as missing values and predictions will be generated
        for them.
    ppc : np.array
        Posterior predictive samples of shape samples x
        len(returns_test).
    plot_train_len : int (optional)
        How many data points to plot of returns_train. Useful to zoom in on
        the prediction if there is a long backtest period.
    ax : matplotlib.Axis (optional)
        Axes upon which to plot.

    Returns
    -------
    score : float
        Consistency score (see compute_consistency_score)
    trace : pymc3.sampling.BaseTrace
        A PyMC3 trace object that contains samples for each parameter
        of the posterior.
    """

    score = compute_consistency_score(returns_test,
                                      ppc)

    ax = _plot_bayes_cone(
        returns_train,
        returns_test,
        ppc,
        plot_train_len=plot_train_len,
        ax=ax)
    ax.text(
        0.40,
        0.90,
        'Consistency score: %.1f' %
        score,
        verticalalignment='bottom',
        horizontalalignment='right',
        transform=ax.transAxes,
    )

    ax.set_ylabel('Cumulative returns')
    return score

def
plot_bayes_cone
(
returns_train
,
returns_test
,
ppc
,
plot_train_len
=
50
,
ax
=
)
:
score
=
compute_consistency_score
(
returns_test
,
ppc
)
ax
=
_plot_bayes_cone
(
returns_train
,
returns_test
,
ppc
,
plot_train_len
=
plot_train_len
,
ax
=
ax
)
ax
.
text
(
0.40
,
0.90
,
'Consistency score: %.1f'
%
score
,
verticalalignment
=
'bottom'
,
horizontalalignment
=
'right'
,
transform
=
ax
.
transAxes
,
)
ax
.
set_ylabel
(
'Cumulative returns'
)
return
score

Generate cumulative returns plot with Bayesian cone .


def _GetNextLogCountPerToken(token):
    """Wrapper for _log_counter_per_token.

    Args:
    token: The token for which to look up the count.

    Returns:
    The number of times this function has been called with
    *token* as an argument (starting at 0)
    """
    global _log_counter_per_token  # pylint: disable=global-variable-not-assigned
    _log_counter_per_token[token] = 1 + _log_counter_per_token.get(token, -1)
    return _log_counter_per_token[token]


def log_every_n(level, msg, n, *args):
    """Log 'msg % args' at level 'level' once per 'n' times.

    Logs the 1st call, (N+1)st call, (2N+1)st call,  etc.
    Not threadsafe.

    Args:
    level: The level at which to log.
    msg: The message to be logged.
    n: The number of times this should be called before it is logged.
    *args: The args to be substituted into the msg.
    """
    count = _GetNextLogCountPerToken(_GetFileAndLine())
    log_if(level, msg, not (count % n), *args)

def log_every_n ( level , msg , n , * args ) : count = _GetNextLogCountPerToken ( _GetFileAndLine ( ) ) log_if ( level , msg , not ( count % n ) , * args )

Log msg % args at level level once per n times .


def log_if(level, msg, condition, *args):
    """Log 'msg % args' at level 'level' only if condition is fulfilled."""
    if condition:
        vlog(level, msg, *args)

def log_if ( level , msg , condition , * args ) : if condition : vlog ( level , msg , * args )

Log msg % args at level level only if condition is fulfilled .


def google2_log_prefix(level, timestamp=None, file_and_line=None):
    """Assemble a logline prefix using the google2 format."""
    # pylint: disable=global-variable-not-assigned
    global _level_names
    # pylint: enable=global-variable-not-assigned

    # Record current time
    now = timestamp or _time.time()
    now_tuple = _time.localtime(now)
    now_microsecond = int(1e6 * (now % 1.0))

    (filename, line) = file_and_line or _GetFileAndLine()
    basename = _os.path.basename(filename)

    # Severity string
    severity = 'I'
    if level in _level_names:
        severity = _level_names[level][0]

    s = '%c%02d%02d %02d: %02d: %02d.%06d %5d %s: %d] ' % (
        severity,
        now_tuple[1],  # month
        now_tuple[2],  # day
        now_tuple[3],  # hour
        now_tuple[4],  # min
        now_tuple[5],  # sec
        now_microsecond,
        _get_thread_id(),
        basename,
        line
    )

    return s

Assemble a logline prefix using the google2 format .


def create_distributed_session(
        task_spec=None, checkpoint_dir=None, scaffold=None, hooks=None, chief_only_hooks=None, save_checkpoint_secs=600,
        save_summaries_steps=object(), save_summaries_secs=object(), config=None, stop_grace_period_secs=120,
        log_step_count_steps=100
):
    """Creates a distributed session.

    It calls `MonitoredTrainingSession` to create a :class:`MonitoredSession` for distributed training.

    Parameters
    ----------
    task_spec : :class:`TaskSpecDef`.
        The task spec definition from create_task_spec_def()
    checkpoint_dir : str.
        Optional path to a directory where to restore variables.
    scaffold : ``Scaffold``
        A `Scaffold` used for gathering or building supportive ops.
        If not specified, a default one is created. It's used to finalize the graph.
    hooks : list of ``SessionRunHook`` objects.
        Optional
    chief_only_hooks : list of ``SessionRunHook`` objects.
        Activate these hooks if `is_chief==True`, ignore otherwise.
    save_checkpoint_secs : int
        The frequency, in seconds, that a checkpoint is saved
        using a default checkpoint saver. If `save_checkpoint_secs` is set to
        `None`, then the default checkpoint saver isn't used.
    save_summaries_steps : int
        The frequency, in number of global steps, that the
        summaries are written to disk using a default summary saver. If both
        `save_summaries_steps` and `save_summaries_secs` are set to `None`, then
        the default summary saver isn't used. Default 100.
    save_summaries_secs : int
        The frequency, in secs, that the summaries are written
        to disk using a default summary saver.  If both `save_summaries_steps` and
        `save_summaries_secs` are set to `None`, then the default summary saver
        isn't used. Default not enabled.
    config : ``tf.ConfigProto``
        an instance of `tf.ConfigProto` proto used to configure the session.
        It's the `config` argument of constructor of `tf.Session`.
    stop_grace_period_secs : int
        Number of seconds given to threads to stop after
        `close()` has been called.
    log_step_count_steps : int
        The frequency, in number of global steps, that the
        global step/sec is logged.

    Examples
    --------
    A simple example for distributed training where all the workers use the same dataset:

    >>> task_spec = TaskSpec()
    >>> with tf.device(task_spec.device_fn()):
    >>>      tensors = create_graph()
    >>> with tl.DistributedSession(task_spec=task_spec,
    ...                            checkpoint_dir='/tmp/ckpt') as session:
    >>>      while not session.should_stop():
    >>>           session.run(tensors)

    An example where the dataset is shared among the workers
    (see https://www.tensorflow.org/programmers_guide/datasets):

    >>> task_spec = TaskSpec()
    >>> # dataset is a :class:`tf.data.Dataset` with the raw data
    >>> dataset = create_dataset()
    >>> if task_spec is not None:
    >>>     dataset = dataset.shard(task_spec.num_workers, task_spec.shard_index)
    >>> # shuffle or apply a map function to the new sharded dataset, for example:
    >>> dataset = dataset.shuffle(buffer_size=10000)
    >>> dataset = dataset.batch(batch_size)
    >>> dataset = dataset.repeat(num_epochs)
    >>> # create the iterator for the dataset and the input tensor
    >>> iterator = dataset.make_one_shot_iterator()
    >>> next_element = iterator.get_next()
    >>> with tf.device(task_spec.device_fn()):
    >>>      # next_element is the input for the graph
    >>>      tensors = create_graph(next_element)
    >>> with tl.DistributedSession(task_spec=task_spec,
    ...                            checkpoint_dir='/tmp/ckpt') as session:
    >>>      while not session.should_stop():
    >>>           session.run(tensors)

    References
    ----------
    - `MonitoredTrainingSession <https://www.tensorflow.org/api_docs/python/tf/train/MonitoredTrainingSession>`__

    """
    target = task_spec.target() if task_spec is not None else None
    is_chief = task_spec.is_master() if task_spec is not None else True
    return tf.train.MonitoredTrainingSession(
        master=target, is_chief=is_chief, checkpoint_dir=checkpoint_dir, scaffold=scaffold,
        save_checkpoint_secs=save_checkpoint_secs, save_summaries_steps=save_summaries_steps,
        save_summaries_secs=save_summaries_secs, log_step_count_steps=log_step_count_steps,
        stop_grace_period_secs=stop_grace_period_secs, config=config, hooks=hooks, chief_only_hooks=chief_only_hooks
    )

def create_distributed_session ( task_spec = None , checkpoint_dir = None , scaffold = None , hooks = None , chief_only_hooks = None , save_checkpoint_secs = 600 , save_summaries_steps = object ( ) , save_summaries_secs = object ( ) , config = None , stop_grace_period_secs = 120 , log_step_count_steps = 100 ) : target = task_spec . target ( ) if task_spec is not None else None is_chief = task_spec . is_master ( ) if task_spec is not None else True return tf . train . MonitoredTrainingSession ( master = target , is_chief = is_chief , checkpoint_dir = checkpoint_dir , scaffold = scaffold , save_checkpoint_secs = save_checkpoint_secs , save_summaries_steps = save_summaries_steps , save_summaries_secs = save_summaries_secs , log_step_count_steps = log_step_count_steps , stop_grace_period_secs = stop_grace_period_secs , config = config , hooks = hooks , chief_only_hooks = chief_only_hooks )


def validation_metrics(self):
        """A helper function to compute validation related metrics"""

        if (self._validation_iterator is None) or (self._validation_metrics is None):
            raise AttributeError('Validation is not setup.')

        n = 0.0
        metric_sums = [0.0] * len(self._validation_metrics)
        self._sess.run(self._validation_iterator.initializer)
        while True:
            try:
                metrics = self._sess.run(self._validation_metrics)
                for i, m in enumerate(metrics):
                    metric_sums[i] += m
                n += 1.0
            except tf.errors.OutOfRangeError:
                break
        for i, m in enumerate(metric_sums):
            metric_sums[i] = metric_sums[i] / n
        return zip(self._validation_metrics, metric_sums)

def
validation_metrics
(
self
)
:
if
(
self
.
_validation_iterator
is
)
or
(
self
.
_validation_metrics
is
)
:
raise
AttributeError
(
'Validation is not setup.'
)
n
=
0.0
metric_sums
=
[
0.0
]
*
len
(
self
.
_validation_metrics
)
self
.
_sess
.
run
(
self
.
_validation_iterator
.
initializer
)
while
True
:
try
:
metrics
=
self
.
_sess
.
run
(
self
.
_validation_metrics
)
for
i
,
m
in
enumerate
(
metrics
)
:
metric_sums
[
i
]
+=
m
n
+=
1.0
except
tf
.
errors
.
OutOfRangeError
:
break
for
i
,
m
in
enumerate
(
metric_sums
)
:
metric_sums
[
i
]
=
metric_sums
[
i
]
/
n
return
zip
(
self
.
_validation_metrics
,
metric_sums
)


def train_and_validate_to_end(self, validate_step_size=50):
        """A helper function that shows how to train and validate a model at the same time.

        Parameters
        ----------
        validate_step_size : int
            Validate the training network every N steps.

        """
        while not self._sess.should_stop():
            self.train_on_batch()  # Run a training step synchronously.
            if self.global_step % validate_step_size == 0:
                # logging.info("Average loss for validation dataset: %s" % self.get_validation_metrics())
                log_str = 'step: %d, ' % self.global_step
                for n, m in self.validation_metrics:
                    log_str += '%s: %f, ' % (n.name, m)
                logging.info(log_str)

A helper function that shows how to train and validate a model at the same time .


def _load_mnist_dataset(shape, path, name='mnist', url='http://yann.lecun.com/exdb/mnist/'):
    """A generic function to load mnist-like dataset.

    Parameters:
    ----------
    shape : tuple
        The shape of digit images.
    path : str
        The path that the data is downloaded to.
    name : str
        The dataset name you want to use(the default is 'mnist').
    url : str
        The url of dataset(the default is 'http://yann.lecun.com/exdb/mnist/').
    """
    path = os.path.join(path, name)

    # Define functions for loading mnist-like data's images and labels.
    # For convenience, they also download the requested files if needed.
    def load_mnist_images(path, filename):
        filepath = maybe_download_and_extract(filename, path, url)

        logging.info(filepath)
        # Read the inputs in Yann LeCun's binary format.
        with gzip.open(filepath, 'rb') as f:
            data = np.frombuffer(f.read(), np.uint8, offset=16)
        # The inputs are vectors now, we reshape them to monochrome 2D images,
        # following the shape convention: (examples, channels, rows, columns)
        data = data.reshape(shape)
        # The inputs come as bytes, we convert them to float32 in range [0,1].
        # (Actually to range [0, 255/256], for compatibility to the version
        # provided at http://deeplearning.net/data/mnist/mnist.pkl.gz.)
        return data / np.float32(256)

    def load_mnist_labels(path, filename):
        filepath = maybe_download_and_extract(filename, path, url)
        # Read the labels in Yann LeCun's binary format.
        with gzip.open(filepath, 'rb') as f:
            data = np.frombuffer(f.read(), np.uint8, offset=8)
        # The labels are vectors of integers now, that's exactly what we want.
        return data

    # Download and read the training and test set images and labels.
    logging.info("Load or Download {0} > {1}".format(name.upper(), path))
    X_train = load_mnist_images(path, 'train-images-idx3-ubyte.gz')
    y_train = load_mnist_labels(path, 'train-labels-idx1-ubyte.gz')
    X_test = load_mnist_images(path, 't10k-images-idx3-ubyte.gz')
    y_test = load_mnist_labels(path, 't10k-labels-idx1-ubyte.gz')

    # We reserve the last 10000 training examples for validation.
    X_train, X_val = X_train[:-10000], X_train[-10000:]
    y_train, y_val = y_train[:-10000], y_train[-10000:]

    # We just return all the arrays in order, as expected in main().
    # (It doesn't matter how we do this as long as we can read them again.)
    X_train = np.asarray(X_train, dtype=np.float32)
    y_train = np.asarray(y_train, dtype=np.int32)
    X_val = np.asarray(X_val, dtype=np.float32)
    y_val = np.asarray(y_val, dtype=np.int32)
    X_test = np.asarray(X_test, dtype=np.float32)
    y_test = np.asarray(y_test, dtype=np.int32)
    return X_train, y_train, X_val, y_val, X_test, y_test

A generic function to load mnist - like dataset .


def load_matt_mahoney_text8_dataset(path='data'):
    """Load Matt Mahoney's dataset.

    Download a text file from Matt Mahoney's website
    if not present, and make sure it's the right size.
    Extract the first file enclosed in a zip file as a list of words.
    This dataset can be used for Word Embedding.

    Parameters
    ----------
    path : str
        The path that the data is downloaded to, defaults is ``data/mm_test8/``.

    Returns
    --------
    list of str
        The raw text data e.g. [.... 'their', 'families', 'who', 'were', 'expelled', 'from', 'jerusalem', ...]

    Examples
    --------
    >>> words = tl.files.load_matt_mahoney_text8_dataset()
    >>> print('Data size', len(words))

    """
    path = os.path.join(path, 'mm_test8')
    logging.info("Load or Download matt_mahoney_text8 Dataset> {}".format(path))

    filename = 'text8.zip'
    url = 'http://mattmahoney.net/dc/'
    maybe_download_and_extract(filename, path, url, expected_bytes=31344016)

    with zipfile.ZipFile(os.path.join(path, filename)) as f:
        word_list = f.read(f.namelist()[0]).split()
        for idx, _ in enumerate(word_list):
            word_list[idx] = word_list[idx].decode()
    return word_list


def load_imdb_dataset(
        path='data', nb_words=None, skip_top=0, maxlen=None, test_split=0.2, seed=113, start_char=1, oov_char=2,
        index_from=3
):
    """Load IMDB dataset.

    Parameters
    ----------
    path : str
        The path that the data is downloaded to, defaults is ``data/imdb/``.
    nb_words : int
        Number of words to get.
    skip_top : int
        Top most frequent words to ignore (they will appear as oov_char value in the sequence data).
    maxlen : int
        Maximum sequence length. Any longer sequence will be truncated.
    seed : int
        Seed for reproducible data shuffling.
    start_char : int
        The start of a sequence will be marked with this character. Set to 1 because 0 is usually the padding character.
    oov_char : int
        Words that were cut out because of the num_words or skip_top limit will be replaced with this character.
    index_from : int
        Index actual words with this index and higher.

    Examples
    --------
    >>> X_train, y_train, X_test, y_test = tl.files.load_imdb_dataset(
    ...                                 nb_words=20000, test_split=0.2)
    >>> print('X_train.shape', X_train.shape)
    (20000,)  [[1, 62, 74, ... 1033, 507, 27],[1, 60, 33, ... 13, 1053, 7]..]
    >>> print('y_train.shape', y_train.shape)
    (20000,)  [1 0 0 ..., 1 0 1]

    References
    -----------
    - `Modified from keras. <https://github.com/fchollet/keras/blob/master/keras/datasets/imdb.py>`__

    """
    path = os.path.join(path, 'imdb')

    filename = "imdb.pkl"
    url = 'https://s3.amazonaws.com/text-datasets/'
    maybe_download_and_extract(filename, path, url)

    if filename.endswith(".gz"):
        f = gzip.open(os.path.join(path, filename), 'rb')
    else:
        f = open(os.path.join(path, filename), 'rb')

    X, labels = cPickle.load(f)
    f.close()

    np.random.seed(seed)
    np.random.shuffle(X)
    np.random.seed(seed)
    np.random.shuffle(labels)

    if start_char is not None:
        X = [[start_char] + [w + index_from for w in x] for x in X]
    elif index_from:
        X = [[w + index_from for w in x] for x in X]

    if maxlen:
        new_X = []
        new_labels = []
        for x, y in zip(X, labels):
            if len(x) < maxlen:
                new_X.append(x)
                new_labels.append(y)
        X = new_X
        labels = new_labels
    if not X:
        raise Exception(
            'After filtering for sequences shorter than maxlen=' + str(maxlen) + ', no sequence was kept. '
            'Increase maxlen.'
        )
    if not nb_words:
        nb_words = max([max(x) for x in X])

    # by convention, use 2 as OOV word
    # reserve 'index_from' (=3 by default) characters: 0 (padding), 1 (start), 2 (OOV)
    if oov_char is not None:
        X = [[oov_char if (w >= nb_words or w < skip_top) else w for w in x] for x in X]
    else:
        nX = []
        for x in X:
            nx = []
            for w in x:
                if (w >= nb_words or w < skip_top):
                    nx.append(w)
            nX.append(nx)
        X = nX

    X_train = np.array(X[:int(len(X) * (1 - test_split))])
    y_train = np.array(labels[:int(len(X) * (1 - test_split))])

    X_test = np.array(X[int(len(X) * (1 - test_split)):])
    y_test = np.array(labels[int(len(X) * (1 - test_split)):])

    return X_train, y_train, X_test, y_test


def load_nietzsche_dataset(path='data'):
    """Load Nietzsche dataset.

    Parameters
    ----------
    path : str
        The path that the data is downloaded to, defaults is ``data/nietzsche/``.

    Returns
    --------
    str
        The content.

    Examples
    --------
    >>> see tutorial_generate_text.py
    >>> words = tl.files.load_nietzsche_dataset()
    >>> words = basic_clean_str(words)
    >>> words = words.split()

    """
    logging.info("Load or Download nietzsche dataset > {}".format(path))
    path = os.path.join(path, 'nietzsche')

    filename = "nietzsche.txt"
    url = 'https://s3.amazonaws.com/text-datasets/'
    filepath = maybe_download_and_extract(filename, path, url)

    with open(filepath, "r") as f:
        words = f.read()
        return words


def load_wmt_en_fr_dataset(path='data'):
    """Load WMT'15 English-to-French translation dataset.

    It will download the data from the WMT'15 Website (10^9-French-English corpus), and the 2013 news test from the same site as development set.
    Returns the directories of training data and test data.

    Parameters
    ----------
    path : str
        The path that the data is downloaded to, defaults is ``data/wmt_en_fr/``.

    References
    ----------
    - Code modified from /tensorflow/models/rnn/translation/data_utils.py

    Notes
    -----
    Usually, it will take a long time to download this dataset.

    """
    path = os.path.join(path, 'wmt_en_fr')
    # URLs for WMT data.
    _WMT_ENFR_TRAIN_URL = "http://www.statmt.org/wmt10/"
    _WMT_ENFR_DEV_URL = "http://www.statmt.org/wmt15/"

    def gunzip_file(gz_path, new_path):
        """Unzips from gz_path into new_path."""
        logging.info("Unpacking %s to %s" % (gz_path, new_path))
        with gzip.open(gz_path, "rb") as gz_file:
            with open(new_path, "wb") as new_file:
                for line in gz_file:
                    new_file.write(line)

    def get_wmt_enfr_train_set(path):
        """Download the WMT en-fr training corpus to directory unless it's there."""
        filename = "training-giga-fren.tar"
        maybe_download_and_extract(filename, path, _WMT_ENFR_TRAIN_URL, extract=True)
        train_path = os.path.join(path, "giga-fren.release2.fixed")
        gunzip_file(train_path + ".fr.gz", train_path + ".fr")
        gunzip_file(train_path + ".en.gz", train_path + ".en")
        return train_path

    def get_wmt_enfr_dev_set(path):
        """Download the WMT en-fr training corpus to directory unless it's there."""
        filename = "dev-v2.tgz"
        dev_file = maybe_download_and_extract(filename, path, _WMT_ENFR_DEV_URL, extract=False)
        dev_name = "newstest2013"
        dev_path = os.path.join(path, "newstest2013")
        if not (gfile.Exists(dev_path + ".fr") and gfile.Exists(dev_path + ".en")):
            logging.info("Extracting tgz file %s" % dev_file)
            with tarfile.open(dev_file, "r:gz") as dev_tar:
                fr_dev_file = dev_tar.getmember("dev/" + dev_name + ".fr")
                en_dev_file = dev_tar.getmember("dev/" + dev_name + ".en")
                fr_dev_file.name = dev_name + ".fr"  # Extract without "dev/" prefix.
                en_dev_file.name = dev_name + ".en"
                dev_tar.extract(fr_dev_file, path)
                dev_tar.extract(en_dev_file, path)
        return dev_path

    logging.info("Load or Download WMT English-to-French translation > {}".format(path))

    train_path = get_wmt_enfr_train_set(path)
    dev_path = get_wmt_enfr_dev_set(path)

    return train_path, dev_path

Load WMT 15 English - to - French translation dataset .


def load_flickr25k_dataset(tag='sky', path="data", n_threads=50, printable=False):
    """Load Flickr25K dataset.

    Returns a list of images by a given tag from Flick25k dataset,
    it will download Flickr25k from `the official website <http://press.liacs.nl/mirflickr/mirdownload.html>`__
    at the first time you use it.

    Parameters
    ------------
    tag : str or None
        What images to return.
            - If you want to get images with tag, use string like 'dog', 'red', see `Flickr Search <https://www.flickr.com/search/>`__.
            - If you want to get all images, set to ``None``.

    path : str
        The path that the data is downloaded to, defaults is ``data/flickr25k/``.
    n_threads : int
        The number of thread to read image.
    printable : boolean
        Whether to print infomation when reading images, default is ``False``.

    Examples
    -----------
    Get images with tag of sky

    >>> images = tl.files.load_flickr25k_dataset(tag='sky')

    Get all images

    >>> images = tl.files.load_flickr25k_dataset(tag=None, n_threads=100, printable=True)

    """
    path = os.path.join(path, 'flickr25k')

    filename = 'mirflickr25k.zip'
    url = 'http://press.liacs.nl/mirflickr/mirflickr25k/'

    # download dataset
    if folder_exists(os.path.join(path, "mirflickr")) is False:
        logging.info("[*] Flickr25k is nonexistent in {}".format(path))
        maybe_download_and_extract(filename, path, url, extract=True)
        del_file(os.path.join(path, filename))

    # return images by the given tag.
    # 1. image path list
    folder_imgs = os.path.join(path, "mirflickr")
    path_imgs = load_file_list(path=folder_imgs, regx='\\.jpg', printable=False)
    path_imgs.sort(key=natural_keys)

    # 2. tag path list
    folder_tags = os.path.join(path, "mirflickr", "meta", "tags")
    path_tags = load_file_list(path=folder_tags, regx='\\.txt', printable=False)
    path_tags.sort(key=natural_keys)

    # 3. select images
    if tag is None:
        logging.info("[Flickr25k] reading all images")
    else:
        logging.info("[Flickr25k] reading images with tag: {}".format(tag))
    images_list = []
    for idx, _v in enumerate(path_tags):
        tags = read_file(os.path.join(folder_tags, path_tags[idx])).split('\n')
        # logging.info(idx+1, tags)
        if tag is None or tag in tags:
            images_list.append(path_imgs[idx])

    images = visualize.read_images(images_list, folder_imgs, n_threads=n_threads, printable=printable)
    return images


def download_file_from_google_drive(ID, destination):
    """Download file from Google Drive.

    See ``tl.files.load_celebA_dataset`` for example.

    Parameters
    --------------
    ID : str
        The driver ID.
    destination : str
        The destination for save file.

    """

    def save_response_content(response, destination, chunk_size=32 * 1024):
        total_size = int(response.headers.get('content-length', 0))
        with open(destination, "wb") as f:
            for chunk in tqdm(response.iter_content(chunk_size), total=total_size, unit='B', unit_scale=True,
                              desc=destination):
                if chunk:  # filter out keep-alive new chunks
                    f.write(chunk)

    def get_confirm_token(response):
        for key, value in response.cookies.items():
            if key.startswith('download_warning'):
                return value
        return None

    URL = "https://docs.google.com/uc?export=download"
    session = requests.Session()

    response = session.get(URL, params={'id': ID}, stream=True)
    token = get_confirm_token(response)

    if token:
        params = {'id': ID, 'confirm': token}
        response = session.get(URL, params=params, stream=True)
    save_response_content(response, destination)


def load_celebA_dataset(path='data'):
    """Load CelebA dataset

    Return a list of image path.

    Parameters
    -----------
    path : str
        The path that the data is downloaded to, defaults is ``data/celebA/``.

    """
    data_dir = 'celebA'
    filename, drive_id = "img_align_celeba.zip", "0B7EVK8r0v71pZjFTYXZWM3FlRnM"
    save_path = os.path.join(path, filename)
    image_path = os.path.join(path, data_dir)
    if os.path.exists(image_path):
        logging.info('[*] {} already exists'.format(save_path))
    else:
        exists_or_mkdir(path)
        download_file_from_google_drive(drive_id, save_path)
        zip_dir = ''
        with zipfile.ZipFile(save_path) as zf:
            zip_dir = zf.namelist()[0]
            zf.extractall(path)
        os.remove(save_path)
        os.rename(os.path.join(path, zip_dir), image_path)

    data_files = load_file_list(path=image_path, regx='\\.jpg', printable=False)
    for i, _v in enumerate(data_files):
        data_files[i] = os.path.join(image_path, data_files[i])
    return data_files

def
load_celebA_dataset
(
path
=
'data'
)
:
data_dir
=
'celebA'
filename
,
drive_id
=
"img_align_celeba.zip"
,
"0B7EVK8r0v71pZjFTYXZWM3FlRnM"
save_path
=
os
.
path
.
join
(
path
,
filename
)
image_path
=
os
.
path
.
join
(
path
,
data_dir
)
if
os
.
path
.
exists
(
image_path
)
:
logging
.
info
(
'[*] {} already exists'
.
format
(
save_path
)
)
else
:
exists_or_mkdir
(
path
)
download_file_from_google_drive
(
drive_id
,
save_path
)
zip_dir
=
''
with
zipfile
.
ZipFile
(
save_path
)
as
zf
:
zip_dir
=
zf
.
namelist
(
)
[
0
]
zf
.
extractall
(
path
)
os
.
remove
(
save_path
)
os
.
rename
(
os
.
path
.
join
(
path
,
zip_dir
)
,
image_path
)
data_files
=
load_file_list
(
path
=
image_path
,
regx
=
'\\.jpg'
,
printable
=
False
)
for
i
,
_v
in
enumerate
(
data_files
)
:
data_files
[
i
]
=
os
.
path
.
join
(
image_path
,
data_files
[
i
]
)
return
data_files


def assign_params(sess, params, network):
    """Assign the given parameters to the TensorLayer network.

    Parameters
    ----------
    sess : Session
        TensorFlow Session.
    params : list of array
        A list of parameters (array) in order.
    network : :class:`Layer`
        The network to be assigned.

    Returns
    --------
    list of operations
        A list of tf ops in order that assign params. Support sess.run(ops) manually.

    Examples
    --------
    - See ``tl.files.save_npz``

    References
    ----------
    - `Assign value to a TensorFlow variable <http://stackoverflow.com/questions/34220532/how-to-assign-value-to-a-tensorflow-variable>`__

    """
    ops = []
    for idx, param in enumerate(params):
        ops.append(network.all_params[idx].assign(param))
    if sess is not None:
        sess.run(ops)
    return ops

def assign_params ( sess , params , network ) : ops = [ ] for idx , param in enumerate ( params ) : ops . append ( network . all_params [ idx ] . assign ( param ) ) if sess is not None : sess . run ( ops ) return ops

Assign the given parameters to the TensorLayer network .


def load_and_assign_npz(sess=None, name=None, network=None):
    """Load model from npz and assign to a network.

    Parameters
    -------------
    sess : Session
        TensorFlow Session.
    name : str
        The name of the `.npz` file.
    network : :class:`Layer`
        The network to be assigned.

    Returns
    --------
    False or network
        Returns False, if the model is not exist.

    Examples
    --------
    - See ``tl.files.save_npz``

    """
    if network is None:
        raise ValueError("network is None.")
    if sess is None:
        raise ValueError("session is None.")
    if not os.path.exists(name):
        logging.error("file {} doesn't exist.".format(name))
        return False
    else:
        params = load_npz(name=name)
        assign_params(sess, params, network)
        logging.info("[*] Load {} SUCCESS!".format(name))
        return network

def
load_and_assign_npz
(
sess
=
,
name
=
,
network
=
)
:
if
network
is
:
raise
ValueError
(
"network is None."
)
if
sess
is
:
raise
ValueError
(
"session is None."
)
if
not
os
.
path
.
exists
(
name
)
:
logging
.
error
(
"file {} doesn't exist."
.
format
(
name
)
)
return
False
else
:
params
=
load_npz
(
name
=
name
)
assign_params
(
sess
,
params
,
network
)
logging
.
info
(
"[*] Load {} SUCCESS!"
.
format
(
name
)
)
return
network

Load model from npz and assign to a network .


def save_npz_dict(save_list=None, name='model.npz', sess=None):
    """Input parameters and the file name, save parameters as a dictionary into .npz file.

    Use ``tl.files.load_and_assign_npz_dict()`` to restore.

    Parameters
    ----------
    save_list : list of parameters
        A list of parameters (tensor) to be saved.
    name : str
        The name of the `.npz` file.
    sess : Session
        TensorFlow Session.

    """
    if sess is None:
        raise ValueError("session is None.")
    if save_list is None:
        save_list = []

    save_list_names = [tensor.name for tensor in save_list]
    save_list_var = sess.run(save_list)
    save_var_dict = {save_list_names[idx]: val for idx, val in enumerate(save_list_var)}
    np.savez(name, **save_var_dict)
    save_list_var = None
    save_var_dict = None
    del save_list_var
    del save_var_dict
    logging.info("[*] Model saved in npz_dict %s" % name)

def
save_npz_dict
(
save_list
=
,
name
=
'model.npz'
,
sess
=
)
:
if
sess
is
:
raise
ValueError
(
"session is None."
)
if
save_list
is
:
save_list
=
[
]
save_list_names
=
[
tensor
.
name
for
tensor
in
save_list
]
save_list_var
=
sess
.
run
(
save_list
)
save_var_dict
=
{
save_list_names
[
idx
]
:
val
for
idx
,
val
in
enumerate
(
save_list_var
)
}
np
.
savez
(
name
,
*
*
save_var_dict
)
save_list_var
=
save_var_dict
=
del
save_list_var
del
save_var_dict
logging
.
info
(
"[*] Model saved in npz_dict %s"
%
name
)

Input parameters and the file name save parameters as a dictionary into . npz file .


def save_ckpt(
        sess=None, mode_name='model.ckpt', save_dir='checkpoint', var_list=None, global_step=None, printable=False
):
    """Save parameters into `ckpt` file.

    Parameters
    ------------
    sess : Session
        TensorFlow Session.
    mode_name : str
        The name of the model, default is ``model.ckpt``.
    save_dir : str
        The path / file directory to the `ckpt`, default is ``checkpoint``.
    var_list : list of tensor
        The parameters / variables (tensor) to be saved. If empty, save all global variables (default).
    global_step : int or None
        Step number.
    printable : boolean
        Whether to print all parameters information.

    See Also
    --------
    load_ckpt

    """
    if sess is None:
        raise ValueError("session is None.")
    if var_list is None:
        var_list = []

    ckpt_file = os.path.join(save_dir, mode_name)
    if var_list == []:
        var_list = tf.global_variables()

    logging.info("[*] save %s n_params: %d" % (ckpt_file, len(var_list)))

    if printable:
        for idx, v in enumerate(var_list):
            logging.info("  param {:3}: {:15}   {}".format(idx, v.name, str(v.get_shape())))

    saver = tf.train.Saver(var_list)
    saver.save(sess, ckpt_file, global_step=global_step)

def
save_ckpt
(
sess
=
,
mode_name
=
'model.ckpt'
,
save_dir
=
'checkpoint'
,
var_list
=
,
global_step
=
,
printable
=
False
)
:
if
sess
is
:
raise
ValueError
(
"session is None."
)
if
var_list
is
:
var_list
=
[
]
ckpt_file
=
os
.
path
.
join
(
save_dir
,
mode_name
)
if
var_list
==
[
]
:
var_list
=
tf
.
global_variables
(
)
logging
.
info
(
"[*] save %s n_params: %d"
%
(
ckpt_file
,
len
(
var_list
)
)
)
if
printable
:
for
idx
,
v
in
enumerate
(
var_list
)
:
logging
.
info
(
"  param {:3}: {:15}   {}"
.
format
(
idx
,
v
.
name
,
str
(
v
.
get_shape
(
)
)
)
)
saver
=
tf
.
train
.
Saver
(
var_list
)
saver
.
save
(
sess
,
ckpt_file
,
global_step
=
global_step
)


def load_ckpt(sess=None, mode_name='model.ckpt', save_dir='checkpoint', var_list=None, is_latest=True, printable=False):
    """Load parameters from `ckpt` file.

    Parameters
    ------------
    sess : Session
        TensorFlow Session.
    mode_name : str
        The name of the model, default is ``model.ckpt``.
    save_dir : str
        The path / file directory to the `ckpt`, default is ``checkpoint``.
    var_list : list of tensor
        The parameters / variables (tensor) to be saved. If empty, save all global variables (default).
    is_latest : boolean
        Whether to load the latest `ckpt`, if False, load the `ckpt` with the name of ```mode_name``.
    printable : boolean
        Whether to print all parameters information.

    Examples
    ----------
    - Save all global parameters.

    >>> tl.files.save_ckpt(sess=sess, mode_name='model.ckpt', save_dir='model', printable=True)

    - Save specific parameters.

    >>> tl.files.save_ckpt(sess=sess, mode_name='model.ckpt', var_list=net.all_params, save_dir='model', printable=True)

    - Load latest ckpt.

    >>> tl.files.load_ckpt(sess=sess, var_list=net.all_params, save_dir='model', printable=True)

    - Load specific ckpt.

    >>> tl.files.load_ckpt(sess=sess, mode_name='model.ckpt', var_list=net.all_params, save_dir='model', is_latest=False, printable=True)

    """
    if sess is None:
        raise ValueError("session is None.")
    if var_list is None:
        var_list = []

    if is_latest:
        ckpt_file = tf.train.latest_checkpoint(save_dir)
    else:
        ckpt_file = os.path.join(save_dir, mode_name)

    if not var_list:
        var_list = tf.global_variables()

    logging.info("[*] load %s n_params: %d" % (ckpt_file, len(var_list)))

    if printable:
        for idx, v in enumerate(var_list):
            logging.info("  param {:3}: {:15}   {}".format(idx, v.name, str(v.get_shape())))

    try:
        saver = tf.train.Saver(var_list)
        saver.restore(sess, ckpt_file)
    except Exception as e:
        logging.info(e)
        logging.info("[*] load ckpt fail ...")

def
load_ckpt
(
sess
=
,
mode_name
=
'model.ckpt'
,
save_dir
=
'checkpoint'
,
var_list
=
,
is_latest
=
True
,
printable
=
False
)
:
if
sess
is
:
raise
ValueError
(
"session is None."
)
if
var_list
is
:
var_list
=
[
]
if
is_latest
:
ckpt_file
=
tf
.
train
.
latest_checkpoint
(
save_dir
)
else
:
ckpt_file
=
os
.
path
.
join
(
save_dir
,
mode_name
)
if
not
var_list
:
var_list
=
tf
.
global_variables
(
)
logging
.
info
(
"[*] load %s n_params: %d"
%
(
ckpt_file
,
len
(
var_list
)
)
)
if
printable
:
for
idx
,
v
in
enumerate
(
var_list
)
:
logging
.
info
(
"  param {:3}: {:15}   {}"
.
format
(
idx
,
v
.
name
,
str
(
v
.
get_shape
(
)
)
)
)
try
:
saver
=
tf
.
train
.
Saver
(
var_list
)
saver
.
restore
(
sess
,
ckpt_file
)
except
Exception
as
e
:
logging
.
info
(
e
)
logging
.
info
(
"[*] load ckpt fail ..."
)


def load_npy_to_any(path='', name='file.npy'):
    """Load `.npy` file.

    Parameters
    ------------
    path : str
        Path to the file (optional).
    name : str
        File name.

    Examples
    ---------
    - see tl.files.save_any_to_npy()

    """
    file_path = os.path.join(path, name)
    try:
        return np.load(file_path).item()
    except Exception:
        return np.load(file_path)
    raise Exception("[!] Fail to load %s" % file_path)

def
load_npy_to_any
(
path
=
''
,
name
=
'file.npy'
)
:
file_path
=
os
.
path
.
join
(
path
,
name
)
try
:
return
np
.
load
(
file_path
)
.
item
(
)
except
Exception
:
return
np
.
load
(
file_path
)
raise
Exception
(
"[!] Fail to load %s"
%
file_path
)


def load_file_list(path=None, regx='\.jpg', printable=True, keep_prefix=False):
    r"""Return a file list in a folder by given a path and regular expression.

    Parameters
    ----------
    path : str or None
        A folder path, if `None`, use the current directory.
    regx : str
        The regx of file name.
    printable : boolean
        Whether to print the files infomation.
    keep_prefix : boolean
        Whether to keep path in the file name.

    Examples
    ----------
    >>> file_list = tl.files.load_file_list(path=None, regx='w1pre_[0-9]+\.(npz)')

    """
    if path is None:
        path = os.getcwd()
    file_list = os.listdir(path)
    return_list = []
    for _, f in enumerate(file_list):
        if re.search(regx, f):
            return_list.append(f)
    # return_list.sort()
    if keep_prefix:
        for i, f in enumerate(return_list):
            return_list[i] = os.path.join(path, f)

    if printable:
        logging.info('Match file list = %s' % return_list)
        logging.info('Number of files = %d' % len(return_list))
    return return_list

def
load_file_list
(
path
=
,
regx
=
'\.jpg'
,
printable
=
True
,
keep_prefix
=
False
)
:
if
path
is
:
path
=
os
.
getcwd
(
)
file_list
=
os
.
listdir
(
path
)
return_list
=
[
]
for
_
,
f
in
enumerate
(
file_list
)
:
if
re
.
search
(
regx
,
f
)
:
return_list
.
append
(
f
)
# return_list.sort()
if
keep_prefix
:
for
i
,
f
in
enumerate
(
return_list
)
:
return_list
[
i
]
=
os
.
path
.
join
(
path
,
f
)
if
printable
:
logging
.
info
(
'Match file list = %s'
%
return_list
)
logging
.
info
(
'Number of files = %d'
%
len
(
return_list
)
)
return
return_list

r Return a file list in a folder by given a path and regular expression .


def load_folder_list(path=""):
    """Return a folder list in a folder by given a folder path.

    Parameters
    ----------
    path : str
        A folder path.

    """
    return [os.path.join(path, o) for o in os.listdir(path) if os.path.isdir(os.path.join(path, o))]

def load_folder_list ( path = "" ) : return [ os . path . join ( path , o ) for o in os . listdir ( path ) if os . path . isdir ( os . path . join ( path , o ) ) ]

Return a folder list in a folder by given a folder path .


def exists_or_mkdir(path, verbose=True):
    """Check a folder by given name, if not exist, create the folder and return False,
    if directory exists, return True.

    Parameters
    ----------
    path : str
        A folder path.
    verbose : boolean
        If True (default), prints results.

    Returns
    --------
    boolean
        True if folder already exist, otherwise, returns False and create the folder.

    Examples
    --------
    >>> tl.files.exists_or_mkdir("checkpoints/train")

    """
    if not os.path.exists(path):
        if verbose:
            logging.info("[*] creates %s ..." % path)
        os.makedirs(path)
        return False
    else:
        if verbose:
            logging.info("[!] %s exists ..." % path)
        return True

def
exists_or_mkdir
(
path
,
verbose
=
True
)
:
if
not
os
.
path
.
exists
(
path
)
:
if
verbose
:
logging
.
info
(
"[*] creates %s ..."
%
path
)
os
.
makedirs
(
path
)
return
False
else
:
if
verbose
:
logging
.
info
(
"[!] %s exists ..."
%
path
)
return
True

Check a folder by given name if not exist create the folder and return False if directory exists return True .


def maybe_download_and_extract(filename, working_directory, url_source, extract=False, expected_bytes=None):
    """Checks if file exists in working_directory otherwise tries to dowload the file,
    and optionally also tries to extract the file if format is ".zip" or ".tar"

    Parameters
    -----------
    filename : str
        The name of the (to be) dowloaded file.
    working_directory : str
        A folder path to search for the file in and dowload the file to
    url : str
        The URL to download the file from
    extract : boolean
        If True, tries to uncompress the dowloaded file is ".tar.gz/.tar.bz2" or ".zip" file, default is False.
    expected_bytes : int or None
        If set tries to verify that the downloaded file is of the specified size, otherwise raises an Exception, defaults is None which corresponds to no check being performed.

    Returns
    ----------
    str
        File path of the dowloaded (uncompressed) file.

    Examples
    --------
    >>> down_file = tl.files.maybe_download_and_extract(filename='train-images-idx3-ubyte.gz',
    ...                                            working_directory='data/',
    ...                                            url_source='http://yann.lecun.com/exdb/mnist/')
    >>> tl.files.maybe_download_and_extract(filename='ADEChallengeData2016.zip',
    ...                                             working_directory='data/',
    ...                                             url_source='http://sceneparsing.csail.mit.edu/data/',
    ...                                             extract=True)

    """

    # We first define a download function, supporting both Python 2 and 3.
    def _download(filename, working_directory, url_source):

        progress_bar = progressbar.ProgressBar()

        def _dlProgress(count, blockSize, totalSize, pbar=progress_bar):
            if (totalSize != 0):

                if not pbar.max_value:
                    totalBlocks = math.ceil(float(totalSize) / float(blockSize))
                    pbar.max_value = int(totalBlocks)

                pbar.update(count, force=True)

        filepath = os.path.join(working_directory, filename)

        logging.info('Downloading %s...\n' % filename)

        urlretrieve(url_source + filename, filepath, reporthook=_dlProgress)

    exists_or_mkdir(working_directory, verbose=False)
    filepath = os.path.join(working_directory, filename)

    if not os.path.exists(filepath):

        _download(filename, working_directory, url_source)
        statinfo = os.stat(filepath)
        logging.info('Succesfully downloaded %s %s bytes.' % (filename, statinfo.st_size))  # , 'bytes.')
        if (not (expected_bytes is None) and (expected_bytes != statinfo.st_size)):
            raise Exception('Failed to verify ' + filename + '. Can you get to it with a browser?')
        if (extract):
            if tarfile.is_tarfile(filepath):
                logging.info('Trying to extract tar file')
                tarfile.open(filepath, 'r').extractall(working_directory)
                logging.info('... Success!')
            elif zipfile.is_zipfile(filepath):
                logging.info('Trying to extract zip file')
                with zipfile.ZipFile(filepath) as zf:
                    zf.extractall(working_directory)
                logging.info('... Success!')
            else:
                logging.info("Unknown compression_format only .tar.gz/.tar.bz2/.tar and .zip supported")
    return filepath

Checks if file exists in working_directory otherwise tries to dowload the file and optionally also tries to extract the file if format is . zip or . tar


def natural_keys(text):
    """Sort list of string with number in human order.

    Examples
    ----------
    >>> l = ['im1.jpg', 'im31.jpg', 'im11.jpg', 'im21.jpg', 'im03.jpg', 'im05.jpg']
    >>> l.sort(key=tl.files.natural_keys)
    ['im1.jpg', 'im03.jpg', 'im05', 'im11.jpg', 'im21.jpg', 'im31.jpg']
    >>> l.sort() # that is what we dont want
    ['im03.jpg', 'im05', 'im1.jpg', 'im11.jpg', 'im21.jpg', 'im31.jpg']

    References
    ----------
    - `link <http://nedbatchelder.com/blog/200712/human_sorting.html>`__

    """

    # - alist.sort(key=natural_keys) sorts in human order
    # http://nedbatchelder.com/blog/200712/human_sorting.html
    # (See Toothy's implementation in the comments)
    def atoi(text):
        return int(text) if text.isdigit() else text

    return [atoi(c) for c in re.split('(\d+)', text)]

Sort list of string with number in human order .


def threading_data(data=None, fn=None, thread_count=None, **kwargs):
    """Process a batch of data by given function by threading.

    Usually be used for data augmentation.

    Parameters
    -----------
    data : numpy.array or others
        The data to be processed.
    thread_count : int
        The number of threads to use.
    fn : function
        The function for data processing.
    more args : the args for `fn`
        Ssee Examples below.

    Examples
    --------
    Process images.

    >>> images, _, _, _ = tl.files.load_cifar10_dataset(shape=(-1, 32, 32, 3))
    >>> images = tl.prepro.threading_data(images[0:32], tl.prepro.zoom, zoom_range=[0.5, 1])

    Customized image preprocessing function.

    >>> def distort_img(x):
    >>>     x = tl.prepro.flip_axis(x, axis=0, is_random=True)
    >>>     x = tl.prepro.flip_axis(x, axis=1, is_random=True)
    >>>     x = tl.prepro.crop(x, 100, 100, is_random=True)
    >>>     return x
    >>> images = tl.prepro.threading_data(images, distort_img)

    Process images and masks together (Usually be used for image segmentation).

    >>> X, Y --> [batch_size, row, col, 1]
    >>> data = tl.prepro.threading_data([_ for _ in zip(X, Y)], tl.prepro.zoom_multi, zoom_range=[0.5, 1], is_random=True)
    data --> [batch_size, 2, row, col, 1]
    >>> X_, Y_ = data.transpose((1,0,2,3,4))
    X_, Y_ --> [batch_size, row, col, 1]
    >>> tl.vis.save_image(X_, 'images.png')
    >>> tl.vis.save_image(Y_, 'masks.png')

    Process images and masks together by using ``thread_count``.

    >>> X, Y --> [batch_size, row, col, 1]
    >>> data = tl.prepro.threading_data(X, tl.prepro.zoom_multi, 8, zoom_range=[0.5, 1], is_random=True)
    data --> [batch_size, 2, row, col, 1]
    >>> X_, Y_ = data.transpose((1,0,2,3,4))
    X_, Y_ --> [batch_size, row, col, 1]
    >>> tl.vis.save_image(X_, 'after.png')
    >>> tl.vis.save_image(Y_, 'before.png')

    Customized function for processing images and masks together.

    >>> def distort_img(data):
    >>>    x, y = data
    >>>    x, y = tl.prepro.flip_axis_multi([x, y], axis=0, is_random=True)
    >>>    x, y = tl.prepro.flip_axis_multi([x, y], axis=1, is_random=True)
    >>>    x, y = tl.prepro.crop_multi([x, y], 100, 100, is_random=True)
    >>>    return x, y

    >>> X, Y --> [batch_size, row, col, channel]
    >>> data = tl.prepro.threading_data([_ for _ in zip(X, Y)], distort_img)
    >>> X_, Y_ = data.transpose((1,0,2,3,4))

    Returns
    -------
    list or numpyarray
        The processed results.

    References
    ----------
    - `python queue <https://pymotw.com/2/Queue/index.html#module-Queue>`__
    - `run with limited queue <http://effbot.org/librarybook/queue.htm>`__

    """

    def apply_fn(results, i, data, kwargs):
        results[i] = fn(data, **kwargs)

    if thread_count is None:
        results = [None] * len(data)
        threads = []
        # for i in range(len(data)):
        #     t = threading.Thread(name='threading_and_return', target=apply_fn, args=(results, i, data[i], kwargs))
        for i, d in enumerate(data):
            t = threading.Thread(name='threading_and_return', target=apply_fn, args=(results, i, d, kwargs))
            t.start()
            threads.append(t)
    else:
        divs = np.linspace(0, len(data), thread_count + 1)
        divs = np.round(divs).astype(int)
        results = [None] * thread_count
        threads = []
        for i in range(thread_count):
            t = threading.Thread(
                name='threading_and_return', target=apply_fn, args=(results, i, data[divs[i]:divs[i + 1]], kwargs)
            )
            t.start()
            threads.append(t)

    for t in threads:
        t.join()

    if thread_count is None:
        try:
            return np.asarray(results)
        except Exception:
            return results
    else:
        return np.concatenate(results)

Process a batch of data by given function by threading .


def affine_transform_keypoints(coords_list, transform_matrix):
    """Transform keypoint coordinates according to a given affine transform matrix.
    OpenCV format, x is width.

    Note that, for pose estimation task, flipping requires maintaining the left and right body information.
    We should not flip the left and right body, so please use ``tl.prepro.keypoint_random_flip``.

    Parameters
    -----------
    coords_list : list of list of tuple/list
        The coordinates
        e.g., the keypoint coordinates of every person in an image.
    transform_matrix : numpy.array
        Transform matrix, OpenCV format.

    Examples
    ---------
    >>> # 1. get all affine transform matrices
    >>> M_rotate = tl.prepro.affine_rotation_matrix(angle=20)
    >>> M_flip = tl.prepro.affine_horizontal_flip_matrix(prob=1)
    >>> # 2. combine all affine transform matrices to one matrix
    >>> M_combined = dot(M_flip).dot(M_rotate)
    >>> # 3. transfrom the matrix from Cartesian coordinate (the origin in the middle of image)
    >>> # to Image coordinate (the origin on the top-left of image)
    >>> transform_matrix = tl.prepro.transform_matrix_offset_center(M_combined, x=w, y=h)
    >>> # 4. then we can transfrom the image once for all transformations
    >>> result = tl.prepro.affine_transform_cv2(image, transform_matrix)  # 76 times faster
    >>> # 5. transform keypoint coordinates
    >>> coords = [[(50, 100), (100, 100), (100, 50), (200, 200)], [(250, 50), (200, 50), (200, 100)]]
    >>> coords_result = tl.prepro.affine_transform_keypoints(coords, transform_matrix)
    """
    coords_result_list = []
    for coords in coords_list:
        coords = np.asarray(coords)
        coords = coords.transpose([1, 0])
        coords = np.insert(coords, 2, 1, axis=0)
        # print(coords)
        # print(transform_matrix)
        coords_result = np.matmul(transform_matrix, coords)
        coords_result = coords_result[0:2, :].transpose([1, 0])
        coords_result_list.append(coords_result)
    return coords_result_list

def
affine_transform_keypoints
(
coords_list
,
transform_matrix
)
:
coords_result_list
=
[
]
for
coords
in
coords_list
:
coords
=
np
.
asarray
(
coords
)
coords
=
coords
.
transpose
(
[
1
,
0
]
)
coords
=
np
.
insert
(
coords
,
2
,
1
,
axis
=
0
)
# print(coords)
# print(transform_matrix)
coords_result
=
np
.
matmul
(
transform_matrix
,
coords
)
coords_result
=
coords_result
[
0
:
2
,
:
]
.
transpose
(
[
1
,
0
]
)
coords_result_list
.
append
(
coords_result
)
return
coords_result_list

Transform keypoint coordinates according to a given affine transform matrix . OpenCV format x is width .


def projective_transform_by_points(
        x, src, dst, map_args=None, output_shape=None, order=1, mode='constant', cval=0.0, clip=True,
        preserve_range=False
):
    """Projective transform by given coordinates, usually 4 coordinates.

    see `scikit-image <http://scikit-image.org/docs/dev/auto_examples/applications/plot_geometric.html>`__.

    Parameters
    -----------
    x : numpy.array
        An image with dimension of [row, col, channel] (default).
    src : list or numpy
        The original coordinates, usually 4 coordinates of (width, height).
    dst : list or numpy
        The coordinates after transformation, the number of coordinates is the same with src.
    map_args : dictionary or None
        Keyword arguments passed to inverse map.
    output_shape : tuple of 2 int
        Shape of the output image generated. By default the shape of the input image is preserved. Note that, even for multi-band images, only rows and columns need to be specified.
    order : int
        The order of interpolation. The order has to be in the range 0-5:
            - 0 Nearest-neighbor
            - 1 Bi-linear (default)
            - 2 Bi-quadratic
            - 3 Bi-cubic
            - 4 Bi-quartic
            - 5 Bi-quintic
    mode : str
        One of `constant` (default), `edge`, `symmetric`, `reflect` or `wrap`.
        Points outside the boundaries of the input are filled according to the given mode. Modes match the behaviour of numpy.pad.
    cval : float
        Used in conjunction with mode `constant`, the value outside the image boundaries.
    clip : boolean
        Whether to clip the output to the range of values of the input image. This is enabled by default, since higher order interpolation may produce values outside the given input range.
    preserve_range : boolean
        Whether to keep the original range of values. Otherwise, the input image is converted according to the conventions of img_as_float.

    Returns
    -------
    numpy.array
        A processed image.

    Examples
    --------
    Assume X is an image from CIFAR-10, i.e. shape == (32, 32, 3)

    >>> src = [[0,0],[0,32],[32,0],[32,32]]     # [w, h]
    >>> dst = [[10,10],[0,32],[32,0],[32,32]]
    >>> x = tl.prepro.projective_transform_by_points(X, src, dst)

    References
    -----------
    - `scikit-image : geometric transformations <http://scikit-image.org/docs/dev/auto_examples/applications/plot_geometric.html>`__
    - `scikit-image : examples <http://scikit-image.org/docs/dev/auto_examples/index.html>`__

    """
    if map_args is None:
        map_args = {}
    # if type(src) is list:
    if isinstance(src, list):  # convert to numpy
        src = np.array(src)
    # if type(dst) is list:
    if isinstance(dst, list):
        dst = np.array(dst)
    if np.max(x) > 1:  # convert to [0, 1]
        x = x / 255

    m = transform.ProjectiveTransform()
    m.estimate(dst, src)
    warped = transform.warp(
        x, m, map_args=map_args, output_shape=output_shape, order=order, mode=mode, cval=cval, clip=clip,
        preserve_range=preserve_range
    )
    return warped

def
projective_transform_by_points
(
x
,
src
,
dst
,
map_args
=
,
output_shape
=
,
order
=
1
,
mode
=
'constant'
,
cval
=
0.0
,
clip
=
True
,
preserve_range
=
False
)
:
if
map_args
is
:
map_args
=
{
}
# if type(src) is list:
if
isinstance
(
src
,
list
)
:
# convert to numpy
src
=
np
.
array
(
src
)
# if type(dst) is list:
if
isinstance
(
dst
,
list
)
:
dst
=
np
.
array
(
dst
)
if
np
.
max
(
x
)
>
1
:
# convert to [0, 1]
x
=
x
/
255
m
=
transform
.
ProjectiveTransform
(
)
m
.
estimate
(
dst
,
src
)
warped
=
transform
.
warp
(
x
,
m
,
map_args
=
map_args
,
output_shape
=
output_shape
,
order
=
order
,
mode
=
mode
,
cval
=
cval
,
clip
=
clip
,
preserve_range
=
preserve_range
)
return
warped

Projective transform by given coordinates usually 4 coordinates .


def rotation(
        x, rg=20, is_random=False, row_index=0, col_index=1, channel_index=2, fill_mode='nearest', cval=0., order=1
):
    """Rotate an image randomly or non-randomly.

    Parameters
    -----------
    x : numpy.array
        An image with dimension of [row, col, channel] (default).
    rg : int or float
        Degree to rotate, usually 0 ~ 180.
    is_random : boolean
        If True, randomly rotate. Default is False
    row_index col_index and channel_index : int
        Index of row, col and channel, default (0, 1, 2), for theano (1, 2, 0).
    fill_mode : str
        Method to fill missing pixel, default `nearest`, more options `constant`, `reflect` or `wrap`, see `scipy ndimage affine_transform <https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.ndimage.interpolation.affine_transform.html>`__
    cval : float
        Value used for points outside the boundaries of the input if mode=`constant`. Default is 0.0
    order : int
        The order of interpolation. The order has to be in the range 0-5. See ``tl.prepro.affine_transform`` and `scipy ndimage affine_transform <https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.ndimage.interpolation.affine_transform.html>`__

    Returns
    -------
    numpy.array
        A processed image.

    Examples
    ---------
    >>> x --> [row, col, 1]
    >>> x = tl.prepro.rotation(x, rg=40, is_random=False)
    >>> tl.vis.save_image(x, 'im.png')

    """
    if is_random:
        theta = np.pi / 180 * np.random.uniform(-rg, rg)
    else:
        theta = np.pi / 180 * rg
    rotation_matrix = np.array([[np.cos(theta), -np.sin(theta), 0], [np.sin(theta), np.cos(theta), 0], [0, 0, 1]])

    h, w = x.shape[row_index], x.shape[col_index]
    transform_matrix = transform_matrix_offset_center(rotation_matrix, h, w)
    x = affine_transform(x, transform_matrix, channel_index, fill_mode, cval, order)
    return x

def rotation ( x , rg = 20 , is_random = False , row_index = 0 , col_index = 1 , channel_index = 2 , fill_mode = 'nearest' , cval = 0. , order = 1 ) : if is_random : theta = np . pi / 180 * np . random . uniform ( - rg , rg ) else : theta = np . pi / 180 * rg rotation_matrix = np . array ( [ [ np . cos ( theta ) , - np . sin ( theta ) , 0 ] , [ np . sin ( theta ) , np . cos ( theta ) , 0 ] , [ 0 , 0 , 1 ] ] ) h , w = x . shape [ row_index ] , x . shape [ col_index ] transform_matrix = transform_matrix_offset_center ( rotation_matrix , h , w ) x = affine_transform ( x , transform_matrix , channel_index , fill_mode , cval , order ) return x

Rotate an image randomly or non - randomly .