def roll_up(
        df,
        levels: List[str],
        groupby_vars: List[str],
        extra_groupby_cols: List[str] = None,
        var_name: str = 'type',
        value_name: str = 'value',
        agg_func: str = 'sum',
        drop_levels: List[str] = None
):
    """
    Creates aggregates following a given hierarchy

    ---

    ### Parameters

    *mandatory :*
    - `levels` (*list of str*): name of the columns composing the hierarchy (from the top to the bottom level).
    - `groupby_vars` (*list of str*): name of the columns with value to aggregate.
    - `extra_groupby_cols` (*list of str*) optional: other columns used to group in each level.

    *optional :*
    - `var_name` (*str*) : name of the result variable column. By default, `“type”`.
    - `value_name` (*str*): name of the result value column. By default, `“value”`.
    - `agg_func` (*str*): name of the aggregation operation. By default, `“sum”`.
    - `drop_levels` (*list of str*): the names of the levels that you may want to discard from the output.

    ---

    ### Example

    **Input**

    |    Region |     City |  Population |
    |:---------:|:--------:|:-----------:|
    |       Idf |     Panam|         200 |
    |       Idf |   Antony |          50 |
    |      Nord |    Lille |          20 |

    ```cson
    roll_up:
      levels: ["Region", "City"]
      groupby_vars: "Population"
    ```

    **Output**

    |    Region |     City |  Population |    value |   type |
    |:---------:|:--------:|:-----------:|:--------:|:------:|
    |       Idf |     Panam|         200 |    Panam |   City |
    |       Idf |   Antony |          50 |   Antony |   City |
    |      Nord |    Lille |          20 |    Lille |   City |
    |       Idf |      Nan |         250 |      Idf | Region |
    |      Nord |      Nan |          20 |     Nord | Region |
    """
    dfs = list()
    groupby_cols_cpy = list(levels)
    levels_cpy = list(levels)
    levels_cpy.reverse()

    extra_groupby_cols = extra_groupby_cols or []
    drop_levels = drop_levels or []
    previous_level = None
    for top_level in levels_cpy:
        # Aggregation
        gb_df = getattr(
            df.groupby(groupby_cols_cpy + extra_groupby_cols)[groupby_vars],
            agg_func)().reset_index()

        # Melt-like columns
        gb_df[var_name] = top_level
        gb_df[value_name] = gb_df[top_level]
        dfs.append(gb_df)
        if previous_level in drop_levels:
            del dfs[-2]
        previous_level = top_level

        # Remove one level each time in the groupby: lowest level column needs
        # a groupby with every levels, the next level needs every one except
        # the lowest, etc. until the top level column that needs only itself
        # inside the groupby.
        groupby_cols_cpy.pop()
    return pd.concat(dfs, sort=False).reset_index()


def argmax(df, column: str, groups: Union[str, List[str]] = None):
    """
    Keep the row of the data corresponding to the maximal value in a column

    ---

    ### Parameters

    *mandatory :*
    - `column` (*str*): name of the column containing the value you want to keep the maximum

    *optional :*
    - `groups` (*str or list(str)*): name of the column(s) used for 'groupby' logic
    (the function will return the argmax by group)

    ---

    ### Example

    **Input**

    | variable |   wave  |  year    | value |
    |:--------:|:-------:|:--------:|:-----:|
    |   toto   |  wave 1 |  2014    |  300  |
    |   toto   |  wave 1 |  2015    |  250  |
    |   toto   |  wave 1 |  2016    |  450  |

    ```cson
    argmax:
      column: 'year'
    ```

    **Output**

    | variable |   wave  |  year    | value |
    |:--------:|:-------:|:--------:|:-----:|
    |   toto   |  wave 1 |  2016    |  450  |
    """
    if groups is None:
        df = df[df[column] == df[column].max()].reset_index(drop=True)
    else:
        group_max = df.groupby(groups)[column].transform('max')
        df = (df
              .loc[df[column] == group_max, :]
              .drop_duplicates()
              .reset_index(drop=True)
              )
    return df

def argmax ( df , column : str , groups : Union [ str , List [ str ] ] = None ) : if groups is None : df = df [ df [ column ] == df [ column ] . max ( ) ] . reset_index ( drop = True ) else : group_max = df . groupby ( groups ) [ column ] . transform ( 'max' ) df = ( df . loc [ df [ column ] == group_max , : ] . drop_duplicates ( ) . reset_index ( drop = True ) ) return df


def argmin(df, column: str, groups: Union[str, List[str]] = None):
    """
    Keep the row of the data corresponding to the minimal value in a column

    ---

    ### Parameters

    *mandatory :*
    - `column` (str): name of the column containing the value you want to keep the minimum

    *optional :*
    - `groups` (*str or list(str)*): name of the column(s) used for 'groupby' logic
    (the function will return the argmax by group)
    ---

    ### Example

    **Input**

    | variable |   wave  |  year    | value |
    |:--------:|:-------:|:--------:|:-----:|
    |   toto   |  wave 1 |  2014    |  300  |
    |   toto   |  wave 1 |  2015    |  250  |
    |   toto   |  wave 1 |  2016    |  450  |

    ```cson
    argmin:
      column: 'year'
    ]
    ```

    **Output**

    | variable |   wave  |  year    | value |
    |:--------:|:-------:|:--------:|:-----:|
    |   toto   |  wave 1 |  2015    |  250  |
    """
    if groups is None:
        df = df[df[column] == df[column].min()].reset_index(drop=True)
    else:
        group_min = df.groupby(groups)[column].transform('min')
        df = (df
              .loc[df[column] == group_min, :]
              .drop_duplicates()
              .reset_index(drop=True)
              )
    return df

def argmin ( df , column : str , groups : Union [ str , List [ str ] ] = None ) : if groups is None : df = df [ df [ column ] == df [ column ] . min ( ) ] . reset_index ( drop = True ) else : group_min = df . groupby ( groups ) [ column ] . transform ( 'min' ) df = ( df . loc [ df [ column ] == group_min , : ] . drop_duplicates ( ) . reset_index ( drop = True ) ) return df


def fillna(df, column: str, value=None,  column_value=None):
    """
    Can fill NaN values from a column with a given value or a column

    ---

    ### Parameters

    - `column` (*str*): name of column you want to fill
    - `value`: NaN will be replaced by this value
    - `column_value`: NaN will be replaced by value from this column

    *NOTE*: You must set either the 'value' parameter or the 'column_value' parameter

    ---

    ### Example

    **Input**

    | variable |   wave  |  year    | my_value |
    |:--------:|:-------:|:--------:|:--------:|
    |   toto   |  wave 1 |  2014    |  300     |
    |   toto   |  wave 1 |  2015    |          |
    |   toto   |  wave 1 |  2016    |  450     |

    ```cson
    fillna:
      column: 'my_value'
      value: 0
    ```

    **Output**

    | variable |   wave  |  year    | my_value |
    |:--------:|:-------:|:--------:|:--------:|
    |   toto   |  wave 1 |  2014    |  300     |
    |   toto   |  wave 1 |  2015    |    0     |
    |   toto   |  wave 1 |  2016    |  450     |
    """
    if column not in df.columns:
        df[column] = nan

    if value is not None and column_value is not None:
        raise ValueError('You cannot set both the parameters value and column_value')

    if value is not None:
        df[column] = df[column].fillna(value)

    if column_value is not None:
        if column_value not in df.columns:
            raise ValueError(f'"{column_value}" is not a valid column name')
        df[column] = df[column].fillna(df[column_value])

    return df


def add_offset(dateobj, hr_offset: str, sign: str):
    """add a human readable offset to `dateobj` and return corresponding date.

    rely on `pandas.Timedelta` and add the following extra shortcuts:
    - "w", "week" and "weeks" for a week (i.e. 7days)
    - "month', "months" for a month (i.e. no day computation, just increment the month)
    - "y", "year', "years" for a year (i.e. no day computation, just increment the year)
    """
    sign_coeff = 1 if sign == '+' else -1
    try:
        return dateobj + sign_coeff * pd.Timedelta(hr_offset)
    except ValueError:
        # pd.Timedelta could not parse the offset, let's try harder
        match = TIMEDELTA_RGX.match(hr_offset)
        if match is not None:
            groups = match.groupdict()
            unit = groups['unit'].lower()[0]
            num = sign_coeff * int(groups['num'])
            # is it a week ?
            if unit == 'w':
                return dateobj + num * timedelta(weeks=1)
            # or a month ?
            if unit == 'm':
                return add_months(dateobj, num)
            # or a year ?
            if unit == 'y':
                return add_years(dateobj, num)
        # we did what we could, just re-raise the original exception
        raise

add a human readable offset to dateobj and return corresponding date .


def add_months(dateobj, nb_months: int):
    """return `dateobj` + `nb_months`

    If landing date doesn't exist (e.g. february, 30th), return the last
    day of the landing month.

    >>> add_months(date(2018, 1, 1), 1)
    datetime.date(2018, 1, 1)
    >>> add_months(date(2018, 1, 1), -1)
    datetime.date(2017, 12, 1)
    >>> add_months(date(2018, 1, 1), 25)
    datetime.date(2020, 2, 1)
    >>> add_months(date(2018, 1, 1), -25)
    datetime.date(2015, 12, 1)
    >>> add_months(date(2018, 1, 31), 1)
    datetime.date(2018, 2, 28)
    """
    nb_years, nb_months = divmod(nb_months, 12)
    month = dateobj.month + nb_months
    if month > 12:
        nb_years += 1
        month -= 12
    year = dateobj.year + nb_years
    lastday = monthrange(year, month)[1]
    return dateobj.replace(year=year, month=month, day=min(lastday, dateobj.day))

def add_months ( dateobj , nb_months : int ) : nb_years , nb_months = divmod ( nb_months , 12 ) month = dateobj . month + nb_months if month > 12 : nb_years += 1 month -= 12 year = dateobj . year + nb_years lastday = monthrange ( year , month ) [ 1 ] return dateobj . replace ( year = year , month = month , day = min ( lastday , dateobj . day ) )


def add_years(dateobj, nb_years):
    """return `dateobj` + `nb_years`

    If landing date doesn't exist (e.g. february, 30th), return the last
    day of the landing month.

    >>> add_years(date(2018, 1, 1), 1)
    datetime.date(2019, 1, 1)
    >>> add_years(date(2018, 1, 1), -1)
    datetime.date(2017, 1, 1)
    >>> add_years(date(2020, 2, 29), 1)
    datetime.date(2021, 2, 28)
    >>> add_years(date(2020, 2, 29), -1)
    datetime.date(2019, 2, 28)
    """
    year = dateobj.year + nb_years
    lastday = monthrange(year, dateobj.month)[1]
    return dateobj.replace(year=year, day=min(lastday, dateobj.day))

def add_years ( dateobj , nb_years ) : year = dateobj . year + nb_years lastday = monthrange ( year , dateobj . month ) [ 1 ] return dateobj . replace ( year = year , day = min ( lastday , dateobj . day ) )


def parse_date(datestr: str, date_fmt: str) -> date:
    """parse `datestr` and return corresponding date object.

    `datestr` should be a string matching `date_fmt` and parseable by `strptime`
    but some offset can also be added using `(datestr) + OFFSET` or `(datestr) -
    OFFSET` syntax. When using this syntax, `OFFSET` should be understable by
    `pandas.Timedelta` (cf.
    http://pandas.pydata.org/pandas-docs/stable/timedeltas.html) and `w`, `week`
    `month` and `year` offset keywords are also accepted. `datestr` MUST be wrapped
    with parenthesis.

    Additionally, the following symbolic names are supported: `TODAY`,
    `YESTERDAY`, `TOMORROW`.

    Example usage:

    >>> parse_date('2018-01-01', '%Y-%m-%d') datetime.date(2018, 1, 1)
    parse_date('(2018-01-01) + 1day', '%Y-%m-%d') datetime.date(2018, 1, 2)
    parse_date('(2018-01-01) + 2weeks', '%Y-%m-%d') datetime.date(2018, 1, 15)

    Parameters: `datestr`: the date to parse, formatted as `date_fmt`
        `date_fmt`: expected date format

    Returns: The `date` object. If date could not be parsed, a ValueError will
        be raised.
    """
    rgx = re.compile(r'\((?P<date>.*)\)(\s*(?P<sign>[+-])(?P<offset>.*))?$')
    datestr = datestr.strip()
    match = rgx.match(datestr)
    # if regexp doesn't match, date must match the expected format
    if match is None:
        return _norm_date(datestr, date_fmt)
    datestr = match.group('date').strip()
    dateobj = _norm_date(datestr, date_fmt)
    offset = match.group('offset')
    if offset:
        return add_offset(dateobj, offset, match.group('sign'))
    return dateobj

parse datestr and return corresponding date object .


def filter_by_date(
    df,
    date_col: str,
    date_format: str = '%Y-%m-%d',
    start: str = None,
    stop: str = None,
    atdate: str = None
):
    """
    Filter dataframe your data by date.

    This function will interpret `start`, `stop` and `atdate` and build
    the corresponding date range. The caller must specify either:

    - `atdate`: keep all rows matching this date exactly,
    - `start`: keep all rows matching this date onwards.
    - `stop`: keep all rows matching dates before this one.
    - `start` and `stop`: keep all rows between `start` and `stop`,

    Any other combination will raise an error. The lower bound of the date range
    will be included, the upper bound will be excluded.

    When specified, `start`, `stop` and `atdate` values are expected to match the
    `date_format` format or a known symbolic value (i.e. 'TODAY', 'YESTERDAY' or 'TOMORROW').

    Additionally, the offset syntax "(date) + offset" is also supported (Mind
    the parenthesis around the date string). In that case, the offset must be
    one of the syntax supported by `pandas.Timedelta` (see [pandas doc](
    http://pandas.pydata.org/pandas-docs/stable/timedeltas.html))

    ---

    ### Parameters

    *mandatory :*
    - `date_col` (*str*): the name of the dataframe's column to filter on

    *optional :*
    - `date_format` (*str*): expected date format in column `date_col` (see [available formats](
    https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior)
    - `start` (*str*): if specified, lower bound (included) of the date range
    - `stop` (*str*): if specified, upper bound (excluded) of the date range
    - `atdate` (*str*): if specified, the exact date we're filtering on
    """
    mask = None
    if start is None and stop is None and atdate is None:
        raise TypeError('either "start", "stop" or "atdate" must be specified')
    if start is not None and atdate is not None:
        raise TypeError('"start" and "atdate" are mutually exclusive')
    if stop is not None and atdate is not None:
        raise TypeError('"stop" and "atdate" are mutually exclusive')
    # add a new column that will hold actual date objects instead of strings.
    # This column is just temporary and will be removed before returning the
    # filtered dataframe.
    filtercol = str(uuid4())
    df[filtercol] = pd.to_datetime(df[date_col], format=date_format)
    if atdate is not None:
        mask = df[filtercol] == parse_date(atdate, date_format)
    elif start is not None and stop is not None:
        mask = ((df[filtercol] >= parse_date(start, date_format)) &
                (df[filtercol] < parse_date(stop, date_format)))
    elif stop is None:
        mask = df[filtercol] >= parse_date(start, date_format)
    elif start is None:
        mask = df[filtercol] < parse_date(stop, date_format)
    return df[mask].drop(filtercol, axis=1)


def percentage(
        df,
        column: str,
        group_cols: Union[str, List[str]] = None,
        new_column: str = None
):
    """
    Add a column to the dataframe according to the groupby logic on group_cols

    ---

    ### Parameters

    *mandatory :*
    - `column` (*str*): name of the desired column you need percentage on

    *optional :*
    - `group_cols` (*list*): names of columns for the groupby logic
    - `new_column` (*str*): name of the output column. By default `column` will be overwritten.

    ---

    **Input**

    | gender |    sport   | number |
    |:------:|:----------:|:------:|
    |  male  |   bicycle  |   17   |
    | female | basketball |   17   |
    |  male  | basketball |    3   |
    | female |  football  |    7   |
    | female |   running  |   30   |
    |  male  |   running  |   20   |
    |  male  |  football  |   21   |
    | female |   bicycle  |   17   |

    ```cson
    percentage:
      new_column: 'number_percentage'
      column: 'number'
      group_cols: ['sport']
    ```

    **Output**

    | gender |    sport   | number | number_percentage |
    |:------:|:----------:|:------:|:-----------------:|
    |  male  |   bicycle  |   17   |        50.0       |
    | female | basketball |   17   |        85.0       |
    |  male  | basketball |    3   |        15.0       |
    | female |  football  |    7   |        25.0       |
    | female |   running  |   30   |        60.0       |
    |  male  |   running  |   20   |        40.0       |
    |  male  |  football  |   21   |        75.0       |
    | female |   bicycle  |   17   |        50.0       |
    """
    new_column = new_column or column
    if group_cols is None:
        df[new_column] = 100. * df[column] / sum(df[column])
    else:
        df[new_column] = 100. * df[column] / df.groupby(group_cols)[column].transform(sum)
    return df

def percentage ( df , column : str , group_cols : Union [ str , List [ str ] ] = None , new_column : str = None ) : new_column = new_column or column if group_cols is None : df [ new_column ] = 100. * df [ column ] / sum ( df [ column ] ) else : df [ new_column ] = 100. * df [ column ] / df . groupby ( group_cols ) [ column ] . transform ( sum ) return df


def ada_family_core(params, gparams, learning_rate = 0.01, eps= 1e-6, rho=0.95, method="ADADELTA",
                        beta=0.0, gsum_regularization = 0.0001):
    """
    Optimize by SGD, AdaGrad, or AdaDelta.
    """

    _, _, _, args = inspect.getargvalues(inspect.currentframe())
    logging.info("ada_family_core: %s" % str(args.items()))
    free_parameters = []

    if method == "FINETUNING_ADAGRAD":
        method = "ADAGRAD"
        gsum_regularization = 0

    oneMinusBeta = 1 - beta

    gsums   = [theano.shared(np.zeros_like(param.get_value(borrow=True), dtype=FLOATX), name="gsum_%s" % param.name) if (method == 'ADADELTA' or method == 'ADAGRAD') else None for param in params]
    xsums   = [theano.shared(np.zeros_like(param.get_value(borrow=True), dtype=FLOATX), name="xsum_%s" % param.name) if method == 'ADADELTA' else None for param in params]

    # Fix for AdaGrad, init gsum to 1
    if method == 'ADAGRAD':
        for gsum in gsums:
            gsum.set_value(gsum.get_value() ** 0)

    updates = OrderedDict()
    # Updates
    for gparam, param, gsum, xsum in zip(gparams, params, gsums, xsums):

        if method == 'ADADELTA':
            updates[gsum] = rho * gsum + (1. - rho) * (gparam **2)
            dparam = -T.sqrt((xsum + eps) / (updates[gsum] + eps)) * gparam
            updates[xsum] =rho * xsum + (1. - rho) * (dparam **2)
            updates[param] = param * oneMinusBeta + dparam
        elif method == 'ADAGRAD':
            updates[gsum] = gsum + (gparam **2) - gsum_regularization * gsum
            updates[param] =  param * oneMinusBeta - learning_rate * (gparam / (T.sqrt(updates[gsum] + eps)))

        else:
            updates[param] = param * oneMinusBeta - gparam * learning_rate
    # Add free parameters
    if method == 'ADADELTA':
        free_parameters.extend(gsums + xsums)
    elif method == 'ADAGRAD':
        free_parameters.extend(gsums)
    # Check dtype
    for k in updates:
        if updates[k].dtype != FLOATX:
            updates[k] = updates[k].astype(FLOATX)
    return updates.items(), free_parameters

def
ada_family_core
(
params
,
gparams
,
learning_rate
=
0.01
,
eps
=
1e-6
,
rho
=
0.95
,
method
=
"ADADELTA"
,
beta
=
0.0
,
gsum_regularization
=
0.0001
)
:
_
,
_
,
_
,
args
=
inspect
.
getargvalues
(
inspect
.
currentframe
(
)
)
logging
.
info
(
"ada_family_core: %s"
%
str
(
args
.
items
(
)
)
)
free_parameters
=
[
]
if
method
==
"FINETUNING_ADAGRAD"
:
method
=
"ADAGRAD"
gsum_regularization
=
0
oneMinusBeta
=
1
beta
gsums
=
[
theano
.
shared
(
np
.
zeros_like
(
param
.
get_value
(
borrow
=
True
)
,
dtype
=
FLOATX
)
,
name
=
"gsum_%s"
%
param
.
name
)
if
(
method
==
'ADADELTA'
or
method
==
'ADAGRAD'
)
else
for
param
in
params
]
xsums
=
[
theano
.
shared
(
np
.
zeros_like
(
param
.
get_value
(
borrow
=
True
)
,
dtype
=
FLOATX
)
,
name
=
"xsum_%s"
%
param
.
name
)
if
method
==
'ADADELTA'
else
for
param
in
params
]
# Fix for AdaGrad, init gsum to 1
if
method
==
'ADAGRAD'
:
for
gsum
in
gsums
:
gsum
.
set_value
(
gsum
.
get_value
(
)
**
0
)
updates
=
OrderedDict
(
)
# Updates
for
gparam
,
param
,
gsum
,
xsum
in
zip
(
gparams
,
params
,
gsums
,
xsums
)
:
if
method
==
'ADADELTA'
:
updates
[
gsum
]
=
rho
*
gsum
+
(
1.
rho
)
*
(
gparam
**
2
)
dparam
=
T
.
sqrt
(
(
xsum
+
eps
)
/
(
updates
[
gsum
]
+
eps
)
)
*
gparam
updates
[
xsum
]
=
rho
*
xsum
+
(
1.
rho
)
*
(
dparam
**
2
)
updates
[
param
]
=
param
*
oneMinusBeta
+
dparam
elif
method
==
'ADAGRAD'
:
updates
[
gsum
]
=
gsum
+
(
gparam
**
2
)
gsum_regularization
*
gsum
updates
[
param
]
=
param
*
oneMinusBeta
learning_rate
*
(
gparam
/
(
T
.
sqrt
(
updates
[
gsum
]
+
eps
)
)
)
else
:
updates
[
param
]
=
param
*
oneMinusBeta
gparam
*
learning_rate
# Add free parameters
if
method
==
'ADADELTA'
:
free_parameters
.
extend
(
gsums
+
xsums
)
elif
method
==
'ADAGRAD'
:
free_parameters
.
extend
(
gsums
)
# Check dtype
for
k
in
updates
:
if
updates
[
k
]
.
dtype
!=
FLOATX
:
updates
[
k
]
=
updates
[
k
]
.
astype
(
FLOATX
)
return
updates
.
items
(
)
,
free_parameters


def _learning_updates(self):
        """
        Return updates in the training.
        """
        params = self.training_params()
        gradients = self.get_gradients(params)
        return self.optimization_updates(params, gradients)

def _learning_updates ( self ) : params = self . training_params ( ) gradients = self . get_gradients ( params ) return self . optimization_updates ( params , gradients )


def training_params(self):
        """
        Get parameters to be optimized.
        """
        params = self.network.parameters
        # Freeze parameters
        if self.config.fixed_parameters:
            logging.info("fixed parameters: %s" % ", ".join(map(str, self.config.fixed_parameters)))
            params = [p for p in params if p not in self.config.fixed_parameters]
        return params

def
training_params
(
self
)
:
params
=
self
.
network
.
parameters
# Freeze parameters
if
self
.
config
.
fixed_parameters
:
logging
.
info
(
"fixed parameters: %s"
%
", "
.
join
(
map
(
str
,
self
.
config
.
fixed_parameters
)
)
)
params
=
[
p
for
p
in
params
if
p
not
in
self
.
config
.
fixed_parameters
]
return
params


def optimization_updates(self, params, gradients):
        """
        Return updates from optimization.
        """
        updates, free_parameters = optimize_updates(params, gradients, self.config)
        self.network.free_parameters.extend(free_parameters)
        logging.info("Added %d free parameters for optimization" % len(free_parameters))
        return updates


def _first_glimpse_sensor(self, x_t):
        """
        Compute first glimpse position using down-sampled image.
        """
        downsampled_img = theano.tensor.signal.downsample.max_pool_2d(x_t, (4,4))
        downsampled_img = downsampled_img.flatten()
        first_l = T.dot(downsampled_img, self.W_f)
        if self.disable_reinforce:
            wf_grad = self.W_f
            if self.random_glimpse:
                first_l = self.srng.uniform((2,), low=-1.7, high=1.7)
        else:
            sampled_l_t = self._sample_gaussian(first_l, self.cov)
            sampled_pdf = self._multi_gaussian_pdf(disconnected_grad(sampled_l_t), first_l)
            wf_grad = T.grad(T.log(sampled_pdf), self.W_f)
            first_l = sampled_l_t
        return first_l, wf_grad

def _first_glimpse_sensor ( self , x_t ) : downsampled_img = theano . tensor . signal . downsample . max_pool_2d ( x_t , ( 4 , 4 ) ) downsampled_img = downsampled_img . flatten ( ) first_l = T . dot ( downsampled_img , self . W_f ) if self . disable_reinforce : wf_grad = self . W_f if self . random_glimpse : first_l = self . srng . uniform ( ( 2 , ) , low = - 1.7 , high = 1.7 ) else : sampled_l_t = self . _sample_gaussian ( first_l , self . cov ) sampled_pdf = self . _multi_gaussian_pdf ( disconnected_grad ( sampled_l_t ) , first_l ) wf_grad = T . grad ( T . log ( sampled_pdf ) , self . W_f ) first_l = sampled_l_t return first_l , wf_grad

Compute first glimpse position using down - sampled image .


def prepare(self):
        """
        All codes that create parameters should be put into 'setup' function.
        """
        self.output_dim = 10
        self.encoder = Chain(self.input_dim).stack(Dense(self.internal_layer_size, 'tanh'))
        self.decoder = Chain(self.internal_layer_size).stack(Dense(self.input_dim))
        self.classifier = Chain(self.internal_layer_size).stack(Dense(50, 'tanh'),
                                                      Dense(self.output_dim),
                                                      Softmax())

        self.register_inner_layers(self.encoder, self.decoder, self.classifier)

        self.target_input = T.ivector('target')
        self.register_external_inputs(self.target_input)

def prepare ( self ) : self . output_dim = 10 self . encoder = Chain ( self . input_dim ) . stack ( Dense ( self . internal_layer_size , 'tanh' ) ) self . decoder = Chain ( self . internal_layer_size ) . stack ( Dense ( self . input_dim ) ) self . classifier = Chain ( self . internal_layer_size ) . stack ( Dense ( 50 , 'tanh' ) , Dense ( self . output_dim ) , Softmax ( ) ) self . register_inner_layers ( self . encoder , self . decoder , self . classifier ) self . target_input = T . ivector ( 'target' ) self . register_external_inputs ( self . target_input )

All codes that create parameters should be put into setup function .


def compute_tensor(self, x):
        """
        Build the computation graph here.
        """
        internal_variable = self.encoder.compute_tensor(x)

        decoding_output = self.decoder.compute_tensor(internal_variable)

        classification_output = self.classifier.compute_tensor(internal_variable)

        auto_encoder_cost = AutoEncoderCost(decoding_output, x).get()

        classification_cost = CrossEntropyCost(classification_output, self.target_input).get()

        final_cost = 0.01 * auto_encoder_cost + classification_cost

        error_rate = ErrorRateCost(classification_output, self.target_input).get()

        self.register_monitors(("err", error_rate),
                               ("encoder_cost", auto_encoder_cost),
                               ("classify_cost", classification_cost))

        return final_cost

def compute_tensor ( self , x ) : internal_variable = self . encoder . compute_tensor ( x ) decoding_output = self . decoder . compute_tensor ( internal_variable ) classification_output = self . classifier . compute_tensor ( internal_variable ) auto_encoder_cost = AutoEncoderCost ( decoding_output , x ) . get ( ) classification_cost = CrossEntropyCost ( classification_output , self . target_input ) . get ( ) final_cost = 0.01 * auto_encoder_cost + classification_cost error_rate = ErrorRateCost ( classification_output , self . target_input ) . get ( ) self . register_monitors ( ( "err" , error_rate ) , ( "encoder_cost" , auto_encoder_cost ) , ( "classify_cost" , classification_cost ) ) return final_cost


def map(self, func):
        """
        Process all data with given function.
        The scheme of function should be x,y -> x,y.
        """
        if self._train_set:
            self._train_set = map(func, self._train_set)
        if self._valid_set:
            self._valid_set = map(func, self._valid_set)
        if self._test_set:
            self._test_set = map(func, self._test_set)

def map ( self , func ) : if self . _train_set : self . _train_set = map ( func , self . _train_set ) if self . _valid_set : self . _valid_set = map ( func , self . _valid_set ) if self . _test_set : self . _test_set = map ( func , self . _test_set )

Process all data with given function . The scheme of function should be x y - > x y .


def vectorize_target(self, size):
        """
        Make targets be one-hot vectors.
        """
        if self._train_set:
            self._train_set = self._vectorize_set(self._train_set, size)
        if self._valid_set:
            self._valid_set = self._vectorize_set(self._valid_set, size)
        if self._test_set:
            self._test_set = self._vectorize_set(self._test_set, size)

def vectorize_target ( self , size ) : if self . _train_set : self . _train_set = self . _vectorize_set ( self . _train_set , size ) if self . _valid_set : self . _valid_set = self . _vectorize_set ( self . _valid_set , size ) if self . _test_set : self . _test_set = self . _vectorize_set ( self . _test_set , size )


def report(self):
        """
        Print dataset statistics.
        """
        logging.info("%s train=%d valid=%d test=%d" % (self.__class__.__name__,
                                                       len(list(self._train_set)) if self._train_set else 0,
                                                       len(list(self._valid_set)) if self._valid_set else 0,
                                                       len(list(self._test_set)) if self._test_set else 0))

def
report
(
self
)
:
logging
.
info
(
"%s train=%d valid=%d test=%d"
%
(
self
.
__class__
.
__name__
,
len
(
list
(
self
.
_train_set
)
)
if
self
.
_train_set
else
0
,
len
(
list
(
self
.
_valid_set
)
)
if
self
.
_valid_set
else
0
,
len
(
list
(
self
.
_test_set
)
)
if
self
.
_test_set
else
0
)
)


def train(self, train_set, valid_set=None, test_set=None, train_size=None):
        '''We train over mini-batches and evaluate periodically.'''
        iteration = 0
        while True:
            if not iteration % self.config.test_frequency and test_set:
                try:
                    self.test(iteration, test_set)
                except KeyboardInterrupt:
                    logging.info('interrupted!')
                    break

            if not iteration % self.validation_frequency and valid_set:
                try:
                    if not self.evaluate(iteration, valid_set):
                        logging.info('patience elapsed, bailing out')
                        break
                except KeyboardInterrupt:
                    logging.info('interrupted!')
                    break

            train_message = ""
            try:
                train_message = self.train_func(train_set)
            except KeyboardInterrupt:
                logging.info('interrupted!')
                break
            if not iteration % self.config.monitor_frequency:
                logging.info('monitor (iter=%i) %s', iteration + 1, train_message)

            iteration += 1
            if hasattr(self.network, "iteration_callback"):
                self.network.iteration_callback()

            yield train_message

        if valid_set:
            self.set_params(self.best_params)
        if test_set:
            self.test(0, test_set)

def
train
(
self
,
train_set
,
valid_set
=
,
test_set
=
,
train_size
=
)
:
iteration
=
0
while
True
:
if
not
iteration
%
self
.
config
.
test_frequency
and
test_set
:
try
:
self
.
test
(
iteration
,
test_set
)
except
KeyboardInterrupt
:
logging
.
info
(
'interrupted!'
)
break
if
not
iteration
%
self
.
validation_frequency
and
valid_set
:
try
:
if
not
self
.
evaluate
(
iteration
,
valid_set
)
:
logging
.
info
(
'patience elapsed, bailing out'
)
break
except
KeyboardInterrupt
:
logging
.
info
(
'interrupted!'
)
break
train_message
=
""
try
:
train_message
=
self
.
train_func
(
train_set
)
except
KeyboardInterrupt
:
logging
.
info
(
'interrupted!'
)
break
if
not
iteration
%
self
.
config
.
monitor_frequency
:
logging
.
info
(
'monitor (iter=%i) %s'
,
iteration
+
1
,
train_message
)
iteration
+=
1
if
hasattr
(
self
.
network
,
"iteration_callback"
)
:
self
.
network
.
iteration_callback
(
)
yield
train_message
if
valid_set
:
self
.
set_params
(
self
.
best_params
)
if
test_set
:
self
.
test
(
0
,
test_set
)

We train over mini - batches and evaluate periodically .


def sample(self, input, steps):
        """
        Sample outputs from LM.
        """
        inputs = [[onehot(self.input_dim, x) for x in input]]
        for _ in range(steps):
            target = self.compute(inputs)[0,-1].argmax()
            input.append(target)
            inputs[0].append(onehot(self.input_dim, target))
        return input

def sample ( self , input , steps ) : inputs = [ [ onehot ( self . input_dim , x ) for x in input ] ] for _ in range ( steps ) : target = self . compute ( inputs ) [ 0 , - 1 ] . argmax ( ) input . append ( target ) inputs [ 0 ] . append ( onehot ( self . input_dim , target ) ) return input


def compute_alignments(self, prev_state, precomputed_values, mask=None):
        """
        Compute the alignment weights based on the previous state.
        """

        WaSp = T.dot(prev_state, self.Wa)
        UaH = precomputed_values
        # For test time the UaH will be (time, output_dim)
        if UaH.ndim == 2:
            preact = WaSp[:, None, :] + UaH[None, :, :]
        else:
            preact = WaSp[:, None, :] + UaH
        act = T.activate(preact, 'tanh')
        align_scores = T.dot(act, self.Va)  # ~ (batch, time)
        if mask:
            mask = (1 - mask) * -99.00
            if align_scores.ndim == 3:
                align_scores += mask[None, :]
            else:
                align_scores += mask
        align_weights = T.nnet.softmax(align_scores)
        return align_weights

Compute the alignment weights based on the previous state .


def compute_context_vector(self, prev_state, inputs, precomputed_values=None, mask=None):
        """
        Compute the context vector with soft attention.
        """
        precomputed_values = precomputed_values if precomputed_values else self.precompute(inputs)
        align_weights = self.compute_alignments(prev_state, precomputed_values, mask)
        context_vector = T.sum(align_weights[:, :, None] * inputs, axis=1)
        return context_vector

def compute_context_vector ( self , prev_state , inputs , precomputed_values = None , mask = None ) : precomputed_values = precomputed_values if precomputed_values else self . precompute ( inputs ) align_weights = self . compute_alignments ( prev_state , precomputed_values , mask ) context_vector = T . sum ( align_weights [ : , : , None ] * inputs , axis = 1 ) return context_vector

Compute the context vector with soft attention .


def concatenate(vars, axis=-1):
    """
    A utility function of concatenate.
    """
    from deepy.core.neural_var import NeuralVariable
    if isinstance(vars[0], NeuralVariable):
        concat_var = Concatenate(axis=axis).compute(*vars)
        if axis == -1 or axis == vars[0].tensor.ndim - 1:
            concat_var.output_dim = sum([x.output_dim for x in vars], 0)
    else:
        concat_var = TT.concatenate(vars, axis)
    return concat_var

def concatenate ( vars , axis = - 1 ) : from deepy . core . neural_var import NeuralVariable if isinstance ( vars [ 0 ] , NeuralVariable ) : concat_var = Concatenate ( axis = axis ) . compute ( * vars ) if axis == - 1 or axis == vars [ 0 ] . tensor . ndim - 1 : concat_var . output_dim = sum ( [ x . output_dim for x in vars ] , 0 ) else : concat_var = TT . concatenate ( vars , axis ) return concat_var


def _pad(self, side, length):
        """
        Pad sequences to given length in the left or right side.
        """
        if self._train_set:
            self._train_set = pad_dataset(self._train_set, side, length)
        if self._valid_set:
            self._valid_set = pad_dataset(self._valid_set, side, length)
        if self._test_set:
            self._test_set = pad_dataset(self._test_set, side, length)

def _pad ( self , side , length ) : if self . _train_set : self . _train_set = pad_dataset ( self . _train_set , side , length ) if self . _valid_set : self . _valid_set = pad_dataset ( self . _valid_set , side , length ) if self . _test_set : self . _test_set = pad_dataset ( self . _test_set , side , length )

Pad sequences to given length in the left or right side .


def rmsprop_core(params, gradients, momentum=0.9, learning_rate=0.01):
    """
    RMSPROP optimization core.
    """
    for param, grad in zip(params, gradients):
            rms_ = theano.shared(np.zeros_like(param.get_value()), name=param.name + '_rms')
            rms = momentum * rms_ + (1 - momentum) * grad * grad
            yield rms_, rms
            yield param, param - learning_rate * grad / T.sqrt(rms + 1e-8)

def rmsprop_core ( params , gradients , momentum = 0.9 , learning_rate = 0.01 ) : for param , grad in zip ( params , gradients ) : rms_ = theano . shared ( np . zeros_like ( param . get_value ( ) ) , name = param . name + '_rms' ) rms = momentum * rms_ + ( 1 - momentum ) * grad * grad yield rms_ , rms yield param , param - learning_rate * grad / T . sqrt ( rms + 1e-8 )


def report(self):
        """
        Report elapsed time.
        """
        if not self.end_time:
            self.end()
        print ("Time: {} mins".format((self.end_time - self.start_time )/ 60))

def
report
(
self
)
:
if
not
self
.
end_time
:
self
.
end
(
)
print
(
"Time: {} mins"
.
format
(
(
self
.
end_time
self
.
start_time
)
/
60
)
)


def run(self, data_x):
        """
        Run the model with validation data and return costs.
        """
        output_vars = self.compute(*data_x)
        return self._extract_costs(output_vars)

def run ( self , data_x ) : output_vars = self . compute ( * data_x ) return self . _extract_costs ( output_vars )

Run the model with validation data and return costs .


def invoke(self):
        """
        This function will be called after each iteration.
        """
        self._counter += 1
        if self._counter % self._freq == 0:
            cnt = 0.
            sum_map = defaultdict(float)
            for x in self._trainer.get_data(self._data_split):
                val_map = self.run(x)
                if not isinstance(val_map, dict):
                    raise Exception("Monitor.run must return a dict.")
                for k, val in val_map.items():
                    sum_map[k] += val
                cnt += 1
            for k in sum_map:
                sum_map[k] /= cnt
            new_best = self.compare(sum_map)
            self._trainer.report(sum_map, self._data_split, new_best=new_best)
            if new_best:
                self._trainer.save_checkpoint(self._save_path)

def
invoke
(
self
)
:
self
.
_counter
+=
1
if
self
.
_counter
%
self
.
_freq
==
0
:
cnt
=
0.
sum_map
=
defaultdict
(
float
)
for
x
in
self
.
_trainer
.
get_data
(
self
.
_data_split
)
:
val_map
=
self
.
run
(
x
)
if
not
isinstance
(
val_map
,
dict
)
:
raise
Exception
(
"Monitor.run must return a dict."
)
for
k
,
val
in
val_map
.
items
(
)
:
sum_map
[
k
]
+=
val
cnt
+=
1
for
k
in
sum_map
:
sum_map
[
k
]
/=
cnt
new_best
=
self
.
compare
(
sum_map
)
self
.
_trainer
.
report
(
sum_map
,
self
.
_data_split
,
new_best
=
new_best
)
if
new_best
:
self
.
_trainer
.
save_checkpoint
(
self
.
_save_path
)

This function will be called after each iteration .


def _build_loop_vars(self):
        """
        Create inner loop variables.
        """
        from theano.tensor.var import TensorVariable
        from deepy.core.neural_var import NeuralVariable
        if not self._loop_vars:
            self._ordered_out_keys = self._outputs.keys()
            seq_keys = self._sequences.keys()
            filled_out_keys = [k for k in self._ordered_out_keys if self._outputs[k]]
            nonseq_keys = self._non_sequences.keys()
            dummy_tensors, self._scan_local_vars = get_dummy_args(
                sequences=[self._sequences[k].tensor for k in seq_keys],
                outputs_info=[self._outputs[k].tensor for k in self._ordered_out_keys],
                non_sequences=[self._non_sequences[k].tensor for k in nonseq_keys],
                **self._kwargs
            )
            dummy_map = dict(zip(seq_keys + filled_out_keys + nonseq_keys, dummy_tensors))
            arg_map = self._sequences.copy()
            arg_map.update(self._outputs)
            arg_map.update(self._non_sequences)
            self._loop_vars = LoopVars()
            for k, dummy_tensor in dummy_map.items():
                dummy_var = NeuralVariable(dummy_tensor, dim=arg_map[k].dim())
                self._loop_vars[k] = dummy_var

def _build_loop_vars ( self ) : from theano . tensor . var import TensorVariable from deepy . core . neural_var import NeuralVariable if not self . _loop_vars : self . _ordered_out_keys = self . _outputs . keys ( ) seq_keys = self . _sequences . keys ( ) filled_out_keys = [ k for k in self . _ordered_out_keys if self . _outputs [ k ] ] nonseq_keys = self . _non_sequences . keys ( ) dummy_tensors , self . _scan_local_vars = get_dummy_args ( sequences = [ self . _sequences [ k ] . tensor for k in seq_keys ] , outputs_info = [ self . _outputs [ k ] . tensor for k in self . _ordered_out_keys ] , non_sequences = [ self . _non_sequences [ k ] . tensor for k in nonseq_keys ] , * * self . _kwargs ) dummy_map = dict ( zip ( seq_keys + filled_out_keys + nonseq_keys , dummy_tensors ) ) arg_map = self . _sequences . copy ( ) arg_map . update ( self . _outputs ) arg_map . update ( self . _non_sequences ) self . _loop_vars = LoopVars ( ) for k , dummy_tensor in dummy_map . items ( ) : dummy_var = NeuralVariable ( dummy_tensor , dim = arg_map [ k ] . dim ( ) ) self . _loop_vars [ k ] = dummy_var


def _scan_step(self, vars):
        """
        Internal scan with dummy input variables.
        """
        from neural_var import NeuralVariable
        if not self._loop_vars:
            raise Exception("The loop is not initialized. To initialize the loop, use `with loop as vars`")
        replace_map = {}
        for k, var in vars.items():
            if var is not None:
                replace_map[self._dummy_nodes[k].tensor] = var.tensor
        outputs = {}
        for k in self._outputs:
            if k not in self._loop_vars:
                raise Exception("{} can not be found in loop vars.".format(k))
            output_node = theano.clone(self._loop_vars[k].tensor, replace_map)
            outputs[k] = NeuralVariable(output_node, self._loop_vars[k].dim())
        return outputs

Internal scan with dummy input variables .


def momentum_core(params, gradients, momentum=0.9, learning_rate=0.01):
    """
    Momentum SGD optimization core.
    """
    free_parameters = []
    updates = []
    for param, grad in zip(params, gradients):
            delta = learning_rate * grad
            velocity = theano.shared(np.zeros_like(param.get_value()), name=param.name + '_vel')
            updates.append((velocity, momentum * velocity - delta))
            updates.append((param, param + velocity))
            free_parameters.append(velocity)
    return updates, free_parameters

def momentum_core ( params , gradients , momentum = 0.9 , learning_rate = 0.01 ) : free_parameters = [ ] updates = [ ] for param , grad in zip ( params , gradients ) : delta = learning_rate * grad velocity = theano . shared ( np . zeros_like ( param . get_value ( ) ) , name = param . name + '_vel' ) updates . append ( ( velocity , momentum * velocity - delta ) ) updates . append ( ( param , param + velocity ) ) free_parameters . append ( velocity ) return updates , free_parameters


def iftrain(self, then_branch, else_branch):
        """
        Execute `then_branch` when training.
        """
        return ifelse(self._training_flag, then_branch, else_branch, name="iftrain")

def iftrain ( self , then_branch , else_branch ) : return ifelse ( self . _training_flag , then_branch , else_branch , name = "iftrain" )


def skip(self, n_batches, n_epochs=0):
        """
        Skip N batches in the training.
        """
        logging.info("skip %d epochs and %d batches" % (n_epochs, n_batches))
        self._skip_batches = n_batches
        self._skip_epochs = n_epochs

def
skip
(
self
,
n_batches
,
n_epochs
=
0
)
:
logging
.
info
(
"skip %d epochs and %d batches"
%
(
n_epochs
,
n_batches
)
)
self
.
_skip_batches
=
n_batches
self
.
_skip_epochs
=
n_epochs


def load_params(self, path, exclude_free_params=False):
        """
        Load parameters for the training.
        This method can load free parameters and resume the training progress.
        """
        self.network.load_params(path, exclude_free_params=exclude_free_params)
        self.best_params = self.copy_params()
        # Resume the progress
        if self.network.train_logger.progress() > 0 or self.network.train_logger.epoch() > 0:
            self.skip(self.network.train_logger.progress(), self.network.train_logger.epoch() - 1)

def
load_params
(
self
,
path
,
exclude_free_params
=
False
)
:
self
.
network
.
load_params
(
path
,
exclude_free_params
=
exclude_free_params
)
self
.
best_params
=
self
.
copy_params
(
)
# Resume the progress
if
self
.
network
.
train_logger
.
progress
(
)
>
0
or
self
.
network
.
train_logger
.
epoch
(
)
>
0
:
self
.
skip
(
self
.
network
.
train_logger
.
progress
(
)
,
self
.
network
.
train_logger
.
epoch
(
)
1
)

Load parameters for the training . This method can load free parameters and resume the training progress .


def train(self, train_set, valid_set=None, test_set=None, train_size=None):
        """
        Train the model and return costs.
        """
        self._epoch = 0
        while True:
            if self._skip_epochs > 0:
                logging.info("skipping one epoch ...")
                self._skip_epochs -= 1
                self._epoch += 1
                yield None
                continue
            # Test
            if not self._epoch % self.config.test_frequency and test_set:
                try:
                    self._run_test(self._epoch, test_set)
                except KeyboardInterrupt:
                    logging.info('interrupted!')
                    break
            # Validate
            if not self._epoch % self.validation_frequency and valid_set:
                try:

                    if not self._run_valid(self._epoch, valid_set):
                        logging.info('patience elapsed, bailing out')
                        break
                except KeyboardInterrupt:
                    logging.info('interrupted!')
                    break
            # Train one step

            try:
                costs = self._run_train(self._epoch, train_set, train_size)
            except KeyboardInterrupt:
                logging.info('interrupted!')
                break
            # Check costs
            if np.isnan(costs[0][1]):
                logging.info("NaN detected in costs, rollback to last parameters")
                self.set_params(*self.checkpoint)
            else:
                self._epoch += 1
                self.network.epoch_callback()

            yield dict(costs)

        if valid_set and self.config.get("save_best_parameters", True):
            self.set_params(*self.best_params)
        if test_set:
            self._run_test(-1, test_set)


def _run_train(self, epoch, train_set, train_size=None):
        """
        Run one training iteration.
        """
        self.network.train_logger.record_epoch(epoch + 1)
        costs = self.train_step(train_set, train_size)
        if not epoch % self.config.monitor_frequency:
            self.report(dict(costs), "train", epoch)
        self.last_run_costs = costs
        return costs

def _run_train ( self , epoch , train_set , train_size = None ) : self . network . train_logger . record_epoch ( epoch + 1 ) costs = self . train_step ( train_set , train_size ) if not epoch % self . config . monitor_frequency : self . report ( dict ( costs ) , "train" , epoch ) self . last_run_costs = costs return costs


def _run_valid(self, epoch, valid_set, dry_run=False, save_path=None):
        """
        Run one valid iteration, return true if to continue training.
        """
        costs = self.valid_step(valid_set)
        # this is the same as: (J_i - J_f) / J_i > min improvement
        _, J = costs[0]
        new_best = False
        if self.best_cost - J > self.best_cost * self.min_improvement:
            # save the best cost and parameters
            self.best_params = self.copy_params()
            new_best = True
            if not dry_run:
                self.best_cost = J
                self.best_epoch = epoch
            self.save_checkpoint(save_path)

        self.report(dict(costs), type="valid", epoch=0 if dry_run else epoch, new_best=new_best)
        self.last_run_costs = costs
        return epoch - self.best_epoch < self.patience

Run one valid iteration return true if to continue training .


def report(self, score_map, type="valid", epoch=-1, new_best=False):
        """
        Report the scores and record them in the log.
        """
        type_str = type
        if len(type_str) < 5:
            type_str += " " * (5 - len(type_str))
        info = " ".join("%s=%.2f" % el for el in score_map.items())
        current_epoch = epoch if epoch > 0 else self.current_epoch()
        epoch_str = "epoch={}".format(current_epoch + 1)
        if epoch < 0:
            epoch_str = "dryrun"
            sys.stdout.write("\r")
            sys.stdout.flush()
        marker = " *" if new_best else ""
        message = "{} ({}) {}{}".format(type_str, epoch_str, info, marker)
        self.network.train_logger.record(message)
        logging.info(message)

def
report
(
self
,
score_map
,
type
=
"valid"
,
epoch
=
1
,
new_best
=
False
)
:
type_str
=
type
if
len
(
type_str
)
<
5
:
type_str
+=
" "
*
(
5
len
(
type_str
)
)
info
=
" "
.
join
(
"%s=%.2f"
%
el
for
el
in
score_map
.
items
(
)
)
current_epoch
=
epoch
if
epoch
>
0
else
self
.
current_epoch
(
)
epoch_str
=
"epoch={}"
.
format
(
current_epoch
+
1
)
if
epoch
<
0
:
epoch_str
=
"dryrun"
sys
.
stdout
.
write
(
"\r"
)
sys
.
stdout
.
flush
(
)
marker
=
" *"
if
new_best
else
""
message
=
"{} ({}) {}{}"
.
format
(
type_str
,
epoch_str
,
info
,
marker
)
self
.
network
.
train_logger
.
record
(
message
)
logging
.
info
(
message
)

Report the scores and record them in the log .


def get_data(self, data_split="train"):
        """
        Get specified split of data.
        """
        if data_split == 'train':
            return self._current_train_set
        elif data_split == 'valid':
            return self._current_valid_set
        elif data_split == 'test':
            return self._current_test_set
        else:
            return None

def get_data ( self , data_split = "train" ) : if data_split == 'train' : return self . _current_train_set elif data_split == 'valid' : return self . _current_valid_set elif data_split == 'test' : return self . _current_test_set else : return None


def apply(self, func, dim=None):
        """
        Apply a function to tensors.
        """
        output_dim = dim if dim else self.output_dim
        return NeuralVariable(func(self.tensor), output_dim)

def apply ( self , func , dim = None ) : output_dim = dim if dim else self . output_dim return NeuralVariable ( func ( self . tensor ) , output_dim )


def report(self):
        """
        Report usage of training parameters.
        """
        if self.logger:
            self.logger.info("accessed parameters:")
            for key in self.used_parameters:
                self.logger.info(" - %s %s" % (key, "(undefined)" if key in self.undefined_parameters else ""))

def
report
(
self
)
:
if
self
.
logger
:
self
.
logger
.
info
(
"accessed parameters:"
)
for
key
in
self
.
used_parameters
:
self
.
logger
.
info
(
" - %s %s"
%
(
key
,
"(undefined)"
if
key
in
self
.
undefined_parameters
else
""
)
)


def var(self, tensor_type, last_dim=0, test_shape=None):
        """
        An alias of deepy.tensor.var.
        """
        from deepy.tensor import var
        return var(tensor_type, last_dim=last_dim, test_shape=test_shape)

def var ( self , tensor_type , last_dim = 0 , test_shape = None ) : from deepy . tensor import var return var ( tensor_type , last_dim = last_dim , test_shape = test_shape )


def create_vars_from_data(self, dataset, split="train"):
        """
        Create vars given a dataset and set test values.
        Useful when dataset is already defined.
        """
        from deepy.core.neural_var import NeuralVariable
        vars = []
        if split == "valid":
            data_split = dataset.valid_set()
        elif split == "test":
            data_split = dataset.test_set()
        else:
            data_split = dataset.train_set()
        first_data_piece = list(data_split)[0]
        for i, numpy_tensor in enumerate(first_data_piece):
            if numpy_tensor.dtype == "int64":
                numpy_tensor = numpy_tensor.astype("int32")
            if numpy_tensor.dtype == "float64":
                numpy_tensor = numpy_tensor.astype(env.FLOATX)
            type_map = {
                0: "scalar",
                1: "vector",
                2: "matrix",
                3: "tensor3",
                4: "tensor4",
                5: "tensor5",
            }
            tensor_type = type_map[numpy_tensor.ndim] if numpy_tensor.ndim in type_map else type_map[0]
            if numpy_tensor.dtype.kind == "i":
                tensor_type = "i" + tensor_type
            theano_tensor = getattr(TT, tensor_type)("input_{}_{}".format(i + 1, tensor_type))
            last_dim = numpy_tensor.shape[-1]
            var = NeuralVariable(theano_tensor, dim=last_dim)
            var.set_test_value(numpy_tensor)
            vars.append(var)
        return vars

def create_vars_from_data ( self , dataset , split = "train" ) : from deepy . core . neural_var import NeuralVariable vars = [ ] if split == "valid" : data_split = dataset . valid_set ( ) elif split == "test" : data_split = dataset . test_set ( ) else : data_split = dataset . train_set ( ) first_data_piece = list ( data_split ) [ 0 ] for i , numpy_tensor in enumerate ( first_data_piece ) : if numpy_tensor . dtype == "int64" : numpy_tensor = numpy_tensor . astype ( "int32" ) if numpy_tensor . dtype == "float64" : numpy_tensor = numpy_tensor . astype ( env . FLOATX ) type_map = { 0 : "scalar" , 1 : "vector" , 2 : "matrix" , 3 : "tensor3" , 4 : "tensor4" , 5 : "tensor5" , } tensor_type = type_map [ numpy_tensor . ndim ] if numpy_tensor . ndim in type_map else type_map [ 0 ] if numpy_tensor . dtype . kind == "i" : tensor_type = "i" + tensor_type theano_tensor = getattr ( TT , tensor_type ) ( "input_{}_{}" . format ( i + 1 , tensor_type ) ) last_dim = numpy_tensor . shape [ - 1 ] var = NeuralVariable ( theano_tensor , dim = last_dim ) var . set_test_value ( numpy_tensor ) vars . append ( var ) return vars

Create vars given a dataset and set test values . Useful when dataset is already defined .


def shared(self, value, name=None):
        """
        Create a shared theano scalar value.
        """
        if type(value) == int:
            final_value = np.array(value, dtype="int32")
        elif type(value) == float:
            final_value = np.array(value, dtype=env.FLOATX)
        else:
            final_value = value

        return theano.shared(final_value, name=name)

def shared ( self , value , name = None ) : if type ( value ) == int : final_value = np . array ( value , dtype = "int32" ) elif type ( value ) == float : final_value = np . array ( value , dtype = env . FLOATX ) else : final_value = value return theano . shared ( final_value , name = name )


def stack_encoders(self, *layers):
        """
        Stack encoding layers, this must be done before stacking decoding layers.
        """
        self.stack(*layers)
        self.encoding_layes.extend(layers)

def stack_encoders ( self , * layers ) : self . stack ( * layers ) self . encoding_layes . extend ( layers )

Stack encoding layers this must be done before stacking decoding layers .


def stack_decoders(self, *layers):
        """
        Stack decoding layers.
        """
        self.stack(*layers)
        self.decoding_layers.extend(layers)

def stack_decoders ( self , * layers ) : self . stack ( * layers ) self . decoding_layers . extend ( layers )


def encode(self, x):
        """
        Encode given input.
        """
        if not self.encoding_network:
            self.encoding_network = NeuralNetwork(self.input_dim, self.input_tensor)
            self.encoding_network.input_variables = self.input_variables
            for layer in self.encoding_layes:
                self.encoding_network.stack_layer(layer, no_setup=True)
        return self.encoding_network.compute(*x)

def encode ( self , x ) : if not self . encoding_network : self . encoding_network = NeuralNetwork ( self . input_dim , self . input_tensor ) self . encoding_network . input_variables = self . input_variables for layer in self . encoding_layes : self . encoding_network . stack_layer ( layer , no_setup = True ) return self . encoding_network . compute ( * x )


def decode(self, x):
        """
        Decode given representation.
        """
        if not self.rep_dim:
            raise Exception("rep_dim must be set to decode.")
        if not self.decoding_network:
            self.decoding_network = NeuralNetwork(self.rep_dim)
            for layer in self.decoding_layers:
                self.decoding_network.stack_layer(layer, no_setup=True)
        return self.decoding_network.compute(x)

def
decode
(
self
,
x
)
:
if
not
self
.
rep_dim
:
raise
Exception
(
"rep_dim must be set to decode."
)
if
not
self
.
decoding_network
:
self
.
decoding_network
=
NeuralNetwork
(
self
.
rep_dim
)
for
layer
in
self
.
decoding_layers
:
self
.
decoding_network
.
stack_layer
(
layer
,
no_setup
=
True
)
return
self
.
decoding_network
.
compute
(
x
)


def create_2d_gaussian(dim, sigma):
    """
    This function creates a 2d gaussian kernel with the standard deviation
    denoted by sigma

    :param dim: integer denoting a side (1-d) of gaussian kernel
    :param sigma: floating point indicating the standard deviation

    :returns: a numpy 2d array
    """

    # check if the dimension is odd
    if dim % 2 == 0:
        raise ValueError("Kernel dimension should be odd")

    # initialize the kernel
    kernel = np.zeros((dim, dim), dtype=np.float16)

    # calculate the center point
    center = dim/2

    # calculate the variance
    variance = sigma ** 2

    # calculate the normalization coefficeint
    coeff = 1. / (2 * variance)

    # create the kernel
    for x in range(0, dim):
        for y in range(0, dim):
            x_val = abs(x - center)
            y_val = abs(y - center)
            numerator = x_val**2 + y_val**2
            denom = 2*variance

            kernel[x,y] = coeff * np.exp(-1. * numerator/denom)

    return kernel/sum(sum(kernel))


def register_layer(self, layer):
        """
        Register the layer so that it's param will be trained.
        But the output of the layer will not be stacked.
        """
        if type(layer) == Block:
            layer.fix()
        self.parameter_count += layer.parameter_count
        self.parameters.extend(layer.parameters)
        self.free_parameters.extend(layer.free_parameters)
        self.training_monitors.extend(layer.training_monitors)
        self.testing_monitors.extend(layer.testing_monitors)
        self.updates.extend(layer.updates)
        self.training_updates.extend(layer.training_updates)
        self.input_variables.extend(layer.external_inputs)
        self.target_variables.extend(layer.external_targets)

        self.training_callbacks.extend(layer.training_callbacks)
        self.testing_callbacks.extend(layer.testing_callbacks)
        self.epoch_callbacks.extend(layer.epoch_callbacks)

def register_layer ( self , layer ) : if type ( layer ) == Block : layer . fix ( ) self . parameter_count += layer . parameter_count self . parameters . extend ( layer . parameters ) self . free_parameters . extend ( layer . free_parameters ) self . training_monitors . extend ( layer . training_monitors ) self . testing_monitors . extend ( layer . testing_monitors ) self . updates . extend ( layer . updates ) self . training_updates . extend ( layer . training_updates ) self . input_variables . extend ( layer . external_inputs ) self . target_variables . extend ( layer . external_targets ) self . training_callbacks . extend ( layer . training_callbacks ) self . testing_callbacks . extend ( layer . testing_callbacks ) self . epoch_callbacks . extend ( layer . epoch_callbacks )

Register the layer so that it s param will be trained . But the output of the layer will not be stacked .


def monitor_layer_outputs(self):
        """
        Monitoring the outputs of each layer.
        Useful for troubleshooting convergence problems.
        """
        for layer, hidden in zip(self.layers, self._hidden_outputs):
            self.training_monitors.append(('mean(%s)' % (layer.name), abs(hidden).mean()))

def monitor_layer_outputs ( self ) : for layer , hidden in zip ( self . layers , self . _hidden_outputs ) : self . training_monitors . append ( ( 'mean(%s)' % ( layer . name ) , abs ( hidden ) . mean ( ) ) )

Monitoring the outputs of each layer . Useful for troubleshooting convergence problems .


def all_parameters(self):
        """
        Return all parameters.
        """
        params = []
        params.extend(self.parameters)
        params.extend(self.free_parameters)

        return params

def all_parameters ( self ) : params = [ ] params . extend ( self . parameters ) params . extend ( self . free_parameters ) return params


def setup_variables(self):
        """
        Set up variables.
        """
        if self.input_tensor:
            if type(self.input_tensor) == int:
                x = dim_to_var(self.input_tensor, name="x")
            else:
                x = self.input_tensor
        else:
            x = T.matrix('x')
        self.input_variables.append(x)
        self._output = x
        self._test_output = x

def setup_variables ( self ) : if self . input_tensor : if type ( self . input_tensor ) == int : x = dim_to_var ( self . input_tensor , name = "x" ) else : x = self . input_tensor else : x = T . matrix ( 'x' ) self . input_variables . append ( x ) self . _output = x self . _test_output = x


def compute(self, *x):
        """
        Return network output.
        """
        self._compile()
        outs = self._compute(*x)
        if self._output_keys:
            return MapDict(dict(zip(self._output_keys, outs)))
        else:
            return outs

def compute ( self , * x ) : self . _compile ( ) outs = self . _compute ( * x ) if self . _output_keys : return MapDict ( dict ( zip ( self . _output_keys , outs ) ) ) else : return outs


def save_params(self, path, new_thread=False):
        """
        Save parameters to file.
        """
        save_logger.info(path)
        param_variables = self.all_parameters
        params = [p.get_value().copy() for p in param_variables]
        if new_thread:
            thread = Thread(target=save_network_params, args=(params, path))
            thread.start()
        else:
            save_network_params(params, path)
        self.train_logger.save(path)

def save_params ( self , path , new_thread = False ) : save_logger . info ( path ) param_variables = self . all_parameters params = [ p . get_value ( ) . copy ( ) for p in param_variables ] if new_thread : thread = Thread ( target = save_network_params , args = ( params , path ) ) thread . start ( ) else : save_network_params ( params , path ) self . train_logger . save ( path )


def load_params(self, path, exclude_free_params=False):
        """
        Load parameters from file.
        """
        if not os.path.exists(path): return;
        logging.info("loading parameters from %s" % path)
        # Decide which parameters to load
        if exclude_free_params:
            params_to_load = self.parameters
        else:
            params_to_load = self.all_parameters
        # Load parameters
        if path.endswith(".gz"):
            opener = gzip.open if path.lower().endswith('.gz') else open
            handle = opener(path, 'rb')
            saved_params = pickle.load(handle)
            handle.close()
            # Write parameters
            for target, source in zip(params_to_load, saved_params):
                logging.info('%s: setting value %s', target.name, source.shape)
                target.set_value(source)
        elif path.endswith(".npz"):
            arrs = np.load(path)
            # Write parameters
            for target, idx in zip(params_to_load, range(len(arrs.keys()))):
                source = arrs['arr_%d' % idx]
                logging.info('%s: setting value %s', target.name, source.shape)
                target.set_value(source)
        else:
            raise Exception("File format of %s is not supported, use '.gz' or '.npz' or '.uncompressed.gz'" % path)

        self.train_logger.load(path)


def report(self):
        """
        Print network statistics.
        """
        logging.info("network inputs: %s", " ".join(map(str, self.input_variables)))
        logging.info("network targets: %s", " ".join(map(str, self.target_variables)))
        logging.info("network parameters: %s", " ".join(map(str, self.all_parameters)))
        logging.info("parameter count: %d", self.parameter_count)

def
report
(
self
)
:
logging
.
info
(
"network inputs: %s"
,
" "
.
join
(
map
(
str
,
self
.
input_variables
)
)
)
logging
.
info
(
"network targets: %s"
,
" "
.
join
(
map
(
str
,
self
.
target_variables
)
)
)
logging
.
info
(
"network parameters: %s"
,
" "
.
join
(
map
(
str
,
self
.
all_parameters
)
)
)
logging
.
info
(
"parameter count: %d"
,
self
.
parameter_count
)


def register_parameters(self, *parameters):
        """
        Register parameters.
        """
        for param in parameters:
            self.parameter_count += np.prod(param.get_value().shape)
        self.parameters.extend(parameters)

def register_parameters ( self , * parameters ) : for param in parameters : self . parameter_count += np . prod ( param . get_value ( ) . shape ) self . parameters . extend ( parameters )


def register_updates(self, *updates):
        """
        Register updates that will be executed in each iteration.
        """
        for key, node in updates:
            if key not in self._registered_updates:
                self.updates.append((key, node))
                self._registered_updates.add(key)

def register_updates ( self , * updates ) : for key , node in updates : if key not in self . _registered_updates : self . updates . append ( ( key , node ) ) self . _registered_updates . add ( key )

Register updates that will be executed in each iteration .


def register_training_updates(self, *updates):
        """
        Register updates that will only be executed in training phase.
        """
        for key, node in updates:
            if key not in self._registered_training_updates:
                self.training_updates.append((key, node))
                self._registered_training_updates.add(key)

def register_training_updates ( self , * updates ) : for key , node in updates : if key not in self . _registered_training_updates : self . training_updates . append ( ( key , node ) ) self . _registered_training_updates . add ( key )

Register updates that will only be executed in training phase .


def register_monitors(self, *monitors):
        """
        Register monitors they should be tuple of name and Theano variable.
        """
        for key, node in monitors:
            if key not in self._registered_monitors:
                node *= 1.0 # Avoid CudaNdarray
                self.training_monitors.append((key, node))
                self.testing_monitors.append((key, node))
                self._registered_monitors.add(key)

def
register_monitors
(
self
,
*
monitors
)
:
for
key
,
node
in
monitors
:
if
key
not
in
self
.
_registered_monitors
:
node
*=
1.0
# Avoid CudaNdarray
self
.
training_monitors
.
append
(
(
key
,
node
)
)
self
.
testing_monitors
.
append
(
(
key
,
node
)
)
self
.
_registered_monitors
.
add
(
key
)

Register monitors they should be tuple of name and Theano variable .


def multiple_l2_norm(tensors):
    """
    Get the L2 norm of multiple tensors.
    This function is taken from blocks.
    """
    # Another way for doing this, I don't know which one is fast
    # return T.sqrt(sum(T.sum(t ** 2) for t in tensors))
    flattened = [T.as_tensor_variable(t).flatten() for t in tensors]
    flattened = [(t if t.ndim > 0 else t.dimshuffle('x'))
                 for t in flattened]
    joined = T.join(0, *flattened)
    return T.sqrt(T.sqr(joined).sum())

Get the L2 norm of multiple tensors . This function is taken from blocks .


def dump_one(elt_to_pickle, file_obj):
        """
        dumps one element to file_obj, a file opened in write mode
        """
        pickled_elt_str = dumps(elt_to_pickle)
        file_obj.write(pickled_elt_str)
        # record separator is a blank line
        # (since pickled_elt_str might contain its own newlines)
        file_obj.write('\n\n')

dumps one element to file_obj a file opened in write mode


def load(file_obj):
        """
        load contents from file_obj, returning a generator that yields one
        element at a time
        """
        cur_elt = []
        for line in file_obj:
          cur_elt.append(line)

          if line == '\n':
            pickled_elt_str = ''.join(cur_elt)
            cur_elt = []
            try:
                elt = loads(pickled_elt_str)
            except ValueError:
                continue

            yield elt

def load ( file_obj ) : cur_elt = [ ] for line in file_obj : cur_elt . append ( line ) if line == '\n' : pickled_elt_str = '' . join ( cur_elt ) cur_elt = [ ] try : elt = loads ( pickled_elt_str ) except ValueError : continue yield elt

load contents from file_obj returning a generator that yields one element at a time


def load_params(self, path, exclude_free_params=False):
        from deepy.core import graph
        """
        Load parameters to the block.
        """
        from deepy.core.comp_graph import ComputationalGraph
        model = graph.compile(blocks=[self])
        model.load_params(path, exclude_free_params=exclude_free_params)

def load_params ( self , path , exclude_free_params = False ) : from deepy . core import graph from deepy . core . comp_graph import ComputationalGraph model = graph . compile ( blocks = [ self ] ) model . load_params ( path , exclude_free_params = exclude_free_params )


def create_request_elements(
            cls, request_type, credentials, url, method='GET', params=None,
            headers=None, body='', secret=None, redirect_uri='', scope='',
            csrf='', user_state=''
    ):
        """
        Creates |oauth2| request elements.
        """

        headers = headers or {}
        params = params or {}

        consumer_key = credentials.consumer_key or ''
        consumer_secret = credentials.consumer_secret or ''
        token = credentials.token or ''
        refresh_token = credentials.refresh_token or credentials.token or ''

        # Separate url base and query parameters.
        url, base_params = cls._split_url(url)

        # Add params extracted from URL.
        params.update(dict(base_params))

        if request_type == cls.USER_AUTHORIZATION_REQUEST_TYPE:
            # User authorization request.
            # TODO: Raise error for specific message for each missing argument.
            if consumer_key and redirect_uri and (
                    csrf or not cls.supports_csrf_protection):
                params['client_id'] = consumer_key
                params['redirect_uri'] = redirect_uri
                params['scope'] = scope
                if cls.supports_user_state:
                    params['state'] = base64.urlsafe_b64encode(
                        json.dumps(
                            {"csrf": csrf, "user_state": user_state}
                        ).encode('utf-8')
                    )
                else:
                    params['state'] = csrf
                params['response_type'] = 'code'

                # Add authorization header
                headers.update(cls._authorization_header(credentials))
            else:
                raise OAuth2Error(
                    'Credentials with valid consumer_key and arguments '
                    'redirect_uri, scope and state are required to create '
                    'OAuth 2.0 user authorization request elements!')

        elif request_type == cls.ACCESS_TOKEN_REQUEST_TYPE:
            # Access token request.
            if consumer_key and consumer_secret:
                params['code'] = token
                params['client_id'] = consumer_key
                params['client_secret'] = consumer_secret
                params['redirect_uri'] = redirect_uri
                params['grant_type'] = 'authorization_code'

                # TODO: Check whether all providers accept it
                headers.update(cls._authorization_header(credentials))
            else:
                raise OAuth2Error(
                    'Credentials with valid token, consumer_key, '
                    'consumer_secret and argument redirect_uri are required '
                    'to create OAuth 2.0 access token request elements!')

        elif request_type == cls.REFRESH_TOKEN_REQUEST_TYPE:
            # Refresh access token request.
            if refresh_token and consumer_key and consumer_secret:
                params['refresh_token'] = refresh_token
                params['client_id'] = consumer_key
                params['client_secret'] = consumer_secret
                params['grant_type'] = 'refresh_token'
            else:
                raise OAuth2Error(
                    'Credentials with valid refresh_token, consumer_key, '
                    'consumer_secret are required to create OAuth 2.0 '
                    'refresh token request elements!')

        elif request_type == cls.PROTECTED_RESOURCE_REQUEST_TYPE:
            # Protected resource request.

            # Add Authorization header. See:
            # http://tools.ietf.org/html/rfc6749#section-7.1
            if credentials.token_type == cls.BEARER:
                # http://tools.ietf.org/html/rfc6750#section-2.1
                headers.update(
                    {'Authorization': 'Bearer {0}'.format(credentials.token)})

            elif token:
                params['access_token'] = token
            else:
                raise OAuth2Error(
                    'Credentials with valid token are required to create '
                    'OAuth 2.0 protected resources request elements!')

        request_elements = core.RequestElements(
            url, method, params, headers, body)

        return cls._x_request_elements_filter(
            request_type, request_elements, credentials)


def decode_state(cls, state, param='user_state'):
        """
        Decode state and return param.

        :param str state:
            state parameter passed through by provider

        :param str param:
            key to query from decoded state variable. Options include 'csrf'
            and 'user_state'.

        :returns:
            string value from decoded state

        """
        if state and cls.supports_user_state:
            # urlsafe_b64 may include = which the browser quotes so must
            # unquote Cast to str to void b64decode translation error. Base64
            # should be str compatible.
            return json.loads(base64.urlsafe_b64decode(
                unquote(str(state))).decode('utf-8'))[param]
        else:
            return state if param == 'csrf' else ''


def _x_credentials_parser(credentials, data):
        """
        We need to override this method to fix Facebooks naming deviation.
        """

        # Facebook returns "expires" instead of "expires_in".
        credentials.expire_in = data.get('expires')

        if data.get('token_type') == 'bearer':
            # TODO: cls is not available here, hardcode for now.
            credentials.token_type = 'Bearer'

        return credentials

We need to override this method to fix Facebooks naming deviation .


def _x_request_elements_filter(cls, request_type, request_elements,
                                   credentials):
        """
        Google doesn't accept client ID and secret to be at the same time in
        request parameters and in the basic authorization header in the access
        token request.
        """
        if request_type is cls.ACCESS_TOKEN_REQUEST_TYPE:
            params = request_elements[2]
            del params['client_id']
            del params['client_secret']
        return request_elements

def _x_request_elements_filter ( cls , request_type , request_elements , credentials ) : if request_type is cls . ACCESS_TOKEN_REQUEST_TYPE : params = request_elements [ 2 ] del params [ 'client_id' ] del params [ 'client_secret' ] return request_elements

Google doesn t accept client ID and secret to be at the same time in request parameters and in the basic authorization header in the access token request .


def login(provider_name):
    """
    Login handler, must accept both GET and POST to be able to use OpenID.
    """

    # We need response object for the WerkzeugAdapter.
    response = make_response()

    # Log the user in, pass it the adapter and the provider name.
    result = authomatic.login(
        WerkzeugAdapter(
            request,
            response),
        provider_name)

    # If there is no LoginResult object, the login procedure is still pending.
    if result:
        if result.user:
            # We need to update the user to get more info.
            result.user.update()

        # The rest happens inside the template.
        return render_template('login.html', result=result)

    # Don't forget to return the response.
    return response

Login handler must accept both GET and POST to be able to use OpenID .


def normalize_dict(dict_):
    """
    Replaces all values that are single-item iterables with the value of its
    index 0.

    :param dict dict_:
        Dictionary to normalize.

    :returns:
        Normalized dictionary.

    """

    return dict([(k, v[0] if not isinstance(v, str) and len(v) == 1 else v)
                 for k, v in list(dict_.items())])

def normalize_dict ( dict_ ) : return dict ( [ ( k , v [ 0 ] if not isinstance ( v , str ) and len ( v ) == 1 else v ) for k , v in list ( dict_ . items ( ) ) ] )

Replaces all values that are single - item iterables with the value of its index 0 .


def items_to_dict(items):
    """
    Converts list of tuples to dictionary with duplicate keys converted to
    lists.

    :param list items:
        List of tuples.

    :returns:
        :class:`dict`

    """

    res = collections.defaultdict(list)

    for k, v in items:
        res[k].append(v)

    return normalize_dict(dict(res))

def items_to_dict ( items ) : res = collections . defaultdict ( list ) for k , v in items : res [ k ] . append ( v ) return normalize_dict ( dict ( res ) )

Converts list of tuples to dictionary with duplicate keys converted to lists .


def json_qs_parser(body):
    """
    Parses response body from JSON, XML or query string.

    :param body:
        string

    :returns:
        :class:`dict`, :class:`list` if input is JSON or query string,
        :class:`xml.etree.ElementTree.Element` if XML.

    """
    try:
        # Try JSON first.
        return json.loads(body)
    except (OverflowError, TypeError, ValueError):
        pass

    try:
        # Then XML.
        return ElementTree.fromstring(body)
    except (ElementTree.ParseError, TypeError, ValueError):
        pass

    # Finally query string.
    return dict(parse.parse_qsl(body))

def
json_qs_parser
(
body
)
:
try
:
# Try JSON first.
return
json
.
loads
(
body
)
except
(
OverflowError
,
TypeError
,
ValueError
)
:
pass
try
:
# Then XML.
return
ElementTree
.
fromstring
(
body
)
except
(
ElementTree
.
ParseError
,
TypeError
,
ValueError
)
:
pass
# Finally query string.
return
dict
(
parse
.
parse_qsl
(
body
)
)

Parses response body from JSON XML or query string .


def resolve_provider_class(class_):
    """
    Returns a provider class.

    :param class_name: :class:`string` or
    :class:`authomatic.providers.BaseProvider` subclass.

    """

    if isinstance(class_, str):
        # prepare path for authomatic.providers package
        path = '.'.join([__package__, 'providers', class_])

        # try to import class by string from providers module or by fully
        # qualified path
        return import_string(class_, True) or import_string(path)
    else:
        return class_


def create_cookie(self, delete=None):
        """
        Creates the value for ``Set-Cookie`` HTTP header.

        :param bool delete:
            If ``True`` the cookie value will be ``deleted`` and the
            Expires value will be ``Thu, 01-Jan-1970 00:00:01 GMT``.

        """
        value = 'deleted' if delete else self._serialize(self.data)
        split_url = parse.urlsplit(self.adapter.url)
        domain = split_url.netloc.split(':')[0]

        # Work-around for issue #11, failure of WebKit-based browsers to accept
        # cookies set as part of a redirect response in some circumstances.
        if '.' not in domain:
            template = '{name}={value}; Path={path}; HttpOnly{secure}{expires}'
        else:
            template = ('{name}={value}; Domain={domain}; Path={path}; '
                        'HttpOnly{secure}{expires}')

        return template.format(
            name=self.name,
            value=value,
            domain=domain,
            path=split_url.path,
            secure='; Secure' if self.secure else '',
            expires='; Expires=Thu, 01-Jan-1970 00:00:01 GMT' if delete else ''
        )

Creates the value for Set - Cookie HTTP header .


def save(self):
        """
        Adds the session cookie to headers.
        """
        if self.data:
            cookie = self.create_cookie()
            cookie_len = len(cookie)

            if cookie_len > 4093:
                raise SessionError('Cookie too long! The cookie size {0} '
                                   'is more than 4093 bytes.'
                                   .format(cookie_len))

            self.adapter.set_header('Set-Cookie', cookie)

            # Reset data
            self._data = {}

def
save
(
self
)
:
if
self
.
data
:
cookie
=
self
.
create_cookie
(
)
cookie_len
=
len
(
cookie
)
if
cookie_len
>
4093
:
raise
SessionError
(
'Cookie too long! The cookie size {0} '
'is more than 4093 bytes.'
.
format
(
cookie_len
)
)
self
.
adapter
.
set_header
(
'Set-Cookie'
,
cookie
)
# Reset data
self
.
_data
=
{
}


def _get_data(self):
        """
        Extracts the session data from cookie.
        """
        cookie = self.adapter.cookies.get(self.name)
        return self._deserialize(cookie) if cookie else {}

def _get_data ( self ) : cookie = self . adapter . cookies . get ( self . name ) return self . _deserialize ( cookie ) if cookie else { }


def data(self):
        """
        Gets session data lazily.
        """
        if not self._data:
            self._data = self._get_data()
        # Always return a dict, even if deserialization returned nothing
        if self._data is None:
            self._data = {}
        return self._data


def _signature(self, *parts):
        """
        Creates signature for the session.
        """
        signature = hmac.new(six.b(self.secret), digestmod=hashlib.sha1)
        signature.update(six.b('|'.join(parts)))
        return signature.hexdigest()

def _signature ( self , * parts ) : signature = hmac . new ( six . b ( self . secret ) , digestmod = hashlib . sha1 ) signature . update ( six . b ( '|' . join ( parts ) ) ) return signature . hexdigest ( )


def _serialize(self, value):
        """
        Converts the value to a signed string with timestamp.

        :param value:
            Object to be serialized.

        :returns:
            Serialized value.

        """

        # data = copy.deepcopy(value)
        data = value

        # 1. Serialize
        serialized = pickle.dumps(data).decode('latin-1')

        # 2. Encode
        # Percent encoding produces smaller result then urlsafe base64.
        encoded = parse.quote(serialized, '')

        # 3. Concatenate
        timestamp = str(int(time.time()))
        signature = self._signature(self.name, encoded, timestamp)
        concatenated = '|'.join([encoded, timestamp, signature])

        return concatenated

Converts the value to a signed string with timestamp .


def valid(self):
        """
        ``True`` if credentials are valid, ``False`` if expired.
        """

        if self.expiration_time:
            return self.expiration_time > int(time.time())
        else:
            return True

def valid ( self ) : if self . expiration_time : return self . expiration_time > int ( time . time ( ) ) else : return True

True if credentials are valid False if expired .


def expire_soon(self, seconds):
        """
        Returns ``True`` if credentials expire sooner than specified.

        :param int seconds:
            Number of seconds.

        :returns:
            ``True`` if credentials expire sooner than specified,
            else ``False``.

        """

        if self.expiration_time:
            return self.expiration_time < int(time.time()) + int(seconds)
        else:
            return False

def expire_soon ( self , seconds ) : if self . expiration_time : return self . expiration_time < int ( time . time ( ) ) + int ( seconds ) else : return False

Returns True if credentials expire sooner than specified .


def serialize(self):
        """
        Converts the credentials to a percent encoded string to be stored for
        later use.

        :returns:
            :class:`string`

        """

        if self.provider_id is None:
            raise ConfigError(
                'To serialize credentials you need to specify a '
                'unique integer under the "id" key in the config '
                'for each provider!')

        # Get the provider type specific items.
        rest = self.provider_type_class().to_tuple(self)

        # Provider ID and provider type ID are always the first two items.
        result = (self.provider_id, self.provider_type_id) + rest

        # Make sure that all items are strings.
        stringified = [str(i) for i in result]

        # Concatenate by newline.
        concatenated = '\n'.join(stringified)

        # Percent encode.
        return parse.quote(concatenated, '')

Converts the credentials to a percent encoded string to be stored for later use .


def is_binary_string(content):
        """
        Return true if string is binary data.
        """

        textchars = (bytearray([7, 8, 9, 10, 12, 13, 27]) +
                     bytearray(range(0x20, 0x100)))
        return bool(content.translate(None, textchars))

def is_binary_string ( content ) : textchars = ( bytearray ( [ 7 , 8 , 9 , 10 , 12 , 13 , 27 ] ) + bytearray ( range ( 0x20 , 0x100 ) ) ) return bool ( content . translate ( None , textchars ) )


def content(self):
        """
        The whole response content.
        """

        if not self._content:
            content = self.httplib_response.read()
            if self.is_binary_string(content):
                self._content = content
            else:
                self._content = content.decode('utf-8')
        return self._content

def content ( self ) : if not self . _content : content = self . httplib_response . read ( ) if self . is_binary_string ( content ) : self . _content = content else : self . _content = content . decode ( 'utf-8' ) return self . _content


def create_request_elements(
            cls, request_type, credentials, url, params=None, headers=None,
            body='', method='GET', verifier='', callback=''
    ):
        """
        Creates |oauth1| request elements.
        """

        params = params or {}
        headers = headers or {}

        consumer_key = credentials.consumer_key or ''
        consumer_secret = credentials.consumer_secret or ''
        token = credentials.token or ''
        token_secret = credentials.token_secret or ''

        # separate url base and query parameters
        url, base_params = cls._split_url(url)

        # add extracted params to future params
        params.update(dict(base_params))

        if request_type == cls.USER_AUTHORIZATION_REQUEST_TYPE:
            # no need for signature
            if token:
                params['oauth_token'] = token
            else:
                raise OAuth1Error(
                    'Credentials with valid token are required to create '
                    'User Authorization URL!')
        else:
            # signature needed
            if request_type == cls.REQUEST_TOKEN_REQUEST_TYPE:
                # Request Token URL
                if consumer_key and consumer_secret and callback:
                    params['oauth_consumer_key'] = consumer_key
                    params['oauth_callback'] = callback
                else:
                    raise OAuth1Error(
                        'Credentials with valid consumer_key, consumer_secret '
                        'and callback are required to create Request Token '
                        'URL!')

            elif request_type == cls.ACCESS_TOKEN_REQUEST_TYPE:
                # Access Token URL
                if consumer_key and consumer_secret and token and verifier:
                    params['oauth_token'] = token
                    params['oauth_consumer_key'] = consumer_key
                    params['oauth_verifier'] = verifier
                else:
                    raise OAuth1Error(
                        'Credentials with valid consumer_key, '
                        'consumer_secret, token and argument verifier'
                        ' are required to create Access Token URL!')

            elif request_type == cls.PROTECTED_RESOURCE_REQUEST_TYPE:
                # Protected Resources URL
                if consumer_key and consumer_secret and token and token_secret:
                    params['oauth_token'] = token
                    params['oauth_consumer_key'] = consumer_key
                else:
                    raise OAuth1Error(
                        'Credentials with valid consumer_key, ' +
                        'consumer_secret, token and token_secret are required '
                        'to create Protected Resources URL!')

            # Sign request.
            # http://oauth.net/core/1.0a/#anchor13

            # Prepare parameters for signature base string
            # http://oauth.net/core/1.0a/#rfc.section.9.1
            params['oauth_signature_method'] = cls._signature_generator.method
            params['oauth_timestamp'] = str(int(time.time()))
            params['oauth_nonce'] = cls.csrf_generator(str(uuid.uuid4()))
            params['oauth_version'] = '1.0'

            # add signature to params
            params['oauth_signature'] = cls._signature_generator.create_signature(  # noqa
                method, url, params, consumer_secret, token_secret)

        request_elements = core.RequestElements(
            url, method, params, headers, body)

        return cls._x_request_elements_filter(
            request_type, request_elements, credentials)


def _access_user_info(self):
        """
        Email is available in separate method so second request is needed.
        """
        response = super(Bitbucket, self)._access_user_info()

        response.data.setdefault("email", None)

        email_response = self.access(self.user_email_url)
        if email_response.data:
            for item in email_response.data:
                if item.get("primary", False):
                    response.data.update(email=item.get("email", None))

        return response

def _access_user_info ( self ) : response = super ( Bitbucket , self ) . _access_user_info ( ) response . data . setdefault ( "email" , None ) email_response = self . access ( self . user_email_url ) if email_response . data : for item in email_response . data : if item . get ( "primary" , False ) : response . data . update ( email = item . get ( "email" , None ) ) return response

Email is available in separate method so second request is needed .


def login(self, *login_args, **login_kwargs):
        """
        Decorator for Flask view functions.
        """

        def decorator(f):
            @wraps(f)
            def decorated(*args, **kwargs):
                self.response = make_response()
                adapter = WerkzeugAdapter(request, self.response)
                login_kwargs.setdefault('session', session)
                login_kwargs.setdefault('session_saver', self.session_saver)
                self.result = super(FlaskAuthomatic, self).login(
                    adapter,
                    *login_args,
                    **login_kwargs)
                return f(*args, **kwargs)
            return decorated
        return decorator

def login ( self , * login_args , * * login_kwargs ) : def decorator ( f ) : @ wraps ( f ) def decorated ( * args , * * kwargs ) : self . response = make_response ( ) adapter = WerkzeugAdapter ( request , self . response ) login_kwargs . setdefault ( 'session' , session ) login_kwargs . setdefault ( 'session_saver' , self . session_saver ) self . result = super ( FlaskAuthomatic , self ) . login ( adapter , * login_args , * * login_kwargs ) return f ( * args , * * kwargs ) return decorated return decorator


def login(self):
        """
        Launches the OpenID authentication procedure.
        """

        if self.params.get(self.identifier_param):
            # =================================================================
            # Phase 1 before redirect.
            # =================================================================
            self._log(
                logging.INFO,
                u'Starting OpenID authentication procedure.')

            url = users.create_login_url(
                dest_url=self.url, federated_identity=self.identifier)

            self._log(logging.INFO, u'Redirecting user to {0}.'.format(url))

            self.redirect(url)
        else:
            # =================================================================
            # Phase 2 after redirect.
            # =================================================================

            self._log(
                logging.INFO,
                u'Continuing OpenID authentication procedure after redirect.')

            user = users.get_current_user()

            if user:
                self._log(logging.INFO, u'Authentication successful.')
                self._log(logging.INFO, u'Creating user.')
                self.user = core.User(self,
                                      id=user.federated_identity(),
                                      email=user.email(),
                                      gae_user=user)

                # =============================================================
                # We're done
                # =============================================================
            else:
                raise FailureError(
                    'Unable to authenticate identifier "{0}"!'.format(
                        self.identifier))

Launches the OpenID authentication procedure .


def _session_key(self, key):
        """
        Generates session key string.

        :param str key:
            e.g. ``"authomatic:facebook:key"``

        """

        return '{0}:{1}:{2}'.format(self.settings.prefix, self.name, key)

def _session_key ( self , key ) : return '{0}:{1}:{2}' . format ( self . settings . prefix , self . name , key )


def _session_set(self, key, value):
        """
        Saves a value to session.
        """

        self.session[self._session_key(key)] = value

def _session_set ( self , key , value ) : self . session [ self . _session_key ( key ) ] = value


def csrf_generator(secret):
        """
        Generates CSRF token.

        Inspired by this article:
        http://blog.ptsecurity.com/2012/10/random-number-security-in-python.html

        :returns:
            :class:`str` Random unguessable string.

        """

        # Create hash from random string plus salt.
        hashed = hashlib.md5(uuid.uuid4().bytes + six.b(secret)).hexdigest()

        # Each time return random portion of the hash.
        span = 5
        shift = random.randint(0, span)
        return hashed[shift:shift - span - 1]


def _log(cls, level, msg, **kwargs):
        """
        Logs a message with pre-formatted prefix.

        :param int level:
            Logging level as specified in the
            `login module <http://docs.python.org/2/library/logging.html>`_ of
            Python standard library.

        :param str msg:
            The actual message.

        """

        logger = getattr(cls, '_logger', None) or authomatic.core._logger
        logger.log(
            level, ': '.join(
                ('authomatic', cls.__name__, msg)), **kwargs)

def
_log
(
cls
,
level
,
msg
,
*
*
kwargs
)
:
logger
=
getattr
(
cls
,
'_logger'
,
)
or
authomatic
.
core
.
_logger
logger
.
log
(
level
,
': '
.
join
(
(
'authomatic'
,
cls
.
__name__
,
msg
)
)
,
*
*
kwargs
)

Logs a message with pre - formatted prefix .


def _http_status_in_category(status, category):
        """
        Checks whether a HTTP status code is in the category denoted by the
        hundreds digit.
        """

        assert category < 10, 'HTTP status category must be a one-digit int!'
        cat = category * 100
        return status >= cat and status < cat + 100

Checks whether a HTTP status code is in the category denoted by the hundreds digit .


def _split_url(url):
        """
        Splits given url to url base and params converted to list of tuples.
        """

        split = parse.urlsplit(url)
        base = parse.urlunsplit((split.scheme, split.netloc, split.path, 0, 0))
        params = parse.parse_qsl(split.query, True)

        return base, params

def _split_url ( url ) : split = parse . urlsplit ( url ) base = parse . urlunsplit ( ( split . scheme , split . netloc , split . path , 0 , 0 ) ) params = parse . parse_qsl ( split . query , True ) return base , params

Splits given url to url base and params converted to list of tuples .