def _create_hstore_required(self, table_name, field, key):
        """Creates a REQUIRED CONSTRAINT for the specified hstore key."""

        name = self._required_constraint_name(
            table_name, field, key)

        sql = self.sql_hstore_required_create.format(
            name=self.quote_name(name),
            table=self.quote_name(table_name),
            field=self.quote_name(field.column),
            key=key
        )
        self.execute(sql)

def _create_hstore_required ( self , table_name , field , key ) : name = self . _required_constraint_name ( table_name , field , key ) sql = self . sql_hstore_required_create . format ( name = self . quote_name ( name ) , table = self . quote_name ( table_name ) , field = self . quote_name ( field . column ) , key = key ) self . execute ( sql )

Creates a REQUIRED CONSTRAINT for the specified hstore key .


def _rename_hstore_required(self, old_table_name, new_table_name,
                                old_field, new_field, key):
        """Renames an existing REQUIRED CONSTRAINT for the specified
        hstore key."""

        old_name = self._required_constraint_name(
            old_table_name, old_field, key)
        new_name = self._required_constraint_name(
            new_table_name, new_field, key)

        sql = self.sql_hstore_required_rename.format(
            table=self.quote_name(new_table_name),
            old_name=self.quote_name(old_name),
            new_name=self.quote_name(new_name)
        )
        self.execute(sql)

def _rename_hstore_required ( self , old_table_name , new_table_name , old_field , new_field , key ) : old_name = self . _required_constraint_name ( old_table_name , old_field , key ) new_name = self . _required_constraint_name ( new_table_name , new_field , key ) sql = self . sql_hstore_required_rename . format ( table = self . quote_name ( new_table_name ) , old_name = self . quote_name ( old_name ) , new_name = self . quote_name ( new_name ) ) self . execute ( sql )

Renames an existing REQUIRED CONSTRAINT for the specified hstore key .


def _drop_hstore_required(self, table_name, field, key):
        """Drops a REQUIRED CONSTRAINT for the specified hstore key."""

        name = self._required_constraint_name(
            table_name, field, key)

        sql = self.sql_hstore_required_drop.format(
            table=self.quote_name(table_name),
            name=self.quote_name(name)
        )
        self.execute(sql)

def _drop_hstore_required ( self , table_name , field , key ) : name = self . _required_constraint_name ( table_name , field , key ) sql = self . sql_hstore_required_drop . format ( table = self . quote_name ( table_name ) , name = self . quote_name ( name ) ) self . execute ( sql )

Drops a REQUIRED CONSTRAINT for the specified hstore key .


def _required_constraint_name(table: str, field, key):
        """Gets the name for a CONSTRAINT that applies
        to a single hstore key.

        Arguments:
            table:
                The name of the table the field is
                a part of.

            field:
                The hstore field to create a
                UNIQUE INDEX for.

            key:
                The name of the hstore key
                to create the name for.

        Returns:
            The name for the UNIQUE index.
        """

        return '{table}_{field}_required_{postfix}'.format(
            table=table,
            field=field.column,
            postfix=key
        )

def _required_constraint_name ( table : str , field , key ) : return '{table}_{field}_required_{postfix}' . format ( table = table , field = field . column , postfix = key )

Gets the name for a CONSTRAINT that applies to a single hstore key .


def create_sql(self, model, schema_editor, using=''):
        """Creates the actual SQL used when applying the migration."""
        if django.VERSION >= (2, 0):
            statement = super().create_sql(model, schema_editor, using)
            statement.template = self.sql_create_index
            statement.parts['condition'] = self.condition
            return statement
        else:
            sql_create_index = self.sql_create_index
            sql_parameters = {
                **Index.get_sql_create_template_values(self, model, schema_editor, using),
                'condition': self.condition
            }
            return sql_create_index % sql_parameters

def create_sql ( self , model , schema_editor , using = '' ) : if django . VERSION >= ( 2 , 0 ) : statement = super ( ) . create_sql ( model , schema_editor , using ) statement . template = self . sql_create_index statement . parts [ 'condition' ] = self . condition return statement else : sql_create_index = self . sql_create_index sql_parameters = { * * Index . get_sql_create_template_values ( self , model , schema_editor , using ) , 'condition' : self . condition } return sql_create_index % sql_parameters

Creates the actual SQL used when applying the migration .


def create_command(text, commands):
    """Creates a custom setup.py command."""

    class CustomCommand(BaseCommand):
        description = text

        def run(self):
            for cmd in commands:
                subprocess.check_call(cmd)

    return CustomCommand

def create_command ( text , commands ) : class CustomCommand ( BaseCommand ) : description = text def run ( self ) : for cmd in commands : subprocess . check_call ( cmd ) return CustomCommand


def _get_backend_base():
    """Gets the base class for the custom database back-end.

    This should be the Django PostgreSQL back-end. However,
    some people are already using a custom back-end from
    another package. We are nice people and expose an option
    that allows them to configure the back-end we base upon.

    As long as the specified base eventually also has
    the PostgreSQL back-end as a base, then everything should
    work as intended.
    """
    base_class_name = getattr(
        settings,
        'POSTGRES_EXTRA_DB_BACKEND_BASE',
        'django.db.backends.postgresql'
    )

    base_class_module = importlib.import_module(base_class_name + '.base')
    base_class = getattr(base_class_module, 'DatabaseWrapper', None)

    if not base_class:
        raise ImproperlyConfigured((
            '\'%s\' is not a valid database back-end.'
            ' The module does not define a DatabaseWrapper class.'
            ' Check the value of POSTGRES_EXTRA_DB_BACKEND_BASE.'
        ) % base_class_name)

    if isinstance(base_class, Psycopg2DatabaseWrapper):
        raise ImproperlyConfigured((
            '\'%s\' is not a valid database back-end.'
            ' It does inherit from the PostgreSQL back-end.'
            ' Check the value of POSTGRES_EXTRA_DB_BACKEND_BASE.'
        ) % base_class_name)

    return base_class

Gets the base class for the custom database back - end .


def prepare_database(self):
        """Ran to prepare the configured database.

        This is where we enable the `hstore` extension
        if it wasn't enabled yet."""

        super().prepare_database()
        with self.cursor() as cursor:
            try:
                cursor.execute('CREATE EXTENSION IF NOT EXISTS hstore')
            except ProgrammingError:  # permission denied
                logger.warning(
                    'Failed to create "hstore" extension. '
                    'Tables with hstore columns may fail to migrate. '
                    'If hstore is needed, make sure you are connected '
                    'to the database as a superuser '
                    'or add the extension manually.',
                    exc_info=True)

Ran to prepare the configured database .


def get_prep_value(self, value):
        """Override the base class so it doesn't cast all values
        to strings.

        psqlextra supports expressions in hstore fields, so casting
        all values to strings is a bad idea."""

        value = Field.get_prep_value(self, value)

        if isinstance(value, dict):
            prep_value = {}
            for key, val in value.items():
                if isinstance(val, Expression):
                    prep_value[key] = val
                elif val is not None:
                    prep_value[key] = str(val)
                else:
                    prep_value[key] = val

            value = prep_value

        if isinstance(value, list):
            value = [str(item) for item in value]

        return value

def get_prep_value ( self , value ) : value = Field . get_prep_value ( self , value ) if isinstance ( value , dict ) : prep_value = { } for key , val in value . items ( ) : if isinstance ( val , Expression ) : prep_value [ key ] = val elif val is not None : prep_value [ key ] = str ( val ) else : prep_value [ key ] = val value = prep_value if isinstance ( value , list ) : value = [ str ( item ) for item in value ] return value

Override the base class so it doesn t cast all values to strings .


def _form_returning(self):
        """Builds the RETURNING part of the query."""

        qn = self.connection.ops.quote_name
        return ' RETURNING %s' % qn(self.query.model._meta.pk.attname)

def
_form_returning
(
self
)
:
qn
=
self
.
connection
.
ops
.
quote_name
return
' RETURNING %s'
%
qn
(
self
.
query
.
model
.
_meta
.
pk
.
attname
)

Builds the RETURNING part of the query .


def as_sql(self, return_id=False):
        """Builds the SQL INSERT statement."""

        queries = [
            self._rewrite_insert(sql, params, return_id)
            for sql, params in super().as_sql()
        ]

        return queries

def as_sql ( self , return_id = False ) : queries = [ self . _rewrite_insert ( sql , params , return_id ) for sql , params in super ( ) . as_sql ( ) ] return queries


def _rewrite_insert(self, sql, params, return_id=False):
        """Rewrites a formed SQL INSERT query to include
        the ON CONFLICT clause.

        Arguments:
            sql:
                The SQL INSERT query to rewrite.

            params:
                The parameters passed to the query.

            returning:
                What to put in the `RETURNING` clause
                of the resulting query.

        Returns:
            A tuple of the rewritten SQL query and new params.
        """

        returning = self.qn(self.query.model._meta.pk.attname) if return_id else '*'

        if self.query.conflict_action.value == 'UPDATE':
            return self._rewrite_insert_update(sql, params, returning)
        elif self.query.conflict_action.value == 'NOTHING':
            return self._rewrite_insert_nothing(sql, params, returning)

        raise SuspiciousOperation((
            '%s is not a valid conflict action, specify '
            'ConflictAction.UPDATE or ConflictAction.NOTHING.'
        ) % str(self.query.conflict_action))

Rewrites a formed SQL INSERT query to include the ON CONFLICT clause .


def _rewrite_insert_update(self, sql, params, returning):
        """Rewrites a formed SQL INSERT query to include
        the ON CONFLICT DO UPDATE clause."""

        update_columns = ', '.join([
            '{0} = EXCLUDED.{0}'.format(self.qn(field.column))
            for field in self.query.update_fields
        ])

        # build the conflict target, the columns to watch
        # for conflicts
        conflict_target = self._build_conflict_target()

        index_predicate = self.query.index_predicate

        sql_template = (
            '{insert} ON CONFLICT {conflict_target} DO UPDATE '
            'SET {update_columns} RETURNING {returning}'
        )

        if index_predicate:
            sql_template = (
                '{insert} ON CONFLICT {conflict_target} WHERE {index_predicate} DO UPDATE '
                'SET {update_columns} RETURNING {returning}'
            )

        return (
            sql_template.format(
                insert=sql,
                conflict_target=conflict_target,
                update_columns=update_columns,
                returning=returning,
                index_predicate=index_predicate,
            ),
            params
        )

Rewrites a formed SQL INSERT query to include the ON CONFLICT DO UPDATE clause .


def _rewrite_insert_nothing(self, sql, params, returning):
        """Rewrites a formed SQL INSERT query to include
        the ON CONFLICT DO NOTHING clause."""

        # build the conflict target, the columns to watch
        # for conflicts
        conflict_target = self._build_conflict_target()

        where_clause = ' AND '.join([
            '{0} = %s'.format(self._format_field_name(field_name))
            for field_name in self.query.conflict_target
        ])

        where_clause_params = [
            self._format_field_value(field_name)
            for field_name in self.query.conflict_target
        ]

        params = params + tuple(where_clause_params)

        # this looks complicated, and it is, but it is for a reason... a normal
        # ON CONFLICT DO NOTHING doesn't return anything if the row already exists
        # so we do DO UPDATE instead that never executes to lock the row, and then
        # select from the table in case we're dealing with an existing row..
        return (
            (
                'WITH insdata AS ('
                '{insert} ON CONFLICT {conflict_target} DO UPDATE'
                ' SET {pk_column} = NULL WHERE FALSE RETURNING {returning})'
                ' SELECT * FROM insdata UNION ALL'
                ' SELECT {returning} FROM {table} WHERE {where_clause} LIMIT 1;'
            ).format(
                insert=sql,
                conflict_target=conflict_target,
                pk_column=self.qn(self.query.model._meta.pk.column),
                returning=returning,
                table=self.query.objs[0]._meta.db_table,
                where_clause=where_clause
            ),
            params
        )

Rewrites a formed SQL INSERT query to include the ON CONFLICT DO NOTHING clause .


def _build_conflict_target(self):
        """Builds the `conflict_target` for the ON CONFLICT
        clause."""

        conflict_target = []

        if not isinstance(self.query.conflict_target, list):
            raise SuspiciousOperation((
                '%s is not a valid conflict target, specify '
                'a list of column names, or tuples with column '
                'names and hstore key.'
            ) % str(self.query.conflict_target))

        def _assert_valid_field(field_name):
            field_name = self._normalize_field_name(field_name)
            if self._get_model_field(field_name):
                return

            raise SuspiciousOperation((
                '%s is not a valid conflict target, specify '
                'a list of column names, or tuples with column '
                'names and hstore key.'
            ) % str(field_name))

        for field_name in self.query.conflict_target:
            _assert_valid_field(field_name)

            # special handling for hstore keys
            if isinstance(field_name, tuple):
                conflict_target.append(
                    '(%s->\'%s\')' % (
                        self._format_field_name(field_name),
                        field_name[1]
                    )
                )
            else:
                conflict_target.append(
                    self._format_field_name(field_name))

        return '(%s)' % ','.join(conflict_target)

Builds the conflict_target for the ON CONFLICT clause .


def _get_model_field(self, name: str):
        """Gets the field on a model with the specified name.

        Arguments:
            name:
                The name of the field to look for.

                This can be both the actual field name, or
                the name of the column, both will work :)

        Returns:
            The field with the specified name or None if
            no such field exists.
        """

        field_name = self._normalize_field_name(name)

        # 'pk' has special meaning and always refers to the primary
        # key of a model, we have to respect this de-facto standard behaviour
        if field_name == 'pk' and self.query.model._meta.pk:
            return self.query.model._meta.pk

        for field in self.query.model._meta.local_concrete_fields:
            if field.name == field_name or field.column == field_name:
                return field

        return None

Gets the field on a model with the specified name .


def _format_field_name(self, field_name) -> str:
        """Formats a field's name for usage in SQL.

        Arguments:
            field_name:
                The field name to format.

        Returns:
            The specified field name formatted for
            usage in SQL.
        """

        field = self._get_model_field(field_name)
        return self.qn(field.column)

def _format_field_name ( self , field_name ) -> str : field = self . _get_model_field ( field_name ) return self . qn ( field . column )

Formats a field s name for usage in SQL .


def _format_field_value(self, field_name) -> str:
        """Formats a field's value for usage in SQL.

        Arguments:
            field_name:
                The name of the field to format
                the value of.

        Returns:
            The field's value formatted for usage
            in SQL.
        """

        field_name = self._normalize_field_name(field_name)
        field = self._get_model_field(field_name)

        return SQLInsertCompiler.prepare_value(
            self,
            field,
            # Note: this deliberately doesn't use `pre_save_val` as we don't
            # want things like auto_now on DateTimeField (etc.) to change the
            # value. We rely on pre_save having already been done by the
            # underlying compiler so that things like FileField have already had
            # the opportunity to save out their data.
            getattr(self.query.objs[0], field.attname)
        )

Formats a field s value for usage in SQL .


def _create_hstore_unique(self, model, field, keys):
        """Creates a UNIQUE constraint for the specified hstore keys."""

        name = self._unique_constraint_name(
            model._meta.db_table, field, keys)
        columns = [
            '(%s->\'%s\')' % (field.column, key)
            for key in keys
        ]
        sql = self.sql_hstore_unique_create.format(
            name=self.quote_name(name),
            table=self.quote_name(model._meta.db_table),
            columns=','.join(columns)
        )
        self.execute(sql)

def _create_hstore_unique ( self , model , field , keys ) : name = self . _unique_constraint_name ( model . _meta . db_table , field , keys ) columns = [ '(%s->\'%s\')' % ( field . column , key ) for key in keys ] sql = self . sql_hstore_unique_create . format ( name = self . quote_name ( name ) , table = self . quote_name ( model . _meta . db_table ) , columns = ',' . join ( columns ) ) self . execute ( sql )

Creates a UNIQUE constraint for the specified hstore keys .


def _rename_hstore_unique(self, old_table_name, new_table_name,
                              old_field, new_field, keys):
        """Renames an existing UNIQUE constraint for the specified
        hstore keys."""

        old_name = self._unique_constraint_name(
            old_table_name, old_field, keys)
        new_name = self._unique_constraint_name(
            new_table_name, new_field, keys)

        sql = self.sql_hstore_unique_rename.format(
            old_name=self.quote_name(old_name),
            new_name=self.quote_name(new_name)
        )
        self.execute(sql)

def _rename_hstore_unique ( self , old_table_name , new_table_name , old_field , new_field , keys ) : old_name = self . _unique_constraint_name ( old_table_name , old_field , keys ) new_name = self . _unique_constraint_name ( new_table_name , new_field , keys ) sql = self . sql_hstore_unique_rename . format ( old_name = self . quote_name ( old_name ) , new_name = self . quote_name ( new_name ) ) self . execute ( sql )

Renames an existing UNIQUE constraint for the specified hstore keys .


def _drop_hstore_unique(self, model, field, keys):
        """Drops a UNIQUE constraint for the specified hstore keys."""

        name = self._unique_constraint_name(
            model._meta.db_table, field, keys)
        sql = self.sql_hstore_unique_drop.format(name=self.quote_name(name))
        self.execute(sql)

def _drop_hstore_unique ( self , model , field , keys ) : name = self . _unique_constraint_name ( model . _meta . db_table , field , keys ) sql = self . sql_hstore_unique_drop . format ( name = self . quote_name ( name ) ) self . execute ( sql )

Drops a UNIQUE constraint for the specified hstore keys .


def _unique_constraint_name(table: str, field, keys):
        """Gets the name for a UNIQUE INDEX that applies
        to one or more keys in a hstore field.

        Arguments:
            table:
                The name of the table the field is
                a part of.

            field:
                The hstore field to create a
                UNIQUE INDEX for.

            key:
                The name of the hstore key
                to create the name for.

                This can also be a tuple
                of multiple names.

        Returns:
            The name for the UNIQUE index.
        """
        postfix = '_'.join(keys)
        return '{table}_{field}_unique_{postfix}'.format(
            table=table,
            field=field.column,
            postfix=postfix
        )

def _unique_constraint_name ( table : str , field , keys ) : postfix = '_' . join ( keys ) return '{table}_{field}_unique_{postfix}' . format ( table = table , field = field . column , postfix = postfix )

Gets the name for a UNIQUE INDEX that applies to one or more keys in a hstore field .


def _iterate_uniqueness_keys(self, field):
        """Iterates over the keys marked as "unique"
        in the specified field.

        Arguments:
            field:
                The field of which key's to
                iterate over.
        """

        uniqueness = getattr(field, 'uniqueness', None)
        if not uniqueness:
            return

        for keys in uniqueness:
            composed_keys = self._compose_keys(keys)
            yield composed_keys

def _iterate_uniqueness_keys ( self , field ) : uniqueness = getattr ( field , 'uniqueness' , None ) if not uniqueness : return for keys in uniqueness : composed_keys = self . _compose_keys ( keys ) yield composed_keys

Iterates over the keys marked as unique in the specified field .


def add_condition(self, field, value: Any) -> None:
        """Adds an extra condition to this join.

        Arguments:
            field:
                The field that the condition will apply to.

            value:
                The value to compare.
        """

        self.extra_conditions.append((field, value))

def add_condition ( self , field , value : Any ) -> None : self . extra_conditions . append ( ( field , value ) )


def as_sql(self, compiler, connection) -> Tuple[str, List[Any]]:
        """Compiles this JOIN into a SQL string."""

        sql, params = super().as_sql(compiler, connection)
        qn = compiler.quote_name_unless_alias

        # generate the extra conditions
        extra_conditions = ' AND '.join([
            '{}.{} = %s'.format(
                qn(self.table_name),
                qn(field.column)
            )
            for field, value in self.extra_conditions
        ])

        # add to the existing params, so the connector will
        # actually nicely format the value for us
        for _, value in self.extra_conditions:
            params.append(value)

        # rewrite the sql to include the extra conditions
        rewritten_sql = sql.replace(')', ' AND {})'.format(extra_conditions))
        return rewritten_sql, params


def tdist95conf_level(df):
    """Approximate the 95% confidence interval for Student's T distribution.

    Given the degrees of freedom, returns an approximation to the 95%
    confidence interval for the Student's T distribution.

    Args:
        df: An integer, the number of degrees of freedom.

    Returns:
        A float.
    """
    df = int(round(df))
    highest_table_df = len(_T_DIST_95_CONF_LEVELS)
    if df >= 200:
        return 1.960
    if df >= 100:
        return 1.984
    if df >= 80:
        return 1.990
    if df >= 60:
        return 2.000
    if df >= 50:
        return 2.009
    if df >= 40:
        return 2.021
    if df >= highest_table_df:
        return _T_DIST_95_CONF_LEVELS[highest_table_df - 1]
    return _T_DIST_95_CONF_LEVELS[df]

def tdist95conf_level ( df ) : df = int ( round ( df ) ) highest_table_df = len ( _T_DIST_95_CONF_LEVELS ) if df >= 200 : return 1.960 if df >= 100 : return 1.984 if df >= 80 : return 1.990 if df >= 60 : return 2.000 if df >= 50 : return 2.009 if df >= 40 : return 2.021 if df >= highest_table_df : return _T_DIST_95_CONF_LEVELS [ highest_table_df - 1 ] return _T_DIST_95_CONF_LEVELS [ df ]

Approximate the 95% confidence interval for Student s T distribution .


def pooled_sample_variance(sample1, sample2):
    """Find the pooled sample variance for two samples.

    Args:
        sample1: one sample.
        sample2: the other sample.

    Returns:
        Pooled sample variance, as a float.
    """
    deg_freedom = len(sample1) + len(sample2) - 2
    mean1 = statistics.mean(sample1)
    squares1 = ((x - mean1) ** 2 for x in sample1)
    mean2 = statistics.mean(sample2)
    squares2 = ((x - mean2) ** 2 for x in sample2)

    return (math.fsum(squares1) + math.fsum(squares2)) / float(deg_freedom)

def pooled_sample_variance ( sample1 , sample2 ) : deg_freedom = len ( sample1 ) + len ( sample2 ) - 2 mean1 = statistics . mean ( sample1 ) squares1 = ( ( x - mean1 ) ** 2 for x in sample1 ) mean2 = statistics . mean ( sample2 ) squares2 = ( ( x - mean2 ) ** 2 for x in sample2 ) return ( math . fsum ( squares1 ) + math . fsum ( squares2 ) ) / float ( deg_freedom )

Find the pooled sample variance for two samples .


def tscore(sample1, sample2):
    """Calculate a t-test score for the difference between two samples.

    Args:
        sample1: one sample.
        sample2: the other sample.

    Returns:
        The t-test score, as a float.
    """
    if len(sample1) != len(sample2):
        raise ValueError("different number of values")
    error = pooled_sample_variance(sample1, sample2) / len(sample1)
    diff = statistics.mean(sample1) - statistics.mean(sample2)
    return diff / math.sqrt(error * 2)

def
tscore
(
sample1
,
sample2
)
:
if
len
(
sample1
)
!=
len
(
sample2
)
:
raise
ValueError
(
"different number of values"
)
error
=
pooled_sample_variance
(
sample1
,
sample2
)
/
len
(
sample1
)
diff
=
statistics
.
mean
(
sample1
)
statistics
.
mean
(
sample2
)
return
diff
/
math
.
sqrt
(
error
*
2
)

Calculate a t - test score for the difference between two samples .


def is_significant(sample1, sample2):
    """Determine whether two samples differ significantly.

    This uses a Student's two-sample, two-tailed t-test with alpha=0.95.

    Args:
        sample1: one sample.
        sample2: the other sample.

    Returns:
        (significant, t_score) where significant is a bool indicating whether
        the two samples differ significantly; t_score is the score from the
        two-sample T test.
    """
    deg_freedom = len(sample1) + len(sample2) - 2
    critical_value = tdist95conf_level(deg_freedom)
    t_score = tscore(sample1, sample2)
    return (abs(t_score) >= critical_value, t_score)

Determine whether two samples differ significantly .


def topoSort(roots, getParents):
    """Return a topological sorting of nodes in a graph.

    roots - list of root nodes to search from
    getParents - function which returns the parents of a given node
    """

    results = []
    visited = set()

    # Use iterative version to avoid stack limits for large datasets
    stack = [(node, 0) for node in roots]
    while stack:
        current, state = stack.pop()
        if state == 0:
            # before recursing
            if current not in visited:
                visited.add(current)
                stack.append((current, 1))
                stack.extend((parent, 0) for parent in getParents(current))
        else:
            # after recursing
            assert(current in visited)
            results.append(current)
    return results

Return a topological sorting of nodes in a graph .


def n_queens(queen_count):
    """N-Queens solver.

    Args:
        queen_count: the number of queens to solve for. This is also the
            board size.

    Yields:
        Solutions to the problem. Each yielded value is looks like
        (3, 8, 2, 1, 4, ..., 6) where each number is the column position for the
        queen, and the index into the tuple indicates the row.
    """
    cols = range(queen_count)
    for vec in permutations(cols):
        if (queen_count == len(set(vec[i] + i for i in cols))
                        == len(set(vec[i] - i for i in cols))):
            yield vec

def n_queens ( queen_count ) : cols = range ( queen_count ) for vec in permutations ( cols ) : if ( queen_count == len ( set ( vec [ i ] + i for i in cols ) ) == len ( set ( vec [ i ] - i for i in cols ) ) ) : yield vec


def play(self, board):
        """ uct tree search """
        color = board.color
        node = self
        path = [node]
        while True:
            pos = node.select(board)
            if pos == PASS:
                break
            board.move(pos)
            child = node.pos_child[pos]
            if not child:
                child = node.pos_child[pos] = UCTNode()
                child.unexplored = board.useful_moves()
                child.pos = pos
                child.parent = node
                path.append(child)
                break
            path.append(child)
            node = child
        self.random_playout(board)
        self.update_path(board, color, path)

def play ( self , board ) : color = board . color node = self path = [ node ] while True : pos = node . select ( board ) if pos == PASS : break board . move ( pos ) child = node . pos_child [ pos ] if not child : child = node . pos_child [ pos ] = UCTNode ( ) child . unexplored = board . useful_moves ( ) child . pos = pos child . parent = node path . append ( child ) break path . append ( child ) node = child self . random_playout ( board ) self . update_path ( board , color , path )


def select(self, board):
        """ select move; unexplored children first, then according to uct value """
        if self.unexplored:
            i = random.randrange(len(self.unexplored))
            pos = self.unexplored[i]
            self.unexplored[i] = self.unexplored[len(self.unexplored) - 1]
            self.unexplored.pop()
            return pos
        elif self.bestchild:
            return self.bestchild.pos
        else:
            return PASS

def select ( self , board ) : if self . unexplored : i = random . randrange ( len ( self . unexplored ) ) pos = self . unexplored [ i ] self . unexplored [ i ] = self . unexplored [ len ( self . unexplored ) - 1 ] self . unexplored . pop ( ) return pos elif self . bestchild : return self . bestchild . pos else : return PASS

select move ; unexplored children first then according to uct value


def random_playout(self, board):
        """ random play until both players pass """
        for x in range(MAXMOVES):  # XXX while not self.finished?
            if board.finished:
                break
            board.move(board.random_move())

def
random_playout
(
self
,
board
)
:
for
x
in
range
(
MAXMOVES
)
:
# XXX while not self.finished?
if
board
.
finished
:
break
board
.
move
(
board
.
random_move
(
)
)


def filter_benchmarks(benchmarks, bench_funcs, base_ver):
    """Filters out benchmarks not supported by both Pythons.

    Args:
        benchmarks: a set() of benchmark names
        bench_funcs: dict mapping benchmark names to functions
        python: the interpereter commands (as lists)

    Returns:
        The filtered set of benchmark names
    """
    for bm in list(benchmarks):
        func = bench_funcs[bm]
        if getattr(func, '_python2_only', False) and (3, 0) <= base_ver:
            benchmarks.discard(bm)
            logging.info("Skipping Python2-only benchmark %s; "
                         "not compatible with Python %s" % (bm, base_ver))
            continue
    return benchmarks

def
filter_benchmarks
(
benchmarks
,
bench_funcs
,
base_ver
)
:
for
bm
in
list
(
benchmarks
)
:
func
=
bench_funcs
[
bm
]
if
getattr
(
func
,
'_python2_only'
,
False
)
and
(
3
,
0
)
<=
base_ver
:
benchmarks
.
discard
(
bm
)
logging
.
info
(
"Skipping Python2-only benchmark %s; "
"not compatible with Python %s"
%
(
bm
,
base_ver
)
)
continue
return
benchmarks

Filters out benchmarks not supported by both Pythons .


def expand_benchmark_name(bm_name, bench_groups):
    """Recursively expand name benchmark names.

    Args:
        bm_name: string naming a benchmark or benchmark group.

    Yields:
        Names of actual benchmarks, with all group names fully expanded.
    """
    expansion = bench_groups.get(bm_name)
    if expansion:
        for name in expansion:
            for name in expand_benchmark_name(name, bench_groups):
                yield name
    else:
        yield bm_name

def expand_benchmark_name ( bm_name , bench_groups ) : expansion = bench_groups . get ( bm_name ) if expansion : for name in expansion : for name in expand_benchmark_name ( name , bench_groups ) : yield name else : yield bm_name

Recursively expand name benchmark names .


def gen_string_table(n):
    """Generates the list of strings that will be used in the benchmarks.

    All strings have repeated prefixes and suffices, and n specifies the
    number of repetitions.
    """
    strings = []

    def append(s):
        if USE_BYTES_IN_PY3K:
            strings.append(s.encode('latin1'))
        else:
            strings.append(s)
    append('-' * n + 'Perl' + '-' * n)
    append('P' * n + 'Perl' + 'P' * n)
    append('-' * n + 'Perl' + '-' * n)
    append('-' * n + 'Perl' + '-' * n)
    append('-' * n + 'Python' + '-' * n)
    append('P' * n + 'Python' + 'P' * n)
    append('-' * n + 'Python' + '-' * n)
    append('-' * n + 'Python' + '-' * n)
    append('-' * n + 'Python' + '-' * n)
    append('-' * n + 'Python' + '-' * n)
    append('-' * n + 'Perl' + '-' * n)
    append('P' * n + 'Perl' + 'P' * n)
    append('-' * n + 'Perl' + '-' * n)
    append('-' * n + 'Perl' + '-' * n)
    append('-' * n + 'PythonPython' + '-' * n)
    append('P' * n + 'PythonPython' + 'P' * n)
    append('-' * n + 'a5,b7,c9,' + '-' * n)
    append('-' * n + 'a5,b7,c9,' + '-' * n)
    append('-' * n + 'a5,b7,c9,' + '-' * n)
    append('-' * n + 'a5,b7,c9,' + '-' * n)
    append('-' * n + 'Python' + '-' * n)
    return strings

def gen_string_table ( n ) : strings = [ ] def append ( s ) : if USE_BYTES_IN_PY3K : strings . append ( s . encode ( 'latin1' ) ) else : strings . append ( s ) append ( '-' * n + 'Perl' + '-' * n ) append ( 'P' * n + 'Perl' + 'P' * n ) append ( '-' * n + 'Perl' + '-' * n ) append ( '-' * n + 'Perl' + '-' * n ) append ( '-' * n + 'Python' + '-' * n ) append ( 'P' * n + 'Python' + 'P' * n ) append ( '-' * n + 'Python' + '-' * n ) append ( '-' * n + 'Python' + '-' * n ) append ( '-' * n + 'Python' + '-' * n ) append ( '-' * n + 'Python' + '-' * n ) append ( '-' * n + 'Perl' + '-' * n ) append ( 'P' * n + 'Perl' + 'P' * n ) append ( '-' * n + 'Perl' + '-' * n ) append ( '-' * n + 'Perl' + '-' * n ) append ( '-' * n + 'PythonPython' + '-' * n ) append ( 'P' * n + 'PythonPython' + 'P' * n ) append ( '-' * n + 'a5,b7,c9,' + '-' * n ) append ( '-' * n + 'a5,b7,c9,' + '-' * n ) append ( '-' * n + 'a5,b7,c9,' + '-' * n ) append ( '-' * n + 'a5,b7,c9,' + '-' * n ) append ( '-' * n + 'Python' + '-' * n ) return strings

Generates the list of strings that will be used in the benchmarks .


def init_benchmarks(n_values=None):
    """Initialize the strings we'll run the regexes against.

    The strings used in the benchmark are prefixed and suffixed by
    strings that are repeated n times.

    The sequence n_values contains the values for n.
    If n_values is None the values of n from the original benchmark
    are used.

    The generated list of strings is cached in the string_tables
    variable, which is indexed by n.

    Returns:
    A list of string prefix/suffix lengths.
    """

    if n_values is None:
        n_values = (0, 5, 50, 250, 1000, 5000, 10000)

    string_tables = {n: gen_string_table(n) for n in n_values}
    regexs = gen_regex_table()

    data = []
    for n in n_values:
        for id in xrange(len(regexs)):
            regex = regexs[id]
            string = string_tables[n][id]
            data.append((regex, string))
    return data

def init_benchmarks ( n_values = None ) : if n_values is None : n_values = ( 0 , 5 , 50 , 250 , 1000 , 5000 , 10000 ) string_tables = { n : gen_string_table ( n ) for n in n_values } regexs = gen_regex_table ( ) data = [ ] for n in n_values : for id in xrange ( len ( regexs ) ) : regex = regexs [ id ] string = string_tables [ n ] [ id ] data . append ( ( regex , string ) ) return data

Initialize the strings we ll run the regexes against .


def GetDomain(self):
        """Returns the domain of the B-Spline"""
        return (self.knots[self.degree - 1],
                self.knots[len(self.knots) - self.degree])

def GetDomain ( self ) : return ( self . knots [ self . degree - 1 ] , self . knots [ len ( self . knots ) - self . degree ] )


def fetch_items(self, category, **kwargs):
        """Fetch the messages.

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items
        """
        from_date = kwargs['from_date']

        logger.info("Fetching messages of '%s' - '%s' channel from %s",
                    self.url, self.channel, str(from_date))

        fetching = True
        page = 0
        nposts = 0

        # Convert timestamp to integer for comparing
        since = int(from_date.timestamp() * 1000)

        while fetching:
            raw_posts = self.client.posts(self.channel, page=page)

            posts_before = nposts

            for post in self._parse_posts(raw_posts):
                if post['update_at'] < since:
                    fetching = False
                    break

                # Fetch user data
                user_id = post['user_id']
                user = self._get_or_fetch_user(user_id)
                post['user_data'] = user

                yield post
                nposts += 1

            if fetching:
                # If no new posts were fetched; stop the process
                if posts_before == nposts:
                    fetching = False
                else:
                    page += 1

        logger.info("Fetch process completed: %s posts fetched", nposts)


def _parse_posts(self, raw_posts):
        """Parse posts and returns in order."""

        parsed_posts = self.parse_json(raw_posts)

        # Posts are not sorted. The order is provided by
        # 'order' key.
        for post_id in parsed_posts['order']:
            yield parsed_posts['posts'][post_id]


def user(self, user):
        """Fetch user data."""

        entrypoint = self.RUSERS + '/' + user
        response = self._fetch(entrypoint, None)

        return response

def user ( self , user ) : entrypoint = self . RUSERS + '/' + user response = self . _fetch ( entrypoint , None ) return response


def fetch(self, category=CATEGORY_ENTRY):
        """Fetch the entries from the url.

        The method retrieves all entries from a RSS url

        :param category: the category of items to fetch

        :returns: a generator of entries
        """
        kwargs = {}
        items = super().fetch(category, **kwargs)

        return items

def fetch ( self , category = CATEGORY_ENTRY ) : kwargs = { } items = super ( ) . fetch ( category , * * kwargs ) return items


def fetch_items(self, category, **kwargs):
        """Fetch the entries

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items
        """
        logger.info("Looking for rss entries at feed '%s'", self.url)

        nentries = 0  # number of entries

        raw_entries = self.client.get_entries()
        entries = self.parse_feed(raw_entries)['entries']
        for item in entries:
            yield item
            nentries += 1

        logger.info("Total number of entries: %i", nentries)

def
fetch_items
(
self
,
category
,
*
*
kwargs
)
:
logger
.
info
(
"Looking for rss entries at feed '%s'"
,
self
.
url
)
nentries
=
0
# number of entries
raw_entries
=
self
.
client
.
get_entries
(
)
entries
=
self
.
parse_feed
(
raw_entries
)
[
'entries'
]
for
item
in
entries
:
yield
item
nentries
+=
1
logger
.
info
(
"Total number of entries: %i"
,
nentries
)


def setup_cmd_parser(cls):
        """Returns the RSS argument parser."""

        parser = BackendCommandArgumentParser(cls.BACKEND.CATEGORIES,
                                              archive=True)

        # Required arguments
        parser.parser.add_argument('url',
                                   help="URL of the RSS feed")

        return parser

def
setup_cmd_parser
(
cls
)
:
parser
=
BackendCommandArgumentParser
(
cls
.
BACKEND
.
CATEGORIES
,
archive
=
True
)
# Required arguments
parser
.
parser
.
add_argument
(
'url'
,
help
=
"URL of the RSS feed"
)
return
parser


def fetch(self, category=CATEGORY_BUG, from_date=DEFAULT_DATETIME):
        """Fetch the bugs from the repository.

        The method retrieves, from a Bugzilla repository, the bugs
        updated since the given date.

        :param category: the category of items to fetch
        :param from_date: obtain bugs updated since this date

        :returns: a generator of bugs
        """
        if not from_date:
            from_date = DEFAULT_DATETIME

        kwargs = {'from_date': from_date}
        items = super().fetch(category, **kwargs)

        return items

def fetch ( self , category = CATEGORY_BUG , from_date = DEFAULT_DATETIME ) : if not from_date : from_date = DEFAULT_DATETIME kwargs = { 'from_date' : from_date } items = super ( ) . fetch ( category , * * kwargs ) return items


def bugs(self, from_date=DEFAULT_DATETIME, offset=None, max_bugs=MAX_BUGS):
        """Get the information of a list of bugs.

        :param from_date: retrieve bugs that where updated from that date;
            dates are converted to UTC
        :param offset: starting position for the search; i.e to return 11th
            element, set this value to 10.
        :param max_bugs: maximum number of bugs to reteurn per query
        """
        date = datetime_to_utc(from_date)
        date = date.strftime("%Y-%m-%dT%H:%M:%SZ")

        params = {
            self.PLAST_CHANGE_TIME: date,
            self.PLIMIT: max_bugs,
            self.PORDER: self.VCHANGE_DATE_ORDER,
            self.PINCLUDE_FIELDS: self.VINCLUDE_ALL
        }

        if offset:
            params[self.POFFSET] = offset

        response = self.call(self.RBUG, params)

        return response

def bugs ( self , from_date = DEFAULT_DATETIME , offset = None , max_bugs = MAX_BUGS ) : date = datetime_to_utc ( from_date ) date = date . strftime ( "%Y-%m-%dT%H:%M:%SZ" ) params = { self . PLAST_CHANGE_TIME : date , self . PLIMIT : max_bugs , self . PORDER : self . VCHANGE_DATE_ORDER , self . PINCLUDE_FIELDS : self . VINCLUDE_ALL } if offset : params [ self . POFFSET ] = offset response = self . call ( self . RBUG , params ) return response


def comments(self, *bug_ids):
        """Get the comments of the given bugs.

        :param bug_ids: list of bug identifiers
        """
        # Hack. The first value must be a valid bug id
        resource = urijoin(self.RBUG, bug_ids[0], self.RCOMMENT)

        params = {
            self.PIDS: bug_ids
        }

        response = self.call(resource, params)

        return response


def history(self, *bug_ids):
        """Get the history of the given bugs.

        :param bug_ids: list of bug identifiers
        """
        resource = urijoin(self.RBUG, bug_ids[0], self.RHISTORY)

        params = {
            self.PIDS: bug_ids
        }

        response = self.call(resource, params)

        return response

def history ( self , * bug_ids ) : resource = urijoin ( self . RBUG , bug_ids [ 0 ] , self . RHISTORY ) params = { self . PIDS : bug_ids } response = self . call ( resource , params ) return response


def attachments(self, *bug_ids):
        """Get the attachments of the given bugs.

        :param bug_id: list of bug identifiers
        """
        resource = urijoin(self.RBUG, bug_ids[0], self.RATTACHMENT)

        params = {
            self.PIDS: bug_ids,
            self.PEXCLUDE_FIELDS: self.VEXCLUDE_ATTCH_DATA
        }

        response = self.call(resource, params)

        return response

def attachments ( self , * bug_ids ) : resource = urijoin ( self . RBUG , bug_ids [ 0 ] , self . RATTACHMENT ) params = { self . PIDS : bug_ids , self . PEXCLUDE_FIELDS : self . VEXCLUDE_ATTCH_DATA } response = self . call ( resource , params ) return response


def __get_issue_notes(self, issue_id):
        """Get issue notes"""

        notes = []

        group_notes = self.client.notes(GitLabClient.ISSUES, issue_id)

        for raw_notes in group_notes:

            for note in json.loads(raw_notes):
                note_id = note['id']
                note['award_emoji_data'] = \
                    self.__get_note_award_emoji(GitLabClient.ISSUES, issue_id, note_id)
                notes.append(note)

        return notes

def __get_issue_notes ( self , issue_id ) : notes = [ ] group_notes = self . client . notes ( GitLabClient . ISSUES , issue_id ) for raw_notes in group_notes : for note in json . loads ( raw_notes ) : note_id = note [ 'id' ] note [ 'award_emoji_data' ] = self . __get_note_award_emoji ( GitLabClient . ISSUES , issue_id , note_id ) notes . append ( note ) return notes


def __fetch_merge_requests(self, from_date):
        """Fetch the merge requests"""

        merges_groups = self.client.merges(from_date=from_date)

        for raw_merges in merges_groups:
            merges = json.loads(raw_merges)
            for merge in merges:
                merge_id = merge['iid']

                if self.blacklist_ids and merge_id in self.blacklist_ids:
                    logger.warning("Skipping blacklisted merge request %s", merge_id)
                    continue

                # The single merge_request API call returns a more
                # complete merge request, thus we inflate it with
                # other data (e.g., notes, emojis, versions)
                merge_full_raw = self.client.merge(merge_id)
                merge_full = json.loads(merge_full_raw)

                self.__init_merge_extra_fields(merge_full)

                merge_full['notes_data'] = self.__get_merge_notes(merge_id)
                merge_full['award_emoji_data'] = self.__get_award_emoji(GitLabClient.MERGES, merge_id)
                merge_full['versions_data'] = self.__get_merge_versions(merge_id)

                yield merge_full


def __get_merge_notes(self, merge_id):
        """Get merge notes"""

        notes = []

        group_notes = self.client.notes(GitLabClient.MERGES, merge_id)

        for raw_notes in group_notes:
            for note in json.loads(raw_notes):
                note_id = note['id']
                note['award_emoji_data'] = \
                    self.__get_note_award_emoji(GitLabClient.MERGES, merge_id, note_id)
                notes.append(note)

        return notes

def __get_merge_notes ( self , merge_id ) : notes = [ ] group_notes = self . client . notes ( GitLabClient . MERGES , merge_id ) for raw_notes in group_notes : for note in json . loads ( raw_notes ) : note_id = note [ 'id' ] note [ 'award_emoji_data' ] = self . __get_note_award_emoji ( GitLabClient . MERGES , merge_id , note_id ) notes . append ( note ) return notes


def __get_merge_versions(self, merge_id):
        """Get merge versions"""

        versions = []

        group_versions = self.client.merge_versions(merge_id)

        for raw_versions in group_versions:
            for version in json.loads(raw_versions):
                version_id = version['id']
                version_full_raw = self.client.merge_version(merge_id, version_id)
                version_full = json.loads(version_full_raw)

                version_full.pop('diffs', None)
                versions.append(version_full)

        return versions

def __get_merge_versions ( self , merge_id ) : versions = [ ] group_versions = self . client . merge_versions ( merge_id ) for raw_versions in group_versions : for version in json . loads ( raw_versions ) : version_id = version [ 'id' ] version_full_raw = self . client . merge_version ( merge_id , version_id ) version_full = json . loads ( version_full_raw ) version_full . pop ( 'diffs' , None ) versions . append ( version_full ) return versions


def merges(self, from_date=None):
        """Get the merge requests from pagination"""

        payload = {
            'state': 'all',
            'order_by': 'updated_at',
            'sort': 'asc',
            'view': 'simple',
            'per_page': PER_PAGE
        }

        if from_date:
            payload['updated_after'] = from_date.isoformat()

        return self.fetch_items(GitLabClient.MERGES, payload)

def merges ( self , from_date = None ) : payload = { 'state' : 'all' , 'order_by' : 'updated_at' , 'sort' : 'asc' , 'view' : 'simple' , 'per_page' : PER_PAGE } if from_date : payload [ 'updated_after' ] = from_date . isoformat ( ) return self . fetch_items ( GitLabClient . MERGES , payload )


def merge(self, merge_id):
        """Get the merge full data"""

        path = urijoin(self.base_url,
                       GitLabClient.PROJECTS, self.owner + '%2F' + self.repository,
                       GitLabClient.MERGES, merge_id)

        response = self.fetch(path)

        return response.text

def merge ( self , merge_id ) : path = urijoin ( self . base_url , GitLabClient . PROJECTS , self . owner + '%2F' + self . repository , GitLabClient . MERGES , merge_id ) response = self . fetch ( path ) return response . text


def merge_versions(self, merge_id):
        """Get the merge versions from pagination"""

        payload = {
            'order_by': 'updated_at',
            'sort': 'asc',
            'per_page': PER_PAGE
        }

        path = urijoin(GitLabClient.MERGES, str(merge_id), GitLabClient.VERSIONS)
        return self.fetch_items(path, payload)

def merge_versions ( self , merge_id ) : payload = { 'order_by' : 'updated_at' , 'sort' : 'asc' , 'per_page' : PER_PAGE } path = urijoin ( GitLabClient . MERGES , str ( merge_id ) , GitLabClient . VERSIONS ) return self . fetch_items ( path , payload )


def merge_version(self, merge_id, version_id):
        """Get merge version detail"""

        path = urijoin(self.base_url,
                       GitLabClient.PROJECTS, self.owner + '%2F' + self.repository,
                       GitLabClient.MERGES, merge_id, GitLabClient.VERSIONS, version_id)

        response = self.fetch(path)

        return response.text

def merge_version ( self , merge_id , version_id ) : path = urijoin ( self . base_url , GitLabClient . PROJECTS , self . owner + '%2F' + self . repository , GitLabClient . MERGES , merge_id , GitLabClient . VERSIONS , version_id ) response = self . fetch ( path ) return response . text


def notes(self, item_type, item_id):
        """Get the notes from pagination"""

        payload = {
            'order_by': 'updated_at',
            'sort': 'asc',
            'per_page': PER_PAGE
        }

        path = urijoin(item_type, str(item_id), GitLabClient.NOTES)

        return self.fetch_items(path, payload)

def notes ( self , item_type , item_id ) : payload = { 'order_by' : 'updated_at' , 'sort' : 'asc' , 'per_page' : PER_PAGE } path = urijoin ( item_type , str ( item_id ) , GitLabClient . NOTES ) return self . fetch_items ( path , payload )


def emojis(self, item_type, item_id):
        """Get emojis from pagination"""

        payload = {
            'order_by': 'updated_at',
            'sort': 'asc',
            'per_page': PER_PAGE
        }

        path = urijoin(item_type, str(item_id), GitLabClient.EMOJI)

        return self.fetch_items(path, payload)

def emojis ( self , item_type , item_id ) : payload = { 'order_by' : 'updated_at' , 'sort' : 'asc' , 'per_page' : PER_PAGE } path = urijoin ( item_type , str ( item_id ) , GitLabClient . EMOJI ) return self . fetch_items ( path , payload )


def note_emojis(self, item_type, item_id, note_id):
        """Get emojis of a note"""

        payload = {
            'order_by': 'updated_at',
            'sort': 'asc',
            'per_page': PER_PAGE
        }

        path = urijoin(item_type, str(item_id), GitLabClient.NOTES,
                       str(note_id), GitLabClient.EMOJI)

        return self.fetch_items(path, payload)

def note_emojis ( self , item_type , item_id , note_id ) : payload = { 'order_by' : 'updated_at' , 'sort' : 'asc' , 'per_page' : PER_PAGE } path = urijoin ( item_type , str ( item_id ) , GitLabClient . NOTES , str ( note_id ) , GitLabClient . EMOJI ) return self . fetch_items ( path , payload )


def calculate_time_to_reset(self):
        """Calculate the seconds to reset the token requests, by obtaining the different
        between the current date and the next date when the token is fully regenerated.
        """

        time_to_reset = self.rate_limit_reset_ts - (datetime_utcnow().replace(microsecond=0).timestamp() + 1)

        if time_to_reset < 0:
            time_to_reset = 0

        return time_to_reset

def calculate_time_to_reset ( self ) : time_to_reset = self . rate_limit_reset_ts - ( datetime_utcnow ( ) . replace ( microsecond = 0 ) . timestamp ( ) + 1 ) if time_to_reset < 0 : time_to_reset = 0 return time_to_reset

Calculate the seconds to reset the token requests by obtaining the different between the current date and the next date when the token is fully regenerated .


def fetch_items(self, path, payload):
        """Return the items from GitLab API using links pagination"""

        page = 0  # current page
        last_page = None  # last page
        url_next = urijoin(self.base_url, GitLabClient.PROJECTS, self.owner + '%2F' + self.repository, path)

        logger.debug("Get GitLab paginated items from " + url_next)

        response = self.fetch(url_next, payload=payload)

        items = response.text
        page += 1

        if 'last' in response.links:
            last_url = response.links['last']['url']
            last_page = last_url.split('&page=')[1].split('&')[0]
            last_page = int(last_page)
            logger.debug("Page: %i/%i" % (page, last_page))

        while items:
            yield items

            items = None

            if 'next' in response.links:
                url_next = response.links['next']['url']  # Loving requests :)
                response = self.fetch(url_next, payload=payload)
                page += 1

                items = response.text
                logger.debug("Page: %i/%i" % (page, last_page))

def
fetch_items
(
self
,
path
,
payload
)
:
page
=
0
# current page
last_page
=
# last page
url_next
=
urijoin
(
self
.
base_url
,
GitLabClient
.
PROJECTS
,
self
.
owner
+
'%2F'
+
self
.
repository
,
path
)
logger
.
debug
(
"Get GitLab paginated items from "
+
url_next
)
response
=
self
.
fetch
(
url_next
,
payload
=
payload
)
items
=
response
.
text
page
+=
1
if
'last'
in
response
.
links
:
last_url
=
response
.
links
[
'last'
]
[
'url'
]
last_page
=
last_url
.
split
(
'&page='
)
[
1
]
.
split
(
'&'
)
[
0
]
last_page
=
int
(
last_page
)
logger
.
debug
(
"Page: %i/%i"
%
(
page
,
last_page
)
)
while
items
:
yield
items
items
=
if
'next'
in
response
.
links
:
url_next
=
response
.
links
[
'next'
]
[
'url'
]
# Loving requests :)
response
=
self
.
fetch
(
url_next
,
payload
=
payload
)
page
+=
1
items
=
response
.
text
logger
.
debug
(
"Page: %i/%i"
%
(
page
,
last_page
)
)


def _init_rate_limit(self):
        """Initialize rate limit information"""

        url = urijoin(self.base_url, 'projects', self.owner + '%2F' + self.repository)
        try:
            response = super().fetch(url)
            self.update_rate_limit(response)
        except requests.exceptions.HTTPError as error:
            if error.response.status_code == 401:
                raise error
            else:
                logger.warning("Rate limit not initialized: %s", error)

def
_init_rate_limit
(
self
)
:
url
=
urijoin
(
self
.
base_url
,
'projects'
,
self
.
owner
+
'%2F'
+
self
.
repository
)
try
:
response
=
super
(
)
.
fetch
(
url
)
self
.
update_rate_limit
(
response
)
except
requests
.
exceptions
.
HTTPError
as
error
:
if
error
.
response
.
status_code
==
401
:
raise
error
else
:
logger
.
warning
(
"Rate limit not initialized: %s"
,
error
)


def setup_cmd_parser(cls):
        """Returns the GitLab argument parser."""

        parser = BackendCommandArgumentParser(cls.BACKEND.CATEGORIES,
                                              from_date=True,
                                              token_auth=True,
                                              archive=True)

        # GitLab options
        group = parser.parser.add_argument_group('GitLab arguments')
        group.add_argument('--enterprise-url', dest='base_url',
                           help="Base URL for GitLab Enterprise instance")
        group.add_argument('--sleep-for-rate', dest='sleep_for_rate',
                           action='store_true',
                           help="sleep for getting more rate")
        group.add_argument('--min-rate-to-sleep', dest='min_rate_to_sleep',
                           default=MIN_RATE_LIMIT, type=int,
                           help="sleep until reset when the rate limit \
                               reaches this value")
        group.add_argument('--blacklist-ids', dest='blacklist_ids',
                           nargs='*', type=int,
                           help="Ids of items that must not be retrieved.")

        # Generic client options
        group.add_argument('--max-retries', dest='max_retries',
                           default=MAX_RETRIES, type=int,
                           help="number of API call retries")
        group.add_argument('--sleep-time', dest='sleep_time',
                           default=DEFAULT_SLEEP_TIME, type=int,
                           help="sleeping time between API call retries")

        # Positional arguments
        parser.parser.add_argument('owner',
                                   help="GitLab owner")
        parser.parser.add_argument('repository',
                                   help="GitLab repository")

        return parser


def fetch(self, category=CATEGORY_MESSAGE, from_date=DEFAULT_DATETIME):
        """Fetch the messages from the channel.

        This method fetches the messages stored on the channel that were
        sent since the given date.

        :param category: the category of items to fetch
        :param from_date: obtain messages sent since this date

        :returns: a generator of messages
        """
        if not from_date:
            from_date = DEFAULT_DATETIME

        from_date = datetime_to_utc(from_date)
        latest = datetime_utcnow().timestamp()

        kwargs = {'from_date': from_date, 'latest': latest}
        items = super().fetch(category, **kwargs)

        return items

def fetch ( self , category = CATEGORY_MESSAGE , from_date = DEFAULT_DATETIME ) : if not from_date : from_date = DEFAULT_DATETIME from_date = datetime_to_utc ( from_date ) latest = datetime_utcnow ( ) . timestamp ( ) kwargs = { 'from_date' : from_date , 'latest' : latest } items = super ( ) . fetch ( category , * * kwargs ) return items


def metadata_id(item):
        """Extracts the identifier from a Slack item.

        This identifier will be the mix of two fields because Slack
        messages does not have any unique identifier. In this case,
        'ts' and 'user' values (or 'bot_id' when the message is sent by a bot)
        are combined because there have been cases where two messages were sent
        by different users at the same time.
        """
        if 'user' in item:
            nick = item['user']
        elif 'comment' in item:
            nick = item['comment']['user']
        else:
            nick = item['bot_id']

        return item['ts'] + nick

def metadata_id ( item ) : if 'user' in item : nick = item [ 'user' ] elif 'comment' in item : nick = item [ 'comment' ] [ 'user' ] else : nick = item [ 'bot_id' ] return item [ 'ts' ] + nick

Extracts the identifier from a Slack item .


def conversation_members(self, conversation):
        """Fetch the number of members in a conversation, which is a supertype for public and
        private ones, DM and group DM.

        :param conversation: the ID of the conversation
        """
        members = 0

        resource = self.RCONVERSATION_INFO

        params = {
            self.PCHANNEL: conversation,
        }

        raw_response = self._fetch(resource, params)
        response = json.loads(raw_response)

        members += len(response["members"])
        while 'next_cursor' in response['response_metadata'] and response['response_metadata']['next_cursor']:
            params['cursor'] = response['response_metadata']['next_cursor']
            raw_response = self._fetch(resource, params)
            response = json.loads(raw_response)
            members += len(response["members"])

        return members

def conversation_members ( self , conversation ) : members = 0 resource = self . RCONVERSATION_INFO params = { self . PCHANNEL : conversation , } raw_response = self . _fetch ( resource , params ) response = json . loads ( raw_response ) members += len ( response [ "members" ] ) while 'next_cursor' in response [ 'response_metadata' ] and response [ 'response_metadata' ] [ 'next_cursor' ] : params [ 'cursor' ] = response [ 'response_metadata' ] [ 'next_cursor' ] raw_response = self . _fetch ( resource , params ) response = json . loads ( raw_response ) members += len ( response [ "members" ] ) return members

Fetch the number of members in a conversation which is a supertype for public and private ones DM and group DM .


def channel_info(self, channel):
        """Fetch information about a channel."""

        resource = self.RCHANNEL_INFO

        params = {
            self.PCHANNEL: channel,
        }

        response = self._fetch(resource, params)

        return response

def channel_info ( self , channel ) : resource = self . RCHANNEL_INFO params = { self . PCHANNEL : channel , } response = self . _fetch ( resource , params ) return response


def user(self, user_id):
        """Fetch user info."""

        resource = self.RUSER_INFO

        params = {
            self.PUSER: user_id
        }

        response = self._fetch(resource, params)

        return response

def user ( self , user_id ) : resource = self . RUSER_INFO params = { self . PUSER : user_id } response = self . _fetch ( resource , params ) return response


def setup_cmd_parser(cls):
        """Returns the Slack argument parser."""

        parser = BackendCommandArgumentParser(cls.BACKEND.CATEGORIES,
                                              from_date=True,
                                              token_auth=True,
                                              archive=True)

        # Backend token is required
        action = parser.parser._option_string_actions['--api-token']
        action.required = True

        # Slack options
        group = parser.parser.add_argument_group('Slack arguments')
        group.add_argument('--max-items', dest='max_items',
                           type=int, default=MAX_ITEMS,
                           help="Maximum number of items requested on the same query")

        # Required arguments
        parser.parser.add_argument('channel',
                                   help="Slack channel identifier")

        return parser


def metadata_updated_on(item):
        """Extracts and coverts the update time from a Bugzilla item.

        The timestamp is extracted from 'delta_ts' field. This date is
        converted to UNIX timestamp format. Due Bugzilla servers ignore
        the timezone on HTTP requests, it will be ignored during the
        conversion, too.

        :param item: item generated by the backend

        :returns: a UNIX timestamp
        """
        ts = item['delta_ts'][0]['__text__']
        ts = str_to_datetime(ts)
        ts = ts.replace(tzinfo=dateutil.tz.tzutc())

        return ts.timestamp()

def metadata_updated_on ( item ) : ts = item [ 'delta_ts' ] [ 0 ] [ '__text__' ] ts = str_to_datetime ( ts ) ts = ts . replace ( tzinfo = dateutil . tz . tzutc ( ) ) return ts . timestamp ( )

Extracts and coverts the update time from a Bugzilla item .


def parse_buglist(raw_csv):
        """Parse a Bugzilla CSV bug list.

        The method parses the CSV file and returns an iterator of
        dictionaries. Each one of this, contains the summary of a bug.

        :param raw_csv: CSV string to parse

        :returns: a generator of parsed bugs
        """
        reader = csv.DictReader(raw_csv.split('\n'),
                                delimiter=',', quotechar='"')
        for row in reader:
            yield row

def parse_buglist ( raw_csv ) : reader = csv . DictReader ( raw_csv . split ( '\n' ) , delimiter = ',' , quotechar = '"' ) for row in reader : yield row


def parse_bugs_details(raw_xml):
        """Parse a Bugilla bugs details XML stream.

        This method returns a generator which parses the given XML,
        producing an iterator of dictionaries. Each dictionary stores
        the information related to a parsed bug.

        If the given XML is invalid or does not contains any bug, the
        method will raise a ParseError exception.

        :param raw_xml: XML string to parse

        :returns: a generator of parsed bugs

        :raises ParseError: raised when an error occurs parsing
            the given XML stream
        """
        bugs = xml_to_dict(raw_xml)

        if 'bug' not in bugs:
            cause = "No bugs found. XML stream seems to be invalid."
            raise ParseError(cause=cause)

        for bug in bugs['bug']:
            yield bug

Parse a Bugilla bugs details XML stream .


def parse_bug_activity(raw_html):
        """Parse a Bugzilla bug activity HTML stream.

        This method extracts the information about activity from the
        given HTML stream. The bug activity is stored into a HTML
        table. Each parsed activity event is returned into a dictionary.

        If the given HTML is invalid, the method will raise a ParseError
        exception.

        :param raw_html: HTML string to parse

        :returns: a generator of parsed activity events

        :raises ParseError: raised when an error occurs parsing
            the given HTML stream
        """
        def is_activity_empty(bs):
            EMPTY_ACTIVITY = "No changes have been made to this (?:bug|issue) yet."
            tag = bs.find(text=re.compile(EMPTY_ACTIVITY))
            return tag is not None

        def find_activity_table(bs):
            # The first table with 5 columns is the table of activity
            tables = bs.find_all('table')

            for tb in tables:
                nheaders = len(tb.tr.find_all('th', recursive=False))
                if nheaders == 5:
                    return tb
            raise ParseError(cause="Table of bug activity not found.")

        def remove_tags(bs):
            HTML_TAGS_TO_REMOVE = ['a', 'i', 'span']

            for tag in bs.find_all(HTML_TAGS_TO_REMOVE):
                tag.replaceWith(tag.text)

        def format_text(bs):
            strings = [s.strip(' \n\t') for s in bs.stripped_strings]
            s = ' '.join(strings)
            return s

        # Parsing starts here
        bs = bs4.BeautifulSoup(raw_html, 'html.parser')

        if is_activity_empty(bs):
            fields = []
        else:
            activity_tb = find_activity_table(bs)
            remove_tags(activity_tb)
            fields = activity_tb.find_all('td')

        while fields:
            # First two fields: 'Who' and 'When'.
            who = fields.pop(0)
            when = fields.pop(0)

            # The attribute 'rowspan' of 'who' field tells how many
            # changes were made on the same date.
            n = int(who.get('rowspan'))

            # Next fields are split into chunks of three elements:
            # 'What', 'Removed' and 'Added'. These chunks share
            # 'Who' and 'When' values.
            for _ in range(n):
                what = fields.pop(0)
                removed = fields.pop(0)
                added = fields.pop(0)
                event = {'Who': format_text(who),
                         'When': format_text(when),
                         'What': format_text(what),
                         'Removed': format_text(removed),
                         'Added': format_text(added)}
                yield event

Parse a Bugzilla bug activity HTML stream .


def logout(self):
        """Logout from the server."""

        params = {
            self.PLOGOUT: '1'
        }

        self.call(self.CGI_LOGIN, params)
        self._close_http_session()

        logger.debug("Bugzilla user logged out from %s",
                     self.base_url)

def
logout
(
self
)
:
params
=
{
self
.
PLOGOUT
:
'1'
}
self
.
call
(
self
.
CGI_LOGIN
,
params
)
self
.
_close_http_session
(
)
logger
.
debug
(
"Bugzilla user logged out from %s"
,
self
.
base_url
)


def metadata(self):
        """Get metadata information in XML format."""

        params = {
            self.PCTYPE: self.CTYPE_XML
        }

        response = self.call(self.CGI_BUG, params)

        return response

def metadata ( self ) : params = { self . PCTYPE : self . CTYPE_XML } response = self . call ( self . CGI_BUG , params ) return response

Get metadata information in XML format .


def buglist(self, from_date=DEFAULT_DATETIME):
        """Get a summary of bugs in CSV format.

        :param from_date: retrieve bugs that where updated from that date
        """
        if not self.version:
            self.version = self.__fetch_version()

        if self.version in self.OLD_STYLE_VERSIONS:
            order = 'Last+Changed'
        else:
            order = 'changeddate'

        date = from_date.strftime("%Y-%m-%d %H:%M:%S")

        params = {
            self.PCHFIELD_FROM: date,
            self.PCTYPE: self.CTYPE_CSV,
            self.PLIMIT: self.max_bugs_csv,
            self.PORDER: order
        }

        response = self.call(self.CGI_BUGLIST, params)

        return response

def
buglist
(
self
,
from_date
=
DEFAULT_DATETIME
)
:
if
not
self
.
version
:
self
.
version
=
self
.
__fetch_version
(
)
if
self
.
version
in
self
.
OLD_STYLE_VERSIONS
:
order
=
'Last+Changed'
else
:
order
=
'changeddate'
date
=
from_date
.
strftime
(
"%Y-%m-%d %H:%M:%S"
)
params
=
{
self
.
PCHFIELD_FROM
:
date
,
self
.
PCTYPE
:
self
.
CTYPE_CSV
,
self
.
PLIMIT
:
self
.
max_bugs_csv
,
self
.
PORDER
:
order
}
response
=
self
.
call
(
self
.
CGI_BUGLIST
,
params
)
return
response


def bugs(self, *bug_ids):
        """Get the information of a list of bugs in XML format.

        :param bug_ids: list of bug identifiers
        """
        params = {
            self.PBUG_ID: bug_ids,
            self.PCTYPE: self.CTYPE_XML,
            self.PEXCLUDE_FIELD: 'attachmentdata'
        }

        response = self.call(self.CGI_BUG, params)

        return response

def bugs ( self , * bug_ids ) : params = { self . PBUG_ID : bug_ids , self . PCTYPE : self . CTYPE_XML , self . PEXCLUDE_FIELD : 'attachmentdata' } response = self . call ( self . CGI_BUG , params ) return response

Get the information of a list of bugs in XML format .


def bug_activity(self, bug_id):
        """Get the activity of a bug in HTML format.

        :param bug_id: bug identifier
        """
        params = {
            self.PBUG_ID: bug_id
        }

        response = self.call(self.CGI_BUG_ACTIVITY, params)

        return response

def bug_activity ( self , bug_id ) : params = { self . PBUG_ID : bug_id } response = self . call ( self . CGI_BUG_ACTIVITY , params ) return response

Get the activity of a bug in HTML format .


def fetch(self, category=CATEGORY_EVENT, from_date=DEFAULT_DATETIME, to_date=None,
              filter_classified=False):
        """Fetch the events from the server.

        This method fetches those events of a group stored on the server
        that were updated since the given date. Data comments and rsvps
        are included within each event.

        :param category: the category of items to fetch
        :param from_date: obtain events updated since this date
        :param to_date: obtain events updated before this date
        :param filter_classified: remove classified fields from the resulting items

        :returns: a generator of events
        """
        if not from_date:
            from_date = DEFAULT_DATETIME

        from_date = datetime_to_utc(from_date)

        kwargs = {"from_date": from_date, "to_date": to_date}
        items = super().fetch(category,
                              filter_classified=filter_classified,
                              **kwargs)

        return items

def fetch ( self , category = CATEGORY_EVENT , from_date = DEFAULT_DATETIME , to_date = None , filter_classified = False ) : if not from_date : from_date = DEFAULT_DATETIME from_date = datetime_to_utc ( from_date ) kwargs = { "from_date" : from_date , "to_date" : to_date } items = super ( ) . fetch ( category , filter_classified = filter_classified , * * kwargs ) return items


def fetch_items(self, category, **kwargs):
        """Fetch the events

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items
        """
        from_date = kwargs['from_date']
        to_date = kwargs['to_date']

        logger.info("Fetching events of '%s' group from %s to %s",
                    self.group, str(from_date),
                    str(to_date) if to_date else '--')

        to_date_ts = datetime_to_utc(to_date).timestamp() if to_date else None

        nevents = 0
        stop_fetching = False

        ev_pages = self.client.events(self.group, from_date=from_date)

        for evp in ev_pages:
            events = [event for event in self.parse_json(evp)]

            for event in events:
                event_id = event['id']

                event['comments'] = self.__fetch_and_parse_comments(event_id)
                event['rsvps'] = self.__fetch_and_parse_rsvps(event_id)

                # Check events updated before 'to_date'
                event_ts = self.metadata_updated_on(event)

                if to_date_ts and event_ts >= to_date_ts:
                    stop_fetching = True
                    continue

                yield event
                nevents += 1

            if stop_fetching:
                break

        logger.info("Fetch process completed: %s events fetched", nevents)


def events(self, group, from_date=DEFAULT_DATETIME):
        """Fetch the events pages of a given group."""

        date = datetime_to_utc(from_date)
        date = date.strftime("since:%Y-%m-%dT%H:%M:%S.000Z")

        resource = urijoin(group, self.REVENTS)

        # Hack required due to Metup API does not support list
        # values with the format `?param=value1&param=value2`.
        # It only works with `?param=value1,value2`.
        # Morever, urrlib3 encodes comma characters when values
        # are given using params dict, which it doesn't work
        # with Meetup, either.
        fixed_params = '?' + self.PFIELDS + '=' + ','.join(self.VEVENT_FIELDS)
        fixed_params += '&' + self.PSTATUS + '=' + ','.join(self.VSTATUS)
        resource += fixed_params

        params = {
            self.PORDER: self.VUPDATED,
            self.PSCROLL: date,
            self.PPAGE: self.max_items
        }

        try:
            for page in self._fetch(resource, params):
                yield page
        except requests.exceptions.HTTPError as error:
            if error.response.status_code == 410:
                msg = "Group is no longer accessible: {}".format(error)
                raise RepositoryError(cause=msg)
            else:
                raise error

Fetch the events pages of a given group .


def comments(self, group, event_id):
        """Fetch the comments of a given event."""

        resource = urijoin(group, self.REVENTS, event_id, self.RCOMMENTS)

        params = {
            self.PPAGE: self.max_items
        }

        for page in self._fetch(resource, params):
            yield page

def comments ( self , group , event_id ) : resource = urijoin ( group , self . REVENTS , event_id , self . RCOMMENTS ) params = { self . PPAGE : self . max_items } for page in self . _fetch ( resource , params ) : yield page


def rsvps(self, group, event_id):
        """Fetch the rsvps of a given event."""

        resource = urijoin(group, self.REVENTS, event_id, self.RRSVPS)

        # Same hack that in 'events' method
        fixed_params = '?' + self.PFIELDS + '=' + ','.join(self.VRSVP_FIELDS)
        fixed_params += '&' + self.PRESPONSE + '=' + ','.join(self.VRESPONSE)
        resource += fixed_params

        params = {
            self.PPAGE: self.max_items
        }

        for page in self._fetch(resource, params):
            yield page

def
rsvps
(
self
,
group
,
event_id
)
:
resource
=
urijoin
(
group
,
self
.
REVENTS
,
event_id
,
self
.
RRSVPS
)
# Same hack that in 'events' method
fixed_params
=
'?'
+
self
.
PFIELDS
+
'='
+
','
.
join
(
self
.
VRSVP_FIELDS
)
fixed_params
+=
'&'
+
self
.
PRESPONSE
+
'='
+
','
.
join
(
self
.
VRESPONSE
)
resource
+=
fixed_params
params
=
{
self
.
PPAGE
:
self
.
max_items
}
for
page
in
self
.
_fetch
(
resource
,
params
)
:
yield
page


def __fetch_question(self, question):
        """Fetch an Askbot HTML question body.

        The method fetchs the HTML question retrieving the
        question body of the item question received

        :param question: item with the question itself

        :returns: a list of HTML page/s for the question
        """
        html_question_items = []

        npages = 1
        next_request = True

        while next_request:
            try:
                html_question = self.client.get_html_question(question['id'], npages)
                html_question_items.append(html_question)
                tpages = self.ab_parser.parse_number_of_html_pages(html_question)

                if npages == tpages:
                    next_request = False

                npages = npages + 1
            except requests.exceptions.TooManyRedirects as e:
                logger.warning("%s, data not retrieved for question %s", e, question['id'])
                next_request = False

        return html_question_items


def __fetch_comments(self, question):
        """Fetch all the comments of an Askbot question and answers.

        The method fetchs the list of every comment existing in a question and
        its answers.

        :param question: item with the question itself

        :returns: a list of comments with the ids as hashes
        """
        comments = {}
        comments[question['id']] = json.loads(self.client.get_comments(question['id']))
        for object_id in question['answer_ids']:
            comments[object_id] = json.loads(self.client.get_comments(object_id))
        return comments

def __fetch_comments ( self , question ) : comments = { } comments [ question [ 'id' ] ] = json . loads ( self . client . get_comments ( question [ 'id' ] ) ) for object_id in question [ 'answer_ids' ] : comments [ object_id ] = json . loads ( self . client . get_comments ( object_id ) ) return comments

Fetch all the comments of an Askbot question and answers .


def __build_question(html_question, question, comments):
        """Build an Askbot HTML response.

        The method puts together all the information regarding a question

        :param html_question: array of HTML raw pages
        :param question: question object from the API
        :param comments: list of comments to add

        :returns: a dict item with the parsed question information
        """
        question_object = {}
        # Parse the user info from the soup container
        question_container = AskbotParser.parse_question_container(html_question[0])
        # Add the info to the question object
        question_object.update(question_container)
        # Add the comments of the question (if any)
        if comments[int(question['id'])]:
            question_object['comments'] = comments[int(question['id'])]

        answers = []

        for page in html_question:
            answers.extend(AskbotParser.parse_answers(page))

        if len(answers) != 0:
            question_object['answers'] = answers
            for answer in question_object['answers']:
                if comments[int(answer['id'])]:
                    answer['comments'] = comments[int(answer['id'])]

        return question_object


def get_api_questions(self, path):
        """Retrieve a question page using the API.

        :param page: page to retrieve
        """
        npages = 1
        next_request = True

        path = urijoin(self.base_url, path)
        while next_request:

            try:
                params = {
                    'page': npages,
                    'sort': self.ORDER_API
                }

                response = self.fetch(path, payload=params)

                whole_page = response.text

                raw_questions = json.loads(whole_page)
                tpages = raw_questions['pages']

                logger.debug("Fetching questions from '%s': page %s/%s",
                             self.base_url, npages, tpages)

                if npages == tpages:
                    next_request = False

                npages = npages + 1
                yield raw_questions

            except requests.exceptions.TooManyRedirects as e:
                logger.warning("%s, data not retrieved for resource %s", e, path)
                next_request = False

Retrieve a question page using the API .


def get_html_question(self, question_id, page=1):
        """Retrieve a raw HTML question and all it's information.

        :param question_id: question identifier
        :param page: page to retrieve
        """
        path = urijoin(self.base_url, self.HTML_QUESTION, question_id)
        params = {
            'page': page,
            'sort': self.ORDER_HTML
        }

        response = self.fetch(path, payload=params)
        return response.text

def get_html_question ( self , question_id , page = 1 ) : path = urijoin ( self . base_url , self . HTML_QUESTION , question_id ) params = { 'page' : page , 'sort' : self . ORDER_HTML } response = self . fetch ( path , payload = params ) return response . text

Retrieve a raw HTML question and all it s information .


def get_comments(self, post_id):
        """Retrieve a list of comments by a given id.

        :param object_id: object identifiere
        """
        path = urijoin(self.base_url, self.COMMENTS if self._use_new_urls else self.COMMENTS_OLD)
        params = {
            'post_id': post_id,
            'post_type': 'answer',
            'avatar_size': 0
        }
        headers = {'X-Requested-With': 'XMLHttpRequest'}

        try:
            response = self.fetch(path, payload=params, headers=headers)
            raw = response.text
        except requests.exceptions.HTTPError as ex:
            if ex.response.status_code == 404:
                logger.debug("Comments URL did not work. Using old URL schema.")
                self._use_new_urls = False
                path = urijoin(self.base_url, self.COMMENTS_OLD)
                response = self.fetch(path, payload=params, headers=headers)
                raw = response.text
            elif ex.response.status_code == 500:
                logger.warning("Comments not retrieved due to %s", ex)
                raw = '[]'
            else:
                raise ex

        return raw

Retrieve a list of comments by a given id .


def parse_question_container(html_question):
        """Parse the question info container of a given HTML question.

        The method parses the information available in the question information
        container. The container can have up to 2 elements: the first one
        contains the information related with the user who generated the question
        and the date (if any). The second one contains the date of the updated,
        and the user who updated it (if not the same who generated the question).

        :param html_question: raw HTML question element

        :returns: an object with the parsed information
        """
        container_info = {}
        bs_question = bs4.BeautifulSoup(html_question, "html.parser")
        question = AskbotParser._find_question_container(bs_question)
        container = question.select("div.post-update-info")
        created = container[0]
        container_info['author'] = AskbotParser.parse_user_info(created)
        try:
            container[1]
        except IndexError:
            pass
        else:
            updated = container[1]
            if AskbotParser.parse_user_info(updated):
                container_info['updated_by'] = AskbotParser.parse_user_info(updated)

        return container_info

def parse_question_container ( html_question ) : container_info = { } bs_question = bs4 . BeautifulSoup ( html_question , "html.parser" ) question = AskbotParser . _find_question_container ( bs_question ) container = question . select ( "div.post-update-info" ) created = container [ 0 ] container_info [ 'author' ] = AskbotParser . parse_user_info ( created ) try : container [ 1 ] except IndexError : pass else : updated = container [ 1 ] if AskbotParser . parse_user_info ( updated ) : container_info [ 'updated_by' ] = AskbotParser . parse_user_info ( updated ) return container_info

Parse the question info container of a given HTML question .


def parse_answers(html_question):
        """Parse the answers of a given HTML question.

        The method parses the answers related with a given HTML question,
        as well as all the comments related to the answer.

        :param html_question: raw HTML question element

        :returns: a list with the answers
        """

        def parse_answer_container(update_info):
            """Parse the answer info container of a given HTML question.

            The method parses the information available in the answer information
            container. The container can have up to 2 elements: the first one
            contains the information related with the user who generated the question
            and the date (if any). The second one contains the date of the updated,
            and the user who updated it (if not the same who generated the question).

            :param update_info: beautiful soup update_info container element

            :returns: an object with the parsed information
            """
            container_info = {}
            created = update_info[0]
            answered_at = created.abbr.attrs["title"]
            # Convert date to UNIX timestamp
            container_info['added_at'] = str(str_to_datetime(answered_at).timestamp())
            container_info['answered_by'] = AskbotParser.parse_user_info(created)
            try:
                update_info[1]
            except IndexError:
                pass
            else:
                updated = update_info[1]
                updated_at = updated.abbr.attrs["title"]
                # Convert date to UNIX timestamp
                container_info['updated_at'] = str(str_to_datetime(updated_at).timestamp())
                if AskbotParser.parse_user_info(updated):
                    container_info['updated_by'] = AskbotParser.parse_user_info(updated)
            return container_info

        answer_list = []
        # Select all the answers
        bs_question = bs4.BeautifulSoup(html_question, "html.parser")
        bs_answers = bs_question.select("div.answer")
        for bs_answer in bs_answers:
            answer_id = bs_answer.attrs["data-post-id"]
            votes_element = bs_answer.select("div.vote-number")[0].text
            accepted_answer = bs_answer.select("div.answer-img-accept")[0].get('title').endswith("correct")
            # Select the body of the answer
            body = bs_answer.select("div.post-body")
            # Get the user information container and parse it
            update_info = body[0].select("div.post-update-info")
            answer_container = parse_answer_container(update_info)
            # Remove the update-info-container div to be able to get the body
            body[0].div.extract().select("div.post-update-info-container")
            # Override the body with a clean one
            body = body[0].get_text(strip=True)
            # Generate the answer object
            answer = {'id': answer_id,
                      'score': votes_element,
                      'summary': body,
                      'accepted': accepted_answer
                      }
            # Update the object with the information in the answer container
            answer.update(answer_container)
            answer_list.append(answer)
        return answer_list

Parse the answers of a given HTML question .


def parse_number_of_html_pages(html_question):
        """Parse number of answer pages to paginate over them.

        :param html_question: raw HTML question element

        :returns: an integer with the number of pages
        """
        bs_question = bs4.BeautifulSoup(html_question, "html.parser")
        try:
            bs_question.select('div.paginator')[0]
        except IndexError:
            return 1
        else:
            return int(bs_question.select('div.paginator')[0].attrs['data-num-pages'])

def parse_number_of_html_pages ( html_question ) : bs_question = bs4 . BeautifulSoup ( html_question , "html.parser" ) try : bs_question . select ( 'div.paginator' ) [ 0 ] except IndexError : return 1 else : return int ( bs_question . select ( 'div.paginator' ) [ 0 ] . attrs [ 'data-num-pages' ] )

Parse number of answer pages to paginate over them .


def parse_user_info(update_info):
        """Parse the user information of a given HTML container.

        The method parses all the available user information in the container.
        If the class "user-info" exists, the method will get all the available
        information in the container. If not, if a class "tip" exists, it will be
        a wiki post with no user associated. Else, it can be an empty container.

        :param update_info: beautiful soup answer container element

        :returns: an object with the parsed information
        """
        user_info = {}
        if update_info.select("div.user-info"):
            # Get all the <a> elements in the container. First <a> contains the user
            # information, second one (if exists), the website of the user.
            elements = update_info.select("div.user-info")[0].find_all("a")
            href = elements[0].attrs["href"]
            user_info['id'] = re.search(r'\d+', href).group(0)
            user_info['username'] = elements[0].text
            user_info['reputation'] = update_info.select('span.reputation-score')[0].text
            user_info['badges'] = update_info.select("span.badges")[0].attrs["title"]
            try:
                elements[1]
            except IndexError:
                pass
            else:
                user_info['website'] = elements[1].attrs["href"]
            if update_info.select("img.flag"):
                flag = update_info.select("img.flag")[0].attrs["alt"]
                user_info['country'] = re.sub("flag of ", "", flag)

        return user_info

Parse the user information of a given HTML container .


def fetch_items(self, category, **kwargs):
        """Fetch the reviews

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items
        """
        from_date = kwargs['from_date']

        if self.client.version[0] == 2 and self.client.version[1] == 8:
            fetcher = self._fetch_gerrit28(from_date)
        else:
            fetcher = self._fetch_gerrit(from_date)

        for review in fetcher:
            yield review

def fetch_items ( self , category , * * kwargs ) : from_date = kwargs [ 'from_date' ] if self . client . version [ 0 ] == 2 and self . client . version [ 1 ] == 8 : fetcher = self . _fetch_gerrit28 ( from_date ) else : fetcher = self . _fetch_gerrit ( from_date ) for review in fetcher : yield review


def parse_reviews(raw_data):
        """Parse a Gerrit reviews list."""

        # Join isolated reviews in JSON in array for parsing
        items_raw = "[" + raw_data.replace("\n", ",") + "]"
        items_raw = items_raw.replace(",]", "]")
        items = json.loads(items_raw)
        reviews = []

        for item in items:
            if 'project' in item.keys():
                reviews.append(item)

        return reviews


def _fetch_gerrit28(self, from_date=DEFAULT_DATETIME):
        """ Specific fetch for gerrit 2.8 version.

        Get open and closed reviews in different queries.
        Take the newer review from both lists and iterate.
        """

        # Convert date to Unix time
        from_ut = datetime_to_utc(from_date)
        from_ut = from_ut.timestamp()

        filter_open = "status:open"
        filter_closed = "status:closed"

        last_item_open = self.client.next_retrieve_group_item()
        last_item_closed = self.client.next_retrieve_group_item()
        reviews_open = self._get_reviews(last_item_open, filter_open)
        reviews_closed = self._get_reviews(last_item_closed, filter_closed)
        last_nreviews_open = len(reviews_open)
        last_nreviews_closed = len(reviews_closed)

        while reviews_open or reviews_closed:
            if reviews_open and reviews_closed:
                if reviews_open[0]['lastUpdated'] >= reviews_closed[0]['lastUpdated']:
                    review_open = reviews_open.pop(0)
                    review = review_open
                else:
                    review_closed = reviews_closed.pop(0)
                    review = review_closed
            elif reviews_closed:
                review_closed = reviews_closed.pop(0)
                review = review_closed
            else:
                review_open = reviews_open.pop(0)
                review = review_open

            updated = review['lastUpdated']
            if updated <= from_ut:
                logger.debug("No more updates for %s" % (self.hostname))
                break
            else:
                yield review

            if not reviews_open and last_nreviews_open >= self.max_reviews:
                last_item_open = self.client.next_retrieve_group_item(last_item_open, review_open)
                reviews_open = self._get_reviews(last_item_open, filter_open)
                last_nreviews_open = len(reviews_open)
            if not reviews_closed and last_nreviews_closed >= self.max_reviews:
                last_item_closed = self.client.next_retrieve_group_item(last_item_closed, review_closed)
                reviews_closed = self._get_reviews(last_item_closed, filter_closed)
                last_nreviews_closed = len(reviews_closed)

def
_fetch_gerrit28
(
self
,
from_date
=
DEFAULT_DATETIME
)
:
# Convert date to Unix time
from_ut
=
datetime_to_utc
(
from_date
)
from_ut
=
from_ut
.
timestamp
(
)
filter_open
=
"status:open"
filter_closed
=
"status:closed"
last_item_open
=
self
.
client
.
next_retrieve_group_item
(
)
last_item_closed
=
self
.
client
.
next_retrieve_group_item
(
)
reviews_open
=
self
.
_get_reviews
(
last_item_open
,
filter_open
)
reviews_closed
=
self
.
_get_reviews
(
last_item_closed
,
filter_closed
)
last_nreviews_open
=
len
(
reviews_open
)
last_nreviews_closed
=
len
(
reviews_closed
)
while
reviews_open
or
reviews_closed
:
if
reviews_open
and
reviews_closed
:
if
reviews_open
[
0
]
[
'lastUpdated'
]
>=
reviews_closed
[
0
]
[
'lastUpdated'
]
:
review_open
=
reviews_open
.
pop
(
0
)
review
=
review_open
else
:
review_closed
=
reviews_closed
.
pop
(
0
)
review
=
review_closed
elif
reviews_closed
:
review_closed
=
reviews_closed
.
pop
(
0
)
review
=
review_closed
else
:
review_open
=
reviews_open
.
pop
(
0
)
review
=
review_open
updated
=
review
[
'lastUpdated'
]
if
updated
<=
from_ut
:
logger
.
debug
(
"No more updates for %s"
%
(
self
.
hostname
)
)
break
else
:
yield
review
if
not
reviews_open
and
last_nreviews_open
>=
self
.
max_reviews
:
last_item_open
=
self
.
client
.
next_retrieve_group_item
(
last_item_open
,
review_open
)
reviews_open
=
self
.
_get_reviews
(
last_item_open
,
filter_open
)
last_nreviews_open
=
len
(
reviews_open
)
if
not
reviews_closed
and
last_nreviews_closed
>=
self
.
max_reviews
:
last_item_closed
=
self
.
client
.
next_retrieve_group_item
(
last_item_closed
,
review_closed
)
reviews_closed
=
self
.
_get_reviews
(
last_item_closed
,
filter_closed
)
last_nreviews_closed
=
len
(
reviews_closed
)

Specific fetch for gerrit 2 . 8 version .


def version(self):
        """Return the Gerrit server version."""

        if self._version:
            return self._version

        cmd = self.gerrit_cmd + " %s " % (GerritClient.CMD_VERSION)

        logger.debug("Getting version: %s" % (cmd))
        raw_data = self.__execute(cmd)
        raw_data = str(raw_data, "UTF-8")
        logger.debug("Gerrit version: %s" % (raw_data))

        # output: gerrit version 2.10-rc1-988-g333a9dd
        m = re.match(GerritClient.VERSION_REGEX, raw_data)

        if not m:
            cause = "Invalid gerrit version %s" % raw_data
            raise BackendError(cause=cause)

        try:
            mayor = int(m.group(1))
            minor = int(m.group(2))
        except Exception:
            cause = "Gerrit client could not determine the server version."
            raise BackendError(cause=cause)

        self._version = [mayor, minor]
        return self._version


def reviews(self, last_item, filter_=None):
        """Get the reviews starting from last_item."""

        cmd = self._get_gerrit_cmd(last_item, filter_)

        logger.debug("Getting reviews with command: %s", cmd)
        raw_data = self.__execute(cmd)
        raw_data = str(raw_data, "UTF-8")

        return raw_data

def
reviews
(
self
,
last_item
,
filter_
=
)
:
cmd
=
self
.
_get_gerrit_cmd
(
last_item
,
filter_
)
logger
.
debug
(
"Getting reviews with command: %s"
,
cmd
)
raw_data
=
self
.
__execute
(
cmd
)
raw_data
=
str
(
raw_data
,
"UTF-8"
)
return
raw_data

Get the reviews starting from last_item .