def diffs2persistence(rev_docs, window_size=50, revert_radius=15, sunset=None,
                      verbose=False):
    """
    Processes a sorted and page-partitioned sequence of revision documents into
    and adds a 'persistence' field to them containing statistics about how each
    token "added" in the revision persisted through future revisions.

    :Parameters:
        rev_docs : `iterable` ( `dict` )
            JSON documents of revision data containing a 'diff' field as
            generated by ``dump2diffs``.  It's assumed that rev_docs are
            partitioned by page and otherwise in chronological order.
        window_size : `int`
            The size of the window of revisions from which persistence data
            will be generated.
        revert_radius : `int`
            The number of revisions back that a revert can reference.
        sunset : :class:`mwtypes.Timestamp`
            The date of the database dump we are generating from.  This is
            used to apply a 'time visible' statistic.  If not set, now() will
            be assumed.
        keep_diff : `bool`
            Do not drop the `diff` field from the revision document after
            processing is complete.
        verbose : `bool`
            Prints out dots and stuff to stderr

    :Returns:
        A generator of rev_docs with a 'persistence' field containing
        statistics about individual tokens.
    """
    rev_docs = mwxml.utilities.normalize(rev_docs)
    window_size = int(window_size)
    revert_radius = int(revert_radius)
    sunset = Timestamp(sunset) if sunset is not None \
                               else Timestamp(time.time())

    # Group the docs by page
    page_docs = groupby(rev_docs, key=lambda d: d['page']['title'])

    for page_title, rev_docs in page_docs:

        if verbose:
            sys.stderr.write(page_title + ": ")

        # We need a look-ahead to know how long this revision was visible
        rev_docs = peekable(rev_docs)

        # The window allows us to manage memory
        window = deque(maxlen=window_size)

        # The state does the actual processing work
        state = DiffState(revert_radius=revert_radius)

        while rev_docs:
            rev_doc = next(rev_docs)
            next_doc = rev_docs.peek(None)

            if next_doc is not None:
                seconds_visible = Timestamp(next_doc['timestamp']) - \
                                  Timestamp(rev_doc['timestamp'])
            else:
                seconds_visible = sunset - Timestamp(rev_doc['timestamp'])

            if seconds_visible < 0:
                logger.warn("Seconds visible {0} is less than zero."
                            .format(seconds_visible))
                seconds_visible = 0

            _, tokens_added, _ = \
                state.update_opdocs(rev_doc['sha1'], rev_doc['diff']['ops'],
                                    (rev_doc['user'], seconds_visible))

            if len(window) == window_size:
                # Time to start writing some stats
                old_doc, old_added = window[0]
                window.append((rev_doc, tokens_added))
                persistence = token_persistence(old_doc, old_added, window,
                                                None)
                old_doc['persistence'] = persistence
                yield old_doc
                if verbose:
                    sys.stderr.write(".")
                    sys.stderr.flush()
            else:
                window.append((rev_doc, tokens_added))

        while len(window) > 0:
            old_doc, old_added = window.popleft()
            persistence = token_persistence(old_doc, old_added, window, sunset)
            old_doc['persistence'] = persistence
            yield old_doc
            if verbose:
                sys.stderr.write("_")
                sys.stderr.flush()

        if verbose:
            sys.stderr.write("\n")
def diffs2persistence ( rev_docs , window_size = 50 , revert_radius = 15 , sunset = None , verbose = False ) : rev_docs = mwxml . utilities . normalize ( rev_docs ) window_size = int ( window_size ) revert_radius = int ( revert_radius ) sunset = Timestamp ( sunset ) if sunset is not None else Timestamp ( time . time ( ) ) page_docs = groupby ( rev_docs , key = lambda d : d [ 'page' ] [ 'title' ] ) for page_title , rev_docs in page_docs : if verbose : sys . stderr . write ( page_title + ": " ) rev_docs = peekable ( rev_docs ) window = deque ( maxlen = window_size ) state = DiffState ( revert_radius = revert_radius ) while rev_docs : rev_doc = next ( rev_docs ) next_doc = rev_docs . peek ( None ) if next_doc is not None : seconds_visible = Timestamp ( next_doc [ 'timestamp' ] ) - Timestamp ( rev_doc [ 'timestamp' ] ) else : seconds_visible = sunset - Timestamp ( rev_doc [ 'timestamp' ] ) if seconds_visible < 0 : logger . warn ( "Seconds visible {0} is less than zero." . format ( seconds_visible ) ) seconds_visible = 0 _ , tokens_added , _ = state . update_opdocs ( rev_doc [ 'sha1' ] , rev_doc [ 'diff' ] [ 'ops' ] , ( rev_doc [ 'user' ] , seconds_visible ) ) if len ( window ) == window_size : old_doc , old_added = window [ 0 ] window . append ( ( rev_doc , tokens_added ) ) persistence = token_persistence ( old_doc , old_added , window , None ) old_doc [ 'persistence' ] = persistence yield old_doc if verbose : sys . stderr . write ( "." ) sys . stderr . flush ( ) else : window . append ( ( rev_doc , tokens_added ) ) while len ( window ) > 0 : old_doc , old_added = window . popleft ( ) persistence = token_persistence ( old_doc , old_added , window , sunset ) old_doc [ 'persistence' ] = persistence yield old_doc if verbose : sys . stderr . write ( "_" ) sys . stderr . flush ( ) if verbose : sys . stderr . write ( "\n" )
Processes a sorted and page-partitioned sequence of revision documents into
    and adds a 'persistence' field to them containing statistics about how each
    token "added" in the revision persisted through future revisions.

    :Parameters:
        rev_docs : `iterable` ( `dict` )
            JSON documents of revision data containing a 'diff' field as
            generated by ``dump2diffs``.  It's assumed that rev_docs are
            partitioned by page and otherwise in chronological order.
        window_size : `int`
            The size of the window of revisions from which persistence data
            will be generated.
        revert_radius : `int`
            The number of revisions back that a revert can reference.
        sunset : :class:`mwtypes.Timestamp`
            The date of the database dump we are generating from.  This is
            used to apply a 'time visible' statistic.  If not set, now() will
            be assumed.
        keep_diff : `bool`
            Do not drop the `diff` field from the revision document after
            processing is complete.
        verbose : `bool`
            Prints out dots and stuff to stderr

    :Returns:
        A generator of rev_docs with a 'persistence' field containing
        statistics about individual tokens.