perceval/backends/core/mediawiki.py
def __fetch_1_27(self, from_date=None):
        """Fetch the pages from the backend url for MediaWiki >=1.27

        The method retrieves, from a MediaWiki url, the
        wiki pages.

        :returns: a generator of pages
        """

        logger.info("Looking for pages at url '%s'", self.url)

        npages = 0  # number of pages processed
        tpages = 0  # number of total pages
        pages_done = []  # pages already retrieved in reviews API

        namespaces_contents = self.__get_namespaces_contents()

        arvcontinue = ''  # pagination for getting revisions and their pages
        while arvcontinue is not None:
            raw_pages = self.client.get_pages_from_allrevisions(namespaces_contents, from_date, arvcontinue)
            data_json = json.loads(raw_pages)
            arvcontinue = data_json['continue']['arvcontinue'] if 'continue' in data_json else None
            pages_json = data_json['query']['allrevisions']
            for page in pages_json:

                if page['pageid'] in pages_done:
                    logger.debug("Page %s already processed; skipped", page['pageid'])
                    continue

                tpages += 1
                pages_done.append(page['pageid'])
                page_reviews = self.__get_page_reviews(page)

                if not page_reviews:
                    logger.warning("Revisions not found in %s [page id: %s], page skipped",
                                   page['title'], page['pageid'])
                    continue

                yield page_reviews
                npages += 1

        logger.info("Total number of pages: %i, skipped %i", tpages, tpages - npages)
def __fetch_1_27(self, from_date=None):
        logger.info("Looking for pages at url '%s'", self.url)

        npages = 0  # number of pages processed
        tpages = 0  # number of total pages
        pages_done = []  # pages already retrieved in reviews API

        namespaces_contents = self.__get_namespaces_contents()

        arvcontinue = ''  # pagination for getting revisions and their pages
        while arvcontinue is not None:
            raw_pages = self.client.get_pages_from_allrevisions(namespaces_contents, from_date, arvcontinue)
            data_json = json.loads(raw_pages)
            arvcontinue = data_json['continue']['arvcontinue'] if 'continue' in data_json else None
            pages_json = data_json['query']['allrevisions']
            for page in pages_json:

                if page['pageid'] in pages_done:
                    logger.debug("Page %s already processed; skipped", page['pageid'])
                    continue

                tpages += 1
                pages_done.append(page['pageid'])
                page_reviews = self.__get_page_reviews(page)

                if not page_reviews:
                    logger.warning("Revisions not found in %s [page id: %s], page skipped",
                                   page['title'], page['pageid'])
                    continue

                yield page_reviews
                npages += 1

        logger.info("Total number of pages: %i, skipped %i", tpages, tpages - npages)
def __fetch_1_27 ( self , from_date = None ) : logger . info ( "Looking for pages at url '%s'" , self . url ) npages = 0 # number of pages processed tpages = 0 # number of total pages pages_done = [ ] # pages already retrieved in reviews API namespaces_contents = self . __get_namespaces_contents ( ) arvcontinue = '' # pagination for getting revisions and their pages while arvcontinue is not None : raw_pages = self . client . get_pages_from_allrevisions ( namespaces_contents , from_date , arvcontinue ) data_json = json . loads ( raw_pages ) arvcontinue = data_json [ 'continue' ] [ 'arvcontinue' ] if 'continue' in data_json else None pages_json = data_json [ 'query' ] [ 'allrevisions' ] for page in pages_json : if page [ 'pageid' ] in pages_done : logger . debug ( "Page %s already processed; skipped" , page [ 'pageid' ] ) continue tpages += 1 pages_done . append ( page [ 'pageid' ] ) page_reviews = self . __get_page_reviews ( page ) if not page_reviews : logger . warning ( "Revisions not found in %s [page id: %s], page skipped" , page [ 'title' ] , page [ 'pageid' ] ) continue yield page_reviews npages += 1 logger . info ( "Total number of pages: %i, skipped %i" , tpages , tpages - npages )
Fetch the pages from the backend url for MediaWiki >=1.27

        The method retrieves, from a MediaWiki url, the
        wiki pages.

        :returns: a generator of pages
Fetch the pages from the backend url for MediaWiki > = 1 . 27