perceval/backends/mozilla/crates.py
def __fetch_items(self, path, page=1):
        """Return the items from Crates.io API using pagination"""

        fetch_data = True
        parsed_crates = 0
        total_crates = 0

        while fetch_data:
            logger.debug("Fetching page: %i", page)

            try:
                payload = {'sort': 'alphabetical', 'page': page}
                raw_content = self.fetch(path, payload=payload)
                content = json.loads(raw_content)

                parsed_crates += len(content['crates'])

                if not total_crates:
                    total_crates = content['meta']['total']

            except requests.exceptions.HTTPError as e:
                logger.error("HTTP exception raised - %s", e.response.text)
                raise e

            yield raw_content
            page += 1

            if parsed_crates >= total_crates:
                fetch_data = False
def __fetch_items(self, path, page=1):
        fetch_data = True
        parsed_crates = 0
        total_crates = 0

        while fetch_data:
            logger.debug("Fetching page: %i", page)

            try:
                payload = {'sort': 'alphabetical', 'page': page}
                raw_content = self.fetch(path, payload=payload)
                content = json.loads(raw_content)

                parsed_crates += len(content['crates'])

                if not total_crates:
                    total_crates = content['meta']['total']

            except requests.exceptions.HTTPError as e:
                logger.error("HTTP exception raised - %s", e.response.text)
                raise e

            yield raw_content
            page += 1

            if parsed_crates >= total_crates:
                fetch_data = False
def __fetch_items ( self , path , page = 1 ) : fetch_data = True parsed_crates = 0 total_crates = 0 while fetch_data : logger . debug ( "Fetching page: %i" , page ) try : payload = { 'sort' : 'alphabetical' , 'page' : page } raw_content = self . fetch ( path , payload = payload ) content = json . loads ( raw_content ) parsed_crates += len ( content [ 'crates' ] ) if not total_crates : total_crates = content [ 'meta' ] [ 'total' ] except requests . exceptions . HTTPError as e : logger . error ( "HTTP exception raised - %s" , e . response . text ) raise e yield raw_content page += 1 if parsed_crates >= total_crates : fetch_data = False
Return the items from Crates.io API using pagination
Return the items from Crates . io API using pagination