def read_component_sitemap(
            self, sitemapindex_uri, sitemap_uri, sitemap, sitemapindex_is_file):
        """Read a component sitemap of a Resource List with index.

        Each component must be a sitemap with the
        """
        if (sitemapindex_is_file):
            if (not self.is_file_uri(sitemap_uri)):
                # Attempt to map URI to local file
                remote_uri = sitemap_uri
                sitemap_uri = self.mapper.src_to_dst(remote_uri)
                self.logger.info(
                    "Mapped %s to local file %s" %
                    (remote_uri, sitemap_uri))
            else:
                # The individual sitemaps should be at a URL (scheme/server/path)
                # that the sitemapindex URL can speak authoritatively about
                if (self.check_url_authority and
                        not UrlAuthority(sitemapindex_uri).has_authority_over(sitemap_uri)):
                    raise ListBaseIndexError(
                        "The sitemapindex (%s) refers to sitemap at a location it does not have authority over (%s)" %
                        (sitemapindex_uri, sitemap_uri))
        try:
            fh = URLopener().open(sitemap_uri)
            self.num_files += 1
        except IOError as e:
            raise ListBaseIndexError(
                "Failed to load sitemap from %s listed in sitemap index %s (%s)" %
                (sitemap_uri, sitemapindex_uri, str(e)))
        # Get the Content-Length if we can (works fine for local files)
        try:
            self.content_length = int(fh.info()['Content-Length'])
            self.bytes_read += self.content_length
        except KeyError:
            # If we don't get a length then c'est la vie
            pass
        self.logger.info(
            "Reading sitemap from %s (%d bytes)" %
            (sitemap_uri, self.content_length))
        component = sitemap.parse_xml(fh=fh, sitemapindex=False)
        # Copy resources into self, check any metadata
        for r in component:
            self.resources.add(r)
def read_component_sitemap ( self , sitemapindex_uri , sitemap_uri , sitemap , sitemapindex_is_file ) : if ( sitemapindex_is_file ) : if ( not self . is_file_uri ( sitemap_uri ) ) : remote_uri = sitemap_uri sitemap_uri = self . mapper . src_to_dst ( remote_uri ) self . logger . info ( "Mapped %s to local file %s" % ( remote_uri , sitemap_uri ) ) else : if ( self . check_url_authority and not UrlAuthority ( sitemapindex_uri ) . has_authority_over ( sitemap_uri ) ) : raise ListBaseIndexError ( "The sitemapindex (%s) refers to sitemap at a location it does not have authority over (%s)" % ( sitemapindex_uri , sitemap_uri ) ) try : fh = URLopener ( ) . open ( sitemap_uri ) self . num_files += 1 except IOError as e : raise ListBaseIndexError ( "Failed to load sitemap from %s listed in sitemap index %s (%s)" % ( sitemap_uri , sitemapindex_uri , str ( e ) ) ) try : self . content_length = int ( fh . info ( ) [ 'Content-Length' ] ) self . bytes_read += self . content_length except KeyError : pass self . logger . info ( "Reading sitemap from %s (%d bytes)" % ( sitemap_uri , self . content_length ) ) component = sitemap . parse_xml ( fh = fh , sitemapindex = False ) for r in component : self . resources . add ( r )
Read a component sitemap of a Resource List with index.

        Each component must be a sitemap with the