def content_allows_robots (self):
        """
        Return False if the content of this URL forbids robots to
        search for recursive links.
        """
        if not self.is_html():
            return True
        # construct parser object
        handler = linkparse.MetaRobotsFinder()
        parser = htmlsax.parser(handler)
        handler.parser = parser
        if self.charset:
            parser.encoding = self.charset
        # parse
        try:
            parser.feed(self.get_content())
            parser.flush()
        except linkparse.StopParse as msg:
            log.debug(LOG_CHECK, "Stopped parsing: %s", msg)
            pass
        # break cyclic dependencies
        handler.parser = None
        parser.handler = None
        return handler.follow
def content_allows_robots ( self ) : if not self . is_html ( ) : return True handler = linkparse . MetaRobotsFinder ( ) parser = htmlsax . parser ( handler ) handler . parser = parser if self . charset : parser . encoding = self . charset try : parser . feed ( self . get_content ( ) ) parser . flush ( ) except linkparse . StopParse as msg : log . debug ( LOG_CHECK , "Stopped parsing: %s" , msg ) pass handler . parser = None parser . handler = None return handler . follow
Return False if the content of this URL forbids robots to
        search for recursive links.