def load_robots_txt(self, url_info: URLInfo, text: str):
        '''Load the robot.txt file.'''
        key = self.url_info_key(url_info)
        parser = robotexclusionrulesparser.RobotExclusionRulesParser()
        parser.parse(text)

        self._parsers[key] = parser
def load_robots_txt ( self , url_info : URLInfo , text : str ) : key = self . url_info_key ( url_info ) parser = robotexclusionrulesparser . RobotExclusionRulesParser ( ) parser . parse ( text ) self . _parsers [ key ] = parser
Load the robot.txt file.