

def detect_charset(self):
    '\n        Detect charset of the response.\n\n        Try following methods:\n        * meta[name="Http-Equiv"]\n        * XML declaration\n        * HTTP Content-Type header\n\n        Ignore unknown charsets.\n\n        Use utf-8 as fallback charset.\n        '
    charset = None
    body_chunk = self.get_body_chunk()
    if body_chunk:
        match_charset = RE_META_CHARSET.search(body_chunk)
        if match_charset:
            charset = match_charset.group(1)
        else:
            match_charset_html5 = RE_META_CHARSET_HTML5.search(body_chunk)
            if match_charset_html5:
                charset = match_charset_html5.group(1)
        (bom_enc, bom) = read_bom(body_chunk)
        if bom_enc:
            charset = bom_enc
            self.bom = bom
        if (not charset):
            if body_chunk.startswith(b'<?xml'):
                match = RE_XML_DECLARATION.search(body_chunk)
                if match:
                    enc_match = RE_DECLARATION_ENCODING.search(match.group(0))
                    if enc_match:
                        charset = enc_match.group(1)
    if (not charset):
        if ('Content-Type' in self.headers):
            pos = self.headers['Content-Type'].find('charset=')
            if (pos > (- 1)):
                charset = self.headers['Content-Type'][(pos + 8):]
    if charset:
        charset = charset.lower()
        if (not isinstance(charset, str)):
            charset = charset.decode('utf-8')
        try:
            codecs.lookup(charset)
        except LookupError:
            logger.error(('Unknown charset found: %s. Using utf-8 istead.' % charset))
            self.charset = 'utf-8'
        else:
            self.charset = charset

dataset/ETHPy150Open lorien/grab/grab/document.py Document.detect_charset/original