streamcorpus_pipeline/_guess_media_type.py
def file_type_stats(config):
    '''
    returns a kba.pipeline "transform" function that generates file
    type stats from the stream_items that it sees.  Currently, these
    stats are just the first five non-whitespace characters.
    '''
    ## make a closure around config
    def _file_type_stats(stream_item, context):
        if stream_item.body and stream_item.body.raw:
            #print repr(stream_item.body.raw[:250])
            #sys.stdout.flush()
            #doctype_m = doctype_re.match(stream_item.body.raw[:250])
            #if doctype_m:
                #print 'DOCTYPE: %s' % repr(doctype_m.group('doctype').lower())
            if 'doctype html' in stream_item.body.raw[:250].lower():
                print 'DOCTYPE: html'
            else:
                #if probably_html.search(stream_item.body.raw):
                if has_tags(stream_item.body.raw[:400]):
                    print 'PROBABLY_HTML'
                else:
                    xml = xml_ish.search(stream_item.body.raw)
                    if xml:
                        print 'XML: %s' % repr(xml.group('intro'))
                    else:
                        pdf = pdf_start.search(stream_item.body.raw)
                        if pdf:
                            print 'PDF %s' % repr(pdf.group('version'))
                        else:
                            ext = stream_item.abs_url.split('.')[-1]
                            if len(ext) < 6:
                                print 'UNK ext: %s' % repr(ext)
                            else:
                                first = first_letters.match(stream_item.body.raw)
                                if first and False:
                                    print 'UNK letters: %s' % repr(first.group('first_letters'))
                                else:
                                    print 'UNK first bytes: %s' % repr(stream_item.body.raw[:50])
                    #m = first_three_letters.search(stream_item.body.raw)
                    #if m:
                    #    print repr(m.group('first_three_letters')).lower().strip()
                    #else:
                    #    print repr(stream_item.body.raw[:50]).lower().strip()
        return stream_item

    return _file_type_stats
def file_type_stats(config):
    '''
    returns a kba.pipeline "transform" function that generates file
    type stats from the stream_items that it sees.  Currently, these
    stats are just the first five non-whitespace characters.
    '''
    ## make a closure around config
    def _file_type_stats(stream_item, context):
        if stream_item.body and stream_item.body.raw:
            #print repr(stream_item.body.raw[:250])
            #sys.stdout.flush()
            #doctype_m = doctype_re.match(stream_item.body.raw[:250])
            #if doctype_m:
                #print 'DOCTYPE: %s' % repr(doctype_m.group('doctype').lower())
            if 'doctype html' in stream_item.body.raw[:250].lower():
                print 'DOCTYPE: html'
            else:
                #if probably_html.search(stream_item.body.raw):
                if has_tags(stream_item.body.raw[:400]):
                    print 'PROBABLY_HTML'
                else:
                    xml = xml_ish.search(stream_item.body.raw)
                    if xml:
                        print 'XML: %s' % repr(xml.group('intro'))
                    else:
                        pdf = pdf_start.search(stream_item.body.raw)
                        if pdf:
                            print 'PDF %s' % repr(pdf.group('version'))
                        else:
                            ext = stream_item.abs_url.split('.')[-1]
                            if len(ext) < 6:
                                print 'UNK ext: %s' % repr(ext)
                            else:
                                first = first_letters.match(stream_item.body.raw)
                                if first and False:
                                    print 'UNK letters: %s' % repr(first.group('first_letters'))
                                else:
                                    print 'UNK first bytes: %s' % repr(stream_item.body.raw[:50])
                    #m = first_three_letters.search(stream_item.body.raw)
                    #if m:
                    #    print repr(m.group('first_three_letters')).lower().strip()
                    #else:
                    #    print repr(stream_item.body.raw[:50]).lower().strip()
        return stream_item

    return _file_type_stats
def file_type_stats ( config ) : ## make a closure around config def _file_type_stats ( stream_item , context ) : if stream_item . body and stream_item . body . raw : #print repr(stream_item.body.raw[:250]) #sys.stdout.flush() #doctype_m = doctype_re.match(stream_item.body.raw[:250]) #if doctype_m: #print 'DOCTYPE: %s' % repr(doctype_m.group('doctype').lower()) if 'doctype html' in stream_item . body . raw [ : 250 ] . lower ( ) : print 'DOCTYPE: html' else : #if probably_html.search(stream_item.body.raw): if has_tags ( stream_item . body . raw [ : 400 ] ) : print 'PROBABLY_HTML' else : xml = xml_ish . search ( stream_item . body . raw ) if xml : print 'XML: %s' % repr ( xml . group ( 'intro' ) ) else : pdf = pdf_start . search ( stream_item . body . raw ) if pdf : print 'PDF %s' % repr ( pdf . group ( 'version' ) ) else : ext = stream_item . abs_url . split ( '.' ) [ - 1 ] if len ( ext ) < 6 : print 'UNK ext: %s' % repr ( ext ) else : first = first_letters . match ( stream_item . body . raw ) if first and False : print 'UNK letters: %s' % repr ( first . group ( 'first_letters' ) ) else : print 'UNK first bytes: %s' % repr ( stream_item . body . raw [ : 50 ] ) #m = first_three_letters.search(stream_item.body.raw) #if m: #    print repr(m.group('first_three_letters')).lower().strip() #else: #    print repr(stream_item.body.raw[:50]).lower().strip() return stream_item return _file_type_stats
returns a kba.pipeline "transform" function that generates file
    type stats from the stream_items that it sees.  Currently, these
    stats are just the first five non-whitespace characters.
returns a kba . pipeline transform function that generates file type stats from the stream_items that it sees . Currently these stats are just the first five non - whitespace characters .