<s> import glob <EOL> import re <EOL> from gluon . sql import DAL , Field <EOL> from utils . UnicodeIO import UnicodeReader , UnicodeWriter <EOL> class Corpus_DB ( ) : <EOL> FILENAME = '<STR_LIT>' <EOL> CONNECTION = '<STR_LIT>' . format ( FILENAME ) <EOL> DOC_IDS = [ '<STR_LIT>' , '<STR_LIT>' ] <EOL> DOC_CONTENTS = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> MODEL_KEY = '<STR_LIT>' <EOL> MODEL_DESC = '<STR_LIT>' <EOL> MODEL_ENTRY = { '<STR_LIT>' : MODEL_KEY , '<STR_LIT>' : MODEL_DESC } <EOL> LINEBREAKS_TABS = re . compile ( r'<STR_LIT>' ) <EOL> DEFAULT_OPTIONS = { <EOL> '<STR_LIT>' : r'<STR_LIT>' , <EOL> '<STR_LIT>' : <NUM_LIT:5> , <EOL> '<STR_LIT>' : <NUM_LIT:3> <EOL> } <EOL> def __init__ ( self , path = None , isInit = False ) : <EOL> self . isInit = isInit <EOL> if path is not None : <EOL> self . db = DAL ( Corpus_DB . CONNECTION , lazy_tables = not self . isInit , migrate_enabled = self . isInit , folder = path ) <EOL> else : <EOL> self . db = DAL ( Corpus_DB . CONNECTION , lazy_tables = not self . isInit , migrate_enabled = self . isInit ) <EOL> def __enter__ ( self ) : <EOL> self . DefineOptionsTable ( ) <EOL> self . DefineModelsTable ( ) <EOL> self . DefineCorpusTable ( ) <EOL> self . DefineMetadataTables ( ) <EOL> return self <EOL> def __exit__ ( self , type , value , traceback ) : <EOL> self . DefineCorpusTextSearch ( ) <EOL> self . db . commit ( ) <EOL> def DefineOptionsTable ( self ) : <EOL> self . db . define_table ( '<STR_LIT>' , <EOL> Field ( '<STR_LIT:key>' , '<STR_LIT:string>' , required = True , unique = True ) , <EOL> Field ( '<STR_LIT:value>' , '<STR_LIT:string>' , required = True ) , <EOL> migrate = self . isInit <EOL> ) <EOL> for key , value in Corpus_DB . DEFAULT_OPTIONS . iteritems ( ) : <EOL> self . SetOption ( key , value , overwrite = self . isInit ) <EOL> def SetOption ( self , key , value , overwrite = True ) : <EOL> where = self . db . options . key == key <EOL> if self . db ( where ) . count ( ) > <NUM_LIT:0> : <EOL> if overwrite : <EOL> self . db ( where ) . update ( value = value ) <EOL> else : <EOL> self . db . options . insert ( key = key , value = value ) <EOL> self . db . commit ( ) <EOL> def GetOption ( self , key ) : <EOL> where = self . db . options . key == key <EOL> keyValue = self . db ( where ) . select ( self . db . options . value ) . first ( ) <EOL> if keyValue : <EOL> return keyValue . value <EOL> else : <EOL> return None <EOL> def DefineModelsTable ( self ) : <EOL> self . db . define_table ( '<STR_LIT>' , <EOL> Field ( '<STR_LIT>' , '<STR_LIT:string>' , required = True , unique = True ) , <EOL> Field ( '<STR_LIT>' , '<STR_LIT:string>' , required = True ) , <EOL> migrate = self . isInit <EOL> ) <EOL> def AddModel ( self , model_key , model_desc ) : <EOL> where = self . db . models . model_key == model_key <EOL> if self . db ( where ) . count ( ) > <NUM_LIT:0> : <EOL> self . db ( where ) . update ( model_desc = model_desc ) <EOL> else : <EOL> self . db . models . insert ( model_key = model_key , model_desc = model_desc ) <EOL> self . db . commit ( ) <EOL> def GetModel ( self , model_key ) : <EOL> if model_key == Corpus_DB . MODEL_KEY : <EOL> return Corpus_DB . MODEL_ENTRY <EOL> where = self . db . models . model_key == model_key <EOL> keyValue = self . db ( where ) . select ( self . db . models . ALL ) . first ( ) <EOL> if keyValue : <EOL> return { '<STR_LIT>' : keyValue . model_key , '<STR_LIT>' : keyValue . model_desc } <EOL> else : <EOL> return None <EOL> def GetModels ( self ) : <EOL> rows = self . db ( self . db . models ) . select ( self . db . models . model_key , self . db . models . model_desc ) . as_list ( ) <EOL> return [ Corpus_DB . MODEL_ENTRY ] + rows <EOL> def DefineCorpusTable ( self ) : <EOL> self . db . define_table ( '<STR_LIT>' , <EOL> Field ( '<STR_LIT>' , '<STR_LIT>' , required = True , unique = True , default = - <NUM_LIT:1> ) , <EOL> Field ( '<STR_LIT>' , '<STR_LIT:string>' , required = True , unique = True ) , <EOL> Field ( '<STR_LIT>' , '<STR_LIT:text>' , required = True ) , <EOL> migrate = self . isInit <EOL> ) <EOL> if self . isInit : <EOL> self . db . executesql ( '<STR_LIT>' ) <EOL> self . db . executesql ( '<STR_LIT>' ) <EOL> def DefineMetadataTables ( self ) : <EOL> self . db . define_table ( '<STR_LIT>' , <EOL> Field ( '<STR_LIT>' , '<STR_LIT>' , required = True , unique = True , default = - <NUM_LIT:1> ) , <EOL> Field ( '<STR_LIT>' , '<STR_LIT:string>' , required = True , unique = True ) , <EOL> Field ( '<STR_LIT>' , '<STR_LIT:string>' , required = True ) , <EOL> migrate = self . isInit <EOL> ) <EOL> if self . isInit : <EOL> self . db . executesql ( '<STR_LIT>' ) <EOL> self . db . executesql ( '<STR_LIT>' ) <EOL> self . db . executesql ( '<STR_LIT>' ) <EOL> self . db . define_table ( '<STR_LIT>' , <EOL> Field ( '<STR_LIT>' , '<STR_LIT>' , required = True , default = - <NUM_LIT:1> ) , <EOL> Field ( '<STR_LIT>' , '<STR_LIT>' , required = True , default = - <NUM_LIT:1> ) , <EOL> Field ( '<STR_LIT:value>' , '<STR_LIT:text>' , required = True ) , <EOL> migrate = self . isInit <EOL> ) <EOL> if self . isInit : <EOL> self . db . executesql ( '<STR_LIT>' ) <EOL> self . db . executesql ( '<STR_LIT>' ) <EOL> self . db . executesql ( '<STR_LIT>' ) <EOL> def DefineCorpusTextSearch ( self ) : <EOL> if self . isInit : <EOL> pass <EOL> def SanitizeText ( self , text ) : <EOL> text = Corpus_DB . LINEBREAKS_TABS . sub ( u'<STR_LIT:U+0020>' , text ) . strip ( ) <EOL> return text <EOL> def ImportFromFile ( self , filename ) : <EOL> """<STR_LIT>""" <EOL> def ReadFile ( ) : <EOL> with open ( filename , '<STR_LIT:r>' ) as f : <EOL> for index , line in enumerate ( f ) : <EOL> doc_index = index <EOL> values = line . decode ( '<STR_LIT:utf-8>' , '<STR_LIT:ignore>' ) . rstrip ( '<STR_LIT:\n>' ) . split ( '<STR_LIT:\t>' ) <EOL> if len ( values ) == <NUM_LIT:1> : <EOL> doc_id = '<STR_LIT>' . format ( doc_index + <NUM_LIT:1> ) <EOL> doc_content = values [ <NUM_LIT:0> ] <EOL> else : <EOL> doc_id = values [ <NUM_LIT:0> ] <EOL> doc_content = values [ <NUM_LIT:1> ] <EOL> yield { <EOL> '<STR_LIT>' : doc_index , <EOL> '<STR_LIT>' : doc_id , <EOL> '<STR_LIT>' : doc_content . encode ( '<STR_LIT:utf-8>' , '<STR_LIT:ignore>' ) <EOL> } <EOL> self . db . corpus . bulk_insert ( ReadFile ( ) ) <EOL> def ImportFromFolder ( self , glob_pattern ) : <EOL> """<STR_LIT>""" <EOL> def ReadFolder ( ) : <EOL> filenames = sorted ( glob . glob ( glob_pattern ) ) <EOL> for index , filename in enumerate ( filenames ) : <EOL> doc_index = index <EOL> doc_id = filename <EOL> with open ( filename , '<STR_LIT:r>' ) as f : <EOL> doc_content = f . read ( ) . decode ( '<STR_LIT:utf-8>' , '<STR_LIT:ignore>' ) <EOL> yield { <EOL> '<STR_LIT>' : doc_index , <EOL> '<STR_LIT>' : doc_id , <EOL> '<STR_LIT>' : doc_content . encode ( '<STR_LIT:utf-8>' , '<STR_LIT:ignore>' ) <EOL> } <EOL> self . db . corpus . bulk_insert ( ReadFolder ( ) ) <EOL> def ImportFromSpreadsheet ( self , filename , id_key = None , content_key = None , is_csv = False ) : <EOL> """<STR_LIT>""" <EOL> doc_id_keys = frozenset ( [ id_key ] if id_key is not None else Corpus_DB . DOC_IDS ) <EOL> doc_content_keys = frozenset ( [ content_key ] if content_key is not None else Corpus_DB . DOC_CONTENTS ) <EOL> field_indexes = [ ] <EOL> field_names = [ ] <EOL> field_types = [ ] <EOL> metadata = [ ] <EOL> def ReadCSV ( ) : <EOL> with open ( filename , '<STR_LIT:r>' ) as f : <EOL> reader = UnicodeReader ( f ) <EOL> for row in reader : <EOL> yield row <EOL> def ReadTSV ( ) : <EOL> with open ( filename , '<STR_LIT:r>' ) as f : <EOL> for line in f : <EOL> yield line . decode ( '<STR_LIT:utf-8>' , '<STR_LIT:ignore>' ) . rstrip ( '<STR_LIT:\n>' ) . split ( '<STR_LIT:\t>' ) <EOL> def ReadSpreadsheet ( reader ) : <EOL> field_doc_id = None <EOL> field_doc_content = None <EOL> for row_index , values in enumerate ( reader ) : <EOL> if row_index == <NUM_LIT:0> : <EOL> for index , field in enumerate ( values ) : <EOL> if field . lower ( ) in doc_id_keys : <EOL> field_doc_id = index <EOL> elif field . lower ( ) in doc_content_keys : <EOL> field_doc_content = index <EOL> else : <EOL> field_index = len ( field_indexes ) <EOL> field_indexes . append ( field_index ) <EOL> field_names . append ( field ) <EOL> field_types . append ( '<STR_LIT>' ) <EOL> else : <EOL> doc_index = row_index - <NUM_LIT:1> <EOL> doc_id = '<STR_LIT>' . format ( doc_index + <NUM_LIT:1> ) <EOL> doc_content = '<STR_LIT>' <EOL> field_index = <NUM_LIT:0> <EOL> for index , value in enumerate ( values ) : <EOL> if field_doc_id == index : <EOL> doc_id = value <EOL> elif field_doc_content == index : <EOL> doc_content = value <EOL> else : <EOL> metadata . append ( { <EOL> '<STR_LIT>' : doc_index , <EOL> '<STR_LIT>' : field_index , <EOL> '<STR_LIT:value>' : value . encode ( '<STR_LIT:utf-8>' , '<STR_LIT:ignore>' ) <EOL> } ) <EOL> field_type = field_types [ field_index ] <EOL> if field_type == '<STR_LIT>' : <EOL> try : <EOL> int ( value ) <EOL> except ValueError : <EOL> field_type = '<STR_LIT>' <EOL> if field_type == '<STR_LIT>' : <EOL> try : <EOL> float ( value ) <EOL> except ValueError : <EOL> field_type = '<STR_LIT:string>' <EOL> field_types [ field_index ] = field_type <EOL> field_index += <NUM_LIT:1> <EOL> yield { <EOL> '<STR_LIT>' : doc_index , <EOL> '<STR_LIT>' : doc_id , <EOL> '<STR_LIT>' : doc_content . encode ( '<STR_LIT:utf-8>' , '<STR_LIT:ignore>' ) <EOL> } <EOL> def GetFields ( ) : <EOL> for field_index in field_indexes : <EOL> field_name = field_names [ field_index ] <EOL> field_type = field_types [ field_index ] <EOL> yield { <EOL> '<STR_LIT>' : field_index , <EOL> '<STR_LIT>' : field_name , <EOL> '<STR_LIT>' : field_type <EOL> } <EOL> reader = ReadCSV ( ) if is_csv else ReadTSV ( ) <EOL> self . db . corpus . bulk_insert ( ReadSpreadsheet ( reader ) ) <EOL> self . db . fields . bulk_insert ( GetFields ( ) ) <EOL> self . db . metadata . bulk_insert ( metadata ) <EOL> def ExportToFile ( self , filename ) : <EOL> """<STR_LIT>""" <EOL> def WriteFile ( rows ) : <EOL> with open ( filename , '<STR_LIT:w>' ) as f : <EOL> for row in rows : <EOL> doc_id = row . doc_id <EOL> doc_content = self . SanitizeText ( row . doc_content . decode ( '<STR_LIT:utf-8>' ) ) <EOL> f . write ( u'<STR_LIT>' . format ( doc_id , doc_content ) . encode ( '<STR_LIT:utf-8>' , '<STR_LIT:ignore>' ) ) <EOL> rows = self . db ( ) . select ( self . db . corpus . doc_id , self . db . corpus . doc_content , orderby = self . db . corpus . doc_index ) <EOL> WriteFile ( rows ) <EOL> def ExportToSpreadsheet ( self , filename , id_key = None , content_key = None , is_csv = False ) : <EOL> """<STR_LIT>""" <EOL> field_names = [ row . field_name for row in self . db ( ) . select ( self . db . fields . field_name , orderby = self . db . fields . field_index ) ] <EOL> field_count = len ( field_names ) <EOL> all_field_names = [ <EOL> id_key if id_key is not None else '<STR_LIT>' , <EOL> content_key if content_key is not None else '<STR_LIT>' <EOL> ] + field_names <EOL> def WriteCSV ( rows ) : <EOL> with open ( filename , '<STR_LIT:w>' ) as f : <EOL> writer = UnicodeWriter ( f ) <EOL> writer . writerow ( all_field_names ) <EOL> for row in rows : <EOL> doc_index = row . doc_index <EOL> doc_id = row . doc_id <EOL> doc_content = self . SanitizeText ( row . doc_content . decode ( '<STR_LIT:utf-8>' ) ) <EOL> values = [ u'<STR_LIT>' ] * field_count <EOL> for d in self . db ( self . db . metadata . doc_index == doc_index ) . select ( self . db . metadata . field_index , self . db . metadata . value ) : <EOL> values [ d . field_index ] = self . SanitizeText ( d . value . decode ( '<STR_LIT:utf-8>' ) ) <EOL> all_values = [ doc_id , doc_content ] + values <EOL> writer . writerow ( all_values ) <EOL> def WriteTSV ( rows ) : <EOL> with open ( filename , '<STR_LIT:w>' ) as f : <EOL> f . write ( u'<STR_LIT>' . format ( u'<STR_LIT:\t>' . join ( all_field_names ) ) . encode ( '<STR_LIT:utf-8>' , '<STR_LIT:ignore>' ) ) <EOL> for row in rows : <EOL> doc_index = row . doc_index <EOL> doc_id = row . doc_id <EOL> doc_content = self . SanitizeText ( row . doc_content . decode ( '<STR_LIT:utf-8>' ) ) <EOL> values = [ u'<STR_LIT>' ] * field_count <EOL> for d in self . db ( self . db . metadata . doc_index == doc_index ) . select ( self . db . metadata . field_index , self . db . metadata . value ) : <EOL> values [ d . field_index ] = self . SanitizeText ( d . value . decode ( '<STR_LIT:utf-8>' ) ) <EOL> all_values = [ doc_id , doc_content ] + values <EOL> f . write ( u'<STR_LIT>' . format ( u'<STR_LIT:\t>' . join ( all_values ) ) . encode ( '<STR_LIT:utf-8>' , '<STR_LIT:ignore>' ) ) <EOL> rows = self . db ( ) . select ( self . db . corpus . doc_index , self . db . corpus . doc_id , self . db . corpus . doc_content , orderby = self . db . corpus . doc_index ) <EOL> if is_csv : <EOL> WriteCSV ( rows ) <EOL> else : <EOL> WriteTSV ( rows ) </s>
