<s> import subprocess <EOL> import MySQLdb <EOL> import re <EOL> import sys <EOL> import json <EOL> import os <EOL> from variableSet import dataField <EOL> from variableSet import variableSet <EOL> from variableSet import splitMySQLcode <EOL> import logging <EOL> import warnings <EOL> import anydbm <EOL> if logging . getLogger ( ) . isEnabledFor ( logging . DEBUG ) : <EOL> warnings . filterwarnings ( '<STR_LIT:error>' , category = MySQLdb . Warning ) <EOL> warnings . filterwarnings ( '<STR_LIT:ignore>' , '<STR_LIT>' ) <EOL> warnings . filterwarnings ( "<STR_LIT:ignore>" , "<STR_LIT>" ) <EOL> warnings . filterwarnings ( "<STR_LIT:ignore>" , "<STR_LIT>" ) <EOL> warnings . filterwarnings ( "<STR_LIT:ignore>" , "<STR_LIT>" ) <EOL> def text_id_dbm ( ) : <EOL> """<STR_LIT>""" <EOL> dbm = anydbm . open ( "<STR_LIT>" , "<STR_LIT:c>" ) <EOL> for file in os . listdir ( "<STR_LIT>" ) : <EOL> for line in open ( "<STR_LIT>" + file ) : <EOL> line = line . rstrip ( "<STR_LIT:\n>" ) <EOL> splat = line . split ( "<STR_LIT:\t>" ) <EOL> try : <EOL> dbm [ splat [ <NUM_LIT:1> ] ] = splat [ <NUM_LIT:0> ] <EOL> except IndexError : <EOL> if line == "<STR_LIT>" : <EOL> continue <EOL> else : <EOL> raise <EOL> class DB : <EOL> def __init__ ( self , dbname = None ) : <EOL> from bookwormDB . configuration import Configfile <EOL> try : <EOL> configuration = Configfile ( "<STR_LIT>" ) <EOL> logging . debug ( "<STR_LIT>" ) <EOL> except IOError : <EOL> try : <EOL> configuration = Configfile ( "<STR_LIT>" ) <EOL> logging . debug ( "<STR_LIT>" ) <EOL> except IOError : <EOL> configuration = Configfile ( "<STR_LIT>" ) <EOL> logging . debug ( "<STR_LIT>" ) <EOL> configuration . read_config_files ( ) <EOL> config = configuration . config <EOL> if dbname == None : <EOL> self . dbname = config . get ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> else : <EOL> self . dbname = dbname <EOL> self . username = config . get ( "<STR_LIT>" , "<STR_LIT:user>" ) <EOL> self . password = config . get ( "<STR_LIT>" , "<STR_LIT:password>" ) <EOL> self . conn = None <EOL> def connect ( self , setengine = True ) : <EOL> self . conn = MySQLdb . connect ( read_default_file = "<STR_LIT>" , use_unicode = '<STR_LIT:True>' , charset = '<STR_LIT:utf8>' , db = '<STR_LIT>' , local_infile = <NUM_LIT:1> ) <EOL> cursor = self . conn . cursor ( ) <EOL> cursor . execute ( "<STR_LIT>" % self . dbname ) <EOL> cursor . execute ( "<STR_LIT>" ) <EOL> cursor . execute ( "<STR_LIT>" ) <EOL> if setengine : <EOL> try : <EOL> cursor . execute ( "<STR_LIT>" ) <EOL> except : <EOL> logging . error ( "<STR_LIT>" ) <EOL> self . connect ( setengine = False ) <EOL> logging . debug ( "<STR_LIT>" % self . dbname ) <EOL> cursor . execute ( "<STR_LIT>" % self . dbname ) <EOL> def query ( self , sql ) : <EOL> """<STR_LIT>""" <EOL> logging . debug ( "<STR_LIT>" + sql ) <EOL> try : <EOL> cursor = self . conn . cursor ( ) <EOL> cursor . execute ( sql ) <EOL> except : <EOL> try : <EOL> self . connect ( ) <EOL> cursor = self . conn . cursor ( ) <EOL> cursor . execute ( sql ) <EOL> except : <EOL> logging . error ( "<STR_LIT>" + sql + "<STR_LIT:\n>" ) <EOL> raise <EOL> return cursor <EOL> class BookwormSQLDatabase : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , dbname = None , <EOL> variableFile = "<STR_LIT>" ) : <EOL> """<STR_LIT>""" <EOL> import bookwormDB . configuration <EOL> self . config_manager = bookwormDB . configuration . Configfile ( "<STR_LIT>" ) <EOL> self . config_manager . read_config_files ( ) <EOL> config = self . config_manager . config <EOL> if dbname == None : <EOL> self . dbname = config . get ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> else : <EOL> self . dbname = dbname <EOL> self . conn = None <EOL> self . db = DB ( dbname = self . dbname ) <EOL> if variableFile is not None : <EOL> self . setVariables ( originFile = variableFile ) <EOL> def grantPrivileges ( self ) : <EOL> """<STR_LIT>""" <EOL> import ConfigParser <EOL> config = ConfigParser . ConfigParser ( allow_no_value = True ) <EOL> config . read ( [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] ) <EOL> username = config . get ( "<STR_LIT>" , "<STR_LIT:user>" ) <EOL> password = config . get ( "<STR_LIT>" , "<STR_LIT:password>" ) <EOL> self . db . query ( "<STR_LIT>" % ( self . dbname , username , password ) ) <EOL> def setVariables ( self , originFile , anchorField = "<STR_LIT>" , <EOL> jsonDefinition = "<STR_LIT>" ) : <EOL> self . variableSet = variableSet ( originFile = originFile , anchorField = anchorField , jsonDefinition = jsonDefinition , db = self . db ) <EOL> def importNewFile ( self , originFile , anchorField , jsonDefinition ) : <EOL> """<STR_LIT>""" <EOL> self . setVariables ( originFile , anchorField = anchorField , jsonDefinition = jsonDefinition ) <EOL> self . variableSet . writeMetadata ( ) <EOL> self . load_book_list ( ) <EOL> self . variableSet . updateMasterVariableTable ( ) <EOL> for variable in self . variableSet . variables : <EOL> variable . clear_associated_memory_tables ( ) <EOL> self . reloadMemoryTables ( ) <EOL> def create_database ( self ) : <EOL> dbname = self . dbname <EOL> dbuser = self . dbuser <EOL> dbpassword = self . dbpassword <EOL> db = self . db <EOL> try : <EOL> db . query ( "<STR_LIT>" + dbname ) <EOL> except : <EOL> logging . info ( "<STR_LIT>" % dbname ) <EOL> "<STR_LIT>" <EOL> db . query ( "<STR_LIT>" + dbname + "<STR_LIT>" + "<STR_LIT>" + dbuser + "<STR_LIT>" + dbpassword + "<STR_LIT:'>" ) <EOL> db . query ( "<STR_LIT>" ) <EOL> try : <EOL> db . query ( "<STR_LIT>" ) <EOL> except : <EOL> raise <EOL> def load_word_list ( self ) : <EOL> db = self . db <EOL> logging . info ( "<STR_LIT>" ) <EOL> db . query ( """<STR_LIT>""" ) <EOL> db . query ( """<STR_LIT>""" ) <EOL> db . query ( "<STR_LIT>" ) <EOL> logging . info ( "<STR_LIT>" ) <EOL> db . query ( """<STR_LIT>""" ) <EOL> logging . info ( "<STR_LIT>" ) <EOL> db . query ( "<STR_LIT>" ) <EOL> db . query ( "<STR_LIT>" ) <EOL> def load_book_list ( self ) : <EOL> """<STR_LIT>""" <EOL> self . variableSet . loadMetadata ( ) <EOL> def create_unigram_book_counts ( self ) : <EOL> db = self . db <EOL> db . query ( """<STR_LIT>""" ) <EOL> logging . info ( "<STR_LIT>" ) <EOL> db . query ( """<STR_LIT>""" ) <EOL> db . query ( "<STR_LIT>" ) <EOL> logging . info ( "<STR_LIT>" ) <EOL> for filename in os . listdir ( "<STR_LIT>" ) : <EOL> if not filename . endswith ( '<STR_LIT>' ) : <EOL> continue <EOL> try : <EOL> db . query ( "<STR_LIT>" + filename + "<STR_LIT>" ) <EOL> except : <EOL> raise <EOL> logging . info ( "<STR_LIT>" ) <EOL> db . query ( "<STR_LIT>" ) <EOL> def create_bigram_book_counts ( self ) : <EOL> db = self . db <EOL> logging . info ( "<STR_LIT>" ) <EOL> db . query ( """<STR_LIT>""" ) <EOL> db . query ( """<STR_LIT>""" ) <EOL> db . query ( "<STR_LIT>" ) <EOL> logging . info ( "<STR_LIT>" ) <EOL> for filename in os . listdir ( "<STR_LIT>" ) : <EOL> try : <EOL> db . query ( "<STR_LIT>" + filename + "<STR_LIT>" ) <EOL> except : <EOL> raise <EOL> logging . info ( "<STR_LIT>" ) <EOL> db . query ( "<STR_LIT>" ) <EOL> def loadVariableDescriptionsIntoDatabase ( self ) : <EOL> """<STR_LIT>""" <EOL> db = self . db <EOL> db . query ( "<STR_LIT>" ) <EOL> m = db . query ( """<STR_LIT>""" ) <EOL> tableTable = db . query ( """<STR_LIT>""" ) <EOL> self . addFilesToMasterVariableTable ( ) <EOL> self . addWordsToMasterVariableTable ( ) <EOL> self . variableSet . updateMasterVariableTable ( ) <EOL> def reloadMemoryTables ( self , force = False ) : <EOL> """<STR_LIT>""" <EOL> existingCreateCodes = self . db . query ( "<STR_LIT>" ) . fetchall ( ) ; <EOL> for row in existingCreateCodes : <EOL> """<STR_LIT>""" <EOL> tablename = row [ <NUM_LIT:0> ] <EOL> try : <EOL> cursor = self . db . query ( "<STR_LIT>" % ( tablename ) ) <EOL> currentLength = cursor . fetchall ( ) [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] <EOL> logging . debug ( "<STR_LIT>" % currentLength ) <EOL> except : <EOL> currentLength = <NUM_LIT:0> <EOL> if currentLength == <NUM_LIT:0> or force : <EOL> for query in splitMySQLcode ( row [ <NUM_LIT:1> ] ) : <EOL> self . db . query ( query ) <EOL> def addFilesToMasterVariableTable ( self ) : <EOL> fastFieldsCreateList = [ "<STR_LIT>" , "<STR_LIT>" ] + [ variable . fastSQL ( ) for variable in self . variableSet . variables if ( variable . unique and variable . fastSQL ( ) is not None ) ] <EOL> fileCommand = """<STR_LIT>""" + "<STR_LIT>" . join ( fastFieldsCreateList ) + """<STR_LIT>""" <EOL> fastFields = [ "<STR_LIT>" , "<STR_LIT>" ] + [ variable . fastField for variable in self . variableSet . variables if variable . unique and variable . fastSQL ( ) is not None ] <EOL> fileCommand += "<STR_LIT>" + "<STR_LIT:U+002C>" . join ( fastFields ) + "<STR_LIT>" + "<STR_LIT:U+0020>" . join ( [ "<STR_LIT>" % variable . __dict__ for variable in self . variableSet . variables if variable . unique and variable . fastSQL ( ) is not None and variable . datatype == "<STR_LIT>" ] ) + "<STR_LIT:;>" <EOL> fileCommand += "<STR_LIT>" <EOL> fileCommand += "<STR_LIT>" <EOL> self . db . query ( '<STR_LIT>' ) <EOL> self . db . query ( """<STR_LIT>""" + fileCommand + """<STR_LIT>""" ) <EOL> def addWordsToMasterVariableTable ( self ) : <EOL> wordCommand = "<STR_LIT>" <EOL> wordCommand += "<STR_LIT>" <EOL> wordCommand += "<STR_LIT>" <EOL> wordCommand += "<STR_LIT>" <EOL> wordCommand += "<STR_LIT>" <EOL> query = """<STR_LIT>""" + MySQLdb . escape_string ( wordCommand ) + """<STR_LIT>""" <EOL> logging . info ( "<STR_LIT>" ) <EOL> self . db . query ( query ) <EOL> def jsonify_data ( self ) : <EOL> variables = self . variableSet . variables <EOL> dbname = self . dbname <EOL> output = dict ( ) <EOL> output [ '<STR_LIT>' ] = { <EOL> "<STR_LIT>" : self . dbname , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : self . dbname , <EOL> "<STR_LIT>" : self . dbname <EOL> } <EOL> ui_components = [ <EOL> { <EOL> "<STR_LIT:type>" : "<STR_LIT:text>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT:name>" : "<STR_LIT>" <EOL> } <EOL> ] <EOL> for variable in variables : <EOL> newdict = variable . jsonDict ( ) <EOL> if newdict : <EOL> ui_components . append ( newdict ) <EOL> try : <EOL> mytime = [ variable . field for variable in variables if variable . datatype == '<STR_LIT:time>' ] [ <NUM_LIT:0> ] <EOL> output [ '<STR_LIT>' ] = [ <EOL> { <EOL> "<STR_LIT>" : [ { "<STR_LIT>" : [ "<STR_LIT:test>" ] } ] , <EOL> "<STR_LIT>" : mytime , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : <NUM_LIT:0> <EOL> } <EOL> ] <EOL> except : <EOL> logging . warning ( "<STR_LIT>" ) <EOL> output [ '<STR_LIT>' ] = ui_components <EOL> outfile = open ( '<STR_LIT>' % dbname , '<STR_LIT:w>' ) <EOL> outfile . write ( json . dumps ( output ) ) <EOL> def create_API_settings ( self ) : <EOL> db = self . db <EOL> try : <EOL> db . query ( "<STR_LIT>" ) <EOL> db . query ( "<STR_LIT>" ) <EOL> except : <EOL> pass <EOL> api_info = { <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : self . dbname , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> } <EOL> addCode = json . dumps ( api_info ) <EOL> logging . info ( addCode ) <EOL> db . query ( "<STR_LIT>" % addCode ) <EOL> def update_Porter_stemming ( self ) : <EOL> """<STR_LIT>""" <EOL> logging . info ( "<STR_LIT>" ) <EOL> from nltk import PorterStemmer <EOL> stemmer = PorterStemmer ( ) <EOL> cursor = db . query ( """<STR_LIT>""" ) <EOL> words = cursor . fetchall ( ) <EOL> for local in words : <EOL> word = '<STR_LIT>' . join ( local ) <EOL> word = word . replace ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> if re . match ( "<STR_LIT>" , word ) : <EOL> query = """<STR_LIT>""" + stemmer . stem ( '<STR_LIT>' . join ( local ) ) + """<STR_LIT>""" + '<STR_LIT>' . join ( local ) + """<STR_LIT>""" <EOL> z = cursor . execute ( query ) </s>
