<s> """<STR_LIT>""" <EOL> from rdflib . term import Literal <EOL> from rdflib . namespace import RDF <EOL> from rdflib . graph import QuotedGraph <EOL> from rdflib . graph import Graph <EOL> from rdfextras . utils . termutils import OBJECT <EOL> from rdfextras . utils . termutils import escape_quotes <EOL> from rdfextras . store . REGEXMatching import REGEXTerm <EOL> from rdfextras . store . FOPLRelationalModel . QuadSlot import normalizeValue <EOL> Any = None <EOL> COLLISION_DETECTION = False <EOL> REGEX_IDX = False <EOL> CREATE_HASH_TABLE = """<STR_LIT>""" <EOL> IDENTIFIER_GARBAGE_COLLECTION_SQL = """<STR_LIT>""" <EOL> VALUE_GARBAGE_COLLECTION_SQL = """<STR_LIT>""" <EOL> PURGE_KEY_SQL = """<STR_LIT>""" <EOL> def GarbageCollectionQUERY ( idHash , valueHash , aBoxPart , binRelPart , litPart ) : <EOL> """<STR_LIT>""" <EOL> purgeQueries = [ "<STR_LIT>" ] <EOL> rdfTypeInt = normalizeValue ( RDF . type , '<STR_LIT>' ) <EOL> idHashKeyName = idHash . columns [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] <EOL> valueHashKeyName = valueHash . columns [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] <EOL> idHashJoinees = [ aBoxPart , binRelPart , litPart ] <EOL> idJoinClauses = [ ] <EOL> idJoinColumnCandidates = [ ] <EOL> explicitJoins = [ ] <EOL> for part in idHashJoinees : <EOL> partJoinClauses = [ ] <EOL> for colName in part . columnNames : <EOL> if part . columnNames . index ( colName ) >= <NUM_LIT:4> : <EOL> colName , sqlType , index = colName <EOL> if sqlType . lower ( ) [ : <NUM_LIT:6> ] == '<STR_LIT>' : <EOL> partJoinClauses . append ( "<STR_LIT>" % ( part , colName , idHash , idHashKeyName ) ) <EOL> idJoinColumnCandidates . append ( "<STR_LIT>" % ( part , colName ) ) <EOL> elif colName : <EOL> partJoinClauses . append ( "<STR_LIT>" % ( part , colName , idHash , idHashKeyName ) ) <EOL> idJoinColumnCandidates . append ( "<STR_LIT>" % ( part , colName ) ) <EOL> explicitJoins . append ( "<STR_LIT>" % ( part , '<STR_LIT>' . join ( partJoinClauses ) ) ) <EOL> idJoinClauses . extend ( partJoinClauses ) <EOL> intersectionClause = "<STR_LIT>" . join ( [ col + "<STR_LIT>" <EOL> for col in idJoinColumnCandidates ] ) <EOL> idGCQuery = IDENTIFIER_GARBAGE_COLLECTION_SQL % ( <EOL> idHash , <EOL> idHashKeyName , <EOL> idHash , <EOL> '<STR_LIT:U+0020>' . join ( explicitJoins ) , <EOL> intersectionClause , <EOL> idHash , <EOL> idHashKeyName , <EOL> rdfTypeInt <EOL> ) <EOL> idPurgeQuery = PURGE_KEY_SQL % ( idHash , idHash , idHashKeyName , idHash , idHashKeyName ) <EOL> purgeQueries . append ( idGCQuery ) <EOL> purgeQueries . append ( idPurgeQuery ) <EOL> partJoinClauses = [ ] <EOL> idJoinColumnCandidates = [ ] <EOL> explicitJoins = [ ] <EOL> partJoinClauses . append ( "<STR_LIT>" % ( litPart , litPart . columnNames [ OBJECT ] , valueHash , valueHashKeyName ) ) <EOL> idJoinColumnCandidates . append ( "<STR_LIT>" % ( litPart , litPart . columnNames [ OBJECT ] ) ) <EOL> intersectionClause = "<STR_LIT>" . join ( [ col + "<STR_LIT>" <EOL> for col in idJoinColumnCandidates ] ) <EOL> valueGCQuery = VALUE_GARBAGE_COLLECTION_SQL % ( <EOL> valueHash , <EOL> valueHashKeyName , <EOL> valueHash , <EOL> "<STR_LIT>" % ( litPart , '<STR_LIT>' . join ( partJoinClauses ) ) , <EOL> intersectionClause <EOL> ) <EOL> valuePurgeQuery = PURGE_KEY_SQL % ( valueHash , valueHash , valueHashKeyName , valueHash , valueHashKeyName ) <EOL> purgeQueries . append ( "<STR_LIT>" ) <EOL> purgeQueries . append ( valueGCQuery ) <EOL> purgeQueries . append ( valuePurgeQuery ) <EOL> return purgeQueries <EOL> class Table ( object ) : <EOL> def get_name ( self ) : <EOL> '''<STR_LIT>''' <EOL> raise NotImplementedError ( '<STR_LIT>' ) <EOL> def createStatements ( self ) : <EOL> '''<STR_LIT>''' <EOL> raise NotImplementedError ( '<STR_LIT>' ) <EOL> def defaultStatements ( self ) : <EOL> '''<STR_LIT>''' <EOL> return [ ] <EOL> def indexingStatements ( self ) : <EOL> '''<STR_LIT>''' <EOL> return [ ] <EOL> def removeIndexingStatements ( self ) : <EOL> '''<STR_LIT>''' <EOL> return [ ] <EOL> def foreignKeyStatements ( self ) : <EOL> '''<STR_LIT>''' <EOL> return [ ] <EOL> def removeForeignKeyStatements ( self ) : <EOL> '''<STR_LIT>''' <EOL> return [ ] <EOL> class RelationalHash ( Table ) : <EOL> def __init__ ( <EOL> self , identifier , <EOL> useSignedInts = False , hashFieldType = '<STR_LIT>' , <EOL> engine = '<STR_LIT>' , declareEnums = False ) : <EOL> self . useSignedInts = useSignedInts <EOL> self . hashFieldType = hashFieldType <EOL> self . engine = engine <EOL> self . declareEnums = declareEnums <EOL> self . identifier = identifier <EOL> self . hashUpdateQueue = { } <EOL> def EscapeQuotes ( self , qstr ) : <EOL> return escape_quotes ( qstr ) <EOL> def normalizeTerm ( self , term ) : <EOL> if isinstance ( term , ( QuotedGraph , Graph ) ) : <EOL> return term . identifier . encode ( '<STR_LIT:utf-8>' ) <EOL> elif isinstance ( term , Literal ) : <EOL> return self . EscapeQuotes ( term ) . encode ( '<STR_LIT:utf-8>' ) <EOL> elif term is None or isinstance ( term , ( tuple , list , REGEXTerm ) ) : <EOL> return term <EOL> else : <EOL> return term . encode ( '<STR_LIT:utf-8>' ) <EOL> def __repr__ ( self ) : <EOL> return "<STR_LIT>" % ( self . identifier , self . tableNameSuffix ) <EOL> def get_name ( self ) : <EOL> return "<STR_LIT>" % ( self . identifier , self . tableNameSuffix ) <EOL> def indexingStatements ( self ) : <EOL> idxSQLStmts = [ ] <EOL> for colName , colType , indexMD in self . columns : <EOL> if indexMD : <EOL> indexName , indexCol = indexMD <EOL> if indexName : <EOL> idxSQLStmts . append ( "<STR_LIT>" % <EOL> ( self , indexName , self , indexCol ) ) <EOL> else : <EOL> idxSQLStmts . append ( <EOL> "<STR_LIT>" % <EOL> ( self , indexCol ) ) <EOL> return idxSQLStmts <EOL> def removeIndexingStatements ( self ) : <EOL> idxSQLStmts = [ ] <EOL> for colName , colType , indexMD in self . columns : <EOL> if indexMD : <EOL> indexName , indexCol = indexMD <EOL> if indexName : <EOL> idxSQLStmts . append ( "<STR_LIT>" % <EOL> ( self , indexName , self ) ) <EOL> else : <EOL> idxSQLStmts . append ( "<STR_LIT>" % <EOL> ( self , ) ) <EOL> return idxSQLStmts <EOL> def createStatements ( self ) : <EOL> statements = [ ] <EOL> columnSQLStmts = [ ] <EOL> for colName , colType , indexMD in self . columns : <EOL> if isinstance ( colType , list ) : <EOL> if self . declareEnums : <EOL> if False : <EOL> enumName = '<STR_LIT>' % ( colName , ) <EOL> statements . append ( <EOL> '<STR_LIT>' % <EOL> ( enumName , '<STR_LIT:U+002CU+0020>' . join ( <EOL> [ "<STR_LIT>" % item for item in colType ] ) ) ) <EOL> enumName = '<STR_LIT>' <EOL> columnSQLStmts . append ( "<STR_LIT>" % <EOL> ( colName , enumName ) ) <EOL> else : <EOL> columnSQLStmts . append ( "<STR_LIT>" % <EOL> ( colName , '<STR_LIT>' % '<STR_LIT:U+002CU+0020>' . join ( <EOL> [ "<STR_LIT>" % item for item in colType ] ) ) ) <EOL> else : <EOL> columnSQLStmts . append ( "<STR_LIT>" % <EOL> ( colName , colType ) ) <EOL> statements . append ( CREATE_HASH_TABLE % ( <EOL> self , <EOL> '<STR_LIT>' . join ( columnSQLStmts ) , <EOL> self . engine ) ) <EOL> return statements <EOL> def getRowsByHash ( self , db , hash ) : <EOL> c = db . cursor ( ) <EOL> keyCol = self . columns [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] <EOL> c . execute ( '<STR_LIT>' % ( self , keyCol , hash ) ) <EOL> return c . fetchall ( ) <EOL> def dropSQL ( self ) : <EOL> pass <EOL> class IdentifierHash ( RelationalHash ) : <EOL> tableNameSuffix = '<STR_LIT>' <EOL> def __init__ ( <EOL> self , identifier , <EOL> useSignedInts = False , hashFieldType = '<STR_LIT>' , <EOL> engine = '<STR_LIT>' , declareEnums = False ) : <EOL> super ( IdentifierHash , self ) . __init__ ( <EOL> identifier , useSignedInts , hashFieldType , engine , declareEnums ) <EOL> self . columns = [ <EOL> ( '<STR_LIT:id>' , self . hashFieldType , [ None , '<STR_LIT:id>' ] ) , <EOL> ( '<STR_LIT>' , <EOL> [ '<STR_LIT>' , '<STR_LIT:B>' , '<STR_LIT:F>' , '<STR_LIT>' , '<STR_LIT:L>' ] , <EOL> [ '<STR_LIT>' , '<STR_LIT>' ] ) , <EOL> ] <EOL> if REGEX_IDX : <EOL> self . columns . append ( <EOL> ( '<STR_LIT>' , '<STR_LIT:text>' , [ '<STR_LIT>' , '<STR_LIT>' ] ) , ) <EOL> else : <EOL> self . columns . append ( ( '<STR_LIT>' , '<STR_LIT:text>' , None ) ) <EOL> def viewUnionSelectExpression ( self , relations_only = False ) : <EOL> return "<STR_LIT>" % ( repr ( self ) ) <EOL> def defaultStatements ( self ) : <EOL> """<STR_LIT>""" <EOL> return [ "<STR_LIT>" % <EOL> ( self , normalizeValue ( RDF . type , '<STR_LIT>' , self . useSignedInts ) , <EOL> RDF . type ) ] <EOL> def generateDict ( self , db ) : <EOL> c = db . cursor ( ) <EOL> c . execute ( "<STR_LIT>" % self ) <EOL> rtDict = { } <EOL> for rt in c . fetchall ( ) : <EOL> rtDict [ rt [ <NUM_LIT:0> ] ] = ( rt [ <NUM_LIT:1> ] , rt [ <NUM_LIT:2> ] ) <EOL> c . close ( ) <EOL> return rtDict <EOL> def updateIdentifierQueue ( self , termList ) : <EOL> for term , termType in termList : <EOL> md5Int = normalizeValue ( term , termType , self . useSignedInts ) <EOL> self . hashUpdateQueue [ md5Int ] = ( termType , <EOL> self . normalizeTerm ( term ) ) <EOL> def insertIdentifiers ( self , db ) : <EOL> c = db . cursor ( ) <EOL> keyCol = self . columns [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] <EOL> if self . hashUpdateQueue : <EOL> params = [ ( md5Int , termType , lexical ) <EOL> for md5Int , ( termType , lexical ) <EOL> in self . hashUpdateQueue . items ( ) <EOL> if len ( self . getRowsByHash ( db , md5Int ) ) == <NUM_LIT:0> ] <EOL> if len ( params ) > <NUM_LIT:0> : <EOL> c . executemany ( <EOL> "<STR_LIT>" % ( self , ) + "<STR_LIT>" , params ) <EOL> if COLLISION_DETECTION : <EOL> insertedIds = self . hashUpdateQueue . keys ( ) <EOL> if len ( insertedIds ) > <NUM_LIT:1> : <EOL> c . execute ( "<STR_LIT>" % ( self ) + "<STR_LIT>" % keyCol + "<STR_LIT>" , ( tuple ( insertedIds ) , ) ) <EOL> else : <EOL> c . execute ( "<STR_LIT>" % ( self ) + "<STR_LIT>" % keyCol + "<STR_LIT>" , tuple ( insertedIds ) ) <EOL> for key , termType , lexical in c . fetchall ( ) : <EOL> if self . hashUpdateQueue [ key ] != ( termType , lexical ) : <EOL> raise Exception ( <EOL> "<STR_LIT>" % ( self , termType , lexical , <EOL> self . hashUpdateQueue [ key ] [ <NUM_LIT:0> ] , <EOL> self . hashUpdateQueue [ key ] [ <NUM_LIT:1> ] ) ) <EOL> self . hashUpdateQueue = { } <EOL> c . close ( ) <EOL> class LiteralHash ( RelationalHash ) : <EOL> tableNameSuffix = '<STR_LIT>' <EOL> def __init__ ( <EOL> self , identifier , <EOL> useSignedInts = False , hashFieldType = '<STR_LIT>' , <EOL> engine = '<STR_LIT>' , declareEnums = False ) : <EOL> super ( LiteralHash , self ) . __init__ ( <EOL> identifier , useSignedInts , hashFieldType , engine , declareEnums ) <EOL> self . columns = [ ( '<STR_LIT:id>' , self . hashFieldType , [ None , '<STR_LIT:id>' ] ) , ] <EOL> if REGEX_IDX : <EOL> self . columns . append ( <EOL> ( '<STR_LIT>' , '<STR_LIT:text>' , [ '<STR_LIT>' , '<STR_LIT>' ] ) , ) <EOL> else : <EOL> self . columns . append ( ( '<STR_LIT>' , '<STR_LIT:text>' , None ) ) <EOL> def viewUnionSelectExpression ( self , relations_only = False ) : <EOL> return "<STR_LIT>" % ( self . columns [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] , repr ( self ) ) <EOL> def generateDict ( self , db ) : <EOL> c = db . cursor ( ) <EOL> c . execute ( "<STR_LIT>" % self ) <EOL> rtDict = { } <EOL> for rt in c . fetchall ( ) : <EOL> rtDict [ rt [ <NUM_LIT:0> ] ] = rt [ <NUM_LIT:1> ] <EOL> c . close ( ) <EOL> return rtDict <EOL> def updateIdentifierQueue ( self , termList ) : <EOL> for term , termType in termList : <EOL> md5Int = normalizeValue ( term , termType , self . useSignedInts ) <EOL> self . hashUpdateQueue [ md5Int ] = self . normalizeTerm ( term ) <EOL> def insertIdentifiers ( self , db ) : <EOL> c = db . cursor ( ) <EOL> keyCol = self . columns [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] <EOL> if self . hashUpdateQueue : <EOL> params = [ ( md5Int , lexical ) for md5Int , lexical <EOL> in self . hashUpdateQueue . items ( ) <EOL> if len ( self . getRowsByHash ( db , md5Int ) ) == <NUM_LIT:0> ] <EOL> if len ( params ) > <NUM_LIT:0> : <EOL> c . executemany ( <EOL> "<STR_LIT>" % ( self , ) + "<STR_LIT>" , params ) <EOL> if COLLISION_DETECTION : <EOL> insertedIds = self . hashUpdateQueue . keys ( ) <EOL> if len ( insertedIds ) > <NUM_LIT:1> : <EOL> c . execute ( "<STR_LIT>" % ( self ) + "<STR_LIT>" % keyCol + "<STR_LIT>" , ( tuple ( insertedIds ) , ) ) <EOL> else : <EOL> c . execute ( "<STR_LIT>" % ( self ) + "<STR_LIT>" % keyCol + "<STR_LIT>" , tuple ( insertedIds ) ) <EOL> for key , lexical in c . fetchall ( ) : <EOL> if self . hashUpdateQueue [ key ] != lexical : <EOL> raise Exception ( <EOL> "<STR_LIT>" % ( self , lexical , self . hashUpdateQueue [ key ] [ <NUM_LIT:0> ] ) ) <EOL> self . hashUpdateQueue = { } <EOL> c . close ( ) </s>
