<s> '''<STR_LIT>''' <EOL> import os , io , time , json , socket , logging , zlib , datetime <EOL> from arelle . ModelDtsObject import ModelConcept , ModelResource , ModelRelationship <EOL> from arelle . ModelInstanceObject import ModelFact <EOL> from arelle . ModelDocument import Type <EOL> from arelle import XbrlConst , XmlUtil , UrlUtil <EOL> import urllib . request <EOL> from urllib . error import HTTPError , URLError <EOL> from lxml import etree <EOL> from decimal import Decimal <EOL> import datetime <EOL> TRACEJSONFILE = None <EOL> JSONFILE_HOSTNAME = "<STR_LIT>" <EOL> def insertIntoDB ( modelXbrl , <EOL> user = None , password = None , host = None , port = None , database = None , timeout = None , <EOL> product = None , rssItem = None , ** kwargs ) : <EOL> jsondb = None <EOL> try : <EOL> jsondb = XbrlSemanticJsonDatabaseConnection ( modelXbrl , user , password , host , port , database , timeout ) <EOL> jsondb . insertXbrl ( rssItem = rssItem ) <EOL> jsondb . close ( ) <EOL> except Exception as ex : <EOL> if jsondb is not None : <EOL> try : <EOL> jsondb . close ( rollback = True ) <EOL> except Exception as ex2 : <EOL> pass <EOL> raise <EOL> def isDBPort ( host , port , db , timeout = <NUM_LIT:10> ) : <EOL> if host == JSONFILE_HOSTNAME : <EOL> return True <EOL> t = <NUM_LIT:2> <EOL> while t < timeout : <EOL> try : <EOL> conn = urllib . request . urlopen ( "<STR_LIT>" . format ( host , port or '<STR_LIT>' , db ) ) <EOL> return True <EOL> except HTTPError : <EOL> return False <EOL> except URLError : <EOL> return False <EOL> except socket . timeout : <EOL> t = t + <NUM_LIT:2> <EOL> return False <EOL> FILINGS = "<STR_LIT>" <EOL> DOCUMENTS = "<STR_LIT>" <EOL> def modelObjectDocumentUri ( modelObject ) : <EOL> return UrlUtil . ensureUrl ( modelObject . modelDocument . uri ) <EOL> def modelObjectUri ( modelObject ) : <EOL> return '<STR_LIT:#>' . join ( ( modelObjectDocumentUri ( modelObject ) , <EOL> XmlUtil . elementFragmentIdentifier ( modelObject ) ) ) <EOL> def qnameUri ( qname , sep = '<STR_LIT:#>' ) : <EOL> return sep . join ( ( qname . namespaceURI , qname . localName ) ) <EOL> def qnamePrefix_Name ( qname , sep = '<STR_LIT::>' ) : <EOL> prefix = { XbrlConst . xsd : '<STR_LIT>' , <EOL> XbrlConst . xml : '<STR_LIT>' , <EOL> XbrlConst . xbrli : '<STR_LIT>' , <EOL> XbrlConst . link : '<STR_LIT>' , <EOL> XbrlConst . gen : '<STR_LIT>' , <EOL> XbrlConst . xlink : '<STR_LIT>' <EOL> } . get ( qname . namespaceURI , qname . prefix ) <EOL> return sep . join ( ( prefix , qname . localName ) ) <EOL> def modelObjectQnameUri ( modelObject , sep = '<STR_LIT:#>' ) : <EOL> return qnameUri ( modelObject . qname , sep ) <EOL> def modelObjectNameUri ( modelObject , sep = '<STR_LIT:#>' ) : <EOL> return '<STR_LIT:#>' . join ( ( modelObjectDocumentUri ( modelObject ) , <EOL> modelObject . name ) ) <EOL> class XJDBException ( Exception ) : <EOL> def __init__ ( self , code , message , ** kwargs ) : <EOL> self . code = code <EOL> self . message = message <EOL> self . kwargs = kwargs <EOL> self . args = ( self . __repr__ ( ) , ) <EOL> def __repr__ ( self ) : <EOL> return _ ( '<STR_LIT>' ) . format ( self . code , self . message % self . kwargs ) <EOL> def jsonDefaultEncoder ( obj ) : <EOL> if isinstance ( obj , Decimal ) : <EOL> return float ( obj ) <EOL> elif isinstance ( obj , ( datetime . date , datetime . datetime ) ) : <EOL> return XmlUtil . dateunionValue ( obj ) <EOL> raise TypeError ( "<STR_LIT>" . format ( type ( obj ) . __name__ ) ) <EOL> class XbrlSemanticJsonDatabaseConnection ( ) : <EOL> def __init__ ( self , modelXbrl , user , password , host , port , database , timeout ) : <EOL> self . modelXbrl = modelXbrl <EOL> self . disclosureSystem = modelXbrl . modelManager . disclosureSystem <EOL> self . isJsonFile = host == JSONFILE_HOSTNAME <EOL> if self . isJsonFile : <EOL> self . jsonFile = database <EOL> else : <EOL> connectionUrl = "<STR_LIT>" . format ( host , port or '<STR_LIT>' ) <EOL> self . url = connectionUrl + '<STR_LIT:/>' + database <EOL> auth_handler = urllib . request . HTTPBasicAuthHandler ( ) <EOL> if user : <EOL> auth_handler . add_password ( realm = None , <EOL> uri = connectionUrl , <EOL> user = user , <EOL> passwd = password ) <EOL> self . conn = urllib . request . build_opener ( auth_handler ) <EOL> self . timeout = timeout or <NUM_LIT> <EOL> self . verticePropTypes = { } <EOL> def close ( self , rollback = False ) : <EOL> try : <EOL> if not self . isJsonFile : <EOL> self . conn . close ( ) <EOL> self . __dict__ . clear ( ) <EOL> except Exception as ex : <EOL> self . __dict__ . clear ( ) <EOL> raise <EOL> @ property <EOL> def isClosed ( self ) : <EOL> return not bool ( self . __dict__ ) <EOL> def showStatus ( self , msg , clearAfter = None ) : <EOL> self . modelXbrl . modelManager . showStatus ( msg , clearAfter ) <EOL> def execute ( self , activity , graph = None , query = None ) : <EOL> if graph is not None : <EOL> headers = { '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT:application/json>' , <EOL> '<STR_LIT:Content-Type>' : "<STR_LIT>" } <EOL> data = _STR_UNICODE ( json . dumps ( graph , <EOL> sort_keys = True , <EOL> ensure_ascii = False , <EOL> indent = <NUM_LIT:2> , <EOL> default = jsonDefaultEncoder ) ) <EOL> elif query is not None : <EOL> headers = { '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT:application/json>' } <EOL> data = ( "<STR_LIT>" + query ) <EOL> else : <EOL> return None <EOL> if TRACEJSONFILE : <EOL> with io . open ( TRACEJSONFILE , '<STR_LIT>' , encoding = '<STR_LIT:utf-8>' ) as fh : <EOL> fh . write ( "<STR_LIT>" ) <EOL> fh . write ( data ) <EOL> if self . isJsonFile and data is not None : <EOL> with io . open ( self . jsonFile , '<STR_LIT>' , encoding = '<STR_LIT:utf-8>' ) as fh : <EOL> fh . write ( data ) <EOL> return None <EOL> if graph is not None or query is not None : <EOL> url = self . url + "<STR_LIT>" <EOL> request = urllib . request . Request ( url , <EOL> data = data . encode ( '<STR_LIT:utf-8>' ) , <EOL> headers = headers ) <EOL> try : <EOL> with self . conn . open ( request , timeout = self . timeout ) as fp : <EOL> results = fp . read ( ) . decode ( '<STR_LIT:utf-8>' ) <EOL> try : <EOL> results = json . loads ( results ) <EOL> except ValueError : <EOL> pass <EOL> except HTTPError as err : <EOL> results = err . fp . read ( ) . decode ( '<STR_LIT:utf-8>' ) <EOL> if TRACEJSONFILE : <EOL> with io . open ( TRACEJSONFILE , "<STR_LIT:a>" , encoding = '<STR_LIT:utf-8>' ) as fh : <EOL> fh . write ( "<STR_LIT>" . format ( str ( results ) ) ) <EOL> if isinstance ( results , str ) and query is not None : <EOL> parser = etree . HTMLParser ( ) <EOL> htmlDoc = etree . parse ( io . StringIO ( results ) , parser ) <EOL> body = htmlDoc . find ( "<STR_LIT>" ) <EOL> if body is not None : <EOL> error = "<STR_LIT>" . join ( text for text in body . itertext ( ) ) <EOL> else : <EOL> error = results <EOL> raise XJDBException ( "<STR_LIT>" , <EOL> _ ( "<STR_LIT>" ) , <EOL> activity = activity , error = error ) <EOL> return results <EOL> def commit ( self , graph ) : <EOL> self . execute ( "<STR_LIT>" , graph = graph ) <EOL> def loadGraphRootVertices ( self ) : <EOL> self . showStatus ( "<STR_LIT>" ) <EOL> pass <EOL> def getDBsize ( self ) : <EOL> self . showStatus ( "<STR_LIT>" ) <EOL> return <NUM_LIT:0> <EOL> def insertXbrl ( self , rssItem ) : <EOL> try : <EOL> from arelle import ValidateXbrlDimensions <EOL> ValidateXbrlDimensions . loadDimensionDefaults ( self . modelXbrl ) <EOL> startedAt = time . time ( ) <EOL> self . identifyPreexistingDocuments ( ) <EOL> g = { FILINGS : { } , <EOL> DOCUMENTS : { } } <EOL> self . insertSchema ( g ) <EOL> self . insertFiling ( rssItem , g ) <EOL> self . insertDocuments ( g ) <EOL> self . insertDataDictionary ( ) <EOL> self . modelXbrl . profileStat ( _ ( "<STR_LIT>" ) , time . time ( ) - startedAt ) <EOL> startedAt = time . time ( ) <EOL> self . insertDataPoints ( ) <EOL> self . modelXbrl . profileStat ( _ ( "<STR_LIT>" ) , time . time ( ) - startedAt ) <EOL> startedAt = time . time ( ) <EOL> self . insertRelationshipSets ( ) <EOL> self . modelXbrl . profileStat ( _ ( "<STR_LIT>" ) , time . time ( ) - startedAt ) <EOL> self . insertValidationResults ( ) <EOL> self . modelXbrl . profileStat ( _ ( "<STR_LIT>" ) , time . time ( ) - startedAt ) <EOL> self . showStatus ( "<STR_LIT>" ) <EOL> self . commit ( g ) <EOL> self . modelXbrl . profileStat ( _ ( "<STR_LIT>" ) , time . time ( ) - startedAt ) <EOL> self . showStatus ( "<STR_LIT>" , clearAfter = <NUM_LIT> ) <EOL> except Exception as ex : <EOL> self . showStatus ( "<STR_LIT>" , clearAfter = <NUM_LIT> ) <EOL> raise <EOL> def insertSchema ( self , g ) : <EOL> if True : <EOL> self . showStatus ( "<STR_LIT>" ) <EOL> def insertFiling ( self , rssItem , g ) : <EOL> self . showStatus ( "<STR_LIT>" ) <EOL> new_filing = { '<STR_LIT>' : [ ] } <EOL> if self . modelXbrl . modelDocument . creationSoftwareComment : <EOL> new_filing [ '<STR_LIT>' ] = self . modelXbrl . modelDocument . creationSoftwareComment <EOL> datetimeNow = datetime . datetime . now ( ) <EOL> datetimeNowStr = XmlUtil . dateunionValue ( datetimeNow ) <EOL> entryUri = modelObjectDocumentUri ( self . modelXbrl ) <EOL> if rssItem is not None : <EOL> new_filing [ '<STR_LIT>' ] = "<STR_LIT>" <EOL> new_filing [ '<STR_LIT>' ] = filingNumber = rssItem . accessionNumber <EOL> new_filing [ '<STR_LIT>' ] = XmlUtil . dateunionValue ( rssItem . acceptanceDatetime ) <EOL> new_filing [ '<STR_LIT>' ] = XmlUtil . dateunionValue ( rssItem . filingDate ) <EOL> new_filing [ '<STR_LIT>' ] = rssItem . cikNumber <EOL> new_filing [ '<STR_LIT>' ] = rssItem . companyName <EOL> new_filing [ '<STR_LIT>' ] = rssItem . assignedSic <EOL> new_filing [ '<STR_LIT>' ] = rssItem . htmlUrl <EOL> new_filing [ '<STR_LIT>' ] = rssItem . url <EOL> self . filingURI = rssItem . htmlUrl <EOL> else : <EOL> intNow = int ( time . time ( ) ) <EOL> new_filing [ '<STR_LIT>' ] = filingNumber = str ( intNow ) <EOL> self . filingId = int ( time . time ( ) ) <EOL> new_filing [ '<STR_LIT>' ] = "<STR_LIT>" <EOL> new_filing [ '<STR_LIT>' ] = datetimeNowStr <EOL> new_filing [ '<STR_LIT>' ] = datetimeNowStr <EOL> new_filing [ '<STR_LIT>' ] = UrlUtil . ensureUrl ( self . modelXbrl . fileSource . url ) <EOL> self . filingURI = filingNumber <EOL> g [ FILINGS ] [ self . filingURI ] = new_filing <EOL> self . filing = new_filing <EOL> self . reportURI = modelObjectDocumentUri ( self . modelXbrl ) <EOL> self . report = { '<STR_LIT>' : self . filingURI , <EOL> '<STR_LIT>' : { } , <EOL> '<STR_LIT>' : { } , <EOL> '<STR_LIT>' : { } , <EOL> '<STR_LIT>' : { } } <EOL> new_filing [ '<STR_LIT>' ] = { self . reportURI : self . report } <EOL> self . relationshipSets = [ ( arcrole , ELR , linkqname , arcqname ) <EOL> for arcrole , ELR , linkqname , arcqname in self . modelXbrl . baseSets . keys ( ) <EOL> if ELR and ( arcrole . startswith ( "<STR_LIT>" ) or ( linkqname and arcqname ) ) ] <EOL> def identifyPreexistingDocuments ( self ) : <EOL> self . existingDocumentUris = set ( ) <EOL> if not self . isJsonFile : <EOL> docFilters = [ ] <EOL> for modelDocument in self . modelXbrl . urlDocs . values ( ) : <EOL> if modelDocument . type == Type . SCHEMA : <EOL> docFilters . append ( '<STR_LIT>' . format ( UrlUtil . ensureUrl ( modelDocument . uri ) ) ) <EOL> results = self . execute ( <EOL> "<STR_LIT>" , <EOL> query = """<STR_LIT>""" + '<STR_LIT>' . join ( docFilters ) + "<STR_LIT>" ) <EOL> try : <EOL> for result in results [ '<STR_LIT>' ] [ '<STR_LIT>' ] : <EOL> doc = result [ '<STR_LIT>' ] <EOL> if doc . get ( '<STR_LIT:type>' ) == '<STR_LIT>' : <EOL> self . existingDocumentUris . add ( doc [ '<STR_LIT:value>' ] ) <EOL> except KeyError : <EOL> pass <EOL> def insertDocuments ( self , g ) : <EOL> self . showStatus ( "<STR_LIT>" ) <EOL> documents = self . documents = g [ DOCUMENTS ] <EOL> for modelDocument in self . modelXbrl . urlDocs . values ( ) : <EOL> docUri = modelObjectDocumentUri ( modelDocument ) <EOL> if docUri not in self . existingDocumentUris : <EOL> documents [ docUri ] = { <EOL> '<STR_LIT:url>' : docUri , <EOL> '<STR_LIT>' : Type . typeName [ modelDocument . type ] , <EOL> '<STR_LIT>' : [ modelObjectDocumentUri ( doc ) <EOL> for doc , ref in modelDocument . referencesDocument . items ( ) <EOL> if doc . inDTS and ref . referenceType in ( "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ) ] , <EOL> '<STR_LIT>' : { } <EOL> } <EOL> self . filing [ '<STR_LIT>' ] . append ( docUri ) <EOL> if modelDocument . uri == self . modelXbrl . modelDocument . uri : <EOL> self . report [ '<STR_LIT>' ] = docUri <EOL> def conceptsUsed ( self ) : <EOL> conceptsUsed = set ( f . qname for f in self . modelXbrl . factsInInstance ) <EOL> for cntx in self . modelXbrl . contexts . values ( ) : <EOL> for dim in cntx . qnameDims . values ( ) : <EOL> conceptsUsed . add ( dim . dimensionQname ) <EOL> if dim . isExplicit : <EOL> conceptsUsed . add ( dim . memberQname ) <EOL> else : <EOL> conceptsUsed . add ( dim . typedMember . qname ) <EOL> for defaultDim , defaultDimMember in self . modelXbrl . qnameDimensionDefaults . items ( ) : <EOL> conceptsUsed . add ( defaultDim ) <EOL> conceptsUsed . add ( defaultDimMember ) <EOL> for roleTypes in ( self . modelXbrl . roleTypes , self . modelXbrl . arcroleTypes ) : <EOL> for modelRoleTypes in roleTypes . values ( ) : <EOL> for modelRoleType in modelRoleTypes : <EOL> for qn in modelRoleType . usedOns : <EOL> conceptsUsed . add ( qn ) <EOL> for relationshipSetKey in self . relationshipSets : <EOL> relationshipSet = self . modelXbrl . relationshipSet ( * relationshipSetKey ) <EOL> for rel in relationshipSet . modelRelationships : <EOL> if isinstance ( rel . fromModelObject , ModelConcept ) : <EOL> conceptsUsed . add ( rel . fromModelObject ) <EOL> if isinstance ( rel . toModelObject , ModelConcept ) : <EOL> conceptsUsed . add ( rel . toModelObject ) <EOL> for qn in ( XbrlConst . qnXbrliIdentifier , XbrlConst . qnXbrliPeriod , XbrlConst . qnXbrliUnit ) : <EOL> conceptsUsed . add ( qn ) <EOL> conceptsUsed -= { None } <EOL> return conceptsUsed <EOL> def insertDataDictionary ( self ) : <EOL> self . type_id = { } <EOL> self . aspect_id = { } <EOL> self . aspect_proxy = { } <EOL> self . aspect_proxy_uri = { } <EOL> self . roleType_id = { } <EOL> self . arcroleType_id = { } <EOL> '''<STR_LIT>''' <EOL> conceptsUsed = self . conceptsUsed ( ) <EOL> for modelDocument in self . modelXbrl . urlDocs . values ( ) : <EOL> self . showStatus ( "<STR_LIT>" + modelDocument . basename ) <EOL> docUri = modelObjectDocumentUri ( modelDocument ) <EOL> document = self . documents [ docUri ] <EOL> if modelDocument . type == Type . SCHEMA : <EOL> isNewDocument = True <EOL> modelConcepts = [ modelConcept <EOL> for modelConcept in self . modelXbrl . qnameConcepts . values ( ) <EOL> if modelConcept . modelDocument is modelDocument and <EOL> ( isNewDocument or modelConcept in conceptsUsed ) ] <EOL> if docUri not in self . existingDocumentUris : <EOL> document [ '<STR_LIT>' ] = dataTypes = { } <EOL> for modelType in self . modelXbrl . qnameTypes . values ( ) : <EOL> if modelType . modelDocument is modelDocument : <EOL> dataTypes [ modelType . name ] = dataType = { <EOL> '<STR_LIT>' : modelObjectNameUri ( modelType ) , <EOL> '<STR_LIT>' : modelObjectDocumentUri ( modelType ) , <EOL> '<STR_LIT:url>' : modelObjectUri ( modelType ) , <EOL> '<STR_LIT>' : modelType . qname . namespaceURI , <EOL> '<STR_LIT>' : modelType . name , <EOL> } <EOL> xbrliBaseType = modelType . baseXbrliTypeQname <EOL> if not isinstance ( xbrliBaseType , ( tuple , list ) ) : <EOL> xbrliBaseType = ( xbrliBaseType , ) <EOL> for baseType in xbrliBaseType : <EOL> if baseType is not None : <EOL> dataType [ '<STR_LIT>' ] = qnameUri ( baseType ) <EOL> if baseType . namespaceURI == "<STR_LIT>" : <EOL> dataType [ '<STR_LIT>' ] = qnameUri ( baseType ) <EOL> typeDerivedFrom = modelType . typeDerivedFrom <EOL> if not isinstance ( typeDerivedFrom , ( tuple , list ) ) : <EOL> typeDerivedFrom = ( typeDerivedFrom , ) <EOL> for dt in typeDerivedFrom : <EOL> if dt is not None : <EOL> dataType [ '<STR_LIT>' ] = modelObjectNameUri ( dt ) <EOL> for prop in ( '<STR_LIT>' , '<STR_LIT>' ) : <EOL> propertyValue = getattr ( modelType , prop , None ) <EOL> if propertyValue : <EOL> dataType [ prop ] = propertyValue <EOL> document [ '<STR_LIT>' ] = aspects = { } <EOL> for modelConcept in modelConcepts : <EOL> aspects [ modelConcept . name ] = aspect = { <EOL> '<STR_LIT>' : modelObjectDocumentUri ( modelConcept ) , <EOL> '<STR_LIT:url>' : modelObjectUri ( modelConcept ) , <EOL> '<STR_LIT>' : modelConcept . qname . namespaceURI , <EOL> '<STR_LIT>' : modelConcept . name , <EOL> '<STR_LIT>' : modelConcept . isAbstract <EOL> } <EOL> if modelConcept . periodType : <EOL> aspect [ '<STR_LIT>' ] = modelConcept . periodType <EOL> if modelConcept . balance : <EOL> aspect [ '<STR_LIT>' ] = modelConcept . balance <EOL> for prop in ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) : <EOL> propertyValue = getattr ( modelConcept , prop , None ) <EOL> if propertyValue : <EOL> aspect [ prop ] = propertyValue <EOL> conceptType = modelConcept . type <EOL> if conceptType is not None : <EOL> aspect [ '<STR_LIT>' ] = modelObjectNameUri ( conceptType ) <EOL> substitutionGroup = modelConcept . substitutionGroup <EOL> if substitutionGroup is not None : <EOL> aspect [ '<STR_LIT>' ] = modelObjectNameUri ( substitutionGroup ) <EOL> document [ '<STR_LIT>' ] = roleTypes = { } <EOL> for modelRoleTypes in self . modelXbrl . roleTypes . values ( ) : <EOL> for modelRoleType in modelRoleTypes : <EOL> roleTypes [ modelRoleType . roleURI ] = roleType = { <EOL> '<STR_LIT>' : modelObjectDocumentUri ( modelRoleType ) , <EOL> '<STR_LIT:url>' : modelObjectUri ( modelRoleType ) , <EOL> '<STR_LIT>' : modelRoleType . roleURI , <EOL> '<STR_LIT>' : modelRoleType . definition , <EOL> '<STR_LIT>' : [ modelObjectUri ( self . modelXbrl . qnameConcepts [ qn ] ) <EOL> for qn in modelRoleType . usedOns ] <EOL> } <EOL> document [ '<STR_LIT>' ] = arcroleTypes = { } <EOL> for modelArcroleTypes in self . modelXbrl . arcroleTypes . values ( ) : <EOL> for modelArcroleType in modelArcroleTypes : <EOL> arcroleTypes [ modelRoleType . roleURI ] = arcroleType = { <EOL> '<STR_LIT>' : modelObjectDocumentUri ( modelArcroleType ) , <EOL> '<STR_LIT:url>' : modelObjectUri ( modelArcroleType ) , <EOL> '<STR_LIT>' : modelArcroleType . roleURI , <EOL> '<STR_LIT>' : modelArcroleType . definition , <EOL> '<STR_LIT>' : [ modelObjectUri ( self . modelXbrl . qnameConcepts [ qn ] ) <EOL> for qn in modelArcroleType . usedOns ] , <EOL> '<STR_LIT>' : modelArcroleType . cyclesAllowed <EOL> } <EOL> activity = "<STR_LIT>" + modelDocument . uri <EOL> '''<STR_LIT>''' <EOL> def insertAspectProxies ( self , qnames ) : <EOL> aspectQnames = [ qname <EOL> for qname in qnames <EOL> if qname not in self . aspect_proxy_uri and qname in self . modelXbrl . qnameConcepts ] <EOL> for qname in aspectQnames : <EOL> self . insertAspectProxy ( qname , qnamePrefix_Name ( qname ) ) <EOL> def insertAspectProxy ( self , aspectQName , aspectProxyUri ) : <EOL> concept = self . modelXbrl . qnameConcepts [ aspectQName ] <EOL> self . report [ '<STR_LIT>' ] [ aspectProxyUri ] = aspectProxy = { <EOL> '<STR_LIT>' : self . reportURI , <EOL> '<STR_LIT>' : modelObjectDocumentUri ( concept ) , <EOL> '<STR_LIT:name>' : concept . name <EOL> } <EOL> self . aspect_proxy [ aspectQName ] = aspectProxy <EOL> self . aspect_proxy_uri [ aspectQName ] = aspectProxyUri <EOL> return aspectProxy <EOL> def aspectQnameProxy ( self , qname ) : <EOL> if hasattr ( qname , "<STR_LIT>" ) : <EOL> return self . aspect_proxy . get ( qname . qname ) <EOL> elif qname in self . modelXbrl . qnameConcepts : <EOL> return self . aspect_proxy . get ( qname ) <EOL> return None <EOL> def aspectQnameProxyId ( self , qname ) : <EOL> if hasattr ( qname , "<STR_LIT>" ) : <EOL> return self . aspect_proxy_uri . get ( qname . qname ) <EOL> elif qname in self . modelXbrl . qnameConcepts : <EOL> return self . aspect_proxy_uri . get ( qname ) <EOL> return None <EOL> def insertDataPoints ( self ) : <EOL> self . showStatus ( "<STR_LIT>" ) <EOL> dimensions = [ ] <EOL> dimensionIds = { } <EOL> if self . modelXbrl . modelDocument . type in ( Type . INSTANCE , Type . INLINEXBRL ) : <EOL> contextAspectValueSelections = { } <EOL> unitIDs = set ( ) <EOL> periodProxies = { } <EOL> entityIdentifierAspectProxies = { } <EOL> dataPoints = self . report [ '<STR_LIT>' ] <EOL> for fact in self . modelXbrl . factsInInstance : <EOL> self . insertAspectProxies ( ( fact . qname , ) ) <EOL> factId = XmlUtil . elementFragmentIdentifier ( fact ) <EOL> dataPoints [ factId ] = dataPoint = { <EOL> '<STR_LIT>' : modelObjectDocumentUri ( fact ) , <EOL> '<STR_LIT:id>' : factId , <EOL> '<STR_LIT>' : fact . sourceline , <EOL> '<STR_LIT>' : modelObjectUri ( fact ) , <EOL> '<STR_LIT>' : self . aspectQnameProxyId ( fact . qname ) <EOL> } <EOL> context = fact . context <EOL> concept = fact . concept <EOL> if context is not None : <EOL> if context . entityIdentifier not in entityIdentifierAspectProxies : <EOL> entityScheme , entityIdentifier = context . entityIdentifier <EOL> entityIdentifierAspectProxy = "<STR_LIT>" . format ( <EOL> qnamePrefix_Name ( XbrlConst . qnXbrliIdentifier ) , <EOL> entityIdentifier ) <EOL> e = self . insertAspectProxy ( XbrlConst . qnXbrliIdentifier , entityIdentifierAspectProxy ) <EOL> e [ '<STR_LIT>' ] = entityScheme <EOL> e [ '<STR_LIT>' ] = entityIdentifier <EOL> entityIdentifierAspectProxies [ context . entityIdentifier ] = entityIdentifierAspectProxy <EOL> else : <EOL> entityIdentifierAspectProxy = entityIdentifierAspectProxies [ context . entityIdentifier ] <EOL> dataPoint [ '<STR_LIT>' ] = entityIdentifierAspectProxy <EOL> if context . isForeverPeriod : <EOL> period = "<STR_LIT>" <EOL> if context . isInstantPeriod : <EOL> endDate = XmlUtil . dateunionValue ( context . instantDatetime , subtractOneDay = True ) . replace ( '<STR_LIT::>' , '<STR_LIT:_>' ) <EOL> period = "<STR_LIT>" . format ( endDate ) <EOL> else : <EOL> startDate = XmlUtil . dateunionValue ( context . startDatetime ) . replace ( '<STR_LIT::>' , '<STR_LIT:_>' ) <EOL> endDate = XmlUtil . dateunionValue ( context . endDatetime , subtractOneDay = True ) . replace ( '<STR_LIT::>' , '<STR_LIT:_>' ) <EOL> period = "<STR_LIT>" . format ( startDate , endDate ) <EOL> if period not in periodProxies : <EOL> periodProxy = "<STR_LIT>" . format ( <EOL> qnamePrefix_Name ( XbrlConst . qnXbrliPeriod ) , <EOL> period ) <EOL> p = self . insertAspectProxy ( XbrlConst . qnXbrliPeriod , periodProxy ) <EOL> p [ '<STR_LIT>' ] = context . isForeverPeriod <EOL> p [ '<STR_LIT>' ] = context . isInstantPeriod <EOL> if context . isStartEndPeriod : <EOL> d = context . startDatetime <EOL> if d . hour == <NUM_LIT:0> and d . minute == <NUM_LIT:0> and d . second == <NUM_LIT:0> : <EOL> d = d . date ( ) <EOL> p [ '<STR_LIT>' ] = d <EOL> if context . isStartEndPeriod or context . isInstantPeriod : <EOL> d = context . endDatetime <EOL> if d . hour == <NUM_LIT:0> and d . minute == <NUM_LIT:0> and d . second == <NUM_LIT:0> : <EOL> d = ( d - datetime . timedelta ( <NUM_LIT:1> ) ) . date ( ) <EOL> p [ '<STR_LIT>' ] = d <EOL> periodProxies [ period ] = periodProxy <EOL> else : <EOL> periodProxy = periodProxies [ period ] <EOL> dataPoint [ '<STR_LIT>' ] = periodProxy <EOL> dataPoint [ '<STR_LIT>' ] = modelObjectUri ( context ) <EOL> dataPoint [ '<STR_LIT>' ] = context . id <EOL> if context . id not in contextAspectValueSelections : <EOL> contextAspectValueSelections [ context . id ] = contextAspectValueSelection = [ ] <EOL> for dimVal in context . qnameDims . values ( ) : <EOL> dim = qnamePrefix_Name ( dimVal . dimensionQname ) <EOL> if dimVal . isExplicit : <EOL> self . insertAspectProxies ( ( dimVal . memberQname , ) ) <EOL> v = self . aspectQnameProxyId ( dimVal . memberQname ) <EOL> else : <EOL> v = dimVal . typedMember . stringValue <EOL> dimProxy = "<STR_LIT>" . format ( dim , v ) <EOL> d = self . insertAspectProxy ( dimVal . dimensionQname , dimProxy ) <EOL> contextAspectValueSelection . append ( dimProxy ) <EOL> d [ '<STR_LIT>' ] = dim <EOL> if dimVal . isExplicit : <EOL> d [ '<STR_LIT>' ] = v <EOL> else : <EOL> d [ '<STR_LIT>' ] = v <EOL> else : <EOL> contextAspectValueSelection = contextAspectValueSelections [ context . id ] <EOL> dataPoint [ '<STR_LIT>' ] = contextAspectValueSelection <EOL> if fact . isNumeric : <EOL> if fact . precision == "<STR_LIT>" : <EOL> dataPoint [ '<STR_LIT>' ] = "<STR_LIT>" <EOL> elif fact . precision is not None : <EOL> dataPoint [ '<STR_LIT>' ] = fact . precision <EOL> if fact . decimals == "<STR_LIT>" : <EOL> dataPoint [ '<STR_LIT>' ] = "<STR_LIT>" <EOL> elif fact . decimals is not None : <EOL> dataPoint [ '<STR_LIT>' ] = fact . decimals <EOL> if fact . unit is not None : <EOL> unit = fact . unit <EOL> unitProxy = "<STR_LIT>" . format ( <EOL> qnamePrefix_Name ( XbrlConst . qnXbrliUnit ) , <EOL> unit . id ) <EOL> dataPoint [ '<STR_LIT>' ] = unitProxy <EOL> if unit . id not in unitIDs : <EOL> unitIDs . add ( unit . id ) <EOL> u = self . insertAspectProxy ( XbrlConst . qnXbrliUnit , unitProxy ) <EOL> u [ '<STR_LIT>' ] = unit . id <EOL> mults , divs = unit . measures <EOL> u [ '<STR_LIT>' ] = [ qnameUri ( qn ) for qn in mults ] <EOL> if divs : <EOL> u [ '<STR_LIT>' ] = [ qnameUri ( qn ) for qn in divs ] <EOL> if fact . xmlLang is None and fact . concept is not None and fact . concept . baseXsdType is not None : <EOL> dataPoint [ '<STR_LIT:value>' ] = fact . xValue <EOL> elif fact . xmlLang is not None : <EOL> dataPoint [ '<STR_LIT>' ] = fact . xmlLang <EOL> dataPoint [ '<STR_LIT:value>' ] = fact . value <EOL> else : <EOL> dataPoint [ '<STR_LIT:value>' ] = fact . value <EOL> if fact . modelTupleFacts : <EOL> dataPoint [ '<STR_LIT>' ] = [ XmlUtil . elementFragmentIdentifier ( tupleFact ) <EOL> for tupleFact in fact . modelTupleFacts ] <EOL> def resourceId ( self , i ) : <EOL> return "<STR_LIT>" . format ( accessionPrefix = self . thisAccessionPrefix , <EOL> i = i ) <EOL> def insertRelationshipSets ( self ) : <EOL> self . showStatus ( "<STR_LIT>" ) <EOL> aspectQnamesUsed = set ( ) <EOL> for i , relationshipSetKey in enumerate ( self . relationshipSets ) : <EOL> arcrole , linkrole , linkqname , arcqname = relationshipSetKey <EOL> if linkqname : <EOL> aspectQnamesUsed . add ( linkqname ) <EOL> if arcqname : <EOL> aspectQnamesUsed . add ( arcqname ) <EOL> self . insertAspectProxies ( aspectQnamesUsed ) <EOL> relationshipSets = self . report [ '<STR_LIT>' ] <EOL> relSetIds = { } <EOL> for i , relationshipSetKey in enumerate ( self . relationshipSets ) : <EOL> arcrole , linkrole , linkqname , arcqname = relationshipSetKey <EOL> if arcrole not in ( "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ) and linkrole and linkqname and arcqname : <EOL> relSetId = "<STR_LIT>" . format ( <EOL> os . path . basename ( arcrole ) , <EOL> os . path . basename ( linkrole ) ) <EOL> relSetIds [ relationshipSetKey ] = relSetId <EOL> relationshipSets [ relSetId ] = relationshipSet = { <EOL> '<STR_LIT>' : arcrole , <EOL> '<STR_LIT>' : linkrole , <EOL> '<STR_LIT>' : self . aspectQnameProxyId ( arcqname ) , <EOL> '<STR_LIT>' : self . aspectQnameProxyId ( linkqname ) , <EOL> '<STR_LIT>' : self . reportURI , <EOL> '<STR_LIT>' : [ ] , <EOL> '<STR_LIT>' : [ ] <EOL> } <EOL> relE = [ ] <EOL> resources = set ( ) <EOL> aspectQnamesUsed = set ( ) <EOL> resourceIDs = { } <EOL> def walkTree ( rels , parentRelId , seq , depth , relationshipSetKey , relationshipSet , visited , relSetId , doVertices ) : <EOL> for rel in rels : <EOL> if rel not in visited : <EOL> visited . add ( rel ) <EOL> if not doVertices : <EOL> _relProp = { '<STR_LIT>' : seq , <EOL> '<STR_LIT>' : depth , <EOL> '<STR_LIT>' : rel . orderDecimal , <EOL> '<STR_LIT>' : rel . priority , <EOL> '<STR_LIT>' : relSetId <EOL> } <EOL> if isinstance ( rel . fromModelObject , ModelConcept ) : <EOL> if doVertices : <EOL> aspectQnamesUsed . add ( rel . fromModelObject . qname ) <EOL> sourceUri = True <EOL> else : <EOL> sourceQname = rel . fromModelObject . qname <EOL> sourceUri = self . aspectQnameProxyId ( sourceQname ) <EOL> sourceId = qnamePrefix_Name ( rel . fromModelObject . qname ) <EOL> else : <EOL> sourceUri = None <EOL> toModelObject = rel . toModelObject <EOL> if isinstance ( toModelObject , ModelConcept ) : <EOL> if doVertices : <EOL> aspectQnamesUsed . add ( toModelObject . qname ) <EOL> targetUri = True <EOL> else : <EOL> targetUri = self . aspectQnameProxyId ( toModelObject . qname ) <EOL> targetId = qnamePrefix_Name ( toModelObject . qname ) <EOL> elif isinstance ( toModelObject , ModelResource ) : <EOL> if doVertices : <EOL> resources . add ( toModelObject ) <EOL> targetUri = <NUM_LIT:0> <EOL> else : <EOL> if rel . preferredLabel : <EOL> _relProp [ '<STR_LIT>' ] = rel . preferredLabel <EOL> if rel . arcrole in ( XbrlConst . all , XbrlConst . notAll ) : <EOL> _relProp [ '<STR_LIT>' ] = rel . closed <EOL> elif rel . arcrole in ( XbrlConst . dimensionDomain , XbrlConst . domainMember ) : <EOL> _relProp [ '<STR_LIT>' ] = rel . usable <EOL> elif rel . arcrole == XbrlConst . summationItem : <EOL> _relProp [ '<STR_LIT>' ] = rel . weightDecimal <EOL> if relationshipSet . arcrole == "<STR_LIT>" : <EOL> _relProp [ '<STR_LIT>' ] = rel . arcrole <EOL> if toModelObject . role : <EOL> _relProp [ '<STR_LIT>' ] = toModelObject . role <EOL> targetUri = modelObjectUri ( toModelObject ) <EOL> targetId = toModelObject . modelDocument . basename + '<STR_LIT:#>' + XmlUtil . elementFragmentIdentifier ( toModelObject ) <EOL> else : <EOL> targetUri = None <EOL> if sourceUri is not None and targetUri is not None : <EOL> targetRelSetId = relSetId <EOL> targetRelSetKey = relationshipSetKey <EOL> if relationshipSet . arcrole == "<STR_LIT>" and rel . targetRole : <EOL> targetRelSet = self . modelXbrl . relationshipSet ( relationshipSet . arcrole , rel . targetRole ) <EOL> for i , relSetKey in enumerate ( self . relationshipSets ) : <EOL> arcrole , ELR , linkqname , arcqname = relSetKey <EOL> if arcrole == "<STR_LIT>" and ELR == rel . targetRole : <EOL> targetRelationshipSetId = relSetIds [ relSetKey ] <EOL> targetRelSetKey = relSetKey <EOL> break <EOL> if not doVertices : <EOL> _relProp [ '<STR_LIT>' ] = rel . targetRole <EOL> _relProp [ '<STR_LIT>' ] = targetRelationshipSetId <EOL> else : <EOL> targetRelSetKey = relationshipSetKey <EOL> targetRelSet = relationshipSet <EOL> if doVertices : <EOL> relId = None <EOL> else : <EOL> _relProp [ '<STR_LIT>' ] = sourceUri <EOL> _relProp [ '<STR_LIT>' ] = sourceQname <EOL> _relProp [ '<STR_LIT:to>' ] = targetUri <EOL> _arcrole = os . path . basename ( rel . arcrole ) <EOL> relId = "<STR_LIT>" . format ( <EOL> _arcrole , <EOL> os . path . basename ( rel . linkrole ) , <EOL> sourceId , <EOL> targetId ) <EOL> _relProp [ '<STR_LIT>' ] = relId <EOL> _relProp [ '<STR_LIT>' ] = relationshipSetKey <EOL> relE . append ( _relProp ) <EOL> seq += <NUM_LIT:1> <EOL> seq = walkTree ( targetRelSet . fromModelObject ( toModelObject ) , relId , seq , depth + <NUM_LIT:1> , targetRelSetKey , targetRelSet , visited , targetRelSetId , doVertices ) <EOL> visited . remove ( rel ) <EOL> return seq <EOL> for doVertices in range ( <NUM_LIT:1> , - <NUM_LIT:1> , - <NUM_LIT:1> ) : <EOL> for i , relationshipSetKey in enumerate ( self . relationshipSets ) : <EOL> arcrole , linkrole , linkqname , arcqname = relationshipSetKey <EOL> if arcrole not in ( "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ) and linkrole and linkqname and arcqname : <EOL> relSetId = relSetIds [ relationshipSetKey ] <EOL> relationshipSet = self . modelXbrl . relationshipSet ( * relationshipSetKey ) <EOL> seq = <NUM_LIT:1> <EOL> for rootConcept in relationshipSet . rootConcepts : <EOL> seq = walkTree ( relationshipSet . fromModelObject ( rootConcept ) , None , seq , <NUM_LIT:1> , relationshipSetKey , relationshipSet , set ( ) , relSetId , doVertices ) <EOL> if doVertices : <EOL> if resources : <EOL> for resource in resources : <EOL> resourceUri = modelObjectUri ( resource ) <EOL> r = { '<STR_LIT:url>' : resourceUri , <EOL> '<STR_LIT:value>' : resource . stringValue <EOL> } <EOL> if resource . xmlLang : <EOL> r [ '<STR_LIT>' ] = resource . xmlLang <EOL> if resource . role : <EOL> r [ '<STR_LIT>' ] = resource . role <EOL> self . documents [ modelObjectDocumentUri ( resource ) ] [ '<STR_LIT>' ] [ <EOL> XmlUtil . elementFragmentIdentifier ( resource ) ] = r <EOL> self . insertAspectProxies ( aspectQnamesUsed ) <EOL> else : <EOL> for j , rel in enumerate ( relE ) : <EOL> relId = rel [ '<STR_LIT>' ] <EOL> relSetId = rel [ '<STR_LIT>' ] <EOL> relSet = relationshipSets [ relSetId ] <EOL> r = dict ( ( k , v ) <EOL> for k , v in rel . items ( ) <EOL> if k not in ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> relSet [ '<STR_LIT>' ] . append ( r ) <EOL> if rel . get ( '<STR_LIT>' , <NUM_LIT:0> ) == <NUM_LIT:1> : <EOL> relSet [ '<STR_LIT>' ] . append ( r ) <EOL> sourceQname = rel [ '<STR_LIT>' ] <EOL> if sourceQname in self . aspect_proxy : <EOL> self . aspect_proxy [ sourceQname ] . setdefault ( '<STR_LIT>' , { } ) . setdefault ( rel [ '<STR_LIT>' ] , [ ] ) . append ( rel [ '<STR_LIT:to>' ] ) <EOL> resourceIDs . clear ( ) <EOL> resources = None <EOL> def insertValidationResults ( self ) : <EOL> logEntries = [ ] <EOL> for handler in logging . getLogger ( "<STR_LIT>" ) . handlers : <EOL> if hasattr ( handler , "<STR_LIT>" ) : <EOL> logEntries = handler . dbHandlerLogEntries ( ) <EOL> break <EOL> messages = [ ] <EOL> messageRefs = [ ] <EOL> for i , logEntry in enumerate ( logEntries ) : <EOL> messageId = "<STR_LIT>" . format ( i + <NUM_LIT:1> ) <EOL> self . report [ '<STR_LIT>' ] [ messageId ] = m = { <EOL> '<STR_LIT:code>' : logEntry [ '<STR_LIT:code>' ] , <EOL> '<STR_LIT>' : logEntry [ '<STR_LIT>' ] , <EOL> '<STR_LIT:value>' : logEntry [ '<STR_LIT:message>' ] [ '<STR_LIT:text>' ] , <EOL> '<STR_LIT>' : self . reportURI , <EOL> '<STR_LIT>' : messageId <EOL> } <EOL> for ref in logEntry [ '<STR_LIT>' ] : <EOL> modelObject = self . modelXbrl . modelObject ( ref . get ( '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> aspectObj = None <EOL> if isinstance ( modelObject , ModelFact ) : <EOL> factId = XmlUtil . elementFragmentIdentifier ( modelObject ) <EOL> dataPoint = self . report [ '<STR_LIT>' ] [ factId ] <EOL> dataPoint . setdefault ( '<STR_LIT>' , [ ] ) . append ( messageId ) <EOL> elif isinstance ( modelObject , ModelConcept ) : <EOL> self . insertAspectProxies ( ( modelObject . qname , ) ) <EOL> self . aspectQnameProxy ( modelObject . qname ) . setdefault ( '<STR_LIT>' , [ ] ) . append ( messageId ) <EOL> elif isinstance ( modelObject , ModelRelationship ) : <EOL> '''<STR_LIT>''' <EOL> else : <EOL> continue <EOL> if messages : <EOL> self . showStatus ( "<STR_LIT>" ) </s>
