<s> """<STR_LIT>""" <EOL> import os <EOL> import logging <EOL> logger = logging . getLogger ( "<STR_LIT>" ) <EOL> from fom . errors import Fluid412Error <EOL> from fom . mapping import Namespace , Tag , Object , tag_value <EOL> from flimp import NAMESPACE_DESC , TAG_DESC <EOL> def make_namespace ( path , name , desc ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> logger . info ( '<STR_LIT>' % path ) <EOL> ns_head , ns_tail = os . path . split ( path ) <EOL> result = Namespace ( path ) <EOL> result . create ( NAMESPACE_DESC % ( ns_tail , name , desc ) ) <EOL> logger . info ( '<STR_LIT>' ) <EOL> except Fluid412Error : <EOL> result = Namespace ( path ) <EOL> logger . info ( '<STR_LIT>' % result . path ) <EOL> return result <EOL> def make_tag ( parent_ns , name , dataset , desc , indexed = False ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> logger . info ( '<STR_LIT>' % ( name , parent_ns . path ) ) <EOL> tag = parent_ns . create_tag ( name , TAG_DESC % ( name , dataset , desc ) , False ) <EOL> logger . info ( '<STR_LIT>' % tag . path ) <EOL> except Fluid412Error : <EOL> tag = Tag ( parent_ns . path + '<STR_LIT:/>' + name ) <EOL> logger . info ( '<STR_LIT>' % tag . path ) <EOL> return tag <EOL> def make_namespace_path ( path , name , desc ) : <EOL> """<STR_LIT>""" <EOL> namespace_path = path . split ( '<STR_LIT:/>' ) <EOL> checked_namespaces = namespace_path [ <NUM_LIT:0> ] <EOL> for namespace in namespace_path [ <NUM_LIT:1> : ] : <EOL> if checked_namespaces : <EOL> ns_path = os . path . join ( checked_namespaces , namespace ) <EOL> else : <EOL> ns_path = namespace <EOL> ns = make_namespace ( ns_path , name , desc ) <EOL> checked_namespaces = ns . path <EOL> return ns <EOL> def process_data_list ( raw_data , root_path , name , desc , about , allowEmpty = True ) : <EOL> """<STR_LIT>""" <EOL> logger . info ( '<STR_LIT>' ) <EOL> tag_dict = create_schema ( raw_data , root_path , name , desc ) <EOL> logger . info ( '<STR_LIT>' % len ( tag_dict ) ) <EOL> logger . info ( tag_dict . keys ( ) ) <EOL> logger . info ( '<STR_LIT>' ) <EOL> fom_class = create_class ( tag_dict ) <EOL> logger . info ( dir ( fom_class ) ) <EOL> logger . info ( '<STR_LIT>' ) <EOL> push_to_fluiddb ( raw_data , root_path , fom_class , about , name , allowEmpty ) <EOL> def validate ( raw_data ) : <EOL> """<STR_LIT>""" <EOL> default = raw_data [ <NUM_LIT:0> ] <EOL> missing_log = [ ] <EOL> extras_log = [ ] <EOL> for record in raw_data [ <NUM_LIT:1> : ] : <EOL> validate_dict ( default , record , record , missing_log , extras_log ) <EOL> return missing_log , extras_log <EOL> def validate_dict ( template , to_be_checked , parent , missing_log , extras_log ) : <EOL> """<STR_LIT>""" <EOL> for k in to_be_checked : <EOL> k = k . strip ( ) <EOL> if not k in template : <EOL> extras_log . append ( "<STR_LIT>" % ( k , parent ) ) <EOL> for k in template : <EOL> k = k . strip ( ) <EOL> if not k in to_be_checked : <EOL> missing_log . append ( "<STR_LIT>" % ( k , parent ) ) <EOL> for key , val in template . iteritems ( ) : <EOL> if isinstance ( val , dict ) : <EOL> validate_dict ( val , to_be_checked [ key ] , parent , missing_log , extras_log ) <EOL> def create_schema ( raw_data , root_path , name , desc ) : <EOL> """<STR_LIT>""" <EOL> template = raw_data [ <NUM_LIT:0> ] <EOL> root_ns = make_namespace_path ( root_path , name , desc ) <EOL> tags = { } <EOL> generate ( root_ns , None , template , desc , name , tags ) <EOL> return tags <EOL> def generate ( parent , child_name , template , description , name , tags ) : <EOL> """<STR_LIT>""" <EOL> if child_name : <EOL> ns = make_namespace ( '<STR_LIT:/>' . join ( [ parent . path , child_name ] ) , name , description ) <EOL> else : <EOL> ns = parent <EOL> for key , value in template . iteritems ( ) : <EOL> key = key . strip ( ) <EOL> if isinstance ( value , dict ) : <EOL> generate ( ns , key , value , description , name , tags ) <EOL> else : <EOL> tag = make_tag ( ns , key , description , False ) <EOL> defaultType = None <EOL> if isinstance ( value , list ) and not all ( isinstance ( x , basestring ) for <EOL> x in value ) : <EOL> defaultType = '<STR_LIT:application/json>' <EOL> logger . info ( "<STR_LIT>" <EOL> "<STR_LIT>" % defaultType ) <EOL> attribute_name = tag . path <EOL> logger . info ( '<STR_LIT>' % ( tag . path , attribute_name ) ) <EOL> tags [ attribute_name ] = tag_value ( tag . path , defaultType ) <EOL> def create_class ( tags ) : <EOL> """<STR_LIT>""" <EOL> return type ( '<STR_LIT>' , ( Object , ) , tags ) <EOL> def push_to_fluiddb ( raw_data , root_path , klass , about , name , allowEmpty = True ) : <EOL> """<STR_LIT>""" <EOL> length = len ( raw_data ) <EOL> for counter , item in enumerate ( raw_data ) : <EOL> logger . info ( "<STR_LIT>" % ( counter + <NUM_LIT:1> , length ) ) <EOL> if about : <EOL> about_value = "<STR_LIT>" % ( name , item [ about ] ) <EOL> logger . info ( '<STR_LIT>' % <EOL> about_value ) <EOL> obj = klass ( about = about_value ) <EOL> else : <EOL> logger . info ( '<STR_LIT>' ) <EOL> obj = klass ( ) <EOL> obj . create ( ) <EOL> logger . info ( '<STR_LIT>' % obj . uid ) <EOL> tag_values = get_values ( item , root_path ) <EOL> for key , value in tag_values . iteritems ( ) : <EOL> set_tag_value ( klass , obj , key , value , allowEmpty ) <EOL> if about : <EOL> logger . info ( '<STR_LIT>' % <EOL> ( about_value , obj . uid ) ) <EOL> else : <EOL> logger . info ( '<STR_LIT>' % <EOL> obj . uid ) <EOL> def set_tag_value ( klass , obj , key , value , allowEmpty ) : <EOL> if key in klass . __dict__ : <EOL> if allowEmpty or not value is None : <EOL> setattr ( obj , key , value ) <EOL> logger . info ( '<STR_LIT>' % ( key , value ) ) <EOL> else : <EOL> logger . info ( '<STR_LIT>' % key ) <EOL> else : <EOL> logger . error ( '<STR_LIT>' % key ) <EOL> def get_values ( item , parent ) : <EOL> """<STR_LIT>""" <EOL> vals = { } <EOL> for key , value in item . iteritems ( ) : <EOL> if isinstance ( value , dict ) : <EOL> vals . update ( get_values ( value , os . path . join ( parent , key ) ) ) <EOL> else : <EOL> vals [ os . path . join ( parent , key ) ] = value <EOL> return vals </s>
