<s> """<STR_LIT>""" <EOL> import datetime <EOL> import os <EOL> import re <EOL> import sys <EOL> import urllib2 <EOL> try : <EOL> import json <EOL> except ImportError : <EOL> try : <EOL> import simplejson as json <EOL> except : <EOL> import gluon . contrib . simplejson as json <EOL> try : <EOL> from lxml import etree <EOL> except ImportError : <EOL> print >> sys . stderr , "<STR_LIT>" <EOL> raise <EOL> from gluon import * <EOL> from gluon . storage import Storage <EOL> from s3codec import S3Codec <EOL> from s3datetime import s3_decode_iso_datetime , s3_encode_iso_datetime , s3_utc <EOL> from s3fields import S3RepresentLazy <EOL> from s3utils import s3_get_foreign_key , s3_unicode , s3_strip_markup , s3_validate , s3_represent_value <EOL> ogetattr = object . __getattribute__ <EOL> SEPARATORS = ( "<STR_LIT:U+002C>" , "<STR_LIT::>" ) <EOL> class S3XML ( S3Codec ) : <EOL> """<STR_LIT>""" <EOL> namespace = "<STR_LIT>" <EOL> CACHE_TTL = <NUM_LIT:20> <EOL> UID = "<STR_LIT>" <EOL> MCI = "<STR_LIT>" <EOL> DELETED = "<STR_LIT>" <EOL> REPLACEDBY = "<STR_LIT>" <EOL> APPROVED = "<STR_LIT>" <EOL> CTIME = "<STR_LIT>" <EOL> CUSER = "<STR_LIT>" <EOL> MTIME = "<STR_LIT>" <EOL> MUSER = "<STR_LIT>" <EOL> OGROUP = "<STR_LIT>" <EOL> OUSER = "<STR_LIT>" <EOL> Lat = "<STR_LIT>" <EOL> Lon = "<STR_LIT>" <EOL> WKT = "<STR_LIT>" <EOL> IGNORE_FIELDS = [ <EOL> "<STR_LIT:id>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> ] <EOL> FIELDS_TO_ATTRIBUTES = [ <EOL> "<STR_LIT:id>" , <EOL> "<STR_LIT>" , <EOL> CUSER , <EOL> MUSER , <EOL> OGROUP , <EOL> OUSER , <EOL> CTIME , <EOL> MTIME , <EOL> UID , <EOL> MCI , <EOL> DELETED , <EOL> ] <EOL> ATTRIBUTES_TO_FIELDS = [ <EOL> "<STR_LIT>" , <EOL> CUSER , <EOL> MUSER , <EOL> OGROUP , <EOL> OUSER , <EOL> CTIME , <EOL> MTIME , <EOL> MCI , <EOL> DELETED , <EOL> APPROVED , <EOL> ] <EOL> TAG = Storage ( <EOL> col = "<STR_LIT>" , <EOL> contents = "<STR_LIT>" , <EOL> data = "<STR_LIT:data>" , <EOL> description = "<STR_LIT:description>" , <EOL> field = "<STR_LIT>" , <EOL> fields = "<STR_LIT>" , <EOL> item = "<STR_LIT>" , <EOL> list = "<STR_LIT:list>" , <EOL> meta = "<STR_LIT>" , <EOL> object = "<STR_LIT:object>" , <EOL> option = "<STR_LIT>" , <EOL> options = "<STR_LIT>" , <EOL> reference = "<STR_LIT>" , <EOL> resource = "<STR_LIT>" , <EOL> root = "<STR_LIT>" , <EOL> row = "<STR_LIT>" , <EOL> select = "<STR_LIT>" , <EOL> table = "<STR_LIT>" , <EOL> ) <EOL> ATTRIBUTE = Storage ( <EOL> alias = "<STR_LIT>" , <EOL> attributes = "<STR_LIT>" , <EOL> comment = "<STR_LIT>" , <EOL> domain = "<STR_LIT>" , <EOL> error = "<STR_LIT:error>" , <EOL> field = "<STR_LIT>" , <EOL> filename = "<STR_LIT:filename>" , <EOL> has_options = "<STR_LIT>" , <EOL> hashtag = "<STR_LIT>" , <EOL> id = "<STR_LIT:id>" , <EOL> label = "<STR_LIT:label>" , <EOL> lat = "<STR_LIT>" , <EOL> latmin = "<STR_LIT>" , <EOL> latmax = "<STR_LIT>" , <EOL> limit = "<STR_LIT>" , <EOL> lon = "<STR_LIT>" , <EOL> lonmin = "<STR_LIT>" , <EOL> lonmax = "<STR_LIT>" , <EOL> marker = "<STR_LIT>" , <EOL> marker_url = "<STR_LIT>" , <EOL> marker_height = "<STR_LIT>" , <EOL> marker_width = "<STR_LIT>" , <EOL> name = "<STR_LIT:name>" , <EOL> parent = "<STR_LIT>" , <EOL> ref = "<STR_LIT>" , <EOL> replaced_by = "<STR_LIT>" , <EOL> resource = "<STR_LIT>" , <EOL> results = "<STR_LIT>" , <EOL> sym = "<STR_LIT>" , <EOL> start = "<STR_LIT:start>" , <EOL> style = "<STR_LIT>" , <EOL> success = "<STR_LIT:success>" , <EOL> table = "<STR_LIT>" , <EOL> tuid = "<STR_LIT>" , <EOL> type = "<STR_LIT:type>" , <EOL> readable = "<STR_LIT>" , <EOL> url = "<STR_LIT:url>" , <EOL> value = "<STR_LIT:value>" , <EOL> writable = "<STR_LIT>" , <EOL> wkt = "<STR_LIT>" , <EOL> ) <EOL> ACTION = Storage ( <EOL> create = "<STR_LIT>" , <EOL> read = "<STR_LIT>" , <EOL> update = "<STR_LIT>" , <EOL> delete = "<STR_LIT>" , <EOL> ) <EOL> PREFIX = Storage ( <EOL> attribute = "<STR_LIT:@>" , <EOL> options = "<STR_LIT>" , <EOL> reference = "<STR_LIT>" , <EOL> resource = "<STR_LIT:$>" , <EOL> text = "<STR_LIT:$>" , <EOL> ) <EOL> def __init__ ( self ) : <EOL> """<STR_LIT>""" <EOL> self . domain = current . request . env . server_name <EOL> self . error = None <EOL> self . filter_mci = False <EOL> self . show_ids = False <EOL> self . show_urls = True <EOL> def parse ( self , source ) : <EOL> """<STR_LIT>""" <EOL> self . error = None <EOL> if isinstance ( source , basestring ) and source [ : <NUM_LIT:5> ] == "<STR_LIT>" : <EOL> try : <EOL> source = urllib2 . urlopen ( source ) <EOL> except : <EOL> pass <EOL> try : <EOL> parser = etree . XMLParser ( no_network = False , <EOL> remove_blank_text = True , <EOL> ) <EOL> result = etree . parse ( source , parser ) <EOL> return result <EOL> except : <EOL> e = sys . exc_info ( ) [ <NUM_LIT:1> ] <EOL> self . error = e <EOL> return None <EOL> def transform ( self , tree , stylesheet_path , ** args ) : <EOL> """<STR_LIT>""" <EOL> self . error = None <EOL> if args : <EOL> _args = dict ( ( k , "<STR_LIT>" % args [ k ] ) for k in args ) <EOL> else : <EOL> _args = None <EOL> if isinstance ( stylesheet_path , ( etree . _ElementTree , etree . _Element ) ) : <EOL> stylesheet = stylesheet_path <EOL> else : <EOL> stylesheet = self . parse ( stylesheet_path ) <EOL> if stylesheet is not None : <EOL> try : <EOL> ac = etree . XSLTAccessControl ( read_file = True , read_network = True ) <EOL> transformer = etree . XSLT ( stylesheet , access_control = ac ) <EOL> if _args : <EOL> result = transformer ( tree , ** _args ) <EOL> else : <EOL> result = transformer ( tree ) <EOL> return result <EOL> except : <EOL> e = sys . exc_info ( ) [ <NUM_LIT:1> ] <EOL> self . error = e <EOL> current . log . error ( e ) <EOL> return None <EOL> else : <EOL> return None <EOL> def envelope ( self , tree , stylesheet_path , ** args ) : <EOL> """<STR_LIT>""" <EOL> if not isinstance ( tree , ( list , tuple ) ) : <EOL> tree = [ tree ] <EOL> root = etree . Element ( self . TAG . contents ) <EOL> for subtree in tree : <EOL> contentDescription = None <EOL> if isinstance ( subtree , ( list , tuple ) ) : <EOL> contentObject = subtree [ <NUM_LIT:0> ] <EOL> if len ( subtree ) > <NUM_LIT:1> : <EOL> contentDescription = subtree [ <NUM_LIT:1> ] <EOL> if isinstance ( subtree , etree . _ElementTree ) : <EOL> contentObject = subtree . getroot ( ) <EOL> else : <EOL> contentObject = subtree <EOL> obj = etree . SubElement ( root , self . TAG . object ) <EOL> if contentDescription : <EOL> desc = etree . SubElement ( obj , self . TAG . description ) <EOL> desc . text = contentDescription <EOL> from copy import deepcopy <EOL> obj . append ( deepcopy ( contentObject ) ) <EOL> return self . transform ( root , stylesheet_path , ** args ) <EOL> @ staticmethod <EOL> def tostring ( tree , xml_declaration = True , pretty_print = True ) : <EOL> """<STR_LIT>""" <EOL> return etree . tostring ( tree , <EOL> xml_declaration = xml_declaration , <EOL> encoding = "<STR_LIT:utf-8>" , <EOL> pretty_print = pretty_print ) <EOL> def tree ( self , elements , <EOL> root = None , <EOL> domain = None , <EOL> url = None , <EOL> start = None , <EOL> limit = None , <EOL> results = None , <EOL> maxbounds = False ) : <EOL> """<STR_LIT>""" <EOL> ATTRIBUTE = self . ATTRIBUTE <EOL> success = False <EOL> if root is None : <EOL> root = etree . Element ( self . TAG . root ) <EOL> if elements is not None or len ( root ) : <EOL> success = True <EOL> set_attribute = root . set <EOL> set_attribute ( ATTRIBUTE . success , json . dumps ( success ) ) <EOL> if start is not None : <EOL> set_attribute ( ATTRIBUTE . start , str ( start ) ) <EOL> if limit is not None : <EOL> set_attribute ( ATTRIBUTE . limit , str ( limit ) ) <EOL> if results is not None : <EOL> set_attribute ( ATTRIBUTE . results , str ( results ) ) <EOL> if elements is not None : <EOL> root . extend ( elements ) <EOL> if domain : <EOL> set_attribute ( ATTRIBUTE . domain , self . domain ) <EOL> if url : <EOL> set_attribute ( ATTRIBUTE . url , current . response . s3 . base_url ) <EOL> if maxbounds : <EOL> bounds = current . gis . get_bounds ( ) <EOL> set_attribute ( ATTRIBUTE . latmin , <EOL> str ( bounds [ "<STR_LIT>" ] ) ) <EOL> set_attribute ( ATTRIBUTE . latmax , <EOL> str ( bounds [ "<STR_LIT>" ] ) ) <EOL> set_attribute ( ATTRIBUTE . lonmin , <EOL> str ( bounds [ "<STR_LIT>" ] ) ) <EOL> set_attribute ( ATTRIBUTE . lonmax , <EOL> str ( bounds [ "<STR_LIT>" ] ) ) <EOL> return etree . ElementTree ( root ) <EOL> def export_uid ( self , uid ) : <EOL> """<STR_LIT>""" <EOL> if not uid : <EOL> return uid <EOL> if uid [ : <NUM_LIT:4> ] == "<STR_LIT>" : <EOL> return uid <EOL> else : <EOL> domain = self . domain <EOL> if domain and "<STR_LIT:/>" not in uid [ <NUM_LIT:1> : - <NUM_LIT:1> ] : <EOL> return "<STR_LIT>" % ( domain , uid . strip ( "<STR_LIT:/>" ) ) <EOL> else : <EOL> return uid <EOL> def import_uid ( self , uid ) : <EOL> """<STR_LIT>""" <EOL> domain = self . domain <EOL> if not uid or uid . startswith ( "<STR_LIT>" ) or not domain : <EOL> return uid <EOL> else : <EOL> if "<STR_LIT:/>" in uid [ <NUM_LIT:1> : - <NUM_LIT:1> ] : <EOL> ( _domain , _uid ) = uid . strip ( "<STR_LIT:/>" ) . split ( "<STR_LIT:/>" , <NUM_LIT:1> ) <EOL> if _domain == domain : <EOL> return _uid <EOL> else : <EOL> return uid <EOL> else : <EOL> return uid <EOL> def represent ( self , table , f , v ) : <EOL> """<STR_LIT>""" <EOL> if f in ( self . CUSER , self . MUSER , self . OUSER ) : <EOL> represent = current . cache . ram ( "<STR_LIT>" % v , <EOL> lambda : self . represent_user ( v ) , <EOL> time_expire = <NUM_LIT> ) <EOL> elif f in ( self . OGROUP ) : <EOL> represent = current . cache . ram ( "<STR_LIT>" % v , <EOL> lambda : self . represent_role ( v ) , <EOL> time_expire = <NUM_LIT> ) <EOL> else : <EOL> represent = s3_represent_value ( table [ f ] , <EOL> value = v , <EOL> strip_markup = True ) <EOL> return represent <EOL> @ staticmethod <EOL> def represent_user ( user_id ) : <EOL> utable = current . auth . settings . table_user <EOL> user = current . db ( utable . id == user_id ) . select ( utable . email , <EOL> limitby = ( <NUM_LIT:0> , <NUM_LIT:1> ) <EOL> ) . first ( ) <EOL> if user : <EOL> return user . email <EOL> return None <EOL> @ staticmethod <EOL> def represent_role ( role_id ) : <EOL> gtable = current . auth . settings . table_group <EOL> role = current . db ( gtable . id == role_id ) . select ( gtable . role , <EOL> limitby = ( <NUM_LIT:0> , <NUM_LIT:1> ) , <EOL> cache = ( current . cache . ram , <EOL> S3XML . CACHE_TTL ) <EOL> ) . first ( ) <EOL> if role : <EOL> return role . role <EOL> return None <EOL> def rmap ( self , table , record , fields ) : <EOL> """<STR_LIT>""" <EOL> reference_map = [ ] <EOL> DELETED = self . DELETED <EOL> REPLACEDBY = self . REPLACEDBY <EOL> if DELETED in record and record [ DELETED ] and REPLACEDBY in record and record [ REPLACEDBY ] : <EOL> fields = [ REPLACEDBY ] <EOL> else : <EOL> fields = [ f for f in fields if f in record and record [ f ] ] <EOL> if not fields : <EOL> return reference_map <EOL> db = current . db <EOL> UID = self . UID <EOL> MCI = self . MCI <EOL> export_uid = self . export_uid <EOL> represent = self . represent <EOL> filter_mci = self . filter_mci <EOL> tablename = table . _tablename <EOL> gtablename = current . auth . settings . table_group_name <EOL> load_table = current . s3db . table <EOL> for f in fields : <EOL> if f == REPLACEDBY : <EOL> val = ogetattr ( record , f ) <EOL> if not val : <EOL> continue <EOL> row = db ( table . _id == val ) . select ( table [ UID ] , <EOL> limitby = ( <NUM_LIT:0> , <NUM_LIT:1> ) ) . first ( ) <EOL> if not row : <EOL> continue <EOL> else : <EOL> uids = [ export_uid ( row [ UID ] ) ] <EOL> entry = { "<STR_LIT>" : f , <EOL> "<STR_LIT>" : tablename , <EOL> "<STR_LIT>" : False , <EOL> "<STR_LIT:id>" : [ val ] , <EOL> "<STR_LIT>" : uids , <EOL> "<STR_LIT:text>" : None , <EOL> "<STR_LIT>" : False , <EOL> "<STR_LIT:value>" : None } <EOL> reference_map . append ( Storage ( entry ) ) <EOL> continue <EOL> try : <EOL> dbfield = ogetattr ( table , f ) <EOL> except : <EOL> continue <EOL> ktablename , pkey , multiple = s3_get_foreign_key ( dbfield ) <EOL> if not ktablename : <EOL> continue <EOL> val = ids = ogetattr ( record , f ) <EOL> ktable = load_table ( ktablename ) <EOL> if not ktable : <EOL> continue <EOL> ktable_fields = ktable . fields <EOL> k_id = ktable . _id <EOL> uid = None <EOL> uids = None <EOL> if pkey is None : <EOL> pkey = k_id . name <EOL> if pkey != "<STR_LIT:id>" and "<STR_LIT>" in ktable_fields : <EOL> if multiple : <EOL> continue <EOL> else : <EOL> query = ( k_id == ids ) <EOL> srecord = db ( query ) . select ( ogetattr ( ktable , UID ) , <EOL> ktable . instance_type , <EOL> limitby = ( <NUM_LIT:0> , <NUM_LIT:1> ) ) . first ( ) <EOL> if not srecord : <EOL> continue <EOL> ktablename = srecord . instance_type <EOL> uid = ogetattr ( srecord , UID ) <EOL> if ktablename == tablename and UID in record and ogetattr ( record , UID ) == uid and not self . show_ids : <EOL> continue <EOL> ktable = load_table ( ktablename ) <EOL> if not ktable : <EOL> continue <EOL> query = current . auth . s3_accessible_query ( "<STR_LIT>" , ktable ) & ( ktable [ UID ] == uid ) <EOL> krecord = db ( query ) . select ( ktable . _id , limitby = ( <NUM_LIT:0> , <NUM_LIT:1> ) ) . first ( ) <EOL> if not krecord : <EOL> continue <EOL> ids = [ krecord [ ktable . _id ] ] <EOL> uids = [ export_uid ( uid ) ] <EOL> else : <EOL> query = current . auth . s3_accessible_query ( "<STR_LIT>" , ktable ) <EOL> if multiple : <EOL> query &= ( k_id . belongs ( ids ) ) <EOL> limitby = None <EOL> else : <EOL> query &= ( k_id == ids ) <EOL> limitby = ( <NUM_LIT:0> , <NUM_LIT:1> ) <EOL> if DELETED in ktable_fields : <EOL> query = ( ktable . deleted != True ) & query <EOL> if filter_mci and MCI in ktable_fields : <EOL> query = ( ktable . mci >= <NUM_LIT:0> ) & query <EOL> if UID in ktable_fields : <EOL> krecords = db ( query ) . select ( ogetattr ( ktable , UID ) , <EOL> limitby = limitby ) <EOL> if krecords : <EOL> uids = [ r [ UID ] for r in krecords if r [ UID ] ] <EOL> if ktablename != gtablename : <EOL> uids = map ( export_uid , uids ) <EOL> else : <EOL> continue <EOL> else : <EOL> krecord = db ( query ) . select ( k_id , limitby = ( <NUM_LIT:0> , <NUM_LIT:1> ) ) . first ( ) <EOL> if not krecord : <EOL> continue <EOL> value = s3_unicode ( dbfield . formatter ( val ) ) <EOL> lazy = None <EOL> renderer = dbfield . represent <EOL> if renderer is not None : <EOL> if hasattr ( renderer , "<STR_LIT>" ) : <EOL> text = None <EOL> lazy = S3RepresentLazy ( val , renderer ) <EOL> else : <EOL> text = represent ( table , f , val ) <EOL> else : <EOL> text = value <EOL> entry = { "<STR_LIT>" : f , <EOL> "<STR_LIT>" : ktablename , <EOL> "<STR_LIT>" : multiple , <EOL> "<STR_LIT:id>" : ids if type ( ids ) is list else [ ids ] , <EOL> "<STR_LIT>" : uids , <EOL> "<STR_LIT:text>" : text , <EOL> "<STR_LIT>" : lazy , <EOL> "<STR_LIT:value>" : value } <EOL> reference_map . append ( Storage ( entry ) ) <EOL> return reference_map <EOL> def add_references ( self , element , rmap , show_ids = False , lazy = None ) : <EOL> """<STR_LIT>""" <EOL> REFERENCE = self . TAG . reference <EOL> ATTRIBUTE = self . ATTRIBUTE <EOL> RESOURCE = ATTRIBUTE . resource <EOL> FIELD = ATTRIBUTE . field <EOL> VALUE = ATTRIBUTE . value <EOL> ID = ATTRIBUTE . id <EOL> RB = ATTRIBUTE . replaced_by <EOL> UID = self . UID <EOL> REPLACEDBY = self . REPLACEDBY <EOL> as_json = json . dumps <EOL> SubElement = etree . SubElement <EOL> for i in xrange ( <NUM_LIT:0> , len ( rmap ) ) : <EOL> r = rmap [ i ] <EOL> f = r . field <EOL> if f == REPLACEDBY : <EOL> element . set ( RB , r . uid [ <NUM_LIT:0> ] ) <EOL> continue <EOL> reference = SubElement ( element , REFERENCE ) <EOL> attr = reference . attrib <EOL> attr [ FIELD ] = f <EOL> attr [ RESOURCE ] = r . table <EOL> if show_ids : <EOL> if r . multiple : <EOL> ids = str ( as_json ( r . id ) ) <EOL> else : <EOL> ids = str ( r . id [ <NUM_LIT:0> ] ) <EOL> attr [ ID ] = ids <EOL> if r . uid : <EOL> if r . multiple : <EOL> uids = str ( as_json ( r . uid ) ) <EOL> else : <EOL> uids = str ( r . uid [ <NUM_LIT:0> ] ) <EOL> attr [ UID ] = uids . decode ( "<STR_LIT:utf-8>" ) <EOL> if r . lazy is not None : <EOL> if lazy is not None : <EOL> lazy . append ( ( r . lazy , reference , None , None ) ) <EOL> else : <EOL> r . lazy . render_node ( reference , None , None ) <EOL> else : <EOL> reference . text = r . text <EOL> else : <EOL> attr [ VALUE ] = r . value <EOL> r . element = reference <EOL> def latlon ( self , rmap ) : <EOL> """<STR_LIT>""" <EOL> ATTRIBUTE = self . ATTRIBUTE <EOL> locations = { } <EOL> for reference in rmap : <EOL> if reference . table == "<STR_LIT>" and len ( reference . id ) == <NUM_LIT:1> : <EOL> location_id = reference . id [ <NUM_LIT:0> ] <EOL> if location_id not in locations : <EOL> locations [ location_id ] = [ reference ] <EOL> else : <EOL> locations [ location_id ] . append ( reference ) <EOL> if locations : <EOL> ltable = current . s3db . gis_location <EOL> rows = current . db ( ltable . _id . belongs ( locations . keys ( ) ) ) . select ( ltable . id , <EOL> ltable . lat , <EOL> ltable . lon , <EOL> ) . as_dict ( ) <EOL> for location_id , row in rows . items ( ) : <EOL> lat = row [ "<STR_LIT>" ] <EOL> lon = row [ "<STR_LIT>" ] <EOL> if lat is not None and lon is not None : <EOL> references = locations . get ( location_id , ( ) ) <EOL> for reference in references : <EOL> attr = reference . element . attrib <EOL> attr [ ATTRIBUTE . lat ] = "<STR_LIT>" % lat <EOL> attr [ ATTRIBUTE . lon ] = "<STR_LIT>" % lon <EOL> return <EOL> def gis_encode ( self , <EOL> resource , <EOL> record , <EOL> element , <EOL> location_data = { } , <EOL> ) : <EOL> """<STR_LIT>""" <EOL> format = current . auth . permission . format <EOL> if format not in ( "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ) : <EOL> return <EOL> tablename = resource . tablename <EOL> if tablename == "<STR_LIT>" : <EOL> return <EOL> gis = current . gis <EOL> request = current . request <EOL> settings = current . deployment_settings <EOL> ATTRIBUTE = self . ATTRIBUTE <EOL> latlons = location_data . get ( "<STR_LIT>" , [ ] ) <EOL> geojsons = location_data . get ( "<STR_LIT>" , [ ] ) <EOL> attributes = location_data . get ( "<STR_LIT>" , [ ] ) <EOL> markers = location_data . get ( "<STR_LIT>" , [ ] ) <EOL> styles = location_data . get ( "<STR_LIT>" , [ ] ) <EOL> map_data = element . find ( "<STR_LIT>" ) <EOL> if map_data : <EOL> map_data = map_data [ <NUM_LIT:0> ] <EOL> else : <EOL> map_data = etree . SubElement ( element , "<STR_LIT>" ) <EOL> attr = map_data . attrib <EOL> record_id = record . id <EOL> if tablename == "<STR_LIT>" : <EOL> if tablename in geojsons : <EOL> geojson = geojsons [ tablename ] . get ( record_id , None ) <EOL> if geojson : <EOL> geometry = etree . SubElement ( map_data , "<STR_LIT>" ) <EOL> geometry . set ( "<STR_LIT:value>" , geojson ) <EOL> elif tablename in latlons : <EOL> LatLon = latlons [ tablename ] . get ( record_id , None ) <EOL> if LatLon : <EOL> lat = LatLon [ <NUM_LIT:0> ] <EOL> lon = LatLon [ <NUM_LIT:1> ] <EOL> if lat is not None and lon is not None : <EOL> attr [ ATTRIBUTE . lat ] = "<STR_LIT>" % lat <EOL> attr [ ATTRIBUTE . lon ] = "<STR_LIT>" % lon <EOL> else : <EOL> return <EOL> if format == "<STR_LIT>" : <EOL> if tablename in attributes : <EOL> attrs = attributes [ tablename ] [ record_id ] <EOL> if attrs : <EOL> _attr = json . dumps ( attrs , separators = ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> ensure_ascii = False ) <EOL> attr [ ATTRIBUTE . attributes ] = "<STR_LIT>" % _attr . replace ( '<STR_LIT:">' , "<STR_LIT>" ) <EOL> if tablename in markers : <EOL> _markers = markers [ tablename ] <EOL> if _markers . get ( "<STR_LIT:image>" , None ) : <EOL> m = _markers <EOL> else : <EOL> m = _markers [ record_id ] <EOL> if m : <EOL> download_url = "<STR_LIT>" % request . application <EOL> attr [ ATTRIBUTE . marker_url ] = "<STR_LIT>" % ( download_url , <EOL> m [ "<STR_LIT:image>" ] ) <EOL> attr [ ATTRIBUTE . marker_height ] = str ( m [ "<STR_LIT>" ] ) <EOL> attr [ ATTRIBUTE . marker_width ] = str ( m [ "<STR_LIT:width>" ] ) <EOL> if tablename in styles : <EOL> style = styles [ tablename ] . get ( record_id ) <EOL> if style : <EOL> _style = etree . SubElement ( map_data , "<STR_LIT>" ) <EOL> _style . set ( "<STR_LIT:value>" , style ) <EOL> return <EOL> elif format == "<STR_LIT>" : <EOL> marker = gis . get_marker ( ) <EOL> marker_download_url = "<STR_LIT>" % ( current . deployment_settings . get_base_public_url ( ) , <EOL> request . application ) <EOL> marker_url = "<STR_LIT>" % ( marker_download_url , marker . image ) <EOL> attr [ ATTRIBUTE . marker ] = marker_url <EOL> elif format == "<STR_LIT>" : <EOL> symbol = "<STR_LIT>" <EOL> attr [ ATTRIBUTE . sym ] = symbol <EOL> return <EOL> elif len ( tablename ) > <NUM_LIT> and tablename . startswith ( "<STR_LIT>" ) : <EOL> if tablename in geojsons : <EOL> geojson = geojsons [ tablename ] . get ( record_id , None ) <EOL> if geojson : <EOL> geometry = etree . SubElement ( map_data , "<STR_LIT>" ) <EOL> geometry . set ( "<STR_LIT:value>" , geojson ) <EOL> if tablename in attributes : <EOL> attrs = attributes [ tablename ] [ record_id ] <EOL> if attrs : <EOL> _attr = json . dumps ( attrs , separators = ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> ensure_ascii = False ) <EOL> attr [ ATTRIBUTE . attributes ] = "<STR_LIT>" % _attr . replace ( '<STR_LIT:">' , "<STR_LIT>" ) <EOL> else : <EOL> return <EOL> return <EOL> if format == "<STR_LIT>" : <EOL> if tablename in geojsons : <EOL> geojson = geojsons [ tablename ] . get ( record_id , None ) <EOL> if geojson : <EOL> for g in geojson : <EOL> geometry = etree . SubElement ( map_data , "<STR_LIT>" ) <EOL> geometry . set ( "<STR_LIT:value>" , g ) <EOL> elif tablename in latlons : <EOL> LatLon = latlons [ tablename ] . get ( record_id , None ) <EOL> if LatLon : <EOL> lat = LatLon [ <NUM_LIT:0> ] <EOL> lon = LatLon [ <NUM_LIT:1> ] <EOL> if lat is not None and lon is not None : <EOL> attr [ ATTRIBUTE . lat ] = "<STR_LIT>" % lat <EOL> attr [ ATTRIBUTE . lon ] = "<STR_LIT>" % lon <EOL> else : <EOL> raise RuntimeError ( "<STR_LIT>" % tablename ) <EOL> if tablename in attributes : <EOL> try : <EOL> attrs = attributes [ tablename ] [ record_id ] <EOL> except KeyError : <EOL> from s3utils import s3_debug <EOL> s3_debug ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> attrs = { } <EOL> if attrs : <EOL> _attr = json . dumps ( attrs , separators = ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> ensure_ascii = False ) <EOL> attr [ ATTRIBUTE . attributes ] = "<STR_LIT>" % _attr . replace ( '<STR_LIT:">' , "<STR_LIT>" ) <EOL> if tablename in markers : <EOL> _markers = markers [ tablename ] <EOL> if _markers : <EOL> if _markers . get ( "<STR_LIT:image>" , None ) : <EOL> m = _markers <EOL> else : <EOL> m = _markers [ record_id ] <EOL> if m : <EOL> download_url = "<STR_LIT>" % request . application <EOL> attr [ ATTRIBUTE . marker_url ] = "<STR_LIT>" % ( download_url , <EOL> m [ "<STR_LIT:image>" ] ) <EOL> attr [ ATTRIBUTE . marker_height ] = str ( m [ "<STR_LIT>" ] ) <EOL> attr [ ATTRIBUTE . marker_width ] = str ( m [ "<STR_LIT:width>" ] ) <EOL> if tablename in styles : <EOL> style = styles [ tablename ] . get ( record_id ) <EOL> if style : <EOL> _style = etree . SubElement ( map_data , "<STR_LIT>" ) <EOL> _style . set ( "<STR_LIT:value>" , style ) <EOL> return <EOL> elif tablename in latlons : <EOL> LatLon = latlons [ tablename ] . get ( record_id , None ) <EOL> if LatLon : <EOL> lat = LatLon [ <NUM_LIT:0> ] <EOL> lon = LatLon [ <NUM_LIT:1> ] <EOL> if lat is not None and lon is not None : <EOL> attr [ ATTRIBUTE . lat ] = "<STR_LIT>" % lat <EOL> attr [ ATTRIBUTE . lon ] = "<STR_LIT>" % lon <EOL> else : <EOL> return <EOL> if tablename in markers : <EOL> _markers = markers [ tablename ] <EOL> if _markers . get ( "<STR_LIT:image>" , None ) : <EOL> m = _markers <EOL> else : <EOL> m = _markers [ record_id ] <EOL> if m : <EOL> if format == "<STR_LIT>" : <EOL> attr [ ATTRIBUTE . sym ] = m . get ( "<STR_LIT>" , <EOL> gis . DEFAULT_SYMBOL ) <EOL> else : <EOL> download_url = "<STR_LIT>" % ( settings . get_base_public_url ( ) , request . application ) <EOL> attr [ ATTRIBUTE . marker ] = "<STR_LIT>" % ( download_url , <EOL> m [ "<STR_LIT:image>" ] ) <EOL> def resource ( self , <EOL> parent , <EOL> table , <EOL> record , <EOL> alias = None , <EOL> fields = [ ] , <EOL> url = None , <EOL> lazy = None , <EOL> postprocess = None ) : <EOL> """<STR_LIT>""" <EOL> SubElement = etree . SubElement <EOL> UID = self . UID <EOL> MCI = self . MCI <EOL> DELETED = self . DELETED <EOL> TAG = self . TAG <EOL> RESOURCE = TAG [ "<STR_LIT>" ] <EOL> DATA = TAG [ "<STR_LIT:data>" ] <EOL> ATTRIBUTE = self . ATTRIBUTE <EOL> NAME = ATTRIBUTE [ "<STR_LIT:name>" ] <EOL> ALIAS = ATTRIBUTE [ "<STR_LIT>" ] <EOL> FIELD = ATTRIBUTE [ "<STR_LIT>" ] <EOL> VALUE = ATTRIBUTE [ "<STR_LIT:value>" ] <EOL> FILEURL = ATTRIBUTE [ "<STR_LIT:url>" ] <EOL> tablename = table . _tablename <EOL> deleted = False <EOL> download_url = current . response . s3 . download_url or "<STR_LIT>" <EOL> auth_group = current . auth . settings . table_group_name <EOL> if parent is not None : <EOL> elem = SubElement ( parent , RESOURCE ) <EOL> else : <EOL> elem = etree . Element ( RESOURCE ) <EOL> attrib = elem . attrib <EOL> attrib [ NAME ] = tablename <EOL> if alias : <EOL> attrib [ ALIAS ] = alias <EOL> if UID in table . fields and UID in record : <EOL> uid = record [ UID ] <EOL> uid = str ( table [ UID ] . formatter ( uid ) ) . decode ( "<STR_LIT:utf-8>" ) <EOL> if tablename != auth_group : <EOL> attrib [ UID ] = self . export_uid ( uid ) <EOL> else : <EOL> attrib [ UID ] = uid <EOL> if DELETED in record and record [ DELETED ] : <EOL> deleted = True <EOL> attrib [ DELETED ] = "<STR_LIT:True>" <EOL> fields = [ self . MTIME ] <EOL> FIELDS_TO_ATTRIBUTES = self . FIELDS_TO_ATTRIBUTES <EOL> _repr = self . represent <EOL> to_json = json . dumps <EOL> for f in fields : <EOL> if f == DELETED : <EOL> continue <EOL> v = record . get ( f , None ) <EOL> if f == MCI : <EOL> if v is None : <EOL> v = <NUM_LIT:0> <EOL> attrib [ MCI ] = str ( int ( v ) + <NUM_LIT:1> ) <EOL> continue <EOL> if v is None or not hasattr ( table , f ) : <EOL> continue <EOL> dbfield = ogetattr ( table , f ) <EOL> fieldtype = str ( dbfield . type ) <EOL> formatter = dbfield . formatter <EOL> represent = dbfield . represent <EOL> value = None <EOL> if fieldtype == "<STR_LIT>" : <EOL> value = s3_encode_iso_datetime ( v ) . decode ( "<STR_LIT:utf-8>" ) <EOL> elif fieldtype in ( "<STR_LIT:date>" , "<STR_LIT:time>" ) or fieldtype [ : <NUM_LIT:7> ] == "<STR_LIT>" : <EOL> value = str ( formatter ( v ) ) . decode ( "<STR_LIT:utf-8>" ) <EOL> is_lazy = False <EOL> if fieldtype not in ( "<STR_LIT>" , "<STR_LIT:password>" , "<STR_LIT>" ) : <EOL> if represent is not None and fieldtype != "<STR_LIT:id>" and f not in ( "<STR_LIT>" , "<STR_LIT>" ) : <EOL> if lazy is not None and hasattr ( represent , "<STR_LIT>" ) : <EOL> is_lazy = True <EOL> text = S3RepresentLazy ( v , represent ) <EOL> else : <EOL> text = _repr ( table , f , v ) <EOL> elif value is not None : <EOL> text = value <EOL> else : <EOL> text = s3_unicode ( formatter ( v ) ) <EOL> else : <EOL> text = None <EOL> if f in FIELDS_TO_ATTRIBUTES : <EOL> if text is not None : <EOL> if is_lazy : <EOL> lazy . append ( ( text , None , attrib , f ) ) <EOL> else : <EOL> attrib [ f ] = s3_unicode ( text ) <EOL> elif fieldtype == "<STR_LIT>" : <EOL> if v : <EOL> fileurl = None <EOL> if dbfield . custom_retrieve_file_properties : <EOL> prop = dbfield . custom_retrieve_file_properties ( v ) <EOL> else : <EOL> prop = dbfield . retrieve_file_properties ( v ) <EOL> filename = prop [ "<STR_LIT:filename>" ] <EOL> folder = prop [ "<STR_LIT:path>" ] <EOL> if folder : <EOL> path = os . path . relpath ( folder , current . request . folder ) . split ( os . sep ) <EOL> if path [ <NUM_LIT:0> ] == "<STR_LIT>" and len ( path ) > <NUM_LIT:1> : <EOL> path . append ( filename ) <EOL> fileurl = URL ( c = path [ <NUM_LIT:0> ] , f = path [ <NUM_LIT:1> ] , args = path [ <NUM_LIT:2> : ] ) <EOL> if fileurl is None : <EOL> fileurl = "<STR_LIT>" % ( download_url , v ) <EOL> data = SubElement ( elem , DATA ) <EOL> attr = data . attrib <EOL> attr [ FIELD ] = f <EOL> attr [ FILEURL ] = fileurl <EOL> attr [ ATTRIBUTE . filename ] = filename <EOL> elif fieldtype == "<STR_LIT:password>" : <EOL> data = SubElement ( elem , DATA ) <EOL> data . attrib [ FIELD ] = f <EOL> data . text = v <EOL> elif fieldtype == "<STR_LIT>" : <EOL> continue <EOL> else : <EOL> data = SubElement ( elem , DATA ) <EOL> attr = data . attrib <EOL> attr [ FIELD ] = f <EOL> if represent or fieldtype not in ( "<STR_LIT:string>" , "<STR_LIT:text>" ) : <EOL> if value is None : <EOL> value = to_json ( v ) . decode ( "<STR_LIT:utf-8>" ) <EOL> attr [ VALUE ] = value <EOL> if is_lazy : <EOL> lazy . append ( ( text , data , None , None ) ) <EOL> else : <EOL> data . text = text <EOL> if url and not deleted : <EOL> attrib [ FILEURL ] = url <EOL> if postprocess : <EOL> postprocess ( elem , record ) <EOL> return elem <EOL> @ classmethod <EOL> def select_resources ( cls , tree , tablename ) : <EOL> """<STR_LIT>""" <EOL> resources = [ ] <EOL> if isinstance ( tree , etree . _ElementTree ) : <EOL> root = tree . getroot ( ) <EOL> if root is None or root . tag != cls . TAG . root : <EOL> return resources <EOL> else : <EOL> root = tree <EOL> if root is None or not len ( root ) : <EOL> return resources <EOL> expr = '<STR_LIT>' % ( cls . TAG . resource , cls . ATTRIBUTE . name , tablename ) <EOL> resources = root . xpath ( expr ) <EOL> return resources <EOL> @ classmethod <EOL> def components ( cls , element , names = None ) : <EOL> """<STR_LIT>""" <EOL> RESOURCE = cls . TAG . resource <EOL> NAME = cls . ATTRIBUTE . name <EOL> for child in element . iterchildren ( ) : <EOL> if child . tag == RESOURCE : <EOL> if names is None or child . get ( NAME , None ) in names : <EOL> yield child <EOL> return <EOL> @ staticmethod <EOL> def _dtparse ( dtstr , field_type = "<STR_LIT>" ) : <EOL> """<STR_LIT>""" <EOL> error = None <EOL> value = None <EOL> try : <EOL> dt = s3_decode_iso_datetime ( str ( dtstr ) ) <EOL> value = s3_utc ( dt ) <EOL> except : <EOL> error = sys . exc_info ( ) [ <NUM_LIT:1> ] <EOL> if error is None : <EOL> if field_type == "<STR_LIT:date>" : <EOL> value = value . date ( ) <EOL> elif field_type == "<STR_LIT:time>" : <EOL> value = value . time ( ) <EOL> return ( value , error ) <EOL> @ classmethod <EOL> def record ( cls , table , element , <EOL> original = None , <EOL> files = [ ] , <EOL> skip = [ ] , <EOL> postprocess = None ) : <EOL> """<STR_LIT>""" <EOL> valid = True <EOL> record = Storage ( ) <EOL> db = current . db <EOL> auth = current . auth <EOL> utable = auth . settings . table_user <EOL> gtable = auth . settings . table_group <EOL> UID = cls . UID <EOL> uid = None <EOL> if UID in table . fields and UID not in skip : <EOL> uid = current . xml . import_uid ( element . get ( UID , None ) ) <EOL> if uid : <EOL> record [ UID ] = uid <EOL> ATTRIBUTE = cls . ATTRIBUTE <EOL> FIELD = ATTRIBUTE [ "<STR_LIT>" ] <EOL> VALUE = ATTRIBUTE [ "<STR_LIT:value>" ] <EOL> ERROR = ATTRIBUTE [ "<STR_LIT:error>" ] <EOL> DELETED = cls . DELETED <EOL> APPROVED = cls . APPROVED <EOL> IGNORE_FIELDS = cls . IGNORE_FIELDS <EOL> OGROUP = cls . OGROUP <EOL> USER_FIELDS = ( cls . CUSER , cls . MUSER , cls . OUSER ) <EOL> deleted = False <EOL> for f in cls . ATTRIBUTES_TO_FIELDS : <EOL> if f == DELETED : <EOL> if f in table and element . get ( f , "<STR_LIT:false>" ) . lower ( ) == "<STR_LIT:true>" : <EOL> record [ f ] = deleted = True <EOL> replaced_by = element . get ( ATTRIBUTE [ "<STR_LIT>" ] , None ) <EOL> if replaced_by : <EOL> record [ cls . REPLACEDBY ] = replaced_by <EOL> break <EOL> else : <EOL> continue <EOL> if f == APPROVED : <EOL> if "<STR_LIT>" in table : <EOL> approved = element . get ( f ) <EOL> if approved : <EOL> if approved . lower ( ) == "<STR_LIT:false>" : <EOL> record [ "<STR_LIT>" ] = None <EOL> else : <EOL> if table [ "<STR_LIT>" ] . default is None : <EOL> auth . permission . set_default_approver ( table , force = True ) <EOL> else : <EOL> if table [ "<STR_LIT>" ] . default is None : <EOL> auth . permission . set_default_approver ( table ) <EOL> continue <EOL> if f in IGNORE_FIELDS or f in skip : <EOL> continue <EOL> elif f in USER_FIELDS : <EOL> v = element . get ( f , None ) <EOL> if v and utable and "<STR_LIT:email>" in utable : <EOL> query = ( utable . email == v ) <EOL> user = db ( query ) . select ( utable . id , limitby = ( <NUM_LIT:0> , <NUM_LIT:1> ) ) . first ( ) <EOL> if user : <EOL> record [ f ] = user . id <EOL> continue <EOL> elif f == OGROUP : <EOL> v = element . get ( f , None ) <EOL> if v and gtable and "<STR_LIT>" in gtable : <EOL> query = ( gtable . role == v ) <EOL> role = db ( query ) . select ( gtable . id , limitby = ( <NUM_LIT:0> , <NUM_LIT:1> ) ) . first ( ) <EOL> if role : <EOL> record [ f ] = role . id <EOL> continue <EOL> if hasattr ( table , f ) : <EOL> v = value = element . get ( f , None ) <EOL> if value is not None : <EOL> field_type = str ( table [ f ] . type ) <EOL> if field_type in ( "<STR_LIT>" , "<STR_LIT:date>" , "<STR_LIT:time>" ) : <EOL> ( value , error ) = cls . _dtparse ( v , <EOL> field_type = field_type ) <EOL> else : <EOL> try : <EOL> ( value , error ) = s3_validate ( table , f , v , original ) <EOL> except AttributeError : <EOL> continue <EOL> if error : <EOL> element . set ( ERROR , "<STR_LIT>" % ( f , error ) ) <EOL> valid = False <EOL> continue <EOL> record [ f ] = value <EOL> if deleted and uid : <EOL> return record <EOL> xml_decode = cls . xml_decode <EOL> for child in element . findall ( "<STR_LIT:data>" ) : <EOL> error = None <EOL> f = child . get ( FIELD , None ) <EOL> if not f or not hasattr ( table , f ) : <EOL> continue <EOL> if f in IGNORE_FIELDS or f in skip : <EOL> continue <EOL> field_type = str ( table [ f ] . type ) <EOL> if field_type in ( "<STR_LIT:id>" , "<STR_LIT>" ) : <EOL> continue <EOL> elif field_type == "<STR_LIT>" : <EOL> download_url = child . get ( ATTRIBUTE [ "<STR_LIT:url>" ] ) <EOL> filename = child . get ( ATTRIBUTE [ "<STR_LIT:filename>" ] ) <EOL> upload = None <EOL> if filename and filename in files : <EOL> upload = files [ filename ] <EOL> elif download_url == "<STR_LIT>" : <EOL> value = filename <EOL> elif download_url : <EOL> if not isinstance ( download_url , str ) : <EOL> try : <EOL> download_url = download_url . encode ( "<STR_LIT:utf-8>" ) <EOL> except UnicodeEncodeError : <EOL> continue <EOL> if not filename : <EOL> filename = download_url . split ( "<STR_LIT:?>" ) [ <NUM_LIT:0> ] or "<STR_LIT>" <EOL> try : <EOL> upload = urllib2 . urlopen ( download_url ) <EOL> except IOError : <EOL> continue <EOL> if upload : <EOL> if not isinstance ( filename , str ) : <EOL> try : <EOL> filename = filename . encode ( "<STR_LIT:utf-8>" ) <EOL> except UnicodeEncodeError : <EOL> continue <EOL> field = table [ f ] <EOL> value = field . store ( upload , filename ) <EOL> elif download_url != "<STR_LIT>" : <EOL> continue <EOL> else : <EOL> value = child . get ( VALUE , None ) <EOL> skip_validation = False <EOL> is_text = field_type in ( "<STR_LIT:string>" , "<STR_LIT:text>" ) <EOL> if value is None : <EOL> decode_value = not is_text <EOL> if field_type == "<STR_LIT:password>" : <EOL> value = child . text <EOL> skip_validation = True <EOL> else : <EOL> value = xml_decode ( child . text ) <EOL> else : <EOL> decode_value = True <EOL> if value is None and is_text : <EOL> value = "<STR_LIT>" <EOL> elif value == "<STR_LIT>" and not is_text : <EOL> value = None <EOL> if value is not None : <EOL> if field_type in ( "<STR_LIT>" , "<STR_LIT:date>" , "<STR_LIT:time>" ) : <EOL> ( value , error ) = cls . _dtparse ( value , <EOL> field_type = field_type ) <EOL> skip_validation = True <EOL> v = value <EOL> elif field_type == "<STR_LIT>" : <EOL> pass <EOL> elif isinstance ( value , basestring ) and len ( value ) and decode_value : <EOL> try : <EOL> _value = json . loads ( value ) <EOL> if _value != float ( "<STR_LIT>" ) : <EOL> value = _value <EOL> except : <EOL> error = sys . exc_info ( ) [ <NUM_LIT:1> ] <EOL> if not skip_validation : <EOL> if not isinstance ( value , ( basestring , list , tuple , bool ) ) : <EOL> v = str ( value ) <EOL> elif isinstance ( value , basestring ) : <EOL> v = value . encode ( "<STR_LIT:utf-8>" ) <EOL> else : <EOL> v = value <EOL> filename = None <EOL> try : <EOL> if field_type == "<STR_LIT>" and download_url != "<STR_LIT>" and table [ f ] . requires : <EOL> filename , stream = field . retrieve ( value ) <EOL> v = filename <EOL> if isinstance ( stream , basestring ) : <EOL> stream = open ( stream , "<STR_LIT:rb>" ) <EOL> if not error : <EOL> dummy = Storage ( { "<STR_LIT:filename>" : filename , <EOL> "<STR_LIT:file>" : stream } ) <EOL> ( dummy , error ) = s3_validate ( table , f , dummy , original ) <EOL> elif field_type == "<STR_LIT:password>" : <EOL> v = str ( value ) <EOL> ( value , error ) = s3_validate ( table , f , v ) <EOL> else : <EOL> ( value , error ) = s3_validate ( table , f , v , original ) <EOL> except AttributeError : <EOL> continue <EOL> except IOError : <EOL> if filename : <EOL> error = "<STR_LIT>" % filename <EOL> else : <EOL> error = sys . exc_info ( ) [ <NUM_LIT:1> ] <EOL> except : <EOL> error = sys . exc_info ( ) [ <NUM_LIT:1> ] <EOL> child . set ( VALUE , s3_unicode ( v ) ) <EOL> if error : <EOL> child . set ( ERROR , s3_unicode ( "<STR_LIT>" % ( f , error ) ) ) <EOL> valid = False <EOL> continue <EOL> record [ f ] = value <EOL> if valid : <EOL> if postprocess : <EOL> postprocess ( element , record ) <EOL> return record <EOL> else : <EOL> return None <EOL> @ classmethod <EOL> def get_field_options ( cls , <EOL> table , <EOL> fieldname , <EOL> parent = None , <EOL> show_uids = False , <EOL> hierarchy = False ) : <EOL> """<STR_LIT>""" <EOL> options = None <EOL> try : <EOL> field = table [ fieldname ] <EOL> except AttributeError : <EOL> pass <EOL> else : <EOL> requires = field . requires <EOL> if requires : <EOL> if not isinstance ( requires , ( list , tuple ) ) : <EOL> requires = [ requires ] <EOL> requires = requires [ <NUM_LIT:0> ] <EOL> if isinstance ( requires , IS_EMPTY_OR ) : <EOL> requires = requires . other <EOL> try : <EOL> options = requires . options ( ) <EOL> except AttributeError : <EOL> pass <EOL> TAG = cls . TAG <EOL> if options : <EOL> ATTRIBUTE = cls . ATTRIBUTE <EOL> UID = cls . UID <EOL> SubElement = etree . SubElement <EOL> if parent is not None : <EOL> select = SubElement ( parent , TAG . select ) <EOL> else : <EOL> select = etree . Element ( TAG . select ) <EOL> select . set ( ATTRIBUTE . name , fieldname ) <EOL> select . set ( ATTRIBUTE . id , "<STR_LIT>" % ( table . _tablename , fieldname ) ) <EOL> uids = { } <EOL> parents = { } <EOL> if show_uids or hierarchy : <EOL> ids = set ( int ( o [ <NUM_LIT:0> ] ) for o in options if str ( o [ <NUM_LIT:0> ] ) . isdigit ( ) ) <EOL> ktablename , key = s3_get_foreign_key ( field , m2m = False ) [ : <NUM_LIT:2> ] <EOL> try : <EOL> ktable = current . s3db [ ktablename ] <EOL> except AttributeError : <EOL> pass <EOL> else : <EOL> if show_uids : <EOL> query = ktable [ key ] . belongs ( ids ) <EOL> rows = current . db ( query ) . select ( ktable [ key ] , <EOL> ktable [ UID ] ) <EOL> for row in rows : <EOL> uids [ str ( row [ key ] ) ] = row [ UID ] <EOL> if hierarchy : <EOL> from s3hierarchy import S3Hierarchy <EOL> h = S3Hierarchy ( ktablename ) <EOL> if h . config : <EOL> for _id in ids : <EOL> parents [ str ( _id ) ] = h . parent ( _id ) <EOL> OPTION = TAG . option <EOL> VALUE = ATTRIBUTE . value <EOL> PARENT = ATTRIBUTE . parent <EOL> for value , text in options : <EOL> option = SubElement ( select , OPTION ) <EOL> option . text = s3_unicode ( s3_strip_markup ( text ) ) <EOL> attr = option . attrib <EOL> value = s3_unicode ( value ) <EOL> attr [ VALUE ] = value <EOL> if show_uids and value in uids : <EOL> uid = uids [ value ] <EOL> if uid : <EOL> attr [ UID ] = uid <EOL> if hierarchy and value in parents : <EOL> p = parents [ value ] <EOL> if p : <EOL> attr [ PARENT ] = str ( p ) <EOL> elif parent is not None : <EOL> select = None <EOL> else : <EOL> select = etree . Element ( TAG . select ) <EOL> return select <EOL> def get_options ( self , <EOL> table , <EOL> fields = None , <EOL> show_uids = False , <EOL> hierarchy = False ) : <EOL> """<STR_LIT>""" <EOL> if fields : <EOL> if not isinstance ( fields , ( list , tuple , set ) ) : <EOL> fields = [ fields ] <EOL> if len ( fields ) == <NUM_LIT:1> : <EOL> return self . get_field_options ( table , <EOL> fields [ <NUM_LIT:0> ] , <EOL> show_uids = show_uids , <EOL> hierarchy = hierarchy , <EOL> ) <EOL> options = etree . Element ( self . TAG . options ) <EOL> if table : <EOL> options . set ( self . ATTRIBUTE . resource , table . _tablename ) <EOL> get_field_options = self . get_field_options <EOL> for f in table . fields : <EOL> if not fields or f in fields : <EOL> get_field_options ( table , f , <EOL> parent = options , <EOL> show_uids = show_uids , <EOL> hierarchy = hierarchy , <EOL> ) <EOL> return options <EOL> def get_fields ( self , prefix , name , <EOL> parent = None , <EOL> meta = False , <EOL> options = False , <EOL> references = False , <EOL> labels = False ) : <EOL> """<STR_LIT>""" <EOL> db = current . db <EOL> tablename = "<STR_LIT>" % ( prefix , name ) <EOL> table = db . get ( tablename , None ) <EOL> if parent is not None : <EOL> fields = parent <EOL> else : <EOL> fields = etree . Element ( self . TAG . fields ) <EOL> if table : <EOL> ATTRIBUTE = self . ATTRIBUTE <EOL> if parent is None : <EOL> fields . set ( ATTRIBUTE . resource , tablename ) <EOL> for f in table . fields : <EOL> ftype = str ( table [ f ] . type ) <EOL> if ftype [ : <NUM_LIT:9> ] == "<STR_LIT>" : <EOL> ktablename = ftype [ <NUM_LIT:10> : ] <EOL> s = current . s3db . get_config ( tablename , "<STR_LIT>" ) <EOL> if not isinstance ( s , ( list , tuple ) ) : <EOL> s = [ s ] <EOL> if ktablename in s : <EOL> continue <EOL> if f in self . IGNORE_FIELDS or ftype == "<STR_LIT:id>" : <EOL> continue <EOL> if f in self . FIELDS_TO_ATTRIBUTES : <EOL> if not meta : <EOL> continue <EOL> tag = self . TAG . meta <EOL> else : <EOL> tag = self . TAG . field <EOL> readable = table [ f ] . readable <EOL> writable = table [ f ] . writable <EOL> field = etree . SubElement ( fields , tag ) <EOL> if options : <EOL> p = field <EOL> if not references and ftype [ : <NUM_LIT:9> ] in ( "<STR_LIT>" , "<STR_LIT>" ) : <EOL> p = None <EOL> else : <EOL> p = None <EOL> opts = self . get_field_options ( table , f , parent = p ) <EOL> set_attribute = field . set <EOL> set_attribute ( ATTRIBUTE . name , f ) <EOL> set_attribute ( ATTRIBUTE . type , ftype ) <EOL> set_attribute ( ATTRIBUTE . readable , str ( readable ) ) <EOL> set_attribute ( ATTRIBUTE . writable , str ( writable ) ) <EOL> has_options = str ( opts is not None and <EOL> len ( opts ) and True or False ) <EOL> set_attribute ( ATTRIBUTE . has_options , has_options ) <EOL> if labels : <EOL> label = s3_unicode ( table [ f ] . label ) <EOL> set_attribute ( ATTRIBUTE . label , label ) <EOL> comment = table [ f ] . comment <EOL> if comment : <EOL> comment = s3_unicode ( comment ) <EOL> if comment and "<STR_LIT:<>" in comment : <EOL> comment = s3_strip_markup ( comment ) <EOL> if comment : <EOL> set_attribute ( ATTRIBUTE . comment , comment ) <EOL> return fields <EOL> def get_struct ( self , prefix , name , <EOL> alias = None , <EOL> parent = None , <EOL> meta = False , <EOL> options = True , <EOL> references = False ) : <EOL> """<STR_LIT>""" <EOL> db = current . db <EOL> tablename = "<STR_LIT>" % ( prefix , name ) <EOL> table = db . get ( tablename , None ) <EOL> if table is not None : <EOL> if parent is not None : <EOL> e = etree . SubElement ( parent , self . TAG . resource ) <EOL> else : <EOL> e = etree . Element ( self . TAG . resource ) <EOL> e . set ( self . ATTRIBUTE . name , tablename ) <EOL> if alias and alias != name : <EOL> e . set ( self . ATTRIBUTE . alias , alias ) <EOL> self . get_fields ( prefix , name , <EOL> parent = e , <EOL> meta = meta , <EOL> options = options , <EOL> references = references , <EOL> labels = True ) <EOL> return e <EOL> else : <EOL> raise AttributeError ( "<STR_LIT>" % tablename ) <EOL> @ classmethod <EOL> def __json2element ( cls , key , value , native = False ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( value , dict ) : <EOL> element = cls . __obj2element ( key , value , native = native ) <EOL> elif isinstance ( value , ( list , tuple ) ) : <EOL> if key != cls . TAG . item : <EOL> element = etree . Element ( key ) <EOL> else : <EOL> element = etree . Element ( cls . TAG . list ) <EOL> json2element = cls . __json2element <EOL> item_tag = cls . TAG . item <EOL> for obj in value : <EOL> item = json2element ( item_tag , obj , native = native ) <EOL> element . append ( item ) <EOL> else : <EOL> if native : <EOL> element = etree . Element ( cls . TAG . data ) <EOL> element . set ( cls . ATTRIBUTE . field , key ) <EOL> else : <EOL> element = etree . Element ( key ) <EOL> if not isinstance ( value , basestring ) : <EOL> value = str ( value ) <EOL> element . text = value <EOL> return element <EOL> @ classmethod <EOL> def __obj2element ( cls , tag , obj , native = False ) : <EOL> """<STR_LIT>""" <EOL> resource = field = None <EOL> if not tag : <EOL> tag = cls . TAG . object <EOL> elif native : <EOL> if tag . startswith ( cls . PREFIX . reference ) : <EOL> field = tag [ len ( cls . PREFIX . reference ) + <NUM_LIT:1> : ] <EOL> tag = cls . TAG . reference <EOL> elif tag . startswith ( cls . PREFIX . options ) : <EOL> resource = tag [ len ( cls . PREFIX . options ) + <NUM_LIT:1> : ] <EOL> tag = cls . TAG . options <EOL> elif tag . startswith ( cls . PREFIX . resource ) : <EOL> resource = tag [ len ( cls . PREFIX . resource ) + <NUM_LIT:1> : ] <EOL> tag = cls . TAG . resource <EOL> elif not tag == cls . TAG . root : <EOL> field = tag <EOL> tag = cls . TAG . data <EOL> element = etree . Element ( tag ) <EOL> if native : <EOL> if resource : <EOL> if tag == cls . TAG . resource : <EOL> element . set ( cls . ATTRIBUTE . name , resource ) <EOL> else : <EOL> element . set ( cls . ATTRIBUTE . resource , resource ) <EOL> if field : <EOL> element . set ( cls . ATTRIBUTE . field , field ) <EOL> attribute_prefix = cls . PREFIX . attribute <EOL> text_prefix = cls . PREFIX . text <EOL> obj2element = cls . __obj2element <EOL> json2element = cls . __json2element <EOL> for k in obj : <EOL> m = obj [ k ] <EOL> if isinstance ( m , dict ) : <EOL> child = obj2element ( k , m , native = native ) <EOL> element . append ( child ) <EOL> elif isinstance ( m , ( list , tuple ) ) : <EOL> if native and k . startswith ( attribute_prefix ) : <EOL> a = k [ len ( attribute_prefix ) : ] <EOL> element . set ( a , json . dumps ( m ) ) <EOL> continue <EOL> for obj_ in m : <EOL> child = json2element ( k , obj_ , native = native ) <EOL> element . append ( child ) <EOL> else : <EOL> if k == text_prefix : <EOL> element . text = m <EOL> elif k . startswith ( attribute_prefix ) : <EOL> a = k [ len ( attribute_prefix ) : ] <EOL> if not isinstance ( m , basestring ) : <EOL> m = str ( m ) <EOL> element . set ( a , m ) <EOL> else : <EOL> child = json2element ( k , m , native = native ) <EOL> element . append ( child ) <EOL> return element <EOL> @ classmethod <EOL> def json2tree ( cls , source , format = None ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> root_dict = json . load ( source ) <EOL> except ( ValueError , ) : <EOL> e = sys . exc_info ( ) [ <NUM_LIT:1> ] <EOL> raise HTTP ( <NUM_LIT> , body = cls . json_message ( False , <NUM_LIT> , e ) ) <EOL> native = False <EOL> if not format : <EOL> format = cls . TAG . root <EOL> native = True <EOL> if root_dict and isinstance ( root_dict , dict ) : <EOL> root = cls . __obj2element ( format , root_dict , native = native ) <EOL> if root is not None : <EOL> return etree . ElementTree ( root ) <EOL> return None <EOL> @ classmethod <EOL> def __element2json ( cls , element , native = False ) : <EOL> """<STR_LIT>""" <EOL> TAG = cls . TAG <EOL> ATTRIBUTE = cls . ATTRIBUTE <EOL> PREFIX = cls . PREFIX <EOL> element2json = cls . __element2json <EOL> if element . tag == TAG . list : <EOL> obj = [ ] <EOL> append = obj . append <EOL> for child in element : <EOL> tag = child . tag <EOL> if not isinstance ( tag , basestring ) : <EOL> continue <EOL> if tag [ <NUM_LIT:0> ] == "<STR_LIT:{>" : <EOL> tag = tag . rsplit ( "<STR_LIT:}>" , <NUM_LIT:1> ) [ <NUM_LIT:1> ] <EOL> child_obj = element2json ( child , native = native ) <EOL> if child_obj : <EOL> append ( child_obj ) <EOL> return obj <EOL> else : <EOL> obj = { } <EOL> iterchildren = element . iterchildren <EOL> xpath = element . xpath <EOL> is_single = lambda t , a , v : len ( xpath ( "<STR_LIT>" % ( t , a , v ) ) ) == <NUM_LIT:1> <EOL> for child in iterchildren ( tag = etree . Element ) : <EOL> tag = child . tag <EOL> if tag [ <NUM_LIT:0> ] == "<STR_LIT:{>" : <EOL> tag = tag . rsplit ( "<STR_LIT:}>" , <NUM_LIT:1> ) [ <NUM_LIT:1> ] <EOL> collapse = True <EOL> single = False <EOL> attributes = child . attrib <EOL> if native : <EOL> if tag == TAG . resource : <EOL> resource = attributes . get ( ATTRIBUTE . name ) <EOL> tag = "<STR_LIT>" % ( PREFIX . resource , resource ) <EOL> collapse = False <EOL> elif tag == TAG . options : <EOL> r = attributes . get ( ATTRIBUTE . resource ) <EOL> tag = "<STR_LIT>" % ( PREFIX . options , r ) <EOL> single = is_single ( TAG . options , ATTRIBUTE . resource , r ) <EOL> elif tag == TAG . reference : <EOL> f = attributes . get ( ATTRIBUTE . field ) <EOL> tag = "<STR_LIT>" % ( PREFIX . reference , f ) <EOL> single = is_single ( TAG . reference , ATTRIBUTE . field , f ) <EOL> elif tag == TAG . data : <EOL> tag = attributes . get ( ATTRIBUTE . field ) <EOL> single = is_single ( TAG . data , ATTRIBUTE . field , tag ) <EOL> else : <EOL> for s in iterchildren ( tag = tag ) : <EOL> if single is True : <EOL> single = False <EOL> break <EOL> else : <EOL> single = True <EOL> child_obj = element2json ( child , native = native ) <EOL> if child_obj is not None and child_obj != "<STR_LIT>" : <EOL> if tag not in obj : <EOL> if single and collapse : <EOL> obj [ tag ] = child_obj <EOL> else : <EOL> obj [ tag ] = [ child_obj ] <EOL> else : <EOL> if type ( obj [ tag ] ) is not list : <EOL> obj [ tag ] = [ obj [ tag ] ] <EOL> obj [ tag ] . append ( child_obj ) <EOL> attributes = element . attrib <EOL> skip_text = False <EOL> tag = element . tag <EOL> numeric = False <EOL> for a in attributes : <EOL> v = attributes [ a ] <EOL> if native : <EOL> if a == ATTRIBUTE . name and tag == TAG . resource : <EOL> continue <EOL> if a == ATTRIBUTE . resource and tag == TAG . options : <EOL> continue <EOL> if a == ATTRIBUTE . field and tag in ( TAG . data , TAG . reference ) : <EOL> continue <EOL> if a == ATTRIBUTE . value : <EOL> try : <EOL> v = json . loads ( v ) <EOL> except : <EOL> pass <EOL> else : <EOL> if a == ATTRIBUTE . value : <EOL> try : <EOL> obj [ TAG . item ] = json . loads ( v ) <EOL> except : <EOL> pass <EOL> else : <EOL> skip_text = True <EOL> continue <EOL> elif a == ATTRIBUTE . type and v == "<STR_LIT>" : <EOL> numeric = True <EOL> continue <EOL> obj [ PREFIX . attribute + a ] = v <EOL> if element . text and not skip_text : <EOL> represent = cls . xml_decode ( element . text ) <EOL> if numeric : <EOL> try : <EOL> float_represent = float ( represent . replace ( "<STR_LIT:U+002C>" , "<STR_LIT>" ) ) <EOL> int_represent = int ( float_represent ) <EOL> if int_represent == float_represent : <EOL> represent = int_represent <EOL> else : <EOL> represent = float_represent <EOL> except : <EOL> pass <EOL> obj [ PREFIX . text ] = represent <EOL> if len ( obj ) == <NUM_LIT:1> and obj . keys ( ) [ <NUM_LIT:0> ] in ( PREFIX . text , TAG . item , TAG . list ) : <EOL> obj = obj [ obj . keys ( ) [ <NUM_LIT:0> ] ] <EOL> return obj <EOL> @ classmethod <EOL> def tree2json ( cls , tree , pretty_print = False , native = False ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( tree , etree . _ElementTree ) : <EOL> root = tree . getroot ( ) <EOL> else : <EOL> root = tree <EOL> if native or root . tag == cls . TAG . root : <EOL> native = True <EOL> else : <EOL> native = False <EOL> root_dict = cls . __element2json ( root , native = native ) <EOL> if "<STR_LIT>" in root_dict : <EOL> if root_dict [ "<STR_LIT>" ] == { } : <EOL> del root_dict [ "<STR_LIT>" ] <EOL> else : <EOL> root_dict [ "<STR_LIT>" ] = json . loads ( root_dict [ "<STR_LIT>" ] ) <EOL> if pretty_print : <EOL> js = json . dumps ( root_dict , indent = <NUM_LIT:4> ) <EOL> return "<STR_LIT:\n>" . join ( [ l . rstrip ( ) for l in js . splitlines ( ) ] ) <EOL> else : <EOL> return json . dumps ( root_dict , separators = SEPARATORS ) <EOL> @ staticmethod <EOL> def collect_errors ( job ) : <EOL> """<STR_LIT>""" <EOL> errors = [ ] <EOL> try : <EOL> if isinstance ( job , etree . _Element ) : <EOL> error_tree = job <EOL> else : <EOL> error_tree = job . error_tree <EOL> except AttributeError : <EOL> return errors <EOL> if error_tree is None : <EOL> return errors <EOL> elements = error_tree . xpath ( "<STR_LIT>" ) <EOL> for element in elements : <EOL> get = element . get <EOL> if element . tag == "<STR_LIT:data>" : <EOL> resource = element . getparent ( ) <EOL> value = get ( "<STR_LIT:value>" ) <EOL> if not value : <EOL> value = element . text <EOL> error = "<STR_LIT>" % ( <EOL> resource . get ( "<STR_LIT:name>" , None ) , <EOL> get ( "<STR_LIT>" , None ) , <EOL> get ( "<STR_LIT:error>" , None ) , <EOL> value ) <EOL> if element . tag == "<STR_LIT>" : <EOL> resource = element . getparent ( ) <EOL> error = "<STR_LIT>" % ( <EOL> resource . get ( "<STR_LIT:name>" , None ) , <EOL> get ( "<STR_LIT>" , None ) , <EOL> get ( "<STR_LIT:error>" , None ) ) <EOL> elif element . tag == "<STR_LIT>" : <EOL> error = "<STR_LIT>" % ( get ( "<STR_LIT:name>" , None ) , <EOL> get ( "<STR_LIT:error>" , None ) ) <EOL> else : <EOL> error = "<STR_LIT:%s>" % get ( "<STR_LIT:error>" , None ) <EOL> errors . append ( error ) <EOL> return errors <EOL> @ classmethod <EOL> def xls2tree ( cls , source , <EOL> resourcename = None , <EOL> extra_data = None , <EOL> hashtags = None , <EOL> sheet = None , <EOL> rows = None , <EOL> cols = None , <EOL> fields = None , <EOL> header_row = True ) : <EOL> """<STR_LIT>""" <EOL> import xlrd <EOL> ATTRIBUTE = cls . ATTRIBUTE <EOL> FIELD = ATTRIBUTE . field <EOL> HASHTAG = ATTRIBUTE . hashtag <EOL> TAG = cls . TAG <EOL> COL = TAG . col <EOL> SubElement = etree . SubElement <EOL> DEFAULT_SHEET_NAME = "<STR_LIT>" <EOL> root = etree . Element ( TAG . table ) <EOL> if resourcename is not None : <EOL> root . set ( ATTRIBUTE . name , resourcename ) <EOL> if isinstance ( sheet , xlrd . sheet . Sheet ) : <EOL> s = sheet <EOL> else : <EOL> if hasattr ( source , "<STR_LIT>" ) : <EOL> if hasattr ( source , "<STR_LIT>" ) : <EOL> source . seek ( <NUM_LIT:0> ) <EOL> wb = xlrd . open_workbook ( file_contents = source . read ( ) , <EOL> on_demand = True ) <EOL> elif isinstance ( source , xlrd . book . Book ) : <EOL> wb = source <EOL> else : <EOL> raise RuntimeError ( "<STR_LIT>" % <EOL> type ( source ) ) <EOL> try : <EOL> if isinstance ( sheet , ( int , long ) ) : <EOL> s = wb . sheet_by_index ( sheet ) <EOL> elif isinstance ( sheet , basestring ) : <EOL> s = wb . sheet_by_name ( sheet ) <EOL> elif sheet is None : <EOL> if DEFAULT_SHEET_NAME in wb . sheet_names ( ) : <EOL> s = wb . sheet_by_name ( DEFAULT_SHEET_NAME ) <EOL> else : <EOL> s = wb . sheet_by_index ( <NUM_LIT:0> ) <EOL> else : <EOL> raise SyntaxError ( "<STR_LIT>" % sheet ) <EOL> except IndexError , xlrd . XLRDError : <EOL> s = None <EOL> def cell_range ( cells , max_cells ) : <EOL> """<STR_LIT>""" <EOL> if not cells : <EOL> cells = ( <NUM_LIT:0> , max_cells ) <EOL> elif not isinstance ( cells , ( tuple , list ) ) : <EOL> cells = ( <NUM_LIT:0> , cells ) <EOL> elif len ( cells ) == <NUM_LIT:1> : <EOL> cells = ( cells [ <NUM_LIT:0> ] , max_cells ) <EOL> else : <EOL> cells = ( cells [ <NUM_LIT:0> ] , cells [ <NUM_LIT:0> ] + cells [ <NUM_LIT:1> ] ) <EOL> return cells <EOL> if s : <EOL> rows = cell_range ( rows , s . nrows ) <EOL> cols = cell_range ( cols , s . ncols ) <EOL> if fields : <EOL> headers = fields <EOL> elif not header_row : <EOL> headers = dict ( ( i , "<STR_LIT:%s>" % i ) <EOL> for i in range ( cols [ <NUM_LIT:1> ] - cols [ <NUM_LIT:0> ] ) ) <EOL> else : <EOL> headers = { } <EOL> decode_date = lambda v : datetime . datetime ( <EOL> * xlrd . xldate_as_tuple ( v , wb . datemode ) ) <EOL> def decode ( t , v ) : <EOL> """<STR_LIT>""" <EOL> text = "<STR_LIT>" <EOL> if v : <EOL> if t is None : <EOL> text = s3_unicode ( v ) . strip ( ) <EOL> elif t == xlrd . XL_CELL_TEXT : <EOL> text = v . strip ( ) <EOL> elif t == xlrd . XL_CELL_NUMBER : <EOL> text = str ( long ( v ) ) if long ( v ) == v else str ( v ) <EOL> elif t == xlrd . XL_CELL_DATE : <EOL> text = s3_encode_iso_datetime ( decode_date ( v ) ) <EOL> elif t == xlrd . XL_CELL_BOOLEAN : <EOL> text = str ( value ) . lower ( ) <EOL> return text <EOL> def add_col ( row , name , t , v , hashtags = None ) : <EOL> """<STR_LIT>""" <EOL> col = SubElement ( row , COL ) <EOL> col . set ( FIELD , name ) <EOL> if hashtags : <EOL> hashtag = hashtags . get ( name ) <EOL> if hashtag and hashtag [ <NUM_LIT:1> : ] : <EOL> col . set ( HASHTAG , hashtag ) <EOL> col . text = decode ( t , v ) <EOL> hashtags = dict ( hashtags ) if hashtags else { } <EOL> ROW = TAG . row <EOL> record_idx = <NUM_LIT:0> <EOL> extra_fields = set ( extra_data ) if extra_data else None <EOL> check_headers = extra_fields is not None <EOL> for ridx in range ( * rows ) : <EOL> types = s . row_types ( ridx , * cols ) <EOL> values = s . row_values ( ridx , * cols ) <EOL> if not any ( v != "<STR_LIT>" for v in values ) : <EOL> continue <EOL> if header_row and record_idx == <NUM_LIT:0> : <EOL> if not fields : <EOL> for cidx , value in enumerate ( values ) : <EOL> header = decode ( types [ cidx ] , value ) <EOL> headers [ cidx ] = header <EOL> if check_headers : <EOL> extra_fields . discard ( header ) <EOL> check_headers = False <EOL> else : <EOL> if not fields and ( header_row and record_idx == <NUM_LIT:1> or record_idx == <NUM_LIT:0> ) : <EOL> items = { } <EOL> for cidx , name in headers . items ( ) : <EOL> try : <EOL> t = types [ cidx ] <EOL> v = values [ cidx ] <EOL> except IndexError : <EOL> continue <EOL> if t not in ( xlrd . XL_CELL_TEXT , xlrd . XL_CELL_EMPTY ) : <EOL> items = None <EOL> break <EOL> elif v : <EOL> items [ name ] = v <EOL> if items and all ( v [ <NUM_LIT:0> ] == '<STR_LIT:#>' for v in items . values ( ) ) : <EOL> hashtags . update ( items ) <EOL> continue <EOL> orow = SubElement ( root , ROW ) <EOL> for cidx , name in headers . items ( ) : <EOL> if check_headers : <EOL> extra_fields . discard ( name ) <EOL> try : <EOL> t = types [ cidx ] <EOL> v = values [ cidx ] <EOL> except IndexError : <EOL> pass <EOL> else : <EOL> add_col ( orow , name , t , v , hashtags = hashtags ) <EOL> check_headers = False <EOL> if extra_fields : <EOL> for key in extra_fields : <EOL> add_col ( orow , key , None , extra_data [ key ] , hashtags = hashtags ) <EOL> record_idx += <NUM_LIT:1> <EOL> return etree . ElementTree ( root ) <EOL> @ classmethod <EOL> def csv2tree ( cls , source , <EOL> resourcename = None , <EOL> extra_data = None , <EOL> hashtags = None , <EOL> delimiter = "<STR_LIT:U+002C>" , <EOL> quotechar = '<STR_LIT:">' ) : <EOL> """<STR_LIT>""" <EOL> import csv <EOL> csv . field_size_limit ( <NUM_LIT:2> ** <NUM_LIT:20> * <NUM_LIT:100> ) <EOL> ATTRIBUTE = cls . ATTRIBUTE <EOL> FIELD = ATTRIBUTE . field <EOL> HASHTAG = ATTRIBUTE . hashtag <EOL> TAG = cls . TAG <EOL> COL = TAG . col <EOL> SubElement = etree . SubElement <EOL> root = etree . Element ( TAG . table ) <EOL> if resourcename is not None : <EOL> root . set ( ATTRIBUTE . name , resourcename ) <EOL> def add_col ( row , key , value , hashtags = None ) : <EOL> col = SubElement ( row , COL ) <EOL> col . set ( FIELD , s3_unicode ( key ) ) <EOL> if hashtags : <EOL> hashtag = hashtags . get ( key ) <EOL> if hashtag and hashtag [ <NUM_LIT:1> : ] : <EOL> col . set ( HASHTAG , hashtag ) <EOL> if value : <EOL> text = s3_unicode ( value ) . strip ( ) <EOL> if text [ : <NUM_LIT:6> ] . lower ( ) not in ( "<STR_LIT:null>" , "<STR_LIT>" ) : <EOL> try : <EOL> col . text = text <EOL> except MemoryError : <EOL> current . log . error ( "<STR_LIT>" % key ) <EOL> else : <EOL> col . text = "<STR_LIT>" <EOL> def utf_8_encode ( source ) : <EOL> """<STR_LIT>""" <EOL> encodings = ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> e = encodings [ <NUM_LIT:0> ] <EOL> for line in source : <EOL> if e : <EOL> try : <EOL> yield unicode ( line , e , "<STR_LIT:strict>" ) . encode ( "<STR_LIT:utf-8>" ) <EOL> except : <EOL> pass <EOL> else : <EOL> continue <EOL> for encoding in encodings : <EOL> try : <EOL> yield unicode ( line , encoding , "<STR_LIT:strict>" ) . encode ( "<STR_LIT:utf-8>" ) <EOL> except : <EOL> continue <EOL> else : <EOL> e = encoding <EOL> break <EOL> hashtags = dict ( hashtags ) if hashtags else { } <EOL> try : <EOL> import StringIO <EOL> if not isinstance ( source , StringIO . StringIO ) : <EOL> source = utf_8_encode ( source ) <EOL> reader = csv . DictReader ( source , <EOL> delimiter = delimiter , <EOL> quotechar = quotechar ) <EOL> ROW = TAG . row <EOL> for i , r in enumerate ( reader ) : <EOL> if not any ( r . values ( ) ) : <EOL> continue <EOL> if i == <NUM_LIT:0> : <EOL> items = dict ( ( k , s3_unicode ( v . strip ( ) ) ) <EOL> for k , v in r . items ( ) if k and v and v . strip ( ) ) <EOL> if all ( v [ <NUM_LIT:0> ] == "<STR_LIT:#>" for v in items . values ( ) ) : <EOL> hashtags . update ( items ) <EOL> continue <EOL> row = SubElement ( root , ROW ) <EOL> for k in r : <EOL> if k : <EOL> add_col ( row , k , r [ k ] , hashtags = hashtags ) <EOL> if extra_data : <EOL> for key in extra_data : <EOL> if key not in r : <EOL> add_col ( row , key , extra_data [ key ] , hashtags = hashtags ) <EOL> except csv . Error : <EOL> e = sys . exc_info ( ) [ <NUM_LIT:1> ] <EOL> raise HTTP ( <NUM_LIT> , body = cls . json_message ( False , <NUM_LIT> , e ) ) <EOL> return etree . ElementTree ( root ) <EOL> class S3XMLFormat ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , stylesheet ) : <EOL> """<STR_LIT>""" <EOL> self . tree = current . xml . parse ( stylesheet ) <EOL> if not self . tree : <EOL> current . log . error ( "<STR_LIT>" % <EOL> ( stylesheet , current . xml . error ) ) <EOL> self . select = None <EOL> self . skip = None <EOL> def get_fields ( self , tablename ) : <EOL> """<STR_LIT>""" <EOL> ANY = "<STR_LIT>" <EOL> default = ( None , None ) <EOL> tree = self . tree <EOL> if not tree : <EOL> return default <EOL> if self . select is None : <EOL> self . __inspect ( ) <EOL> def find_match ( items , tablename , default ) : <EOL> if tablename in items : <EOL> match = items [ tablename ] <EOL> else : <EOL> match = False <EOL> maxlen = <NUM_LIT:0> <EOL> for tn , fields in items . iteritems ( ) : <EOL> if "<STR_LIT:*>" in tn : <EOL> m = re . match ( tn . replace ( "<STR_LIT:*>" , "<STR_LIT>" ) , tablename ) <EOL> if not m : <EOL> continue <EOL> l = m . span ( ) [ - <NUM_LIT:1> ] <EOL> if l > maxlen : <EOL> match = fields <EOL> if match is False : <EOL> match = items . get ( ANY , default ) <EOL> return match <EOL> select = find_match ( self . select , tablename , None ) <EOL> skip = find_match ( self . skip , tablename , set ( ) ) <EOL> if skip : <EOL> if select : <EOL> include = [ fn for fn in select if fn not in skip ] <EOL> exclude = [ fn for fn in skip if fn not in select ] <EOL> else : <EOL> include = None <EOL> exclude = list ( skip ) <EOL> else : <EOL> include = list ( select ) if select else None <EOL> exclude = [ ] <EOL> return ( include , exclude ) <EOL> def __inspect ( self ) : <EOL> """<STR_LIT>""" <EOL> ALL = "<STR_LIT>" <EOL> ANY = "<STR_LIT>" <EOL> tree = self . tree <EOL> ns = { "<STR_LIT>" : "<STR_LIT>" } <EOL> elements = tree . xpath ( "<STR_LIT>" , namespaces = ns ) <EOL> select = { } <EOL> skip = { } <EOL> for element in elements : <EOL> tables = element . get ( "<STR_LIT>" , ANY ) . split ( "<STR_LIT:U+002C>" ) <EOL> fields = element . get ( "<STR_LIT>" , None ) <EOL> if fields is not None and fields != ALL : <EOL> fields = set ( [ f . strip ( ) for f in fields . split ( "<STR_LIT:U+002C>" ) ] ) <EOL> exclude = element . get ( "<STR_LIT>" , None ) <EOL> if exclude is not None : <EOL> exclude = set ( [ f . strip ( ) for f in exclude . split ( "<STR_LIT:U+002C>" ) ] ) <EOL> for table in tables : <EOL> tablename = table . strip ( ) <EOL> if fields : <EOL> select [ tablename ] = None if fields == ALL else fields <EOL> if exclude : <EOL> skip [ tablename ] = exclude <EOL> self . select = select <EOL> self . skip = skip <EOL> return <EOL> def transform ( self , tree , ** args ) : <EOL> """<STR_LIT>""" <EOL> if not self . tree : <EOL> current . log . error ( "<STR_LIT>" ) <EOL> return tree <EOL> return current . xml . transform ( tree , self . tree , ** args ) </s>
