<s> """<STR_LIT>""" <EOL> import sys <EOL> from decimal import Decimal , InvalidOperation as DecimalInvalidOperation <EOL> from django . contrib . gis . db . models import GeometryField <EOL> from django . contrib . gis . gdal import ( <EOL> CoordTransform , DataSource , GDALException , OGRGeometry , OGRGeomType , <EOL> SpatialReference , <EOL> ) <EOL> from django . contrib . gis . gdal . field import ( <EOL> OFTDate , OFTDateTime , OFTInteger , OFTInteger64 , OFTReal , OFTString , <EOL> OFTTime , <EOL> ) <EOL> from django . core . exceptions import FieldDoesNotExist , ObjectDoesNotExist <EOL> from django . db import connections , models , router , transaction <EOL> from django . utils import six <EOL> from django . utils . encoding import force_text <EOL> class LayerMapError ( Exception ) : <EOL> pass <EOL> class InvalidString ( LayerMapError ) : <EOL> pass <EOL> class InvalidDecimal ( LayerMapError ) : <EOL> pass <EOL> class InvalidInteger ( LayerMapError ) : <EOL> pass <EOL> class MissingForeignKey ( LayerMapError ) : <EOL> pass <EOL> class LayerMapping ( object ) : <EOL> "<STR_LIT>" <EOL> MULTI_TYPES = { <NUM_LIT:1> : OGRGeomType ( '<STR_LIT>' ) , <EOL> <NUM_LIT:2> : OGRGeomType ( '<STR_LIT>' ) , <EOL> <NUM_LIT:3> : OGRGeomType ( '<STR_LIT>' ) , <EOL> OGRGeomType ( '<STR_LIT>' ) . num : OGRGeomType ( '<STR_LIT>' ) , <EOL> OGRGeomType ( '<STR_LIT>' ) . num : OGRGeomType ( '<STR_LIT>' ) , <EOL> OGRGeomType ( '<STR_LIT>' ) . num : OGRGeomType ( '<STR_LIT>' ) , <EOL> } <EOL> FIELD_TYPES = { <EOL> models . AutoField : OFTInteger , <EOL> models . BigAutoField : OFTInteger64 , <EOL> models . IntegerField : ( OFTInteger , OFTReal , OFTString ) , <EOL> models . FloatField : ( OFTInteger , OFTReal ) , <EOL> models . DateField : OFTDate , <EOL> models . DateTimeField : OFTDateTime , <EOL> models . EmailField : OFTString , <EOL> models . TimeField : OFTTime , <EOL> models . DecimalField : ( OFTInteger , OFTReal ) , <EOL> models . CharField : OFTString , <EOL> models . SlugField : OFTString , <EOL> models . TextField : OFTString , <EOL> models . URLField : OFTString , <EOL> models . BigIntegerField : ( OFTInteger , OFTReal , OFTString ) , <EOL> models . SmallIntegerField : ( OFTInteger , OFTReal , OFTString ) , <EOL> models . PositiveSmallIntegerField : ( OFTInteger , OFTReal , OFTString ) , <EOL> } <EOL> def __init__ ( self , model , data , mapping , layer = <NUM_LIT:0> , <EOL> source_srs = None , encoding = '<STR_LIT:utf-8>' , <EOL> transaction_mode = '<STR_LIT>' , <EOL> transform = True , unique = None , using = None ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( data , six . string_types ) : <EOL> self . ds = DataSource ( data , encoding = encoding ) <EOL> else : <EOL> self . ds = data <EOL> self . layer = self . ds [ layer ] <EOL> self . using = using if using is not None else router . db_for_write ( model ) <EOL> self . spatial_backend = connections [ self . using ] . ops <EOL> self . mapping = mapping <EOL> self . model = model <EOL> self . check_layer ( ) <EOL> if connections [ self . using ] . features . supports_transform : <EOL> self . geo_field = self . geometry_field ( ) <EOL> else : <EOL> transform = False <EOL> if transform : <EOL> self . source_srs = self . check_srs ( source_srs ) <EOL> self . transform = self . coord_transform ( ) <EOL> else : <EOL> self . transform = transform <EOL> if encoding : <EOL> from codecs import lookup <EOL> lookup ( encoding ) <EOL> self . encoding = encoding <EOL> else : <EOL> self . encoding = None <EOL> if unique : <EOL> self . check_unique ( unique ) <EOL> transaction_mode = '<STR_LIT>' <EOL> self . unique = unique <EOL> else : <EOL> self . unique = None <EOL> self . transaction_mode = transaction_mode <EOL> if transaction_mode == '<STR_LIT>' : <EOL> self . transaction_decorator = None <EOL> elif transaction_mode == '<STR_LIT>' : <EOL> self . transaction_decorator = transaction . atomic <EOL> else : <EOL> raise LayerMapError ( '<STR_LIT>' % transaction_mode ) <EOL> def check_fid_range ( self , fid_range ) : <EOL> "<STR_LIT>" <EOL> if fid_range : <EOL> if isinstance ( fid_range , ( tuple , list ) ) : <EOL> return slice ( * fid_range ) <EOL> elif isinstance ( fid_range , slice ) : <EOL> return fid_range <EOL> else : <EOL> raise TypeError <EOL> else : <EOL> return None <EOL> def check_layer ( self ) : <EOL> """<STR_LIT>""" <EOL> self . geom_field = False <EOL> self . fields = { } <EOL> ogr_fields = self . layer . fields <EOL> ogr_field_types = self . layer . field_types <EOL> def check_ogr_fld ( ogr_map_fld ) : <EOL> try : <EOL> idx = ogr_fields . index ( ogr_map_fld ) <EOL> except ValueError : <EOL> raise LayerMapError ( '<STR_LIT>' % ogr_map_fld ) <EOL> return idx <EOL> for field_name , ogr_name in self . mapping . items ( ) : <EOL> try : <EOL> model_field = self . model . _meta . get_field ( field_name ) <EOL> except FieldDoesNotExist : <EOL> raise LayerMapError ( '<STR_LIT>' % field_name ) <EOL> fld_name = model_field . __class__ . __name__ <EOL> if isinstance ( model_field , GeometryField ) : <EOL> if self . geom_field : <EOL> raise LayerMapError ( '<STR_LIT>' ) <EOL> coord_dim = model_field . dim <EOL> try : <EOL> if coord_dim == <NUM_LIT:3> : <EOL> gtype = OGRGeomType ( ogr_name + '<STR_LIT>' ) <EOL> else : <EOL> gtype = OGRGeomType ( ogr_name ) <EOL> except GDALException : <EOL> raise LayerMapError ( '<STR_LIT>' % field_name ) <EOL> ltype = self . layer . geom_type <EOL> if not ( ltype . name . startswith ( gtype . name ) or self . make_multi ( ltype , model_field ) ) : <EOL> raise LayerMapError ( '<STR_LIT>' <EOL> '<STR_LIT>' % <EOL> ( fld_name , '<STR_LIT>' if coord_dim == <NUM_LIT:3> else '<STR_LIT>' , ltype ) ) <EOL> self . geom_field = field_name <EOL> self . coord_dim = coord_dim <EOL> fields_val = model_field <EOL> elif isinstance ( model_field , models . ForeignKey ) : <EOL> if isinstance ( ogr_name , dict ) : <EOL> rel_model = model_field . remote_field . model <EOL> for rel_name , ogr_field in ogr_name . items ( ) : <EOL> idx = check_ogr_fld ( ogr_field ) <EOL> try : <EOL> rel_model . _meta . get_field ( rel_name ) <EOL> except FieldDoesNotExist : <EOL> raise LayerMapError ( '<STR_LIT>' % <EOL> ( rel_name , rel_model . __class__ . __name__ ) ) <EOL> fields_val = rel_model <EOL> else : <EOL> raise TypeError ( '<STR_LIT>' ) <EOL> else : <EOL> if model_field . __class__ not in self . FIELD_TYPES : <EOL> raise LayerMapError ( '<STR_LIT>' % fld_name ) <EOL> idx = check_ogr_fld ( ogr_name ) <EOL> ogr_field = ogr_field_types [ idx ] <EOL> if not issubclass ( ogr_field , self . FIELD_TYPES [ model_field . __class__ ] ) : <EOL> raise LayerMapError ( '<STR_LIT>' % <EOL> ( ogr_field , ogr_field . __name__ , fld_name ) ) <EOL> fields_val = model_field <EOL> self . fields [ field_name ] = fields_val <EOL> def check_srs ( self , source_srs ) : <EOL> "<STR_LIT>" <EOL> if isinstance ( source_srs , SpatialReference ) : <EOL> sr = source_srs <EOL> elif isinstance ( source_srs , self . spatial_backend . spatial_ref_sys ( ) ) : <EOL> sr = source_srs . srs <EOL> elif isinstance ( source_srs , ( int , six . string_types ) ) : <EOL> sr = SpatialReference ( source_srs ) <EOL> else : <EOL> sr = self . layer . srs <EOL> if not sr : <EOL> raise LayerMapError ( '<STR_LIT>' ) <EOL> else : <EOL> return sr <EOL> def check_unique ( self , unique ) : <EOL> "<STR_LIT>" <EOL> if isinstance ( unique , ( list , tuple ) ) : <EOL> for attr in unique : <EOL> if attr not in self . mapping : <EOL> raise ValueError <EOL> elif isinstance ( unique , six . string_types ) : <EOL> if unique not in self . mapping : <EOL> raise ValueError <EOL> else : <EOL> raise TypeError ( '<STR_LIT>' ) <EOL> def feature_kwargs ( self , feat ) : <EOL> """<STR_LIT>""" <EOL> kwargs = { } <EOL> for field_name , ogr_name in self . mapping . items ( ) : <EOL> model_field = self . fields [ field_name ] <EOL> if isinstance ( model_field , GeometryField ) : <EOL> try : <EOL> val = self . verify_geom ( feat . geom , model_field ) <EOL> except GDALException : <EOL> raise LayerMapError ( '<STR_LIT>' ) <EOL> elif isinstance ( model_field , models . base . ModelBase ) : <EOL> val = self . verify_fk ( feat , model_field , ogr_name ) <EOL> else : <EOL> val = self . verify_ogr_field ( feat [ ogr_name ] , model_field ) <EOL> kwargs [ field_name ] = val <EOL> return kwargs <EOL> def unique_kwargs ( self , kwargs ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( self . unique , six . string_types ) : <EOL> return { self . unique : kwargs [ self . unique ] } <EOL> else : <EOL> return { fld : kwargs [ fld ] for fld in self . unique } <EOL> def verify_ogr_field ( self , ogr_field , model_field ) : <EOL> """<STR_LIT>""" <EOL> if ( isinstance ( ogr_field , OFTString ) and <EOL> isinstance ( model_field , ( models . CharField , models . TextField ) ) ) : <EOL> if self . encoding : <EOL> val = force_text ( ogr_field . value , self . encoding ) <EOL> else : <EOL> val = ogr_field . value <EOL> if model_field . max_length and len ( val ) > model_field . max_length : <EOL> raise InvalidString ( '<STR_LIT>' % <EOL> ( model_field . name , model_field . max_length , len ( val ) ) ) <EOL> elif isinstance ( ogr_field , OFTReal ) and isinstance ( model_field , models . DecimalField ) : <EOL> try : <EOL> d = Decimal ( str ( ogr_field . value ) ) <EOL> except DecimalInvalidOperation : <EOL> raise InvalidDecimal ( '<STR_LIT>' % ogr_field . value ) <EOL> dtup = d . as_tuple ( ) <EOL> digits = dtup [ <NUM_LIT:1> ] <EOL> d_idx = dtup [ <NUM_LIT:2> ] <EOL> max_prec = model_field . max_digits - model_field . decimal_places <EOL> if d_idx < <NUM_LIT:0> : <EOL> n_prec = len ( digits [ : d_idx ] ) <EOL> else : <EOL> n_prec = len ( digits ) + d_idx <EOL> if n_prec > max_prec : <EOL> raise InvalidDecimal ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' % <EOL> ( model_field . max_digits , model_field . decimal_places , max_prec ) <EOL> ) <EOL> val = d <EOL> elif isinstance ( ogr_field , ( OFTReal , OFTString ) ) and isinstance ( model_field , models . IntegerField ) : <EOL> try : <EOL> val = int ( ogr_field . value ) <EOL> except ValueError : <EOL> raise InvalidInteger ( '<STR_LIT>' % ogr_field . value ) <EOL> else : <EOL> val = ogr_field . value <EOL> return val <EOL> def verify_fk ( self , feat , rel_model , rel_mapping ) : <EOL> """<STR_LIT>""" <EOL> fk_kwargs = { } <EOL> for field_name , ogr_name in rel_mapping . items ( ) : <EOL> fk_kwargs [ field_name ] = self . verify_ogr_field ( feat [ ogr_name ] , rel_model . _meta . get_field ( field_name ) ) <EOL> try : <EOL> return rel_model . objects . using ( self . using ) . get ( ** fk_kwargs ) <EOL> except ObjectDoesNotExist : <EOL> raise MissingForeignKey ( <EOL> '<STR_LIT>' % <EOL> ( rel_model . __name__ , fk_kwargs ) <EOL> ) <EOL> def verify_geom ( self , geom , model_field ) : <EOL> """<STR_LIT>""" <EOL> if self . coord_dim != geom . coord_dim : <EOL> geom . coord_dim = self . coord_dim <EOL> if self . make_multi ( geom . geom_type , model_field ) : <EOL> multi_type = self . MULTI_TYPES [ geom . geom_type . num ] <EOL> g = OGRGeometry ( multi_type ) <EOL> g . add ( geom ) <EOL> else : <EOL> g = geom <EOL> if self . transform : <EOL> g . transform ( self . transform ) <EOL> return g . wkt <EOL> def coord_transform ( self ) : <EOL> "<STR_LIT>" <EOL> SpatialRefSys = self . spatial_backend . spatial_ref_sys ( ) <EOL> try : <EOL> target_srs = SpatialRefSys . objects . using ( self . using ) . get ( srid = self . geo_field . srid ) . srs <EOL> return CoordTransform ( self . source_srs , target_srs ) <EOL> except Exception as msg : <EOL> new_msg = '<STR_LIT>' % msg <EOL> six . reraise ( LayerMapError , LayerMapError ( new_msg ) , sys . exc_info ( ) [ <NUM_LIT:2> ] ) <EOL> def geometry_field ( self ) : <EOL> "<STR_LIT>" <EOL> opts = self . model . _meta <EOL> return opts . get_field ( self . geom_field ) <EOL> def make_multi ( self , geom_type , model_field ) : <EOL> """<STR_LIT>""" <EOL> return ( geom_type . num in self . MULTI_TYPES and <EOL> model_field . __class__ . __name__ == '<STR_LIT>' % geom_type . django ) <EOL> def save ( self , verbose = False , fid_range = False , step = False , <EOL> progress = False , silent = False , stream = sys . stdout , strict = False ) : <EOL> """<STR_LIT>""" <EOL> default_range = self . check_fid_range ( fid_range ) <EOL> if progress : <EOL> if progress is True or not isinstance ( progress , int ) : <EOL> progress_interval = <NUM_LIT:1000> <EOL> else : <EOL> progress_interval = progress <EOL> def _save ( feat_range = default_range , num_feat = <NUM_LIT:0> , num_saved = <NUM_LIT:0> ) : <EOL> if feat_range : <EOL> layer_iter = self . layer [ feat_range ] <EOL> else : <EOL> layer_iter = self . layer <EOL> for feat in layer_iter : <EOL> num_feat += <NUM_LIT:1> <EOL> try : <EOL> kwargs = self . feature_kwargs ( feat ) <EOL> except LayerMapError as msg : <EOL> if strict : <EOL> raise <EOL> elif not silent : <EOL> stream . write ( '<STR_LIT>' % ( feat . fid , msg ) ) <EOL> else : <EOL> is_update = False <EOL> if self . unique : <EOL> try : <EOL> u_kwargs = self . unique_kwargs ( kwargs ) <EOL> m = self . model . objects . using ( self . using ) . get ( ** u_kwargs ) <EOL> is_update = True <EOL> geom = getattr ( m , self . geom_field ) . ogr <EOL> new = OGRGeometry ( kwargs [ self . geom_field ] ) <EOL> for g in new : <EOL> geom . add ( g ) <EOL> setattr ( m , self . geom_field , geom . wkt ) <EOL> except ObjectDoesNotExist : <EOL> m = self . model ( ** kwargs ) <EOL> else : <EOL> m = self . model ( ** kwargs ) <EOL> try : <EOL> m . save ( using = self . using ) <EOL> num_saved += <NUM_LIT:1> <EOL> if verbose : <EOL> stream . write ( '<STR_LIT>' % ( '<STR_LIT>' if is_update else '<STR_LIT>' , m ) ) <EOL> except Exception as msg : <EOL> if strict : <EOL> if not silent : <EOL> stream . write ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' % feat . fid <EOL> ) <EOL> stream . write ( '<STR_LIT>' % kwargs ) <EOL> raise <EOL> elif not silent : <EOL> stream . write ( '<STR_LIT>' % ( kwargs , msg ) ) <EOL> if progress and num_feat % progress_interval == <NUM_LIT:0> : <EOL> stream . write ( '<STR_LIT>' % ( num_feat , num_saved ) ) <EOL> return num_saved , num_feat <EOL> if self . transaction_decorator is not None : <EOL> _save = self . transaction_decorator ( _save ) <EOL> nfeat = self . layer . num_feat <EOL> if step and isinstance ( step , int ) and step < nfeat : <EOL> if default_range : <EOL> raise LayerMapError ( '<STR_LIT>' ) <EOL> beg , num_feat , num_saved = ( <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:0> ) <EOL> indices = range ( step , nfeat , step ) <EOL> n_i = len ( indices ) <EOL> for i , end in enumerate ( indices ) : <EOL> if i + <NUM_LIT:1> == n_i : <EOL> step_slice = slice ( beg , None ) <EOL> else : <EOL> step_slice = slice ( beg , end ) <EOL> try : <EOL> num_feat , num_saved = _save ( step_slice , num_feat , num_saved ) <EOL> beg = end <EOL> except Exception : <EOL> stream . write ( '<STR_LIT>' % ( '<STR_LIT>' * <NUM_LIT:20> , step_slice ) ) <EOL> raise <EOL> else : <EOL> _save ( ) </s>
