<s> import os <EOL> import six <EOL> from hashlib import sha1 <EOL> from normality import slugify <EOL> DATA_FIXTURES = os . path . join ( os . path . dirname ( __file__ ) , '<STR_LIT:data>' ) <EOL> SCHEMA_FIXTURES = os . path . join ( os . path . dirname ( __file__ ) , '<STR_LIT>' ) <EOL> def remove_namespace ( doc , namespace ) : <EOL> """<STR_LIT>""" <EOL> ns = u'<STR_LIT>' % namespace <EOL> nsl = len ( ns ) <EOL> for elem in doc . getiterator ( ) : <EOL> if elem . tag . startswith ( ns ) : <EOL> elem . tag = elem . tag [ nsl : ] <EOL> def make_id ( * parts ) : <EOL> parts = [ unicode ( p ) for p in parts if p is not None ] <EOL> parts = [ slugify ( p , sep = '<STR_LIT:->' ) for p in parts if len ( p ) ] <EOL> return '<STR_LIT::>' . join ( parts ) <EOL> def clean_obj ( data ) : <EOL> if isinstance ( data , dict ) : <EOL> if not len ( data . keys ( ) ) : <EOL> return None <EOL> out = { } <EOL> for k , v in data . items ( ) : <EOL> v = clean_obj ( v ) <EOL> if v is not None : <EOL> out [ k ] = v <EOL> return out <EOL> elif isinstance ( data , ( list , tuple , set ) ) : <EOL> if not len ( data ) : <EOL> return <EOL> return [ clean_obj ( o ) for o in data ] <EOL> elif isinstance ( data , six . string_types ) : <EOL> if not len ( data ) : <EOL> return None <EOL> return data <EOL> def obj_hash ( data ) : <EOL> if isinstance ( data , dict ) : <EOL> sum_ = sha1 ( ) <EOL> for k , v in data . items ( ) : <EOL> sum_ . update ( unicode ( k ) . encode ( '<STR_LIT:utf-8>' ) ) <EOL> sum_ . update ( unicode ( v ) . encode ( '<STR_LIT:utf-8>' ) ) <EOL> return sum_ . hexdigest ( ) <EOL> return hash ( data ) <EOL> def unique_objs ( objs ) : <EOL> out = [ ] <EOL> if objs is None : <EOL> return out <EOL> seen = set ( ) <EOL> for obj in objs : <EOL> sum_ = obj_hash ( obj ) <EOL> if sum_ not in seen : <EOL> seen . add ( sum_ ) <EOL> out . append ( obj ) <EOL> return out </s>
