<s> import os <EOL> import time <EOL> import math <EOL> import logging <EOL> import mimetypes <EOL> import requests <EOL> from requests . auth import HTTPDigestAuth <EOL> from urlparse import urljoin <EOL> from xml . parsers . expat import ExpatError <EOL> from cStringIO import StringIO <EOL> from django . core . files . base import ContentFile <EOL> from django . core . files . storage import default_storage <EOL> from django . core . files . uploadedfile import InMemoryUploadedFile <EOL> from onadata . apps . logger . xform_instance_parser import clean_and_parse_xml <EOL> from onadata . libs . utils . logger_tools import publish_xml_form , publish_form , create_instance <EOL> NUM_RETRIES = <NUM_LIT:3> <EOL> def django_file ( file_obj , field_name , content_type ) : <EOL> return InMemoryUploadedFile ( <EOL> file = file_obj , <EOL> field_name = field_name , <EOL> name = file_obj . name , <EOL> content_type = content_type , <EOL> size = file_obj . size , <EOL> charset = None <EOL> ) <EOL> def retry ( tries , delay = <NUM_LIT:3> , backoff = <NUM_LIT:2> ) : <EOL> '''<STR_LIT>''' <EOL> if backoff <= <NUM_LIT:1> : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> tries = math . floor ( tries ) <EOL> if tries < <NUM_LIT:0> : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> if delay <= <NUM_LIT:0> : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> def decorator_retry ( func ) : <EOL> def function_retry ( self , * args , ** kwargs ) : <EOL> mtries , mdelay = tries , delay <EOL> result = func ( self , * args , ** kwargs ) <EOL> while mtries > <NUM_LIT:0> : <EOL> if result : <EOL> return result <EOL> mtries -= <NUM_LIT:1> <EOL> time . sleep ( mdelay ) <EOL> mdelay *= backoff <EOL> result = func ( self , * args , ** kwargs ) <EOL> return False <EOL> return function_retry <EOL> return decorator_retry <EOL> def node_value ( node , tag_name ) : <EOL> tag = node . getElementsByTagName ( tag_name ) [ <NUM_LIT:0> ] <EOL> return tag . childNodes [ <NUM_LIT:0> ] . nodeValue <EOL> class BriefcaseClient ( object ) : <EOL> def __init__ ( self , url , username , password , user ) : <EOL> self . url = url <EOL> self . user = user <EOL> self . auth = HTTPDigestAuth ( username , password ) <EOL> self . form_list_url = urljoin ( self . url , '<STR_LIT>' ) <EOL> self . submission_list_url = urljoin ( self . url , '<STR_LIT>' ) <EOL> self . download_submission_url = urljoin ( self . url , <EOL> '<STR_LIT>' ) <EOL> self . forms_path = os . path . join ( <EOL> self . user . username , '<STR_LIT>' , '<STR_LIT>' ) <EOL> self . resumption_cursor = <NUM_LIT:0> <EOL> self . logger = logging . getLogger ( '<STR_LIT>' ) <EOL> def _get_form_list ( self , xml_text ) : <EOL> xml_doc = clean_and_parse_xml ( xml_text ) <EOL> forms = [ ] <EOL> for childNode in xml_doc . childNodes : <EOL> if childNode . nodeName == '<STR_LIT>' : <EOL> for xformNode in childNode . childNodes : <EOL> if xformNode . nodeName == '<STR_LIT>' : <EOL> id_string = node_value ( xformNode , '<STR_LIT>' ) <EOL> download_url = node_value ( xformNode , '<STR_LIT>' ) <EOL> manifest_url = node_value ( xformNode , '<STR_LIT>' ) <EOL> forms . append ( ( id_string , download_url , manifest_url ) ) <EOL> return forms <EOL> def download_manifest ( self , manifest_url , id_string ) : <EOL> if self . _get_response ( manifest_url ) : <EOL> manifest_res = self . _current_response <EOL> try : <EOL> manifest_doc = clean_and_parse_xml ( manifest_res . content ) <EOL> except ExpatError : <EOL> return <EOL> manifest_path = os . path . join ( <EOL> self . forms_path , id_string , '<STR_LIT>' ) <EOL> self . logger . debug ( "<STR_LIT>" % id_string ) <EOL> self . download_media_files ( manifest_doc , manifest_path ) <EOL> def download_xforms ( self , include_instances = False ) : <EOL> if not self . _get_response ( self . form_list_url ) : <EOL> response = self . _current_response . content if self . _current_response else "<STR_LIT>" <EOL> self . logger . error ( "<STR_LIT>" % response ) <EOL> return <EOL> response = self . _current_response <EOL> forms = self . _get_form_list ( response . content ) <EOL> self . logger . debug ( '<STR_LIT>' % self . form_list_url ) <EOL> for id_string , download_url , manifest_url in forms : <EOL> form_path = os . path . join ( <EOL> self . forms_path , id_string , '<STR_LIT>' % id_string ) <EOL> if not default_storage . exists ( form_path ) : <EOL> if not self . _get_response ( download_url ) : <EOL> self . logger . error ( "<STR_LIT>" <EOL> % download_url ) <EOL> continue <EOL> form_res = self . _current_response <EOL> content = ContentFile ( form_res . content . strip ( ) ) <EOL> default_storage . save ( form_path , content ) <EOL> else : <EOL> form_res = default_storage . open ( form_path ) <EOL> content = form_res . read ( ) <EOL> self . logger . debug ( "<STR_LIT>" % download_url ) <EOL> self . download_manifest ( manifest_url , id_string ) <EOL> if include_instances : <EOL> self . download_instances ( id_string ) <EOL> self . logger . debug ( "<STR_LIT>" % <EOL> id_string ) <EOL> @ retry ( NUM_RETRIES ) <EOL> def _get_response ( self , url , params = None ) : <EOL> self . _current_response = None <EOL> response = requests . get ( url , auth = self . auth , params = params ) <EOL> success = response . status_code == <NUM_LIT:200> <EOL> self . _current_response = response <EOL> return success <EOL> @ retry ( NUM_RETRIES ) <EOL> def _get_media_response ( self , url ) : <EOL> self . _current_response = None <EOL> head_response = requests . head ( url , auth = self . auth ) <EOL> if head_response . status_code == <NUM_LIT> : <EOL> url = head_response . headers . get ( '<STR_LIT:location>' ) <EOL> response = requests . get ( url ) <EOL> success = response . status_code == <NUM_LIT:200> <EOL> self . _current_response = response <EOL> return success <EOL> def download_media_files ( self , xml_doc , media_path ) : <EOL> for media_node in xml_doc . getElementsByTagName ( '<STR_LIT>' ) : <EOL> filename_node = media_node . getElementsByTagName ( '<STR_LIT:filename>' ) <EOL> url_node = media_node . getElementsByTagName ( '<STR_LIT>' ) <EOL> if filename_node and url_node : <EOL> filename = filename_node [ <NUM_LIT:0> ] . childNodes [ <NUM_LIT:0> ] . nodeValue <EOL> path = os . path . join ( media_path , filename ) <EOL> if default_storage . exists ( path ) : <EOL> continue <EOL> download_url = url_node [ <NUM_LIT:0> ] . childNodes [ <NUM_LIT:0> ] . nodeValue <EOL> if self . _get_media_response ( download_url ) : <EOL> download_res = self . _current_response <EOL> media_content = ContentFile ( download_res . content ) <EOL> default_storage . save ( path , media_content ) <EOL> self . logger . debug ( "<STR_LIT>" % filename ) <EOL> else : <EOL> self . logger . error ( "<STR_LIT>" % filename ) <EOL> def get_instances_uuids ( self , xml_doc ) : <EOL> uuids = [ ] <EOL> for child_node in xml_doc . childNodes : <EOL> if child_node . nodeName == '<STR_LIT>' : <EOL> for id_node in child_node . getElementsByTagName ( '<STR_LIT:id>' ) : <EOL> if id_node . childNodes : <EOL> uuid = id_node . childNodes [ <NUM_LIT:0> ] . nodeValue <EOL> uuids . append ( uuid ) <EOL> return uuids <EOL> def download_instances ( self , form_id , cursor = <NUM_LIT:0> , num_entries = <NUM_LIT:100> ) : <EOL> self . logger . debug ( "<STR_LIT>" % form_id ) <EOL> if not self . _get_response ( self . submission_list_url , <EOL> params = { '<STR_LIT>' : form_id , <EOL> '<STR_LIT>' : num_entries , <EOL> '<STR_LIT>' : cursor } ) : <EOL> self . logger . error ( "<STR_LIT>" % <EOL> ( self . submission_list_url , form_id , cursor ) ) <EOL> return <EOL> response = self . _current_response <EOL> self . logger . debug ( "<STR_LIT>" % <EOL> ( self . submission_list_url , form_id , cursor ) ) <EOL> try : <EOL> xml_doc = clean_and_parse_xml ( response . content ) <EOL> except ExpatError : <EOL> return <EOL> instances = self . get_instances_uuids ( xml_doc ) <EOL> path = os . path . join ( self . forms_path , form_id , '<STR_LIT>' ) <EOL> for uuid in instances : <EOL> self . logger . debug ( "<STR_LIT>" % ( uuid , form_id ) ) <EOL> form_str = u'<STR_LIT>' u'<STR_LIT>' % { <EOL> '<STR_LIT>' : form_id , <EOL> '<STR_LIT>' : uuid <EOL> } <EOL> instance_path = os . path . join ( path , uuid . replace ( '<STR_LIT::>' , '<STR_LIT>' ) , <EOL> '<STR_LIT>' ) <EOL> if not default_storage . exists ( instance_path ) : <EOL> if self . _get_response ( self . download_submission_url , <EOL> params = { '<STR_LIT>' : form_str } ) : <EOL> instance_res = self . _current_response <EOL> content = instance_res . content . strip ( ) <EOL> default_storage . save ( instance_path , ContentFile ( content ) ) <EOL> else : <EOL> continue <EOL> else : <EOL> instance_res = default_storage . open ( instance_path ) <EOL> content = instance_res . read ( ) <EOL> try : <EOL> instance_doc = clean_and_parse_xml ( content ) <EOL> except ExpatError : <EOL> continue <EOL> media_path = os . path . join ( path , uuid . replace ( '<STR_LIT::>' , '<STR_LIT>' ) ) <EOL> self . download_media_files ( instance_doc , media_path ) <EOL> self . logger . debug ( "<STR_LIT>" % ( form_id , uuid ) ) <EOL> if xml_doc . getElementsByTagName ( '<STR_LIT>' ) : <EOL> rs_node = xml_doc . getElementsByTagName ( '<STR_LIT>' ) [ <NUM_LIT:0> ] <EOL> cursor = rs_node . childNodes [ <NUM_LIT:0> ] . nodeValue <EOL> if self . resumption_cursor != cursor : <EOL> self . resumption_cursor = cursor <EOL> self . download_instances ( form_id , cursor ) <EOL> def _upload_xform ( self , path , file_name ) : <EOL> class PublishXForm ( object ) : <EOL> def __init__ ( self , xml_file , user ) : <EOL> self . xml_file = xml_file <EOL> self . user = user <EOL> def publish_xform ( self ) : <EOL> return publish_xml_form ( self . xml_file , self . user ) <EOL> xml_file = default_storage . open ( path ) <EOL> xml_file . name = file_name <EOL> k = PublishXForm ( xml_file , self . user ) <EOL> return publish_form ( k . publish_xform ) <EOL> def _upload_instance ( self , xml_file , instance_dir_path , files ) : <EOL> xml_doc = clean_and_parse_xml ( xml_file . read ( ) ) <EOL> xml = StringIO ( ) <EOL> de_node = xml_doc . documentElement <EOL> for node in de_node . firstChild . childNodes : <EOL> xml . write ( node . toxml ( ) ) <EOL> new_xml_file = ContentFile ( xml . getvalue ( ) ) <EOL> new_xml_file . content_type = '<STR_LIT>' <EOL> xml . close ( ) <EOL> attachments = [ ] <EOL> for attach in de_node . getElementsByTagName ( '<STR_LIT>' ) : <EOL> filename_node = attach . getElementsByTagName ( '<STR_LIT:filename>' ) <EOL> filename = filename_node [ <NUM_LIT:0> ] . childNodes [ <NUM_LIT:0> ] . nodeValue <EOL> if filename in files : <EOL> file_obj = default_storage . open ( <EOL> os . path . join ( instance_dir_path , filename ) ) <EOL> mimetype , encoding = mimetypes . guess_type ( file_obj . name ) <EOL> media_obj = django_file ( file_obj , '<STR_LIT>' , mimetype ) <EOL> attachments . append ( media_obj ) <EOL> create_instance ( self . user . username , new_xml_file , attachments ) <EOL> def _upload_instances ( self , path ) : <EOL> instances_count = <NUM_LIT:0> <EOL> dirs , not_in_use = default_storage . listdir ( path ) <EOL> for instance_dir in dirs : <EOL> instance_dir_path = os . path . join ( path , instance_dir ) <EOL> i_dirs , files = default_storage . listdir ( instance_dir_path ) <EOL> xml_file = None <EOL> if '<STR_LIT>' in files : <EOL> file_obj = default_storage . open ( <EOL> os . path . join ( instance_dir_path , '<STR_LIT>' ) ) <EOL> xml_file = file_obj <EOL> if xml_file : <EOL> try : <EOL> self . _upload_instance ( xml_file , instance_dir_path , files ) <EOL> except ExpatError : <EOL> continue <EOL> except Exception : <EOL> pass <EOL> else : <EOL> instances_count += <NUM_LIT:1> <EOL> return instances_count <EOL> def push ( self ) : <EOL> dirs , files = default_storage . listdir ( self . forms_path ) <EOL> for form_dir in dirs : <EOL> dir_path = os . path . join ( self . forms_path , form_dir ) <EOL> form_dirs , form_files = default_storage . listdir ( dir_path ) <EOL> form_xml = '<STR_LIT>' % form_dir <EOL> if form_xml in form_files : <EOL> form_xml_path = os . path . join ( dir_path , form_xml ) <EOL> x = self . _upload_xform ( form_xml_path , form_xml ) <EOL> if isinstance ( x , dict ) : <EOL> self . logger . error ( "<STR_LIT>" % form_dir ) <EOL> else : <EOL> self . logger . debug ( "<STR_LIT>" % form_dir ) <EOL> if '<STR_LIT>' in form_dirs : <EOL> self . logger . debug ( "<STR_LIT>" ) <EOL> c = self . _upload_instances ( os . path . join ( dir_path , '<STR_LIT>' ) ) <EOL> self . logger . debug ( "<STR_LIT>" % <EOL> ( c , form_dir ) ) </s>
