<s> """<STR_LIT>""" <EOL> from urllib3 . exceptions import HTTPError , SSLError <EOL> from simplejson import dump as json_dump , load as json_load , loads as json_loads , JSONDecodeError <EOL> from os import stat , sep , error , rename , remove , makedirs , utime , access , R_OK , walk <EOL> from os . path import join , basename , abspath , splitext , sep , isdir , dirname <EOL> from errno import EEXIST <EOL> from stat import S_ISREG <EOL> from glob import iglob <EOL> from logging import getLogger <EOL> from mimetypes import guess_type <EOL> from gzip import GzipFile <EOL> from shutil import rmtree <EOL> from Queue import Queue <EOL> from threading import Thread <EOL> from time import time <EOL> from subprocess import Popen , PIPE <EOL> from poster . encode import gen_boundary , get_headers , MultipartParam <EOL> from turbulenz_local . tools import get_absolute_path , get_7zip_path <EOL> from turbulenz_tools . utils . hash import hash_file_sha256_md5 , hash_file_sha256 , hash_file_md5 <EOL> from turbulenz_local import __version__ <EOL> LOG = getLogger ( __name__ ) <EOL> def _update_file_mtime ( file_path , mtime ) : <EOL> mtime = long ( mtime ) + <NUM_LIT:1> <EOL> utime ( file_path , ( mtime , mtime ) ) <EOL> def _get_upload_file_token ( index , filename ) : <EOL> return '<STR_LIT>' % ( index , splitext ( filename ) [ <NUM_LIT:1> ] ) <EOL> def _get_cached_file_name ( file_name , file_hash , file_length ) : <EOL> return '<STR_LIT>' % ( file_hash , file_length , splitext ( file_name ) [ <NUM_LIT:1> ] ) <EOL> class Deployment ( object ) : <EOL> _batch_checks = True <EOL> _empty_meta_data = { '<STR_LIT>' : <NUM_LIT:0> , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' } <EOL> _base_check_url = '<STR_LIT>' <EOL> _check_url_format = '<STR_LIT>' <EOL> _cached_hash_folder = '<STR_LIT>' <EOL> _cached_hash_ttl = ( <NUM_LIT:30> * <NUM_LIT> * <NUM_LIT> * <NUM_LIT> ) <EOL> _do_not_compress = set ( [ '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' ] ) <EOL> _directories_to_ignore = set ( [ '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' ] ) <EOL> def __init__ ( self , game , hub_pool , hub_project , hub_version , hub_versiontitle , hub_cookie , cache_dir ) : <EOL> self . path = abspath ( get_absolute_path ( game . path ) ) <EOL> self . plugin_main = game . plugin_main <EOL> self . canvas_main = game . canvas_main <EOL> self . flash_main = game . flash_main <EOL> self . mapping_table = game . mapping_table <EOL> self . files = game . deploy_files . items <EOL> self . engine_version = game . engine_version <EOL> self . is_multiplayer = game . is_multiplayer <EOL> self . aspect_ratio = game . aspect_ratio <EOL> self . cache_dir = cache_dir <EOL> self . game_cache_dir = join ( abspath ( cache_dir ) , game . slug ) <EOL> self . stopped = False <EOL> self . hub_project = hub_project <EOL> self . hub_version = hub_version <EOL> self . hub_versiontitle = hub_versiontitle <EOL> self . hub_session = None <EOL> self . hub_pool = hub_pool <EOL> self . hub_cookie = hub_cookie <EOL> self . hub_timeout = <NUM_LIT:200> <EOL> self . total_files = <NUM_LIT:0> <EOL> self . num_files = <NUM_LIT:0> <EOL> self . num_bytes = <NUM_LIT:0> <EOL> self . uploaded_files = <NUM_LIT:0> <EOL> self . uploaded_bytes = <NUM_LIT:0> <EOL> self . done = False <EOL> self . error = None <EOL> try : <EOL> makedirs ( self . get_gzip_dir ( ) ) <EOL> except OSError as e : <EOL> if e . errno != EEXIST : <EOL> LOG . error ( str ( e ) ) <EOL> def get_meta_data_path ( self ) : <EOL> return self . game_cache_dir + '<STR_LIT>' <EOL> def get_gzip_dir ( self ) : <EOL> return self . game_cache_dir . replace ( '<STR_LIT:\\>' , '<STR_LIT:/>' ) <EOL> def deploy ( self , ultra = False ) : <EOL> self . done = self . upload_files ( ultra ) <EOL> if self . hub_session : <EOL> headers = { '<STR_LIT>' : self . hub_cookie } <EOL> fields = { '<STR_LIT>' : self . hub_session } <EOL> try : <EOL> if self . done : <EOL> self . hub_pool . request ( '<STR_LIT:POST>' , <EOL> '<STR_LIT>' , <EOL> fields = fields , <EOL> headers = headers , <EOL> redirect = False , <EOL> retries = <NUM_LIT:5> , <EOL> timeout = self . hub_timeout ) <EOL> else : <EOL> self . hub_pool . request ( '<STR_LIT:POST>' , <EOL> '<STR_LIT>' , <EOL> fields = fields , <EOL> headers = headers , <EOL> redirect = False , <EOL> retries = <NUM_LIT:5> , <EOL> timeout = self . hub_timeout ) <EOL> except ( HTTPError , SSLError ) as e : <EOL> LOG . error ( e ) <EOL> def cancel ( self ) : <EOL> self . stopped = True <EOL> self . error = '<STR_LIT>' <EOL> def stop ( self , error_msg ) : <EOL> self . stopped = True <EOL> self . error = error_msg <EOL> def read_metadata_cache ( self ) : <EOL> try : <EOL> file_name = self . get_meta_data_path ( ) <EOL> gzip_file = GzipFile ( filename = file_name , <EOL> mode = '<STR_LIT:rb>' ) <EOL> meta_data_cache = json_load ( gzip_file ) <EOL> gzip_file . close ( ) <EOL> cache_time = stat ( file_name ) . st_mtime <EOL> except ( IOError , ValueError ) : <EOL> cache_time = - <NUM_LIT:1> <EOL> meta_data_cache = { } <EOL> return cache_time , meta_data_cache <EOL> def write_metadata_cache ( self , meta_data , force_mtime ) : <EOL> try : <EOL> file_path = self . get_meta_data_path ( ) <EOL> gzip_file = GzipFile ( filename = file_path , <EOL> mode = '<STR_LIT:wb>' , <EOL> compresslevel = <NUM_LIT:9> ) <EOL> json_dump ( meta_data , gzip_file , separators = ( '<STR_LIT:U+002C>' , '<STR_LIT::>' ) , sort_keys = True ) <EOL> gzip_file . close ( ) <EOL> if force_mtime > <NUM_LIT:0> : <EOL> _update_file_mtime ( file_path , force_mtime ) <EOL> except ( IOError , OSError ) : <EOL> pass <EOL> def delete_unused_cache_files ( self , meta_data , meta_data_cache ) : <EOL> old_files_to_delete = ( set ( meta_data_cache . iterkeys ( ) ) - set ( meta_data . iterkeys ( ) ) ) <EOL> if old_files_to_delete : <EOL> gzip_cache_dir = self . get_gzip_dir ( ) <EOL> for relative_path in old_files_to_delete : <EOL> cache_file_name = '<STR_LIT>' % ( gzip_cache_dir , relative_path ) <EOL> if access ( cache_file_name , R_OK ) : <EOL> remove ( cache_file_name ) <EOL> def batch_check_files ( self , files , checked_queue_put ) : <EOL> urlopen = self . hub_pool . urlopen <EOL> base_url = self . _base_check_url <EOL> url_format = self . _check_url_format <EOL> get_upload_token = _get_upload_file_token <EOL> timeout = self . hub_timeout <EOL> if self . _batch_checks : <EOL> query = '<STR_LIT:&>' . join ( ( url_format % ( get_upload_token ( i , f [ <NUM_LIT:1> ] ) , f [ <NUM_LIT:3> ] , f [ <NUM_LIT:2> ] ) ) for i , f in enumerate ( files ) ) <EOL> r = urlopen ( '<STR_LIT:GET>' , <EOL> base_url + query , <EOL> redirect = False , <EOL> assert_same_host = False , <EOL> timeout = timeout ) <EOL> if r . status == <NUM_LIT:200> : <EOL> missing_files = set ( json_loads ( r . data ) . get ( '<STR_LIT>' , [ ] ) ) <EOL> for i , f in enumerate ( files ) : <EOL> if get_upload_token ( i , f [ <NUM_LIT:1> ] ) in missing_files : <EOL> checked_queue_put ( f ) <EOL> else : <EOL> checked_queue_put ( ( f [ <NUM_LIT:1> ] , f [ <NUM_LIT:2> ] , f [ <NUM_LIT:3> ] , f [ <NUM_LIT:4> ] , f [ <NUM_LIT:5> ] ) ) <EOL> return <EOL> else : <EOL> f = files . pop ( <NUM_LIT:0> ) <EOL> if r . status == <NUM_LIT> : <EOL> checked_queue_put ( ( f [ <NUM_LIT:1> ] , f [ <NUM_LIT:2> ] , f [ <NUM_LIT:3> ] , f [ <NUM_LIT:4> ] , f [ <NUM_LIT:5> ] ) ) <EOL> elif r . status == <NUM_LIT> : <EOL> checked_queue_put ( f ) <EOL> else : <EOL> raise Exception ( r . reason ) <EOL> if len ( files ) == <NUM_LIT:1> : <EOL> return <EOL> self . _batch_checks = False <EOL> r = None <EOL> for f in files : <EOL> query = url_format % ( basename ( f [ <NUM_LIT:1> ] ) , f [ <NUM_LIT:3> ] , f [ <NUM_LIT:2> ] ) <EOL> if urlopen ( '<STR_LIT:GET>' , <EOL> base_url + query , <EOL> redirect = False , <EOL> assert_same_host = False , <EOL> timeout = timeout ) . status == <NUM_LIT> : <EOL> checked_queue_put ( ( f [ <NUM_LIT:1> ] , f [ <NUM_LIT:2> ] , f [ <NUM_LIT:3> ] , f [ <NUM_LIT:4> ] , f [ <NUM_LIT:5> ] ) ) <EOL> else : <EOL> checked_queue_put ( f ) <EOL> def check_files ( self , files , start , end , checked_queue_put , hashes , ultra , cache_time , meta_data_cache ) : <EOL> files_to_batch_check = [ ] <EOL> base_path_len = len ( self . path ) <EOL> if not self . path . endswith ( sep ) : <EOL> base_path_len += <NUM_LIT:1> <EOL> gzip_cache_dir = self . get_gzip_dir ( ) <EOL> compressor_path = get_7zip_path ( ) <EOL> empty_meta_data = self . _empty_meta_data <EOL> get_cached_file_name = _get_cached_file_name <EOL> while start < end : <EOL> if self . stopped : <EOL> checked_queue_put ( None ) <EOL> break <EOL> abs_path = files [ start ] <EOL> start += <NUM_LIT:1> <EOL> relative_path = abs_path [ base_path_len : ] <EOL> try : <EOL> file_stat = stat ( abs_path ) <EOL> file_size = file_stat . st_size <EOL> if not S_ISREG ( file_stat . st_mode ) or file_size <= <NUM_LIT:0> : <EOL> checked_queue_put ( relative_path ) <EOL> continue <EOL> calculate_hash = update_meta_data = False <EOL> file_time = max ( file_stat . st_mtime , file_stat . st_ctime ) <EOL> if cache_time < file_time : <EOL> calculate_hash = True <EOL> else : <EOL> old_meta_data = meta_data_cache . get ( relative_path , empty_meta_data ) <EOL> if file_size != old_meta_data [ '<STR_LIT>' ] : <EOL> calculate_hash = True <EOL> else : <EOL> file_hash = old_meta_data [ '<STR_LIT>' ] <EOL> file_md5 = old_meta_data [ '<STR_LIT>' ] <EOL> if relative_path . split ( '<STR_LIT:.>' ) [ - <NUM_LIT:1> ] not in self . _do_not_compress : <EOL> deploy_file_name = '<STR_LIT>' % ( gzip_cache_dir , relative_path ) <EOL> do_compress = False <EOL> try : <EOL> file_stat = stat ( deploy_file_name ) <EOL> if file_stat . st_mtime < file_time : <EOL> do_compress = True <EOL> elif file_stat . st_size >= file_size : <EOL> deploy_file_name = abs_path <EOL> except error : <EOL> do_compress = True <EOL> if do_compress : <EOL> if compressor_path : <EOL> if ultra : <EOL> process = Popen ( [ compressor_path , <EOL> '<STR_LIT:a>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> deploy_file_name , abs_path ] , <EOL> stdout = PIPE , stderr = PIPE ) <EOL> else : <EOL> process = Popen ( [ compressor_path , <EOL> '<STR_LIT:a>' , '<STR_LIT>' , <EOL> deploy_file_name , abs_path ] , <EOL> stdout = PIPE , stderr = PIPE ) <EOL> update_meta_data = True <EOL> if calculate_hash : <EOL> calculate_hash = False <EOL> file_hash = hash_file_sha256 ( abs_path ) <EOL> output , _ = process . communicate ( ) <EOL> if process . poll ( ) : <EOL> self . stop ( '<STR_LIT>' % ( relative_path , str ( output ) ) ) <EOL> continue <EOL> else : <EOL> try : <EOL> if stat ( deploy_file_name ) . st_size >= file_size : <EOL> deploy_file_name = abs_path <EOL> except error as e : <EOL> self . stop ( '<STR_LIT>' % ( deploy_file_name , str ( e ) ) ) <EOL> continue <EOL> file_md5 = hash_file_md5 ( deploy_file_name ) <EOL> else : <EOL> cache_dir = dirname ( deploy_file_name ) <EOL> try : <EOL> makedirs ( cache_dir ) <EOL> except OSError as e : <EOL> if e . errno != EEXIST : <EOL> self . stop ( '<STR_LIT>' % ( relative_path , str ( e ) ) ) <EOL> continue <EOL> try : <EOL> with GzipFile ( deploy_file_name , mode = '<STR_LIT:wb>' , compresslevel = <NUM_LIT:9> ) as gzipfile : <EOL> with open ( abs_path , '<STR_LIT:rb>' ) as f : <EOL> gzipfile . write ( f . read ( ) ) <EOL> except IOError as e : <EOL> self . stop ( '<STR_LIT>' % ( relative_path , str ( e ) ) ) <EOL> continue <EOL> LOG . warning ( '<STR_LIT>' ) <EOL> update_meta_data = True <EOL> if calculate_hash : <EOL> calculate_hash = False <EOL> file_hash = hash_file_sha256 ( abs_path ) <EOL> try : <EOL> if stat ( deploy_file_name ) . st_size >= file_size : <EOL> deploy_file_name = abs_path <EOL> except error as e : <EOL> self . stop ( '<STR_LIT>' % ( deploy_file_name , str ( e ) ) ) <EOL> continue <EOL> file_md5 = hash_file_md5 ( deploy_file_name ) <EOL> else : <EOL> deploy_file_name = abs_path <EOL> if calculate_hash : <EOL> update_meta_data = True <EOL> if deploy_file_name == abs_path : <EOL> file_hash , file_md5 = hash_file_sha256_md5 ( abs_path ) <EOL> else : <EOL> file_hash = hash_file_sha256 ( abs_path ) <EOL> file_md5 = hash_file_md5 ( deploy_file_name ) <EOL> if get_cached_file_name ( relative_path , file_hash , file_size ) not in hashes : <EOL> file_item = ( deploy_file_name , relative_path , file_size , file_hash , file_md5 , file_time ) <EOL> files_to_batch_check . append ( file_item ) <EOL> if len ( files_to_batch_check ) >= <NUM_LIT:10> : <EOL> self . batch_check_files ( files_to_batch_check , checked_queue_put ) <EOL> files_to_batch_check = [ ] <EOL> elif update_meta_data : <EOL> checked_queue_put ( ( relative_path , file_size , file_hash , file_md5 , file_time ) ) <EOL> else : <EOL> checked_queue_put ( ( relative_path , file_size , file_hash , file_time ) ) <EOL> file_stat = None <EOL> except ( error , IOError ) as e : <EOL> self . stop ( '<STR_LIT>' % ( relative_path , str ( e ) ) ) <EOL> except Exception as e : <EOL> self . stop ( '<STR_LIT>' % ( relative_path , str ( e ) ) ) <EOL> if len ( files_to_batch_check ) > <NUM_LIT:0> : <EOL> try : <EOL> self . batch_check_files ( files_to_batch_check , checked_queue_put ) <EOL> except ( HTTPError , SSLError , ValueError ) as e : <EOL> self . stop ( '<STR_LIT>' % str ( e ) ) <EOL> except Exception as e : <EOL> self . stop ( '<STR_LIT>' % str ( e ) ) <EOL> def find_files ( self ) : <EOL> files = set ( ) <EOL> path = self . path <EOL> directories_to_ignore = self . _directories_to_ignore <EOL> for pattern in self . files : <EOL> if pattern : <EOL> for abs_path in iglob ( join ( path , pattern ) ) : <EOL> if isdir ( abs_path ) : <EOL> for tmp_root , dir_names , list_of_files in walk ( abs_path ) : <EOL> if dir_names : <EOL> dir_names [ : ] = ( dirname for dirname in dir_names <EOL> if dirname not in directories_to_ignore ) <EOL> files . update ( join ( tmp_root , filename ) . replace ( '<STR_LIT:\\>' , '<STR_LIT:/>' ) for filename in list_of_files ) <EOL> else : <EOL> files . add ( abs_path . replace ( '<STR_LIT:\\>' , '<STR_LIT:/>' ) ) <EOL> return list ( files ) <EOL> def load_hashes ( self , project ) : <EOL> hashes = set ( ) <EOL> try : <EOL> hashes_folder = join ( self . cache_dir , self . _cached_hash_folder ) <EOL> stale_time = long ( time ( ) - self . _cached_hash_ttl ) <EOL> for file_path in iglob ( join ( hashes_folder , '<STR_LIT>' ) ) : <EOL> delete_file = True <EOL> try : <EOL> file_time = long ( splitext ( basename ( file_path ) ) [ <NUM_LIT:0> ] ) <EOL> if stale_time < file_time : <EOL> file_obj = open ( file_path , '<STR_LIT:rb>' ) <EOL> hashes_meta = json_load ( file_obj ) <EOL> file_obj . close ( ) <EOL> hashes_version = hashes_meta . get ( '<STR_LIT:version>' , <NUM_LIT:0> ) <EOL> if <NUM_LIT:2> <= hashes_version : <EOL> cached_hashes = hashes_meta . get ( '<STR_LIT>' , None ) <EOL> if cached_hashes : <EOL> delete_file = False <EOL> hashes_host = hashes_meta . get ( '<STR_LIT:host>' , None ) <EOL> if hashes_host == self . hub_pool . host : <EOL> hashes . update ( cached_hashes ) <EOL> except ( TypeError , ValueError ) : <EOL> pass <EOL> if delete_file : <EOL> LOG . info ( '<STR_LIT>' , file_path ) <EOL> remove ( file_path ) <EOL> except ( IOError , error ) : <EOL> pass <EOL> except Exception as e : <EOL> LOG . error ( str ( e ) ) <EOL> hashes . update ( self . request_hashes ( project ) ) <EOL> return hashes <EOL> def request_hashes ( self , project ) : <EOL> try : <EOL> min_version = <NUM_LIT:2> <EOL> r = self . hub_pool . urlopen ( '<STR_LIT:GET>' , <EOL> '<STR_LIT>' % ( min_version , project ) , <EOL> headers = { '<STR_LIT>' : self . hub_cookie , <EOL> '<STR_LIT>' : '<STR_LIT>' } , <EOL> redirect = False , <EOL> assert_same_host = False , <EOL> timeout = self . hub_timeout ) <EOL> if r . status == <NUM_LIT:200> : <EOL> response = json_loads ( r . data ) <EOL> if response . get ( '<STR_LIT:version>' , <NUM_LIT:1> ) >= min_version : <EOL> return response [ '<STR_LIT>' ] <EOL> except ( HTTPError , SSLError , TypeError , ValueError ) : <EOL> pass <EOL> except Exception as e : <EOL> LOG . error ( str ( e ) ) <EOL> return [ ] <EOL> def save_hashes ( self , hashes ) : <EOL> try : <EOL> hashes_folder = join ( self . cache_dir , self . _cached_hash_folder ) <EOL> try : <EOL> makedirs ( hashes_folder ) <EOL> except OSError as e : <EOL> if e . errno != EEXIST : <EOL> LOG . error ( str ( e ) ) <EOL> return <EOL> for file_path in iglob ( join ( hashes_folder , '<STR_LIT>' ) ) : <EOL> try : <EOL> file_obj = open ( file_path , '<STR_LIT:rb>' ) <EOL> hashes_meta = json_load ( file_obj ) <EOL> file_obj . close ( ) <EOL> hashes_host = hashes_meta [ '<STR_LIT:host>' ] <EOL> if hashes_host == self . hub_pool . host : <EOL> hashes . difference_update ( hashes_meta [ '<STR_LIT>' ] ) <EOL> except ( IOError , TypeError , ValueError , KeyError , AttributeError ) : <EOL> pass <EOL> if hashes : <EOL> try : <EOL> file_path = join ( hashes_folder , '<STR_LIT>' % long ( time ( ) ) ) <EOL> file_obj = open ( file_path , '<STR_LIT:wb>' ) <EOL> hashes_meta = { '<STR_LIT:version>' : <NUM_LIT:2> , <EOL> '<STR_LIT:host>' : self . hub_pool . host , <EOL> '<STR_LIT>' : list ( hashes ) } <EOL> json_dump ( hashes_meta , file_obj , separators = ( '<STR_LIT:U+002C>' , '<STR_LIT::>' ) ) <EOL> file_obj . close ( ) <EOL> except IOError : <EOL> pass <EOL> except Exception as e : <EOL> LOG . error ( str ( e ) ) <EOL> def start_scan_workers ( self , files , checked_queue , hashes , ultra , cache_time , meta_data_cache ) : <EOL> num_files = len ( files ) <EOL> num_workers = <NUM_LIT:4> <EOL> if num_workers > num_files : <EOL> num_workers = num_files <EOL> start = <NUM_LIT:0> <EOL> step = int ( ( num_files + ( num_workers - <NUM_LIT:1> ) ) / num_workers ) <EOL> for _ in range ( num_workers ) : <EOL> end = ( start + step ) <EOL> if end > num_files : <EOL> end = num_files <EOL> Thread ( target = self . check_files , args = [ files , start , end , <EOL> checked_queue . put , <EOL> hashes , ultra , cache_time , meta_data_cache ] ) . start ( ) <EOL> start = end <EOL> def scan_files ( self , hashes , ultra ) : <EOL> files = self . find_files ( ) <EOL> num_files = len ( files ) <EOL> self . total_files = num_files <EOL> cache_time , meta_data_cache = self . read_metadata_cache ( ) <EOL> checked_queue = Queue ( ) <EOL> self . start_scan_workers ( files , checked_queue , hashes , ultra , cache_time , meta_data_cache ) <EOL> files_scanned = [ ] <EOL> files_to_upload = [ ] <EOL> meta_data = { } <EOL> update_meta_data = False <EOL> newer_time = - <NUM_LIT:1> <EOL> while True : <EOL> item = checked_queue . get ( ) <EOL> if item is None or self . stopped : <EOL> break <EOL> elif isinstance ( item , basestring ) : <EOL> num_files -= <NUM_LIT:1> <EOL> else : <EOL> if len ( item ) == <NUM_LIT:4> : <EOL> relative_path , file_size , file_hash , file_time = item <EOL> meta_data [ relative_path ] = meta_data_cache [ relative_path ] <EOL> files_scanned . append ( ( relative_path , file_size , file_hash ) ) <EOL> else : <EOL> if len ( item ) == <NUM_LIT:5> : <EOL> relative_path , file_size , file_hash , file_md5 , file_time = item <EOL> files_scanned . append ( ( relative_path , file_size , file_hash ) ) <EOL> else : <EOL> deploy_path , relative_path , file_size , file_hash , file_md5 , file_time = item <EOL> files_to_upload . append ( ( deploy_path , relative_path , file_size , file_hash , file_md5 ) ) <EOL> meta_data [ relative_path ] = { '<STR_LIT>' : file_size , <EOL> '<STR_LIT>' : file_hash , <EOL> '<STR_LIT>' : file_md5 } <EOL> update_meta_data = True <EOL> if newer_time < file_time : <EOL> newer_time = file_time <EOL> self . num_bytes += file_size <EOL> self . num_files += <NUM_LIT:1> <EOL> if self . num_files >= num_files : <EOL> break <EOL> item = None <EOL> if self . stopped : <EOL> meta_data . update ( meta_data_cache ) <EOL> if update_meta_data or newer_time > cache_time or len ( meta_data ) != len ( meta_data_cache ) : <EOL> self . write_metadata_cache ( meta_data , newer_time ) <EOL> self . delete_unused_cache_files ( meta_data , meta_data_cache ) <EOL> return files_scanned , files_to_upload <EOL> def update_num_bytes ( self , x ) : <EOL> self . num_bytes += len ( x ) <EOL> def post ( self , url , params , boundary ) : <EOL> headers = get_headers ( params , boundary ) <EOL> headers [ '<STR_LIT>' ] = self . hub_cookie <EOL> params = MultipartParam . from_params ( params ) <EOL> return self . hub_pool . urlopen ( '<STR_LIT:POST>' , <EOL> url , <EOL> MultipartReader ( params , boundary ) , <EOL> headers = headers , <EOL> timeout = self . hub_timeout ) <EOL> def post_files ( self , files , start , end , uploaded_queue_put , boundary , local_deploy ) : <EOL> hub_session = self . hub_session <EOL> hub_cookie = self . hub_cookie <EOL> hub_pool = self . hub_pool <EOL> while start < end : <EOL> if self . stopped : <EOL> uploaded_queue_put ( None ) <EOL> break <EOL> item = files [ start ] <EOL> start += <NUM_LIT:1> <EOL> deploy_path , relative_path , file_size , file_hash , file_md5 = item <EOL> try : <EOL> if local_deploy : <EOL> guessed_type = guess_type ( relative_path ) [ <NUM_LIT:0> ] <EOL> if guessed_type is None : <EOL> guessed_type = "<STR_LIT>" <EOL> params = { '<STR_LIT>' : guessed_type , <EOL> '<STR_LIT>' : relative_path , <EOL> '<STR_LIT>' : deploy_path , <EOL> '<STR_LIT>' : hub_session , <EOL> '<STR_LIT>' : file_hash , <EOL> '<STR_LIT>' : str ( file_size ) , <EOL> '<STR_LIT>' : file_md5 } <EOL> if deploy_path . endswith ( '<STR_LIT>' ) : <EOL> params [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> r = hub_pool . request ( '<STR_LIT:POST>' , <EOL> '<STR_LIT>' , <EOL> fields = params , <EOL> headers = { '<STR_LIT>' : hub_cookie } , <EOL> timeout = self . hub_timeout ) <EOL> else : <EOL> params = [ MultipartParam ( '<STR_LIT:file>' , <EOL> filename = relative_path , <EOL> filetype = guess_type ( relative_path ) [ <NUM_LIT:0> ] , <EOL> fileobj = open ( deploy_path , '<STR_LIT:rb>' ) ) , <EOL> ( '<STR_LIT>' , hub_session ) , <EOL> ( '<STR_LIT>' , file_hash ) , <EOL> ( '<STR_LIT>' , file_size ) , <EOL> ( '<STR_LIT>' , file_md5 ) ] <EOL> if deploy_path . endswith ( '<STR_LIT>' ) : <EOL> params . append ( ( '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> headers = get_headers ( params , boundary ) <EOL> headers [ '<STR_LIT>' ] = hub_cookie <EOL> params = MultipartParam . from_params ( params ) <EOL> params = MultipartReader ( params , boundary ) <EOL> r = hub_pool . urlopen ( '<STR_LIT:POST>' , <EOL> '<STR_LIT>' , <EOL> params , <EOL> headers = headers , <EOL> timeout = self . hub_timeout ) <EOL> except IOError : <EOL> self . stop ( '<STR_LIT>' % deploy_path ) <EOL> continue <EOL> except ( HTTPError , SSLError , ValueError ) as e : <EOL> self . stop ( '<STR_LIT>' % ( relative_path , e ) ) <EOL> continue <EOL> if r . headers . get ( '<STR_LIT>' , '<STR_LIT>' ) != '<STR_LIT>' : <EOL> self . stop ( '<STR_LIT>' % relative_path ) <EOL> continue <EOL> answer = json_loads ( r . data ) <EOL> if r . status != <NUM_LIT:200> : <EOL> if answer . get ( '<STR_LIT>' , False ) : <EOL> self . stop ( '<STR_LIT>' % relative_path ) <EOL> else : <EOL> msg = answer . get ( '<STR_LIT>' , None ) <EOL> if msg : <EOL> self . stop ( '<STR_LIT>' % ( relative_path , msg ) ) <EOL> else : <EOL> self . stop ( '<STR_LIT>' % ( relative_path , r . reason ) ) <EOL> continue <EOL> if not answer . get ( '<STR_LIT>' , False ) : <EOL> self . stop ( '<STR_LIT>' % relative_path ) <EOL> continue <EOL> uploaded_queue_put ( ( relative_path , file_size , file_hash ) ) <EOL> answer = None <EOL> r = None <EOL> params = None <EOL> relative_path = None <EOL> deploy_path = None <EOL> item = None <EOL> def start_upload_workers ( self , files , uploaded_queue , boundary , local_deploy ) : <EOL> num_files = len ( files ) <EOL> num_workers = <NUM_LIT:4> <EOL> if num_workers > num_files : <EOL> num_workers = num_files <EOL> start = <NUM_LIT:0> <EOL> step = int ( ( num_files + ( num_workers - <NUM_LIT:1> ) ) / num_workers ) <EOL> for _ in range ( num_workers ) : <EOL> end = ( start + step ) <EOL> if end > num_files : <EOL> end = num_files <EOL> Thread ( target = self . post_files , args = [ files , start , end , uploaded_queue . put , boundary , local_deploy ] ) . start ( ) <EOL> start = end <EOL> def upload_files ( self , ultra ) : <EOL> hashes = self . load_hashes ( self . hub_project ) <EOL> files_scanned , files_to_upload = self . scan_files ( hashes , ultra ) <EOL> if self . stopped : <EOL> return False <EOL> num_files = self . num_files <EOL> if num_files <= <NUM_LIT:0> : <EOL> return True <EOL> boundary = gen_boundary ( ) <EOL> local_deploy = self . hub_pool . host in [ '<STR_LIT:127.0.0.1>' , '<STR_LIT>' , '<STR_LIT:localhost>' ] <EOL> try : <EOL> if local_deploy : <EOL> params = { '<STR_LIT>' : self . get_meta_data_path ( ) , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : self . hub_project , <EOL> '<STR_LIT:version>' : self . hub_version , <EOL> '<STR_LIT>' : self . hub_versiontitle , <EOL> '<STR_LIT>' : self . plugin_main , <EOL> '<STR_LIT>' : self . canvas_main , <EOL> '<STR_LIT>' : self . flash_main , <EOL> '<STR_LIT>' : self . mapping_table , <EOL> '<STR_LIT>' : self . engine_version , <EOL> '<STR_LIT>' : self . is_multiplayer , <EOL> '<STR_LIT>' : self . aspect_ratio , <EOL> '<STR_LIT>' : str ( num_files ) , <EOL> '<STR_LIT>' : str ( self . num_bytes ) , <EOL> '<STR_LIT>' : __version__ } <EOL> r = self . hub_pool . request ( '<STR_LIT:POST>' , <EOL> '<STR_LIT>' , <EOL> fields = params , <EOL> headers = { '<STR_LIT>' : self . hub_cookie } , <EOL> timeout = self . hub_timeout ) <EOL> else : <EOL> r = self . post ( '<STR_LIT>' , <EOL> [ MultipartParam ( '<STR_LIT>' , <EOL> filename = '<STR_LIT>' , <EOL> filetype = '<STR_LIT>' , <EOL> fileobj = open ( self . get_meta_data_path ( ) , '<STR_LIT:rb>' ) ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , self . hub_project ) , <EOL> ( '<STR_LIT:version>' , self . hub_version ) , <EOL> ( '<STR_LIT>' , self . hub_versiontitle ) , <EOL> ( '<STR_LIT>' , self . plugin_main ) , <EOL> ( '<STR_LIT>' , self . canvas_main ) , <EOL> ( '<STR_LIT>' , self . flash_main ) , <EOL> ( '<STR_LIT>' , self . mapping_table ) , <EOL> ( '<STR_LIT>' , self . engine_version ) , <EOL> ( '<STR_LIT>' , self . is_multiplayer ) , <EOL> ( '<STR_LIT>' , self . aspect_ratio ) , <EOL> ( '<STR_LIT>' , num_files ) , <EOL> ( '<STR_LIT>' , self . num_bytes ) , <EOL> ( '<STR_LIT>' , __version__ ) ] , <EOL> boundary ) <EOL> except IOError : <EOL> self . stop ( '<STR_LIT>' % self . get_meta_data_path ( ) ) <EOL> return False <EOL> except ( HTTPError , SSLError ) as e : <EOL> self . stop ( '<STR_LIT>' % e ) <EOL> return False <EOL> if r . status == <NUM_LIT> : <EOL> self . stop ( '<STR_LIT>' ) <EOL> return False <EOL> if r . headers . get ( '<STR_LIT>' , '<STR_LIT>' ) == '<STR_LIT>' and r . data != '<STR_LIT>' : <EOL> try : <EOL> answer = json_loads ( r . data ) <EOL> except JSONDecodeError as e : <EOL> LOG . error ( e ) <EOL> answer = { } <EOL> else : <EOL> answer = { } <EOL> if r . status != <NUM_LIT:200> : <EOL> msg = answer . get ( '<STR_LIT>' , False ) <EOL> if msg : <EOL> self . stop ( msg ) <EOL> else : <EOL> self . stop ( '<STR_LIT>' % r . reason ) <EOL> return False <EOL> hub_session = answer . get ( '<STR_LIT>' , None ) <EOL> if not answer . get ( '<STR_LIT>' , False ) or not hub_session : <EOL> self . stop ( '<STR_LIT>' ) <EOL> return False <EOL> self . hub_session = hub_session <EOL> get_cached_file_name = _get_cached_file_name <EOL> for file_name , file_size , file_hash in files_scanned : <EOL> hashes . add ( get_cached_file_name ( file_name , file_hash , file_size ) ) <EOL> self . uploaded_bytes += file_size <EOL> self . uploaded_files += <NUM_LIT:1> <EOL> if self . uploaded_files >= num_files : <EOL> self . save_hashes ( hashes ) <EOL> return True <EOL> uploaded_queue = Queue ( ) <EOL> self . start_upload_workers ( files_to_upload , uploaded_queue , boundary , local_deploy ) <EOL> while True : <EOL> item = uploaded_queue . get ( ) <EOL> if item is None or self . stopped : <EOL> break <EOL> file_name , file_size , file_hash = item <EOL> hashes . add ( get_cached_file_name ( file_name , file_hash , file_size ) ) <EOL> self . uploaded_bytes += file_size <EOL> self . uploaded_files += <NUM_LIT:1> <EOL> if self . uploaded_files >= num_files : <EOL> self . save_hashes ( hashes ) <EOL> return True <EOL> item = None <EOL> self . save_hashes ( hashes ) <EOL> return False <EOL> @ classmethod <EOL> def rename_cache ( cls , cache_dir , old_slug , new_slug ) : <EOL> old_file = join ( cache_dir , old_slug ) + '<STR_LIT>' <EOL> new_file = join ( cache_dir , new_slug ) + '<STR_LIT>' <EOL> old_folder = join ( cache_dir , old_slug ) <EOL> new_folder = join ( cache_dir , new_slug ) <EOL> try : <EOL> remove ( new_file ) <EOL> rmtree ( new_folder ) <EOL> except OSError : <EOL> pass <EOL> try : <EOL> rename ( old_file , new_file ) <EOL> rename ( old_folder , new_folder ) <EOL> except OSError : <EOL> pass <EOL> class MultipartReader ( object ) : <EOL> def __init__ ( self , params , boundary ) : <EOL> self . params = params <EOL> self . boundary = boundary <EOL> self . i = <NUM_LIT:0> <EOL> self . param = None <EOL> self . param_iter = None <EOL> def read ( self , blocksize ) : <EOL> """<STR_LIT>""" <EOL> if self . param_iter is not None : <EOL> try : <EOL> return self . param_iter . next ( ) <EOL> except StopIteration : <EOL> self . param = None <EOL> self . param_iter = None <EOL> if self . i is None : <EOL> return None <EOL> elif self . i >= len ( self . params ) : <EOL> self . param_iter = None <EOL> self . param = None <EOL> self . i = None <EOL> return "<STR_LIT>" % self . boundary <EOL> self . param = self . params [ self . i ] <EOL> self . param_iter = self . param . iter_encode ( self . boundary , blocksize ) <EOL> self . i += <NUM_LIT:1> <EOL> return self . read ( blocksize ) <EOL> def reset ( self ) : <EOL> self . i = <NUM_LIT:0> <EOL> for param in self . params : <EOL> param . reset ( ) </s>
