<s> import os <EOL> import stat <EOL> import logging <EOL> import requests <EOL> from tempfile import NamedTemporaryFile <EOL> from urlparse import urlparse <EOL> import celery <EOL> from celery . task import task <EOL> from django . core . files import File <EOL> from file_store . models import ( FileStoreItem , get_temp_dir , file_path , <EOL> FILE_STORE_BASE_DIR ) <EOL> logger = logging . getLogger ( __name__ ) <EOL> @ task ( ) <EOL> def create ( source , sharename = '<STR_LIT>' , filetype = '<STR_LIT>' , permanent = False , file_size = <NUM_LIT:1> ) : <EOL> """<STR_LIT>""" <EOL> logger . info ( "<STR_LIT>" , source ) <EOL> item = FileStoreItem . objects . create_item ( <EOL> source = source , sharename = sharename , filetype = filetype ) <EOL> if not item : <EOL> logger . error ( "<STR_LIT>" , <EOL> source ) <EOL> return None <EOL> logger . info ( "<STR_LIT>" , item . uuid ) <EOL> return item . uuid <EOL> @ task ( track_started = True ) <EOL> def import_file ( uuid , permanent = False , refresh = False , file_size = <NUM_LIT:0> ) : <EOL> """<STR_LIT>""" <EOL> logger . debug ( "<STR_LIT>" , uuid ) <EOL> item = FileStoreItem . objects . get_item ( uuid = uuid ) <EOL> if not item : <EOL> logger . error ( "<STR_LIT>" , uuid ) <EOL> return None <EOL> item . import_task_id = import_file . request . id <EOL> item . save ( ) <EOL> if item . is_local ( ) : <EOL> if refresh : <EOL> item . delete_datafile ( ) <EOL> else : <EOL> return item <EOL> if os . path . isabs ( item . source ) : <EOL> if os . path . isfile ( item . source ) : <EOL> try : <EOL> srcfile = File ( open ( item . source ) ) <EOL> except IOError : <EOL> logger . error ( "<STR_LIT>" , item . source ) <EOL> return None <EOL> srcfilename = os . path . basename ( item . source ) <EOL> item . datafile . save ( srcfilename , srcfile ) <EOL> srcfile . close ( ) <EOL> logger . info ( "<STR_LIT>" ) <EOL> else : <EOL> logger . error ( "<STR_LIT>" ) <EOL> return None <EOL> else : <EOL> try : <EOL> response = requests . get ( item . source , stream = True ) <EOL> except requests . exceptions . HTTPError as e : <EOL> logger . error ( "<STR_LIT>" , item . source ) <EOL> return None <EOL> except requests . exceptions . ConnectionError as e : <EOL> logger . error ( "<STR_LIT>" , <EOL> item . source , e . reason ) <EOL> return None <EOL> except ValueError as e : <EOL> logger . error ( "<STR_LIT>" , <EOL> item . source , e . message ) <EOL> return None <EOL> tmpfile = NamedTemporaryFile ( dir = get_temp_dir ( ) , delete = False ) <EOL> remote_file_size = int ( <EOL> response . headers . get ( '<STR_LIT>' , file_size ) <EOL> ) <EOL> logger . debug ( "<STR_LIT>" , item . source ) <EOL> local_file_size = <NUM_LIT:0> <EOL> block_size = <NUM_LIT:10> * <NUM_LIT> * <NUM_LIT> <EOL> for buf in iter ( lambda : response . raw . read ( block_size ) , '<STR_LIT>' ) : <EOL> local_file_size += len ( buf ) <EOL> try : <EOL> tmpfile . write ( buf ) <EOL> except IOError as exc : <EOL> logger . error ( "<STR_LIT>" , <EOL> item . source , exc ) <EOL> import_file . update_state ( state = celery . states . FAILURE ) <EOL> return <EOL> if remote_file_size > <NUM_LIT:0> : <EOL> percent_done = local_file_size * <NUM_LIT> / remote_file_size <EOL> else : <EOL> percent_done = <NUM_LIT:0> <EOL> import_file . update_state ( <EOL> state = "<STR_LIT>" , <EOL> meta = { <EOL> "<STR_LIT>" : "<STR_LIT>" . format ( percent_done ) , <EOL> "<STR_LIT>" : local_file_size , <EOL> "<STR_LIT>" : remote_file_size <EOL> } ) <EOL> tmpfile . flush ( ) <EOL> tmpfile . close ( ) <EOL> response . close ( ) <EOL> logger . debug ( "<STR_LIT>" , item . source ) <EOL> u = urlparse ( item . source ) <EOL> src_file_name = os . path . basename ( u . path ) <EOL> rel_dst_path = item . datafile . storage . get_available_name ( <EOL> file_path ( item , src_file_name ) <EOL> ) <EOL> abs_dst_path = os . path . join ( FILE_STORE_BASE_DIR , rel_dst_path ) <EOL> try : <EOL> if not os . path . exists ( os . path . dirname ( abs_dst_path ) ) : <EOL> os . makedirs ( os . path . dirname ( abs_dst_path ) ) <EOL> os . rename ( tmpfile . name , abs_dst_path ) <EOL> except OSError as e : <EOL> logger . error ( <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" , <EOL> e . errno , e . filename , e . strerror ) <EOL> return False <EOL> try : <EOL> mode = os . stat ( abs_dst_path ) . st_mode <EOL> os . chmod ( abs_dst_path , <EOL> mode | stat . S_IRGRP | stat . S_IWGRP | stat . S_IROTH ) <EOL> except OSError as e : <EOL> logger . error ( "<STR_LIT>" , abs_dst_path ) <EOL> logger . error ( "<STR_LIT>" , <EOL> e . errno , e . filename , e . strerror ) <EOL> item . datafile . name = rel_dst_path <EOL> item . save ( ) <EOL> return item <EOL> @ task ( ) <EOL> def read ( uuid ) : <EOL> '''<STR_LIT>''' <EOL> return FileStoreItem . objects . get_item ( uuid ) <EOL> @ task ( ) <EOL> def delete ( uuid ) : <EOL> '''<STR_LIT>''' <EOL> logger . debug ( "<STR_LIT>" , uuid ) <EOL> item = FileStoreItem . objects . get_item ( uuid ) <EOL> if item : <EOL> item . delete ( ) <EOL> logger . info ( "<STR_LIT>" ) <EOL> return True <EOL> else : <EOL> logger . error ( "<STR_LIT>" , uuid ) <EOL> return False <EOL> @ task ( ) <EOL> def update ( uuid , source ) : <EOL> """<STR_LIT>""" <EOL> FileStoreItem . objects . filter ( uuid = uuid ) . update ( source = source ) <EOL> return import_file ( uuid , refresh = True ) <EOL> @ task ( ) <EOL> def rename ( uuid , name ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> item = FileStoreItem . objects . get_item ( uuid = uuid ) <EOL> except FileStoreItem . DoesNotExist : <EOL> logger . error ( "<STR_LIT>" , uuid ) <EOL> return None <EOL> if item . rename_datafile ( name ) : <EOL> return item <EOL> else : <EOL> return None <EOL> @ task ( ) <EOL> def download_file ( url , target_path , file_size = <NUM_LIT:1> ) : <EOL> '''<STR_LIT>''' <EOL> logger . debug ( "<STR_LIT>" , url ) <EOL> try : <EOL> response = requests . get ( url , stream = True ) <EOL> except requests . exceptions . ConnectionError as e : <EOL> raise DownloadError ( <EOL> "<STR_LIT>" . format ( url , e . reason ) ) <EOL> except ValueError as e : <EOL> raise DownloadError ( "<STR_LIT>" . format ( url ) ) <EOL> remotefilesize = int ( <EOL> response . headers . get ( "<STR_LIT>" , file_size ) ) <EOL> with open ( target_path , '<STR_LIT>' ) as destination : <EOL> localfilesize = <NUM_LIT:0> <EOL> blocksize = <NUM_LIT:8> * <NUM_LIT:2> ** <NUM_LIT:10> <EOL> for buf in iter ( lambda : response . raw . read ( blocksize ) , '<STR_LIT>' ) : <EOL> localfilesize += len ( buf ) <EOL> destination . write ( buf ) <EOL> if remotefilesize > <NUM_LIT:0> : <EOL> percent_done = localfilesize * <NUM_LIT> / remotefilesize <EOL> else : <EOL> percent_done = <NUM_LIT:0> <EOL> import_file . update_state ( <EOL> state = "<STR_LIT>" , <EOL> meta = { "<STR_LIT>" : "<STR_LIT>" % ( percent_done ) , <EOL> '<STR_LIT>' : localfilesize , <EOL> '<STR_LIT>' : remotefilesize } <EOL> ) <EOL> destination . flush ( ) <EOL> response . close ( ) <EOL> logger . debug ( "<STR_LIT>" ) <EOL> class DownloadError ( StandardError ) : <EOL> '''<STR_LIT>''' <EOL> def __init__ ( self , value ) : <EOL> self . value = value <EOL> def __str__ ( self ) : <EOL> return repr ( self . value ) </s>
