<s> import sys <EOL> import md5 <EOL> import zlib <EOL> from os . path import getsize , split , join , abspath , isdir , normpath <EOL> from os import listdir <EOL> from sha import sha <EOL> from copy import copy <EOL> from string import strip <EOL> from threading import Event <EOL> from time import time <EOL> from traceback import print_exc <EOL> try : <EOL> from sys import getfilesystemencoding <EOL> ENCODING = getfilesystemencoding ( ) <EOL> except : <EOL> from sys import getdefaultencoding <EOL> ENCODING = getdefaultencoding ( ) <EOL> from BitTornado . bencode import bencode <EOL> from BitTornado . BT1 . btformats import check_info <EOL> defaults = [ <EOL> ( '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , <NUM_LIT:0> , <EOL> "<STR_LIT>" ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' , <EOL> "<STR_LIT>" ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' , <EOL> "<STR_LIT>" + <EOL> "<STR_LIT>" ) , <EOL> ( '<STR_LIT:target>' , '<STR_LIT>' , <EOL> "<STR_LIT>" ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' , <EOL> "<STR_LIT>" ) <EOL> ] <EOL> default_piece_len_exp = <NUM_LIT> <EOL> ignore = [ '<STR_LIT>' , '<STR_LIT>' ] <EOL> def print_announcelist_details ( ) : <EOL> print ( '<STR_LIT>' ) <EOL> print ( '<STR_LIT>' ) <EOL> print ( '<STR_LIT>' ) <EOL> print ( '<STR_LIT>' ) <EOL> print ( "<STR_LIT>" ) <EOL> print ( '<STR_LIT>' ) <EOL> print ( '<STR_LIT>' ) <EOL> print ( '<STR_LIT>' ) <EOL> print ( '<STR_LIT>' ) <EOL> print ( '<STR_LIT>' ) <EOL> print ( '<STR_LIT>' ) <EOL> print ( '<STR_LIT>' ) <EOL> print ( '<STR_LIT>' ) <EOL> print ( '<STR_LIT>' ) <EOL> print ( '<STR_LIT>' ) <EOL> print ( '<STR_LIT>' ) <EOL> print ( '<STR_LIT>' ) <EOL> def make_meta_file ( file , url , params = None , flag = Event ( ) , <EOL> progress = lambda x : None , progress_percent = <NUM_LIT:1> , gethash = None ) : <EOL> if params is None : <EOL> params = { } <EOL> if '<STR_LIT>' in params : <EOL> piece_len_exp = params [ '<STR_LIT>' ] <EOL> else : <EOL> piece_len_exp = default_piece_len_exp <EOL> if '<STR_LIT:target>' in params and params [ '<STR_LIT:target>' ] : <EOL> f = params [ '<STR_LIT:target>' ] <EOL> else : <EOL> a , b = split ( file ) <EOL> if b == '<STR_LIT>' : <EOL> f = a + '<STR_LIT>' <EOL> else : <EOL> f = join ( a , b + '<STR_LIT>' ) <EOL> if piece_len_exp == <NUM_LIT:0> : <EOL> size = calcsize ( file ) <EOL> if size > <NUM_LIT:8> L * <NUM_LIT> * <NUM_LIT> * <NUM_LIT> : <EOL> piece_len_exp = <NUM_LIT> <EOL> elif size > <NUM_LIT:2> * <NUM_LIT> * <NUM_LIT> * <NUM_LIT> : <EOL> piece_len_exp = <NUM_LIT:20> <EOL> elif size > <NUM_LIT> * <NUM_LIT> * <NUM_LIT> : <EOL> piece_len_exp = <NUM_LIT> <EOL> elif size > <NUM_LIT:64> * <NUM_LIT> * <NUM_LIT> : <EOL> piece_len_exp = <NUM_LIT> <EOL> elif size > <NUM_LIT:16> * <NUM_LIT> * <NUM_LIT> : <EOL> piece_len_exp = <NUM_LIT> <EOL> elif size > <NUM_LIT:4> * <NUM_LIT> * <NUM_LIT> : <EOL> piece_len_exp = <NUM_LIT:16> <EOL> else : <EOL> piece_len_exp = <NUM_LIT:15> <EOL> piece_length = <NUM_LIT:2> ** piece_len_exp <EOL> encoding = None <EOL> if '<STR_LIT>' in params : <EOL> encoding = params [ '<STR_LIT>' ] <EOL> if not encoding : <EOL> encoding = ENCODING <EOL> if not encoding : <EOL> encoding = '<STR_LIT:ascii>' <EOL> info = makeinfo ( file , piece_length , encoding , flag , progress , progress_percent , gethash = gethash ) <EOL> if flag . isSet ( ) : <EOL> return <EOL> check_info ( info ) <EOL> h = open ( f , '<STR_LIT:wb>' ) <EOL> data = { '<STR_LIT:info>' : info , '<STR_LIT>' : strip ( url ) , '<STR_LIT>' : long ( time ( ) ) } <EOL> if '<STR_LIT>' in params and params [ '<STR_LIT>' ] : <EOL> data [ '<STR_LIT>' ] = params [ '<STR_LIT>' ] <EOL> if '<STR_LIT>' in params and params [ '<STR_LIT>' ] : <EOL> data [ '<STR_LIT>' ] = params [ '<STR_LIT>' ] <EOL> if '<STR_LIT>' in params : <EOL> data [ '<STR_LIT>' ] = params [ '<STR_LIT>' ] <EOL> elif '<STR_LIT>' in params and params [ '<STR_LIT>' ] : <EOL> l = [ ] <EOL> for tier in params [ '<STR_LIT>' ] . split ( '<STR_LIT:|>' ) : <EOL> l . append ( tier . split ( '<STR_LIT:U+002C>' ) ) <EOL> data [ '<STR_LIT>' ] = l <EOL> if '<STR_LIT>' in params : <EOL> data [ '<STR_LIT>' ] = params [ '<STR_LIT>' ] <EOL> elif '<STR_LIT>' in params and params [ '<STR_LIT>' ] : <EOL> data [ '<STR_LIT>' ] = params [ '<STR_LIT>' ] . split ( '<STR_LIT:|>' ) <EOL> if '<STR_LIT>' in params and params [ '<STR_LIT>' ] : <EOL> data [ '<STR_LIT>' ] = params [ '<STR_LIT>' ] <EOL> h . write ( bencode ( data ) ) <EOL> h . close ( ) <EOL> def calcsize ( file ) : <EOL> if not isdir ( file ) : <EOL> return getsize ( file ) <EOL> total = <NUM_LIT:0> L <EOL> for s in subfiles ( abspath ( file ) ) : <EOL> total += getsize ( s [ <NUM_LIT:1> ] ) <EOL> return total <EOL> def uniconvertl ( l , e ) : <EOL> r = [ ] <EOL> try : <EOL> for s in l : <EOL> r . append ( uniconvert ( s , e ) ) <EOL> except UnicodeError : <EOL> raise UnicodeError ( '<STR_LIT>' + join ( l ) ) <EOL> return r <EOL> def uniconvert ( s , e ) : <EOL> try : <EOL> s = unicode ( s , e ) <EOL> except UnicodeError : <EOL> raise UnicodeError ( '<STR_LIT>' + s ) <EOL> return s . encode ( '<STR_LIT:utf-8>' ) <EOL> def makeinfo ( file , piece_length , encoding , flag , progress , progress_percent = <NUM_LIT:1> , gethash = None ) : <EOL> if gethash is None : <EOL> gethash = { } <EOL> if not '<STR_LIT>' in gethash : <EOL> gethash [ '<STR_LIT>' ] = False <EOL> if not '<STR_LIT>' in gethash : <EOL> gethash [ '<STR_LIT>' ] = False <EOL> if not '<STR_LIT>' in gethash : <EOL> gethash [ '<STR_LIT>' ] = False <EOL> file = abspath ( file ) <EOL> if isdir ( file ) : <EOL> subs = subfiles ( file ) <EOL> subs . sort ( ) <EOL> pieces = [ ] <EOL> sh = sha ( ) <EOL> done = <NUM_LIT:0> L <EOL> fs = [ ] <EOL> totalsize = <NUM_LIT:0.0> <EOL> totalhashed = <NUM_LIT:0> L <EOL> for p , f in subs : <EOL> totalsize += getsize ( f ) <EOL> for p , f in subs : <EOL> pos = <NUM_LIT:0> L <EOL> size = getsize ( f ) <EOL> h = open ( f , '<STR_LIT:rb>' ) <EOL> if gethash [ '<STR_LIT>' ] : <EOL> hash_md5 = md5 . new ( ) <EOL> if gethash [ '<STR_LIT>' ] : <EOL> hash_sha1 = sha ( ) <EOL> if gethash [ '<STR_LIT>' ] : <EOL> hash_crc32 = zlib . crc32 ( '<STR_LIT>' ) <EOL> while pos < size : <EOL> a = min ( size - pos , piece_length - done ) <EOL> readpiece = h . read ( a ) <EOL> if flag . isSet ( ) : <EOL> return <EOL> sh . update ( readpiece ) <EOL> if flag . isSet ( ) : <EOL> return <EOL> if gethash [ '<STR_LIT>' ] : <EOL> hash_md5 . update ( readpiece ) <EOL> if flag . isSet ( ) : <EOL> return <EOL> if gethash [ '<STR_LIT>' ] : <EOL> hash_crc32 = zlib . crc32 ( readpiece , hash_crc32 ) <EOL> if flag . isSet ( ) : <EOL> return <EOL> if gethash [ '<STR_LIT>' ] : <EOL> hash_sha1 . update ( readpiece ) <EOL> if flag . isSet ( ) : <EOL> return <EOL> done += a <EOL> pos += a <EOL> totalhashed += a <EOL> if done == piece_length : <EOL> pieces . append ( sh . digest ( ) ) <EOL> done = <NUM_LIT:0> <EOL> sh = sha ( ) <EOL> if progress_percent : <EOL> progress ( totalhashed / totalsize ) <EOL> else : <EOL> progress ( a ) <EOL> newdict = { '<STR_LIT>' : size , <EOL> '<STR_LIT:path>' : uniconvertl ( p , encoding ) } <EOL> if gethash [ '<STR_LIT>' ] : <EOL> newdict [ '<STR_LIT>' ] = hash_md5 . hexdigest ( ) <EOL> if gethash [ '<STR_LIT>' ] : <EOL> newdict [ '<STR_LIT>' ] = "<STR_LIT>" % hash_crc32 <EOL> if gethash [ '<STR_LIT>' ] : <EOL> newdict [ '<STR_LIT>' ] = hash_sha1 . digest ( ) <EOL> fs . append ( newdict ) <EOL> h . close ( ) <EOL> if done > <NUM_LIT:0> : <EOL> pieces . append ( sh . digest ( ) ) <EOL> return { '<STR_LIT>' : '<STR_LIT>' . join ( pieces ) , <EOL> '<STR_LIT>' : piece_length , <EOL> '<STR_LIT>' : fs , <EOL> '<STR_LIT:name>' : uniconvert ( split ( file ) [ <NUM_LIT:1> ] , encoding ) } <EOL> else : <EOL> size = getsize ( file ) <EOL> pieces = [ ] <EOL> p = <NUM_LIT:0> L <EOL> h = open ( file , '<STR_LIT:rb>' ) <EOL> if gethash [ '<STR_LIT>' ] : <EOL> hash_md5 = md5 . new ( ) <EOL> if gethash [ '<STR_LIT>' ] : <EOL> hash_crc32 = zlib . crc32 ( '<STR_LIT>' ) <EOL> if gethash [ '<STR_LIT>' ] : <EOL> hash_sha1 = sha ( ) <EOL> while p < size : <EOL> x = h . read ( min ( piece_length , size - p ) ) <EOL> if flag . isSet ( ) : <EOL> return <EOL> if gethash [ '<STR_LIT>' ] : <EOL> hash_md5 . update ( x ) <EOL> if flag . isSet ( ) : <EOL> return <EOL> if gethash [ '<STR_LIT>' ] : <EOL> hash_crc32 = zlib . crc32 ( x , hash_crc32 ) <EOL> if flag . isSet ( ) : <EOL> return <EOL> if gethash [ '<STR_LIT>' ] : <EOL> hash_sha1 . update ( x ) <EOL> if flag . isSet ( ) : <EOL> return <EOL> pieces . append ( sha ( x ) . digest ( ) ) <EOL> if flag . isSet ( ) : <EOL> return <EOL> p += piece_length <EOL> if p > size : <EOL> p = size <EOL> if progress_percent : <EOL> progress ( float ( p ) / size ) <EOL> else : <EOL> progress ( min ( piece_length , size - p ) ) <EOL> h . close ( ) <EOL> newdict = { '<STR_LIT>' : '<STR_LIT>' . join ( pieces ) , <EOL> '<STR_LIT>' : piece_length , <EOL> '<STR_LIT>' : size , <EOL> '<STR_LIT:name>' : uniconvert ( split ( file ) [ <NUM_LIT:1> ] , encoding ) } <EOL> if gethash [ '<STR_LIT>' ] : <EOL> newdict [ '<STR_LIT>' ] = hash_md5 . hexdigest ( ) <EOL> if gethash [ '<STR_LIT>' ] : <EOL> newdict [ '<STR_LIT>' ] = "<STR_LIT>" % hash_crc32 <EOL> if gethash [ '<STR_LIT>' ] : <EOL> newdict [ '<STR_LIT>' ] = hash_sha1 . digest ( ) <EOL> return newdict <EOL> def subfiles ( d ) : <EOL> r = [ ] <EOL> stack = [ ( [ ] , d ) ] <EOL> while stack : <EOL> p , n = stack . pop ( ) <EOL> if isdir ( n ) : <EOL> for s in listdir ( n ) : <EOL> if s not in ignore and s [ : <NUM_LIT:1> ] != '<STR_LIT:.>' : <EOL> stack . append ( ( copy ( p ) + [ s ] , join ( n , s ) ) ) <EOL> else : <EOL> r . append ( ( p , n ) ) <EOL> return r <EOL> def completedir ( dir , url , params = None , flag = Event ( ) , <EOL> vc = lambda x : None , fc = lambda x : None , getmd5 = False ) : <EOL> if params is None : <EOL> params = { } <EOL> files = listdir ( dir ) <EOL> files . sort ( ) <EOL> ext = '<STR_LIT>' <EOL> if '<STR_LIT:target>' in params : <EOL> target = params [ '<STR_LIT:target>' ] <EOL> else : <EOL> target = '<STR_LIT>' <EOL> togen = [ ] <EOL> for f in files : <EOL> if f [ - len ( ext ) : ] != ext and ( f + ext ) not in files : <EOL> togen . append ( join ( dir , f ) ) <EOL> total = <NUM_LIT:0> <EOL> for i in togen : <EOL> total += calcsize ( i ) <EOL> subtotal = [ <NUM_LIT:0> ] <EOL> def callback ( x , subtotal = subtotal , total = total , vc = vc ) : <EOL> subtotal [ <NUM_LIT:0> ] += x <EOL> vc ( float ( subtotal [ <NUM_LIT:0> ] ) / total ) <EOL> for i in togen : <EOL> fc ( i ) <EOL> try : <EOL> t = split ( i ) [ - <NUM_LIT:1> ] <EOL> if t not in ignore and t [ <NUM_LIT:0> ] != '<STR_LIT:.>' : <EOL> if target != '<STR_LIT>' : <EOL> params [ '<STR_LIT:target>' ] = join ( target , t + ext ) <EOL> make_meta_file ( i , url , params , flag , progress = callback , progress_percent = <NUM_LIT:0> , getmd5 = getmd5 ) <EOL> except ValueError : <EOL> print_exc ( ) </s>
