<s> import os <EOL> import Queue <EOL> import threading <EOL> import sys <EOL> import urllib3 <EOL> from . settings import PROGRESS_FORMATTER , NUM_THREADS <EOL> pool_manager = urllib3 . PoolManager ( ) <EOL> photo_queue = Queue . Queue ( ) <EOL> lock = threading . Lock ( ) <EOL> downloaded = <NUM_LIT:0> <EOL> def parallel_download ( photos , user , size , outdir ) : <EOL> threads = [ ] <EOL> for photo in photos : <EOL> photo_queue . put ( photo ) <EOL> thread_pool = [ <EOL> threading . Thread ( <EOL> target = worker , <EOL> args = ( photo_queue , user , size , outdir , len ( photos ) ) <EOL> ) for i in range ( NUM_THREADS ) <EOL> ] <EOL> for t in thread_pool : <EOL> threads . append ( t ) <EOL> t . start ( ) <EOL> for t in threads : <EOL> t . join ( ) <EOL> def worker ( queue , user , size , outdir , total ) : <EOL> while True : <EOL> try : <EOL> photo = queue . get ( False ) <EOL> except Queue . Empty : <EOL> break <EOL> media_url = photo [ <NUM_LIT:1> ] <EOL> urllib3_download ( media_url , size , outdir ) <EOL> with lock : <EOL> global downloaded <EOL> downloaded += <NUM_LIT:1> <EOL> d = { <EOL> '<STR_LIT>' : os . path . basename ( media_url ) , <EOL> '<STR_LIT:user>' : user , <EOL> '<STR_LIT:index>' : downloaded + <NUM_LIT:1> if downloaded < total else total , <EOL> '<STR_LIT>' : total , <EOL> } <EOL> progress = PROGRESS_FORMATTER % d <EOL> sys . stdout . write ( '<STR_LIT>' % progress ) <EOL> sys . stdout . flush ( ) <EOL> def urllib3_download ( media_url , size , outdir ) : <EOL> r = pool_manager . request ( '<STR_LIT:GET>' , media_url + '<STR_LIT::>' + size ) <EOL> bs = os . path . basename ( media_url ) <EOL> filename = os . path . join ( outdir or '<STR_LIT>' , bs ) <EOL> with open ( filename , '<STR_LIT:wb>' ) as fd : <EOL> fd . write ( r . data ) <EOL> return filename </s>
