<s> import mimetypes <EOL> import logging <EOL> import os <EOL> import requests <EOL> from celery . task import task <EOL> from django . conf import settings <EOL> from django . core . files . uploadedfile import TemporaryUploadedFile <EOL> from django . utils . timezone import now <EOL> from localshop . apps . packages import models , forms <EOL> from localshop . apps . packages . utils import md5_hash_file <EOL> from localshop . utils import no_duplicates , enqueue <EOL> @ task ( bind = True ) <EOL> @ no_duplicates <EOL> def fetch_package ( self , repository_pk , slug ) : <EOL> """<STR_LIT:U+0020>""" <EOL> repository = models . Repository . objects . get ( pk = repository_pk ) <EOL> logging . info ( '<STR_LIT>' , slug ) <EOL> response = requests . get ( <EOL> '<STR_LIT>' % ( repository . upstream_pypi_url_api , slug ) ) <EOL> if response . status_code == <NUM_LIT> : <EOL> return <EOL> if response . status_code != <NUM_LIT:200> : <EOL> return <EOL> package_data = response . json ( ) <EOL> name = package_data [ '<STR_LIT:info>' ] [ '<STR_LIT:name>' ] <EOL> try : <EOL> package = repository . packages . get ( name = name ) <EOL> releases = package . get_all_releases ( ) <EOL> except models . Package . DoesNotExist : <EOL> package = repository . packages . create ( name = name ) <EOL> releases = { } <EOL> for version , release_list in package_data [ '<STR_LIT>' ] . items ( ) : <EOL> release , files = releases . get ( version , ( None , { } ) ) <EOL> if not release : <EOL> release = package . releases . create ( version = version ) <EOL> release_data = { <EOL> '<STR_LIT>' : package_data [ '<STR_LIT:info>' ] [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : package_data [ '<STR_LIT:info>' ] [ '<STR_LIT>' ] , <EOL> '<STR_LIT:description>' : package_data [ '<STR_LIT:info>' ] [ '<STR_LIT:description>' ] , <EOL> '<STR_LIT>' : package_data [ '<STR_LIT:info>' ] [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : package_data [ '<STR_LIT:info>' ] [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : package_data [ '<STR_LIT:info>' ] [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : package_data [ '<STR_LIT:info>' ] [ '<STR_LIT>' ] , <EOL> '<STR_LIT:version>' : version , <EOL> } <EOL> release_form = forms . PypiReleaseDataForm ( release_data , instance = release ) <EOL> if release_form . is_valid ( ) : <EOL> release_form . save ( ) <EOL> for data in release_list : <EOL> release_file = files . get ( data [ '<STR_LIT:filename>' ] ) <EOL> if not release_file : <EOL> release_file = models . ReleaseFile ( <EOL> release = release , filename = data [ '<STR_LIT:filename>' ] ) <EOL> release_file . python_version = data [ '<STR_LIT>' ] <EOL> release_file . filetype = data [ '<STR_LIT>' ] <EOL> release_file . url = data [ '<STR_LIT:url>' ] <EOL> release_file . size = data [ '<STR_LIT:size>' ] <EOL> release_file . md5_digest = data [ '<STR_LIT>' ] <EOL> release_file . save ( ) <EOL> package . update_timestamp = now ( ) <EOL> package . save ( ) <EOL> logging . info ( '<STR_LIT>' , slug ) <EOL> @ task <EOL> def download_file ( pk ) : <EOL> """<STR_LIT>""" <EOL> release_file = models . ReleaseFile . objects . get ( pk = pk ) <EOL> logging . info ( "<STR_LIT>" , release_file . url ) <EOL> proxies = None <EOL> if settings . LOCALSHOP_HTTP_PROXY : <EOL> proxies = settings . LOCALSHOP_HTTP_PROXY <EOL> response = requests . get ( release_file . url , stream = True , proxies = proxies ) <EOL> filename = os . path . basename ( release_file . url ) <EOL> if '<STR_LIT>' in response . headers : <EOL> size = int ( response . headers [ '<STR_LIT>' ] ) <EOL> else : <EOL> size = len ( response . content ) <EOL> default_content_type = '<STR_LIT>' <EOL> content_type = response . headers . get ( '<STR_LIT>' ) <EOL> if content_type is None or content_type == default_content_type : <EOL> content_type = mimetypes . guess_type ( filename ) [ <NUM_LIT:0> ] or default_content_type <EOL> with TemporaryUploadedFile ( name = filename , size = size , charset = '<STR_LIT:utf-8>' , <EOL> content_type = content_type ) as temp_file : <EOL> temp_file . write ( response . content ) <EOL> temp_file . seek ( <NUM_LIT:0> ) <EOL> md5_hash = md5_hash_file ( temp_file ) <EOL> if md5_hash != release_file . md5_digest : <EOL> logging . error ( "<STR_LIT>" % ( <EOL> md5_hash , release_file . md5_digest ) ) <EOL> return <EOL> release_file . distribution . save ( filename , temp_file ) <EOL> release_file . save ( ) <EOL> logging . info ( "<STR_LIT>" ) <EOL> @ task <EOL> def update_packages ( ) : <EOL> """<STR_LIT>""" <EOL> logging . info ( '<STR_LIT>' ) <EOL> for package in models . Package . objects . filter ( is_local = False ) : <EOL> logging . info ( '<STR_LIT>' , package . name ) <EOL> enqueue ( fetch_package , package . name ) <EOL> logging . info ( '<STR_LIT>' ) </s>
