<s> from __future__ import unicode_literals <EOL> import concurrent . futures <EOL> import contextlib <EOL> import json <EOL> import logging <EOL> import multiprocessing <EOL> import pkgutil <EOL> import re <EOL> try : <EOL> import urllib . request as urllib_request <EOL> except ImportError : <EOL> import urllib2 as urllib_request <EOL> import xml . parsers . expat <EOL> try : <EOL> import xmlrpc . client as xmlrpc_client <EOL> except ImportError : <EOL> import xmlrpclib as xmlrpc_client <EOL> try : <EOL> CPU_COUNT = max ( <NUM_LIT:2> , multiprocessing . cpu_count ( ) ) <EOL> except NotImplementedError : <EOL> CPU_COUNT = <NUM_LIT:2> <EOL> PROJECT_NAME = re . compile ( r'<STR_LIT>' ) <EOL> def just_name ( supposed_name ) : <EOL> """<STR_LIT>""" <EOL> return PROJECT_NAME . match ( supposed_name ) . group ( <NUM_LIT:0> ) . lower ( ) <EOL> @ contextlib . contextmanager <EOL> def pypi_client ( ) : <EOL> client = xmlrpc_client . ServerProxy ( '<STR_LIT>' ) <EOL> try : <EOL> yield client <EOL> finally : <EOL> try : <EOL> client ( '<STR_LIT>' ) ( ) <EOL> except ( xml . parsers . expat . ExpatError , xmlrpc_client . Fault ) : <EOL> pass <EOL> def overrides ( ) : <EOL> """<STR_LIT>""" <EOL> raw_bytes = pkgutil . get_data ( __name__ , '<STR_LIT>' ) <EOL> return json . loads ( raw_bytes . decode ( '<STR_LIT:utf-8>' ) ) <EOL> def py3_classifiers ( ) : <EOL> """<STR_LIT>""" <EOL> url = '<STR_LIT>' <EOL> response = urllib_request . urlopen ( url ) <EOL> try : <EOL> try : <EOL> status = response . status <EOL> except AttributeError : <EOL> status = response . code <EOL> if status != <NUM_LIT:200> : <EOL> msg = '<STR_LIT>' . format ( status , url ) <EOL> raise ValueError ( msg ) <EOL> data = response . read ( ) <EOL> finally : <EOL> response . close ( ) <EOL> classifiers = data . decode ( '<STR_LIT:utf-8>' ) . splitlines ( ) <EOL> base_classifier = '<STR_LIT>' <EOL> return ( classifier for classifier in classifiers <EOL> if classifier . startswith ( base_classifier ) ) <EOL> def projects_matching_classifier ( classifier ) : <EOL> """<STR_LIT>""" <EOL> log = logging . getLogger ( '<STR_LIT>' ) <EOL> with pypi_client ( ) as client : <EOL> log . info ( '<STR_LIT>' . format ( classifier ) ) <EOL> try : <EOL> return frozenset ( result [ <NUM_LIT:0> ] . lower ( ) <EOL> for result in client . browse ( [ classifier ] ) ) <EOL> except xml . parsers . expat . ExpatError : <EOL> logging . getLogger ( '<STR_LIT>' ) . info ( "<STR_LIT>" ) <EOL> return [ ] <EOL> def all_py3_projects ( manual_overrides = None ) : <EOL> """<STR_LIT>""" <EOL> log = logging . getLogger ( '<STR_LIT>' ) <EOL> projects = set ( ) <EOL> thread_pool_executor = concurrent . futures . ThreadPoolExecutor ( <EOL> max_workers = CPU_COUNT ) <EOL> with thread_pool_executor as executor : <EOL> for result in map ( projects_matching_classifier , py3_classifiers ( ) ) : <EOL> projects . update ( result ) <EOL> if manual_overrides is None : <EOL> manual_overrides = overrides ( ) <EOL> stale_overrides = projects . intersection ( manual_overrides ) <EOL> log . info ( '<STR_LIT>' . format ( len ( manual_overrides ) ) ) <EOL> for override in sorted ( manual_overrides ) : <EOL> msg = override <EOL> try : <EOL> msg += '<STR_LIT>' . format ( manual_overrides [ override ] ) <EOL> except TypeError : <EOL> pass <EOL> log . info ( '<STR_LIT:U+0020>' + msg ) <EOL> if stale_overrides : <EOL> log . warning ( '<STR_LIT>' . format ( stale_overrides ) ) <EOL> projects . update ( manual_overrides ) <EOL> return projects <EOL> def all_projects ( ) : <EOL> """<STR_LIT>""" <EOL> log = logging . getLogger ( '<STR_LIT>' ) <EOL> with pypi_client ( ) as client : <EOL> log . info ( '<STR_LIT>' ) <EOL> return frozenset ( name . lower ( ) for name in client . list_packages ( ) ) </s>
