<s> """<STR_LIT>""" <EOL> import os <EOL> import logging <EOL> import tarfile <EOL> import pickle <EOL> import shutil <EOL> import re <EOL> import codecs <EOL> import numpy as np <EOL> import scipy . sparse as sp <EOL> from . base import get_data_home <EOL> from . base import Bunch <EOL> from . base import load_files <EOL> from . base import _pkl_filepath <EOL> from . . utils import check_random_state <EOL> from . . feature_extraction . text import CountVectorizer <EOL> from . . preprocessing import normalize <EOL> from . . externals import joblib , six <EOL> if six . PY3 : <EOL> from urllib . request import urlopen <EOL> else : <EOL> from urllib2 import urlopen <EOL> logger = logging . getLogger ( __name__ ) <EOL> URL = ( "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> ARCHIVE_NAME = "<STR_LIT>" <EOL> CACHE_NAME = "<STR_LIT>" <EOL> TRAIN_FOLDER = "<STR_LIT>" <EOL> TEST_FOLDER = "<STR_LIT>" <EOL> def download_20newsgroups ( target_dir , cache_path ) : <EOL> """<STR_LIT>""" <EOL> archive_path = os . path . join ( target_dir , ARCHIVE_NAME ) <EOL> train_path = os . path . join ( target_dir , TRAIN_FOLDER ) <EOL> test_path = os . path . join ( target_dir , TEST_FOLDER ) <EOL> if not os . path . exists ( target_dir ) : <EOL> os . makedirs ( target_dir ) <EOL> if os . path . exists ( archive_path ) : <EOL> logger . warning ( "<STR_LIT>" ) <EOL> os . remove ( archive_path ) <EOL> logger . warning ( "<STR_LIT>" , URL ) <EOL> opener = urlopen ( URL ) <EOL> with open ( archive_path , '<STR_LIT:wb>' ) as f : <EOL> f . write ( opener . read ( ) ) <EOL> logger . info ( "<STR_LIT>" , archive_path ) <EOL> tarfile . open ( archive_path , "<STR_LIT>" ) . extractall ( path = target_dir ) <EOL> os . remove ( archive_path ) <EOL> cache = dict ( train = load_files ( train_path , encoding = '<STR_LIT>' ) , <EOL> test = load_files ( test_path , encoding = '<STR_LIT>' ) ) <EOL> compressed_content = codecs . encode ( pickle . dumps ( cache ) , '<STR_LIT>' ) <EOL> with open ( cache_path , '<STR_LIT:wb>' ) as f : <EOL> f . write ( compressed_content ) <EOL> shutil . rmtree ( target_dir ) <EOL> return cache <EOL> def strip_newsgroup_header ( text ) : <EOL> """<STR_LIT>""" <EOL> _before , _blankline , after = text . partition ( '<STR_LIT>' ) <EOL> return after <EOL> _QUOTE_RE = re . compile ( r'<STR_LIT>' <EOL> r'<STR_LIT>' ) <EOL> def strip_newsgroup_quoting ( text ) : <EOL> """<STR_LIT>""" <EOL> good_lines = [ line for line in text . split ( '<STR_LIT:\n>' ) <EOL> if not _QUOTE_RE . search ( line ) ] <EOL> return '<STR_LIT:\n>' . join ( good_lines ) <EOL> def strip_newsgroup_footer ( text ) : <EOL> """<STR_LIT>""" <EOL> lines = text . strip ( ) . split ( '<STR_LIT:\n>' ) <EOL> for line_num in range ( len ( lines ) - <NUM_LIT:1> , - <NUM_LIT:1> , - <NUM_LIT:1> ) : <EOL> line = lines [ line_num ] <EOL> if line . strip ( ) . strip ( '<STR_LIT:->' ) == '<STR_LIT>' : <EOL> break <EOL> if line_num > <NUM_LIT:0> : <EOL> return '<STR_LIT:\n>' . join ( lines [ : line_num ] ) <EOL> else : <EOL> return text <EOL> def fetch_20newsgroups ( data_home = None , subset = '<STR_LIT:train>' , categories = None , <EOL> shuffle = True , random_state = <NUM_LIT> , <EOL> remove = ( ) , <EOL> download_if_missing = True ) : <EOL> """<STR_LIT>""" <EOL> data_home = get_data_home ( data_home = data_home ) <EOL> cache_path = _pkl_filepath ( data_home , CACHE_NAME ) <EOL> twenty_home = os . path . join ( data_home , "<STR_LIT>" ) <EOL> cache = None <EOL> if os . path . exists ( cache_path ) : <EOL> try : <EOL> with open ( cache_path , '<STR_LIT:rb>' ) as f : <EOL> compressed_content = f . read ( ) <EOL> uncompressed_content = codecs . decode ( <EOL> compressed_content , '<STR_LIT>' ) <EOL> cache = pickle . loads ( uncompressed_content ) <EOL> except Exception as e : <EOL> print ( <NUM_LIT> * '<STR_LIT:_>' ) <EOL> print ( '<STR_LIT>' ) <EOL> print ( <NUM_LIT> * '<STR_LIT:_>' ) <EOL> print ( e ) <EOL> if cache is None : <EOL> if download_if_missing : <EOL> print ( '<STR_LIT>' ) <EOL> cache = download_20newsgroups ( target_dir = twenty_home , <EOL> cache_path = cache_path ) <EOL> else : <EOL> raise IOError ( '<STR_LIT>' ) <EOL> if subset in ( '<STR_LIT:train>' , '<STR_LIT:test>' ) : <EOL> data = cache [ subset ] <EOL> elif subset == '<STR_LIT:all>' : <EOL> data_lst = list ( ) <EOL> target = list ( ) <EOL> filenames = list ( ) <EOL> for subset in ( '<STR_LIT:train>' , '<STR_LIT:test>' ) : <EOL> data = cache [ subset ] <EOL> data_lst . extend ( data . data ) <EOL> target . extend ( data . target ) <EOL> filenames . extend ( data . filenames ) <EOL> data . data = data_lst <EOL> data . target = np . array ( target ) <EOL> data . filenames = np . array ( filenames ) <EOL> else : <EOL> raise ValueError ( <EOL> "<STR_LIT>" % subset ) <EOL> data . description = '<STR_LIT>' <EOL> if '<STR_LIT>' in remove : <EOL> data . data = [ strip_newsgroup_header ( text ) for text in data . data ] <EOL> if '<STR_LIT>' in remove : <EOL> data . data = [ strip_newsgroup_footer ( text ) for text in data . data ] <EOL> if '<STR_LIT>' in remove : <EOL> data . data = [ strip_newsgroup_quoting ( text ) for text in data . data ] <EOL> if categories is not None : <EOL> labels = [ ( data . target_names . index ( cat ) , cat ) for cat in categories ] <EOL> labels . sort ( ) <EOL> labels , categories = zip ( * labels ) <EOL> mask = np . in1d ( data . target , labels ) <EOL> data . filenames = data . filenames [ mask ] <EOL> data . target = data . target [ mask ] <EOL> data . target = np . searchsorted ( labels , data . target ) <EOL> data . target_names = list ( categories ) <EOL> data_lst = np . array ( data . data , dtype = object ) <EOL> data_lst = data_lst [ mask ] <EOL> data . data = data_lst . tolist ( ) <EOL> if shuffle : <EOL> random_state = check_random_state ( random_state ) <EOL> indices = np . arange ( data . target . shape [ <NUM_LIT:0> ] ) <EOL> random_state . shuffle ( indices ) <EOL> data . filenames = data . filenames [ indices ] <EOL> data . target = data . target [ indices ] <EOL> data_lst = np . array ( data . data , dtype = object ) <EOL> data_lst = data_lst [ indices ] <EOL> data . data = data_lst . tolist ( ) <EOL> return data <EOL> def fetch_20newsgroups_vectorized ( subset = "<STR_LIT:train>" , remove = ( ) , data_home = None ) : <EOL> """<STR_LIT>""" <EOL> data_home = get_data_home ( data_home = data_home ) <EOL> filebase = '<STR_LIT>' <EOL> if remove : <EOL> filebase += '<STR_LIT>' + ( '<STR_LIT:->' . join ( remove ) ) <EOL> target_file = _pkl_filepath ( data_home , filebase + "<STR_LIT>" ) <EOL> data_train = fetch_20newsgroups ( data_home = data_home , <EOL> subset = '<STR_LIT:train>' , <EOL> categories = None , <EOL> shuffle = True , <EOL> random_state = <NUM_LIT:12> , <EOL> remove = remove ) <EOL> data_test = fetch_20newsgroups ( data_home = data_home , <EOL> subset = '<STR_LIT:test>' , <EOL> categories = None , <EOL> shuffle = True , <EOL> random_state = <NUM_LIT:12> , <EOL> remove = remove ) <EOL> if os . path . exists ( target_file ) : <EOL> X_train , X_test = joblib . load ( target_file ) <EOL> else : <EOL> vectorizer = CountVectorizer ( dtype = np . int16 ) <EOL> X_train = vectorizer . fit_transform ( data_train . data ) . tocsr ( ) <EOL> X_test = vectorizer . transform ( data_test . data ) . tocsr ( ) <EOL> joblib . dump ( ( X_train , X_test ) , target_file , compress = <NUM_LIT:9> ) <EOL> X_train = X_train . astype ( np . float64 ) <EOL> X_test = X_test . astype ( np . float64 ) <EOL> normalize ( X_train , copy = False ) <EOL> normalize ( X_test , copy = False ) <EOL> target_names = data_train . target_names <EOL> if subset == "<STR_LIT:train>" : <EOL> data = X_train <EOL> target = data_train . target <EOL> elif subset == "<STR_LIT:test>" : <EOL> data = X_test <EOL> target = data_test . target <EOL> elif subset == "<STR_LIT:all>" : <EOL> data = sp . vstack ( ( X_train , X_test ) ) . tocsr ( ) <EOL> target = np . concatenate ( ( data_train . target , data_test . target ) ) <EOL> else : <EOL> raise ValueError ( "<STR_LIT>" <EOL> "<STR_LIT>" % subset ) <EOL> return Bunch ( data = data , target = target , target_names = target_names ) </s>
