<s> """<STR_LIT>""" <EOL> import contextlib <EOL> import os <EOL> import shutil <EOL> import numpy as np <EOL> import zipfile <EOL> import tarfile <EOL> import gzip <EOL> from tempfile import mkdtemp , mkstemp <EOL> from nose import with_setup <EOL> from nose . tools import assert_true , assert_false , assert_equal <EOL> from nilearn import datasets <EOL> from nilearn . _utils . testing import ( mock_request , wrap_chunk_read_ , <EOL> FetchFilesMock , assert_raises_regex ) <EOL> currdir = os . path . dirname ( os . path . abspath ( __file__ ) ) <EOL> datadir = os . path . join ( currdir , '<STR_LIT:data>' ) <EOL> tmpdir = None <EOL> url_request = None <EOL> file_mock = None <EOL> def setup_tmpdata ( ) : <EOL> global tmpdir <EOL> tmpdir = mkdtemp ( ) <EOL> def setup_mock ( utils_mod = datasets . utils , dataset_mod = datasets . utils ) : <EOL> global original_url_request <EOL> global mock_url_request <EOL> mock_url_request = mock_request ( ) <EOL> original_url_request = utils_mod . _urllib . request <EOL> utils_mod . _urllib . request = mock_url_request <EOL> global original_chunk_read <EOL> global mock_chunk_read <EOL> mock_chunk_read = wrap_chunk_read_ ( utils_mod . _chunk_read_ ) <EOL> original_chunk_read = utils_mod . _chunk_read_ <EOL> utils_mod . _chunk_read_ = mock_chunk_read <EOL> global original_fetch_files <EOL> global mock_fetch_files <EOL> mock_fetch_files = FetchFilesMock ( ) <EOL> original_fetch_files = dataset_mod . _fetch_files <EOL> dataset_mod . _fetch_files = mock_fetch_files <EOL> def teardown_mock ( utils_mod = datasets . utils , dataset_mod = datasets . utils ) : <EOL> global original_url_request <EOL> utils_mod . _urllib . request = original_url_request <EOL> global original_chunk_read <EOL> utils_mod . chunk_read_ = original_chunk_read <EOL> global original_fetch_files <EOL> dataset_mod . _fetch_files = original_fetch_files <EOL> def teardown_tmpdata ( ) : <EOL> global tmpdir <EOL> if tmpdir is not None : <EOL> shutil . rmtree ( tmpdir ) <EOL> @ with_setup ( setup_tmpdata , teardown_tmpdata ) <EOL> def test_get_dataset_dir ( ) : <EOL> os . environ . pop ( '<STR_LIT>' , None ) <EOL> os . environ . pop ( '<STR_LIT>' , None ) <EOL> expected_base_dir = os . path . expanduser ( '<STR_LIT>' ) <EOL> data_dir = datasets . utils . _get_dataset_dir ( '<STR_LIT:test>' , verbose = <NUM_LIT:0> ) <EOL> assert_equal ( data_dir , os . path . join ( expected_base_dir , '<STR_LIT:test>' ) ) <EOL> assert os . path . exists ( data_dir ) <EOL> shutil . rmtree ( data_dir ) <EOL> expected_base_dir = os . path . join ( tmpdir , '<STR_LIT>' ) <EOL> os . environ [ '<STR_LIT>' ] = expected_base_dir <EOL> data_dir = datasets . utils . _get_dataset_dir ( '<STR_LIT:test>' , verbose = <NUM_LIT:0> ) <EOL> assert_equal ( data_dir , os . path . join ( expected_base_dir , '<STR_LIT:test>' ) ) <EOL> assert os . path . exists ( data_dir ) <EOL> shutil . rmtree ( data_dir ) <EOL> expected_base_dir = os . path . join ( tmpdir , '<STR_LIT>' ) <EOL> os . environ [ '<STR_LIT>' ] = expected_base_dir <EOL> data_dir = datasets . utils . _get_dataset_dir ( '<STR_LIT:test>' , verbose = <NUM_LIT:0> ) <EOL> assert_equal ( data_dir , os . path . join ( expected_base_dir , '<STR_LIT:test>' ) ) <EOL> assert os . path . exists ( data_dir ) <EOL> shutil . rmtree ( data_dir ) <EOL> expected_base_dir = os . path . join ( tmpdir , '<STR_LIT>' ) <EOL> expected_dataset_dir = os . path . join ( expected_base_dir , '<STR_LIT:test>' ) <EOL> data_dir = datasets . utils . _get_dataset_dir ( <EOL> '<STR_LIT:test>' , default_paths = [ expected_dataset_dir ] , verbose = <NUM_LIT:0> ) <EOL> assert_equal ( data_dir , os . path . join ( expected_base_dir , '<STR_LIT:test>' ) ) <EOL> assert os . path . exists ( data_dir ) <EOL> shutil . rmtree ( data_dir ) <EOL> no_write = os . path . join ( tmpdir , '<STR_LIT>' ) <EOL> os . makedirs ( no_write ) <EOL> os . chmod ( no_write , <NUM_LIT> ) <EOL> expected_base_dir = os . path . join ( tmpdir , '<STR_LIT>' ) <EOL> os . environ [ '<STR_LIT>' ] = expected_base_dir <EOL> data_dir = datasets . utils . _get_dataset_dir ( '<STR_LIT:test>' , <EOL> default_paths = [ no_write ] , <EOL> verbose = <NUM_LIT:0> ) <EOL> assert_equal ( data_dir , no_write ) <EOL> assert os . path . exists ( data_dir ) <EOL> os . chmod ( no_write , <NUM_LIT> ) <EOL> shutil . rmtree ( data_dir ) <EOL> test_file = os . path . join ( tmpdir , '<STR_LIT>' ) <EOL> with open ( test_file , '<STR_LIT:w>' ) as out : <EOL> out . write ( '<STR_LIT>' ) <EOL> assert_raises_regex ( OSError , <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' , <EOL> datasets . utils . _get_dataset_dir , <EOL> '<STR_LIT:test>' , test_file , verbose = <NUM_LIT:0> ) <EOL> def test_md5_sum_file ( ) : <EOL> out , f = mkstemp ( ) <EOL> os . write ( out , b'<STR_LIT>' ) <EOL> os . close ( out ) <EOL> assert_equal ( datasets . utils . _md5_sum_file ( f ) , <EOL> '<STR_LIT>' ) <EOL> os . remove ( f ) <EOL> def test_read_md5_sum_file ( ) : <EOL> out , f = mkstemp ( ) <EOL> os . write ( out , b'<STR_LIT>' <EOL> b'<STR_LIT>' ) <EOL> os . close ( out ) <EOL> h = datasets . utils . _read_md5_sum_file ( f ) <EOL> assert_true ( '<STR_LIT>' in h ) <EOL> assert_false ( '<STR_LIT>' in h ) <EOL> assert_equal ( h [ '<STR_LIT>' ] , '<STR_LIT>' ) <EOL> assert_equal ( h [ '<STR_LIT>' ] , '<STR_LIT>' ) <EOL> os . remove ( f ) <EOL> def test_tree ( ) : <EOL> parent = mkdtemp ( ) <EOL> open ( os . path . join ( parent , '<STR_LIT>' ) , '<STR_LIT:w>' ) . close ( ) <EOL> open ( os . path . join ( parent , '<STR_LIT>' ) , '<STR_LIT:w>' ) . close ( ) <EOL> dir1 = os . path . join ( parent , '<STR_LIT>' ) <EOL> dir11 = os . path . join ( dir1 , '<STR_LIT>' ) <EOL> dir12 = os . path . join ( dir1 , '<STR_LIT>' ) <EOL> dir2 = os . path . join ( parent , '<STR_LIT>' ) <EOL> os . mkdir ( dir1 ) <EOL> os . mkdir ( dir11 ) <EOL> os . mkdir ( dir12 ) <EOL> os . mkdir ( dir2 ) <EOL> open ( os . path . join ( dir1 , '<STR_LIT>' ) , '<STR_LIT:w>' ) . close ( ) <EOL> open ( os . path . join ( dir1 , '<STR_LIT>' ) , '<STR_LIT:w>' ) . close ( ) <EOL> open ( os . path . join ( dir11 , '<STR_LIT>' ) , '<STR_LIT:w>' ) . close ( ) <EOL> open ( os . path . join ( dir2 , '<STR_LIT>' ) , '<STR_LIT:w>' ) . close ( ) <EOL> tree_ = datasets . utils . _tree ( parent ) <EOL> assert_equal ( tree_ [ <NUM_LIT:0> ] [ <NUM_LIT:1> ] [ <NUM_LIT:0> ] [ <NUM_LIT:1> ] [ <NUM_LIT:0> ] , os . path . join ( dir11 , '<STR_LIT>' ) ) <EOL> assert_equal ( len ( tree_ [ <NUM_LIT:0> ] [ <NUM_LIT:1> ] [ <NUM_LIT:1> ] [ <NUM_LIT:1> ] ) , <NUM_LIT:0> ) <EOL> assert_equal ( tree_ [ <NUM_LIT:0> ] [ <NUM_LIT:1> ] [ <NUM_LIT:2> ] , os . path . join ( dir1 , '<STR_LIT>' ) ) <EOL> assert_equal ( tree_ [ <NUM_LIT:0> ] [ <NUM_LIT:1> ] [ <NUM_LIT:3> ] , os . path . join ( dir1 , '<STR_LIT>' ) ) <EOL> assert_equal ( tree_ [ <NUM_LIT:1> ] [ <NUM_LIT:1> ] [ <NUM_LIT:0> ] , os . path . join ( dir2 , '<STR_LIT>' ) ) <EOL> assert_equal ( tree_ [ <NUM_LIT:2> ] , os . path . join ( parent , '<STR_LIT>' ) ) <EOL> assert_equal ( tree_ [ <NUM_LIT:3> ] , os . path . join ( parent , '<STR_LIT>' ) ) <EOL> shutil . rmtree ( parent ) <EOL> def test_movetree ( ) : <EOL> parent = mkdtemp ( ) <EOL> dir1 = os . path . join ( parent , '<STR_LIT>' ) <EOL> dir11 = os . path . join ( dir1 , '<STR_LIT>' ) <EOL> dir12 = os . path . join ( dir1 , '<STR_LIT>' ) <EOL> dir2 = os . path . join ( parent , '<STR_LIT>' ) <EOL> os . mkdir ( dir1 ) <EOL> os . mkdir ( dir11 ) <EOL> os . mkdir ( dir12 ) <EOL> os . mkdir ( dir2 ) <EOL> os . mkdir ( os . path . join ( dir2 , '<STR_LIT>' ) ) <EOL> open ( os . path . join ( dir1 , '<STR_LIT>' ) , '<STR_LIT:w>' ) . close ( ) <EOL> open ( os . path . join ( dir1 , '<STR_LIT>' ) , '<STR_LIT:w>' ) . close ( ) <EOL> open ( os . path . join ( dir11 , '<STR_LIT>' ) , '<STR_LIT:w>' ) . close ( ) <EOL> open ( os . path . join ( dir12 , '<STR_LIT>' ) , '<STR_LIT:w>' ) . close ( ) <EOL> open ( os . path . join ( dir2 , '<STR_LIT>' ) , '<STR_LIT:w>' ) . close ( ) <EOL> datasets . utils . movetree ( dir1 , dir2 ) <EOL> assert_false ( os . path . exists ( dir11 ) ) <EOL> assert_false ( os . path . exists ( dir12 ) ) <EOL> assert_false ( os . path . exists ( os . path . join ( dir1 , '<STR_LIT>' ) ) ) <EOL> assert_false ( os . path . exists ( os . path . join ( dir1 , '<STR_LIT>' ) ) ) <EOL> assert_false ( os . path . exists ( os . path . join ( dir11 , '<STR_LIT>' ) ) ) <EOL> assert_false ( os . path . exists ( os . path . join ( dir12 , '<STR_LIT>' ) ) ) <EOL> dir11 = os . path . join ( dir2 , '<STR_LIT>' ) <EOL> dir12 = os . path . join ( dir2 , '<STR_LIT>' ) <EOL> assert_true ( os . path . exists ( dir11 ) ) <EOL> assert_true ( os . path . exists ( dir12 ) ) <EOL> assert_true ( os . path . exists ( os . path . join ( dir2 , '<STR_LIT>' ) ) ) <EOL> assert_true ( os . path . exists ( os . path . join ( dir2 , '<STR_LIT>' ) ) ) <EOL> assert_true ( os . path . exists ( os . path . join ( dir11 , '<STR_LIT>' ) ) ) <EOL> assert_true ( os . path . exists ( os . path . join ( dir12 , '<STR_LIT>' ) ) ) <EOL> def test_filter_columns ( ) : <EOL> value1 = np . arange ( <NUM_LIT> ) <EOL> strings = np . asarray ( [ '<STR_LIT:a>' , '<STR_LIT:b>' , '<STR_LIT:c>' ] ) <EOL> value2 = strings [ value1 % <NUM_LIT:3> ] <EOL> values = np . asarray ( list ( zip ( value1 , value2 ) ) , <EOL> dtype = [ ( '<STR_LIT>' , int ) , ( '<STR_LIT>' , '<STR_LIT>' ) ] ) <EOL> f = datasets . utils . _filter_columns ( values , { '<STR_LIT>' : ( <NUM_LIT> , <NUM_LIT> ) } ) <EOL> assert_equal ( np . sum ( f ) , <NUM_LIT> ) <EOL> f = datasets . utils . _filter_columns ( values , { '<STR_LIT>' : [ <NUM_LIT:0> , <NUM_LIT:9> , ( <NUM_LIT:12> , <NUM_LIT> ) ] } ) <EOL> assert_equal ( np . sum ( f ) , <NUM_LIT:15> ) <EOL> value1 = value1 % <NUM_LIT:2> <EOL> values = np . asarray ( list ( zip ( value1 , value2 ) ) , <EOL> dtype = [ ( '<STR_LIT>' , int ) , ( '<STR_LIT>' , b'<STR_LIT>' ) ] ) <EOL> f = datasets . utils . _filter_columns ( values , [ ] ) <EOL> assert_equal ( np . sum ( f ) , <NUM_LIT> ) <EOL> f = datasets . utils . _filter_columns ( values , { '<STR_LIT>' : b'<STR_LIT:b>' } ) <EOL> assert_equal ( np . sum ( f ) , <NUM_LIT> ) <EOL> f = datasets . utils . _filter_columns ( values , { '<STR_LIT>' : <NUM_LIT:1> , '<STR_LIT>' : b'<STR_LIT:b>' } ) <EOL> assert_equal ( np . sum ( f ) , <NUM_LIT> ) <EOL> f = datasets . utils . _filter_columns ( values , { '<STR_LIT>' : <NUM_LIT:1> , '<STR_LIT>' : b'<STR_LIT:b>' } , <EOL> combination = '<STR_LIT>' ) <EOL> assert_equal ( np . sum ( f ) , <NUM_LIT> ) <EOL> def test_uncompress ( ) : <EOL> fd , temp = mkstemp ( ) <EOL> os . close ( fd ) <EOL> dtemp = mkdtemp ( ) <EOL> ztemp = os . path . join ( dtemp , '<STR_LIT>' ) <EOL> with contextlib . closing ( zipfile . ZipFile ( ztemp , '<STR_LIT:w>' ) ) as testzip : <EOL> testzip . write ( temp ) <EOL> datasets . utils . _uncompress_file ( ztemp , verbose = <NUM_LIT:0> ) <EOL> assert ( os . path . exists ( os . path . join ( dtemp , temp ) ) ) <EOL> shutil . rmtree ( dtemp ) <EOL> dtemp = mkdtemp ( ) <EOL> ztemp = os . path . join ( dtemp , '<STR_LIT>' ) <EOL> with contextlib . closing ( tarfile . open ( ztemp , '<STR_LIT:w>' ) ) as tar : <EOL> tar . add ( temp ) <EOL> datasets . utils . _uncompress_file ( ztemp , verbose = <NUM_LIT:0> ) <EOL> assert ( os . path . exists ( os . path . join ( dtemp , temp ) ) ) <EOL> shutil . rmtree ( dtemp ) <EOL> dtemp = mkdtemp ( ) <EOL> ztemp = os . path . join ( dtemp , '<STR_LIT>' ) <EOL> f = gzip . open ( ztemp , '<STR_LIT:wb>' ) <EOL> f . close ( ) <EOL> datasets . utils . _uncompress_file ( ztemp , verbose = <NUM_LIT:0> ) <EOL> assert ( os . path . exists ( os . path . join ( dtemp , temp ) ) ) <EOL> shutil . rmtree ( dtemp ) <EOL> os . remove ( temp ) <EOL> @ with_setup ( setup_mock , teardown_mock ) <EOL> @ with_setup ( setup_tmpdata , teardown_tmpdata ) <EOL> def test_fetch_file_overwrite ( ) : <EOL> fil = datasets . utils . _fetch_file ( url = '<STR_LIT>' , data_dir = tmpdir , <EOL> verbose = <NUM_LIT:0> , overwrite = True ) <EOL> assert_equal ( len ( mock_url_request . urls ) , <NUM_LIT:1> ) <EOL> assert_true ( os . path . exists ( fil ) ) <EOL> with open ( fil , '<STR_LIT:r>' ) as fp : <EOL> assert_equal ( fp . read ( ) , '<STR_LIT>' ) <EOL> with open ( fil , '<STR_LIT:w>' ) as fp : <EOL> fp . write ( '<STR_LIT>' ) <EOL> fil = datasets . utils . _fetch_file ( url = '<STR_LIT>' , data_dir = tmpdir , <EOL> verbose = <NUM_LIT:0> , overwrite = False ) <EOL> assert_equal ( len ( mock_url_request . urls ) , <NUM_LIT:1> ) <EOL> assert_true ( os . path . exists ( fil ) ) <EOL> with open ( fil , '<STR_LIT:r>' ) as fp : <EOL> assert_equal ( fp . read ( ) , '<STR_LIT>' ) <EOL> fil = datasets . utils . _fetch_file ( url = '<STR_LIT>' , data_dir = tmpdir , <EOL> verbose = <NUM_LIT:0> , overwrite = True ) <EOL> assert_equal ( len ( mock_url_request . urls ) , <NUM_LIT:1> ) <EOL> assert_true ( os . path . exists ( fil ) ) <EOL> with open ( fil , '<STR_LIT:r>' ) as fp : <EOL> assert_equal ( fp . read ( ) , '<STR_LIT>' ) <EOL> @ with_setup ( setup_mock , teardown_mock ) <EOL> @ with_setup ( setup_tmpdata , teardown_tmpdata ) <EOL> def test_fetch_files_overwrite ( ) : <EOL> files = ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> fil = datasets . utils . _fetch_files ( data_dir = tmpdir , verbose = <NUM_LIT:0> , <EOL> files = [ files + ( dict ( overwrite = True ) , ) ] ) <EOL> assert_equal ( len ( mock_url_request . urls ) , <NUM_LIT:1> ) <EOL> assert_true ( os . path . exists ( fil [ <NUM_LIT:0> ] ) ) <EOL> with open ( fil [ <NUM_LIT:0> ] , '<STR_LIT:r>' ) as fp : <EOL> assert_equal ( fp . read ( ) , '<STR_LIT>' ) <EOL> with open ( fil [ <NUM_LIT:0> ] , '<STR_LIT:w>' ) as fp : <EOL> fp . write ( '<STR_LIT>' ) <EOL> fil = datasets . utils . _fetch_files ( data_dir = tmpdir , verbose = <NUM_LIT:0> , <EOL> files = [ files + ( dict ( overwrite = False ) , ) ] ) <EOL> assert_equal ( len ( mock_url_request . urls ) , <NUM_LIT:1> ) <EOL> assert_true ( os . path . exists ( fil [ <NUM_LIT:0> ] ) ) <EOL> with open ( fil [ <NUM_LIT:0> ] , '<STR_LIT:r>' ) as fp : <EOL> assert_equal ( fp . read ( ) , '<STR_LIT>' ) <EOL> fil = datasets . utils . _fetch_files ( data_dir = tmpdir , verbose = <NUM_LIT:0> , <EOL> files = [ files + ( dict ( overwrite = True ) , ) ] ) <EOL> assert_equal ( len ( mock_url_request . urls ) , <NUM_LIT:1> ) <EOL> assert_true ( os . path . exists ( fil [ <NUM_LIT:0> ] ) ) <EOL> with open ( fil [ <NUM_LIT:0> ] , '<STR_LIT:r>' ) as fp : <EOL> assert_equal ( fp . read ( ) , '<STR_LIT>' ) </s>
