<s> from __future__ import absolute_import , unicode_literals <EOL> import os <EOL> import tempfile <EOL> import unittest <EOL> import numpy as np <EOL> from scipy import sparse as sp <EOL> from spacy import attrs <EOL> from textacy import data , fileio <EOL> from textacy . compat import PY2 , str <EOL> class FileIOTestCase ( unittest . TestCase ) : <EOL> def setUp ( self ) : <EOL> text = "<STR_LIT>" <EOL> self . spacy_pipeline = data . load_spacy_pipeline ( lang = '<STR_LIT>' ) <EOL> self . spacy_doc = self . spacy_pipeline ( text ) <EOL> cols = [ attrs . TAG , attrs . HEAD , attrs . DEP ] <EOL> values = np . array ( <EOL> [ [ <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:0> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:2> , <NUM_LIT> ] , <EOL> [ <NUM_LIT> , - <NUM_LIT:3> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:5> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:2> , <NUM_LIT> ] , <EOL> [ <NUM_LIT> , - <NUM_LIT:3> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:0> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:2> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:1> , <NUM_LIT> ] , <EOL> [ <NUM_LIT> , - <NUM_LIT:3> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ] , <EOL> [ <NUM_LIT> , - <NUM_LIT:3> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:9> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:0> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:1> , <NUM_LIT> ] , <EOL> [ <NUM_LIT> , <NUM_LIT:2> , <NUM_LIT> ] , [ <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:4> , <NUM_LIT> ] , [ <NUM_LIT> , - <NUM_LIT:5> , <NUM_LIT> ] ] , dtype = '<STR_LIT>' ) <EOL> self . spacy_doc . from_array ( cols , values ) <EOL> self . tempdir = tempfile . mkdtemp ( <EOL> prefix = '<STR_LIT>' , dir = os . path . dirname ( os . path . abspath ( __file__ ) ) ) <EOL> self . tests_dir = os . path . split ( __file__ ) [ <NUM_LIT:0> ] <EOL> def test_write_conll ( self ) : <EOL> expected = "<STR_LIT>" <EOL> filename = os . path . join ( self . tempdir , '<STR_LIT>' ) <EOL> fileio . write_conll ( self . spacy_doc , filename ) <EOL> observed = fileio . read_file ( filename ) <EOL> self . assertEqual ( observed , expected ) <EOL> def test_read_write_spacy_doc ( self ) : <EOL> expected = [ tok . lemma_ for tok in self . spacy_doc ] <EOL> filename = os . path . join ( self . tempdir , '<STR_LIT>' ) <EOL> fileio . write_spacy_docs ( self . spacy_doc , filename ) <EOL> observed = [ tok . lemma_ for doc in fileio . read_spacy_docs ( self . spacy_pipeline . vocab , filename ) <EOL> for tok in doc ] <EOL> self . assertEqual ( observed , expected ) <EOL> def test_read_write_file_lines ( self ) : <EOL> expected = [ sent . text for sent in self . spacy_doc . sents ] <EOL> filename = os . path . join ( self . tempdir , '<STR_LIT>' ) <EOL> fileio . write_file_lines ( expected , filename ) <EOL> observed = [ line . strip ( ) for line in fileio . read_file_lines ( filename ) ] <EOL> self . assertEqual ( observed , expected ) <EOL> def test_read_write_file_lines_gzip ( self ) : <EOL> expected = [ sent . text for sent in self . spacy_doc . sents ] <EOL> filename = os . path . join ( self . tempdir , '<STR_LIT>' ) <EOL> fileio . write_file_lines ( expected , filename ) <EOL> observed = [ line . strip ( ) for line in fileio . read_file_lines ( filename ) ] <EOL> self . assertEqual ( observed , expected ) <EOL> def test_read_write_file_lines_bz2 ( self ) : <EOL> expected = [ sent . text for sent in self . spacy_doc . sents ] <EOL> filename = os . path . join ( self . tempdir , '<STR_LIT>' ) <EOL> fileio . write_file_lines ( expected , filename , <EOL> mode = '<STR_LIT:wb>' if PY2 else '<STR_LIT>' ) <EOL> observed = [ line . strip ( ) for line <EOL> in fileio . read_file_lines ( filename , mode = '<STR_LIT:r>' if PY2 else '<STR_LIT>' ) ] <EOL> self . assertEqual ( observed , expected ) <EOL> def test_read_write_json ( self ) : <EOL> expected = [ { '<STR_LIT>' : i , '<STR_LIT>' : sent . text } <EOL> for i , sent in enumerate ( self . spacy_doc . sents ) ] <EOL> filename = os . path . join ( self . tempdir , '<STR_LIT>' ) <EOL> fileio . write_json ( expected , filename ) <EOL> observed = list ( fileio . read_json ( filename , prefix = '<STR_LIT>' ) ) [ <NUM_LIT:0> ] <EOL> self . assertEqual ( observed , expected ) <EOL> def test_read_write_json_prefix ( self ) : <EOL> to_write = [ { '<STR_LIT>' : i , '<STR_LIT>' : sent . text } <EOL> for i , sent in enumerate ( self . spacy_doc . sents ) ] <EOL> expected = [ item [ '<STR_LIT>' ] for item in to_write ] <EOL> filename = os . path . join ( self . tempdir , '<STR_LIT>' ) <EOL> fileio . write_json ( to_write , filename ) <EOL> observed = list ( fileio . read_json ( filename , prefix = '<STR_LIT>' ) ) <EOL> self . assertEqual ( observed , expected ) <EOL> def test_read_write_json_lines ( self ) : <EOL> expected = [ { '<STR_LIT>' : i , '<STR_LIT>' : sent . text } <EOL> for i , sent in enumerate ( self . spacy_doc . sents ) ] <EOL> filename = os . path . join ( self . tempdir , '<STR_LIT>' ) <EOL> fileio . write_json_lines ( expected , filename ) <EOL> observed = list ( fileio . read_json_lines ( filename ) ) <EOL> self . assertEqual ( observed , expected ) <EOL> def test_read_write_sparse_csr_matrix ( self ) : <EOL> expected = sp . csr_matrix ( <EOL> ( np . array ( [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT:4> , <NUM_LIT:5> , <NUM_LIT:6> ] ) , <EOL> ( np . array ( [ <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:2> , <NUM_LIT:2> ] ) , np . array ( [ <NUM_LIT:0> , <NUM_LIT:2> , <NUM_LIT:2> , <NUM_LIT:0> , <NUM_LIT:1> , <NUM_LIT:2> ] ) ) ) , <EOL> shape = ( <NUM_LIT:3> , <NUM_LIT:3> ) ) <EOL> filename = os . path . join ( self . tempdir , '<STR_LIT>' ) <EOL> fileio . write_sparse_matrix ( expected , filename , compressed = False ) <EOL> observed = fileio . read_sparse_csr_matrix ( filename ) <EOL> self . assertEqual ( abs ( observed - expected ) . nnz , <NUM_LIT:0> ) <EOL> def test_read_write_sparse_csr_matrix_compressed ( self ) : <EOL> expected = sp . csr_matrix ( <EOL> ( np . array ( [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT:4> , <NUM_LIT:5> , <NUM_LIT:6> ] ) , <EOL> ( np . array ( [ <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:2> , <NUM_LIT:2> ] ) , np . array ( [ <NUM_LIT:0> , <NUM_LIT:2> , <NUM_LIT:2> , <NUM_LIT:0> , <NUM_LIT:1> , <NUM_LIT:2> ] ) ) ) , <EOL> shape = ( <NUM_LIT:3> , <NUM_LIT:3> ) ) <EOL> filename = os . path . join ( self . tempdir , '<STR_LIT>' ) <EOL> fileio . write_sparse_matrix ( expected , filename , compressed = True ) <EOL> observed = fileio . read_sparse_csr_matrix ( filename ) <EOL> self . assertEqual ( abs ( observed - expected ) . nnz , <NUM_LIT:0> ) <EOL> def test_read_write_sparse_csc_matrix ( self ) : <EOL> expected = sp . csc_matrix ( <EOL> ( np . array ( [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT:4> , <NUM_LIT:5> , <NUM_LIT:6> ] ) , <EOL> ( np . array ( [ <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:2> , <NUM_LIT:2> ] ) , np . array ( [ <NUM_LIT:0> , <NUM_LIT:2> , <NUM_LIT:2> , <NUM_LIT:0> , <NUM_LIT:1> , <NUM_LIT:2> ] ) ) ) , <EOL> shape = ( <NUM_LIT:3> , <NUM_LIT:3> ) ) <EOL> filename = os . path . join ( self . tempdir , '<STR_LIT>' ) <EOL> fileio . write_sparse_matrix ( expected , filename , compressed = False ) <EOL> observed = fileio . read_sparse_csc_matrix ( filename ) <EOL> self . assertEqual ( abs ( observed - expected ) . nnz , <NUM_LIT:0> ) <EOL> def test_read_write_sparse_csc_matrix_compressed ( self ) : <EOL> expected = sp . csc_matrix ( <EOL> ( np . array ( [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT:4> , <NUM_LIT:5> , <NUM_LIT:6> ] ) , <EOL> ( np . array ( [ <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:2> , <NUM_LIT:2> ] ) , np . array ( [ <NUM_LIT:0> , <NUM_LIT:2> , <NUM_LIT:2> , <NUM_LIT:0> , <NUM_LIT:1> , <NUM_LIT:2> ] ) ) ) , <EOL> shape = ( <NUM_LIT:3> , <NUM_LIT:3> ) ) <EOL> filename = os . path . join ( self . tempdir , '<STR_LIT>' ) <EOL> fileio . write_sparse_matrix ( expected , filename , compressed = True ) <EOL> observed = fileio . read_sparse_csc_matrix ( filename ) <EOL> self . assertEqual ( abs ( observed - expected ) . nnz , <NUM_LIT:0> ) <EOL> def test_get_filenames ( self ) : <EOL> expected = sorted ( os . path . join ( self . tests_dir , fname ) <EOL> for fname in os . listdir ( self . tests_dir ) <EOL> if os . path . isfile ( os . path . join ( self . tests_dir , fname ) ) ) <EOL> observed = sorted ( fileio . get_filenames ( self . tests_dir , <EOL> ignore_invisible = False , <EOL> recursive = False ) ) <EOL> self . assertEqual ( observed , expected ) <EOL> def test_get_filenames_ignore_invisible ( self ) : <EOL> self . assertTrue ( <EOL> len ( list ( fileio . get_filenames ( self . tests_dir , ignore_invisible = True ) ) ) < len ( list ( fileio . get_filenames ( self . tests_dir , ignore_invisible = False ) ) ) ) <EOL> def test_get_filenames_ignore_substr ( self ) : <EOL> self . assertTrue ( <EOL> len ( list ( fileio . get_filenames ( self . tests_dir , <EOL> ignore_substr = '<STR_LIT>' , <EOL> ignore_invisible = True ) ) ) == <NUM_LIT:0> ) <EOL> def test_get_filenames_match_substr ( self ) : <EOL> self . assertTrue ( <EOL> len ( list ( fileio . get_filenames ( self . tests_dir , <EOL> match_substr = '<STR_LIT>' , <EOL> extension = '<STR_LIT>' ) ) ) == <NUM_LIT:1> ) <EOL> def tearDown ( self ) : <EOL> for fname in os . listdir ( self . tempdir ) : <EOL> os . remove ( os . path . join ( self . tempdir , fname ) ) <EOL> os . rmdir ( self . tempdir ) </s>
