<s> import numpy as np <EOL> import pytest <EOL> from pandas . util . testing import assert_frame_equal <EOL> import pandas as pd <EOL> from ibis . compat import unittest <EOL> from ibis . common import IbisTypeError <EOL> from ibis . impala . pandas_interop import pandas_to_ibis_schema , DataFrameWriter <EOL> from ibis . impala . tests . common import ImpalaE2E <EOL> import ibis . expr . datatypes as dt <EOL> import ibis . expr . types as ir <EOL> import ibis . util as util <EOL> import ibis <EOL> class TestPandasTypeInterop ( unittest . TestCase ) : <EOL> def test_series_to_ibis_literal ( self ) : <EOL> values = [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT:4> ] <EOL> s = pd . Series ( values ) <EOL> expr = ir . as_value_expr ( s ) <EOL> expected = ir . sequence ( list ( s ) ) <EOL> assert expr . equals ( expected ) <EOL> class TestPandasSchemaInference ( unittest . TestCase ) : <EOL> def test_dtype_bool ( self ) : <EOL> df = pd . DataFrame ( { '<STR_LIT>' : [ True , False , False ] } ) <EOL> inferred = pandas_to_ibis_schema ( df ) <EOL> expected = ibis . schema ( [ ( '<STR_LIT>' , '<STR_LIT>' ) ] ) <EOL> assert inferred == expected <EOL> def test_dtype_int8 ( self ) : <EOL> df = pd . DataFrame ( { '<STR_LIT>' : np . int8 ( [ - <NUM_LIT:3> , <NUM_LIT:9> , <NUM_LIT> ] ) } ) <EOL> inferred = pandas_to_ibis_schema ( df ) <EOL> expected = ibis . schema ( [ ( '<STR_LIT>' , '<STR_LIT>' ) ] ) <EOL> assert inferred == expected <EOL> def test_dtype_int16 ( self ) : <EOL> df = pd . DataFrame ( { '<STR_LIT>' : np . int16 ( [ - <NUM_LIT:5> , <NUM_LIT:0> , <NUM_LIT:12> ] ) } ) <EOL> inferred = pandas_to_ibis_schema ( df ) <EOL> expected = ibis . schema ( [ ( '<STR_LIT>' , '<STR_LIT>' ) ] ) <EOL> assert inferred == expected <EOL> def test_dtype_int32 ( self ) : <EOL> df = pd . DataFrame ( { '<STR_LIT>' : np . int32 ( [ - <NUM_LIT:12> , <NUM_LIT:3> , <NUM_LIT> ] ) } ) <EOL> inferred = pandas_to_ibis_schema ( df ) <EOL> expected = ibis . schema ( [ ( '<STR_LIT>' , '<STR_LIT>' ) ] ) <EOL> assert inferred == expected <EOL> def test_dtype_int64 ( self ) : <EOL> df = pd . DataFrame ( { '<STR_LIT>' : np . int64 ( [ <NUM_LIT> , <NUM_LIT> , - <NUM_LIT:0> ] ) } ) <EOL> inferred = pandas_to_ibis_schema ( df ) <EOL> expected = ibis . schema ( [ ( '<STR_LIT>' , '<STR_LIT>' ) ] ) <EOL> assert inferred == expected <EOL> def test_dtype_float32 ( self ) : <EOL> df = pd . DataFrame ( { '<STR_LIT>' : np . float32 ( [ <NUM_LIT> , - <NUM_LIT> , <NUM_LIT> ] ) } ) <EOL> inferred = pandas_to_ibis_schema ( df ) <EOL> expected = ibis . schema ( [ ( '<STR_LIT>' , '<STR_LIT:float>' ) ] ) <EOL> assert inferred == expected <EOL> def test_dtype_float64 ( self ) : <EOL> df = pd . DataFrame ( { '<STR_LIT>' : np . float64 ( [ - <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ] ) } ) <EOL> inferred = pandas_to_ibis_schema ( df ) <EOL> expected = ibis . schema ( [ ( '<STR_LIT>' , '<STR_LIT>' ) ] ) <EOL> assert inferred == expected <EOL> def test_dtype_uint8 ( self ) : <EOL> df = pd . DataFrame ( { '<STR_LIT>' : np . uint8 ( [ <NUM_LIT:3> , <NUM_LIT:0> , <NUM_LIT:16> ] ) } ) <EOL> inferred = pandas_to_ibis_schema ( df ) <EOL> expected = ibis . schema ( [ ( '<STR_LIT>' , '<STR_LIT>' ) ] ) <EOL> assert inferred == expected <EOL> def test_dtype_uint16 ( self ) : <EOL> df = pd . DataFrame ( { '<STR_LIT>' : np . uint16 ( [ <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ] ) } ) <EOL> inferred = pandas_to_ibis_schema ( df ) <EOL> expected = ibis . schema ( [ ( '<STR_LIT>' , '<STR_LIT>' ) ] ) <EOL> assert inferred == expected <EOL> def test_dtype_uint32 ( self ) : <EOL> df = pd . DataFrame ( { '<STR_LIT>' : np . uint32 ( [ <NUM_LIT:100> , <NUM_LIT:0> , <NUM_LIT:6> ] ) } ) <EOL> inferred = pandas_to_ibis_schema ( df ) <EOL> expected = ibis . schema ( [ ( '<STR_LIT>' , '<STR_LIT>' ) ] ) <EOL> assert inferred == expected <EOL> def test_dtype_uint64 ( self ) : <EOL> df = pd . DataFrame ( { '<STR_LIT>' : np . uint64 ( [ <NUM_LIT> , <NUM_LIT:2> , <NUM_LIT:3> ] ) } ) <EOL> with self . assertRaises ( IbisTypeError ) : <EOL> inferred = pandas_to_ibis_schema ( df ) <EOL> def test_dtype_datetime64 ( self ) : <EOL> df = pd . DataFrame ( { <EOL> '<STR_LIT>' : [ pd . Timestamp ( '<STR_LIT>' ) , <EOL> pd . Timestamp ( '<STR_LIT>' ) , <EOL> pd . Timestamp ( '<STR_LIT>' ) ] } ) <EOL> inferred = pandas_to_ibis_schema ( df ) <EOL> expected = ibis . schema ( [ ( '<STR_LIT>' , '<STR_LIT>' ) ] ) <EOL> assert inferred == expected <EOL> def test_dtype_timedelta64 ( self ) : <EOL> df = pd . DataFrame ( { <EOL> '<STR_LIT>' : [ pd . Timedelta ( '<STR_LIT>' ) , <EOL> pd . Timedelta ( '<STR_LIT>' ) , <EOL> pd . Timedelta ( '<STR_LIT>' ) ] } ) <EOL> inferred = pandas_to_ibis_schema ( df ) <EOL> expected = ibis . schema ( [ ( '<STR_LIT>' , '<STR_LIT>' ) ] ) <EOL> assert inferred == expected <EOL> def test_dtype_string ( self ) : <EOL> df = pd . DataFrame ( { '<STR_LIT>' : [ '<STR_LIT:foo>' , '<STR_LIT:bar>' , '<STR_LIT:hello>' ] } ) <EOL> inferred = pandas_to_ibis_schema ( df ) <EOL> expected = ibis . schema ( [ ( '<STR_LIT>' , '<STR_LIT:string>' ) ] ) <EOL> assert inferred == expected <EOL> def test_dtype_categorical ( self ) : <EOL> df = pd . DataFrame ( { '<STR_LIT>' : [ '<STR_LIT:a>' , '<STR_LIT:b>' , '<STR_LIT:c>' , '<STR_LIT:a>' ] } , dtype = '<STR_LIT>' ) <EOL> inferred = pandas_to_ibis_schema ( df ) <EOL> expected = ibis . schema ( [ ( '<STR_LIT>' , dt . Category ( <NUM_LIT:3> ) ) ] ) <EOL> assert inferred == expected <EOL> exhaustive_df = pd . DataFrame ( { <EOL> '<STR_LIT>' : np . array ( [ <NUM_LIT:0> , <NUM_LIT:10> , <NUM_LIT:20> , <NUM_LIT:30> , <NUM_LIT> , <NUM_LIT:50> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ] , <EOL> dtype = '<STR_LIT>' ) , <EOL> '<STR_LIT>' : np . array ( [ True , False , True , False , True , None , <EOL> True , False , True , False ] , dtype = np . bool_ ) , <EOL> '<STR_LIT>' : np . array ( [ True , False , np . nan , False , True , np . nan , <EOL> True , np . nan , True , False ] , dtype = np . object_ ) , <EOL> '<STR_LIT>' : [ '<STR_LIT>' , None , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' ] , <EOL> '<STR_LIT>' : np . array ( [ <NUM_LIT:0.0> , <NUM_LIT> , np . nan , <NUM_LIT> , <EOL> <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <EOL> <NUM_LIT> , <NUM_LIT> , <EOL> <NUM_LIT> ] , dtype = np . float64 ) , <EOL> '<STR_LIT>' : np . array ( [ np . nan , <NUM_LIT> , <NUM_LIT> , <EOL> <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <EOL> <NUM_LIT> , <NUM_LIT> , <EOL> <NUM_LIT> , <EOL> <NUM_LIT> ] , dtype = '<STR_LIT>' ) , <EOL> '<STR_LIT>' : np . array ( [ <NUM_LIT:0> , <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT:4> , <NUM_LIT:5> , <NUM_LIT:6> , <NUM_LIT:7> , <NUM_LIT:8> , <NUM_LIT:9> ] , dtype = '<STR_LIT>' ) , <EOL> '<STR_LIT>' : [ <NUM_LIT:11> , <NUM_LIT:11> , <NUM_LIT:11> , <NUM_LIT:11> , <NUM_LIT:2> , <NUM_LIT:11> , <NUM_LIT:11> , <NUM_LIT:11> , <NUM_LIT:11> , <NUM_LIT:11> ] , <EOL> '<STR_LIT>' : np . array ( [ <NUM_LIT:0> , <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT:4> , <NUM_LIT:5> , <NUM_LIT:6> , <NUM_LIT:7> , <NUM_LIT:8> , <NUM_LIT:9> ] , dtype = '<STR_LIT>' ) , <EOL> '<STR_LIT>' : [ '<STR_LIT:0>' , '<STR_LIT:1>' , None , '<STR_LIT>' , '<STR_LIT:4>' , '<STR_LIT:5>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] , <EOL> '<STR_LIT>' : [ pd . Timestamp ( '<STR_LIT>' ) , <EOL> None , <EOL> pd . Timestamp ( '<STR_LIT>' ) , <EOL> pd . Timestamp ( '<STR_LIT>' ) , <EOL> pd . Timestamp ( '<STR_LIT>' ) , <EOL> pd . Timestamp ( '<STR_LIT>' ) , <EOL> pd . Timestamp ( '<STR_LIT>' ) , <EOL> pd . Timestamp ( '<STR_LIT>' ) , <EOL> pd . Timestamp ( '<STR_LIT>' ) , <EOL> pd . Timestamp ( '<STR_LIT>' ) ] , <EOL> '<STR_LIT>' : np . array ( [ <NUM_LIT:0> , <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT:4> , <NUM_LIT:5> , <NUM_LIT:6> , <NUM_LIT:7> , <NUM_LIT:8> , <NUM_LIT:9> ] , dtype = '<STR_LIT>' ) , <EOL> '<STR_LIT>' : [ <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ] } ) <EOL> class TestPandasInterop ( ImpalaE2E , unittest . TestCase ) : <EOL> @ classmethod <EOL> def setUpClass ( cls ) : <EOL> super ( TestPandasInterop , cls ) . setUpClass ( ) <EOL> cls . alltypes = cls . alltypes . execute ( ) <EOL> def test_alltypes_roundtrip ( self ) : <EOL> pytest . skip ( '<STR_LIT>' ) <EOL> self . _check_roundtrip ( self . alltypes ) <EOL> def test_writer_cleanup_deletes_hdfs_dir ( self ) : <EOL> writer = DataFrameWriter ( self . con , self . alltypes ) <EOL> path = writer . write_temp_csv ( ) <EOL> assert self . con . hdfs . exists ( path ) <EOL> writer . cleanup ( ) <EOL> assert not self . con . hdfs . exists ( path ) <EOL> writer . cleanup ( ) <EOL> assert not self . con . hdfs . exists ( path ) <EOL> def test_create_table_from_dataframe ( self ) : <EOL> pytest . skip ( '<STR_LIT>' ) <EOL> tname = '<STR_LIT>' . format ( util . guid ( ) ) <EOL> self . con . create_table ( tname , self . alltypes , database = self . tmp_db ) <EOL> self . temp_tables . append ( tname ) <EOL> table = self . con . table ( tname , database = self . tmp_db ) <EOL> df = table . execute ( ) <EOL> assert_frame_equal ( df , self . alltypes ) <EOL> def test_insert ( self ) : <EOL> pytest . skip ( '<STR_LIT>' ) <EOL> schema = pandas_to_ibis_schema ( exhaustive_df ) <EOL> table_name = '<STR_LIT>' . format ( util . guid ( ) ) <EOL> self . con . create_table ( table_name , database = self . tmp_db , <EOL> schema = schema ) <EOL> self . temp_tables . append ( table_name ) <EOL> self . con . insert ( table_name , exhaustive_df . iloc [ : <NUM_LIT:4> ] , <EOL> database = self . tmp_db ) <EOL> self . con . insert ( table_name , exhaustive_df . iloc [ <NUM_LIT:4> : ] , <EOL> database = self . tmp_db ) <EOL> table = self . con . table ( table_name , database = self . tmp_db ) <EOL> result = ( table . execute ( ) <EOL> . sort_index ( by = '<STR_LIT>' ) <EOL> . reset_index ( drop = True ) ) <EOL> assert_frame_equal ( result , exhaustive_df ) <EOL> def test_insert_partition ( self ) : <EOL> pass <EOL> def test_round_trip_exhaustive ( self ) : <EOL> pytest . skip ( '<STR_LIT>' ) <EOL> self . _check_roundtrip ( exhaustive_df ) <EOL> def _check_roundtrip ( self , df ) : <EOL> writer = DataFrameWriter ( self . con , df ) <EOL> path = writer . write_temp_csv ( ) <EOL> table = writer . delimited_table ( path ) <EOL> df2 = table . execute ( ) <EOL> assert_frame_equal ( df2 , df ) </s>
