    HAS_PANDAS = True
    HAS_PANDAS = False
MIN_MULTIPART_SIZE = 50 * 2 ** 20  # 50MB
MIN_PART_SIZE = 5 * 2 ** 20  # 5MB
MAX_PART_SIZE = 5 * 2 ** 30  # 5GB
MAX_FILE_SIZE = 5 * 2 ** 40  # 5TB
MAX_THREADS = 4
RETRY_EXCEPTIONS = (requests.HTTPError,
                    requests.ConnectionError,
                    requests.ConnectTimeout)
CHUNK_SIZE = 32 * 1024
def _get_aws_error_message(response):
def _buf_len(buf):
def _single_upload(buf, name, client, **kwargs):
    def _post():
def _multipart_upload(buf, name, file_size, client, **kwargs):
    def _upload_part_base(item, file_path, part_size, file_size):
def file_to_civis(buf, name=None, api_key=None, client=None, **kwargs):
def _file_to_civis(buf, name, api_key=None, client=None, **kwargs):
def civis_to_file(file_id, buf, api_key=None, client=None):
def _civis_to_file(file_id, buf, api_key=None, client=None):
    def _download_url_to_buf():
def file_id_from_run_output(name, job_id, run_id, regex=False, client=None):
def file_to_dataframe(file_id, compression='infer', client=None,
                      **read_kwargs):
def dataframe_to_file(df, name='data.csv', expires_at='DEFAULT',
                      client=None, **to_csv_kws):
def file_to_json(file_id, client=None, **json_kwargs):
def json_to_file(obj, name='file.json', expires_at='DEFAULT',
                 client=None, **json_kwargs):     NO_PANDAS = False
    NO_PANDAS = True
CHUNK_SIZE = 32 * 1024
DELIMITERS = {
    ',': 'comma',
    '\t': 'tab',
    '|': 'pipe',
}
def read_civis(table, database, columns=None, use_pandas=False, encoding=None,
               job_name=None, api_key=None, client=None, credential_id=None,
               polling_interval=None, archive=False, hidden=True, **kwargs):
def export_to_civis_file(sql, database, job_name=None, client=None,
                         credential_id=None, polling_interval=None,
                         hidden=True, csv_settings=None):
def read_civis_sql(sql, database, use_pandas=False,
                   encoding=None, job_name=None,
                   api_key=None, client=None, credential_id=None,
                   polling_interval=None, archive=False,
                   hidden=True, **kwargs):
        def f(x):
def civis_to_csv(filename, sql, database, job_name=None, api_key=None,
                 client=None, credential_id=None, include_header=True,
                 compression='none', delimiter=',', unquoted=False,
                 archive=False, hidden=True, polling_interval=None):
        def f(x):
def civis_to_multifile_csv(sql, database, job_name=None, api_key=None,
                           client=None, credential_id=None,
                           include_header=True,
                           compression='none', delimiter='|',
                           max_file_size=None,
                           unquoted=False, prefix=None,
                           polling_interval=None, hidden=True):
def dataframe_to_civis(df, database, table, api_key=None, client=None,
                       max_errors=None, existing_table_rows="fail",
                       diststyle=None, distkey=None,
                       sortkey1=None, sortkey2=None,
                       table_columns=None,
                       headers=None, credential_id=None,
                       primary_keys=None, last_modified_keys=None,
                       execution="immediate",
                       delimiter=None, polling_interval=None,
                       archive=False, hidden=True, **kwargs):
def csv_to_civis(filename, database, table, api_key=None, client=None,
                 max_errors=None, existing_table_rows="fail",
                 diststyle=None, distkey=None,
                 sortkey1=None, sortkey2=None,
                 table_columns=None,
                 delimiter=",", headers=None,
                 primary_keys=None, last_modified_keys=None,
                 escaped=False, execution="immediate",
                 credential_id=None, polling_interval=None, archive=False,
                 hidden=True):
def civis_file_to_table(file_id, database, table, client=None,
                        max_errors=None, existing_table_rows="fail",
                        diststyle=None, distkey=None,
                        sortkey1=None, sortkey2=None,
                        table_columns=None,
                        primary_keys=None, last_modified_keys=None,
                        escaped=False, execution="immediate",
                        delimiter=None, headers=None,
                        credential_id=None, polling_interval=None,
                        hidden=True):
def _sql_script(client, sql, database, job_name, credential_id, hidden=False,
                csv_settings=None):
def _get_sql_select(table, columns=None):
def _decompress_stream(response, buf, write_bytes=True, encoding="utf-8"):
def _download_file(url, local_path, headers, compression):
def _download_callback(job_id, run_id, filename, headers, compression):
    def callback(future):
def split_schema_tablename(table):
def _run_cleaning(file_ids, client, need_table_columns, headers, delimiter,
                  hidden, polling_interval=None):
def _process_cleaning_results(cleaning_futures, client, headers,
                              need_table_columns, delimiter):
def _check_detected_info(files: List[_File], attr: str, value_from_user=None):
def _check_column_types(files: List[_File]):
def _err_msg_if_inconsistent(items: List, files: List[_File]):
def _format_files_for_err_msg(files: List[_File], indices): class Response(dict):
    """Custom Civis response object.

    Attributes
    ----------
    json_data : dict | None
        This is `json_data` as it is originally returned to the user without
        the key names being changed. See Notes. None is used if the original
        response returned a 204 No Content response.
    headers : dict
        This is the header for the API call without changing the key names.
    calls_remaining : int
        Number of API calls remaining before rate limit is reached.
    rate_limit : int
        Total number of calls per API rate limit period.

    Notes
    -----
    The main features of this class are that it maps camelCase to snake_case
    at the top level of the json object and attaches keys as attributes.
    Nested object keys are not changed.
    """
    def __init__(self, json_data, snake_case=True, headers=None):
        self.json_data = json_data
        if headers is not None:
            # this circumvents recursive calls
            self.headers = headers
            self.calls_remaining = headers.get('X-RateLimit-Remaining')
            self.rate_limit = headers.get('X-RateLimit-Limit')

        # Keys to update for this response object.
        self_updates = {}

        if json_data is not None:
            for key, v in json_data.items():
                if snake_case:
                    key = camel_to_snake(key)

                if isinstance(v, dict):
                    val = Response(v, False)
                elif isinstance(v, list):
                    val = [Response(o) if isinstance(o, dict) else o
                           for o in v]
                else:
                    val = v

                self_updates[key] = val

        self.update(self_updates)
        # Update self.__dict__ at the end to avoid replacing the update method.
        self.__dict__.update(self_updates) class CivisAPIError(Exception):
    def __init__(self, response):
        if response.content:  # the API itself gave an error response
            json = response.json()
            self.error_message = json["errorDescription"]
        else:  # this was something like a 502
            self.error_message = response.reason

        self.status_code = response.status_code
        self._response = response

    def __str__(self):
        if self.status_code:
            return "({}) {}".format(self.status_code, self.error_message)
        else:
            return self.error_message class CivisImportError(Exception):
    pass class EmptyResultError(Exception):
    pass API_SPEC = json.load(f, object_pairs_hook=OrderedDict) @lru_cache(maxsize=4)
def get_api_spec(api_key, api_version="1.0", user_agent="civis-python"):
    """Download the Civis API specification.

    Parameters
    ----------
    api_key : str
        Your API key obtained from the Civis Platform.
    api_version : string, optional
        The version of endpoints to call. May instantiate multiple client
        objects with different versions.  Currently only "1.0" is supported.
    user_agent : string, optional
        Provide this user agent to the the Civis API, along with an
        API client version tag and ``requests`` version tag.
    """
    if api_version == "1.0":
        with open_session(api_key, user_agent=user_agent) as sess:
            request = Request('GET', "{}endpoints".format(get_base_url()))
            pre_request = sess.prepare_request(request)
            response = retry_request('get', pre_request, sess, MAX_RETRIES)
    else:
        msg = "API specification for api version {} cannot be found"
        raise ValueError(msg.format(api_version))
    if response.status_code in (401, 403):
        msg = "{} error downloading API specification. API key may be expired."
        raise requests.exceptions.HTTPError(msg.format(response.status_code))
    response.raise_for_status()
    spec = response.json(object_pairs_hook=OrderedDict)
    return spec @lru_cache(maxsize=4)
@deprecate_param('v2.0.0', 'resources')
def generate_classes(api_key, api_version="1.0", resources="all"):
    """ Dynamically create classes to interface with the Civis API.

    The Civis API documents behavior using an OpenAPI/Swagger specification.
    This function parses the specification and returns a dictionary of
    classes to provide a convenient method to make requests to the Civis
    API and to view documentation.

    https://github.com/OAI/OpenAPI-Specification

    Parameters
    ----------
    api_key : str
        Your API key obtained from the Civis Platform.
    api_version : string, optional
        The version of endpoints to call. May instantiate multiple client
        objects with different versions.  Currently only "1.0" is supported.
    resources : string, optional
        When set to "base", only the default endpoints will be exposed in the
        client object.  Set to "all" to include all endpoints available for
        a given user, including those that may be in development and subject
        to breaking changes at a later date.
    """
    if api_version not in API_VERSIONS:
        raise ValueError(
            f"APIClient api_version must be one of {set(API_VERSIONS)}: "
            f"{api_version}"
        )
    if resources not in _RESOURCES:
        raise ValueError(
            f"resources must be one of {set(_RESOURCES)}: {resources}"
        )
    raw_spec = get_api_spec(api_key, api_version)
    spec = JsonRef.replace_refs(raw_spec)
    return parse_api_spec(spec, api_version, resources) class CivisVCRTestCase(VCRTestCase):

    def _get_vcr_kwargs(self):
        return {
            'filter_headers': ['Authorization'],
            'record_mode': 'once',
            'path_transformer': lambda x: x.replace('yaml', 'yml')
        } def cassette_dir():
    testdir = os.path.dirname(os.path.abspath(__file__))
    return os.path.join(testdir, 'cassettes') POLL_INTERVAL = None def create_client_mock(cache=API_SPEC_PATH):
    """Create an APIClient mock from a cache of the API spec

    Parameters
    ----------
    cache : str, optional
        Location of the API spec on the local filesystem

    Returns
    -------
    mock.Mock
        A `Mock` object which looks like an APIClient and which will
        error if any method calls have non-existent / misspelled parameters
    """
    # Create a client from the cache. We'll use this for auto-speccing.
    real_client = _real_client(cache)

    # Prevent the client from trying to talk to the real API when autospeccing
    with mock.patch('requests.Session', mock.MagicMock):
        mock_client = mock.create_autospec(real_client, spec_set=True)

    return mock_client
import tempfile
import csv
import gzip
import io
import json
import os
import zipfile
import pytest
import requests
import vcr
    import pandas as pd
import civis
from io import StringIO, BytesIO
from unittest import mock
from tempfile import TemporaryDirectory
from civis.io import _files
from civis.io._tables import _File
from civis.response import Response
from civis.base import CivisAPIError, CivisImportError, EmptyResultError
from civis.resources import API_SPEC
from civis.resources._resources import get_api_spec, generate_classes
from civis.tests.testcase import (CivisVCRTestCase,
                                  cassette_dir,
                                  POLL_INTERVAL)
from civis.tests.mocks import create_client_mock
    mock_client.scripts.post_sql.assert_called_once_with(
        'My job',
        remote_host_id=database_id,
        credential_id=credential_id,
        sql=sql,
        hidden=False,
        csv_settings={})
    mock_client.scripts.post_sql_runs.assert_called_once_with(export_job_id)


@mock.patch.object(civis.io._tables, 'requests')
def test_read_civis_sql_no_pandas_special_encoding_sad_path(m_requests):
    # Set up a mock client object for what civis.io.read_civis_sql needs.
    m_client = create_client_mock()
    m_client.scripts.get_sql_runs.return_value = Response(
        {
            "output": [
                {"path": "blah", "file_id": 123, "output_name": "blah"}
            ],
            "state": "success",
        }
    )
    m_response = mock.Mock()
    m_response.raw.read.return_value = gzip.compress(
        # Intentionally using non-ASCII chars like è to mess with encoding
        "foo,bar\n123,très bien\n".encode("latin-1")
    )
    m_requests.get.return_value = m_response

    # The data was encoded in latin-1, but we don't specify this at
    # the `encoding` param in the read_civis_sql call,
    # which should raise UnicodeDecodeError.
    with pytest.raises(UnicodeDecodeError):
        civis.io.read_civis_sql(
            "select 1", "db", use_pandas=False, client=m_client,
            polling_interval=1
        )


@mock.patch.object(civis.io._tables, 'requests')
def test_read_civis_sql_no_pandas_special_encoding_happy_path(m_requests):
    # Set up a mock client object for what civis.io.read_civis_sql needs.
    m_client = create_client_mock()
    m_client.scripts.get_sql_runs.return_value = Response(
        {
            "output": [
                {"path": "blah", "file_id": 123, "output_name": "blah"}
            ],
            "state": "success",
        }
    )

    # Intentionally using non-ASCII chars like è to mess with encoding
    expected_data = "foo,bar\n123,très bien\n"
    encoding = "latin-1"

    m_response = mock.Mock()

    # The helper function _decompress_stream() calls
    # response.raw.read(CHUNK_SIZE) in a while loop.
    # In the mock `m_response.raw.read` here, we use side_effect so that
    # we get the data back from the first call and no data in the second call
    # inside the while loop (because the data has all been consumed in the
    # first call).
    m_response.raw.read.side_effect = [
        gzip.compress(expected_data.encode(encoding)), b""
    ]
    m_requests.get.return_value = m_response

    actual_data = civis.io.read_civis_sql(
        "select 1", "db", use_pandas=False, client=m_client,
        polling_interval=1, encoding=encoding
    )
    assert list(csv.reader(io.StringIO(expected_data))) == actual_data


@mock.patch.object(civis.io._tables, 'requests')
def test_read_civis_sql_no_pandas(m_requests):
    # Set up a mock client object for what civis.io.read_civis_sql needs.
    m_client = create_client_mock()
    m_client.scripts.get_sql_runs.return_value = Response(
        {
            "output": [
                {"path": "blah", "file_id": 123, "output_name": "blah"}
            ],
            "state": "success",
        }
    )

    expected_data = "foo,bar\n123,very good\n"

    m_response = mock.Mock()
    # The helper function _decompress_stream() calls
    # response.raw.read(CHUNK_SIZE) in a while loop.
    # In the mock `m_response.raw.read` here, we use side_effect so that
    # we get the data back from the first call and no data in the second call
    # inside the while loop (because the data has all been consumed in the
    # first call).
    m_response.raw.read.side_effect = [
        gzip.compress(expected_data.encode("utf-8")), b""
    ]
    m_requests.get.return_value = m_response

    actual_data = civis.io.read_civis_sql(
        "select 1", "db", use_pandas=False, client=m_client,
        polling_interval=1,
    )
    assert list(csv.reader(io.StringIO(expected_data))) == actual_data


@pytest.mark.skipif(not has_pandas, reason="pandas not installed")
def test_read_civis_sql_pandas():
    with tempfile.TemporaryDirectory() as tmp_dir:
        tmp_csv_path = os.path.join(tmp_dir, "data.csv")
        expected_data = "foo,bar\n123,very good\n"
        with open(tmp_csv_path, "wb") as f:
            f.write(gzip.compress(expected_data.encode("utf-8")))

        # Set up a mock client object for what civis.io.read_civis_sql needs.
        m_client = create_client_mock()

        m_client.scripts.get_sql_runs.return_value = Response(