"""
Contains the base Client class for accessing Swift services.
"""
"""
Copyright 2011-2013 Gregory Holt

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from swiftly import VERSION
from swiftly.client.utils import quote

class Client(object):
    """
    The base class for accessing Swift services.

    For concrete examples, see
    :py:class:`swiftly.client.standardclient.StandardClient` and
    :py:class:`swiftly.client.directclient.DirectClient`.

    To form a new concrete subclass, you would need to implement
    :py:func:`request` and :py:func:`get_account_hash` minimally and
    optionally :py:func:`reset` and :py:func:`auth`.
    """

    def __init__(self):
        #: The string to use for the User-Agent request header.
        self.user_agent = 'Swiftly v%s' % VERSION
        #: These HTTP methods do not allow contents
        self.no_content_methods = ['COPY', 'DELETE', 'GET', 'HEAD']

    def reset(self):
        """
        Resets the client, closing any connections and discarding any
        state. This can be useful if some exceptional condition
        occurred and the request/response state can no longer be
        certain.
        """
        pass

    def auth(self):
        """
        Just performs any authentication steps without making an
        actual request to the Swift system.
        """
        pass

    def request(self, method, path, contents, headers, decode_json=False,
                stream=False, query=None, cdn=False):
        """
        Performs a direct HTTP request to the Swift service.

        :param method: The request method ('GET', 'HEAD', etc.)
        :param path: The request path.
        :param contents: The body of the request. May be a string or
            a file-like object.
        :param headers: A dict of request headers and values.
        :param decode_json: If set True, the response body will be
            treated as JSON and decoded result returned instead of
            the raw contents.
        :param stream: If set True, the response body will return as
            a file-like object; otherwise, the response body will be
            read in its entirety and returned as a string. Overrides
            decode_json.
        :param query: A dict of query parameters and values to append
            to the path.
        :param cdn: If set True, the request will be sent to the CDN
            management endpoint instead of the default storage
            endpoint.
        :returns: A tuple of (status, reason, headers, contents).

            :status: An int for the HTTP status code.
            :reason: The str for the HTTP status (ex: "Ok").
            :headers: A dict with all lowercase keys of the HTTP
                headers; if a header has multiple values, it will be
                a list.
            :contents: Depending on the decode_json and stream
                settings, this will either be the raw response
                string, the JSON decoded object, or a file-like
                object.
        """
        raise Exception('request method not implemented')

    def get_account_hash(self):
        """
        Returns the account identifier for the Swift account being
        accessed.
        """
        raise Exception('get_account_hash method not implemented')

    def _container_path(self, container):
        container = container.rstrip('/')
        if container.startswith('/'):
            return quote(container)
        else:
            return '/' + quote(container)

    def _object_path(self, container, obj):
        container = self._container_path(container)
        # Leading/trailing slashes are allowed in object names, so don't strip
        # them.
        return container + '/' + quote(obj)

    def head_account(self, headers=None, query=None, cdn=False):
        """
        HEADs the account and returns the results. Useful headers
        returned are:

        =========================== =================================
        x-account-bytes-used        Object storage used for the
                                    account, in bytes.
        x-account-container-count   The number of containers in the
                                    account.
        x-account-object-count      The number of objects in the
                                    account.
        =========================== =================================

        Also, any user headers beginning with x-account-meta- are
        returned.

        These values can be delayed depending the Swift cluster.

        :param headers: Additional headers to send with the request.
        :param query: Set to a dict of query values to send on the
            query string of the request.
        :param cdn: If set True, the CDN management interface will be
            used.
        :returns: A tuple of (status, reason, headers, contents).

            :status: is an int for the HTTP status code.
            :reason: is the str for the HTTP status (ex: "Ok").
            :headers: is a dict with all lowercase keys of the HTTP
                headers; if a header has multiple values, it will be a
                list.
            :contents: is the str for the HTTP body.
        """
        return self.request(
            'HEAD', '', '', headers, query=query, cdn=cdn)

    def get_account(self, headers=None, prefix=None, delimiter=None,
                    marker=None, end_marker=None, limit=None, query=None,
                    cdn=False, decode_json=True):
        """
        GETs the account and returns the results. This is done to list
        the containers for the account. Some useful headers are also
        returned:

        =========================== =================================
        x-account-bytes-used        Object storage used for the
                                    account, in bytes.
        x-account-container-count   The number of containers in the
                                    account.
        x-account-object-count      The number of objects in the
                                    account.
        =========================== =================================

        Also, any user headers beginning with x-account-meta- are
        returned.

        These values can be delayed depending the Swift cluster.

        :param headers: Additional headers to send with the request.
        :param prefix: The prefix container names must match to be
            listed.
        :param delimiter: The delimiter for the listing. Delimiters
            indicate how far to progress through container names
            before "rolling them up". For instance, a delimiter='.'
            query on an account with the containers::

                one.one
                one.two
                two
                three.one

            would return the JSON value of::

                [{'subdir': 'one.'},
                 {'count': 0, 'bytes': 0, 'name': 'two'},
                 {'subdir': 'three.'}]

            Using this with prefix can allow you to traverse a psuedo
            hierarchy.
        :param marker: Only container names after this marker will be
            returned. Swift returns a limited number of containers
            per request (often 10,000). To get the next batch of
            names, you issue another query with the marker set to the
            last name you received. You can continue to issue
            requests until you receive no more names.
        :param end_marker: Only container names before this marker will be
            returned.
        :param limit: Limits the size of the list returned per
            request. The default and maximum depends on the Swift
            cluster (usually 10,000).
        :param query: Set to a dict of query values to send on the
            query string of the request.
        :param cdn: If set True, the CDN management interface will be
            used.
        :param decode_json: If set False, the usual decoding of the
            JSON response will be skipped and the raw contents will
            be returned instead.
        :returns: A tuple of (status, reason, headers, contents).

            :status: is an int for the HTTP status code.
            :reason: is the str for the HTTP status (ex: "Ok").
            :headers: is a dict with all lowercase keys of the HTTP
                headers; if a header has multiple values, it will be a
                list.
            :contents: is the decoded JSON response or the raw str
                for the HTTP body.
        """
        query = dict(query or {})
        query['format'] = 'json'
        if prefix:
            query['prefix'] = prefix
        if delimiter:
            query['delimiter'] = delimiter
        if marker:
            query['marker'] = marker
        if end_marker:
            query['end_marker'] = end_marker
        if limit:
            query['limit'] = limit
        return self.request(
            'GET', '', '', headers, decode_json=decode_json, query=query,
            cdn=cdn)

    def put_account(self, headers=None, query=None, cdn=False, body=None):
        """
        PUTs the account and returns the results. This is usually
        done with the extract-archive bulk upload request and has no
        other use I know of (but the call is left open in case there
        ever is).

        :param headers: Additional headers to send with the request.
        :param query: Set to a dict of query values to send on the
            query string of the request.
        :param cdn: If set True, the CDN management interface will be
            used.
        :param body: Some account PUT requests, like the
            extract-archive bulk upload request, take a body.
        :returns: A tuple of (status, reason, headers, contents).

            :status: is an int for the HTTP status code.
            :reason: is the str for the HTTP status (ex: "Ok").
            :headers: is a dict with all lowercase keys of the HTTP
                headers; if a header has multiple values, it will be a
                list.
            :contents: is the str for the HTTP body.
        """
        return self.request(
            'PUT', '', body or '', headers, query=query, cdn=cdn)

    def post_account(self, headers=None, query=None, cdn=False, body=None):
        """
        POSTs the account and returns the results. This is usually
        done to set X-Account-Meta-xxx headers. Note that any existing
        X-Account-Meta-xxx headers will remain untouched. To remove an
        X-Account-Meta-xxx header, send the header with an empty
        string as its value.

        :param headers: Additional headers to send with the request.
        :param query: Set to a dict of query values to send on the
            query string of the request.
        :param cdn: If set True, the CDN management interface will be
            used.
        :param body: No known Swift POSTs take a body; but the option
            is there for the future.
        :returns: A tuple of (status, reason, headers, contents).

            :status: is an int for the HTTP status code.
            :reason: is the str for the HTTP status (ex: "Ok").
            :headers: is a dict with all lowercase keys of the HTTP
                headers; if a header has multiple values, it will be a
                list.
            :contents: is the str for the HTTP body.
        """
        return self.request(
            'POST', '', body or '', headers, query=query, cdn=cdn)

    def delete_account(self, headers=None,
                       yes_i_mean_delete_the_account=False, query=None,
                       cdn=False, body=None):
        """
        Sends a DELETE request to the account and returns the results.

        With ``query['bulk-delete'] = ''`` this might mean a bulk
        delete request where the body of the request is new-line
        separated, url-encoded list of names to delete. Be careful
        with this! One wrong move and you might mark your account for
        deletion of you have the access to do so!

        For a plain DELETE to the account, on clusters that support
        it and, assuming you have permissions to do so, the account
        will be marked as deleted and immediately begin removing the
        objects from the cluster in the backgound.

        THERE IS NO GOING BACK!

        :param headers: Additional headers to send with the request.
        :param yes_i_mean_delete_the_account: Set to True to verify
            you really mean to delete the entire account. This is
            required unless ``body and 'bulk-delete' in query``.
        :param query: Set to a dict of query values to send on the
            query string of the request.
        :param cdn: If set True, the CDN management interface will be
            used.
        :param body: Some account DELETE requests, like the bulk
            delete request, take a body.
        :returns: A tuple of (status, reason, headers, contents).

            :status: is an int for the HTTP status code.
            :reason: is the str for the HTTP status (ex: "Ok").
            :headers: is a dict with all lowercase keys of the HTTP
                headers; if a header has multiple values, it will be
                a list.
            :contents: is the str for the HTTP body.
        """
        if not yes_i_mean_delete_the_account and (
                not body or not query or 'bulk-delete' not in query):
            return (0, 'yes_i_mean_delete_the_account was not set to True', {},
                    '')
        return self.request(
            'DELETE', '', body or '', headers, query=query, cdn=cdn)

    def head_container(self, container, headers=None, query=None, cdn=False):
        """
        HEADs the container and returns the results. Useful headers
        returned are:

        =========================== =================================
        x-container-bytes-used      Object storage used for the
                                    container, in bytes.
        x-container-object-count    The number of objects in the
                                    container.
        =========================== =================================

        Also, any user headers beginning with x-container-meta- are
        returned.

        These values can be delayed depending the Swift cluster.

        :param container: The name of the container.
        :param headers: Additional headers to send with the request.
        :param query: Set to a dict of query values to send on the
            query string of the request.
        :param cdn: If set True, the CDN management interface will be
            used.
        :returns: A tuple of (status, reason, headers, contents).

            :status: is an int for the HTTP status code.
            :reason: is the str for the HTTP status (ex: "Ok").
            :headers: is a dict with all lowercase keys of the HTTP
                headers; if a header has multiple values, it will be a
                list.
            :contents: is the str for the HTTP body.
        """
        path = self._container_path(container)
        return self.request(
            'HEAD', path, '', headers, query=query, cdn=cdn)

    def get_container(self, container, headers=None, prefix=None,
                      delimiter=None, marker=None, end_marker=None,
                      limit=None, query=None, cdn=False, decode_json=True):
        """
        GETs the container and returns the results. This is done to
        list the objects for the container. Some useful headers are
        also returned:

        =========================== =================================
        x-container-bytes-used      Object storage used for the
                                    container, in bytes.
        x-container-object-count    The number of objects in the
                                    container.
        =========================== =================================

        Also, any user headers beginning with x-container-meta- are
        returned.

        These values can be delayed depending the Swift cluster.

        :param container: The name of the container.
        :param headers: Additional headers to send with the request.
        :param prefix: The prefix object names must match to be
            listed.
        :param delimiter: The delimiter for the listing. Delimiters
            indicate how far to progress through object names before
            "rolling them up". For instance, a delimiter='/' query on
            an container with the objects::

                one/one
                one/two
                two
                three/one

            would return the JSON value of::

                [{'subdir': 'one/'},
                 {'count': 0, 'bytes': 0, 'name': 'two'},
                 {'subdir': 'three/'}]

            Using this with prefix can allow you to traverse a psuedo
            hierarchy.
        :param marker: Only object names after this marker will be
            returned. Swift returns a limited number of objects per
            request (often 10,000). To get the next batch of names,
            you issue another query with the marker set to the last
            name you received. You can continue to issue requests
            until you receive no more names.
        :param end_marker: Only object names before this marker will be
            returned.
        :param limit: Limits the size of the list returned per
            request. The default and maximum depends on the Swift
            cluster (usually 10,000).
        :param query: Set to a dict of query values to send on the
            query string of the request.
        :param cdn: If set True, the CDN management interface will be
            used.
        :param decode_json: If set False, the usual decoding of the
            JSON response will be skipped and the raw contents will
            be returned instead.
        :returns: A tuple of (status, reason, headers, contents).

            :status: is an int for the HTTP status code.
            :reason: is the str for the HTTP status (ex: "Ok").
            :headers: is a dict with all lowercase keys of the HTTP
                headers; if a header has multiple values, it will be a
                list.
            :contents: is the decoded JSON response or the raw str
                for the HTTP body.
        """
        query = dict(query or {})
        query['format'] = 'json'
        if prefix:
            query['prefix'] = prefix
        if delimiter:
            query['delimiter'] = delimiter
        if marker:
            query['marker'] = marker
        if end_marker:
            query['end_marker'] = end_marker
        if limit:
            query['limit'] = limit
        return self.request(
            'GET', self._container_path(container), '', headers,
            decode_json=decode_json, query=query, cdn=cdn)

    def put_container(self, container, headers=None, query=None, cdn=False,
                      body=None):
        """
        PUTs the container and returns the results. This is usually
        done to create new containers and can also be used to set
        X-Container-Meta-xxx headers. Note that if the container
        already exists, any existing X-Container-Meta-xxx headers will
        remain untouched. To remove an X-Container-Meta-xxx header,
        send the header with an empty string as its value.

        :param container: The name of the container.
        :param headers: Additional headers to send with the request.
        :param query: Set to a dict of query values to send on the
            query string of the request.
        :param cdn: If set True, the CDN management interface will be
            used.
        :param body: Some container PUT requests, like the
            extract-archive bulk upload request, take a body.
        :returns: A tuple of (status, reason, headers, contents).

            :status: is an int for the HTTP status code.
            :reason: is the str for the HTTP status (ex: "Ok").
            :headers: is a dict with all lowercase keys of the HTTP
                headers; if a header has multiple values, it will be a
                list.
            :contents: is the str for the HTTP body.
        """
        path = self._container_path(container)
        return self.request(
            'PUT', path, body or '', headers, query=query, cdn=cdn)

    def post_container(self, container, headers=None, query=None, cdn=False,
                       body=None):
        """
        POSTs the container and returns the results. This is usually
        done to set X-Container-Meta-xxx headers. Note that any
        existing X-Container-Meta-xxx headers will remain untouched.
        To remove an X-Container-Meta-xxx header, send the header with
        an empty string as its value.

        :param container: The name of the container.
        :param headers: Additional headers to send with the request.
        :param query: Set to a dict of query values to send on the
            query string of the request.
        :param cdn: If set True, the CDN management interface will be
            used.
        :param body: No known Swift POSTs take a body; but the option
            is there for the future.
        :returns: A tuple of (status, reason, headers, contents).

            :status: is an int for the HTTP status code.
            :reason: is the str for the HTTP status (ex: "Ok").
            :headers: is a dict with all lowercase keys of the HTTP
                headers; if a header has multiple values, it will be a
                list.
            :contents: is the str for the HTTP body.
        """
        path = self._container_path(container)
        return self.request(
            'POST', path, body or '', headers, query=query, cdn=cdn)

    def delete_container(self, container, headers=None, query=None, cdn=False,
                         body=None):
        """
        DELETEs the container and returns the results.

        :param container: The name of the container.
        :param headers: Additional headers to send with the request.
        :param query: Set to a dict of query values to send on the
            query string of the request.
        :param cdn: If set True, the CDN management interface will be
            used.
        :param body: Some container DELETE requests might take a body
            in the future.
        :returns: A tuple of (status, reason, headers, contents).

            :status: is an int for the HTTP status code.
            :reason: is the str for the HTTP status (ex: "Ok").
            :headers: is a dict with all lowercase keys of the HTTP
                headers; if a header has multiple values, it will be a
                list.
            :contents: is the str for the HTTP body.
        """
        path = self._container_path(container)
        return self.request(
            'DELETE', path, body or '', headers, query=query, cdn=cdn)

    def head_object(self, container, obj, headers=None, query=None, cdn=False):
        """
        HEADs the object and returns the results.

        :param container: The name of the container.
        :param obj: The name of the object.
        :param headers: Additional headers to send with the request.
        :param query: Set to a dict of query values to send on the
            query string of the request.
        :param cdn: If set True, the CDN management interface will be
            used.
        :returns: A tuple of (status, reason, headers, contents).

            :status: is an int for the HTTP status code.
            :reason: is the str for the HTTP status (ex: "Ok").
            :headers: is a dict with all lowercase keys of the HTTP
                headers; if a header has multiple values, it will be a
                list.
            :contents: is the str for the HTTP body.
        """
        path = self._object_path(container, obj)
        return self.request(
            'HEAD', path, '', headers, query=query, cdn=cdn)

    def get_object(self, container, obj, headers=None, stream=True, query=None,
                   cdn=False):
        """
        GETs the object and returns the results.

        :param container: The name of the container.
        :param obj: The name of the object.
        :param headers: Additional headers to send with the request.
        :param stream: Indicates whether to stream the contents or
            preread them fully and return them as a str. Default:
            True to stream the contents. When streaming, contents
            will have the standard file-like-object read function,
            which accepts an optional size parameter to limit how
            much data is read per call. When streaming is on, be
            certain to fully read the contents before issuing another
            request.
        :param query: Set to a dict of query values to send on the
            query string of the request.
        :param cdn: If set True, the CDN management interface will be
            used.
        :returns: A tuple of (status, reason, headers, contents).

            :status: is an int for the HTTP status code.
            :reason: is the str for the HTTP status (ex: "Ok").
            :headers: is a dict with all lowercase keys of the HTTP
                headers; if a header has multiple values, it will be a
                list.
            :contents: if *stream* was True, *contents* is a
                file-like-object of the contents of the HTTP body. If
                *stream* was False, *contents* is just a simple str of
                the HTTP body.
        """
        path = self._object_path(container, obj)
        return self.request(
            'GET', path, '', headers, query=query, stream=stream, cdn=cdn)

    def put_object(self, container, obj, contents, headers=None, query=None,
                   cdn=False):
        """
        PUTs the object and returns the results. This is used to
        create or overwrite objects. X-Object-Meta-xxx can optionally
        be sent to be stored with the object. Content-Type,
        Content-Encoding and other standard HTTP headers can often
        also be set, depending on the Swift cluster.

        Note that you can set the ETag header to the MD5 sum of the
        contents for extra verification the object was stored
        correctly.

        :param container: The name of the container.
        :param obj: The name of the object.
        :param contents: The contents of the object to store. This can
            be a simple str, or a file-like-object with at least a
            read function. If the file-like-object also has tell and
            seek functions, the PUT can be reattempted on any server
            error.
        :param headers: Additional headers to send with the request.
        :param query: Set to a dict of query values to send on the
            query string of the request.
        :param cdn: If set True, the CDN management interface will be
            used.
        :returns: A tuple of (status, reason, headers, contents).

            :status: is an int for the HTTP status code.
            :reason: is the str for the HTTP status (ex: "Ok").
            :headers: is a dict with all lowercase keys of the HTTP
                headers; if a header has multiple values, it will be a
                list.
            :contents: is the str for the HTTP body.
        """
        path = self._object_path(container, obj)
        return self.request(
            'PUT', path, contents, headers, query=query, cdn=cdn)

    def post_object(self, container, obj, headers=None, query=None, cdn=False,
                    body=None):
        """
        POSTs the object and returns the results. This is used to
        update the object's header values. Note that all headers must
        be sent with the POST, unlike the account and container POSTs.
        With account and container POSTs, existing headers are
        untouched. But with object POSTs, any existing headers are
        removed. The full list of supported headers depends on the
        Swift cluster, but usually include Content-Type,
        Content-Encoding, and any X-Object-Meta-xxx headers.

        :param container: The name of the container.
        :param obj: The name of the object.
        :param headers: Additional headers to send with the request.
        :param query: Set to a dict of query values to send on the
            query string of the request.
        :param cdn: If set True, the CDN management interface will be
            used.
        :param body: No known Swift POSTs take a body; but the option
            is there for the future.
        :returns: A tuple of (status, reason, headers, contents).

            :status: is an int for the HTTP status code.
            :reason: is the str for the HTTP status (ex: "Ok").
            :headers: is a dict with all lowercase keys of the HTTP
                headers; if a header has multiple values, it will be a
                list.
            :contents: is the str for the HTTP body.
        """
        path = self._object_path(container, obj)
        return self.request(
            'POST', path, body or '', headers, query=query, cdn=cdn)

    def delete_object(self, container, obj, headers=None, query=None,
                      cdn=False, body=None):
        """
        DELETEs the object and returns the results.

        :param container: The name of the container.
        :param obj: The name of the object.
        :param headers: Additional headers to send with the request.
        :param query: Set to a dict of query values to send on the
            query string of the request.
        :param cdn: If set True, the CDN management interface will be
            used.
        :param body: Some object DELETE requests might take a body in
            the future.
        :returns: A tuple of (status, reason, headers, contents).

            :status: is an int for the HTTP status code.
            :reason: is the str for the HTTP status (ex: "Ok").
            :headers: is a dict with all lowercase keys of the HTTP
                headers; if a header has multiple values, it will be a
                list.
            :contents: is the str for the HTTP body.
        """
        path = self._object_path(container, obj)
        return self.request(
            'DELETE', path, body or '', headers, query=query, cdn=cdn)


"""
Contains a CLICommand that implements ping test functionality.

Uses the following from :py:class:`swiftly.cli.context.CLIContext`:

===============  ====================================================
client_manager   For connecting to Swift.
concurrency      The number of concurrent actions that can be
                 performed.
io_manager       For directing output.
limit            The maximum number of Swift nodes to output
                 information about.
object_ring      An instance of swift.common.ring.ring.Ring if you
                 want a report based on Swift nodes with implied
                 usage during the ping test.
ping_begin       The first time.time() when the entire ping test
                 began.
ping_begin_last  The time.time() the last ping task started.
ping_count       The number of objects to use.
ping_verbose     True if you want a full ping report rather than just
                 the overall time.
threshold        Defines the threshold for the threshold node report.
                 This is the multiplier over the average request
                 time.
===============  ====================================================
"""
"""
Copyright 2011-2013 Gregory Holt

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import six
from six import moves
import collections
from six.moves import StringIO
import time
import traceback
import uuid

from swiftly.concurrency import Concurrency
from swiftly.cli.command import CLICommand, ReturnCode

try:
    from eventlet import sleep
except ImportError:
    sleep = time.sleep

def _cli_ping_status(context, heading, ident, status, reason, headers,
                     contents):
    if headers:
        ident = headers.get('x-trans-id') or ident
    if hasattr(contents, 'read'):
        contents.read()
    if status and status // 100 != 2:
        raise ReturnCode(
            'with %s: %s %s %s' % (heading, status, reason, ident))
    now = time.time()
    if context.ping_verbose:
        with context.io_manager.with_stdout() as fp:
            fp.write(
                '% 6.02fs %s %s\n' %
                (now - context.ping_begin_last, heading, ident))
            fp.flush()
    context.ping_begin_last = now

def _cli_ping_objects(context, heading, conc, container, objects, func,
                      results):
    begin = time.time()
    for obj in objects:
        for (exc_type, exc_value, exc_tb, result) in \
                six.itervalues(conc.get_results()):
            if exc_value:
                with context.io_manager.with_stderr() as fp:
                    fp.write(str(exc_value))
                    fp.write('\n')
                    fp.flush()
        conc.spawn(obj, func, context, results, container, obj)
    conc.join()
    for (exc_type, exc_value, exc_tb, result) in \
            six.itervalues(conc.get_results()):
        if exc_value:
            with context.io_manager.with_stderr() as fp:
                fp.write(str(exc_value))
                fp.write('\n')
                fp.flush()
    elapsed = time.time() - begin
    _cli_ping_status(
        context,
        'object %s x%d at %d concurrency, %.02f/s' %
        (heading, len(objects), conc.concurrency, len(objects) / elapsed),
        None, None, None, None, None)
    overall = results.get('overall')
    if overall:
        overall = sorted(overall, key=lambda x: x[0])
        results['overall'] = overall
        if context.ping_verbose or context.graphite:
            best = overall[0][0]
            worst = overall[-1][0]
            mean = overall[len(overall) / 2][0]
            median = sum(x[0] for x in overall) / len(overall)
            threshold = max(2, mean * 2)
            slows = 0
            for x in overall:
                if x[0] > 2 and x[0] > threshold:
                    slows += 1
            slow_percentage = 100.0 * slows / len(overall)
            with context.io_manager.with_stdout() as fp:
                if context.ping_verbose:
                    fp.write(
                        '        best %.02fs, worst %.02fs, mean %.02fs, '
                        'median %.02fs\n        %d slower than 2s and twice '
                        'the mean, %.02f%%\n' % (
                            best, worst, mean, median, slows, slow_percentage))
                    fp.flush()
                if context.graphite:
                    fp.write(
                        '%s.%s.slow_percentage %.02f %d\n' % (
                            context.graphite, heading, slow_percentage,
                            time.time()))

def _cli_ping_object_put(context, results, container, obj):
    with context.client_manager.with_client() as client:
        begin = time.time()
        try:
            status, reason, headers, contents = client.put_object(
                container, obj, StringIO('swiftly-ping'))
        except Exception:
            raise ReturnCode(
                'putting object %r: %s' % (obj, traceback.format_exc()))
        if status // 100 != 2:
            raise ReturnCode(
                'putting object %r: %s %s %s' %
                (obj, status, reason, headers.get('x-trans-id') or '-'))
        elapsed = time.time() - begin
        results['overall'].append((elapsed, headers.get('x-trans-id') or obj))
        if context.object_ring:
            for node in context.object_ring.get_nodes(
                    client.get_account_hash(), container, obj)[1]:
                results[node['ip']].append(
                    (elapsed, headers.get('x-trans-id') or obj))

def _cli_ping_object_get(context, results, container, obj):
    with context.client_manager.with_client() as client:
        begin = time.time()
        try:
            status, reason, headers, contents = client.get_object(
                container, obj, stream=False)
        except Exception:
            raise ReturnCode(
                'getting object %r: %s' % (obj, traceback.format_exc()))
        if status // 100 != 2:
            raise ReturnCode(
                'getting object %r: %s %s %s' %
                (obj, status, reason, headers.get('x-trans-id') or '-'))
        elapsed = time.time() - begin
        results['overall'].append((elapsed, headers.get('x-trans-id') or obj))
        if context.object_ring:
            for node in context.object_ring.get_nodes(
                    client.get_account_hash(), container, obj)[1]:
                results[node['ip']].append(
                    (elapsed, headers.get('x-trans-id') or obj))

def _cli_ping_object_delete(context, results, container, obj):
    with context.client_manager.with_client() as client:
        begin = time.time()
        try:
            status, reason, headers, contents = client.delete_object(
                container, obj)
        except Exception:
            raise ReturnCode(
                'deleting object %r: %s' % (obj, traceback.format_exc()))
        if status // 100 != 2 and status != 404:
            raise ReturnCode(
                'deleting object %r: %s %s %s' %
                (obj, status, reason, headers.get('x-trans-id') or '-'))
        elapsed = time.time() - begin
        results['overall'].append((elapsed, headers.get('x-trans-id') or obj))
        if context.object_ring:
            for node in context.object_ring.get_nodes(
                    client.get_account_hash(), container, obj)[1]:
                results[node['ip']].append(
                    (elapsed, headers.get('x-trans-id') or obj))

def _cli_ping_ring_report(context, timings_dict, label):
    timings_dict.pop('overall', None)  # Not currently used in this function
    if not timings_dict:
        return
    worsts = {}
    for ip, timings in six.iteritems(timings_dict):
        worst = [0, None]
        for timing in timings:
            if timing[0] > worst[0]:
                worst = timing
        worsts[ip] = worst
    with context.io_manager.with_stdout() as fp:
        fp.write(
            'Worst %s times for up to %d nodes with implied usage:\n' %
            (label, context.limit))
        for ip, (elapsed, xid) in sorted(
                six.iteritems(worsts), key=lambda x: x[1][0],
                reverse=True)[:context.limit]:
            fp.write('    %20s % 6.02fs %s\n' % (ip, elapsed, xid))
        fp.flush()
    with context.io_manager.with_stdout() as fp:
        averages = {}
        for ip, timings in six.iteritems(timings_dict):
            averages[ip] = sum(t[0] for t in timings) / len(timings)
        fp.write(
            'Average %s times for up to %d nodes with implied usage:\n' %
            (label, context.limit))
        for ip, elapsed in sorted(
                six.iteritems(averages), key=lambda x: x[1],
                reverse=True)[:context.limit]:
            fp.write('    %20s % 6.02fs\n' % (ip, elapsed))
        fp.flush()
    total = 0.0
    count = 0
    for ip, timings in six.iteritems(timings_dict):
        total += sum(t[0] for t in timings)
        count += len(timings)
    threshold = total / count * context.threshold
    counts = collections.defaultdict(lambda: 0)
    for ip, timings in six.iteritems(timings_dict):
        for t in timings:
            if t[0] > threshold:
                counts[ip] += 1
    with context.io_manager.with_stdout() as fp:
        fp.write(
            'Count of %s times past (average * %d) for up to %d nodes with '
            'implied usage:\n' % (label, context.threshold, context.limit))
        for ip, count in sorted(
                six.iteritems(counts), key=lambda x: x[1],
                reverse=True)[:context.limit]:
            fp.write('    %20s % 6d\n' % (ip, count))
        fp.flush()
    percentages = {}
    for ip, count in six.iteritems(counts):
        percentages[ip] = (
            100.0 * count / len(timings_dict[ip]),
            count, len(timings_dict[ip]))
    with context.io_manager.with_stdout() as fp:
        fp.write(
            'Percentage of %s times past (average * %d) for up to %d nodes '
            'with implied usage:\n' %
            (label, context.threshold, context.limit))
        for ip, percentage in sorted(
                six.iteritems(percentages), key=lambda x: x[1][0],
                reverse=True)[:context.limit]:
            fp.write(
                '    %20s % 6.02f%%  %d of %d\n' %
                (ip, percentage[0], percentage[1], percentage[2]))
        fp.flush()

def cli_ping(context, prefix):
    """
    Performs a ping test.

    See :py:mod:`swiftly.cli.ping` for context usage information.

    See :py:class:`CLIPing` for more information.

    :param context: The :py:class:`swiftly.cli.context.CLIContext` to
        use.
    :param prefix: The container name prefix to use. Default:
        swiftly-ping
    """
    if not prefix:
        prefix = 'swiftly-ping'
    ping_ring_object_puts = collections.defaultdict(lambda: [])
    ping_ring_object_gets = collections.defaultdict(lambda: [])
    ping_ring_object_deletes = collections.defaultdict(lambda: [])
    context.ping_begin = context.ping_begin_last = time.time()
    container = prefix + '-' + uuid.uuid4().hex
    objects = [uuid.uuid4().hex for x in moves.range(context.ping_count)]
    conc = Concurrency(context.concurrency)
    with context.client_manager.with_client() as client:
        client.auth()
        _cli_ping_status(context, 'auth', '-', None, None, None, None)
        _cli_ping_status(context, 'account head', '-', *client.head_account())
        _cli_ping_status(
            context, 'container put', '-', *client.put_container(container))
    if _cli_ping_objects(
            context, 'put', conc, container, objects, _cli_ping_object_put,
            ping_ring_object_puts):
        with context.io_manager.with_stderr() as fp:
            fp.write(
                'ERROR put objects did not complete successfully due to '
                'previous error; but continuing\n')
            fp.flush()
    if _cli_ping_objects(
            context, 'get', conc, container, objects, _cli_ping_object_get,
            ping_ring_object_gets):
        with context.io_manager.with_stderr() as fp:
            fp.write(
                'ERROR get objects did not complete successfully due to '
                'previous error; but continuing\n')
            fp.flush()
    if _cli_ping_objects(
            context, 'delete', conc, container, objects,
            _cli_ping_object_delete, ping_ring_object_deletes):
        with context.io_manager.with_stderr() as fp:
            fp.write(
                'ERROR delete objects did not complete successfully due to '
                'previous error; but continuing\n')
            fp.flush()
    for attempt in moves.range(5):
        if attempt:
            sleep(2**attempt)
        with context.client_manager.with_client() as client:
            try:
                _cli_ping_status(
                    context, 'container delete', '-',
                    *client.delete_container(container))
                break
            except ReturnCode as err:
                with context.io_manager.with_stderr() as fp:
                    fp.write(str(err))
                    fp.write('\n')
                    fp.flush()
    else:
        with context.io_manager.with_stderr() as fp:
            fp.write(
                'ERROR could not confirm deletion of container due to '
                'previous error; but continuing\n')
            fp.flush()
    end = time.time()
    with context.io_manager.with_stdout() as fp:
        if context.graphite:
            fp.write(
                '%s.ping_overall %.02f %d\n' % (
                    context.graphite, end - context.ping_begin, time.time()))
        if context.ping_verbose:
            fp.write('% 6.02fs total\n' % (end - context.ping_begin))
        elif not context.graphite:
            fp.write('%.02fs\n' % (end - context.ping_begin))
        fp.flush()
    ping_ring_overall = collections.defaultdict(lambda: [])
    _cli_ping_ring_report(context, ping_ring_object_puts, 'PUT')
    for ip, timings in six.iteritems(ping_ring_object_puts):
        ping_ring_overall[ip].extend(timings)
    _cli_ping_ring_report(context, ping_ring_object_gets, 'GET')
    for ip, timings in six.iteritems(ping_ring_object_gets):
        ping_ring_overall[ip].extend(timings)
    _cli_ping_ring_report(context, ping_ring_object_deletes, 'DELETE')
    for ip, timings in six.iteritems(ping_ring_object_deletes):
        ping_ring_overall[ip].extend(timings)
    _cli_ping_ring_report(context, ping_ring_overall, 'overall')

class CLIPing(CLICommand):
    """
    A CLICommand that implements ping test functionality.

    See the output of ``swiftly help ping`` for more information.
    """

    def __init__(self, cli):
        super(CLIPing, self).__init__(
            cli, 'ping', max_args=1, usage="""
Usage: %prog [main_options] ping [options] [path]

For help on [main_options] run %prog with no args.

Runs a ping test against the account.
The [path] will be used as a prefix to the random container name used (default:
swiftly-ping).""".strip())
        self.option_parser.add_option(
            '-v', '--verbose', dest='ping_verbose', action='store_true',
            help='Outputs additional information as ping works.')
        self.option_parser.add_option(
            '-c', '--count', dest='ping_count', default=1,
            help='Count of objects to create; default 1.')
        self.option_parser.add_option(
            '-o', '--object-ring', dest='object_ring',
            help='The current object ring of the cluster being pinged. This '
                 'will enable output of which nodes are involved in the '
                 'object requests and their implied behavior. Use of this '
                 'also requires the main Swift code is installed and '
                 'importable.')
        self.option_parser.add_option(
            '-l', '--limit', dest='limit',
            help='Limits the node output tables to LIMIT nodes.')
        self.option_parser.add_option(
            '-t', '--threshold', dest='threshold',
            help='Changes the threshold for the final (average * x) reports. '
                 'This will define the value of x, defaults to 2.')
        self.option_parser.add_option(
            '-g', '--graphite', dest='graphite', metavar='PREFIX',
            help='Switches to "graphite" output. The output will be lines of '
                 '"PREFIX.metric value timestamp" suitable for piping to '
                 'graphite (through netcat or something similar).')

    def __call__(self, args):
        options, args, context = self.parse_args_and_create_context(args)
        context.ping_count = int(options.ping_count or 1)
        context.ping_verbose = options.ping_verbose
        context.object_ring = None
        if options.object_ring:
            import swift.common.ring.ring
            context.object_ring = swift.common.ring.ring.Ring(
                options.object_ring)
        context.limit = int(options.limit or 10)
        context.threshold = int(options.threshold or 2)
        context.graphite = options.graphite
        prefix = args.pop(0) if args else 'swiftly-ping'
        return cli_ping(context, prefix)


"""
Contains a CLICommand that can issue GET requests.

Uses the following from :py:class:`swiftly.cli.context.CLIContext`:

=======================  ============================================
all_objects              True if instead of the listing itself you
                         want all objects that the listing references
                         to be output.
cdn                      True if the CDN Management URL should be
                         used instead of the Storage URL.
client_manager           For connecting to Swift.
concurrency              The number of concurrent actions that can be
                         performed.
full                     True if you want a full listing (additional
                         information like object count, bytes used,
                         and upload date) instead of just the item
                         names.
headers                  A dict of headers to send.
ignore_404               True if 404s should be silently ignored.
io_manager               For directing output.
muted_account_headers    The headers to omit when outputting account
                         response headers.
muted_container_headers  The headers to omit when outputting
                         container response headers.
muted_object_headers     The headers to omit when outputting object
                         response headers.
output_headers           True if you want the headers from the
                         response to also be output.
query                    A dict of query parameters to send. Of
                         important use are limit, delimiter, prefix,
                         marker, and end_marker as they are common
                         listing query parameters.
raw                      Normally the result of the GET is translated
                         as a listing and formatted output is
                         generated. Setting this to True will skip
                         any formatting a just output the raw
                         contents from the response. Note that this
                         will also just issue a single request and
                         will not try additional follow-on marker
                         requests.
remove_empty_files       True if files created on disk should be
                         removed if they result in an empty file.
                         This can be useful with sub_commands that
                         only output information for matches.
suppress_container_name  True if files created on disk should have
                         the container name stripped from the file
                         name. When downloading a single container,
                         this is usually desired.
write_headers            A function used to output the response
                         headers if output_headers is set True.
=======================  ============================================
"""
"""
Copyright 2011-2013 Gregory Holt

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import six
import os
import time

from swiftly.cli.command import CLICommand, ReturnCode
from swiftly.concurrency import Concurrency
from swiftly.dencrypt import AES256CBC, aes_decrypt
from swiftly.filelikeiter import FileLikeIter

def cli_get_account_listing(context):
    """
    Performs a GET on the account as a listing request.

    See :py:mod:`swiftly.cli.get` for context usage information.

    See :py:class:`CLIGet` for more information.
    """
    limit = context.query.get('limit')
    delimiter = context.query.get('delimiter')
    prefix = context.query.get('prefix')
    marker = context.query.get('marker')
    end_marker = context.query.get('end_marker')
    if context.raw:
        with context.client_manager.with_client() as client:
            status, reason, headers, contents = client.get_account(
                decode_json=False, headers=context.headers, limit=limit,
                marker=marker, end_marker=end_marker, query=context.query,
                cdn=context.cdn)
            if hasattr(contents, 'read'):
                contents = contents.read()
        if status // 100 != 2:
            if status == 404 and context.ignore_404:
                return
            raise ReturnCode('listing account: %s %s' % (status, reason))
        with context.io_manager.with_stdout() as fp:
            if context.output_headers:
                context.write_headers(
                    fp, headers, context.muted_account_headers)
            fp.write(contents)
            fp.flush()
        return
    with context.client_manager.with_client() as client:
        status, reason, headers, contents = client.get_account(
            headers=context.headers, limit=limit, marker=marker,
            end_marker=end_marker, query=context.query, cdn=context.cdn)
        if status // 100 != 2:
            if status == 404 and context.ignore_404:
                return
            if hasattr(contents, 'read'):
                contents.read()
            raise ReturnCode('listing account: %s %s' % (status, reason))
    if context.output_headers and not context.all_objects:
        with context.io_manager.with_stdout() as fp:
            context.write_headers(
                fp, headers, context.muted_account_headers)
    while contents:
        if context.all_objects:
            new_context = context.copy()
            new_context.query = dict(new_context.query)
            for remove in (
                    'limit', 'delimiter', 'prefix', 'marker', 'end_marker'):
                if remove in new_context.query:
                    del new_context.query[remove]
            for item in contents:
                if 'name' in item:
                    new_path = item['name'].encode('utf8')
                    cli_get_container_listing(new_context, new_path)
        else:
            with context.io_manager.with_stdout() as fp:
                for item in contents:
                    if context.full:
                        fp.write('%13s %13s ' % (
                            item.get('bytes', '-'),
                            item.get('count', '-')))
                    fp.write(item.get(
                        'name', item.get('subdir')))
                    fp.write('\n')
                fp.flush()
        if limit:
            break
        marker = contents[-1].get('name', contents[-1].get('subdir', ''))
        with context.client_manager.with_client() as client:
            status, reason, headers, contents = client.get_account(
                headers=context.headers, limit=limit, delimiter=delimiter,
                prefix=prefix, end_marker=end_marker, marker=marker,
                query=context.query, cdn=context.cdn)
            if status // 100 != 2:
                if status == 404 and context.ignore_404:
                    return
                if hasattr(contents, 'read'):
                    contents.read()
                raise ReturnCode('listing account: %s %s' % (status, reason))

def cli_get_container_listing(context, path=None):
    """
    Performs a GET on the container as a listing request.

    See :py:mod:`swiftly.cli.get` for context usage information.

    See :py:class:`CLIGet` for more information.
    """
    path = path.strip('/') if path else None
    if not path or '/' in path:
        raise ReturnCode(
            'tried to get a container listing for non-container path %r' %
            path)
    context.suppress_container_name = True
    limit = context.query.get('limit')
    delimiter = context.query.get('delimiter')
    prefix = context.query.get('prefix')
    marker = context.query.get('marker')
    end_marker = context.query.get('end_marker')
    if context.raw:
        with context.client_manager.with_client() as client:
            status, reason, headers, contents = client.get_container(
                path, decode_json=False, headers=context.headers, limit=limit,
                marker=marker, end_marker=end_marker, query=context.query,
                cdn=context.cdn)
            if hasattr(contents, 'read'):
                contents = contents.read()
        if status // 100 != 2:
            if status == 404 and context.ignore_404:
                return
            raise ReturnCode(
                'listing container %r: %s %s' % (path, status, reason))
        with context.io_manager.with_stdout() as fp:
            if context.output_headers:
                context.write_headers(
                    fp, headers, context.muted_container_headers)
            fp.write(contents)
            fp.flush()
        return
    with context.client_manager.with_client() as client:
        status, reason, headers, contents = client.get_container(
            path, headers=context.headers, limit=limit, delimiter=delimiter,
            prefix=prefix, marker=marker, end_marker=end_marker,
            query=context.query, cdn=context.cdn)
        if status // 100 != 2:
            if status == 404 and context.ignore_404:
                return
            if hasattr(contents, 'read'):
                contents.read()
            raise ReturnCode(
                'listing container %r: %s %s' % (path, status, reason))
    if context.output_headers and not context.all_objects:
        with context.io_manager.with_stdout() as fp:
            context.write_headers(
                fp, headers, context.muted_container_headers)
    conc = Concurrency(context.concurrency)
    while contents:
        if context.all_objects:
            new_context = context.copy()
            new_context.query = dict(new_context.query)
            for remove in (
                    'limit', 'delimiter', 'prefix', 'marker', 'end_marker'):
                if remove in new_context.query:
                    del new_context.query[remove]
            for item in contents:
                if 'name' in item:
                    for (exc_type, exc_value, exc_tb, result) in \
                            six.itervalues(conc.get_results()):
                        if exc_value:
                            conc.join()
                            raise exc_value
                    new_path = path + '/' + item['name'].encode('utf8')
                    conc.spawn(new_path, cli_get, new_context, new_path)
        else:
            with context.io_manager.with_stdout() as fp:
                for item in contents:
                    if context.full:
                        fp.write('%13s %22s %32s %25s ' % (
                            item.get('bytes', '-'),
                            item.get('last_modified', '-')[:22].replace(
                                'T', ' '),
                            item.get('hash', '-'),
                            item.get('content_type', '-')))
                    fp.write(item.get(
                        'name', item.get('subdir')))
                    fp.write('\n')
                fp.flush()
        if limit:
            break
        marker = contents[-1].get('name', contents[-1].get('subdir', ''))
        with context.client_manager.with_client() as client:
            status, reason, headers, contents = client.get_container(
                path, headers=context.headers, limit=limit,
                delimiter=delimiter, prefix=prefix, end_marker=end_marker,
                marker=marker, query=context.query, cdn=context.cdn)
            if status // 100 != 2:
                if status == 404 and context.ignore_404:
                    return
                if hasattr(contents, 'read'):
                    contents.read()
                raise ReturnCode(
                    'listing container %r: %s %s' % (path, status, reason))
    conc.join()
    for (exc_type, exc_value, exc_tb, result) in \
        six.itervalues(conc.get_results()):
        if exc_value:
            raise exc_value

def cli_get(context, path=None):
    """
    Performs a GET on the item (account, container, or object).

    See :py:mod:`swiftly.cli.get` for context usage information.

    See :py:class:`CLIGet` for more information.
    """
    path = path.lstrip('/') if path else None
    if not path:
        return cli_get_account_listing(context)
    elif '/' not in path.rstrip('/'):
        return cli_get_container_listing(context, path)
    status, reason, headers, contents = 0, 'Unknown', {}, ''
    with context.client_manager.with_client() as client:
        status, reason, headers, contents = client.get_object(
            *path.split('/', 1), headers=context.headers, query=context.query,
            cdn=context.cdn)
        if status // 100 != 2:
            if status == 404 and context.ignore_404:
                return
            if hasattr(contents, 'read'):
                contents.read()
            raise ReturnCode(
                'getting object %r: %s %s' % (path, status, reason))
        if context.decrypt:
            crypt_type = contents.read(1)
            if crypt_type == AES256CBC:
                contents = FileLikeIter(aes_decrypt(
                    context.decrypt, contents,
                    chunk_size=getattr(client, 'chunk_size', 65536)))
            else:
                raise ReturnCode(
                    'getting object %r: contents encrypted with unsupported '
                    'type %r' % (path, crypt_type))

        def disk_closed_callback(disk_path):
            if context.remove_empty_files and not os.path.getsize(disk_path):
                os.unlink(disk_path)
                if context.io_manager.stdout_root:
                    dirname = os.path.dirname(disk_path)
                    while dirname and dirname.startswith(
                            context.io_manager.stdout_root):
                        try:
                            os.rmdir(dirname)
                        except OSError:
                            pass
                        dirname = os.path.dirname(dirname)
                return
            if (headers.get('content-type') in
                    ['text/directory', 'application/directory'] and
                    headers.get('content-length') == '0'):
                os.unlink(disk_path)
                os.makedirs(disk_path)
            mtime = 0
            if 'x-object-meta-mtime' in headers:
                mtime = float(headers['x-object-meta-mtime'])
            elif 'last-modified' in headers:
                mtime = time.mktime(time.strptime(
                    headers['last-modified'], '%a, %d %b %Y %H:%M:%S %Z'))
            if mtime:
                os.utime(disk_path, (mtime, mtime))

        out_path = path
        if context.suppress_container_name:
            out_path = out_path.split('/', 1)[1]
        out_path = context.io_manager.client_path_to_os_path(out_path)
        with context.io_manager.with_stdout(
                out_path, disk_closed_callback=disk_closed_callback) as fp:
            if context.output_headers:
                context.write_headers(
                    fp, headers, context.muted_object_headers)
                fp.write('\n')
            chunk = contents.read(65536)
            while chunk:
                fp.write(chunk)
                chunk = contents.read(65536)
            fp.flush()

class CLIGet(CLICommand):
    """
    A CLICommand that can issue GET requests.

    See the output of ``swiftly help get`` for more information.
    """

    def __init__(self, cli):
        super(CLIGet, self).__init__(
            cli, 'get', max_args=1, usage="""
Usage: %prog [main_options] get [options] [path]

For help on [main_options] run %prog with no args.

Outputs the resulting contents from a GET request of the [path] given. If no
[path] is given, a GET request on the account is performed.""".strip())
        self.option_parser.add_option(
            '--headers', dest='headers', action='store_true',
            help='Output headers as well as the contents.')
        self.option_parser.add_option(
            '-h', '-H', '--header', dest='header', action='append',
            metavar='HEADER:VALUE',
            help='Add a header to the request. This can be used multiple '
                 'times for multiple headers. Examples: '
                 '-hif-match:6f432df40167a4af05ca593acc6b3e4c -h '
                 '"If-Modified-Since: Wed, 23 Nov 2011 20:03:38 GMT"')
        self.option_parser.add_option(
            '-q', '--query', dest='query', action='append',
            metavar='NAME[=VALUE]',
            help='Add a query parameter to the request. This can be used '
                 'multiple times for multiple query parameters. Example: '
                 '-qmultipart-manifest=get')
        self.option_parser.add_option(
            '-l', '--limit', dest='limit',
            help='For account and container GETs, this limits the number of '
                 'items returned. Without this option, all items are '
                 'returned, even if it requires several backend requests to '
                 'the gather the information.')
        self.option_parser.add_option(
            '-d', '--delimiter', dest='delimiter',
            help='For account and container GETs, this sets the delimiter for '
                 'the listing retrieved. For example, a container with the '
                 'objects "abc/one", "abc/two", "xyz" and a delimiter of "/" '
                 'would return "abc/" and "xyz". Using the same delimiter, '
                 'but with a prefix of "abc/", would return "abc/one" and '
                 '"abc/two".')
        self.option_parser.add_option(
            '-p', '--prefix', dest='prefix',
            help='For account and container GETs, this sets the prefix for '
                 'the listing retrieved; the items returned will all match '
                 'the PREFIX given.')
        self.option_parser.add_option(
            '-m', '--marker', dest='marker',
            help='For account and container GETs, this sets the marker for '
                 'the listing retrieved; the items returned will begin with '
                 'the item just after the MARKER given (note: the marker does '
                 'not have to actually exist).')
        self.option_parser.add_option(
            '-e', '--end-marker', dest='end_marker', metavar='MARKER',
            help='For account and container GETs, this sets the end-marker '
                 'for the listing retrieved; the items returned will stop '
                 'with the item just before the MARKER given (note: the '
                 'marker does not have to actually exist).')
        self.option_parser.add_option(
            '-f', '--full', dest='full', action="store_true",
            help='For account and container GETs, this will output additional '
                 'information about each item. For an account GET, the items '
                 'output will be bytes-used, object-count, and '
                 'container-name. For a container GET, the items output will '
                 'be bytes-used, last-modified-time, etag, content-type, and '
                 'object-name. Note that this is mostly useless for --cdn '
                 'queries; for those it is best to just use --raw and parse '
                 'the results yourself (perhaps through "python -m '
                 'json.tool").')
        self.option_parser.add_option(
            '-r', '--raw', dest='raw', action="store_true",
            help='For account and container GETs, this will return the raw '
                 'JSON from the request. This will only do one request, even '
                 'if subsequent requests would be needed to return all items. '
                 'Use a subsequent call with --marker set to the last item '
                 'name returned to get the next batch of items, if desired.')
        self.option_parser.add_option(
            '--all-objects', dest='all_objects', action="store_true",
            help='For an account GET, performs a container GET --all-objects '
                 'for every container returned by the original account GET. '
                 'For a container GET, performs a GET for every object '
                 'returned by that original container GET. Any headers set '
                 'with --header options are sent for every GET. Any query '
                 'parameter set with --query is sent for every GET.')
        self.option_parser.add_option(
            '-o', '--output', dest='output', metavar='PATH',
            help='Indicates where to send the output; default is standard '
                 'output. If the PATH ends with a slash "/" and --all-objects '
                 'is used, each object will be placed in a similarly named '
                 'file inside the PATH given.')
        self.option_parser.add_option(
            '--ignore-404', dest='ignore_404', action='store_true',
            help='Ignores 404 Not Found responses. Nothing will be output, '
                 'but the exit code will be 0 instead of 1.')
        self.option_parser.add_option(
            '--sub-command', dest='sub_command', metavar='COMMAND',
            help='Sends the contents of each object downloaded as standard '
                 'input to the COMMAND given and outputs the command\'s '
                 'standard output as if it were the object\'s contents. This '
                 'can be useful in combination with --all-objects to filter '
                 'the objects before writing them to disk; for instance, '
                 'downloading logs, gunzipping them, grepping for a keyword, '
                 'and only storing matching lines locally (--sub-command '
                 '"gunzip | grep keyword" or --sub-command "zgrep keyword" if '
                 'your system has that).')
        self.option_parser.add_option(
            '--remove-empty-files', dest='remove_empty_files',
            action='store_true',
            help='Removes files that result as empty. This can be useful in '
                 'conjunction with --sub-command so you are left only with '
                 'the files that generated output.')
        self.option_parser.add_option(
            '--decrypt', dest='decrypt', metavar='KEY',
            help='Will decrypt the downloaded object data with KEY. This '
                 'currently only supports AES 256 in CBC mode but other '
                 'algorithms may be offered in the future. You may specify a '
                 'single dash "-" as the KEY and instead the KEY will be '
                 'loaded from the SWIFTLY_CRYPT_KEY environment variable.')

    def __call__(self, args):
        options, args, context = self.parse_args_and_create_context(args)
        if options.output:
            if options.output.endswith(os.path.sep):
                context.io_manager.stdout_root = options.output
            else:
                context.io_manager.stdout = open(options.output, 'wb')
                context.io_manager.stdout_root = None
                context.concurrency = 1
        context.io_manager.stdout_sub_command = options.sub_command
        context.output_headers = options.headers
        context.headers = self.options_list_to_lowered_dict(options.header)
        context.query = self.options_list_to_lowered_dict(options.query)
        context.raw = options.raw
        context.ignore_404 = options.ignore_404
        context.all_objects = options.all_objects
        context.full = options.full
        context.remove_empty_files = options.remove_empty_files
        if options.limit:
            context.query['limit'] = int(options.limit)
        if options.delimiter:
            context.query['delimiter'] = options.delimiter
        if options.prefix:
            context.query['prefix'] = options.prefix
        if options.marker:
            context.query['marker'] = options.marker
        if options.end_marker:
            context.query['end_marker'] = options.end_marker
        context.decrypt = options.decrypt
        if context.decrypt == '-':
            context.decrypt = os.environ.get('SWIFTLY_CRYPT_KEY')
            if not context.decrypt:
                raise ReturnCode(
                    'A single dash "-" was given as the decryption key, but '
                    'no key was found in the SWIFTLY_CRYPT_KEY environment '
                    'variable.')
        path = args.pop(0).lstrip('/') if args else None
        return cli_get(context, path)


"""A client that uses the local file system pretending to be Swift.
"""
"""
Copyright 2014 Gregory Holt

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import six
from six import moves
from contextlib import contextmanager
from errno import EAGAIN
from fcntl import flock, LOCK_EX, LOCK_NB
from json import dumps, loads
from os import close as os_close, listdir, mkdir, open as os_open, O_CREAT, \
    O_WRONLY, rename, rmdir, unlink
from os.path import exists, getsize, isdir, isfile, join as path_join, \
    sep as path_sep
from sqlite3 import connect, Row
from six.moves import StringIO
from time import time
from uuid import uuid4

from swiftly.client.client import Client
from swiftly.client.utils import quote

try:
    from eventlet import sleep
except ImportError:
    from time import sleep

SUBS = [
    ('_', '__'),
    ('.', '_.'),
    ('/', '_s'),
    ('\\', '_b'),
    (':', '_c'),
    ('*', '_S'),
    ("'", '_q'),
    ('"', '_d'),
    ('?', '_Q'),
    ('<', '_l'),
    ('>', '_g'),
    ('|', '_p')]
"""The list of strings in names to substitute for."""
# Note that _- is reserved for use as the start of internal data file names.

def _encode_name(name):
    for a, b in SUBS:
        name = name.replace(a, b)
    return name

def _decode_name(name):
    for a, b in SUBS:
        name = name.replace(b, a)
    return name

@contextmanager
def lock_dir(path):
    path = path_join(path, '_-lock')
    fd = os_open(path, O_WRONLY | O_CREAT, 0o0600)
    try:
        for x in moves.range(100):
            try:
                flock(fd, LOCK_EX | LOCK_NB)
                break
            except IOError as err:
                if err.errno != EAGAIN:
                    raise
            sleep(0.1)
        else:
            raise Exception('Timeout 10s trying to get lock on %r' % path)
        yield True
    finally:
        os_close(fd)

class LocalClient(Client):
    """A client that uses the local file system pretending to be Swift.

    .. note::

        This is a really early implementation and no-ops a lot of stuff.
        With time it will become a more complete representation.

    :param local_path: This is where the fake Swift will store its data.
        Default: Current working directory.
    :param chunk_size: Maximum size to read or write at one time.
    :param verbose: Set to a ``func(msg, *args)`` that will be called
        with debug messages. Constructing a string for output can be
        done with msg % args.
    :param verbose_id: Set to a string you wish verbose messages to
        be prepended with; can help in identifying output when
        multiple Clients are in use.
    """

    def __init__(self, local_path=None, chunk_size=65536, verbose=None,
                 verbose_id=''):
        super(LocalClient, self).__init__()
        self.local_path = local_path.rstrip(path_sep) if local_path else '.'
        self.chunk_size = chunk_size
        if verbose:
            self.verbose = lambda m, *a, **k: verbose(
                self._verbose_id + m, *a, **k)
        else:
            self.verbose = lambda *a, **k: None
        self.verbose_id = verbose_id
        self._verbose_id = self.verbose_id
        if self._verbose_id:
            self._verbose_id += ' '

    def request(self, method, path, contents, headers, decode_json=False,
                stream=False, query=None, cdn=False):
        """
        See :py:func:`swiftly.client.client.Client.request`
        """
        if cdn:
            raise Exception('CDN not yet supported with LocalClient')
        if isinstance(contents, six.string_types):
            contents = StringIO(contents)
        if not headers:
            headers = {}
        if not query:
            query = {}
        rpath = path.lstrip('/')
        if '/' in rpath:
            container_name, object_name = rpath.split('/', 1)
        else:
            container_name = rpath
            object_name = ''
        if not container_name:
            status, reason, hdrs, body = self._account(
                method, contents, headers, stream, query, cdn)
        elif not object_name:
            status, reason, hdrs, body = self._container(
                method, container_name, contents, headers, stream, query, cdn)
        else:
            status, reason, hdrs, body = self._object(
                method, container_name, object_name, contents, headers, stream,
                query, cdn)
        if status and status // 100 != 5:
            if not stream and decode_json and status // 100 == 2:
                if body:
                    body = loads(body)
                else:
                    body = None
            return (status, reason, hdrs, body)
        raise Exception('%s %s failed: %s %s' % (method, path, status, reason))

    def _connect(self, db_path):
        db = connect(db_path)
        db.row_factory = Row
        db.text_factory = str
        db.executescript('''
            PRAGMA synchronous = NORMAL;
            PRAGMA count_changes = OFF;
            PRAGMA temp_store = MEMORY;
            PRAGMA journal_mode = DELETE;
        ''')
        return db

    def _get_db(self):
        db_path = path_join(self.local_path, '_-account.db')
        if isfile(db_path):
            return self._connect(db_path)
        with lock_dir(self.local_path):
            if isfile(db_path):
                return self._connect(db_path)
            temp_path = path_join(self.local_path, '_-temp-account.db')
            db = self._connect(temp_path)
            db.executescript('''
                CREATE TABLE account_entry (
                    container_count INTEGER,
                    object_count INTEGER,
                    byte_count INTEGER);

                INSERT INTO account_entry (
                    container_count, object_count, byte_count)
                VALUES (0, 0, 0);

                CREATE TABLE container_entry (
                    container_name TEXT PRIMARY KEY,
                    object_count INTEGER,
                    byte_count INTEGER);

                CREATE TRIGGER container_insert
                AFTER INSERT
                ON container_entry
                BEGIN
                    UPDATE account_entry
                    SET container_count = container_count + 1,
                        object_count = object_count + new.object_count,
                        byte_count = byte_count + new.byte_count;
                END;

                CREATE TRIGGER container_update
                AFTER UPDATE
                ON container_entry
                BEGIN
                    UPDATE account_entry
                    SET object_count = object_count + (
                            new.object_count - old.object_count),
                        byte_count = byte_count + (
                            new.byte_count - old.byte_count);
                END;

                CREATE TRIGGER container_delete
                AFTER DELETE
                ON container_entry
                BEGIN
                    UPDATE account_entry
                    SET container_count = container_count - 1,
                        object_count = object_count - old.object_count,
                        byte_count = byte_count - old.byte_count;
                END;

                CREATE TABLE object_entry (
                    container_name TEXT,
                    object_name TEXT,
                    put_timestamp TEXT,
                    byte_count INTEGER);

                CREATE UNIQUE INDEX object_entry_primary_key
                ON object_entry (container_name, object_name);

                CREATE TRIGGER object_insert
                AFTER INSERT
                ON object_entry
                BEGIN
                    UPDATE container_entry
                    SET object_count = object_count + 1,
                        byte_count = byte_count + new.byte_count
                    WHERE container_name = new.container_name;
                END;

                CREATE TRIGGER object_update
                AFTER UPDATE
                ON object_entry
                BEGIN
                    UPDATE container_entry
                    SET byte_count = byte_count + (
                            new.byte_count - old.byte_count)
                    WHERE container_name = new.container_name;
                END;

                CREATE TRIGGER object_delete
                AFTER DELETE
                ON object_entry
                BEGIN
                    UPDATE container_entry
                    SET object_count = object_count - 1,
                        byte_count = byte_count - old.byte_count
                    WHERE container_name = old.container_name;
                END;
            ''')
            db.commit()
            db.close()
            if isfile(db_path):
                unlink(temp_path)
            else:
                rename(temp_path, db_path)
        return self._connect(db_path)

    def _account(self, method, contents, headers, stream, query, cdn):
        if cdn:
            raise Exception('CDN not yet supported with LocalClient')
        status = 503
        reason = 'Internal Server Error'
        hdrs = {}
        body = ''
        if method in ('GET', 'HEAD'):
            db = self._get_db()
            prefix = query.get('prefix')
            delimiter = query.get('delimiter')
            marker = query.get('marker')
            end_marker = query.get('end_marker')
            limit = int(query.get('limit') or 10000)
            if delimiter and not prefix:
                prefix = ''
            orig_marker = marker
            body = []
            done = False
            while len(body) < limit and not done:
                query = '''
                    SELECT container_name AS name, object_count AS count,
                        byte_count AS bytes
                    FROM container_entry
                '''
                query_args = []
                where = []
                if end_marker:
                    where.append('name < ?')
                    query_args.append(end_marker)
                if marker and marker >= prefix:
                    where.append('name > ?')
                    query_args.append(marker)
                elif prefix:
                    where.append('name >= ?')
                    query_args.append(prefix)
                if where:
                    query += ' WHERE ' + ' AND '.join(where)
                query += ' ORDER BY name LIMIT ? '
                query_args.append(limit - len(body))
                curs = db.execute(query, query_args)
                if prefix is None:
                    body = [dict(r) for r in curs]
                    break
                if not delimiter:
                    if not prefix:
                        body = [dict(r) for r in curs]
                    else:
                        body = [
                            dict(r) for r in curs if r[0].startswith(prefix)]
                    break
                rowcount = 0
                for row in curs:
                    rowcount += 1
                    marker = name = row[0]
                    if len(body) >= limit or not name.startswith(prefix):
                        curs.close()
                        done = True
                        break
                    end = name.find(delimiter, len(prefix))
                    if end > 0:
                        marker = name[:end] + chr(ord(delimiter) + 1)
                        dir_name = name[:end + 1]
                        if dir_name != orig_marker:
                            body.append({'subdir': dir_name})
                        curs.close()
                        break
                    body.append(dict(row))
                if not rowcount:
                    break
            status = 200
            reason = 'OK'
            body = dumps(body)
            hdrs['content-length'] = str(len(body))
            if method == 'HEAD':
                body = ''
            row = db.execute('''
                SELECT container_count, object_count, byte_count
                FROM account_entry
            ''').fetchone()
            hdrs['x-account-container-count'] = row['container_count']
            hdrs['x-account-object-count'] = row['object_count']
            hdrs['x-account-bytes-used'] = row['byte_count']
            db.close()
        if stream:
            return status, reason, hdrs, StringIO(body)
        else:
            return status, reason, hdrs, body

    def _container(self, method, container_name, contents, headers, stream,
                   query, cdn):
        if cdn:
            raise Exception('CDN not yet supported with LocalClient')
        fs_container = _encode_name(container_name)
        status = 503
        reason = 'Internal Server Error'
        hdrs = {}
        body = ''
        if method in ('GET', 'HEAD'):
            local_path = path_join(self.local_path, fs_container)
            if not isdir(local_path):
                status = 404
                reason = 'Not Found'
                body = ''
                hdrs['content-length'] = str(len(body))
            else:
                object_count = 0
                bytes_used = 0
                prefix = query.get('prefix')
                delimiter = query.get('delimiter')
                marker = query.get('marker')
                end_marker = query.get('end_marker')
                limit = query.get('limit')
                objects = {}
                for item in listdir(local_path):
                    item_local_path = path_join(local_path, item)
                    if isfile(item_local_path):
                        object_name = _decode_name(item)
                        object_count += 1
                        object_size = getsize(item_local_path)
                        bytes_used += object_size
                        if prefix and not object_name.startswith(prefix):
                            continue
                        if delimiter:
                            index = object_name.find(
                                delimiter, len(prefix) + 1 if prefix else 0)
                            if index >= 0:
                                object_name = object_name[:index + 1]
                        objects[object_name] = {
                            'name': object_name, 'bytes': object_size}
                objects = sorted(six.itervalues(objects), key=lambda x: x['name'])
                if marker:
                    objects = [o for o in objects if o['name'] > marker]
                if end_marker:
                    objects = [o for o in objects if o['name'] < end_marker]
                if limit:
                    objects = objects[:int(limit)]
                status = 200
                reason = 'OK'
                body = dumps([
                    ({'subdir': o} if o['name'][-1] == delimiter else o)
                    for o in objects])
                hdrs['content-length'] = str(len(body))
                hdrs['x-container-object-count'] = str(object_count)
                hdrs['x-container-bytes-used'] = str(bytes_used)
            if method == 'HEAD':
                body = ''
        elif method == 'PUT':
            fs_container_path = path_join(self.local_path, fs_container)
            if isdir(fs_container_path):
                status = 202
                reason = 'Accepted'
            else:
                db = self._get_db()
                with lock_dir(self.local_path):
                    if isdir(fs_container_path):
                        status = 202
                        reason = 'Accepted'
                    else:
                        mkdir(fs_container_path)
                        db.execute('''
                            INSERT INTO container_entry (
                                container_name, object_count, byte_count)
                            VALUES (?, 0, 0)
                        ''', (container_name,))
                        db.commit()
                        db.close()
                        status = 201
                        reason = 'Created'
            body = ''
            hdrs['content-length'] = str(len(body))
        elif method == 'POST':
            status = 204
            reason = 'No Content'
            body = ''
            hdrs['content-length'] = str(len(body))
        elif method == 'DELETE':
            fs_container_path = path_join(self.local_path, fs_container)
            if not isdir(fs_container_path):
                status = 404
                reason = 'Not Found'
            else:
                db = self._get_db()
                with lock_dir(self.local_path):
                    if not isdir(fs_container_path):
                        status = 404
                        reason = 'Not Found'
                    else:
                        rmdir(fs_container_path)
                        db.execute('''
                            DELETE FROM container_entry
                            WHERE container_name = ?
                        ''', (container_name,))
                        db.commit()
                        status = 204
                        reason = 'No Content'
            body = ''
            hdrs['content-length'] = str(len(body))
        if stream:
            return status, reason, hdrs, StringIO(body)
        else:
            return status, reason, hdrs, body

    def _object(self, method, container_name, object_name, contents, headers,
                stream, query, cdn):
        if cdn:
            raise Exception('CDN not yet supported with LocalClient')
        fs_container = _encode_name(container_name)
        fs_object = _encode_name(object_name)
        status = 503
        reason = 'Internal Server Error'
        hdrs = {}
        body = ''
        if method in ('GET', 'HEAD'):
            local_path = path_join(self.local_path, fs_container, fs_object)
            if not exists(local_path):
                status = 404
                reason = 'Not Found'
            else:
                content_length = getsize(local_path)
                hdrs['content-length'] = str(content_length)
                status = 200 if content_length else 204
                if method == 'HEAD':
                    body = ''
                else:
                    body = open(local_path, 'rb')
                    if not stream:
                        body = body.read()
        elif method == 'PUT':
            fs_object_path = path_join(
                self.local_path, fs_container, fs_object)
            temp_path = path_join(
                self.local_path, fs_container, '_-temp' + uuid4().hex)
            content_length = headers.get('content-length')
            if content_length is not None:
                content_length = int(content_length)
            fp = open(temp_path, 'wb')
            left = content_length
            written = 0
            while left is None or left > 0:
                if left is not None:
                    chunk = contents.read(max(left, self.chunk_size))
                    left -= len(chunk)
                else:
                    chunk = contents.read(self.chunk_size)
                if not chunk:
                    break
                fp.write(chunk)
                written += len(chunk)
            fp.flush()
            fp.close()
            if content_length is not None and written != content_length:
                unlink(temp_path)
                status = 503
                reason = 'Internal Server Error'
                body = 'Wrote %d bytes when Content-Length was %d' % (
                    written, content_length)
            else:
                db = self._get_db()
                with lock_dir(self.local_path):
                    if isfile(fs_object_path):
                        rename(temp_path, fs_object_path)
                        db.execute('''
                            UPDATE object_entry
                            SET put_timestamp = ?, byte_count = ?
                            WHERE container_name = ? AND object_name = ?
                        ''', (time(), written, container_name, object_name))
                    else:
                        rename(temp_path, fs_object_path)
                        db.execute('''
                            INSERT INTO object_entry (
                                container_name, object_name, put_timestamp,
                                byte_count)
                            VALUES (?, ?, ?, ?)
                        ''', (container_name, object_name, time(), written))
                    db.commit()
                status = 201
                reason = 'Created'
                body = ''
            hdrs['content-length'] = str(len(body))
        elif method == 'DELETE':
            fs_object_path = path_join(
                self.local_path, fs_container, fs_object)
            if not isfile(fs_object_path):
                status = 404
                reason = 'Not Found'
            else:
                db = self._get_db()
                with lock_dir(self.local_path):
                    if not isfile(fs_object_path):
                        status = 404
                        reason = 'Not Found'
                    else:
                        unlink(fs_object_path)
                        db.execute('''
                            DELETE FROM object_entry
                            WHERE container_name = ? AND object_name = ?
                        ''', (container_name, object_name))
                        db.commit()
                        status = 204
                        reason = 'No Content'
            body = ''
            hdrs['content-length'] = str(len(body))
        if stream and not hasattr(body, 'read'):
            body = StringIO(body)
        elif not stream and hasattr(body, 'read'):
            body = body.read()
        return status, reason, hdrs, body

    def get_account_hash(self):
        """
        See :py:func:`swiftly.client.client.Client.get_account_hash`
        """
        return quote(self.local_path, safe='')