def create_dataset(self, name, shape=None, dtype=None, data=None,
                       sparse_format=None, indptr_dtype=np.int64, indices_dtype=np.int32,
                       **kwargs):
        """Create 3 datasets in a group to represent the sparse array.

        Parameters
        ----------
        sparse_format:
        """
        if isinstance(data, Dataset):
            assert sparse_format is None
            group = self.create_group(name)
            group.attrs['h5sparse_format'] = data.attrs['h5sparse_format']
            group.attrs['h5sparse_shape'] = data.attrs['h5sparse_shape']
            group.create_dataset('data', data=data.h5py_group['data'],
                                 dtype=dtype, **kwargs)
            group.create_dataset('indices', data=data.h5py_group['indices'],
                                 dtype=indices_dtype, **kwargs)
            group.create_dataset('indptr', data=data.h5py_group['indptr'],
                                 dtype=indptr_dtype, **kwargs)
        elif ss.issparse(data):
            if sparse_format is not None:
                format_class = get_format_class(sparse_format)
                data = format_class(data)
            group = self.create_group(name)
            group.attrs['h5sparse_format'] = get_format_str(data)
            group.attrs['h5sparse_shape'] = data.shape
            group.create_dataset('data', data=data.data, dtype=dtype, **kwargs)
            group.create_dataset('indices', data=data.indices, dtype=indices_dtype, **kwargs)
            group.create_dataset('indptr', data=data.indptr, dtype=indptr_dtype, **kwargs)
        elif data is None and sparse_format is not None:
            format_class = get_format_class(sparse_format)
            if dtype is None:
                dtype = np.float64
            if shape is None:
                shape = (0, 0)
            data = format_class(shape, dtype=dtype)
            group = self.create_group(name)
            group.attrs['h5sparse_format'] = get_format_str(data)
            group.attrs['h5sparse_shape'] = data.shape
            group.create_dataset('data', data=data.data, dtype=dtype, **kwargs)
            group.create_dataset('indices', data=data.indices, dtype=indices_dtype, **kwargs)
            group.create_dataset('indptr', data=data.indptr, dtype=indptr_dtype, **kwargs)
        else:
            # forward the arguments to h5py
            assert sparse_format is None
            return super(Group, self).create_dataset(
                name, data=data, shape=shape, dtype=dtype, **kwargs)
        return Dataset(group)

def
create_dataset
(
self
,
name
,
shape
=
,
dtype
=
,
data
=
,
sparse_format
=
,
indptr_dtype
=
np
.
int64
,
indices_dtype
=
np
.
int32
,
*
*
kwargs
)
:
if
isinstance
(
data
,
Dataset
)
:
assert
sparse_format
is
group
=
self
.
create_group
(
name
)
group
.
attrs
[
'h5sparse_format'
]
=
data
.
attrs
[
'h5sparse_format'
]
group
.
attrs
[
'h5sparse_shape'
]
=
data
.
attrs
[
'h5sparse_shape'
]
group
.
create_dataset
(
'data'
,
data
=
data
.
h5py_group
[
'data'
]
,
dtype
=
dtype
,
*
*
kwargs
)
group
.
create_dataset
(
'indices'
,
data
=
data
.
h5py_group
[
'indices'
]
,
dtype
=
indices_dtype
,
*
*
kwargs
)
group
.
create_dataset
(
'indptr'
,
data
=
data
.
h5py_group
[
'indptr'
]
,
dtype
=
indptr_dtype
,
*
*
kwargs
)
elif
ss
.
issparse
(
data
)
:
if
sparse_format
is
not
:
format_class
=
get_format_class
(
sparse_format
)
data
=
format_class
(
data
)
group
=
self
.
create_group
(
name
)
group
.
attrs
[
'h5sparse_format'
]
=
get_format_str
(
data
)
group
.
attrs
[
'h5sparse_shape'
]
=
data
.
shape
group
.
create_dataset
(
'data'
,
data
=
data
.
data
,
dtype
=
dtype
,
*
*
kwargs
)
group
.
create_dataset
(
'indices'
,
data
=
data
.
indices
,
dtype
=
indices_dtype
,
*
*
kwargs
)
group
.
create_dataset
(
'indptr'
,
data
=
data
.
indptr
,
dtype
=
indptr_dtype
,
*
*
kwargs
)
elif
data
is
and
sparse_format
is
not
:
format_class
=
get_format_class
(
sparse_format
)
if
dtype
is
:
dtype
=
np
.
float64
if
shape
is
:
shape
=
(
0
,
0
)
data
=
format_class
(
shape
,
dtype
=
dtype
)
group
=
self
.
create_group
(
name
)
group
.
attrs
[
'h5sparse_format'
]
=
get_format_str
(
data
)
group
.
attrs
[
'h5sparse_shape'
]
=
data
.
shape
group
.
create_dataset
(
'data'
,
data
=
data
.
data
,
dtype
=
dtype
,
*
*
kwargs
)
group
.
create_dataset
(
'indices'
,
data
=
data
.
indices
,
dtype
=
indices_dtype
,
*
*
kwargs
)
group
.
create_dataset
(
'indptr'
,
data
=
data
.
indptr
,
dtype
=
indptr_dtype
,
*
*
kwargs
)
else
:
# forward the arguments to h5py
assert
sparse_format
is
return
super
(
Group
,
self
)
.
create_dataset
(
name
,
data
=
data
,
shape
=
shape
,
dtype
=
dtype
,
*
*
kwargs
)
return
Dataset
(
group
)

Create 3 datasets in a group to represent the sparse array .


def cli_decrypt(context, key):
    """
    Decrypts context.io_manager's stdin and sends that to
    context.io_manager's stdout.

    See :py:mod:`swiftly.cli.decrypt` for context usage information.

    See :py:class:`CLIDecrypt` for more information.
    """
    with context.io_manager.with_stdout() as stdout:
        with context.io_manager.with_stdin() as stdin:
            crypt_type = stdin.read(1)
            if crypt_type == AES256CBC:
                for chunk in aes_decrypt(key, stdin):
                    stdout.write(chunk)
                stdout.flush()
            else:
                raise ReturnCode(
                    'contents encrypted with unsupported type %r' % crypt_type)

Decrypts context . io_manager s stdin and sends that to context . io_manager s stdout .


def get_stdin(self, os_path=None, skip_sub_command=False):
        """
        Returns a stdin-suitable file-like object based on the
        optional os_path and optionally skipping any configured
        sub-command.
        """
        sub_command = None if skip_sub_command else self.stdin_sub_command
        inn, path = self._get_in_and_path(
            self.stdin, self.stdin_root, sub_command, os_path)
        if hasattr(inn, 'stdout'):
            return inn.stdout
        return inn

def get_stdin ( self , os_path = None , skip_sub_command = False ) : sub_command = None if skip_sub_command else self . stdin_sub_command inn , path = self . _get_in_and_path ( self . stdin , self . stdin_root , sub_command , os_path ) if hasattr ( inn , 'stdout' ) : return inn . stdout return inn

Returns a stdin - suitable file - like object based on the optional os_path and optionally skipping any configured sub - command .


def get_stdout(self, os_path=None, skip_sub_command=False):
        """
        Returns a stdout-suitable file-like object based on the
        optional os_path and optionally skipping any configured
        sub-command.
        """
        sub_command = None if skip_sub_command else self.stdout_sub_command
        out, path = self._get_out_and_path(
            self.stdout, self.stdout_root, sub_command, os_path)
        if hasattr(out, 'stdin'):
            return out.stdin
        return out

def get_stdout ( self , os_path = None , skip_sub_command = False ) : sub_command = None if skip_sub_command else self . stdout_sub_command out , path = self . _get_out_and_path ( self . stdout , self . stdout_root , sub_command , os_path ) if hasattr ( out , 'stdin' ) : return out . stdin return out

Returns a stdout - suitable file - like object based on the optional os_path and optionally skipping any configured sub - command .


def get_stderr(self, os_path=None, skip_sub_command=False):
        """
        Returns a stderr-suitable file-like object based on the
        optional os_path and optionally skipping any configured
        sub-command.
        """
        sub_command = None if skip_sub_command else self.stderr_sub_command
        out, path = self._get_out_and_path(
            self.stderr, self.stderr_root, sub_command, os_path)
        if hasattr(out, 'stdin'):
            return out.stdin
        return out

def get_stderr ( self , os_path = None , skip_sub_command = False ) : sub_command = None if skip_sub_command else self . stderr_sub_command out , path = self . _get_out_and_path ( self . stderr , self . stderr_root , sub_command , os_path ) if hasattr ( out , 'stdin' ) : return out . stdin return out

Returns a stderr - suitable file - like object based on the optional os_path and optionally skipping any configured sub - command .


def get_debug(self, os_path=None, skip_sub_command=False):
        """
        Returns a debug-output-suitable file-like object based on the
        optional os_path and optionally skipping any configured
        sub-command.
        """
        sub_command = None if skip_sub_command else self.debug_sub_command
        out, path = self._get_out_and_path(
            self.debug, self.debug_root, sub_command, os_path)
        if hasattr(out, 'stdin'):
            return out.stdin
        return out

def get_debug ( self , os_path = None , skip_sub_command = False ) : sub_command = None if skip_sub_command else self . debug_sub_command out , path = self . _get_out_and_path ( self . debug , self . debug_root , sub_command , os_path ) if hasattr ( out , 'stdin' ) : return out . stdin return out

Returns a debug - output - suitable file - like object based on the optional os_path and optionally skipping any configured sub - command .


def with_stdin(self, os_path=None, skip_sub_command=False,
                   disk_closed_callback=None):
        """
        A context manager yielding a stdin-suitable file-like object
        based on the optional os_path and optionally skipping any
        configured sub-command.

        :param os_path: Optional path to base the file-like object
            on.
        :param skip_sub_command: Set True to skip any configured
            sub-command filter.
        :param disk_closed_callback: If the backing of the file-like
            object is an actual file that will be closed,
            disk_closed_callback (if set) will be called with the
            on-disk path just after closing it.
        """
        sub_command = None if skip_sub_command else self.stdin_sub_command
        inn, path = self._get_in_and_path(
            self.stdin, self.stdin_root, sub_command, os_path)
        try:
            if hasattr(inn, 'stdout'):
                yield inn.stdout
            else:
                yield inn
        finally:
            if hasattr(inn, 'stdout'):
                self._close(inn.stdout)
            self._wait(inn, path)
            self._close(inn)
            if disk_closed_callback and path:
                disk_closed_callback(path)

def with_stdin ( self , os_path = None , skip_sub_command = False , disk_closed_callback = None ) : sub_command = None if skip_sub_command else self . stdin_sub_command inn , path = self . _get_in_and_path ( self . stdin , self . stdin_root , sub_command , os_path ) try : if hasattr ( inn , 'stdout' ) : yield inn . stdout else : yield inn finally : if hasattr ( inn , 'stdout' ) : self . _close ( inn . stdout ) self . _wait ( inn , path ) self . _close ( inn ) if disk_closed_callback and path : disk_closed_callback ( path )

A context manager yielding a stdin - suitable file - like object based on the optional os_path and optionally skipping any configured sub - command .


def with_stdout(self, os_path=None, skip_sub_command=False,
                    disk_closed_callback=None):
        """
        A context manager yielding a stdout-suitable file-like object
        based on the optional os_path and optionally skipping any
        configured sub-command.

        :param os_path: Optional path to base the file-like object
            on.
        :param skip_sub_command: Set True to skip any configured
            sub-command filter.
        :param disk_closed_callback: If the backing of the file-like
            object is an actual file that will be closed,
            disk_closed_callback (if set) will be called with the
            on-disk path just after closing it.
        """
        sub_command = None if skip_sub_command else self.stdout_sub_command
        out, path = self._get_out_and_path(
            self.stdout, self.stdout_root, sub_command, os_path)
        try:
            if hasattr(out, 'stdin'):
                yield out.stdin
            else:
                yield out
        finally:
            if hasattr(out, 'stdin'):
                self._close(out.stdin)
            self._wait(out, path)
            self._close(out)
            if disk_closed_callback and path:
                disk_closed_callback(path)

def with_stdout ( self , os_path = None , skip_sub_command = False , disk_closed_callback = None ) : sub_command = None if skip_sub_command else self . stdout_sub_command out , path = self . _get_out_and_path ( self . stdout , self . stdout_root , sub_command , os_path ) try : if hasattr ( out , 'stdin' ) : yield out . stdin else : yield out finally : if hasattr ( out , 'stdin' ) : self . _close ( out . stdin ) self . _wait ( out , path ) self . _close ( out ) if disk_closed_callback and path : disk_closed_callback ( path )

A context manager yielding a stdout - suitable file - like object based on the optional os_path and optionally skipping any configured sub - command .


def with_stderr(self, os_path=None, skip_sub_command=False,
                    disk_closed_callback=None):
        """
        A context manager yielding a stderr-suitable file-like object
        based on the optional os_path and optionally skipping any
        configured sub-command.

        :param os_path: Optional path to base the file-like object
            on.
        :param skip_sub_command: Set True to skip any configured
            sub-command filter.
        :param disk_closed_callback: If the backing of the file-like
            object is an actual file that will be closed,
            disk_closed_callback (if set) will be called with the
            on-disk path just after closing it.
        """
        sub_command = None if skip_sub_command else self.stderr_sub_command
        out, path = self._get_out_and_path(
            self.stderr, self.stderr_root, sub_command, os_path)
        try:
            if hasattr(out, 'stdin'):
                yield out.stdin
            else:
                yield out
        finally:
            if hasattr(out, 'stdin'):
                self._close(out.stdin)
            self._wait(out, path)
            self._close(out)
            if disk_closed_callback and path:
                disk_closed_callback(path)

def with_stderr ( self , os_path = None , skip_sub_command = False , disk_closed_callback = None ) : sub_command = None if skip_sub_command else self . stderr_sub_command out , path = self . _get_out_and_path ( self . stderr , self . stderr_root , sub_command , os_path ) try : if hasattr ( out , 'stdin' ) : yield out . stdin else : yield out finally : if hasattr ( out , 'stdin' ) : self . _close ( out . stdin ) self . _wait ( out , path ) self . _close ( out ) if disk_closed_callback and path : disk_closed_callback ( path )

A context manager yielding a stderr - suitable file - like object based on the optional os_path and optionally skipping any configured sub - command .


def with_debug(self, os_path=None, skip_sub_command=False,
                   disk_closed_callback=None):
        """
        A context manager yielding a debug-output-suitable file-like
        object based on the optional os_path and optionally skipping
        any configured sub-command.

        :param os_path: Optional path to base the file-like object
            on.
        :param skip_sub_command: Set True to skip any configured
            sub-command filter.
        :param disk_closed_callback: If the backing of the file-like
            object is an actual file that will be closed,
            disk_closed_callback (if set) will be called with the
            on-disk path just after closing it.
        """
        sub_command = None if skip_sub_command else self.debug_sub_command
        out, path = self._get_out_and_path(
            self.debug, self.debug_root, sub_command, os_path)
        try:
            if hasattr(out, 'stdin'):
                yield out.stdin
            else:
                yield out
        finally:
            if hasattr(out, 'stdin'):
                self._close(out.stdin)
            self._wait(out, path)
            self._close(out)
            if disk_closed_callback and path:
                disk_closed_callback(path)

def with_debug ( self , os_path = None , skip_sub_command = False , disk_closed_callback = None ) : sub_command = None if skip_sub_command else self . debug_sub_command out , path = self . _get_out_and_path ( self . debug , self . debug_root , sub_command , os_path ) try : if hasattr ( out , 'stdin' ) : yield out . stdin else : yield out finally : if hasattr ( out , 'stdin' ) : self . _close ( out . stdin ) self . _wait ( out , path ) self . _close ( out ) if disk_closed_callback and path : disk_closed_callback ( path )

A context manager yielding a debug - output - suitable file - like object based on the optional os_path and optionally skipping any configured sub - command .


def cli_empty_account(context, yes_empty_account=False, until_empty=False):
    """
    Deletes all objects and containers in the account.

    You must set yes_empty_account to True to verify you really want to
    do this.

    By default, this will perform one pass at deleting all objects and
    containers; so if objects revert to previous versions or if new
    objects or containers otherwise arise during the process, the
    account may not be empty once done.

    Set `until_empty` to True if you want multiple passes to keep trying
    to fully empty and delete the containers. Note until_empty=True
    could run forever if something else is making new items faster than
    they're being deleted.

    See :py:mod:`swiftly.cli.delete` for context usage information.

    See :py:class:`CLIDelete` for more information.
    """
    if not yes_empty_account:
        raise ReturnCode(
            'called cli_empty_account without setting yes_empty_account=True')
    marker = None
    while True:
        with context.client_manager.with_client() as client:
            status, reason, headers, contents = client.get_account(
                marker=marker, headers=context.headers, query=context.query,
                cdn=context.cdn)
        if status // 100 != 2:
            if status == 404 and context.ignore_404:
                return
            raise ReturnCode('listing account: %s %s' % (status, reason))
        if not contents:
            if until_empty and marker:
                marker = None
                continue
            break
        for item in contents:
            cli_delete(
                context, item['name'], context.headers, recursive=True)
        marker = item['name']

Deletes all objects and containers in the account .


def cli_empty_container(context, path, until_empty=False):
    """
    Deletes all objects in the container.

    By default, this will perform one pass at deleting all objects in
    the container; so if objects revert to previous versions or if new
    objects otherwise arise during the process, the container may not be
    empty once done.

    Set `until_empty` to True if you want multiple passes to keep trying
    to fully empty the container. Note until_empty=True could run
    forever if something else is making new objects faster than they're
    being deleted.

    See :py:mod:`swiftly.cli.delete` for context usage information.

    See :py:class:`CLIDelete` for more information.
    """
    path = path.rstrip('/').decode('utf8')
    conc = Concurrency(context.concurrency)

    def check_conc():
        for (exc_type, exc_value, exc_tb, result) in \
                six.itervalues(conc.get_results()):
            if exc_value:
                with context.io_manager.with_stderr() as fp:
                    fp.write(str(exc_value))
                    fp.write('\n')
                    fp.flush()

    marker = None
    while True:
        with context.client_manager.with_client() as client:
            status, reason, headers, contents = client.get_container(
                path, marker=marker, headers=context.headers,
                query=context.query, cdn=context.cdn)
        if status // 100 != 2:
            if status == 404 and context.ignore_404:
                return
            raise ReturnCode(
                'listing container %r: %s %s' % (path, status, reason))
        if not contents:
            if until_empty and marker:
                marker = None
                continue
            break
        for item in contents:
            newpath = '%s/%s' % (path, item['name'])
            new_context = context.copy()
            new_context.ignore_404 = True
            check_conc()
            conc.spawn(newpath, cli_delete, new_context, newpath)
        marker = item['name']
        conc.join()
        check_conc()

def
cli_empty_container
(
context
,
path
,
until_empty
=
False
)
:
path
=
path
.
rstrip
(
'/'
)
.
decode
(
'utf8'
)
conc
=
Concurrency
(
context
.
concurrency
)
def
check_conc
(
)
:
for
(
exc_type
,
exc_value
,
exc_tb
,
result
)
in
six
.
itervalues
(
conc
.
get_results
(
)
)
:
if
exc_value
:
with
context
.
io_manager
.
with_stderr
(
)
as
fp
:
fp
.
write
(
str
(
exc_value
)
)
fp
.
write
(
'\n'
)
fp
.
flush
(
)
marker
=
while
True
:
with
context
.
client_manager
.
with_client
(
)
as
client
:
status
,
reason
,
headers
,
contents
=
client
.
get_container
(
path
,
marker
=
marker
,
headers
=
context
.
headers
,
query
=
context
.
query
,
cdn
=
context
.
cdn
)
if
status
//
100
!=
2
:
if
status
==
404
and
context
.
ignore_404
:
return
raise
ReturnCode
(
'listing container %r: %s %s'
%
(
path
,
status
,
reason
)
)
if
not
contents
:
if
until_empty
and
marker
:
marker
=
continue
break
for
item
in
contents
:
newpath
=
'%s/%s'
%
(
path
,
item
[
'name'
]
)
new_context
=
context
.
copy
(
)
new_context
.
ignore_404
=
True
check_conc
(
)
conc
.
spawn
(
newpath
,
cli_delete
,
new_context
,
newpath
)
marker
=
item
[
'name'
]
conc
.
join
(
)
check_conc
(
)


def _stdout_filed(func):
    """
    Instance method decorator to convert an optional file keyword
    argument into an actual value, whether it be a passed value, a
    value obtained from an io_manager, or sys.stdout.
    """
    def wrapper(self, file=None):
        if file:
            return func(self, file=file)
        elif self.io_manager:
            with self.io_manager.with_stdout() as stdout:
                return func(self, file=stdout)
        else:
            return func(self, file=sys.stdout)
    wrapper.__doc__ = func.__doc__
    return wrapper

def _stdout_filed ( func ) : def wrapper ( self , file = None ) : if file : return func ( self , file = file ) elif self . io_manager : with self . io_manager . with_stdout ( ) as stdout : return func ( self , file = stdout ) else : return func ( self , file = sys . stdout ) wrapper . __doc__ = func . __doc__ return wrapper

Instance method decorator to convert an optional file keyword argument into an actual value whether it be a passed value a value obtained from an io_manager or sys . stdout .


def _stderr_filed(func):
    """
    Instance method decorator to convert an optional file keyword
    argument into an actual value, whether it be a passed value, a
    value obtained from an io_manager, or sys.stderr.
    """
    def wrapper(self, msg, file=None):
        if file:
            return func(self, msg, file=file)
        elif self.io_manager:
            with self.io_manager.with_stderr() as stderr:
                return func(self, msg, file=stderr)
        else:
            return func(self, msg, file=sys.stderr)
    wrapper.__doc__ = func.__doc__
    return wrapper

def _stderr_filed ( func ) : def wrapper ( self , msg , file = None ) : if file : return func ( self , msg , file = file ) elif self . io_manager : with self . io_manager . with_stderr ( ) as stderr : return func ( self , msg , file = stderr ) else : return func ( self , msg , file = sys . stderr ) wrapper . __doc__ = func . __doc__ return wrapper

Instance method decorator to convert an optional file keyword argument into an actual value whether it be a passed value a value obtained from an io_manager or sys . stderr .


def error(self, msg, file=None):
        """
        Outputs the error msg to the file if specified, or to the
        io_manager's stderr if available, or to sys.stderr.
        """
        self.error_encountered = True
        file.write(self.error_prefix)
        file.write(msg)
        file.write('\n')
        file.flush()

def error ( self , msg , file = None ) : self . error_encountered = True file . write ( self . error_prefix ) file . write ( msg ) file . write ( '\n' ) file . flush ( )

Outputs the error msg to the file if specified or to the io_manager s stderr if available or to sys . stderr .


def print_help(self, file=None):
        """
        Outputs help information to the file if specified, or to the
        io_manager's stdout if available, or to sys.stdout.
        """
        optparse.OptionParser.print_help(self, file)
        if self.raw_epilog:
            file.write(self.raw_epilog)
        file.flush()

def print_help ( self , file = None ) : optparse . OptionParser . print_help ( self , file ) if self . raw_epilog : file . write ( self . raw_epilog ) file . flush ( )

Outputs help information to the file if specified or to the io_manager s stdout if available or to sys . stdout .


def print_usage(self, file=None):
        """
        Outputs usage information to the file if specified, or to the
        io_manager's stdout if available, or to sys.stdout.
        """
        optparse.OptionParser.print_usage(self, file)
        file.flush()

def print_usage ( self , file = None ) : optparse . OptionParser . print_usage ( self , file ) file . flush ( )

Outputs usage information to the file if specified or to the io_manager s stdout if available or to sys . stdout .


def print_version(self, file=None):
        """
        Outputs version information to the file if specified, or to
        the io_manager's stdout if available, or to sys.stdout.
        """
        optparse.OptionParser.print_version(self, file)
        file.flush()

def print_version ( self , file = None ) : optparse . OptionParser . print_version ( self , file ) file . flush ( )

Outputs version information to the file if specified or to the io_manager s stdout if available or to sys . stdout .


def request(self, method, path, contents, headers, decode_json=False,
                stream=False, query=None, cdn=False):
        """
        Performs a direct HTTP request to the Swift service.

        :param method: The request method ('GET', 'HEAD', etc.)
        :param path: The request path.
        :param contents: The body of the request. May be a string or
            a file-like object.
        :param headers: A dict of request headers and values.
        :param decode_json: If set True, the response body will be
            treated as JSON and decoded result returned instead of
            the raw contents.
        :param stream: If set True, the response body will return as
            a file-like object; otherwise, the response body will be
            read in its entirety and returned as a string. Overrides
            decode_json.
        :param query: A dict of query parameters and values to append
            to the path.
        :param cdn: If set True, the request will be sent to the CDN
            management endpoint instead of the default storage
            endpoint.
        :returns: A tuple of (status, reason, headers, contents).

            :status: An int for the HTTP status code.
            :reason: The str for the HTTP status (ex: "Ok").
            :headers: A dict with all lowercase keys of the HTTP
                headers; if a header has multiple values, it will be
                a list.
            :contents: Depending on the decode_json and stream
                settings, this will either be the raw response
                string, the JSON decoded object, or a file-like
                object.
        """
        raise Exception('request method not implemented')

def
request
(
self
,
method
,
path
,
contents
,
headers
,
decode_json
=
False
,
stream
=
False
,
query
=
,
cdn
=
False
)
:
raise
Exception
(
'request method not implemented'
)

Performs a direct HTTP request to the Swift service .


def post_account(self, headers=None, query=None, cdn=False, body=None):
        """
        POSTs the account and returns the results. This is usually
        done to set X-Account-Meta-xxx headers. Note that any existing
        X-Account-Meta-xxx headers will remain untouched. To remove an
        X-Account-Meta-xxx header, send the header with an empty
        string as its value.

        :param headers: Additional headers to send with the request.
        :param query: Set to a dict of query values to send on the
            query string of the request.
        :param cdn: If set True, the CDN management interface will be
            used.
        :param body: No known Swift POSTs take a body; but the option
            is there for the future.
        :returns: A tuple of (status, reason, headers, contents).

            :status: is an int for the HTTP status code.
            :reason: is the str for the HTTP status (ex: "Ok").
            :headers: is a dict with all lowercase keys of the HTTP
                headers; if a header has multiple values, it will be a
                list.
            :contents: is the str for the HTTP body.
        """
        return self.request(
            'POST', '', body or '', headers, query=query, cdn=cdn)

def post_account ( self , headers = None , query = None , cdn = False , body = None ) : return self . request ( 'POST' , '' , body or '' , headers , query = query , cdn = cdn )

POSTs the account and returns the results . This is usually done to set X - Account - Meta - xxx headers . Note that any existing X - Account - Meta - xxx headers will remain untouched . To remove an X - Account - Meta - xxx header send the header with an empty string as its value .


def delete_account(self, headers=None,
                       yes_i_mean_delete_the_account=False, query=None,
                       cdn=False, body=None):
        """
        Sends a DELETE request to the account and returns the results.

        With ``query['bulk-delete'] = ''`` this might mean a bulk
        delete request where the body of the request is new-line
        separated, url-encoded list of names to delete. Be careful
        with this! One wrong move and you might mark your account for
        deletion of you have the access to do so!

        For a plain DELETE to the account, on clusters that support
        it and, assuming you have permissions to do so, the account
        will be marked as deleted and immediately begin removing the
        objects from the cluster in the backgound.

        THERE IS NO GOING BACK!

        :param headers: Additional headers to send with the request.
        :param yes_i_mean_delete_the_account: Set to True to verify
            you really mean to delete the entire account. This is
            required unless ``body and 'bulk-delete' in query``.
        :param query: Set to a dict of query values to send on the
            query string of the request.
        :param cdn: If set True, the CDN management interface will be
            used.
        :param body: Some account DELETE requests, like the bulk
            delete request, take a body.
        :returns: A tuple of (status, reason, headers, contents).

            :status: is an int for the HTTP status code.
            :reason: is the str for the HTTP status (ex: "Ok").
            :headers: is a dict with all lowercase keys of the HTTP
                headers; if a header has multiple values, it will be
                a list.
            :contents: is the str for the HTTP body.
        """
        if not yes_i_mean_delete_the_account and (
                not body or not query or 'bulk-delete' not in query):
            return (0, 'yes_i_mean_delete_the_account was not set to True', {},
                    '')
        return self.request(
            'DELETE', '', body or '', headers, query=query, cdn=cdn)

Sends a DELETE request to the account and returns the results .


def put_container(self, container, headers=None, query=None, cdn=False,
                      body=None):
        """
        PUTs the container and returns the results. This is usually
        done to create new containers and can also be used to set
        X-Container-Meta-xxx headers. Note that if the container
        already exists, any existing X-Container-Meta-xxx headers will
        remain untouched. To remove an X-Container-Meta-xxx header,
        send the header with an empty string as its value.

        :param container: The name of the container.
        :param headers: Additional headers to send with the request.
        :param query: Set to a dict of query values to send on the
            query string of the request.
        :param cdn: If set True, the CDN management interface will be
            used.
        :param body: Some container PUT requests, like the
            extract-archive bulk upload request, take a body.
        :returns: A tuple of (status, reason, headers, contents).

            :status: is an int for the HTTP status code.
            :reason: is the str for the HTTP status (ex: "Ok").
            :headers: is a dict with all lowercase keys of the HTTP
                headers; if a header has multiple values, it will be a
                list.
            :contents: is the str for the HTTP body.
        """
        path = self._container_path(container)
        return self.request(
            'PUT', path, body or '', headers, query=query, cdn=cdn)

def put_container ( self , container , headers = None , query = None , cdn = False , body = None ) : path = self . _container_path ( container ) return self . request ( 'PUT' , path , body or '' , headers , query = query , cdn = cdn )

PUTs the container and returns the results . This is usually done to create new containers and can also be used to set X - Container - Meta - xxx headers . Note that if the container already exists any existing X - Container - Meta - xxx headers will remain untouched . To remove an X - Container - Meta - xxx header send the header with an empty string as its value .


def head_object(self, container, obj, headers=None, query=None, cdn=False):
        """
        HEADs the object and returns the results.

        :param container: The name of the container.
        :param obj: The name of the object.
        :param headers: Additional headers to send with the request.
        :param query: Set to a dict of query values to send on the
            query string of the request.
        :param cdn: If set True, the CDN management interface will be
            used.
        :returns: A tuple of (status, reason, headers, contents).

            :status: is an int for the HTTP status code.
            :reason: is the str for the HTTP status (ex: "Ok").
            :headers: is a dict with all lowercase keys of the HTTP
                headers; if a header has multiple values, it will be a
                list.
            :contents: is the str for the HTTP body.
        """
        path = self._object_path(container, obj)
        return self.request(
            'HEAD', path, '', headers, query=query, cdn=cdn)

def head_object ( self , container , obj , headers = None , query = None , cdn = False ) : path = self . _object_path ( container , obj ) return self . request ( 'HEAD' , path , '' , headers , query = query , cdn = cdn )

HEADs the object and returns the results .


def get_object(self, container, obj, headers=None, stream=True, query=None,
                   cdn=False):
        """
        GETs the object and returns the results.

        :param container: The name of the container.
        :param obj: The name of the object.
        :param headers: Additional headers to send with the request.
        :param stream: Indicates whether to stream the contents or
            preread them fully and return them as a str. Default:
            True to stream the contents. When streaming, contents
            will have the standard file-like-object read function,
            which accepts an optional size parameter to limit how
            much data is read per call. When streaming is on, be
            certain to fully read the contents before issuing another
            request.
        :param query: Set to a dict of query values to send on the
            query string of the request.
        :param cdn: If set True, the CDN management interface will be
            used.
        :returns: A tuple of (status, reason, headers, contents).

            :status: is an int for the HTTP status code.
            :reason: is the str for the HTTP status (ex: "Ok").
            :headers: is a dict with all lowercase keys of the HTTP
                headers; if a header has multiple values, it will be a
                list.
            :contents: if *stream* was True, *contents* is a
                file-like-object of the contents of the HTTP body. If
                *stream* was False, *contents* is just a simple str of
                the HTTP body.
        """
        path = self._object_path(container, obj)
        return self.request(
            'GET', path, '', headers, query=query, stream=stream, cdn=cdn)

def get_object ( self , container , obj , headers = None , stream = True , query = None , cdn = False ) : path = self . _object_path ( container , obj ) return self . request ( 'GET' , path , '' , headers , query = query , stream = stream , cdn = cdn )

GETs the object and returns the results .


def put_object(self, container, obj, contents, headers=None, query=None,
                   cdn=False):
        """
        PUTs the object and returns the results. This is used to
        create or overwrite objects. X-Object-Meta-xxx can optionally
        be sent to be stored with the object. Content-Type,
        Content-Encoding and other standard HTTP headers can often
        also be set, depending on the Swift cluster.

        Note that you can set the ETag header to the MD5 sum of the
        contents for extra verification the object was stored
        correctly.

        :param container: The name of the container.
        :param obj: The name of the object.
        :param contents: The contents of the object to store. This can
            be a simple str, or a file-like-object with at least a
            read function. If the file-like-object also has tell and
            seek functions, the PUT can be reattempted on any server
            error.
        :param headers: Additional headers to send with the request.
        :param query: Set to a dict of query values to send on the
            query string of the request.
        :param cdn: If set True, the CDN management interface will be
            used.
        :returns: A tuple of (status, reason, headers, contents).

            :status: is an int for the HTTP status code.
            :reason: is the str for the HTTP status (ex: "Ok").
            :headers: is a dict with all lowercase keys of the HTTP
                headers; if a header has multiple values, it will be a
                list.
            :contents: is the str for the HTTP body.
        """
        path = self._object_path(container, obj)
        return self.request(
            'PUT', path, contents, headers, query=query, cdn=cdn)

def put_object ( self , container , obj , contents , headers = None , query = None , cdn = False ) : path = self . _object_path ( container , obj ) return self . request ( 'PUT' , path , contents , headers , query = query , cdn = cdn )

PUTs the object and returns the results . This is used to create or overwrite objects . X - Object - Meta - xxx can optionally be sent to be stored with the object . Content - Type Content - Encoding and other standard HTTP headers can often also be set depending on the Swift cluster .


def post_object(self, container, obj, headers=None, query=None, cdn=False,
                    body=None):
        """
        POSTs the object and returns the results. This is used to
        update the object's header values. Note that all headers must
        be sent with the POST, unlike the account and container POSTs.
        With account and container POSTs, existing headers are
        untouched. But with object POSTs, any existing headers are
        removed. The full list of supported headers depends on the
        Swift cluster, but usually include Content-Type,
        Content-Encoding, and any X-Object-Meta-xxx headers.

        :param container: The name of the container.
        :param obj: The name of the object.
        :param headers: Additional headers to send with the request.
        :param query: Set to a dict of query values to send on the
            query string of the request.
        :param cdn: If set True, the CDN management interface will be
            used.
        :param body: No known Swift POSTs take a body; but the option
            is there for the future.
        :returns: A tuple of (status, reason, headers, contents).

            :status: is an int for the HTTP status code.
            :reason: is the str for the HTTP status (ex: "Ok").
            :headers: is a dict with all lowercase keys of the HTTP
                headers; if a header has multiple values, it will be a
                list.
            :contents: is the str for the HTTP body.
        """
        path = self._object_path(container, obj)
        return self.request(
            'POST', path, body or '', headers, query=query, cdn=cdn)

def post_object ( self , container , obj , headers = None , query = None , cdn = False , body = None ) : path = self . _object_path ( container , obj ) return self . request ( 'POST' , path , body or '' , headers , query = query , cdn = cdn )

POSTs the object and returns the results . This is used to update the object s header values . Note that all headers must be sent with the POST unlike the account and container POSTs . With account and container POSTs existing headers are untouched . But with object POSTs any existing headers are removed . The full list of supported headers depends on the Swift cluster but usually include Content - Type Content - Encoding and any X - Object - Meta - xxx headers .


def _resolve_option(self, options, option_name, section_name):
        """Resolves an option value into options.

        Sets options.<option_name> to a resolved value. Any value
        already in options overrides a value in os.environ which
        overrides self.context.conf.

        :param options: The options instance as returned by optparse.
        :param option_name: The name of the option, such as
            ``auth_url``.
        :param section_name: The name of the section, such as
            ``swiftly``.
        """
        if getattr(options, option_name, None) is not None:
            return
        if option_name.startswith(section_name + '_'):
            environ_name = option_name.upper()
            conf_name = option_name[len(section_name) + 1:]
        else:
            environ_name = (section_name + '_' + option_name).upper()
            conf_name = option_name
        setattr(
            options, option_name,
            os.environ.get(
                environ_name,
                (self.context.conf.get(section_name, {})).get(conf_name)))

def _resolve_option ( self , options , option_name , section_name ) : if getattr ( options , option_name , None ) is not None : return if option_name . startswith ( section_name + '_' ) : environ_name = option_name . upper ( ) conf_name = option_name [ len ( section_name ) + 1 : ] else : environ_name = ( section_name + '_' + option_name ) . upper ( ) conf_name = option_name setattr ( options , option_name , os . environ . get ( environ_name , ( self . context . conf . get ( section_name , { } ) ) . get ( conf_name ) ) )


def copy(self):
        """
        Returns a new CLIContext instance that is a shallow copy of
        the original, much like dict's copy method.
        """
        context = CLIContext()
        for item in dir(self):
            if item[0] != '_' and item not in ('copy', 'write_headers'):
                setattr(context, item, getattr(self, item))
        return context

def copy ( self ) : context = CLIContext ( ) for item in dir ( self ) : if item [ 0 ] != '_' and item not in ( 'copy' , 'write_headers' ) : setattr ( context , item , getattr ( self , item ) ) return context

Returns a new CLIContext instance that is a shallow copy of the original much like dict s copy method .


def write_headers(self, fp, headers, mute=None):
        """
        Convenience function to output headers in a formatted fashion
        to a file-like fp, optionally muting any headers in the mute
        list.
        """
        if headers:
            if not mute:
                mute = []
            fmt = '%%-%ds %%s\n' % (max(len(k) for k in headers) + 1)
            for key in sorted(headers):
                if key in mute:
                    continue
                fp.write(fmt % (key.title() + ':', headers[key]))
            fp.flush()

def
write_headers
(
self
,
fp
,
headers
,
mute
=
)
:
if
headers
:
if
not
mute
:
mute
=
[
]
fmt
=
'%%-%ds %%s\n'
%
(
max
(
len
(
k
)
for
k
in
headers
)
+
1
)
for
key
in
sorted
(
headers
)
:
if
key
in
mute
:
continue
fp
.
write
(
fmt
%
(
key
.
title
(
)
+
':'
,
headers
[
key
]
)
)
fp
.
flush
(
)

Convenience function to output headers in a formatted fashion to a file - like fp optionally muting any headers in the mute list .


def cli_auth(context):
    """
    Authenticates and then outputs the resulting information.

    See :py:mod:`swiftly.cli.auth` for context usage information.

    See :py:class:`CLIAuth` for more information.
    """
    with context.io_manager.with_stdout() as fp:
        with context.client_manager.with_client() as client:
            info = []
            client.auth()
            if getattr(client, 'auth_cache_path', None):
                info.append(('Auth Cache', client.auth_cache_path))
            if getattr(client, 'auth_url', None):
                info.append(('Auth URL', client.auth_url))
            if getattr(client, 'auth_user', None):
                info.append(('Auth User', client.auth_user))
            if getattr(client, 'auth_key', None):
                info.append(('Auth Key', client.auth_key))
            if getattr(client, 'auth_tenant', None):
                info.append(('Auth Tenant', client.auth_tenant))
            if getattr(client, 'auth_methods', None):
                info.append(('Auth Methods', client.auth_methods))
            if getattr(client, 'storage_path', None):
                info.append(('Direct Storage Path', client.storage_path))
            if getattr(client, 'cdn_path', None):
                info.append(('Direct CDN Path', client.cdn_path))
            if getattr(client, 'local_path', None):
                info.append(('Local Path', client.local_path))
            if getattr(client, 'regions', None):
                info.append(('Regions', ' '.join(client.regions)))
            if getattr(client, 'default_region', None):
                info.append(('Default Region', client.default_region))
            if getattr(client, 'region', None):
                info.append(('Selected Region', client.region))
            if getattr(client, 'snet', None):
                info.append(('SNet', client.snet))
            if getattr(client, 'storage_url', None):
                info.append(('Storage URL', client.storage_url))
            if getattr(client, 'cdn_url', None):
                info.append(('CDN URL', client.cdn_url))
            if getattr(client, 'auth_token', None):
                info.append(('Auth Token', client.auth_token))
            if not info:
                info.append((
                    'No auth information available',
                    'Maybe no credentials were provided?'))
            fmt = '%%-%ds %%s\n' % (max(len(t) for t, v in info) + 1)
            for t, v in info:
                fp.write(fmt % (t + ':', v))
            fp.flush()

def
cli_auth
(
context
)
:
with
context
.
io_manager
.
with_stdout
(
)
as
fp
:
with
context
.
client_manager
.
with_client
(
)
as
client
:
info
=
[
]
client
.
auth
(
)
if
getattr
(
client
,
'auth_cache_path'
,
)
:
info
.
append
(
(
'Auth Cache'
,
client
.
auth_cache_path
)
)
if
getattr
(
client
,
'auth_url'
,
)
:
info
.
append
(
(
'Auth URL'
,
client
.
auth_url
)
)
if
getattr
(
client
,
'auth_user'
,
)
:
info
.
append
(
(
'Auth User'
,
client
.
auth_user
)
)
if
getattr
(
client
,
'auth_key'
,
)
:
info
.
append
(
(
'Auth Key'
,
client
.
auth_key
)
)
if
getattr
(
client
,
'auth_tenant'
,
)
:
info
.
append
(
(
'Auth Tenant'
,
client
.
auth_tenant
)
)
if
getattr
(
client
,
'auth_methods'
,
)
:
info
.
append
(
(
'Auth Methods'
,
client
.
auth_methods
)
)
if
getattr
(
client
,
'storage_path'
,
)
:
info
.
append
(
(
'Direct Storage Path'
,
client
.
storage_path
)
)
if
getattr
(
client
,
'cdn_path'
,
)
:
info
.
append
(
(
'Direct CDN Path'
,
client
.
cdn_path
)
)
if
getattr
(
client
,
'local_path'
,
)
:
info
.
append
(
(
'Local Path'
,
client
.
local_path
)
)
if
getattr
(
client
,
'regions'
,
)
:
info
.
append
(
(
'Regions'
,
' '
.
join
(
client
.
regions
)
)
)
if
getattr
(
client
,
'default_region'
,
)
:
info
.
append
(
(
'Default Region'
,
client
.
default_region
)
)
if
getattr
(
client
,
'region'
,
)
:
info
.
append
(
(
'Selected Region'
,
client
.
region
)
)
if
getattr
(
client
,
'snet'
,
)
:
info
.
append
(
(
'SNet'
,
client
.
snet
)
)
if
getattr
(
client
,
'storage_url'
,
)
:
info
.
append
(
(
'Storage URL'
,
client
.
storage_url
)
)
if
getattr
(
client
,
'cdn_url'
,
)
:
info
.
append
(
(
'CDN URL'
,
client
.
cdn_url
)
)
if
getattr
(
client
,
'auth_token'
,
)
:
info
.
append
(
(
'Auth Token'
,
client
.
auth_token
)
)
if
not
info
:
info
.
append
(
(
'No auth information available'
,
'Maybe no credentials were provided?'
)
)
fmt
=
'%%-%ds %%s\n'
%
(
max
(
len
(
t
)
for
t
,
v
in
info
)
+
1
)
for
t
,
v
in
info
:
fp
.
write
(
fmt
%
(
t
+
':'
,
v
)
)
fp
.
flush
(
)

Authenticates and then outputs the resulting information .


def generate_temp_url(method, url, seconds, key):
    """
    Returns a TempURL good for the given request method, url, and
    number of seconds from now, signed by the given key.
    """
    method = method.upper()
    base_url, object_path = url.split('/v1/')
    object_path = '/v1/' + object_path
    expires = int(time.time() + seconds)
    hmac_body = '%s\n%s\n%s' % (method, expires, object_path)
    sig = hmac.new(key, hmac_body, hashlib.sha1).hexdigest()
    return '%s%s?temp_url_sig=%s&temp_url_expires=%s' % (
        base_url, object_path, sig, expires)

Returns a TempURL good for the given request method url and number of seconds from now signed by the given key .


def quote(value, safe='/:'):
    """
    Much like parse.quote in that it returns a URL encoded string
    for the given value, protecting the safe characters; but this
    version also ensures the value is UTF-8 encoded.
    """
    if isinstance(value, six.text_type):
        value = value.encode('utf8')
    elif not isinstance(value, six.string_types):
        value = str(value)
    return parse.quote(value, safe)

def quote ( value , safe = '/:' ) : if isinstance ( value , six . text_type ) : value = value . encode ( 'utf8' ) elif not isinstance ( value , six . string_types ) : value = str ( value ) return parse . quote ( value , safe )

Much like parse . quote in that it returns a URL encoded string for the given value protecting the safe characters ; but this version also ensures the value is UTF - 8 encoded .


def cli_fordo(context, path=None):
    """
    Issues commands for each item in an account or container listing.

    See :py:mod:`swiftly.cli.fordo` for context usage information.

    See :py:class:`CLIForDo` for more information.
    """
    path = path.lstrip('/') if path else None
    if path and '/' in path:
        raise ReturnCode(
            'path must be an empty string or a container name; was %r' % path)
    limit = context.query.get('limit')
    delimiter = context.query.get('delimiter')
    prefix = context.query.get('prefix')
    marker = context.query.get('marker')
    end_marker = context.query.get('end_marker')
    conc = Concurrency(context.concurrency)
    while True:
        with context.client_manager.with_client() as client:
            if not path:
                status, reason, headers, contents = client.get_account(
                    headers=context.headers, prefix=prefix,
                    delimiter=delimiter, marker=marker, end_marker=end_marker,
                    limit=limit, query=context.query, cdn=context.cdn)
            else:
                status, reason, headers, contents = client.get_container(
                    path, headers=context.headers, prefix=prefix,
                    delimiter=delimiter, marker=marker, end_marker=end_marker,
                    limit=limit, query=context.query, cdn=context.cdn)
            if status // 100 != 2:
                if status == 404 and context.ignore_404:
                    return
                if hasattr(contents, 'read'):
                    contents.read()
                if not path:
                    raise ReturnCode(
                        'listing account: %s %s' % (status, reason))
                else:
                    raise ReturnCode(
                        'listing container %r: %s %s' % (path, status, reason))
        if not contents:
            break
        for item in contents:
            name = (path + '/' if path else '') + item.get(
                'name', item.get('subdir'))
            args = list(context.remaining_args)
            try:
                index = args.index('<item>')
            except ValueError:
                raise ReturnCode(
                    'No "<item>" designation found in the "do" clause.')
            args[index] = name
            for (exc_type, exc_value, exc_tb, result) in \
                    six.itervalues(conc.get_results()):
                if exc_value:
                    conc.join()
                    raise exc_value
            conc.spawn(name, _cli_call, context, name, args)
        marker = contents[-1]['name']
        if limit:
            break
    conc.join()
    for (exc_type, exc_value, exc_tb, result) in \
            six.itervalues(conc.get_results()):
        if exc_value:
            conc.join()
            raise exc_value

Issues commands for each item in an account or container listing .


def get_client(self):
        """
        Obtains a client for use, whether an existing unused client
        or a brand new one if none are available.
        """
        client = None
        try:
            client = self.clients.get(block=False)
        except queue.Empty:
            pass
        if not client:
            self.client_id += 1
            kwargs = dict(self.kwargs)
            kwargs['verbose_id'] = kwargs.get(
                'verbose_id', '') + str(self.client_id)
            client = self.client_class(*self.args, **kwargs)
        return client

def get_client ( self ) : client = None try : client = self . clients . get ( block = False ) except queue . Empty : pass if not client : self . client_id += 1 kwargs = dict ( self . kwargs ) kwargs [ 'verbose_id' ] = kwargs . get ( 'verbose_id' , '' ) + str ( self . client_id ) client = self . client_class ( * self . args , * * kwargs ) return client

Obtains a client for use whether an existing unused client or a brand new one if none are available .


def aes_encrypt(key, stdin, preamble=None, chunk_size=65536,
                content_length=None):
    """
    Generator that encrypts a content stream using AES 256 in CBC
    mode.

    :param key: Any string to use as the encryption key.
    :param stdin: Where to read the contents from.
    :param preamble: str to yield initially useful for providing a
        hint for future readers as to the algorithm in use.
    :param chunk_size: Largest amount to read at once.
    :param content_length: The number of bytes to read from stdin.
        None or < 0 indicates reading until EOF.
    """
    if not AES256CBC_Support:
        raise Exception(
            'AES256CBC not supported; likely pycrypto is not installed')
    if preamble:
        yield preamble
    # Always use 256-bit key
    key = hashlib.sha256(key).digest()
    # At least 16 and a multiple of 16
    chunk_size = max(16, chunk_size >> 4 << 4)
    iv = Crypto.Random.new().read(16)
    yield iv
    encryptor = Crypto.Cipher.AES.new(key, Crypto.Cipher.AES.MODE_CBC, iv)
    reading = True
    left = None
    if content_length is not None and content_length >= 0:
        left = content_length
    while reading:
        size = chunk_size
        if left is not None and size > left:
            size = left
        chunk = stdin.read(size)
        if not chunk:
            if left is not None and left > 0:
                raise IOError('Early EOF from input')
            # Indicates how many usable bytes in last block
            yield encryptor.encrypt('\x00' * 16)
            break
        if left is not None:
            left -= len(chunk)
            if left <= 0:
                reading = False
        block = chunk
        trailing = len(block) % 16
        while trailing:
            size = 16 - trailing
            if left is not None and size > left:
                size = left
            chunk = stdin.read(size)
            if not chunk:
                if left is not None and left > 0:
                    raise IOError('Early EOF from input')
                reading = False
                # Indicates how many usable bytes in last block
                chunk = chr(trailing) * (16 - trailing)
            elif left is not None:
                left -= len(chunk)
                if left <= 0:
                    reading = False
            block += chunk
            trailing = len(block) % 16
        yield encryptor.encrypt(block)

Generator that encrypts a content stream using AES 256 in CBC mode .


def aes_decrypt(key, stdin, chunk_size=65536):
    """
    Generator that decrypts a content stream using AES 256 in CBC
    mode.

    :param key: Any string to use as the decryption key.
    :param stdin: Where to read the encrypted data from.
    :param chunk_size: Largest amount to read at once.
    """
    if not AES256CBC_Support:
        raise Exception(
            'AES256CBC not supported; likely pycrypto is not installed')
    # Always use 256-bit key
    key = hashlib.sha256(key).digest()
    # At least 16 and a multiple of 16
    chunk_size = max(16, chunk_size >> 4 << 4)
    iv = stdin.read(16)
    while len(iv) < 16:
        chunk = stdin.read(16 - len(iv))
        if not chunk:
            raise IOError('EOF reading IV')
    decryptor = Crypto.Cipher.AES.new(key, Crypto.Cipher.AES.MODE_CBC, iv)
    data = ''
    while True:
        chunk = stdin.read(chunk_size)
        if not chunk:
            if len(data) != 16:
                raise IOError('EOF reading encrypted stream')
            data = decryptor.decrypt(data)
            trailing = ord(data[-1])
            if trailing > 15:
                raise IOError(
                    'EOF reading encrypted stream or trailing value corrupted '
                    '%s' % trailing)
            yield data[:trailing]
            break
        data += chunk
        if len(data) > 16:
            # Always leave at least one byte pending
            trailing = (len(data) % 16) or 16
            yield decryptor.decrypt(data[:-trailing])
            data = data[-trailing:]

Generator that decrypts a content stream using AES 256 in CBC mode .


def cli_put_directory_structure(context, path):
    """
    Performs PUTs rooted at the path using a directory structure
    pointed to by context.input\_.

    See :py:mod:`swiftly.cli.put` for context usage information.

    See :py:class:`CLIPut` for more information.
    """
    if not context.input_:
        raise ReturnCode(
            'called cli_put_directory_structure without context.input_ set')
    if not os.path.isdir(context.input_):
        raise ReturnCode(
            '%r is not a directory' % context.input_)
    if not path:
        raise ReturnCode(
            'uploading a directory structure requires at least a container '
            'name')
    new_context = context.copy()
    new_context.input_ = None
    container = path.split('/', 1)[0]
    cli_put_container(new_context, container)
    ilen = len(context.input_)
    if not context.input_.endswith(os.sep):
        ilen += 1
    conc = Concurrency(context.concurrency)
    for (dirpath, dirnames, filenames) in os.walk(context.input_):
        if not dirnames and not filenames:
            new_context = context.copy()
            new_context.headers = dict(context.headers)
            new_context.headers['content-type'] = 'text/directory'
            new_context.headers['x-object-meta-mtime'] = \
                '%f' % os.path.getmtime(context.input_)
            new_context.input_ = None
            new_context.empty = True
            new_path = path
            if path[-1] != '/':
                new_path += '/'
            new_path += dirpath[ilen:]
            for (exc_type, exc_value, exc_tb, result) in \
                    six.itervalues(conc.get_results()):
                if exc_value:
                    conc.join()
                    raise exc_value
            conc.spawn(new_path, cli_put_object, new_context, new_path)
        else:
            for fname in filenames:
                new_context = context.copy()
                new_context.input_ = os.path.join(dirpath, fname)
                new_path = path
                if path[-1] != '/':
                    new_path += '/'
                if dirpath[ilen:]:
                    new_path += dirpath[ilen:] + '/'
                new_path += fname
                for (exc_type, exc_value, exc_tb, result) in \
                        six.itervalues(conc.get_results()):
                    if exc_value:
                        conc.join()
                        raise exc_value
                conc.spawn(new_path, cli_put_object, new_context, new_path)
    conc.join()
    for (exc_type, exc_value, exc_tb, result) in \
            six.itervalues(conc.get_results()):
        if exc_value:
            raise exc_value

Performs PUTs rooted at the path using a directory structure pointed to by context . input \ _ .


def cli_put_account(context):
    """
    Performs a PUT on the account.

    See :py:mod:`swiftly.cli.put` for context usage information.

    See :py:class:`CLIPut` for more information.
    """
    body = None
    if context.input_:
        if context.input_ == '-':
            body = context.io_manager.get_stdin()
        else:
            body = open(context.input_, 'rb')
    with context.client_manager.with_client() as client:
        status, reason, headers, contents = client.put_account(
            headers=context.headers, query=context.query, cdn=context.cdn,
            body=body)
        if hasattr(contents, 'read'):
            contents.read()
    if status // 100 != 2:
        raise ReturnCode('putting account: %s %s' % (status, reason))

def
cli_put_account
(
context
)
:
body
=
if
context
.
input_
:
if
context
.
input_
==
'-'
:
body
=
context
.
io_manager
.
get_stdin
(
)
else
:
body
=
open
(
context
.
input_
,
'rb'
)
with
context
.
client_manager
.
with_client
(
)
as
client
:
status
,
reason
,
headers
,
contents
=
client
.
put_account
(
headers
=
context
.
headers
,
query
=
context
.
query
,
cdn
=
context
.
cdn
,
body
=
body
)
if
hasattr
(
contents
,
'read'
)
:
contents
.
read
(
)
if
status
//
100
!=
2
:
raise
ReturnCode
(
'putting account: %s %s'
%
(
status
,
reason
)
)


def cli_put_container(context, path):
    """
    Performs a PUT on the container.

    See :py:mod:`swiftly.cli.put` for context usage information.

    See :py:class:`CLIPut` for more information.
    """
    path = path.rstrip('/')
    if '/' in path:
        raise ReturnCode('called cli_put_container with object %r' % path)
    body = None
    if context.input_:
        if context.input_ == '-':
            body = context.io_manager.get_stdin()
        else:
            body = open(context.input_, 'rb')
    with context.client_manager.with_client() as client:
        status, reason, headers, contents = client.put_container(
            path, headers=context.headers, query=context.query,
            cdn=context.cdn, body=body)
        if hasattr(contents, 'read'):
            contents.read()
    if status // 100 != 2:
        raise ReturnCode(
            'putting container %r: %s %s' % (path, status, reason))


def _get_manifest_body(context, prefix, path2info, put_headers):
    """
    Returns body for manifest file and modifies put_headers.

    path2info is a dict like {"path": (size, etag)}
    """
    if context.static_segments:
        body = json.dumps([
            {'path': '/' + p, 'size_bytes': s, 'etag': e}
            for p, (s, e) in sorted(six.iteritems(path2info))
        ])
        put_headers['content-length'] = str(len(body))
        context.query['multipart-manifest'] = 'put'
    else:
        body = ''
        put_headers['content-length'] = '0'
        put_headers['x-object-manifest'] = prefix

    return body

def _get_manifest_body ( context , prefix , path2info , put_headers ) : if context . static_segments : body = json . dumps ( [ { 'path' : '/' + p , 'size_bytes' : s , 'etag' : e } for p , ( s , e ) in sorted ( six . iteritems ( path2info ) ) ] ) put_headers [ 'content-length' ] = str ( len ( body ) ) context . query [ 'multipart-manifest' ] = 'put' else : body = '' put_headers [ 'content-length' ] = '0' put_headers [ 'x-object-manifest' ] = prefix return body

Returns body for manifest file and modifies put_headers .


def _create_container(context, path, l_mtime, size):
    """
    Creates container for segments of file with `path`
    """
    new_context = context.copy()
    new_context.input_ = None
    new_context.headers = None
    new_context.query = None
    container = path.split('/', 1)[0] + '_segments'
    cli_put_container(new_context, container)
    prefix = container + '/' + path.split('/', 1)[1]
    prefix = '%s/%s/%s/' % (prefix, l_mtime, size)

    return prefix

def _create_container ( context , path , l_mtime , size ) : new_context = context . copy ( ) new_context . input_ = None new_context . headers = None new_context . query = None container = path . split ( '/' , 1 ) [ 0 ] + '_segments' cli_put_container ( new_context , container ) prefix = container + '/' + path . split ( '/' , 1 ) [ 1 ] prefix = '%s/%s/%s/' % ( prefix , l_mtime , size ) return prefix

Creates container for segments of file with path


def cli_tempurl(context, method, path, seconds=None, use_container=False):
    """
    Generates a TempURL and sends that to the context.io_manager's
    stdout.

    See :py:mod:`swiftly.cli.tempurl` for context usage information.

    See :py:class:`CLITempURL` for more information.

    :param context: The :py:class:`swiftly.cli.context.CLIContext` to
        use.
    :param method: The method for the TempURL (GET, PUT, etc.)
    :param path: The path the TempURL should direct to.
    :param seconds: The number of seconds the TempURL should be good
        for. Default: 3600
    :param use_container: If True, will create a container level TempURL
        useing X-Container-Meta-Temp-Url-Key instead of
        X-Account-Meta-Temp-Url-Key.
    """
    with contextlib.nested(
            context.io_manager.with_stdout(),
            context.client_manager.with_client()) as (fp, client):
        method = method.upper()
        path = path.lstrip('/')
        seconds = seconds if seconds is not None else 3600
        if '/' not in path:
            raise ReturnCode(
                'invalid tempurl path %r; should have a / within it' % path)
        if use_container:
            key_type = 'container'
            container = path.split('/', 1)[0]
            status, reason, headers, contents = \
                client.head_container(container)
        else:
            key_type = 'account'
            status, reason, headers, contents = \
                client.head_account()
        if status // 100 != 2:
            raise ReturnCode(
                'obtaining X-%s-Meta-Temp-Url-Key: %s %s' %
                (key_type.title(), status, reason))
        key = headers.get('x-%s-meta-temp-url-key' % key_type)
        if not key:
            raise ReturnCode(
                'there is no X-%s-Meta-Temp-Url-Key set for this %s' %
                (key_type.title(), key_type))
        url = client.storage_url + '/' + path
        fp.write(generate_temp_url(method, url, seconds, key))
        fp.write('\n')
        fp.flush()

Generates a TempURL and sends that to the context . io_manager s stdout .


def cli_trans(context, x_trans_id):
    """
    Translates any information that can be determined from the
    x_trans_id and sends that to the context.io_manager's stdout.

    See :py:mod:`swiftly.cli.trans` for context usage information.

    See :py:class:`CLITrans` for more information.
    """
    with context.io_manager.with_stdout() as fp:
        trans_time = get_trans_id_time(x_trans_id)
        trans_info = x_trans_id[34:]
        msg = 'X-Trans-Id:      ' + x_trans_id + '\n'
        if not trans_time:
            msg += 'Time Stamp:      None, old style id with no time ' \
                'embedded\nUTC Time:        None, old style id with no time ' \
                'embedded\n'
        else:
            msg += 'Time Stamp:      %s\nUTC Time:        %s\n' % (
                trans_time,
                time.strftime(
                    '%a %Y-%m-%d %H:%M:%S UTC', time.gmtime(trans_time)))
        msg += 'Additional Info: ' + trans_info + '\n'
        fp.write(msg)
        fp.flush()

'Time Stamp:      %s\nUTC Time:        %s\n'

Translates any information that can be determined from the x_trans_id and sends that to the context . io_manager s stdout .


def cli_help(context, command_name, general_parser, command_parsers):
    """
    Outputs help information.

    See :py:mod:`swiftly.cli.help` for context usage information.

    See :py:class:`CLIHelp` for more information.

    :param context: The :py:class:`swiftly.cli.context.CLIContext` to
        use.
    :param command_name: The command_name to output help information
        for, or set to None or an empty string to output the general
        help information.
    :param general_parser: The
        :py:class:`swiftly.cli.optionparser.OptionParser` for general
        usage.
    :param command_parsers: A dict of (name, :py:class:`CLICommand`)
        for specific command usage.
    """
    if command_name == 'for':
        command_name = 'fordo'
    with context.io_manager.with_stdout() as stdout:
        if not command_name:
            general_parser.print_help(stdout)
        elif command_name in command_parsers:
            command_parsers[command_name].option_parser.print_help(stdout)
        else:
            raise ReturnCode('unknown command %r' % command_name)

def
cli_help
(
context
,
command_name
,
general_parser
,
command_parsers
)
:
if
command_name
==
'for'
:
command_name
=
'fordo'
with
context
.
io_manager
.
with_stdout
(
)
as
stdout
:
if
not
command_name
:
general_parser
.
print_help
(
stdout
)
elif
command_name
in
command_parsers
:
command_parsers
[
command_name
]
.
option_parser
.
print_help
(
stdout
)
else
:
raise
ReturnCode
(
'unknown command %r'
%
command_name
)


def is_empty(self):
        """
        Check whether the "file" is empty reading the single byte.
        """
        something = self.read(1)
        if something:
            if self.buf:
                self.buf = something + self.buf
            else:
                self.buf = something
            return False
        else:
            return True

def is_empty ( self ) : something = self . read ( 1 ) if something : if self . buf : self . buf = something + self . buf else : self . buf = something return False else : return True

Check whether the file is empty reading the single byte .


def cli_encrypt(context, key):
    """
    Encrypts context.io_manager's stdin and sends that to
    context.io_manager's stdout.

    This can be useful to encrypt to disk before attempting to
    upload, allowing uploads retries and segmented encrypted objects.

    See :py:mod:`swiftly.cli.encrypt` for context usage information.

    See :py:class:`CLIEncrypt` for more information.
    """
    with context.io_manager.with_stdout() as stdout:
        with context.io_manager.with_stdin() as stdin:
            for chunk in aes_encrypt(key, stdin, preamble=AES256CBC):
                stdout.write(chunk)
            stdout.flush()

def cli_encrypt ( context , key ) : with context . io_manager . with_stdout ( ) as stdout : with context . io_manager . with_stdin ( ) as stdin : for chunk in aes_encrypt ( key , stdin , preamble = AES256CBC ) : stdout . write ( chunk ) stdout . flush ( )

Encrypts context . io_manager s stdin and sends that to context . io_manager s stdout .


def get_status(app, repo_config, repo_name, sha):
    """Gets the status of a commit.

    .. note::
        ``repo_name`` might not ever be anything other than
        ``repo_config['github_repo']``.

    :param app: Flask app for leeroy
    :param repo_config: configuration for the repo
    :param repo_name: The name of the owner/repo
    :param sha: SHA for the status we are looking for
    :return: returns json response of status
    """
    url = get_api_url(app, repo_config, github_status_url).format(
        repo_name=repo_name, sha=sha)
    logging.debug("Getting status for %s %s", repo_name, sha)
    s = get_session_for_repo(app, repo_config)
    response = s.get(url)
    if not response.ok:
        raise Exception("Unable to get status: {}".format(response.status_code))
    return response

def
get_status
(
app
,
repo_config
,
repo_name
,
sha
)
:
url
=
get_api_url
(
app
,
repo_config
,
github_status_url
)
.
format
(
repo_name
=
repo_name
,
sha
=
sha
)
logging
.
debug
(
"Getting status for %s %s"
,
repo_name
,
sha
)
s
=
get_session_for_repo
(
app
,
repo_config
)
response
=
s
.
get
(
url
)
if
not
response
.
ok
:
raise
Exception
(
"Unable to get status: {}"
.
format
(
response
.
status_code
)
)
return
response


def get_pull_request(app, repo_config, pull_request):
    """Data for a given pull request.

    :param app: Flask app
    :param repo_config: dict with ``github_repo`` key
    :param pull_request: the pull request number
    """
    response = get_api_response(
        app, repo_config,
        "/repos/{{repo_name}}/pulls/{0}".format(pull_request))
    if not response.ok:
        raise Exception("Unable to get pull request: status code {}".format(response.status_code))
    return response.json


def get_pull_requests(app, repo_config):
    """Last 30 pull requests from a repository.

    :param app: Flask app
    :param repo_config: dict with ``github_repo`` key

    :returns: id for a pull request
    """
    response = get_api_response(app, repo_config, "/repos/{repo_name}/pulls")
    if not response.ok:
        raise Exception("Unable to get pull requests: status code {}".format(response.status_code))
    return (item for item in response.json)

Last 30 pull requests from a repository .


def forwards(self, orm):
        "Write your forwards methods here."
        # Note: Remember to use orm['appname.ModelName'] rather than "from appname.models..."
        for title in orm['hero_slider.SliderItemTitle'].objects.all():
            title.is_published = True
            title.save()


def get_slider_items(context, amount=None):
    """Returns the published slider items."""
    req = context.get('request')
    qs = SliderItem.objects.published(req).order_by('position')
    if amount:
        qs = qs[:amount]
    return qs

def get_slider_items ( context , amount = None ) : req = context . get ( 'request' ) qs = SliderItem . objects . published ( req ) . order_by ( 'position' ) if amount : qs = qs [ : amount ] return qs


def render_hero_slider(context):
    """
    Renders the hero slider.

    """
    req = context.get('request')
    qs = SliderItem.objects.published(req).order_by('position')
    return {
        'slider_items': qs,
    }

def render_hero_slider ( context ) : req = context . get ( 'request' ) qs = SliderItem . objects . published ( req ) . order_by ( 'position' ) return { 'slider_items' : qs , }


def reader_acquire(self):
        """Acquire the lock to read"""

        self._order_mutex.acquire()
        self._readers_mutex.acquire()

        if self._readers == 0:
            self._access_mutex.acquire()
        self._readers += 1

        self._order_mutex.release()
        self._readers_mutex.release()

def reader_acquire ( self ) : self . _order_mutex . acquire ( ) self . _readers_mutex . acquire ( ) if self . _readers == 0 : self . _access_mutex . acquire ( ) self . _readers += 1 self . _order_mutex . release ( ) self . _readers_mutex . release ( )


def reader_release(self):
        """Release the lock after reading"""

        self._readers_mutex.acquire()

        self._readers -= 1
        if self._readers == 0:
            self._access_mutex.release()

        self._readers_mutex.release()

def reader_release ( self ) : self . _readers_mutex . acquire ( ) self . _readers -= 1 if self . _readers == 0 : self . _access_mutex . release ( ) self . _readers_mutex . release ( )


def writer_acquire(self):
        """Acquire the lock to write"""

        self._order_mutex.acquire()
        self._access_mutex.acquire()
        self._order_mutex.release()

def writer_acquire ( self ) : self . _order_mutex . acquire ( ) self . _access_mutex . acquire ( ) self . _order_mutex . release ( )


def add(self, task_id, backend, category, backend_args,
            archiving_cfg=None, scheduling_cfg=None):
        """Add a task to the registry.

        This method adds task using `task_id` as identifier. If a task
        with the same identifier already exists on the registry, a
        `AlreadyExistsError` exception will be raised.

        :param task_id: identifier of the task to add
        :param backend: backend used to fetch data from the repository
        :param category: category of the items to fetch
        :param backend_args: dictionary of arguments required to run the backend
        :param archiving_cfg: archiving config for the task, if needed
        :param scheduling_cfg: scheduling config for the task, if needed

        :returns: the new task added to the registry

        :raises AlreadyExistsError: raised when the given task identifier
            exists on the registry
        """
        self._rwlock.writer_acquire()

        if task_id in self._tasks:
            self._rwlock.writer_release()
            raise AlreadyExistsError(element=str(task_id))

        task = Task(task_id, backend, category, backend_args,
                    archiving_cfg=archiving_cfg,
                    scheduling_cfg=scheduling_cfg)
        self._tasks[task_id] = task

        self._rwlock.writer_release()

        logger.debug("Task %s added to the registry", str(task_id))

        return task

def
add
(
self
,
task_id
,
backend
,
category
,
backend_args
,
archiving_cfg
=
,
scheduling_cfg
=
)
:
self
.
_rwlock
.
writer_acquire
(
)
if
task_id
in
self
.
_tasks
:
self
.
_rwlock
.
writer_release
(
)
raise
AlreadyExistsError
(
element
=
str
(
task_id
)
)
task
=
Task
(
task_id
,
backend
,
category
,
backend_args
,
archiving_cfg
=
archiving_cfg
,
scheduling_cfg
=
scheduling_cfg
)
self
.
_tasks
[
task_id
]
=
task
self
.
_rwlock
.
writer_release
(
)
logger
.
debug
(
"Task %s added to the registry"
,
str
(
task_id
)
)
return
task


def remove(self, task_id):
        """Remove a task from the registry.

        To remove it, pass its identifier with `taks_id` parameter.
        When the identifier is not found, a `NotFoundError` exception
        is raised.

        :param task_id: identifier of the task to remove

        :raises NotFoundError: raised when the given task identifier
            is not found on the registry
        """
        try:
            self._rwlock.writer_acquire()
            del self._tasks[task_id]
        except KeyError:
            raise NotFoundError(element=str(task_id))
        finally:
            self._rwlock.writer_release()

        logger.debug("Task %s removed from the registry", str(task_id))

def
remove
(
self
,
task_id
)
:
try
:
self
.
_rwlock
.
writer_acquire
(
)
del
self
.
_tasks
[
task_id
]
except
KeyError
:
raise
NotFoundError
(
element
=
str
(
task_id
)
)
finally
:
self
.
_rwlock
.
writer_release
(
)
logger
.
debug
(
"Task %s removed from the registry"
,
str
(
task_id
)
)


def get(self, task_id):
        """Get a task from the registry.

        Retrieve a task from the registry using its task identifier. When
        the task does not exist, a `NotFoundError` exception will be
        raised.

        :param task_id: task identifier

        :returns: a task object

        :raises NotFoundError: raised when the requested task is not
            found on the registry
        """
        try:
            self._rwlock.reader_acquire()
            task = self._tasks[task_id]
        except KeyError:
            raise NotFoundError(element=str(task_id))
        finally:
            self._rwlock.reader_release()

        return task

def get ( self , task_id ) : try : self . _rwlock . reader_acquire ( ) task = self . _tasks [ task_id ] except KeyError : raise NotFoundError ( element = str ( task_id ) ) finally : self . _rwlock . reader_release ( ) return task


def tasks(self):
        """Get the list of tasks"""

        self._rwlock.reader_acquire()
        tl = [v for v in self._tasks.values()]
        tl.sort(key=lambda x: x.task_id)
        self._rwlock.reader_release()

        return tl

def tasks ( self ) : self . _rwlock . reader_acquire ( ) tl = [ v for v in self . _tasks . values ( ) ] tl . sort ( key = lambda x : x . task_id ) self . _rwlock . reader_release ( ) return tl


def to_dict(self):
        """Returns a dict with the representation of this task configuration object."""

        properties = find_class_properties(self.__class__)
        config = {
            name: self.__getattribute__(name) for name, _ in properties
        }
        return config

def to_dict ( self ) : properties = find_class_properties ( self . __class__ ) config = { name : self . __getattribute__ ( name ) for name , _ in properties } return config

Returns a dict with the representation of this task configuration object .


def from_dict(cls, config):
        """Create an configuration object from a dictionary.

        Key,value pairs will be used to initialize a task configuration
        object. If 'config' contains invalid configuration parameters
        a `ValueError` exception will be raised.

        :param config: dictionary used to create an instance of this object

        :returns: a task config instance

        :raises ValueError: when an invalid configuration parameter is found
        """
        try:
            obj = cls(**config)
        except TypeError as e:
            m = cls.KW_ARGS_ERROR_REGEX.match(str(e))
            if m:
                raise ValueError("unknown '%s' task config parameter" % m.group(1))
            else:
                raise e
        else:
            return obj

def
from_dict
(
cls
,
config
)
:
try
:
obj
=
cls
(
*
*
config
)
except
TypeError
as
e
:
m
=
cls
.
KW_ARGS_ERROR_REGEX
.
match
(
str
(
e
)
)
if
m
:
raise
ValueError
(
"unknown '%s' task config parameter"
%
m
.
group
(
1
)
)
else
:
raise
e
else
:
return
obj

Create an configuration object from a dictionary .


def execute_perceval_job(backend, backend_args, qitems, task_id, category,
                         archive_args=None, max_retries=MAX_JOB_RETRIES):
    """Execute a Perceval job on RQ.

    The items fetched during the process will be stored in a
    Redis queue named `queue`.

    Setting the parameter `archive_path`, raw data will be stored
    with the archive manager. The contents from the archive can
    be retrieved setting the pameter `fetch_from_archive` to `True`,
    too. Take into account this behaviour will be only available
    when the backend supports the use of the archive. If archiving
    is not supported, an `AttributeError` exception will be raised.

    :param backend: backend to execute
    :param bakend_args: dict of arguments for running the backend
    :param qitems: name of the RQ queue used to store the items
    :param task_id: identifier of the task linked to this job
    :param category: category of the items to retrieve
    :param archive_args: archive arguments
    :param max_retries: maximum number of attempts this job can execute
        before failing

    :returns: a `JobResult` instance

    :raises NotFoundError: raised when the backend is not found
    :raises AttributeError: raised when archiving is not supported but
        any of the archive parameters were set
    """
    rq_job = rq.get_current_job()

    job = PercevalJob(rq_job.id, task_id, backend, category,
                      rq_job.connection, qitems)

    logger.debug("Running job #%s (task: %s) (%s) (cat:%s)",
                 job.job_id, task_id, backend, category)

    if not job.has_archiving() and archive_args:
        raise AttributeError("archive attributes set but archive is not supported")

    run_job = True
    resume = False
    failures = 0

    while run_job:
        try:
            job.run(backend_args, archive_args=archive_args, resume=resume)
        except AttributeError as e:
            raise e
        except Exception as e:
            logger.debug("Error running job %s (%s) - %s",
                         job.job_id, backend, str(e))
            failures += 1

            if not job.has_resuming() or failures >= max_retries:
                logger.error("Cancelling job #%s (task: %s) (%s)",
                             job.job_id, task_id, backend)
                raise e

            logger.warning("Resuming job #%s (task: %s) (%s) due to a failure (n %s, max %s)",
                           job.job_id, task_id, backend, failures, max_retries)
            resume = True
        else:
            # No failure, do not retry
            run_job = False

    result = job.result

    logger.debug("Job #%s (task: %s) completed (%s) - %s items (%s) fetched",
                 result.job_id, task_id, result.backend, str(result.nitems), result.category)

    return result


def initialize_archive_manager(self, archive_path):
        """Initialize the archive manager.

        :param archive_path: path where the archive manager is located
        """
        if archive_path == "":
            raise ValueError("Archive manager path cannot be empty")

        if archive_path:
            self.archive_manager = perceval.archive.ArchiveManager(archive_path)

def
initialize_archive_manager
(
self
,
archive_path
)
:
if
archive_path
==
""
:
raise
ValueError
(
"Archive manager path cannot be empty"
)
if
archive_path
:
self
.
archive_manager
=
perceval
.
archive
.
ArchiveManager
(
archive_path
)


def run(self, backend_args, archive_args=None, resume=False):
        """Run the backend with the given parameters.

        The method will run the backend assigned to this job,
        storing the fetched items in a Redis queue. The ongoing
        status of the job, can be accessed through the property
        `result`. When `resume` is set, the job will start from
        the last execution, overwriting 'from_date' and 'offset'
        parameters, if needed.

        Setting to `True` the parameter `fetch_from_archive`, items can
        be fetched from the archive assigned to this job.

        Any exception during the execution of the process will
        be raised.

        :param backend_args: parameters used to un the backend
        :param archive_args: archive arguments
        :param resume: fetch items starting where the last
            execution stopped
        """
        args = backend_args.copy()

        if archive_args:
            self.initialize_archive_manager(archive_args['archive_path'])

        if not resume:
            max_date = backend_args.get('from_date', None)
            offset = backend_args.get('offset', None)

            if max_date:
                max_date = datetime_to_utc(max_date).timestamp()

            self._result = JobResult(self.job_id, self.task_id, self.backend, self.category,
                                     None, max_date, 0, offset=offset,
                                     nresumed=0)
        else:
            if self.result.max_date:
                args['from_date'] = unixtime_to_datetime(self.result.max_date)
            if self.result.offset:
                args['offset'] = self.result.offset
            self._result.nresumed += 1

        for item in self._execute(args, archive_args):
            self.conn.rpush(self.qitems, pickle.dumps(item))

            self._result.nitems += 1
            self._result.last_uuid = item['uuid']

            if not self.result.max_date or self.result.max_date < item['updated_on']:
                self._result.max_date = item['updated_on']
            if 'offset' in item:
                self._result.offset = item['offset']

def run ( self , backend_args , archive_args = None , resume = False ) : args = backend_args . copy ( ) if archive_args : self . initialize_archive_manager ( archive_args [ 'archive_path' ] ) if not resume : max_date = backend_args . get ( 'from_date' , None ) offset = backend_args . get ( 'offset' , None ) if max_date : max_date = datetime_to_utc ( max_date ) . timestamp ( ) self . _result = JobResult ( self . job_id , self . task_id , self . backend , self . category , None , max_date , 0 , offset = offset , nresumed = 0 ) else : if self . result . max_date : args [ 'from_date' ] = unixtime_to_datetime ( self . result . max_date ) if self . result . offset : args [ 'offset' ] = self . result . offset self . _result . nresumed += 1 for item in self . _execute ( args , archive_args ) : self . conn . rpush ( self . qitems , pickle . dumps ( item ) ) self . _result . nitems += 1 self . _result . last_uuid = item [ 'uuid' ] if not self . result . max_date or self . result . max_date < item [ 'updated_on' ] : self . _result . max_date = item [ 'updated_on' ] if 'offset' in item : self . _result . offset = item [ 'offset' ]

Run the backend with the given parameters .


def _execute(self, backend_args, archive_args):
        """Execute a backend of Perceval.

        Run the backend of Perceval assigned to this job using the
        given arguments. It will raise an `AttributeError` when any of
        the required parameters to run the backend are not found.
        Other exceptions related to the execution of the backend
        will be raised too.

        This method will return an iterator of the items fetched
        by the backend. These items will include some metadata
        related to this job.

        It will also be possible to retrieve the items from the
        archive setting to `True` the parameter `fetch_from_archive`.

        :param backend_args: arguments to execute the backend
        :param archive_args: archive arguments

        :returns: iterator of items fetched by the backend

        :raises AttributeError: raised when any of the required
            parameters is not found
        """
        if not archive_args or not archive_args['fetch_from_archive']:
            return perceval.backend.fetch(self._bklass, backend_args, self.category,
                                          manager=self.archive_manager)
        else:
            return perceval.backend.fetch_from_archive(self._bklass, backend_args,
                                                       self.archive_manager, self.category,
                                                       archive_args['archived_after'])

def _execute ( self , backend_args , archive_args ) : if not archive_args or not archive_args [ 'fetch_from_archive' ] : return perceval . backend . fetch ( self . _bklass , backend_args , self . category , manager = self . archive_manager ) else : return perceval . backend . fetch_from_archive ( self . _bklass , backend_args , self . archive_manager , self . category , archive_args [ 'archived_after' ] )


def create_index(idx_url, clean=False):
        """Configure the index to work with"""

        try:
            r = requests.get(idx_url)
        except requests.exceptions.ConnectionError:
            cause = "Error connecting to Elastic Search (index: %s)" % idx_url
            raise ElasticSearchError(cause=cause)

        if r.status_code != 200:
            # The index does not exist
            r = requests.put(idx_url)

            if r.status_code != 200:
                logger.info("Can't create index %s (%s)", idx_url, r.status_code)
                cause = "Error creating Elastic Search index %s" % idx_url
                raise ElasticSearchError(cause=cause)

            logger.info("Index %s created", idx_url)
            return True
        elif r.status_code == 200 and clean:
            requests.delete(idx_url)
            requests.put(idx_url)
            logger.info("Index deleted and created (index: %s)", idx_url)
            return True

        return False


def create_mapping(idx_url, mapping):
        """Create a mapping"""

        mapping_url = idx_url + '/items/_mapping'
        mapping = json.dumps(mapping)

        try:
            r = requests.put(mapping_url, data=mapping,
                             headers={'Content-Type': 'application/json'})
        except requests.exceptions.ConnectionError:
            cause = "Error connecting to Elastic Search (index: %s, url: %s)" \
                % (idx_url, mapping_url)
            raise ElasticSearchError(cause=cause)

        if r.status_code != 200:
            reason = r.json()['error']
            logger.info("Can't create mapping in %s. %s",
                        mapping_url, reason)
            cause = "Error creating Elastic Search mapping %s. %s" % \
                (mapping_url, reason)
            raise ElasticSearchError(cause=cause)
        else:
            logger.info("Mapping created in %s", mapping_url)


def json_encoder(*args, **kwargs):
    """Custom JSON encoder handler"""

    obj = cherrypy.serving.request._json_inner_handler(*args, **kwargs)

    for chunk in JSONEncoder().iterencode(obj):
        yield chunk.encode('utf-8')

def json_encoder ( * args , * * kwargs ) : obj = cherrypy . serving . request . _json_inner_handler ( * args , * * kwargs ) for chunk in JSONEncoder ( ) . iterencode ( obj ) : yield chunk . encode ( 'utf-8' )


def write_items(cls, writer, items_generator):
        """Write items to the queue

        :param writer: the writer object
        :param items_generator: items to be written in the queue
        """
        while True:
            items = items_generator()
            writer.write(items)
            time.sleep(1)

def write_items ( cls , writer , items_generator ) : while True : items = items_generator ( ) writer . write ( items ) time . sleep ( 1 )


def add_task(self, task_id, backend, category, backend_args,
                 archive_args=None, sched_args=None):
        """Add and schedule a task.

        :param task_id: id of the task
        :param backend: name of the backend
        :param category: category of the items to fecth
        :param backend_args: args needed to initialize the backend
        :param archive_args: args needed to initialize the archive
        :param sched_args: scheduling args for this task

        :returns: the task created
        """
        try:
            archiving_cfg = self.__parse_archive_args(archive_args)
            scheduling_cfg = self.__parse_schedule_args(sched_args)
            self.__validate_args(task_id, backend, category, backend_args)
        except ValueError as e:
            raise e

        try:
            task = self._tasks.add(task_id, backend, category, backend_args,
                                   archiving_cfg=archiving_cfg,
                                   scheduling_cfg=scheduling_cfg)
        except AlreadyExistsError as e:
            raise e

        self._scheduler.schedule_task(task.task_id)

        return task

def add_task ( self , task_id , backend , category , backend_args , archive_args = None , sched_args = None ) : try : archiving_cfg = self . __parse_archive_args ( archive_args ) scheduling_cfg = self . __parse_schedule_args ( sched_args ) self . __validate_args ( task_id , backend , category , backend_args ) except ValueError as e : raise e try : task = self . _tasks . add ( task_id , backend , category , backend_args , archiving_cfg = archiving_cfg , scheduling_cfg = scheduling_cfg ) except AlreadyExistsError as e : raise e self . _scheduler . schedule_task ( task . task_id ) return task


def remove_task(self, task_id):
        """Remove and cancel a task.

        :param task_id: id of the task to be removed
        """
        try:
            self._scheduler.cancel_task(task_id)
        except NotFoundError as e:
            logger.info("Cannot cancel %s task because it does not exist.",
                        task_id)
            return False
        else:
            return True


def items(self):
        """Get the items fetched by the jobs."""

        # Get and remove queued items in an atomic transaction
        pipe = self.conn.pipeline()
        pipe.lrange(Q_STORAGE_ITEMS, 0, -1)
        pipe.ltrim(Q_STORAGE_ITEMS, 1, 0)
        items = pipe.execute()[0]

        for item in items:
            item = pickle.loads(item)
            yield item


def __validate_args(task_id, backend, category, backend_args):
        """Check that the task arguments received are valid"""

        if not task_id or task_id.strip() == "":
            msg = "Missing task_id for task"
            raise ValueError(msg)

        if not backend or backend.strip() == "":
            msg = "Missing backend for task '%s'" % task_id
            raise ValueError(msg)

        if backend_args and not isinstance(backend_args, dict):
            msg = "Backend_args is not a dict, task '%s'" % task_id
            raise ValueError(msg)

        if not category or category.strip() == "":
            msg = "Missing category for task '%s'" % task_id
            raise ValueError(msg)

def
__validate_args
(
task_id
,
backend
,
category
,
backend_args
)
:
if
not
task_id
or
task_id
.
strip
(
)
==
""
:
msg
=
"Missing task_id for task"
raise
ValueError
(
msg
)
if
not
backend
or
backend
.
strip
(
)
==
""
:
msg
=
"Missing backend for task '%s'"
%
task_id
raise
ValueError
(
msg
)
if
backend_args
and
not
isinstance
(
backend_args
,
dict
)
:
msg
=
"Backend_args is not a dict, task '%s'"
%
task_id
raise
ValueError
(
msg
)
if
not
category
or
category
.
strip
(
)
==
""
:
msg
=
"Missing category for task '%s'"
%
task_id
raise
ValueError
(
msg
)