def _build_dict_from_key_value(keys_and_values):
    """Return a dict from a list of key=value pairs
    """
    key_dict = {}
    for key_value in keys_and_values:
        if '=' not in key_value:
            raise GhostError('Pair {0} is not of `key=value` format'.format(
                key_value))
        key, value = key_value.split('=', 1)
        key_dict.update({str(key): str(value)})
    return key_dict

def
_build_dict_from_key_value
(
keys_and_values
)
:
key_dict
=
{
}
for
key_value
in
keys_and_values
:
if
'='
not
in
key_value
:
raise
GhostError
(
'Pair {0} is not of `key=value` format'
.
format
(
key_value
)
)
key
,
value
=
key_value
.
split
(
'='
,
1
)
key_dict
.
update
(
{
str
(
key
)
:
str
(
value
)
}
)
return
key_dict

Return a dict from a list of key = value pairs


def _prettify_list(items):
    """Return a human readable format of a list.

    Example:

    Available Keys:
      - my_first_key
      - my_second_key
    """
    assert isinstance(items, list)

    keys_list = 'Available Keys:'
    for item in items:
        keys_list += '\n  - {0}'.format(item)
    return keys_list

def
_prettify_list
(
items
)
:
assert
isinstance
(
items
,
list
)
keys_list
=
'Available Keys:'
for
item
in
items
:
keys_list
+=
'\n  - {0}'
.
format
(
item
)
return
keys_list

Return a human readable format of a list .


def init_stash(stash_path, passphrase, passphrase_size, backend):
    r"""Init a stash

    `STASH_PATH` is the path to the storage endpoint. If this isn't supplied,
    a default path will be used. In the path, you can specify a name
    for the stash (which, if omitted, will default to `ghost`) like so:
    `ghost init http://10.10.1.1:8500;stash1`.

    After initializing a stash, don't forget you can set environment
    variables for both your stash's path and its passphrase.
    On Linux/OSx you can run:

    export GHOST_STASH_PATH='http://10.10.1.1:8500;stash1'

    export GHOST_PASSPHRASE=$(cat passphrase.ghost)

    export GHOST_BACKEND='tinydb'
    """
    stash_path = stash_path or STORAGE_DEFAULT_PATH_MAPPING[backend]
    click.echo('Stash: {0} at {1}'.format(backend, stash_path))
    storage = STORAGE_MAPPING[backend](**_parse_path_string(stash_path))

    try:
        click.echo('Initializing stash...')
        if os.path.isfile(PASSPHRASE_FILENAME):
            raise GhostError(
                '{0} already exists. Overwriting might prevent you '
                'from accessing the stash it was generated for. '
                'Please make sure to save and remove the file before '
                'initializing another stash.'.format(PASSPHRASE_FILENAME))

        stash = Stash(
            storage,
            passphrase=passphrase,
            passphrase_size=passphrase_size)
        passphrase = stash.init()

        if not passphrase:
            click.echo('Stash already initialized.')
            sys.exit(0)

        _write_passphrase_file(passphrase)
    except GhostError as ex:
        sys.exit(ex)
    except (OSError, IOError) as ex:
        click.echo("Seems like we've run into a problem.")
        file_path = _parse_path_string(stash_path)['db_path']
        click.echo(
            'Removing stale stash and passphrase: {0}. Note that any '
            'directories created are not removed for safety reasons and you '
            'might want to remove them manually.'.format(file_path))
        if os.path.isfile(file_path):
            os.remove(file_path)
        sys.exit(ex)

    click.echo('Initialized stash at: {0}'.format(stash_path))
    click.echo(
        'Your passphrase can be found under the `{0}` file in the '
        'current directory.'.format(PASSPHRASE_FILENAME))
    click.echo(
        'Make sure you save your passphrase somewhere safe. '
        'If lost, you will lose access to your stash.')


def put_key(key_name,
            value,
            description,
            meta,
            modify,
            add,
            lock,
            key_type,
            stash,
            passphrase,
            backend):
    """Insert a key to the stash

    `KEY_NAME` is the name of the key to insert

    `VALUE` is a key=value argument which can be provided multiple times.
    it is the encrypted value of your key
    """
    stash = _get_stash(backend, stash, passphrase)

    try:
        click.echo('Stashing {0} key...'.format(key_type))
        stash.put(
            name=key_name,
            value=_build_dict_from_key_value(value),
            modify=modify,
            metadata=_build_dict_from_key_value(meta),
            description=description,
            lock=lock,
            key_type=key_type,
            add=add)
        click.echo('Key stashed successfully')
    except GhostError as ex:
        sys.exit(ex)

def
put_key
(
key_name
,
value
,
description
,
meta
,
modify
,
add
,
lock
,
key_type
,
stash
,
passphrase
,
backend
)
:
stash
=
_get_stash
(
backend
,
stash
,
passphrase
)
try
:
click
.
echo
(
'Stashing {0} key...'
.
format
(
key_type
)
)
stash
.
put
(
name
=
key_name
,
value
=
_build_dict_from_key_value
(
value
)
,
modify
=
modify
,
metadata
=
_build_dict_from_key_value
(
meta
)
,
description
=
description
,
lock
=
lock
,
key_type
=
key_type
,
add
=
add
)
click
.
echo
(
'Key stashed successfully'
)
except
GhostError
as
ex
:
sys
.
exit
(
ex
)


def lock_key(key_name,
             stash,
             passphrase,
             backend):
    """Lock a key to prevent it from being deleted, purged or modified

    `KEY_NAME` is the name of the key to lock
    """
    stash = _get_stash(backend, stash, passphrase)

    try:
        click.echo('Locking key...')
        stash.lock(key_name=key_name)
        click.echo('Key locked successfully')
    except GhostError as ex:
        sys.exit(ex)

def
lock_key
(
key_name
,
stash
,
passphrase
,
backend
)
:
stash
=
_get_stash
(
backend
,
stash
,
passphrase
)
try
:
click
.
echo
(
'Locking key...'
)
stash
.
lock
(
key_name
=
key_name
)
click
.
echo
(
'Key locked successfully'
)
except
GhostError
as
ex
:
sys
.
exit
(
ex
)

Lock a key to prevent it from being deleted purged or modified


def unlock_key(key_name,
               stash,
               passphrase,
               backend):
    """Unlock a key to allow it to be modified, deleted or purged

    `KEY_NAME` is the name of the key to unlock
    """
    stash = _get_stash(backend, stash, passphrase)

    try:
        click.echo('Unlocking key...')
        stash.unlock(key_name=key_name)
        click.echo('Key unlocked successfully')
    except GhostError as ex:
        sys.exit(ex)

def
unlock_key
(
key_name
,
stash
,
passphrase
,
backend
)
:
stash
=
_get_stash
(
backend
,
stash
,
passphrase
)
try
:
click
.
echo
(
'Unlocking key...'
)
stash
.
unlock
(
key_name
=
key_name
)
click
.
echo
(
'Key unlocked successfully'
)
except
GhostError
as
ex
:
sys
.
exit
(
ex
)

Unlock a key to allow it to be modified deleted or purged


def get_key(key_name,
            value_name,
            jsonify,
            no_decrypt,
            stash,
            passphrase,
            backend):
    """Retrieve a key from the stash

    \b
    `KEY_NAME` is the name of the key to retrieve
    `VALUE_NAME` is a single value to retrieve e.g. if the value
     of the key `test` is `a=b,b=c`, `ghost get test a`a will return
     `b`
    """
    if value_name and no_decrypt:
        sys.exit('VALUE_NAME cannot be used in conjuction with --no-decrypt')

    stash = _get_stash(backend, stash, passphrase, quiet=jsonify or value_name)

    try:
        key = stash.get(key_name=key_name, decrypt=not no_decrypt)
    except GhostError as ex:
        sys.exit(ex)

    if not key:
        sys.exit('Key `{0}` not found'.format(key_name))
    if value_name:
        key = key['value'].get(value_name)
        if not key:
            sys.exit(
                'Value name `{0}` could not be found under key `{1}`'.format(
                    value_name, key_name))

    if jsonify or value_name:
        click.echo(
            json.dumps(key, indent=4, sort_keys=False).strip('"'),
            nl=True)
    else:
        click.echo('Retrieving key...')
        click.echo('\n' + _prettify_dict(key))


def delete_key(key_name, stash, passphrase, backend):
    """Delete a key from the stash

    `KEY_NAME` is the name of the key to delete
    You can provide that multiple times to delete multiple keys at once
    """
    stash = _get_stash(backend, stash, passphrase)

    for key in key_name:
        try:
            click.echo('Deleting key {0}...'.format(key))
            stash.delete(key_name=key)
        except GhostError as ex:
            sys.exit(ex)
    click.echo('Keys deleted successfully')

def
delete_key
(
key_name
,
stash
,
passphrase
,
backend
)
:
stash
=
_get_stash
(
backend
,
stash
,
passphrase
)
for
key
in
key_name
:
try
:
click
.
echo
(
'Deleting key {0}...'
.
format
(
key
)
)
stash
.
delete
(
key_name
=
key
)
except
GhostError
as
ex
:
sys
.
exit
(
ex
)
click
.
echo
(
'Keys deleted successfully'
)


def list_keys(key_name,
              max_suggestions,
              cutoff,
              jsonify,
              locked,
              key_type,
              stash,
              passphrase,
              backend):
    """List all keys in the stash

    If `KEY_NAME` is provided, will look for keys containing `KEY_NAME`.
    If `KEY_NAME` starts with `~`, close matches will be provided according
    to `max_suggestions` and `cutoff`.
    """
    stash = _get_stash(backend, stash, passphrase, quiet=jsonify)

    try:
        keys = stash.list(
            key_name=key_name,
            max_suggestions=max_suggestions,
            cutoff=cutoff,
            locked_only=locked,
            key_type=key_type)
    except GhostError as ex:
        sys.exit(ex)
    if jsonify:
        click.echo(json.dumps(keys, indent=4, sort_keys=True))
    elif not keys:
        click.echo('The stash is empty. Go on, put some keys in there...')
    else:
        click.echo('Listing all keys...')
        click.echo(_prettify_list(keys))


def purge_stash(force, stash, passphrase, backend):
    """Purge the stash from all of its keys
    """
    stash = _get_stash(backend, stash, passphrase)

    try:
        click.echo('Purging stash...')
        stash.purge(force)
        # Maybe we should verify that the list is empty
        # afterwards?
        click.echo('Purge complete!')
    except GhostError as ex:
        sys.exit(ex)


def export_keys(output_path, stash, passphrase, backend):
    """Export all keys to a file
    """
    stash = _get_stash(backend, stash, passphrase)

    try:
        click.echo('Exporting stash to {0}...'.format(output_path))
        stash.export(output_path=output_path)
        click.echo('Export complete!')
    except GhostError as ex:
        sys.exit(ex)

def
export_keys
(
output_path
,
stash
,
passphrase
,
backend
)
:
stash
=
_get_stash
(
backend
,
stash
,
passphrase
)
try
:
click
.
echo
(
'Exporting stash to {0}...'
.
format
(
output_path
)
)
stash
.
export
(
output_path
=
output_path
)
click
.
echo
(
'Export complete!'
)
except
GhostError
as
ex
:
sys
.
exit
(
ex
)


def load_keys(key_file, origin_passphrase, stash, passphrase, backend):
    """Load all keys from an exported key file to the stash

    `KEY_FILE` is the exported stash file to load keys from
    """
    stash = _get_stash(backend, stash, passphrase)

    click.echo('Importing all keys from {0}...'.format(key_file))
    stash.load(origin_passphrase, key_file=key_file)
    click.echo('Import complete!')

def
load_keys
(
key_file
,
origin_passphrase
,
stash
,
passphrase
,
backend
)
:
stash
=
_get_stash
(
backend
,
stash
,
passphrase
)
click
.
echo
(
'Importing all keys from {0}...'
.
format
(
key_file
)
)
stash
.
load
(
origin_passphrase
,
key_file
=
key_file
)
click
.
echo
(
'Import complete!'
)


def migrate_stash(source_stash_path,
                  source_passphrase,
                  source_backend,
                  destination_stash_path,
                  destination_passphrase,
                  destination_backend):
    """Migrate all keys from a source stash to a destination stash.

    `SOURCE_STASH_PATH` and `DESTINATION_STASH_PATH` are the paths
    to the stashs you wish to perform the migration on.
    """
    click.echo('Migrating all keys from {0} to {1}...'.format(
        source_stash_path, destination_stash_path))

    try:
        migrate(
            src_path=source_stash_path,
            src_passphrase=source_passphrase,
            src_backend=source_backend,
            dst_path=destination_stash_path,
            dst_passphrase=destination_passphrase,
            dst_backend=destination_backend)
    except GhostError as ex:
        sys.exit(ex)
    click.echo('Migration complete!')

def
migrate_stash
(
source_stash_path
,
source_passphrase
,
source_backend
,
destination_stash_path
,
destination_passphrase
,
destination_backend
)
:
click
.
echo
(
'Migrating all keys from {0} to {1}...'
.
format
(
source_stash_path
,
destination_stash_path
)
)
try
:
migrate
(
src_path
=
source_stash_path
,
src_passphrase
=
source_passphrase
,
src_backend
=
source_backend
,
dst_path
=
destination_stash_path
,
dst_passphrase
=
destination_passphrase
,
dst_backend
=
destination_backend
)
except
GhostError
as
ex
:
sys
.
exit
(
ex
)
click
.
echo
(
'Migration complete!'
)

Migrate all keys from a source stash to a destination stash .


def ssh(key_name, no_tunnel, stash, passphrase, backend):
    """Use an ssh type key to connect to a machine via ssh

    Note that trying to use a key of the wrong type (e.g. `secret`)
    will result in an error.

    `KEY_NAME` is the key to use.

    For additional information on the different configuration options
    for an ssh type key, see the repo's readme.
    """
    # TODO: find_executable or raise
    def execute(command):
        try:
            click.echo('Executing: {0}'.format(' '.join(command)))
            subprocess.check_call(' '.join(command), shell=True)
        except subprocess.CalledProcessError:
            sys.exit(1)

    stash = _get_stash(backend, stash, passphrase)
    key = stash.get(key_name)

    if key:
        _assert_is_ssh_type_key(key)
    else:
        sys.exit('Key `{0}` not found'.format(key_name))

    conn_info = key['value']
    ssh_key_path = conn_info.get('ssh_key_path')
    ssh_key = conn_info.get('ssh_key')
    proxy_key_path = conn_info.get('proxy_key_path')
    proxy_key = conn_info.get('proxy_key')

    id_file = _write_tmp(ssh_key) if ssh_key else ssh_key_path
    conn_info['ssh_key_path'] = id_file

    if conn_info.get('proxy'):
        proxy_id_file = _write_tmp(proxy_key) if proxy_key else proxy_key_path
        conn_info['proxy_key_path'] = proxy_id_file

    ssh_command = _build_ssh_command(conn_info, no_tunnel)
    try:
        execute(ssh_command)
    finally:
        # If they're not equal, that means we've created a temp one which
        # should be deleted, else, it's a path to an existing key file.
        if id_file != ssh_key_path:
            click.echo('Removing temp ssh key file: {0}...'.format(id_file))
            os.remove(id_file)
        if conn_info.get('proxy') and proxy_id_file != proxy_key_path:
            click.echo('Removing temp proxy key file: {0}...'.format(
                proxy_id_file))
            os.remove(proxy_id_file)


def put(self,
            name,
            value=None,
            modify=False,
            metadata=None,
            description='',
            encrypt=True,
            lock=False,
            key_type='secret',
            add=False):
        """Put a key inside the stash

        if key exists and modify true: delete and create
        if key exists and modify false: fail
        if key doesn't exist and modify true: fail
        if key doesn't exist and modify false: create

        `name` is unique and cannot be changed.

        `value` must be provided if the key didn't already exist, otherwise,
        the previous value will be retained.

        `created_at` will be left unmodified if the key
        already existed. Otherwise, the current time will be used.

        `modified_at` will be changed to the current time
        if the field is being modified.

        `metadata` will be updated if provided. If it wasn't
        provided the field from the existing key will be used and the
        same goes for the `uid` which will be generated if it didn't
        previously exist.

        `lock` will lock the key to prevent it from being modified or deleted

        `add` allows to add values to an existing key instead of overwriting.

        Returns the id of the key in the database
        """
        def assert_key_is_unlocked(existing_key):
            if existing_key and existing_key.get('lock'):
                raise GhostError(
                    'Key `{0}` is locked and therefore cannot be modified. '
                    'Unlock the key and try again'.format(name))

        def assert_value_provided_for_new_key(value, existing_key):
            if not value and not existing_key.get('value'):
                raise GhostError('You must provide a value for new keys')

        self._assert_valid_stash()
        self._validate_key_schema(value, key_type)
        if value and encrypt and not isinstance(value, dict):
            raise GhostError('Value must be of type dict')

        # TODO: This should be refactored. `_handle_existing_key` deletes
        # the key rather implicitly. It shouldn't do that.
        # `existing_key` will be an empty dict if it doesn't exist
        key = self._handle_existing_key(name, modify or add)
        assert_key_is_unlocked(key)
        assert_value_provided_for_new_key(value, key)

        new_key = dict(name=name, lock=lock)
        if value:
            # TODO: fix edge case in which encrypt is false and yet we might
            # try to add to an existing key. encrypt=false is only used when
            # `load`ing into a new stash, but someone might use it directly
            # from the API.
            if add:
                value = self._update_existing_key(key, value)
            new_key['value'] = self._encrypt(value) if encrypt else value
        else:
            new_key['value'] = key.get('value')

        # TODO: Treat a case in which we try to update an existing key
        # but don't provide a value in which nothing will happen.
        new_key['description'] = description or key.get('description')
        new_key['created_at'] = key.get('created_at') or _get_current_time()
        new_key['modified_at'] = _get_current_time()
        new_key['metadata'] = metadata or key.get('metadata')
        new_key['uid'] = key.get('uid') or str(uuid.uuid4())
        new_key['type'] = key.get('type') or key_type

        key_id = self._storage.put(new_key)

        audit(
            storage=self._storage.db_path,
            action='MODIFY' if (modify or add) else 'PUT',
            message=json.dumps(dict(
                key_name=new_key['name'],
                value='HIDDEN',
                description=new_key['description'],
                uid=new_key['uid'],
                metadata=json.dumps(new_key['metadata']),
                lock=new_key['lock'],
                type=new_key['type'])))

        return key_id


def get(self, key_name, decrypt=True):
        """Return a key with its parameters if it was found.
        """
        self._assert_valid_stash()

        key = self._storage.get(key_name).copy()
        if not key.get('value'):
            return None
        if decrypt:
            key['value'] = self._decrypt(key['value'])

        audit(
            storage=self._storage.db_path,
            action='GET',
            message=json.dumps(dict(key_name=key_name)))

        return key

def get ( self , key_name , decrypt = True ) : self . _assert_valid_stash ( ) key = self . _storage . get ( key_name ) . copy ( ) if not key . get ( 'value' ) : return None if decrypt : key [ 'value' ] = self . _decrypt ( key [ 'value' ] ) audit ( storage = self . _storage . db_path , action = 'GET' , message = json . dumps ( dict ( key_name = key_name ) ) ) return key

Return a key with its parameters if it was found .


def list(self,
             key_name=None,
             max_suggestions=100,
             cutoff=0.5,
             locked_only=False,
             key_type=None):
        """Return a list of all keys.
        """
        self._assert_valid_stash()

        key_list = [k for k in self._storage.list()
                    if k['name'] != 'stored_passphrase' and
                    (k.get('lock') if locked_only else True)]

        if key_type:
            # To maintain backward compatibility with keys without a type.
            # The default key type is secret, in which case we also look for
            # keys with no (None) types.
            types = ('secret', None) if key_type == 'secret' else [key_type]
            key_list = [k for k in key_list if k.get('type') in types]

        key_list = [k['name'] for k in key_list]
        if key_name:
            if key_name.startswith('~'):
                key_list = difflib.get_close_matches(
                    key_name.lstrip('~'), key_list, max_suggestions, cutoff)
            else:
                key_list = [k for k in key_list if key_name in k]

        audit(
            storage=self._storage.db_path,
            action='LIST' + ('[LOCKED]' if locked_only else ''),
            message=json.dumps(dict()))

        return key_list


def delete(self, key_name):
        """Delete a key if it exists.
        """
        self._assert_valid_stash()

        if key_name == 'stored_passphrase':
            raise GhostError(
                '`stored_passphrase` is a reserved ghost key name '
                'which cannot be deleted')

        # TODO: Optimize. We get from the storage twice here for no reason
        if not self.get(key_name):
            raise GhostError('Key `{0}` not found'.format(key_name))
        key = self._storage.get(key_name)
        if key.get('lock'):
            raise GhostError(
                'Key `{0}` is locked and therefore cannot be deleted '
                'Please unlock the key and try again'.format(key_name))

        deleted = self._storage.delete(key_name)

        audit(
            storage=self._storage.db_path,
            action='DELETE',
            message=json.dumps(dict(key_name=key_name)))

        if not deleted:
            raise GhostError('Failed to delete {0}'.format(key_name))


def purge(self, force=False, key_type=None):
        """Purge the stash from all keys
        """
        self._assert_valid_stash()

        if not force:
            raise GhostError(
                "The `force` flag must be provided to perform a stash purge. "
                "I mean, you don't really want to just delete everything "
                "without precautionary measures eh?")

        audit(
            storage=self._storage.db_path,
            action='PURGE',
            message=json.dumps(dict()))

        for key_name in self.list(key_type=key_type):
            self.delete(key_name)


def export(self, output_path=None, decrypt=False):
        """Export all keys in the stash to a list or a file
        """
        self._assert_valid_stash()

        all_keys = []
        for key in self.list():
            # We `dict` this as a precaution as tinydb returns
            # a tinydb.database.Element instead of a dictionary
            # and well.. I ain't taking no chances
            all_keys.append(dict(self.get(key, decrypt=decrypt)))
        if all_keys:
            if output_path:
                with open(output_path, 'w') as output_file:
                    output_file.write(json.dumps(all_keys, indent=4))
            return all_keys
        else:
            raise GhostError('There are no keys to export')


def load(self, origin_passphrase, keys=None, key_file=None):
        """Import keys to the stash from either a list of keys or a file

        `keys` is a list of dictionaries created by `self.export`
        `stash_path` is a path to a file created by `self.export`
        """
        # TODO: Handle keys not dict or key_file not json
        self._assert_valid_stash()

        # Check if both or none are provided (ahh, the mighty xor)
        if not (bool(keys) ^ bool(key_file)):
            raise GhostError(
                'You must either provide a path to an exported stash file '
                'or a list of key dicts to import')
        if key_file:
            with open(key_file) as stash_file:
                keys = json.loads(stash_file.read())

        # If the passphrases are the same, there's no reason to decrypt
        # and re-encrypt. We can simply pass the value.
        decrypt = origin_passphrase != self.passphrase
        if decrypt:
            # TODO: The fact that we need to create a stub stash just to
            # decrypt means we should probably have some encryptor class.
            stub = Stash(TinyDBStorage('stub'), origin_passphrase)
        # TODO: Handle existing keys when loading
        for key in keys:
            self.put(
                name=key['name'],
                value=stub._decrypt(key['value']) if decrypt else key['value'],
                metadata=key['metadata'],
                description=key['description'],
                lock=key.get('lock'),
                key_type=key.get('type'),
                encrypt=decrypt)


def _encrypt(self, value):
        """Turn a json serializable value into an jsonified, encrypted,
        hexa string.
        """
        value = json.dumps(value)
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            encrypted_value = self.cipher.encrypt(value.encode('utf8'))
        hexified_value = binascii.hexlify(encrypted_value).decode('ascii')
        return hexified_value

def _encrypt ( self , value ) : value = json . dumps ( value ) with warnings . catch_warnings ( ) : warnings . simplefilter ( "ignore" ) encrypted_value = self . cipher . encrypt ( value . encode ( 'utf8' ) ) hexified_value = binascii . hexlify ( encrypted_value ) . decode ( 'ascii' ) return hexified_value

Turn a json serializable value into an jsonified encrypted hexa string .


def _decrypt(self, hexified_value):
        """The exact opposite of _encrypt
        """
        encrypted_value = binascii.unhexlify(hexified_value)
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            jsonified_value = self.cipher.decrypt(
                encrypted_value).decode('ascii')
        value = json.loads(jsonified_value)
        return value

def _decrypt ( self , hexified_value ) : encrypted_value = binascii . unhexlify ( hexified_value ) with warnings . catch_warnings ( ) : warnings . simplefilter ( "ignore" ) jsonified_value = self . cipher . decrypt ( encrypted_value ) . decode ( 'ascii' ) value = json . loads ( jsonified_value ) return value


def get(self, key_name):
        """Return a dictionary consisting of the key itself

        e.g.
        {u'created_at': u'2016-10-10 08:31:53',
         u'description': None,
         u'metadata': None,
         u'modified_at': u'2016-10-10 08:31:53',
         u'name': u'aws',
         u'uid': u'459f12c0-f341-413e-9d7e-7410f912fb74',
         u'value': u'the_value'}

        """
        result = self.db.search(Query().name == key_name)
        if not result:
            return {}
        return result[0]

def get ( self , key_name ) : result = self . db . search ( Query ( ) . name == key_name ) if not result : return { } return result [ 0 ]


def delete(self, key_name):
        """Delete the key and return true if the key was deleted, else false
        """
        self.db.remove(Query().name == key_name)
        return self.get(key_name) == {}

def delete ( self , key_name ) : self . db . remove ( Query ( ) . name == key_name ) return self . get ( key_name ) == { }

Delete the key and return true if the key was deleted else false


def _construct_key(self, values):
        """Return a dictionary representing a key from a list of columns
        and a tuple of values
        """
        key = {}
        for column, value in zip(self.keys.columns, values):
            key.update({column.name: value})
        return key

def _construct_key ( self , values ) : key = { } for column , value in zip ( self . keys . columns , values ) : key . update ( { column . name : value } ) return key


def put(self, key):
        """Put and return the only unique identifier possible, its url
        """
        self._consul_request('PUT', self._key_url(key['name']), json=key)
        return key['name']

def put ( self , key ) : self . _consul_request ( 'PUT' , self . _key_url ( key [ 'name' ] ) , json = key ) return key [ 'name' ]

Put and return the only unique identifier possible its url


def put(self, key):
        """Put and return the only unique identifier possible, its path
        """
        self.client.write(self._key_path(key['name']), **key)
        return self._key_path(key['name'])

def put ( self , key ) : self . client . write ( self . _key_path ( key [ 'name' ] ) , * * key ) return self . _key_path ( key [ 'name' ] )

Put and return the only unique identifier possible its path


def init(self):
        """Create an Elasticsearch index if necessary
        """
        # ignore 400 (IndexAlreadyExistsException) when creating an index
        self.es.indices.create(index=self.params['index'], ignore=400)


def init(self):
        """Create a bucket.
        """
        try:
            self.client.create_bucket(
                Bucket=self.db_path,
                CreateBucketConfiguration=self.bucket_configuration)
        except botocore.exceptions.ClientError as e:
            # If the bucket already exists
            if 'BucketAlreadyOwnedByYou' not in str(
                    e.response['Error']['Code']):
                raise e

def
init
(
self
)
:
try
:
self
.
client
.
create_bucket
(
Bucket
=
self
.
db_path
,
CreateBucketConfiguration
=
self
.
bucket_configuration
)
except
botocore
.
exceptions
.
ClientError
as
e
:
# If the bucket already exists
if
'BucketAlreadyOwnedByYou'
not
in
str
(
e
.
response
[
'Error'
]
[
'Code'
]
)
:
raise
e


def terminal(port=default_port(), baud='9600'):
    """Launch minterm from pyserial"""
    testargs = ['nodemcu-uploader', port, baud]
    # TODO: modifying argv is no good
    sys.argv = testargs
    # resuse miniterm on main function
    miniterm.main()

def
terminal
(
port
=
default_port
(
)
,
baud
=
'9600'
)
:
testargs
=
[
'nodemcu-uploader'
,
port
,
baud
]
# TODO: modifying argv is no good
sys
.
argv
=
testargs
# resuse miniterm on main function
miniterm
.
main
(
)


def __set_baudrate(self, baud):
        """setting baudrate if supported"""
        log.info('Changing communication to %s baud', baud)
        self.__writeln(UART_SETUP.format(baud=baud))
        # Wait for the string to be sent before switching baud
        time.sleep(0.1)
        try:
            self._port.setBaudrate(baud)
        except AttributeError:
            #pySerial 2.7
            self._port.baudrate = baud


def set_timeout(self, timeout):
        """Set the timeout for the communication with the device."""
        timeout = int(timeout) # will raise on Error
        self._timeout = timeout == 0 and 999999 or timeout

def
set_timeout
(
self
,
timeout
)
:
timeout
=
int
(
timeout
)
# will raise on Error
self
.
_timeout
=
timeout
==
0
and
999999
or
timeout

Set the timeout for the communication with the device .


def __clear_buffers(self):
        """Clears the input and output buffers"""
        try:
            self._port.reset_input_buffer()
            self._port.reset_output_buffer()
        except AttributeError:
            #pySerial 2.7
            self._port.flushInput()
            self._port.flushOutput()

def
__clear_buffers
(
self
)
:
try
:
self
.
_port
.
reset_input_buffer
(
)
self
.
_port
.
reset_output_buffer
(
)
except
AttributeError
:
#pySerial 2.7
self
.
_port
.
flushInput
(
)
self
.
_port
.
flushOutput
(
)


def __expect(self, exp='> ', timeout=None):
        """will wait for exp to be returned from nodemcu or timeout"""
        timeout_before = self._port.timeout
        timeout = timeout or self._timeout
        #do NOT set timeout on Windows
        if SYSTEM != 'Windows':
            # Checking for new data every 100us is fast enough
            if self._port.timeout != MINIMAL_TIMEOUT:
                self._port.timeout = MINIMAL_TIMEOUT

        end = time.time() + timeout

        # Finish as soon as either exp matches or we run out of time (work like dump, but faster on success)
        data = ''
        while not data.endswith(exp) and time.time() <= end:
            data += self._port.read()

        log.debug('expect returned: `{0}`'.format(data))
        if time.time() > end:
            raise CommunicationTimeout('Timeout waiting for data', data)

        if not data.endswith(exp) and len(exp) > 0:
            raise BadResponseException('Bad response.', exp, data)

        if SYSTEM != 'Windows':
            self._port.timeout = timeout_before

        return data


def __write(self, output, binary=False):
        """write data on the nodemcu port. If 'binary' is True the debug log
        will show the intended output as hex, otherwise as string"""
        if not binary:
            log.debug('write: %s', output)
        else:
            log.debug('write binary: %s', hexify(output))
        self._port.write(output)
        self._port.flush()

def
__write
(
self
,
output
,
binary
=
False
)
:
if
not
binary
:
log
.
debug
(
'write: %s'
,
output
)
else
:
log
.
debug
(
'write binary: %s'
,
hexify
(
output
)
)
self
.
_port
.
write
(
output
)
self
.
_port
.
flush
(
)

write data on the nodemcu port . If binary is True the debug log will show the intended output as hex otherwise as string


def __exchange(self, output, timeout=None):
        """Write output to the port and wait for response"""
        self.__writeln(output)
        self._port.flush()
        return self.__expect(timeout=timeout or self._timeout)

def __exchange ( self , output , timeout = None ) : self . __writeln ( output ) self . _port . flush ( ) return self . __expect ( timeout = timeout or self . _timeout )


def close(self):
        """restores the nodemcu to default baudrate and then closes the port"""
        try:
            if self.baud != self.start_baud:
                self.__set_baudrate(self.start_baud)
            self._port.flush()
            self.__clear_buffers()
        except serial.serialutil.SerialException:
            pass
        log.debug('closing port')
        self._port.close()

def
close
(
self
)
:
try
:
if
self
.
baud
!=
self
.
start_baud
:
self
.
__set_baudrate
(
self
.
start_baud
)
self
.
_port
.
flush
(
)
self
.
__clear_buffers
(
)
except
serial
.
serialutil
.
SerialException
:
pass
log
.
debug
(
'closing port'
)
self
.
_port
.
close
(
)


def prepare(self):
        """
        This uploads the protocol functions nessecary to do binary
        chunked transfer
        """
        log.info('Preparing esp for transfer.')

        for func in LUA_FUNCTIONS:
            detected = self.__exchange('print({0})'.format(func))
            if detected.find('function:') == -1:
                break
        else:
            log.info('Preparation already done. Not adding functions again.')
            return True
        functions = RECV_LUA + '\n' + SEND_LUA
        data = functions.format(baud=self._port.baudrate)
        ##change any \r\n to just \n and split on that
        lines = data.replace('\r', '').split('\n')

        #remove some unneccesary spaces to conserve some bytes
        for line in lines:
            line = line.strip().replace(', ', ',').replace(' = ', '=')

            if len(line) == 0:
                continue

            resp = self.__exchange(line)
            #do some basic test of the result
            if ('unexpected' in resp) or ('stdin' in resp) or len(resp) > len(functions)+10:
                log.error('error when preparing "%s"', resp)
                return False
        return True


def download_file(self, filename):
        """Download a file from device to local filesystem"""
        res = self.__exchange('send("{filename}")'.format(filename=filename))
        if ('unexpected' in res) or ('stdin' in res):
            log.error('Unexpected error downloading file: %s', res)
            raise Exception('Unexpected error downloading file')

        #tell device we are ready to receive
        self.__write('C')
        #we should get a NUL terminated filename to start with
        sent_filename = self.__expect(NUL).strip()
        log.info('receiveing ' + sent_filename)

        #ACK to start download
        self.__write(ACK, True)
        buf = ''

        data = ''
        chunk, buf = self.__read_chunk(buf)
        #read chunks until we get an empty which is the end
        while chunk != '':
            self.__write(ACK, True)
            data = data + chunk
            chunk, buf = self.__read_chunk(buf)
        return data


def read_file(self, filename, destination=''):
        """reading data from device into local file"""
        if not destination:
            destination = filename
        log.info('Transferring %s to %s', filename, destination)
        data = self.download_file(filename)

        # Just in case, the filename may contain folder, so create it if needed.
        log.info(destination)
        if not os.path.exists(os.path.dirname(destination)):
            try:
                os.makedirs(os.path.dirname(destination))
            except OSError as e:  # Guard against race condition
                if e.errno != errno.EEXIST:
                    raise
        with open(destination, 'w') as fil:
            fil.write(data)


def write_file(self, path, destination='', verify='none'):
        """sends a file to the device using the transfer protocol"""
        filename = os.path.basename(path)
        if not destination:
            destination = filename

        log.info('Transferring %s as %s', path, destination)
        self.__writeln("recv()")

        res = self.__expect('C> ')
        if not res.endswith('C> '):
            log.error('Error waiting for esp "%s"', res)
            raise CommunicationTimeout('Error waiting for device to start receiving', res)

        log.debug('sending destination filename "%s"', destination)
        self.__write(destination + '\x00', True)
        if not self.__got_ack():
            log.error('did not ack destination filename')
            raise NoAckException('Device did not ACK destination filename')

        content = from_file(path)

        log.debug('sending %d bytes in %s', len(content), filename)
        pos = 0
        chunk_size = 128
        while pos < len(content):
            rest = len(content) - pos
            if rest > chunk_size:
                rest = chunk_size

            data = content[pos:pos+rest]
            if not self.__write_chunk(data):
                resp = self.__expect()
                log.error('Bad chunk response "%s" %s', resp, hexify(resp))
                raise BadResponseException('Bad chunk response', ACK, resp)

            pos += chunk_size

        log.debug('sending zero block')
        #zero size block
        self.__write_chunk('')
        if verify != 'none':
            self.verify_file(path, destination, verify)


def verify_file(self, path, destination, verify='none'):
        """Tries to verify if path has same checksum as destination.
            Valid options for verify is 'raw', 'sha1' or 'none'
        """
        content = from_file(path)
        log.info('Verifying using %s...' % verify)
        if verify == 'raw':

            data = self.download_file(destination)
            if content != data:
                log.error('Raw verification failed.')
                raise VerificationError('Verification failed.')
            else:
                log.info('Verification successful. Contents are identical.')
        elif verify == 'sha1':
            #Calculate SHA1 on remote file. Extract just hash from result
            data = self.__exchange('shafile("'+destination+'")').splitlines()[1]
            log.info('Remote SHA1: %s', data)

            #Calculate hash of local data
            filehashhex = hashlib.sha1(content.encode(ENCODING)).hexdigest()
            log.info('Local SHA1: %s', filehashhex)
            if data != filehashhex:
                log.error('SHA1 verification failed.')
                raise VerificationError('SHA1 Verification failed.')
            else:
                log.info('Verification successful. Checksums match')

        elif verify != 'none':
            raise Exception(verify + ' is not a valid verification method.')

Tries to verify if path has same checksum as destination . Valid options for verify is raw sha1 or none


def exec_file(self, path):
        """execute the lines in the local file 'path'"""
        filename = os.path.basename(path)
        log.info('Execute %s', filename)

        content = from_file(path).replace('\r', '').split('\n')

        res = '> '
        for line in content:
            line = line.rstrip('\n')
            retlines = (res + self.__exchange(line)).splitlines()
            # Log all but the last line
            res = retlines.pop()
            for lin in retlines:
                log.info(lin)
        # last line
        log.info(res)

def
exec_file
(
self
,
path
)
:
filename
=
os
.
path
.
basename
(
path
)
log
.
info
(
'Execute %s'
,
filename
)
content
=
from_file
(
path
)
.
replace
(
'\r'
,
''
)
.
split
(
'\n'
)
res
=
'> '
for
line
in
content
:
line
=
line
.
rstrip
(
'\n'
)
retlines
=
(
res
+
self
.
__exchange
(
line
)
)
.
splitlines
(
)
# Log all but the last line
res
=
retlines
.
pop
(
)
for
lin
in
retlines
:
log
.
info
(
lin
)
# last line
log
.
info
(
res
)

execute the lines in the local file path


def __got_ack(self):
        """Returns true if ACK is received"""
        log.debug('waiting for ack')
        res = self._port.read(1)
        log.debug('ack read %s', hexify(res))
        return res == ACK

def
__got_ack
(
self
)
:
log
.
debug
(
'waiting for ack'
)
res
=
self
.
_port
.
read
(
1
)
log
.
debug
(
'ack read %s'
,
hexify
(
res
)
)
return
res
==
ACK


def write_lines(self, data):
        """write lines, one by one, separated by \n to device"""
        lines = data.replace('\r', '').split('\n')
        for line in lines:
            self.__exchange(line)

def write_lines ( self , data ) : lines = data . replace ( '\r' , '' ) . split ( '\n' ) for line in lines : self . __exchange ( line )

write lines one by one separated by \ n to device


def __write_chunk(self, chunk):
        """formats and sends a chunk of data to the device according
        to transfer protocol"""
        log.debug('writing %d bytes chunk', len(chunk))
        data = BLOCK_START + chr(len(chunk)) + chunk
        if len(chunk) < 128:
            padding = 128 - len(chunk)
            log.debug('pad with %d characters', padding)
            data = data + (' ' * padding)
        log.debug("packet size %d", len(data))
        self.__write(data)
        self._port.flush()
        return self.__got_ack()

def
__write_chunk
(
self
,
chunk
)
:
log
.
debug
(
'writing %d bytes chunk'
,
len
(
chunk
)
)
data
=
BLOCK_START
+
chr
(
len
(
chunk
)
)
+
chunk
if
len
(
chunk
)
<
128
:
padding
=
128
len
(
chunk
)
log
.
debug
(
'pad with %d characters'
,
padding
)
data
=
data
+
(
' '
*
padding
)
log
.
debug
(
"packet size %d"
,
len
(
data
)
)
self
.
__write
(
data
)
self
.
_port
.
flush
(
)
return
self
.
__got_ack
(
)


def __read_chunk(self, buf):
        """Read a chunk of data"""
        log.debug('reading chunk')
        timeout_before = self._port.timeout
        if SYSTEM != 'Windows':
            # Checking for new data every 100us is fast enough
            if self._port.timeout != MINIMAL_TIMEOUT:
                self._port.timeout = MINIMAL_TIMEOUT

        end = time.time() + timeout_before

        while len(buf) < 130 and time.time() <= end:
            buf = buf + self._port.read()

        if buf[0] != BLOCK_START or len(buf) < 130:
            log.debug('buffer binary: %s ', hexify(buf))
            raise Exception('Bad blocksize or start byte')

        if SYSTEM != 'Windows':
            self._port.timeout = timeout_before

        chunk_size = ord(buf[1])
        data = buf[2:chunk_size+2]
        buf = buf[130:]
        return (data, buf)


def file_list(self):
        """list files on the device"""
        log.info('Listing files')
        res = self.__exchange(LIST_FILES)
        res = res.split('\r\n')
        # skip first and last lines
        res = res[1:-1]
        files = []
        for line in res:
            files.append(line.split('\t'))
        return files

def
file_list
(
self
)
:
log
.
info
(
'Listing files'
)
res
=
self
.
__exchange
(
LIST_FILES
)
res
=
res
.
split
(
'\r\n'
)
# skip first and last lines
res
=
res
[
1
:
1
]
files
=
[
]
for
line
in
res
:
files
.
append
(
line
.
split
(
'\t'
)
)
return
files


def file_do(self, filename):
        """Execute a file on the device using 'do'"""
        log.info('Executing '+filename)
        res = self.__exchange('dofile("'+filename+'")')
        log.info(res)
        return res

def
file_do
(
self
,
filename
)
:
log
.
info
(
'Executing '
+
filename
)
res
=
self
.
__exchange
(
'dofile("'
+
filename
+
'")'
)
log
.
info
(
res
)
return
res


def file_format(self):
        """Formats device filesystem"""
        log.info('Formating, can take minutes depending on flash size...')
        res = self.__exchange('file.format()', timeout=300)
        if 'format done' not in res:
            log.error(res)
        else:
            log.info(res)
        return res


def file_print(self, filename):
        """Prints a file on the device to console"""
        log.info('Printing ' + filename)
        res = self.__exchange(PRINT_FILE.format(filename=filename))
        log.info(res)
        return res

def
file_print
(
self
,
filename
)
:
log
.
info
(
'Printing '
+
filename
)
res
=
self
.
__exchange
(
PRINT_FILE
.
format
(
filename
=
filename
)
)
log
.
info
(
res
)
return
res


def node_heap(self):
        """Show device heap size"""
        log.info('Heap')
        res = self.__exchange('print(node.heap())')
        log.info(res)
        return int(res.split('\r\n')[1])

def node_heap ( self ) : log . info ( 'Heap' ) res = self . __exchange ( 'print(node.heap())' ) log . info ( res ) return int ( res . split ( '\r\n' ) [ 1 ] )


def file_compile(self, path):
        """Compiles a file specified by path on the device"""
        log.info('Compile '+path)
        cmd = 'node.compile("%s")' % path
        res = self.__exchange(cmd)
        log.info(res)
        return res

def
file_compile
(
self
,
path
)
:
log
.
info
(
'Compile '
+
path
)
cmd
=
'node.compile("%s")'
%
path
res
=
self
.
__exchange
(
cmd
)
log
.
info
(
res
)
return
res


def file_remove(self, path):
        """Removes a file on the device"""
        log.info('Remove '+path)
        cmd = 'file.remove("%s")' % path
        res = self.__exchange(cmd)
        log.info(res)
        return res

def
file_remove
(
self
,
path
)
:
log
.
info
(
'Remove '
+
path
)
cmd
=
'file.remove("%s")'
%
path
res
=
self
.
__exchange
(
cmd
)
log
.
info
(
res
)
return
res


def backup(self, path):
        """Backup all files from the device"""
        log.info('Backing up in '+path)
        # List file to backup
        files = self.file_list()
        # then download each of then
        self.prepare()
        for f in files:
            self.read_file(f[0], os.path.join(path, f[0]))

def
backup
(
self
,
path
)
:
log
.
info
(
'Backing up in '
+
path
)
# List file to backup
files
=
self
.
file_list
(
)
# then download each of then
self
.
prepare
(
)
for
f
in
files
:
self
.
read_file
(
f
[
0
]
,
os
.
path
.
join
(
path
,
f
[
0
]
)
)


def operation_upload(uploader, sources, verify, do_compile, do_file, do_restart):
    """The upload operation"""
    sources, destinations = destination_from_source(sources)
    if len(destinations) == len(sources):
        if uploader.prepare():
            for filename, dst in zip(sources, destinations):
                if do_compile:
                    uploader.file_remove(os.path.splitext(dst)[0]+'.lc')
                uploader.write_file(filename, dst, verify)
                #init.lua is not allowed to be compiled
                if do_compile and dst != 'init.lua':
                    uploader.file_compile(dst)
                    uploader.file_remove(dst)
                    if do_file:
                        uploader.file_do(os.path.splitext(dst)[0]+'.lc')
                elif do_file:
                    uploader.file_do(dst)
        else:
            raise Exception('Error preparing nodemcu for reception')
    else:
        raise Exception('You must specify a destination filename for each file you want to upload.')

    if do_restart:
        uploader.node_restart()
    log.info('All done!')


def operation_download(uploader, sources):
    """The download operation"""
    sources, destinations = destination_from_source(sources, False)
    print('sources', sources)
    print('destinations', destinations)
    if len(destinations) == len(sources):
        if uploader.prepare():
            for filename, dst in zip(sources, destinations):
                uploader.read_file(filename, dst)
    else:
        raise Exception('You must specify a destination filename for each file you want to download.')
    log.info('All done!')


def operation_list(uploader):
    """List file on target"""
    files = uploader.file_list()
    for f in files:
        log.info("{file:30s} {size}".format(file=f[0], size=f[1]))

def
operation_list
(
uploader
)
:
files
=
uploader
.
file_list
(
)
for
f
in
files
:
log
.
info
(
"{file:30s} {size}"
.
format
(
file
=
f
[
0
]
,
size
=
f
[
1
]
)
)


def display(content):
    """
    Display a widget, text or other media in a notebook without the need to import IPython at the top level.

    Also handles wrapping GenePattern Python Library content in widgets.
    :param content:
    :return:
    """
    if isinstance(content, gp.GPServer):
        IPython.display.display(GPAuthWidget(content))
    elif isinstance(content, gp.GPTask):
        IPython.display.display(GPTaskWidget(content))
    elif isinstance(content, gp.GPJob):
        IPython.display.display(GPJobWidget(content))
    else:
        IPython.display.display(content)

def display ( content ) : if isinstance ( content , gp . GPServer ) : IPython . display . display ( GPAuthWidget ( content ) ) elif isinstance ( content , gp . GPTask ) : IPython . display . display ( GPTaskWidget ( content ) ) elif isinstance ( content , gp . GPJob ) : IPython . display . display ( GPJobWidget ( content ) ) else : IPython . display . display ( content )

Display a widget text or other media in a notebook without the need to import IPython at the top level .


def from_timestamp(ts):
    """
    Convert a numeric timestamp to a timezone-aware datetime.

    A client may override this function to change the default behavior,
    such as to use local time or timezone-naïve times.
    """
    return datetime.datetime.utcfromtimestamp(ts).replace(tzinfo=pytz.utc)

def from_timestamp ( ts ) : return datetime . datetime . utcfromtimestamp ( ts ) . replace ( tzinfo = pytz . utc )

Convert a numeric timestamp to a timezone - aware datetime .


def at_time(cls, at, target):
        """
        Construct a DelayedCommand to come due at `at`, where `at` may be
        a datetime or timestamp.
        """
        at = cls._from_timestamp(at)
        cmd = cls.from_datetime(at)
        cmd.delay = at - now()
        cmd.target = target
        return cmd

def at_time ( cls , at , target ) : at = cls . _from_timestamp ( at ) cmd = cls . from_datetime ( at ) cmd . delay = at - now ( ) cmd . target = target return cmd

Construct a DelayedCommand to come due at at where at may be a datetime or timestamp .


def _localize(dt):
        """
        Rely on pytz.localize to ensure new result honors DST.
        """
        try:
            tz = dt.tzinfo
            return tz.localize(dt.replace(tzinfo=None))
        except AttributeError:
            return dt

def _localize ( dt ) : try : tz = dt . tzinfo return tz . localize ( dt . replace ( tzinfo = None ) ) except AttributeError : return dt

Rely on pytz . localize to ensure new result honors DST .


def daily_at(cls, at, target):
        """
        Schedule a command to run at a specific time each day.
        """
        daily = datetime.timedelta(days=1)
        # convert when to the next datetime matching this time
        when = datetime.datetime.combine(datetime.date.today(), at)
        if when < now():
            when += daily
        return cls.at_time(cls._localize(when), daily, target)

Schedule a command to run at a specific time each day .


def strftime(fmt, t):
	"""A class to replace the strftime in datetime package or time module.
	Identical to strftime behavior in those modules except supports any
	year.
	Also supports datetime.datetime times.
	Also supports milliseconds using %s
	Also supports microseconds using %u"""
	if isinstance(t, (time.struct_time, tuple)):
		t = datetime.datetime(*t[:6])
	assert isinstance(t, (datetime.datetime, datetime.time, datetime.date))
	try:
		year = t.year
		if year < 1900:
			t = t.replace(year=1900)
	except AttributeError:
		year = 1900
	subs = (
		('%Y', '%04d' % year),
		('%y', '%02d' % (year % 100)),
		('%s', '%03d' % (t.microsecond // 1000)),
		('%u', '%03d' % (t.microsecond % 1000))
	)

	def doSub(s, sub):
		return s.replace(*sub)

	def doSubs(s):
		return functools.reduce(doSub, subs, s)

	fmt = '%%'.join(map(doSubs, fmt.split('%%')))
	return t.strftime(fmt)

def strftime ( fmt , t ) : if isinstance ( t , ( time . struct_time , tuple ) ) : t = datetime . datetime ( * t [ : 6 ] ) assert isinstance ( t , ( datetime . datetime , datetime . time , datetime . date ) ) try : year = t . year if year < 1900 : t = t . replace ( year = 1900 ) except AttributeError : year = 1900 subs = ( ( '%Y' , '%04d' % year ) , ( '%y' , '%02d' % ( year % 100 ) ) , ( '%s' , '%03d' % ( t . microsecond // 1000 ) ) , ( '%u' , '%03d' % ( t . microsecond % 1000 ) ) ) def doSub ( s , sub ) : return s . replace ( * sub ) def doSubs ( s ) : return functools . reduce ( doSub , subs , s ) fmt = '%%' . join ( map ( doSubs , fmt . split ( '%%' ) ) ) return t . strftime ( fmt )

A class to replace the strftime in datetime package or time module . Identical to strftime behavior in those modules except supports any year . Also supports datetime . datetime times . Also supports milliseconds using %s Also supports microseconds using %u


def strptime(s, fmt, tzinfo=None):
	"""
	A function to replace strptime in the time module.  Should behave
	identically to the strptime function except it returns a datetime.datetime
	object instead of a time.struct_time object.
	Also takes an optional tzinfo parameter which is a time zone info object.
	"""
	res = time.strptime(s, fmt)
	return datetime.datetime(tzinfo=tzinfo, *res[:6])

def strptime ( s , fmt , tzinfo = None ) : res = time . strptime ( s , fmt ) return datetime . datetime ( tzinfo = tzinfo , * res [ : 6 ] )

A function to replace strptime in the time module . Should behave identically to the strptime function except it returns a datetime . datetime object instead of a time . struct_time object . Also takes an optional tzinfo parameter which is a time zone info object .


def get_nearest_year_for_day(day):
	"""
	Returns the nearest year to now inferred from a Julian date.
	"""
	now = time.gmtime()
	result = now.tm_year
	# if the day is far greater than today, it must be from last year
	if day - now.tm_yday > 365 // 2:
		result -= 1
	# if the day is far less than today, it must be for next year.
	if now.tm_yday - day > 365 // 2:
		result += 1
	return result

Returns the nearest year to now inferred from a Julian date .


def get_period_seconds(period):
	"""
	return the number of seconds in the specified period

	>>> get_period_seconds('day')
	86400
	>>> get_period_seconds(86400)
	86400
	>>> get_period_seconds(datetime.timedelta(hours=24))
	86400
	>>> get_period_seconds('day + os.system("rm -Rf *")')
	Traceback (most recent call last):
	...
	ValueError: period not in (second, minute, hour, day, month, year)
	"""
	if isinstance(period, six.string_types):
		try:
			name = 'seconds_per_' + period.lower()
			result = globals()[name]
		except KeyError:
			msg = "period not in (second, minute, hour, day, month, year)"
			raise ValueError(msg)
	elif isinstance(period, numbers.Number):
		result = period
	elif isinstance(period, datetime.timedelta):
		result = period.days * get_period_seconds('day') + period.seconds
	else:
		raise TypeError('period must be a string or integer')
	return result


def divide_timedelta_float(td, divisor):
	"""
	Divide a timedelta by a float value

	>>> one_day = datetime.timedelta(days=1)
	>>> half_day = datetime.timedelta(days=.5)
	>>> divide_timedelta_float(one_day, 2.0) == half_day
	True
	>>> divide_timedelta_float(one_day, 2) == half_day
	True
	"""
	# td is comprised of days, seconds, microseconds
	dsm = [getattr(td, attr) for attr in ('days', 'seconds', 'microseconds')]
	dsm = map(lambda elem: elem / divisor, dsm)
	return datetime.timedelta(*dsm)


def parse_timedelta(str):
	"""
	Take a string representing a span of time and parse it to a time delta.
	Accepts any string of comma-separated numbers each with a unit indicator.

	>>> parse_timedelta('1 day')
	datetime.timedelta(days=1)

	>>> parse_timedelta('1 day, 30 seconds')
	datetime.timedelta(days=1, seconds=30)

	>>> parse_timedelta('47.32 days, 20 minutes, 15.4 milliseconds')
	datetime.timedelta(days=47, seconds=28848, microseconds=15400)

	Supports weeks, months, years

	>>> parse_timedelta('1 week')
	datetime.timedelta(days=7)

	>>> parse_timedelta('1 year, 1 month')
	datetime.timedelta(days=395, seconds=58685)

	Note that months and years strict intervals, not aligned
	to a calendar:

	>>> now = datetime.datetime.now()
	>>> later = now + parse_timedelta('1 year')
	>>> diff = later.replace(year=now.year) - now
	>>> diff.seconds
	20940
	"""
	deltas = (_parse_timedelta_part(part.strip()) for part in str.split(','))
	return sum(deltas, datetime.timedelta())

def parse_timedelta ( str ) : deltas = ( _parse_timedelta_part ( part . strip ( ) ) for part in str . split ( ',' ) ) return sum ( deltas , datetime . timedelta ( ) )

Take a string representing a span of time and parse it to a time delta . Accepts any string of comma - separated numbers each with a unit indicator .


def divide_timedelta(td1, td2):
	"""
	Get the ratio of two timedeltas

	>>> one_day = datetime.timedelta(days=1)
	>>> one_hour = datetime.timedelta(hours=1)
	>>> divide_timedelta(one_hour, one_day) == 1 / 24
	True
	"""
	try:
		return td1 / td2
	except TypeError:
		# Python 3.2 gets division
		# http://bugs.python.org/issue2706
		return td1.total_seconds() / td2.total_seconds()

def
divide_timedelta
(
td1
,
td2
)
:
try
:
return
td1
/
td2
except
TypeError
:
# Python 3.2 gets division
# http://bugs.python.org/issue2706
return
td1
.
total_seconds
(
)
/
td2
.
total_seconds
(
)


def date_range(start=None, stop=None, step=None):
	"""
	Much like the built-in function range, but works with dates

	>>> range_items = date_range(
	...     datetime.datetime(2005,12,21),
	...     datetime.datetime(2005,12,25),
	... )
	>>> my_range = tuple(range_items)
	>>> datetime.datetime(2005,12,21) in my_range
	True
	>>> datetime.datetime(2005,12,22) in my_range
	True
	>>> datetime.datetime(2005,12,25) in my_range
	False
	"""
	if step is None:
		step = datetime.timedelta(days=1)
	if start is None:
		start = datetime.datetime.now()
	while start < stop:
		yield start
		start += step

def date_range ( start = None , stop = None , step = None ) : if step is None : step = datetime . timedelta ( days = 1 ) if start is None : start = datetime . datetime . now ( ) while start < stop : yield start start += step

Much like the built - in function range but works with dates


def construct_datetime(cls, *args, **kwargs):
		"""Construct a datetime.datetime from a number of different time
		types found in python and pythonwin"""
		if len(args) == 1:
			arg = args[0]
			method = cls.__get_dt_constructor(
				type(arg).__module__,
				type(arg).__name__,
			)
			result = method(arg)
			try:
				result = result.replace(tzinfo=kwargs.pop('tzinfo'))
			except KeyError:
				pass
			if kwargs:
				first_key = kwargs.keys()[0]
				tmpl = (
					"{first_key} is an invalid keyword "
					"argument for this function."
				)
				raise TypeError(tmpl.format(**locals()))
		else:
			result = datetime.datetime(*args, **kwargs)
		return result

def
construct_datetime
(
cls
,
*
args
,
*
*
kwargs
)
:
if
len
(
args
)
==
1
:
arg
=
args
[
0
]
method
=
cls
.
__get_dt_constructor
(
type
(
arg
)
.
__module__
,
type
(
arg
)
.
__name__
,
)
result
=
method
(
arg
)
try
:
result
=
result
.
replace
(
tzinfo
=
kwargs
.
pop
(
'tzinfo'
)
)
except
KeyError
:
pass
if
kwargs
:
first_key
=
kwargs
.
keys
(
)
[
0
]
tmpl
=
(
"{first_key} is an invalid keyword "
"argument for this function."
)
raise
TypeError
(
tmpl
.
format
(
*
*
locals
(
)
)
)
else
:
result
=
datetime
.
datetime
(
*
args
,
*
*
kwargs
)
return
result

Construct a datetime . datetime from a number of different time types found in python and pythonwin


def get(self, query, responseformat="geojson", verbosity="body", build=True):
        """Pass in an Overpass query in Overpass QL."""
        # Construct full Overpass query
        if build:
            full_query = self._construct_ql_query(
                query, responseformat=responseformat, verbosity=verbosity
            )
        else:
            full_query = query

        if self.debug:
            logging.getLogger().info(query)

        # Get the response from Overpass
        r = self._get_from_overpass(full_query)
        content_type = r.headers.get("content-type")

        if self.debug:
            print(content_type)
        if content_type == "text/csv":
            result = []
            reader = csv.reader(StringIO(r.text), delimiter="\t")
            for row in reader:
                result.append(row)
            return result
        elif content_type in ("text/xml", "application/xml", "application/osm3s+xml"):
            return r.text
        elif content_type == "application/json":
            response = json.loads(r.text)

        if not build:
            return response

        # Check for valid answer from Overpass.
        # A valid answer contains an 'elements' key at the root level.
        if "elements" not in response:
            raise UnknownOverpassError("Received an invalid answer from Overpass.")

        # If there is a 'remark' key, it spells trouble.
        overpass_remark = response.get("remark", None)
        if overpass_remark and overpass_remark.startswith("runtime error"):
            raise ServerRuntimeError(overpass_remark)

        if responseformat is not "geojson":
            return response

        # construct geojson
        return self._as_geojson(response["elements"])

Pass in an Overpass query in Overpass QL .


def get_ports_count(context, filters=None):
    """Return the number of ports.

    The result depends on the identity of the user making the request
    (as indicated by the context) as well as any filters.
    : param context: neutron api request context
    : param filters: a dictionary with keys that are valid keys for
        a port as listed in the RESOURCE_ATTRIBUTE_MAP object
        in neutron/api/v2/attributes.py.  Values in this dictionary
        are an iterable containing values that will be used for an exact
        match comparison for that value.  Each result returned by this
        function will have matched one of the values for each key in
        filters.

    NOTE: this method is optional, as it was not part of the originally
          defined plugin API.
    """
    LOG.info("get_ports_count for tenant %s filters %s" %
             (context.tenant_id, filters))
    return db_api.port_count_all(context, join_security_groups=True, **filters)


def _allocate_from_v6_subnet(self, context, net_id, subnet,
                                 port_id, reuse_after, ip_address=None,
                                 **kwargs):
        """This attempts to allocate v6 addresses as per RFC2462 and RFC3041.

        To accomodate this, we effectively treat all v6 assignment as a
        first time allocation utilizing the MAC address of the VIF. Because
        we recycle MACs, we will eventually attempt to recreate a previously
        generated v6 address. Instead of failing, we've opted to handle
        reallocating that address in this method.

        This should provide a performance boost over attempting to check
        each and every subnet in the existing reallocate logic, as we'd
        have to iterate over each and every subnet returned
        """

        LOG.info("Attempting to allocate a v6 address - [{0}]".format(
            utils.pretty_kwargs(network_id=net_id, subnet=subnet,
                                port_id=port_id, ip_address=ip_address)))

        if ip_address:
            LOG.info("IP %s explicitly requested, deferring to standard "
                     "allocation" % ip_address)
            return self._allocate_from_subnet(context, net_id=net_id,
                                              subnet=subnet, port_id=port_id,
                                              reuse_after=reuse_after,
                                              ip_address=ip_address, **kwargs)
        else:
            mac = kwargs.get("mac_address")
            if mac:
                mac = kwargs["mac_address"].get("address")

            if subnet and subnet["ip_policy"]:
                ip_policy_cidrs = subnet["ip_policy"].get_cidrs_ip_set()
            else:
                ip_policy_cidrs = netaddr.IPSet([])

            for tries, ip_address in enumerate(
                    generate_v6(mac, port_id, subnet["cidr"])):

                LOG.info("Attempt {0} of {1}".format(
                    tries + 1, CONF.QUARK.v6_allocation_attempts))

                if tries > CONF.QUARK.v6_allocation_attempts - 1:
                    LOG.info("Exceeded v6 allocation attempts, bailing")
                    raise ip_address_failure(net_id)

                ip_address = netaddr.IPAddress(ip_address).ipv6()
                LOG.info("Generated a new v6 address {0}".format(
                    str(ip_address)))

                if (ip_policy_cidrs is not None and
                        ip_address in ip_policy_cidrs):
                    LOG.info("Address {0} excluded by policy".format(
                        str(ip_address)))
                    continue

                try:
                    with context.session.begin():
                        address = db_api.ip_address_create(
                            context, address=ip_address,
                            subnet_id=subnet["id"],
                            version=subnet["ip_version"], network_id=net_id,
                            address_type=kwargs.get('address_type',
                                                    ip_types.FIXED))
                        return address
                except db_exception.DBDuplicateEntry:
                    # This shouldn't ever happen, since we hold a unique MAC
                    # address from the previous IPAM step.
                    LOG.info("{0} exists but was already "
                             "allocated".format(str(ip_address)))
                    LOG.debug("Duplicate entry found when inserting subnet_id"
                              " %s ip_address %s", subnet["id"], ip_address)

This attempts to allocate v6 addresses as per RFC2462 and RFC3041 .


def _create_flip(context, flip, port_fixed_ips):
    """Associates the flip with ports and creates it with the flip driver

    :param context: neutron api request context.
    :param flip: quark.db.models.IPAddress object representing a floating IP
    :param port_fixed_ips: dictionary of the structure:
    {"<id of port>": {"port": <quark.db.models.Port>,
     "fixed_ip": "<fixed ip address>"}}
    :return: None
    """
    if port_fixed_ips:
        context.session.begin()
        try:
            ports = [val['port'] for val in port_fixed_ips.values()]
            flip = db_api.port_associate_ip(context, ports, flip,
                                            port_fixed_ips.keys())

            for port_id in port_fixed_ips:
                fixed_ip = port_fixed_ips[port_id]['fixed_ip']
                flip = db_api.floating_ip_associate_fixed_ip(context, flip,
                                                             fixed_ip)

            flip_driver = registry.DRIVER_REGISTRY.get_driver()

            flip_driver.register_floating_ip(flip, port_fixed_ips)
            context.session.commit()
        except Exception:
            context.session.rollback()
            raise

    # alexm: Notify from this method for consistency with _delete_flip
    billing.notify(context, billing.IP_ASSOC, flip)


def create_floatingip(context, content):
    """Allocate or reallocate a floating IP.

    :param context: neutron api request context.
    :param content: dictionary describing the floating ip, with keys
        as listed in the RESOURCE_ATTRIBUTE_MAP object in
        neutron/api/v2/attributes.py.  All keys will be populated.

    :returns: Dictionary containing details for the new floating IP.  If values
        are declared in the fields parameter, then only those keys will be
        present.
    """
    LOG.info('create_floatingip %s for tenant %s and body %s' %
             (id, context.tenant_id, content))
    network_id = content.get('floating_network_id')
    # TODO(blogan): Since the extension logic will reject any requests without
    # floating_network_id, is this still needed?
    if not network_id:
        raise n_exc.BadRequest(resource='floating_ip',
                               msg='floating_network_id is required.')
    fixed_ip_address = content.get('fixed_ip_address')
    ip_address = content.get('floating_ip_address')
    port_id = content.get('port_id')
    port = None
    port_fixed_ip = {}

    network = _get_network(context, network_id)
    if port_id:
        port = _get_port(context, port_id)
        fixed_ip = _get_fixed_ip(context, fixed_ip_address, port)
        port_fixed_ip = {port.id: {'port': port, 'fixed_ip': fixed_ip}}
    flip = _allocate_ip(context, network, port, ip_address, ip_types.FLOATING)
    _create_flip(context, flip, port_fixed_ip)
    return v._make_floating_ip_dict(flip, port_id)


def update_floatingip(context, id, content):
    """Update an existing floating IP.

    :param context: neutron api request context.
    :param id: id of the floating ip
    :param content: dictionary with keys indicating fields to update.
        valid keys are those that have a value of True for 'allow_put'
        as listed in the RESOURCE_ATTRIBUTE_MAP object in
        neutron/api/v2/attributes.py.

    :returns: Dictionary containing details for the new floating IP.  If values
        are declared in the fields parameter, then only those keys will be
        present.
    """

    LOG.info('update_floatingip %s for tenant %s and body %s' %
             (id, context.tenant_id, content))

    if 'port_id' not in content:
        raise n_exc.BadRequest(resource='floating_ip',
                               msg='port_id is required.')

    requested_ports = []
    if content.get('port_id'):
        requested_ports = [{'port_id': content.get('port_id')}]
    flip = _update_flip(context, id, ip_types.FLOATING, requested_ports)
    return v._make_floating_ip_dict(flip)


def delete_floatingip(context, id):
    """deallocate a floating IP.

    :param context: neutron api request context.
    :param id: id of the floating ip
    """

    LOG.info('delete_floatingip %s for tenant %s' % (id, context.tenant_id))

    _delete_flip(context, id, ip_types.FLOATING)

def
delete_floatingip
(
context
,
id
)
:
LOG
.
info
(
'delete_floatingip %s for tenant %s'
%
(
id
,
context
.
tenant_id
)
)
_delete_flip
(
context
,
id
,
ip_types
.
FLOATING
)


def get_floatingip(context, id, fields=None):
    """Retrieve a floating IP.

    :param context: neutron api request context.
    :param id: The UUID of the floating IP.
    :param fields: a list of strings that are valid keys in a
        floating IP dictionary as listed in the RESOURCE_ATTRIBUTE_MAP
        object in neutron/api/v2/attributes.py. Only these fields
        will be returned.

    :returns: Dictionary containing details for the floating IP.  If values
        are declared in the fields parameter, then only those keys will be
        present.
    """
    LOG.info('get_floatingip %s for tenant %s' % (id, context.tenant_id))

    filters = {'address_type': ip_types.FLOATING, '_deallocated': False}

    floating_ip = db_api.floating_ip_find(context, id=id, scope=db_api.ONE,
                                          **filters)

    if not floating_ip:
        raise q_exc.FloatingIpNotFound(id=id)

    return v._make_floating_ip_dict(floating_ip)

def
get_floatingip
(
context
,
id
,
fields
=
)
:
LOG
.
info
(
'get_floatingip %s for tenant %s'
%
(
id
,
context
.
tenant_id
)
)
filters
=
{
'address_type'
:
ip_types
.
FLOATING
,
'_deallocated'
:
False
}
floating_ip
=
db_api
.
floating_ip_find
(
context
,
id
=
id
,
scope
=
db_api
.
ONE
,
*
*
filters
)
if
not
floating_ip
:
raise
q_exc
.
FloatingIpNotFound
(
id
=
id
)
return
v
.
_make_floating_ip_dict
(
floating_ip
)


def get_floatingips(context, filters=None, fields=None, sorts=['id'],
                    limit=None, marker=None, page_reverse=False):
    """Retrieve a list of floating ips.

    :param context: neutron api request context.
    :param filters: a dictionary with keys that are valid keys for
        a floating ip as listed in the RESOURCE_ATTRIBUTE_MAP object
        in neutron/api/v2/attributes.py.  Values in this dictionary
        are an iterable containing values that will be used for an exact
        match comparison for that value.  Each result returned by this
        function will have matched one of the values for each key in
        filters.
    :param fields: a list of strings that are valid keys in a
        floating IP dictionary as listed in the RESOURCE_ATTRIBUTE_MAP
        object in neutron/api/v2/attributes.py. Only these fields
        will be returned.

    :returns: List of floating IPs that are accessible to the tenant who
        submits the request (as indicated by the tenant id of the context)
        as well as any filters.
    """
    LOG.info('get_floatingips for tenant %s filters %s fields %s' %
             (context.tenant_id, filters, fields))

    floating_ips = _get_ips_by_type(context, ip_types.FLOATING,
                                    filters=filters, fields=fields)

    return [v._make_floating_ip_dict(flip) for flip in floating_ips]


def get_floatingips_count(context, filters=None):
    """Return the number of floating IPs.

    :param context: neutron api request context
    :param filters: a dictionary with keys that are valid keys for
        a floating IP as listed in the RESOURCE_ATTRIBUTE_MAP object
        in neutron/api/v2/attributes.py.  Values in this dictionary
        are an iterable containing values that will be used for an exact
        match comparison for that value.  Each result returned by this
        function will have matched one of the values for each key in
        filters.

    :returns: The number of floating IPs that are accessible to the tenant who
        submits the request (as indicated by the tenant id of the context)
        as well as any filters.

    NOTE: this method is optional, as it was not part of the originally
          defined plugin API.
    """
    LOG.info('get_floatingips_count for tenant %s filters %s' %
             (context.tenant_id, filters))

    if filters is None:
        filters = {}

    filters['_deallocated'] = False
    filters['address_type'] = ip_types.FLOATING
    count = db_api.ip_address_count_all(context, filters)

    LOG.info('Found %s floating ips for tenant %s' % (count,
                                                      context.tenant_id))
    return count


def create_scalingip(context, content):
    """Allocate or reallocate a scaling IP.

    :param context: neutron api request context.
    :param content: dictionary describing the scaling ip, with keys
        as listed in the RESOURCE_ATTRIBUTE_MAP object in
        neutron/api/v2/attributes.py.  All keys will be populated.

    :returns: Dictionary containing details for the new scaling IP.  If values
        are declared in the fields parameter, then only those keys will be
        present.
    """
    LOG.info('create_scalingip for tenant %s and body %s',
             context.tenant_id, content)
    network_id = content.get('scaling_network_id')
    ip_address = content.get('scaling_ip_address')
    requested_ports = content.get('ports', [])

    network = _get_network(context, network_id)
    port_fixed_ips = {}
    for req_port in requested_ports:
        port = _get_port(context, req_port['port_id'])
        fixed_ip = _get_fixed_ip(context, req_port.get('fixed_ip_address'),
                                 port)
        port_fixed_ips[port.id] = {"port": port, "fixed_ip": fixed_ip}
    scip = _allocate_ip(context, network, None, ip_address, ip_types.SCALING)
    _create_flip(context, scip, port_fixed_ips)
    return v._make_scaling_ip_dict(scip)


def update_scalingip(context, id, content):
    """Update an existing scaling IP.

    :param context: neutron api request context.
    :param id: id of the scaling ip
    :param content: dictionary with keys indicating fields to update.
        valid keys are those that have a value of True for 'allow_put'
        as listed in the RESOURCE_ATTRIBUTE_MAP object in
        neutron/api/v2/attributes.py.

    :returns: Dictionary containing details for the new scaling IP.  If values
        are declared in the fields parameter, then only those keys will be
        present.
    """
    LOG.info('update_scalingip %s for tenant %s and body %s' %
             (id, context.tenant_id, content))
    requested_ports = content.get('ports', [])
    flip = _update_flip(context, id, ip_types.SCALING, requested_ports)
    return v._make_scaling_ip_dict(flip)


def delete_scalingip(context, id):
    """Deallocate a scaling IP.

    :param context: neutron api request context.
    :param id: id of the scaling ip
    """
    LOG.info('delete_scalingip %s for tenant %s' % (id, context.tenant_id))
    _delete_flip(context, id, ip_types.SCALING)

def
delete_scalingip
(
context
,
id
)
:
LOG
.
info
(
'delete_scalingip %s for tenant %s'
%
(
id
,
context
.
tenant_id
)
)
_delete_flip
(
context
,
id
,
ip_types
.
SCALING
)


def get_scalingip(context, id, fields=None):
    """Retrieve a scaling IP.

    :param context: neutron api request context.
    :param id: The UUID of the scaling IP.
    :param fields: a list of strings that are valid keys in a
        scaling IP dictionary as listed in the RESOURCE_ATTRIBUTE_MAP
        object in neutron/api/v2/attributes.py. Only these fields
        will be returned.

    :returns: Dictionary containing details for the scaling IP.  If values
        are declared in the fields parameter, then only those keys will be
        present.
    """
    LOG.info('get_scalingip %s for tenant %s' % (id, context.tenant_id))
    filters = {'address_type': ip_types.SCALING, '_deallocated': False}
    scaling_ip = db_api.floating_ip_find(context, id=id, scope=db_api.ONE,
                                         **filters)
    if not scaling_ip:
        raise q_exc.ScalingIpNotFound(id=id)
    return v._make_scaling_ip_dict(scaling_ip)

def
get_scalingip
(
context
,
id
,
fields
=
)
:
LOG
.
info
(
'get_scalingip %s for tenant %s'
%
(
id
,
context
.
tenant_id
)
)
filters
=
{
'address_type'
:
ip_types
.
SCALING
,
'_deallocated'
:
False
}
scaling_ip
=
db_api
.
floating_ip_find
(
context
,
id
=
id
,
scope
=
db_api
.
ONE
,
*
*
filters
)
if
not
scaling_ip
:
raise
q_exc
.
ScalingIpNotFound
(
id
=
id
)
return
v
.
_make_scaling_ip_dict
(
scaling_ip
)


def get_scalingips(context, filters=None, fields=None, sorts=['id'],
                   limit=None, marker=None, page_reverse=False):
    """Retrieve a list of scaling ips.

    :param context: neutron api request context.
    :param filters: a dictionary with keys that are valid keys for
        a scaling ip as listed in the RESOURCE_ATTRIBUTE_MAP object
        in neutron/api/v2/attributes.py.  Values in this dictionary
        are an iterable containing values that will be used for an exact
        match comparison for that value.  Each result returned by this
        function will have matched one of the values for each key in
        filters.
    :param fields: a list of strings that are valid keys in a
        scaling IP dictionary as listed in the RESOURCE_ATTRIBUTE_MAP
        object in neutron/api/v2/attributes.py. Only these fields
        will be returned.

    :returns: List of scaling IPs that are accessible to the tenant who
        submits the request (as indicated by the tenant id of the context)
        as well as any filters.
    """
    LOG.info('get_scalingips for tenant %s filters %s fields %s' %
             (context.tenant_id, filters, fields))
    scaling_ips = _get_ips_by_type(context, ip_types.SCALING,
                                   filters=filters, fields=fields)
    return [v._make_scaling_ip_dict(scip) for scip in scaling_ips]


def is_isonet_vif(vif):
    """Determine if a vif is on isonet

    Returns True if a vif belongs to an isolated network by checking
    for a nicira interface id.
    """
    nicira_iface_id = vif.record.get('other_config').get('nicira-iface-id')

    if nicira_iface_id:
        return True

    return False

def is_isonet_vif ( vif ) : nicira_iface_id = vif . record . get ( 'other_config' ) . get ( 'nicira-iface-id' ) if nicira_iface_id : return True return False


def partition_vifs(xapi_client, interfaces, security_group_states):
    """Splits VIFs into three explicit categories and one implicit

    Added - Groups exist in Redis that have not been ack'd and the VIF
            is not tagged.
            Action: Tag the VIF and apply flows
    Updated - Groups exist in Redis that have not been ack'd and the VIF
              is already tagged
              Action: Do not tag the VIF, do apply flows
    Removed - Groups do NOT exist in Redis but the VIF is tagged
              Action: Untag the VIF, apply default flows
    Self-Heal - Groups are ack'd in Redis but the VIF is untagged. We treat
                this case as if it were an "added" group.
                Action: Tag the VIF and apply flows
    NOOP - The VIF is not tagged and there are no matching groups in Redis.
           This is our implicit category
           Action: Do nothing
    """
    added = []
    updated = []
    removed = []

    for vif in interfaces:
        # Quark should not action on isonet vifs in regions that use FLIP
        if ('floating_ip' in CONF.QUARK.environment_capabilities and
                is_isonet_vif(vif)):
            continue

        vif_has_groups = vif in security_group_states
        if vif.tagged and vif_has_groups and\
                security_group_states[vif][sg_cli.SECURITY_GROUP_ACK]:
            # Already ack'd these groups and VIF is tagged, reapply.
            # If it's not tagged, fall through and have it self-heal
            continue

        if vif.tagged:
            if vif_has_groups:
                updated.append(vif)
            else:
                removed.append(vif)
        else:
            if vif_has_groups:
                added.append(vif)
            # if not tagged and no groups, skip

    return added, updated, removed


def get_groups_to_ack(groups_to_ack, init_sg_states, curr_sg_states):
    """Compares initial security group rules with current sg rules.

    Given the groups that were successfully returned from
        xapi_client.update_interfaces call, compare initial and current
        security group rules to determine if an update occurred during
        the window that the xapi_client.update_interfaces was executing.
        Return a list of vifs whose security group rules have not changed.
    """
    security_groups_changed = []
    # Compare current security group rules with initial rules.
    for vif in groups_to_ack:
        initial_state = init_sg_states[vif][sg_cli.SECURITY_GROUP_HASH_ATTR]
        current_state = curr_sg_states[vif][sg_cli.SECURITY_GROUP_HASH_ATTR]
        bad_match_msg = ('security group rules were changed for vif "%s" while'
                         ' executing xapi_client.update_interfaces.'
                         ' Will not ack rule.' % vif)
        # If lists are different lengths, they're automatically different.
        if len(initial_state) != len(current_state):
            security_groups_changed.append(vif)
            LOG.info(bad_match_msg)
        elif len(initial_state) > 0:
            # Compare rules in equal length lists.
            for rule in current_state:
                if rule not in initial_state:
                    security_groups_changed.append(vif)
                    LOG.info(bad_match_msg)
                    break

    # Only ack groups whose rules have not changed since update. If
    # rules do not match, do not add them to ret so the change
    # can be picked up on the next cycle.
    ret = [group for group in groups_to_ack
           if group not in security_groups_changed]
    return ret

Compares initial security group rules with current sg rules .


def run():
    """Fetches changes and applies them to VIFs periodically

    Process as of RM11449:
    * Get all groups from redis
    * Fetch ALL VIFs from Xen
    * Walk ALL VIFs and partition them into added, updated and removed
    * Walk the final "modified" VIFs list and apply flows to each
    """
    groups_client = sg_cli.SecurityGroupsClient()
    xapi_client = xapi.XapiClient()

    interfaces = set()
    while True:
        try:
            interfaces = xapi_client.get_interfaces()
        except Exception:
            LOG.exception("Unable to get instances/interfaces from xapi")
            _sleep()
            continue

        try:
            sg_states = groups_client.get_security_group_states(interfaces)
            new_sg, updated_sg, removed_sg = partition_vifs(xapi_client,
                                                            interfaces,
                                                            sg_states)
            xapi_client.update_interfaces(new_sg, updated_sg, removed_sg)
            groups_to_ack = [v for v in new_sg + updated_sg if v.success]
            # NOTE(quade): This solves a race condition where a security group
            # rule may have changed between the time the sg_states were called
            # and when they were officially ack'd. It functions as a compare
            # and set. This is a fix until we get onto a proper messaging
            # queue. NCP-2287
            sg_sts_curr = groups_client.get_security_group_states(interfaces)
            groups_to_ack = get_groups_to_ack(groups_to_ack, sg_states,
                                              sg_sts_curr)
            # This list will contain all the security group rules that do not
            # match
            ack_groups(groups_client, groups_to_ack)

        except Exception:
            LOG.exception("Unable to get security groups from registry and "
                          "apply them to xapi")
            _sleep()
            continue

        _sleep()


def delete_tenant_quota(context, tenant_id):
        """Delete the quota entries for a given tenant_id.

        Atfer deletion, this tenant will use default quota values in conf.
        """

        tenant_quotas = context.session.query(Quota)
        tenant_quotas = tenant_quotas.filter_by(tenant_id=tenant_id)
        tenant_quotas.delete()

def delete_tenant_quota ( context , tenant_id ) : tenant_quotas = context . session . query ( Quota ) tenant_quotas = tenant_quotas . filter_by ( tenant_id = tenant_id ) tenant_quotas . delete ( )

Delete the quota entries for a given tenant_id .


def _validate_subnet_cidr(context, network_id, new_subnet_cidr):
    """Validate the CIDR for a subnet.

    Verifies the specified CIDR does not overlap with the ones defined
    for the other subnets specified for this network, or with any other
    CIDR if overlapping IPs are disabled.

    """
    if neutron_cfg.cfg.CONF.allow_overlapping_ips:
        return

    try:
        new_subnet_ipset = netaddr.IPSet([new_subnet_cidr])
    except TypeError:
        LOG.exception("Invalid or missing cidr: %s" % new_subnet_cidr)
        raise n_exc.BadRequest(resource="subnet",
                               msg="Invalid or missing cidr")

    filters = {
        'network_id': network_id,
        'shared': [False]
    }
    # Using admin context here, in case we actually share networks later
    subnet_list = db_api.subnet_find(context=context.elevated(), **filters)

    for subnet in subnet_list:
        if (netaddr.IPSet([subnet.cidr]) & new_subnet_ipset):
            # don't give out details of the overlapping subnet
            err_msg = (_("Requested subnet with cidr: %(cidr)s for "
                         "network: %(network_id)s overlaps with another "
                         "subnet") %
                       {'cidr': new_subnet_cidr,
                        'network_id': network_id})
            LOG.error(_("Validation for CIDR: %(new_cidr)s failed - "
                        "overlaps with subnet %(subnet_id)s "
                        "(CIDR: %(cidr)s)"),
                      {'new_cidr': new_subnet_cidr,
                       'subnet_id': subnet.id,
                       'cidr': subnet.cidr})
            raise n_exc.InvalidInput(error_message=err_msg)


def get_subnet(context, id, fields=None):
    """Retrieve a subnet.

    : param context: neutron api request context
    : param id: UUID representing the subnet to fetch.
    : param fields: a list of strings that are valid keys in a
        subnet dictionary as listed in the RESOURCE_ATTRIBUTE_MAP
        object in neutron/api/v2/attributes.py. Only these fields
        will be returned.
    """
    LOG.info("get_subnet %s for tenant %s with fields %s" %
             (id, context.tenant_id, fields))
    subnet = db_api.subnet_find(context=context, limit=None,
                                page_reverse=False, sorts=['id'],
                                marker_obj=None, fields=None, id=id,
                                join_dns=True, join_routes=True,
                                scope=db_api.ONE)
    if not subnet:
        raise n_exc.SubnetNotFound(subnet_id=id)

    cache = subnet.get("_allocation_pool_cache")
    if not cache:
        new_cache = subnet.allocation_pools
        db_api.subnet_update_set_alloc_pool_cache(context, subnet, new_cache)
    return v._make_subnet_dict(subnet)


def get_subnets(context, limit=None, page_reverse=False, sorts=['id'],
                marker=None, filters=None, fields=None):
    """Retrieve a list of subnets.

    The contents of the list depends on the identity of the user
    making the request (as indicated by the context) as well as any
    filters.
    : param context: neutron api request context
    : param filters: a dictionary with keys that are valid keys for
        a subnet as listed in the RESOURCE_ATTRIBUTE_MAP object
        in neutron/api/v2/attributes.py.  Values in this dictiontary
        are an iterable containing values that will be used for an exact
        match comparison for that value.  Each result returned by this
        function will have matched one of the values for each key in
        filters.
    : param fields: a list of strings that are valid keys in a
        subnet dictionary as listed in the RESOURCE_ATTRIBUTE_MAP
        object in neutron/api/v2/attributes.py. Only these fields
        will be returned.
    """
    LOG.info("get_subnets for tenant %s with filters %s fields %s" %
             (context.tenant_id, filters, fields))
    filters = filters or {}
    subnets = db_api.subnet_find(context, limit=limit,
                                 page_reverse=page_reverse, sorts=sorts,
                                 marker_obj=marker, join_dns=True,
                                 join_routes=True, join_pool=True, **filters)
    for subnet in subnets:
        cache = subnet.get("_allocation_pool_cache")
        if not cache:
            db_api.subnet_update_set_alloc_pool_cache(
                context, subnet, subnet.allocation_pools)
    return v._make_subnets_list(subnets, fields=fields)


def get_subnets_count(context, filters=None):
    """Return the number of subnets.

    The result depends on the identity of the user making the request
    (as indicated by the context) as well as any filters.
    : param context: neutron api request context
    : param filters: a dictionary with keys that are valid keys for
        a network as listed in the RESOURCE_ATTRIBUTE_MAP object
        in neutron/api/v2/attributes.py.  Values in this dictiontary
        are an iterable containing values that will be used for an exact
        match comparison for that value.  Each result returned by this
        function will have matched one of the values for each key in
        filters.

    NOTE: this method is optional, as it was not part of the originally
          defined plugin API.
    """
    LOG.info("get_subnets_count for tenant %s with filters %s" %
             (context.tenant_id, filters))
    return db_api.subnet_count_all(context, **filters)


def delete_subnet(context, id):
    """Delete a subnet.

    : param context: neutron api request context
    : param id: UUID representing the subnet to delete.
    """
    LOG.info("delete_subnet %s for tenant %s" % (id, context.tenant_id))
    with context.session.begin():
        subnet = db_api.subnet_find(context, id=id, scope=db_api.ONE)
        if not subnet:
            raise n_exc.SubnetNotFound(subnet_id=id)

        if not context.is_admin:
            if STRATEGY.is_provider_network(subnet.network_id):
                if subnet.tenant_id == context.tenant_id:
                    # A tenant can't delete subnets on provider network
                    raise n_exc.NotAuthorized(subnet_id=id)
                else:
                    # Raise a NotFound here because the foreign tenant
                    # does not have to know about other tenant's subnet
                    # existence.
                    raise n_exc.SubnetNotFound(subnet_id=id)

        _delete_subnet(context, subnet)


def _perform_async_update_rule(context, id, db_sg_group, rule_id, action):
    """Updates a SG rule async and return the job information.

    Only happens if the security group has associated ports. If the async
    connection fails the update continues (legacy mode).
    """
    rpc_reply = None
    sg_rpc = sg_rpc_api.QuarkSGAsyncProcessClient()
    ports = db_api.sg_gather_associated_ports(context, db_sg_group)
    if len(ports) > 0:
        rpc_reply = sg_rpc.start_update(context, id, rule_id, action)
        if rpc_reply:
            job_id = rpc_reply['job_id']
            job_api.add_job_to_context(context, job_id)
        else:
            LOG.error("Async update failed. Is the worker running?")

Updates a SG rule async and return the job information .


def update_security_group_rule(context, id, security_group_rule):
    '''Updates a rule and updates the ports'''
    LOG.info("update_security_group_rule for tenant %s" %
             (context.tenant_id))
    new_rule = security_group_rule["security_group_rule"]
    # Only allow updatable fields
    new_rule = _filter_update_security_group_rule(new_rule)

    with context.session.begin():
        rule = db_api.security_group_rule_find(context, id=id,
                                               scope=db_api.ONE)
        if not rule:
            raise sg_ext.SecurityGroupRuleNotFound(id=id)

        db_rule = db_api.security_group_rule_update(context, rule, **new_rule)

        group_id = db_rule.group_id
        group = db_api.security_group_find(context, id=group_id,
                                           scope=db_api.ONE)
        if not group:
            raise sg_ext.SecurityGroupNotFound(id=group_id)

    if group:
        _perform_async_update_rule(context, group_id, group, rule.id,
                                   RULE_UPDATE)

    return v._make_security_group_rule_dict(db_rule)


def get_public_net_id(self):
        """Returns the public net id"""
        for id, net_params in self.strategy.iteritems():
            if id == CONF.QUARK.public_net_id:
                return id
        return None

def get_public_net_id ( self ) : for id , net_params in self . strategy . iteritems ( ) : if id == CONF . QUARK . public_net_id : return id return None


def opt_args_decorator(func):
    """A decorator to be used on another decorator

    This is done to allow separate handling on the basis of argument values
    """
    @wraps(func)
    def wrapped_dec(*args, **kwargs):
        if len(args) == 1 and len(kwargs) == 0 and callable(args[0]):
            # actual decorated function
            return func(args[0])
        else:
            # decorator arguments
            return lambda realf: func(realf, *args, **kwargs)

    return wrapped_dec

def
opt_args_decorator
(
func
)
:
@
wraps
(
func
)
def
wrapped_dec
(
*
args
,
*
*
kwargs
)
:
if
len
(
args
)
==
1
and
len
(
kwargs
)
==
0
and
callable
(
args
[
0
]
)
:
# actual decorated function
return
func
(
args
[
0
]
)
else
:
# decorator arguments
return
lambda
realf
:
func
(
realf
,
*
args
,
*
*
kwargs
)
return
wrapped_dec


def _fix_missing_tenant_id(self, context, body, key):
        """Will add the tenant_id to the context from body.

        It is assumed that the body must have a tenant_id because neutron
        core could never have gotten here otherwise.
        """
        if not body:
            raise n_exc.BadRequest(resource=key,
                                   msg="Body malformed")
        resource = body.get(key)
        if not resource:
            raise n_exc.BadRequest(resource=key,
                                   msg="Body malformed")
        if context.tenant_id is None:
            context.tenant_id = resource.get("tenant_id")
        if context.tenant_id is None:
            msg = _("Running without keystone AuthN requires "
                    "that tenant_id is specified")
            raise n_exc.BadRequest(resource=key, msg=msg)

Will add the tenant_id to the context from body .