def _move_temp_binary_to_path(tmp_binary_path):
    """Moves the temporary binary to the location of the binary that's currently being run.
    Preserves owner, group, and permissions of original binary"""
    # pylint: disable=E1101
    binary_path = _get_binary_location()
    if not binary_path.endswith(constants.DUSTY_BINARY_NAME):
        raise RuntimeError('Refusing to overwrite binary {}'.format(binary_path))
    st = os.stat(binary_path)
    permissions = st.st_mode
    owner = st.st_uid
    group = st.st_gid
    shutil.move(tmp_binary_path, binary_path)
    os.chown(binary_path, owner, group)
    os.chmod(binary_path, permissions)
    return binary_path

def
_move_temp_binary_to_path
(
tmp_binary_path
)
:
# pylint: disable=E1101
binary_path
=
_get_binary_location
(
)
if
not
binary_path
.
endswith
(
constants
.
DUSTY_BINARY_NAME
)
:
raise
RuntimeError
(
'Refusing to overwrite binary {}'
.
format
(
binary_path
)
)
st
=
os
.
stat
(
binary_path
)
permissions
=
st
.
st_mode
owner
=
st
.
st_uid
group
=
st
.
st_gid
shutil
.
move
(
tmp_binary_path
,
binary_path
)
os
.
chown
(
binary_path
,
owner
,
group
)
os
.
chmod
(
binary_path
,
permissions
)
return
binary_path

Moves the temporary binary to the location of the binary that s currently being run . Preserves owner group and permissions of original binary


def parallel_task_queue(pool_size=multiprocessing.cpu_count()):
    """Context manager for setting up a TaskQueue. Upon leaving the
    context manager, all tasks that were enqueued will be executed
    in parallel subject to `pool_size` concurrency constraints."""
    task_queue = TaskQueue(pool_size)
    yield task_queue
    task_queue.execute()

def parallel_task_queue ( pool_size = multiprocessing . cpu_count ( ) ) : task_queue = TaskQueue ( pool_size ) yield task_queue task_queue . execute ( )

Context manager for setting up a TaskQueue . Upon leaving the context manager all tasks that were enqueued will be executed in parallel subject to pool_size concurrency constraints .


def _nginx_location_spec(port_spec, bridge_ip):
    """This will output the nginx location config string for specific port spec """
    location_string_spec = "\t \t location / { \n"
    for location_setting in ['proxy_http_version 1.1;',
                             'proxy_set_header Upgrade $http_upgrade;',
                             'proxy_set_header Connection "upgrade";',
                             'proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;',
                             'proxy_set_header Host $http_host;',
                             _nginx_proxy_string(port_spec, bridge_ip)]:
        location_string_spec += "\t \t \t {} \n".format(location_setting)
    location_string_spec += "\t \t } \n"
    return location_string_spec


def _nginx_http_spec(port_spec, bridge_ip):
    """This will output the nginx HTTP config string for specific port spec """
    server_string_spec = "\t server {\n"
    server_string_spec += "\t \t {}\n".format(_nginx_max_file_size_string())
    server_string_spec += "\t \t {}\n".format(_nginx_listen_string(port_spec))
    server_string_spec += "\t \t {}\n".format(_nginx_server_name_string(port_spec))
    server_string_spec += _nginx_location_spec(port_spec, bridge_ip)
    server_string_spec += _custom_502_page()
    server_string_spec += "\t }\n"
    return server_string_spec

def
_nginx_http_spec
(
port_spec
,
bridge_ip
)
:
server_string_spec
=
"\t server {\n"
server_string_spec
+=
"\t \t {}\n"
.
format
(
_nginx_max_file_size_string
(
)
)
server_string_spec
+=
"\t \t {}\n"
.
format
(
_nginx_listen_string
(
port_spec
)
)
server_string_spec
+=
"\t \t {}\n"
.
format
(
_nginx_server_name_string
(
port_spec
)
)
server_string_spec
+=
_nginx_location_spec
(
port_spec
,
bridge_ip
)
server_string_spec
+=
_custom_502_page
(
)
server_string_spec
+=
"\t }\n"
return
server_string_spec


def _nginx_stream_spec(port_spec, bridge_ip):
    """This will output the nginx stream config string for specific port spec """
    server_string_spec = "\t server {\n"
    server_string_spec += "\t \t {}\n".format(_nginx_listen_string(port_spec))
    server_string_spec += "\t \t {}\n".format(_nginx_proxy_string(port_spec, bridge_ip))
    server_string_spec += "\t }\n"
    return server_string_spec

def
_nginx_stream_spec
(
port_spec
,
bridge_ip
)
:
server_string_spec
=
"\t server {\n"
server_string_spec
+=
"\t \t {}\n"
.
format
(
_nginx_listen_string
(
port_spec
)
)
server_string_spec
+=
"\t \t {}\n"
.
format
(
_nginx_proxy_string
(
port_spec
,
bridge_ip
)
)
server_string_spec
+=
"\t }\n"
return
server_string_spec


def get_nginx_configuration_spec(port_spec_dict, docker_bridge_ip):
    """This function will take in a port spec as specified by the port_spec compiler and
    will output an nginx web proxy config string. This string can then be written to a file
    and used running nginx """
    nginx_http_config, nginx_stream_config = "", ""
    for port_spec in port_spec_dict['nginx']:
        if port_spec['type'] == 'http':
            nginx_http_config += _nginx_http_spec(port_spec, docker_bridge_ip)
        elif port_spec['type'] == 'stream':
            nginx_stream_config += _nginx_stream_spec(port_spec, docker_bridge_ip)
    return {'http': nginx_http_config, 'stream': nginx_stream_config}

def get_nginx_configuration_spec ( port_spec_dict , docker_bridge_ip ) : nginx_http_config , nginx_stream_config = "" , "" for port_spec in port_spec_dict [ 'nginx' ] : if port_spec [ 'type' ] == 'http' : nginx_http_config += _nginx_http_spec ( port_spec , docker_bridge_ip ) elif port_spec [ 'type' ] == 'stream' : nginx_stream_config += _nginx_stream_spec ( port_spec , docker_bridge_ip ) return { 'http' : nginx_http_config , 'stream' : nginx_stream_config }

This function will take in a port spec as specified by the port_spec compiler and will output an nginx web proxy config string . This string can then be written to a file and used running nginx


def _load_ssh_auth_post_yosemite(mac_username):
    """Starting with Yosemite, launchd was rearchitected and now only one
    launchd process runs for all users. This allows us to much more easily
    impersonate a user through launchd and extract the environment
    variables from their running processes."""
    user_id = subprocess.check_output(['id', '-u', mac_username])
    ssh_auth_sock = subprocess.check_output(['launchctl', 'asuser', user_id, 'launchctl', 'getenv', 'SSH_AUTH_SOCK']).rstrip()
    _set_ssh_auth_sock(ssh_auth_sock)

def _load_ssh_auth_post_yosemite ( mac_username ) : user_id = subprocess . check_output ( [ 'id' , '-u' , mac_username ] ) ssh_auth_sock = subprocess . check_output ( [ 'launchctl' , 'asuser' , user_id , 'launchctl' , 'getenv' , 'SSH_AUTH_SOCK' ] ) . rstrip ( ) _set_ssh_auth_sock ( ssh_auth_sock )

Starting with Yosemite launchd was rearchitected and now only one launchd process runs for all users . This allows us to much more easily impersonate a user through launchd and extract the environment variables from their running processes .


def _load_ssh_auth_pre_yosemite():
    """For OS X versions before Yosemite, many launchd processes run simultaneously under
    different users and different permission models. The simpler `asuser` trick we use
    in Yosemite doesn't work, since it gets routed to the wrong launchd. We instead need
    to find the running ssh-agent process and use its PID to navigate ourselves
    to the correct launchd."""
    for process in psutil.process_iter():
        if process.name() == 'ssh-agent':
            ssh_auth_sock = subprocess.check_output(['launchctl', 'bsexec', str(process.pid), 'launchctl', 'getenv', 'SSH_AUTH_SOCK']).rstrip()
            if ssh_auth_sock:
                _set_ssh_auth_sock(ssh_auth_sock)
                break
    else:
        daemon_warnings.warn('ssh', 'No running ssh-agent found linked to SSH_AUTH_SOCK')

For OS X versions before Yosemite many launchd processes run simultaneously under different users and different permission models . The simpler asuser trick we use in Yosemite doesn t work since it gets routed to the wrong launchd . We instead need to find the running ssh - agent process and use its PID to navigate ourselves to the correct launchd .


def check_and_load_ssh_auth():
    """
    Will check the mac_username config value; if it is present, will load that user's
    SSH_AUTH_SOCK environment variable to the current environment.  This allows git clones
    to behave the same for the daemon as they do for the user
    """

    mac_username = get_config_value(constants.CONFIG_MAC_USERNAME_KEY)
    if not mac_username:
        logging.info("Can't setup ssh authorization; no mac_username specified")
        return
    if not _running_on_mac(): # give our Linux unit tests a way to not freak out
        logging.info("Skipping SSH load, we are not running on Mac")
        return

    if _mac_version_is_post_yosemite():
        _load_ssh_auth_post_yosemite(mac_username)
    else:
        _load_ssh_auth_pre_yosemite()

Will check the mac_username config value ; if it is present will load that user s SSH_AUTH_SOCK environment variable to the current environment . This allows git clones to behave the same for the daemon as they do for the user


def _cleanup_path(path):
    """Recursively delete a path upon exiting this context
    manager. Supports targets that are files or directories."""
    try:
        yield
    finally:
        if os.path.exists(path):
            if os.path.isdir(path):
                shutil.rmtree(path)
            else:
                os.remove(path)

def _cleanup_path ( path ) : try : yield finally : if os . path . exists ( path ) : if os . path . isdir ( path ) : shutil . rmtree ( path ) else : os . remove ( path )

Recursively delete a path upon exiting this context manager . Supports targets that are files or directories .


def copy_between_containers(source_name, source_path, dest_name, dest_path):
    """Copy a file from the source container to an intermediate staging
    area on the local filesystem, then from that staging area to the
    destination container.

    These moves take place without demotion for two reasons:
      1. There should be no permissions vulnerabilities with copying
         between containers because it is assumed the non-privileged
         user has full access to all Dusty containers.
      2. The temp dir created by mkdtemp is owned by the owner of the
         Dusty daemon process, so if we demoted our moves to/from that location
         they would encounter permission errors."""
    if not container_path_exists(source_name, source_path):
        raise RuntimeError('ERROR: Path {} does not exist inside container {}.'.format(source_path, source_name))
    temp_path = os.path.join(tempfile.mkdtemp(), str(uuid.uuid1()))
    with _cleanup_path(temp_path):
        copy_to_local(temp_path, source_name, source_path, demote=False)
        copy_from_local(temp_path, dest_name, dest_path, demote=False)

Copy a file from the source container to an intermediate staging area on the local filesystem then from that staging area to the destination container .


def copy_from_local(local_path, remote_name, remote_path, demote=True):
    """Copy a path from the local filesystem to a path inside a Dusty
    container. The files on the local filesystem must be accessible
    by the user specified in mac_username."""
    if not os.path.exists(local_path):
        raise RuntimeError('ERROR: Path {} does not exist'.format(local_path))
    temp_identifier = str(uuid.uuid1())
    if os.path.isdir(local_path):
        sync_local_path_to_vm(local_path, os.path.join(vm_cp_path(remote_name), temp_identifier), demote=demote)
        move_dir_inside_container(remote_name, os.path.join(constants.CONTAINER_CP_DIR, temp_identifier), remote_path)
    else:
        sync_local_path_to_vm(local_path, os.path.join(vm_cp_path(remote_name), temp_identifier), demote=demote)
        move_file_inside_container(remote_name, os.path.join(constants.CONTAINER_CP_DIR, temp_identifier), remote_path)

def
copy_from_local
(
local_path
,
remote_name
,
remote_path
,
demote
=
True
)
:
if
not
os
.
path
.
exists
(
local_path
)
:
raise
RuntimeError
(
'ERROR: Path {} does not exist'
.
format
(
local_path
)
)
temp_identifier
=
str
(
uuid
.
uuid1
(
)
)
if
os
.
path
.
isdir
(
local_path
)
:
sync_local_path_to_vm
(
local_path
,
os
.
path
.
join
(
vm_cp_path
(
remote_name
)
,
temp_identifier
)
,
demote
=
demote
)
move_dir_inside_container
(
remote_name
,
os
.
path
.
join
(
constants
.
CONTAINER_CP_DIR
,
temp_identifier
)
,
remote_path
)
else
:
sync_local_path_to_vm
(
local_path
,
os
.
path
.
join
(
vm_cp_path
(
remote_name
)
,
temp_identifier
)
,
demote
=
demote
)
move_file_inside_container
(
remote_name
,
os
.
path
.
join
(
constants
.
CONTAINER_CP_DIR
,
temp_identifier
)
,
remote_path
)

Copy a path from the local filesystem to a path inside a Dusty container . The files on the local filesystem must be accessible by the user specified in mac_username .


def copy_to_local(local_path, remote_name, remote_path, demote=True):
    """Copy a path from inside a Dusty container to a path on the
    local filesystem. The path on the local filesystem must be
    wrist-accessible by the user specified in mac_username."""
    if not container_path_exists(remote_name, remote_path):
        raise RuntimeError('ERROR: Path {} does not exist inside container {}.'.format(remote_path, remote_name))
    temp_identifier = str(uuid.uuid1())
    copy_path_inside_container(remote_name, remote_path, os.path.join(constants.CONTAINER_CP_DIR, temp_identifier))
    vm_path = os.path.join(vm_cp_path(remote_name), temp_identifier)
    is_dir = vm_path_is_directory(vm_path)
    sync_local_path_from_vm(local_path, vm_path, demote=demote, is_dir=is_dir)

Copy a path from inside a Dusty container to a path on the local filesystem . The path on the local filesystem must be wrist - accessible by the user specified in mac_username .


def _mount_repo(repo, wait_for_server=False):
    """
    This function will create the VM directory where a repo will be mounted, if it
    doesn't exist.  If wait_for_server is set, it will wait up to 10 seconds for
    the nfs server to start, by retrying mounts that fail with 'Connection Refused'.

    If wait_for_server is not set, it will attempt to run the mount command once
    """
    check_call_on_vm('sudo mkdir -p {}'.format(repo.vm_path))
    if wait_for_server:
        for i in range(0,10):
            try:
                _run_mount_command(repo)
                return
            except CalledProcessError as e:
                if 'Connection refused' in e.output:
                    logging.info('Failed to mount repo; waiting for nfsd to restart')
                    time.sleep(1)
                else:
                    logging.info(e.output)
                    raise e
        log_to_client('Failed to mount repo {}'.format(repo.short_name))
        raise RuntimeError('Unable to mount repo with NFS')
    else:
        _run_mount_command(repo)

This function will create the VM directory where a repo will be mounted if it doesn t exist . If wait_for_server is set it will wait up to 10 seconds for the nfs server to start by retrying mounts that fail with Connection Refused .


def get_port_spec_document(expanded_active_specs, docker_vm_ip):
    """ Given a dictionary containing the expanded dusty DAG specs this function will
    return a dictionary containing the port mappings needed by downstream methods.  Currently
    this includes docker_compose, virtualbox, nginx and hosts_file."""
    forwarding_port = 65000
    port_spec = {'docker_compose':{}, 'nginx':[], 'hosts_file':[]}
    host_full_addresses, host_names, stream_host_ports = set(), set(), set()
    # No matter the order of apps in expanded_active_specs, we want to produce a consistent
    # port_spec with respect to the apps and the ports they are outputted on
    for app_name in sorted(expanded_active_specs['apps'].keys()):
        app_spec = expanded_active_specs['apps'][app_name]
        if 'host_forwarding' not in app_spec:
            continue
        port_spec['docker_compose'][app_name] = []
        for host_forwarding_spec in app_spec['host_forwarding']:
            # These functions are just used for validating the set of specs all works together
            _add_full_addresses(host_forwarding_spec, host_full_addresses)
            if host_forwarding_spec['type'] == 'stream':
                _add_stream_host_port(host_forwarding_spec, stream_host_ports)

            port_spec['docker_compose'][app_name].append(_docker_compose_port_spec(host_forwarding_spec, forwarding_port))
            port_spec['nginx'].append(_nginx_port_spec(host_forwarding_spec, forwarding_port, docker_vm_ip))

            _add_host_names(host_forwarding_spec, docker_vm_ip, port_spec, host_names)
            forwarding_port += 1
    return port_spec

Given a dictionary containing the expanded dusty DAG specs this function will return a dictionary containing the port mappings needed by downstream methods . Currently this includes docker_compose virtualbox nginx and hosts_file .


def init_yaml_constructor():
    """
    This dark magic is used to make yaml.safe_load encode all strings as utf-8,
    where otherwise python unicode strings would be returned for non-ascii chars
    """
    def utf_encoding_string_constructor(loader, node):
        return loader.construct_scalar(node).encode('utf-8')
    yaml.SafeLoader.add_constructor(u'tag:yaml.org,2002:str', utf_encoding_string_constructor)

def init_yaml_constructor ( ) : def utf_encoding_string_constructor ( loader , node ) : return loader . construct_scalar ( node ) . encode ( 'utf-8' ) yaml . SafeLoader . add_constructor ( u'tag:yaml.org,2002:str' , utf_encoding_string_constructor )

This dark magic is used to make yaml . safe_load encode all strings as utf - 8 where otherwise python unicode strings would be returned for non - ascii chars


def registry_from_image(image_name):
    """Returns the Docker registry host associated with
    a given image name."""
    if '/' not in image_name: # official image
        return constants.PUBLIC_DOCKER_REGISTRY
    prefix = image_name.split('/')[0]
    if '.' not in prefix: # user image on official repository, e.g. thieman/clojure
        return constants.PUBLIC_DOCKER_REGISTRY
    return prefix

Returns the Docker registry host associated with a given image name .


def get_authed_registries():
    """Reads the local Docker client config for the current user
    and returns all registries to which the user may be logged in.
    This is intended to be run client-side, not by the daemon."""
    result = set()
    if not os.path.exists(constants.DOCKER_CONFIG_PATH):
        return result
    config = json.load(open(constants.DOCKER_CONFIG_PATH, 'r'))
    for registry in config.get('auths', {}).iterkeys():
        try:
            parsed = urlparse(registry)
        except Exception:
            log_to_client('Error parsing registry {} from Docker config, will skip this registry').format(registry)
        # This logic assumes the auth is either of the form
        # gamechanger.io (no scheme, no path after host) or
        # of the form https://index.docker.io/v1/ (scheme,
        # netloc parses correctly, additional path does not matter).
        # These are the formats I saw in my personal config file,
        # not sure what other formats it might accept.
        result.add(parsed.netloc) if parsed.netloc else result.add(parsed.path)
    return result

Reads the local Docker client config for the current user and returns all registries to which the user may be logged in . This is intended to be run client - side not by the daemon .


def streaming_to_client():
    """Puts the client logger into streaming mode, which sends
    unbuffered input through to the socket one character at a time.
    We also disable propagation so the root logger does not
    receive many one-byte emissions. This context handler
    was originally created for streaming Compose up's
    terminal output through to the client and should only be
    used for similarly complex circumstances."""
    for handler in client_logger.handlers:
        if hasattr(handler, 'append_newlines'):
            break
    else:
        handler = None
    old_propagate = client_logger.propagate
    client_logger.propagate = False
    if handler is not None:
        old_append = handler.append_newlines
        handler.append_newlines = False
    yield
    client_logger.propagate = old_propagate
    if handler is not None:
        handler.append_newlines = old_append

def streaming_to_client ( ) : for handler in client_logger . handlers : if hasattr ( handler , 'append_newlines' ) : break else : handler = None old_propagate = client_logger . propagate client_logger . propagate = False if handler is not None : old_append = handler . append_newlines handler . append_newlines = False yield client_logger . propagate = old_propagate if handler is not None : handler . append_newlines = old_append

Puts the client logger into streaming mode which sends unbuffered input through to the socket one character at a time . We also disable propagation so the root logger does not receive many one - byte emissions . This context handler was originally created for streaming Compose up s terminal output through to the client and should only be used for similarly complex circumstances .


def pty_fork(*args):
    """Runs a subprocess with a PTY attached via fork and exec.
    The output from the PTY is streamed through log_to_client.
    This should not be necessary for most subprocesses, we
    built this to handle Compose up which only streams pull
    progress if it is attached to a TTY."""

    updated_env = copy(os.environ)
    updated_env.update(get_docker_env())
    args += (updated_env,)
    executable = args[0]
    demote_fn = demote_to_user(get_config_value(constants.CONFIG_MAC_USERNAME_KEY))

    child_pid, pty_fd = pty.fork()
    if child_pid == 0:
        demote_fn()
        os.execle(_executable_path(executable), *args)
    else:
        child_process = psutil.Process(child_pid)
        terminal = os.fdopen(pty_fd, 'r', 0)
        with streaming_to_client():
            while child_process.status() == 'running':
                output = terminal.read(1)
                log_to_client(output)
        _, exit_code = os.waitpid(child_pid, 0)
        if exit_code != 0:
            raise subprocess.CalledProcessError(exit_code, ' '.join(args[:-1]))

def
pty_fork
(
*
args
)
:
updated_env
=
copy
(
os
.
environ
)
updated_env
.
update
(
get_docker_env
(
)
)
args
+=
(
updated_env
,
)
executable
=
args
[
0
]
demote_fn
=
demote_to_user
(
get_config_value
(
constants
.
CONFIG_MAC_USERNAME_KEY
)
)
child_pid
,
pty_fd
=
pty
.
fork
(
)
if
child_pid
==
0
:
demote_fn
(
)
os
.
execle
(
_executable_path
(
executable
)
,
*
args
)
else
:
child_process
=
psutil
.
Process
(
child_pid
)
terminal
=
os
.
fdopen
(
pty_fd
,
'r'
,
0
)
with
streaming_to_client
(
)
:
while
child_process
.
status
(
)
==
'running'
:
output
=
terminal
.
read
(
1
)
log_to_client
(
output
)
_
,
exit_code
=
os
.
waitpid
(
child_pid
,
0
)
if
exit_code
!=
0
:
raise
subprocess
.
CalledProcessError
(
exit_code
,
' '
.
join
(
args
[
:
1
]
)
)

Runs a subprocess with a PTY attached via fork and exec . The output from the PTY is streamed through log_to_client . This should not be necessary for most subprocesses we built this to handle Compose up which only streams pull progress if it is attached to a TTY .


def _compile_docker_commands(app_name, assembled_specs, port_spec):
    """ This is used to compile the command that will be run when the docker container starts
    up. This command has to install any libs that the app uses, run the `always` command, and
    run the `once` command if the container is being launched for the first time """
    app_spec = assembled_specs['apps'][app_name]
    commands = ['set -e']
    commands += _lib_install_commands_for_app(app_name, assembled_specs)
    if app_spec['mount']:
        commands.append("cd {}".format(container_code_path(app_spec)))
        commands.append("export PATH=$PATH:{}".format(container_code_path(app_spec)))
    commands += _copy_assets_commands_for_app(app_spec, assembled_specs)
    commands += _get_once_commands(app_spec, port_spec)
    commands += _get_always_commands(app_spec)
    return commands

def
_compile_docker_commands
(
app_name
,
assembled_specs
,
port_spec
)
:
app_spec
=
assembled_specs
[
'apps'
]
[
app_name
]
commands
=
[
'set -e'
]
commands
+=
_lib_install_commands_for_app
(
app_name
,
assembled_specs
)
if
app_spec
[
'mount'
]
:
commands
.
append
(
"cd {}"
.
format
(
container_code_path
(
app_spec
)
)
)
commands
.
append
(
"export PATH=$PATH:{}"
.
format
(
container_code_path
(
app_spec
)
)
)
commands
+=
_copy_assets_commands_for_app
(
app_spec
,
assembled_specs
)
commands
+=
_get_once_commands
(
app_spec
,
port_spec
)
commands
+=
_get_always_commands
(
app_spec
)
return
commands

This is used to compile the command that will be run when the docker container starts up . This command has to install any libs that the app uses run the always command and run the once command if the container is being launched for the first time


def _increase_file_handle_limit():
    """Raise the open file handles permitted by the Dusty daemon process
    and its child processes. The number we choose here needs to be within
    the OS X default kernel hard limit, which is 10240."""
    logging.info('Increasing file handle limit to {}'.format(constants.FILE_HANDLE_LIMIT))
    resource.setrlimit(resource.RLIMIT_NOFILE,
                       (constants.FILE_HANDLE_LIMIT, resource.RLIM_INFINITY))

def
_increase_file_handle_limit
(
)
:
logging
.
info
(
'Increasing file handle limit to {}'
.
format
(
constants
.
FILE_HANDLE_LIMIT
)
)
resource
.
setrlimit
(
resource
.
RLIMIT_NOFILE
,
(
constants
.
FILE_HANDLE_LIMIT
,
resource
.
RLIM_INFINITY
)
)

Raise the open file handles permitted by the Dusty daemon process and its child processes . The number we choose here needs to be within the OS X default kernel hard limit which is 10240 .


def _start_http_server():
    """Start the daemon's HTTP server on a separate thread.
    This server is only used for servicing container status
    requests from Dusty's custom 502 page."""
    logging.info('Starting HTTP server at {}:{}'.format(constants.DAEMON_HTTP_BIND_IP,
                                                        constants.DAEMON_HTTP_BIND_PORT))
    thread = threading.Thread(target=http_server.app.run, args=(constants.DAEMON_HTTP_BIND_IP,
                                                                constants.DAEMON_HTTP_BIND_PORT))
    thread.daemon = True
    thread.start()

def
_start_http_server
(
)
:
logging
.
info
(
'Starting HTTP server at {}:{}'
.
format
(
constants
.
DAEMON_HTTP_BIND_IP
,
constants
.
DAEMON_HTTP_BIND_PORT
)
)
thread
=
threading
.
Thread
(
target
=
http_server
.
app
.
run
,
args
=
(
constants
.
DAEMON_HTTP_BIND_IP
,
constants
.
DAEMON_HTTP_BIND_PORT
)
)
thread
.
daemon
=
True
thread
.
start
(
)

Start the daemon s HTTP server on a separate thread . This server is only used for servicing container status requests from Dusty s custom 502 page .


def get_docker_client():
    """Ripped off and slightly modified based on docker-py's
    kwargs_from_env utility function."""
    env = get_docker_env()
    host, cert_path, tls_verify = env['DOCKER_HOST'], env['DOCKER_CERT_PATH'], env['DOCKER_TLS_VERIFY']

    params = {'base_url': host.replace('tcp://', 'https://'),
              'timeout': None,
              'version': 'auto'}
    if tls_verify and cert_path:
        params['tls'] = docker.tls.TLSConfig(
            client_cert=(os.path.join(cert_path, 'cert.pem'),
                         os.path.join(cert_path, 'key.pem')),
            ca_cert=os.path.join(cert_path, 'ca.pem'),
            verify=True,
            ssl_version=None,
            assert_hostname=False)
    return docker.Client(**params)

def get_docker_client ( ) : env = get_docker_env ( ) host , cert_path , tls_verify = env [ 'DOCKER_HOST' ] , env [ 'DOCKER_CERT_PATH' ] , env [ 'DOCKER_TLS_VERIFY' ] params = { 'base_url' : host . replace ( 'tcp://' , 'https://' ) , 'timeout' : None , 'version' : 'auto' } if tls_verify and cert_path : params [ 'tls' ] = docker . tls . TLSConfig ( client_cert = ( os . path . join ( cert_path , 'cert.pem' ) , os . path . join ( cert_path , 'key.pem' ) ) , ca_cert = os . path . join ( cert_path , 'ca.pem' ) , verify = True , ssl_version = None , assert_hostname = False ) return docker . Client ( * * params )

Ripped off and slightly modified based on docker - py s kwargs_from_env utility function .


def get_dusty_containers(services, include_exited=False):
    """Get a list of containers associated with the list
    of services. If no services are provided, attempts to
    return all containers associated with Dusty."""
    client = get_docker_client()
    if services:
        containers = [get_container_for_app_or_service(service, include_exited=include_exited) for service in services]
        return [container for container in containers if container]
    else:
        return [container
                for container in client.containers(all=include_exited)
                if any(name.startswith('/dusty') for name in container.get('Names', []))]

def get_dusty_containers ( services , include_exited = False ) : client = get_docker_client ( ) if services : containers = [ get_container_for_app_or_service ( service , include_exited = include_exited ) for service in services ] return [ container for container in containers if container ] else : return [ container for container in client . containers ( all = include_exited ) if any ( name . startswith ( '/dusty' ) for name in container . get ( 'Names' , [ ] ) ) ]

Get a list of containers associated with the list of services . If no services are provided attempts to return all containers associated with Dusty .


def configure_nfs_server():
    """
    This function is used with `dusty up`.  It will check all active repos to see if
    they are exported.  If any are missing, it will replace current dusty exports with
    exports that are needed for currently active repos, and restart
    the nfs server
    """
    repos_for_export = get_all_repos(active_only=True, include_specs_repo=False)

    current_exports = _get_current_exports()
    needed_exports = _get_exports_for_repos(repos_for_export)

    _ensure_managed_repos_dir_exists()

    if not needed_exports.difference(current_exports):
        if not _server_is_running():
            _restart_server()
        return

    _write_exports_config(needed_exports)
    _restart_server()

def configure_nfs_server ( ) : repos_for_export = get_all_repos ( active_only = True , include_specs_repo = False ) current_exports = _get_current_exports ( ) needed_exports = _get_exports_for_repos ( repos_for_export ) _ensure_managed_repos_dir_exists ( ) if not needed_exports . difference ( current_exports ) : if not _server_is_running ( ) : _restart_server ( ) return _write_exports_config ( needed_exports ) _restart_server ( )

This function is used with dusty up . It will check all active repos to see if they are exported . If any are missing it will replace current dusty exports with exports that are needed for currently active repos and restart the nfs server


def _ensure_managed_repos_dir_exists():
    """
    Our exports file will be invalid if this folder doesn't exist, and the NFS server
    will not run correctly.
    """
    if not os.path.exists(constants.REPOS_DIR):
        os.makedirs(constants.REPOS_DIR)

def _ensure_managed_repos_dir_exists ( ) : if not os . path . exists ( constants . REPOS_DIR ) : os . makedirs ( constants . REPOS_DIR )

Our exports file will be invalid if this folder doesn t exist and the NFS server will not run correctly .


def register_consumer():
    """Given a hostname and port attempting to be accessed,
    return a unique consumer ID for accessing logs from
    the referenced container."""
    global _consumers
    hostname, port = request.form['hostname'], request.form['port']

    app_name = _app_name_from_forwarding_info(hostname, port)
    containers = get_dusty_containers([app_name], include_exited=True)
    if not containers:
        raise ValueError('No container exists for app {}'.format(app_name))
    container = containers[0]

    new_id = uuid1()
    new_consumer = Consumer(container['Id'], datetime.utcnow())
    _consumers[str(new_id)] = new_consumer

    response = jsonify({'app_name': app_name, 'consumer_id': new_id})
    response.headers['Access-Control-Allow-Origin'] = '*'
    response.headers['Access-Control-Allow-Methods'] = 'GET, POST'
    return response

def
register_consumer
(
)
:
global
_consumers
hostname
,
port
=
request
.
form
[
'hostname'
]
,
request
.
form
[
'port'
]
app_name
=
_app_name_from_forwarding_info
(
hostname
,
port
)
containers
=
get_dusty_containers
(
[
app_name
]
,
include_exited
=
True
)
if
not
containers
:
raise
ValueError
(
'No container exists for app {}'
.
format
(
app_name
)
)
container
=
containers
[
0
]
new_id
=
uuid1
(
)
new_consumer
=
Consumer
(
container
[
'Id'
]
,
datetime
.
utcnow
(
)
)
_consumers
[
str
(
new_id
)
]
=
new_consumer
response
=
jsonify
(
{
'app_name'
:
app_name
,
'consumer_id'
:
new_id
}
)
response
.
headers
[
'Access-Control-Allow-Origin'
]
=
'*'
response
.
headers
[
'Access-Control-Allow-Methods'
]
=
'GET, POST'
return
response

Given a hostname and port attempting to be accessed return a unique consumer ID for accessing logs from the referenced container .


def consume(consumer_id):
    """Given an existing consumer ID, return any new lines from the
    log since the last time the consumer was consumed."""
    global _consumers
    consumer = _consumers[consumer_id]

    client = get_docker_client()
    try:
        status = client.inspect_container(consumer.container_id)['State']['Status']
    except Exception as e:
        status = 'unknown'
    new_logs = client.logs(consumer.container_id,
                           stdout=True,
                           stderr=True,
                           stream=False,
                           timestamps=False,
                           since=calendar.timegm(consumer.offset.timetuple()))

    updated_consumer = Consumer(consumer.container_id, datetime.utcnow())
    _consumers[str(consumer_id)] = updated_consumer

    response = jsonify({'logs': new_logs, 'status': status})
    response.headers['Access-Control-Allow-Origin'] = '*'
    response.headers['Access-Control-Allow-Methods'] = 'GET, POST'
    return response

def
consume
(
consumer_id
)
:
global
_consumers
consumer
=
_consumers
[
consumer_id
]
client
=
get_docker_client
(
)
try
:
status
=
client
.
inspect_container
(
consumer
.
container_id
)
[
'State'
]
[
'Status'
]
except
Exception
as
e
:
status
=
'unknown'
new_logs
=
client
.
logs
(
consumer
.
container_id
,
stdout
=
True
,
stderr
=
True
,
stream
=
False
,
timestamps
=
False
,
since
=
calendar
.
timegm
(
consumer
.
offset
.
timetuple
(
)
)
)
updated_consumer
=
Consumer
(
consumer
.
container_id
,
datetime
.
utcnow
(
)
)
_consumers
[
str
(
consumer_id
)
]
=
updated_consumer
response
=
jsonify
(
{
'logs'
:
new_logs
,
'status'
:
status
}
)
response
.
headers
[
'Access-Control-Allow-Origin'
]
=
'*'
response
.
headers
[
'Access-Control-Allow-Methods'
]
=
'GET, POST'
return
response

Given an existing consumer ID return any new lines from the log since the last time the consumer was consumed .


def get_app_volume_mounts(app_name, assembled_specs, test=False):
    """ This returns a list of formatted volume specs for an app. These mounts declared in the apps' spec
    and mounts declared in all lib specs the app depends on"""
    app_spec = assembled_specs['apps'][app_name]
    volumes = [get_command_files_volume_mount(app_name, test=test)]
    volumes.append(get_asset_volume_mount(app_name))
    repo_mount = _get_app_repo_volume_mount(app_spec)
    if repo_mount:
        volumes.append(repo_mount)
    volumes += _get_app_libs_volume_mounts(app_name, assembled_specs)
    return volumes

def get_app_volume_mounts ( app_name , assembled_specs , test = False ) : app_spec = assembled_specs [ 'apps' ] [ app_name ] volumes = [ get_command_files_volume_mount ( app_name , test = test ) ] volumes . append ( get_asset_volume_mount ( app_name ) ) repo_mount = _get_app_repo_volume_mount ( app_spec ) if repo_mount : volumes . append ( repo_mount ) volumes += _get_app_libs_volume_mounts ( app_name , assembled_specs ) return volumes

This returns a list of formatted volume specs for an app . These mounts declared in the apps spec and mounts declared in all lib specs the app depends on


def get_lib_volume_mounts(base_lib_name, assembled_specs):
    """ Returns a list of the formatted volume specs for a lib"""
    volumes = [_get_lib_repo_volume_mount(assembled_specs['libs'][base_lib_name])]
    volumes.append(get_command_files_volume_mount(base_lib_name, test=True))
    for lib_name in assembled_specs['libs'][base_lib_name]['depends']['libs']:
        lib_spec = assembled_specs['libs'][lib_name]
        volumes.append(_get_lib_repo_volume_mount(lib_spec))
    return volumes

def get_lib_volume_mounts ( base_lib_name , assembled_specs ) : volumes = [ _get_lib_repo_volume_mount ( assembled_specs [ 'libs' ] [ base_lib_name ] ) ] volumes . append ( get_command_files_volume_mount ( base_lib_name , test = True ) ) for lib_name in assembled_specs [ 'libs' ] [ base_lib_name ] [ 'depends' ] [ 'libs' ] : lib_spec = assembled_specs [ 'libs' ] [ lib_name ] volumes . append ( _get_lib_repo_volume_mount ( lib_spec ) ) return volumes


def _get_app_libs_volume_mounts(app_name, assembled_specs):
    """ Returns a list of the formatted volume mounts for all libs that an app uses """
    volumes = []
    for lib_name in assembled_specs['apps'][app_name]['depends']['libs']:
        lib_spec = assembled_specs['libs'][lib_name]
        volumes.append("{}:{}".format(Repo(lib_spec['repo']).vm_path, container_code_path(lib_spec)))
    return volumes

def _get_app_libs_volume_mounts ( app_name , assembled_specs ) : volumes = [ ] for lib_name in assembled_specs [ 'apps' ] [ app_name ] [ 'depends' ] [ 'libs' ] : lib_spec = assembled_specs [ 'libs' ] [ lib_name ] volumes . append ( "{}:{}" . format ( Repo ( lib_spec [ 'repo' ] ) . vm_path , container_code_path ( lib_spec ) ) ) return volumes


def _dusty_vm_exists():
    """We use VBox directly instead of Docker Machine because it
    shaves about 0.5 seconds off the runtime of this check."""
    existing_vms = check_output_demoted(['VBoxManage', 'list', 'vms'])
    for line in existing_vms.splitlines():
        if '"{}"'.format(constants.VM_MACHINE_NAME) in line:
            return True
    return False

def _dusty_vm_exists ( ) : existing_vms = check_output_demoted ( [ 'VBoxManage' , 'list' , 'vms' ] ) for line in existing_vms . splitlines ( ) : if '"{}"' . format ( constants . VM_MACHINE_NAME ) in line : return True return False

We use VBox directly instead of Docker Machine because it shaves about 0 . 5 seconds off the runtime of this check .


def _init_docker_vm():
    """Initialize the Dusty VM if it does not already exist."""
    if not _dusty_vm_exists():
        log_to_client('Initializing new Dusty VM with Docker Machine')
        machine_options = ['--driver', 'virtualbox',
                           '--virtualbox-cpu-count', '-1',
                           '--virtualbox-boot2docker-url', constants.CONFIG_BOOT2DOCKER_URL,
                           '--virtualbox-memory', str(get_config_value(constants.CONFIG_VM_MEM_SIZE)),
                           '--virtualbox-hostonly-nictype', constants.VM_NIC_TYPE]
        check_call_demoted(['docker-machine', 'create'] + machine_options + [constants.VM_MACHINE_NAME],
                           redirect_stderr=True)

Initialize the Dusty VM if it does not already exist .


def _start_docker_vm():
    """Start the Dusty VM if it is not already running."""
    is_running = docker_vm_is_running()
    if not is_running:
        log_to_client('Starting docker-machine VM {}'.format(constants.VM_MACHINE_NAME))
        _apply_nat_dns_host_resolver()
        _apply_nat_net_less_greedy_subnet()
        check_and_log_output_and_error_demoted(['docker-machine', 'start', constants.VM_MACHINE_NAME], quiet_on_success=True)
    return is_running

def
_start_docker_vm
(
)
:
is_running
=
docker_vm_is_running
(
)
if
not
is_running
:
log_to_client
(
'Starting docker-machine VM {}'
.
format
(
constants
.
VM_MACHINE_NAME
)
)
_apply_nat_dns_host_resolver
(
)
_apply_nat_net_less_greedy_subnet
(
)
check_and_log_output_and_error_demoted
(
[
'docker-machine'
,
'start'
,
constants
.
VM_MACHINE_NAME
]
,
quiet_on_success
=
True
)
return
is_running

Start the Dusty VM if it is not already running .


def docker_vm_is_running():
    """Using VBoxManage is 0.5 seconds or so faster than Machine."""
    running_vms = check_output_demoted(['VBoxManage', 'list', 'runningvms'])
    for line in running_vms.splitlines():
        if '"{}"'.format(constants.VM_MACHINE_NAME) in line:
            return True
    return False

def docker_vm_is_running ( ) : running_vms = check_output_demoted ( [ 'VBoxManage' , 'list' , 'runningvms' ] ) for line in running_vms . splitlines ( ) : if '"{}"' . format ( constants . VM_MACHINE_NAME ) in line : return True return False

Using VBoxManage is 0 . 5 seconds or so faster than Machine .


def _get_localhost_ssh_port():
    """Something in the VM chain, either VirtualBox or Machine, helpfully
    sets up localhost-to-VM forwarding on port 22. We can inspect this
    rule to determine the port on localhost which gets forwarded to
    22 in the VM."""
    for line in _get_vm_config():
        if line.startswith('Forwarding'):
            spec = line.split('=')[1].strip('"')
            name, protocol, host, host_port, target, target_port = spec.split(',')
            if name == 'ssh' and protocol == 'tcp' and target_port == '22':
                return host_port
    raise ValueError('Could not determine localhost port for SSH forwarding')

Something in the VM chain either VirtualBox or Machine helpfully sets up localhost - to - VM forwarding on port 22 . We can inspect this rule to determine the port on localhost which gets forwarded to 22 in the VM .


def _get_host_only_mac_address():
    """Returns the MAC address assigned to the host-only adapter,
    using output from VBoxManage. Returned MAC address has no colons
    and is lower-cased."""
    # Get the number of the host-only adapter
    vm_config = _get_vm_config()
    for line in vm_config:
        if line.startswith('hostonlyadapter'):
            adapter_number = int(line[15:16])
            break
    else:
        raise ValueError('No host-only adapter is defined for the Dusty VM')

    for line in vm_config:
        if line.startswith('macaddress{}'.format(adapter_number)):
            return line.split('=')[1].strip('"').lower()
    raise ValueError('Could not find MAC address for adapter number {}'.format(adapter_number))

Returns the MAC address assigned to the host - only adapter using output from VBoxManage . Returned MAC address has no colons and is lower - cased .


def _ip_for_mac_from_ip_addr_show(ip_addr_show, target_mac):
    """Given the rather-complex output from an 'ip addr show' command
    on the VM, parse the output to determine the IP address
    assigned to the interface with the given MAC."""
    return_next_ip = False
    for line in ip_addr_show.splitlines():
        line = line.strip()
        if line.startswith('link/ether'):
            line_mac = line.split(' ')[1].replace(':', '')
            if line_mac == target_mac:
                return_next_ip = True
        elif return_next_ip and line.startswith('inet') and not line.startswith('inet6'):
            ip = line.split(' ')[1].split('/')[0]
            return ip

def
_ip_for_mac_from_ip_addr_show
(
ip_addr_show
,
target_mac
)
:
return_next_ip
=
False
for
line
in
ip_addr_show
.
splitlines
(
)
:
line
=
line
.
strip
(
)
if
line
.
startswith
(
'link/ether'
)
:
line_mac
=
line
.
split
(
' '
)
[
1
]
.
replace
(
':'
,
''
)
if
line_mac
==
target_mac
:
return_next_ip
=
True
elif
return_next_ip
and
line
.
startswith
(
'inet'
)
and
not
line
.
startswith
(
'inet6'
)
:
ip
=
line
.
split
(
' '
)
[
1
]
.
split
(
'/'
)
[
0
]
return
ip

Given the rather - complex output from an ip addr show command on the VM parse the output to determine the IP address assigned to the interface with the given MAC .


def _get_host_only_ip():
    """Determine the host-only IP of the Dusty VM through Virtualbox and SSH
    directly, bypassing Docker Machine. We do this because Docker Machine is
    much slower, taking about 600ms total. We are basically doing the same
    flow Docker Machine does in its own code."""
    mac = _get_host_only_mac_address()
    ip_addr_show = check_output_demoted(['ssh', '-o', 'StrictHostKeyChecking=no',
                                         '-o', 'UserKnownHostsFile=/dev/null',
                                         '-i', _vm_key_path(), '-p', _get_localhost_ssh_port(),
                                         'docker@127.0.0.1', 'ip addr show'])
    return _ip_for_mac_from_ip_addr_show(ip_addr_show, mac)

def
_get_host_only_ip
(
)
:
mac
=
_get_host_only_mac_address
(
)
ip_addr_show
=
check_output_demoted
(
[
'ssh'
,
'-o'
,
'StrictHostKeyChecking=no'
,
'-o'
,
'UserKnownHostsFile=/dev/null'
,
'-i'
,
_vm_key_path
(
)
,
'-p'
,
_get_localhost_ssh_port
(
)
,
'docker@127.0.0.1'
,
'ip addr show'
]
)
return
_ip_for_mac_from_ip_addr_show
(
ip_addr_show
,
mac
)

Determine the host - only IP of the Dusty VM through Virtualbox and SSH directly bypassing Docker Machine . We do this because Docker Machine is much slower taking about 600ms total . We are basically doing the same flow Docker Machine does in its own code .


def create_local_copy(cookie_file):
    """Make a local copy of the sqlite cookie database and return the new filename.
    This is necessary in case this database is still being written to while the user browses
    to avoid sqlite locking errors.
    """
    # if type of cookie_file is a list, use the first element in the list
    if isinstance(cookie_file, list):
        cookie_file = cookie_file[0]

    # check if cookie file exists
    if os.path.exists(cookie_file):
        # copy to random name in tmp folder
        tmp_cookie_file = tempfile.NamedTemporaryFile(suffix='.sqlite').name
        open(tmp_cookie_file, 'wb').write(open(cookie_file, 'rb').read())
        return tmp_cookie_file
    else:
        raise BrowserCookieError('Can not find cookie file at: ' + cookie_file)

Make a local copy of the sqlite cookie database and return the new filename . This is necessary in case this database is still being written to while the user browses to avoid sqlite locking errors .


def create_cookie(host, path, secure, expires, name, value):
    """Shortcut function to create a cookie
    """
    return http.cookiejar.Cookie(0, name, value, None, False, host, host.startswith('.'), host.startswith('.'), path,
                                 True, secure, expires, False, None, None, {})

def create_cookie ( host , path , secure , expires , name , value ) : return http . cookiejar . Cookie ( 0 , name , value , None , False , host , host . startswith ( '.' ) , host . startswith ( '.' ) , path , True , secure , expires , False , None , None , { } )


def load(domain_name=""):
    """Try to load cookies from all supported browsers and return combined cookiejar
    Optionally pass in a domain name to only load cookies from the specified domain
    """
    cj = http.cookiejar.CookieJar()
    for cookie_fn in [chrome, firefox]:
        try:
            for cookie in cookie_fn(domain_name=domain_name):
                cj.set_cookie(cookie)
        except BrowserCookieError:
            pass
    return cj

def load ( domain_name = "" ) : cj = http . cookiejar . CookieJar ( ) for cookie_fn in [ chrome , firefox ] : try : for cookie in cookie_fn ( domain_name = domain_name ) : cj . set_cookie ( cookie ) except BrowserCookieError : pass return cj


def load(self):
        """Load sqlite cookies into a cookiejar
        """
        con = sqlite3.connect(self.tmp_cookie_file)
        cur = con.cursor()
        try:
            # chrome <=55
            cur.execute('SELECT host_key, path, secure, expires_utc, name, value, encrypted_value '
                        'FROM cookies WHERE host_key like "%{}%";'.format(self.domain_name))
        except sqlite3.OperationalError:
            # chrome >=56
            cur.execute('SELECT host_key, path, is_secure, expires_utc, name, value, encrypted_value '
                        'FROM cookies WHERE host_key like "%{}%";'.format(self.domain_name))

        cj = http.cookiejar.CookieJar()
        for item in cur.fetchall():
            host, path, secure, expires, name = item[:5]
            value = self._decrypt(item[5], item[6])
            c = create_cookie(host, path, secure, expires, name, value)
            cj.set_cookie(c)
        con.close()
        return cj


def _decrypt(self, value, encrypted_value):
        """Decrypt encoded cookies
        """

        if sys.platform == 'win32':
            return self._decrypt_windows_chrome(value, encrypted_value)

        if value or (encrypted_value[:3] != b'v10'):
            return value

        # Encrypted cookies should be prefixed with 'v10' according to the
        # Chromium code. Strip it off.
        encrypted_value = encrypted_value[3:]
        encrypted_value_half_len = int(len(encrypted_value) / 2)

        cipher = pyaes.Decrypter(pyaes.AESModeOfOperationCBC(self.key, self.iv))
        decrypted = cipher.feed(encrypted_value[:encrypted_value_half_len])
        decrypted += cipher.feed(encrypted_value[encrypted_value_half_len:])
        decrypted += cipher.feed()
        return decrypted.decode("utf-8")


def _randone(d, limit=20, grouprefs=None):
    if grouprefs is None:
        grouprefs = {}
    """docstring for _randone"""
    ret = ''
    for i in d:
        if i[0] == sre_parse.IN:
            ret += choice(_in(i[1]))
        elif i[0] == sre_parse.LITERAL:
            ret += unichr(i[1])
        elif i[0] == sre_parse.CATEGORY:
            ret += choice(CATEGORIES.get(i[1], ['']))
        elif i[0] == sre_parse.ANY:
            ret += choice(CATEGORIES['category_any'])
        elif i[0] == sre_parse.MAX_REPEAT or i[0] == sre_parse.MIN_REPEAT:
            if i[1][1] + 1 - i[1][0] >= limit:
                min, max = i[1][0], i[1][0] + limit - 1
            else:
                min, max = i[1][0], i[1][1]
            for _ in range(randint(min, max)):
                ret += _randone(list(i[1][2]), limit, grouprefs)
        elif i[0] == sre_parse.BRANCH:
            ret += _randone(choice(i[1][1]), limit, grouprefs)
        elif i[0] == sre_parse.SUBPATTERN or i[0] == sre_parse.ASSERT:
            subexpr = i[1][1]
            if IS_PY36_OR_GREATER and i[0] == sre_parse.SUBPATTERN:
                subexpr = i[1][3]
            subp = _randone(subexpr, limit, grouprefs)
            if i[1][0]:
                grouprefs[i[1][0]] = subp
            ret += subp
        elif i[0] == sre_parse.AT:
            continue
        elif i[0] == sre_parse.NOT_LITERAL:
            c = list(CATEGORIES['category_any'])
            if unichr(i[1]) in c:
                c.remove(unichr(i[1]))
            ret += choice(c)
        elif i[0] == sre_parse.GROUPREF:
            ret += grouprefs[i[1]]
        elif i[0] == sre_parse.ASSERT_NOT:
            pass
        else:
            print('[!] cannot handle expression "%s"' % str(i))

    return ret

def
_randone
(
d
,
limit
=
20
,
grouprefs
=
)
:
if
grouprefs
is
:
grouprefs
=
{
}
ret
=
''
for
i
in
d
:
if
i
[
0
]
==
sre_parse
.
IN
:
ret
+=
choice
(
_in
(
i
[
1
]
)
)
elif
i
[
0
]
==
sre_parse
.
LITERAL
:
ret
+=
unichr
(
i
[
1
]
)
elif
i
[
0
]
==
sre_parse
.
CATEGORY
:
ret
+=
choice
(
CATEGORIES
.
get
(
i
[
1
]
,
[
''
]
)
)
elif
i
[
0
]
==
sre_parse
.
ANY
:
ret
+=
choice
(
CATEGORIES
[
'category_any'
]
)
elif
i
[
0
]
==
sre_parse
.
MAX_REPEAT
or
i
[
0
]
==
sre_parse
.
MIN_REPEAT
:
if
i
[
1
]
[
1
]
+
1
i
[
1
]
[
0
]
>=
limit
:
min
,
max
=
i
[
1
]
[
0
]
,
i
[
1
]
[
0
]
+
limit
1
else
:
min
,
max
=
i
[
1
]
[
0
]
,
i
[
1
]
[
1
]
for
_
in
range
(
randint
(
min
,
max
)
)
:
ret
+=
_randone
(
list
(
i
[
1
]
[
2
]
)
,
limit
,
grouprefs
)
elif
i
[
0
]
==
sre_parse
.
BRANCH
:
ret
+=
_randone
(
choice
(
i
[
1
]
[
1
]
)
,
limit
,
grouprefs
)
elif
i
[
0
]
==
sre_parse
.
SUBPATTERN
or
i
[
0
]
==
sre_parse
.
ASSERT
:
subexpr
=
i
[
1
]
[
1
]
if
IS_PY36_OR_GREATER
and
i
[
0
]
==
sre_parse
.
SUBPATTERN
:
subexpr
=
i
[
1
]
[
3
]
subp
=
_randone
(
subexpr
,
limit
,
grouprefs
)
if
i
[
1
]
[
0
]
:
grouprefs
[
i
[
1
]
[
0
]
]
=
subp
ret
+=
subp
elif
i
[
0
]
==
sre_parse
.
AT
:
continue
elif
i
[
0
]
==
sre_parse
.
NOT_LITERAL
:
c
=
list
(
CATEGORIES
[
'category_any'
]
)
if
unichr
(
i
[
1
]
)
in
c
:
c
.
remove
(
unichr
(
i
[
1
]
)
)
ret
+=
choice
(
c
)
elif
i
[
0
]
==
sre_parse
.
GROUPREF
:
ret
+=
grouprefs
[
i
[
1
]
]
elif
i
[
0
]
==
sre_parse
.
ASSERT_NOT
:
pass
else
:
print
(
'[!] cannot handle expression "%s"'
%
str
(
i
)
)
return
ret


def parse(s):
    """Regular expression parser

    :param s: Regular expression
    :type s: str
    :rtype: list
    """
    if IS_PY3:
        r = sre_parse.parse(s, flags=U)
    else:
        r = sre_parse.parse(s.decode('utf-8'), flags=U)
    return list(r)

def parse ( s ) : if IS_PY3 : r = sre_parse . parse ( s , flags = U ) else : r = sre_parse . parse ( s . decode ( 'utf-8' ) , flags = U ) return list ( r )


def ib64_patched(self, attrsD, contentparams):
    """ Patch isBase64 to prevent Base64 encoding of JSON content
    """
    if attrsD.get("mode", "") == "base64":
        return 0
    if self.contentparams["type"].startswith("text/"):
        return 0
    if self.contentparams["type"].endswith("+xml"):
        return 0
    if self.contentparams["type"].endswith("/xml"):
        return 0
    if self.contentparams["type"].endswith("/json"):
        return 0
    return 0

def ib64_patched ( self , attrsD , contentparams ) : if attrsD . get ( "mode" , "" ) == "base64" : return 0 if self . contentparams [ "type" ] . startswith ( "text/" ) : return 0 if self . contentparams [ "type" ] . endswith ( "+xml" ) : return 0 if self . contentparams [ "type" ] . endswith ( "/xml" ) : return 0 if self . contentparams [ "type" ] . endswith ( "/json" ) : return 0 return 0


def cleanwrap(func):
    """ Wrapper for Zotero._cleanup
    """

    def enc(self, *args, **kwargs):
        """ Send each item to _cleanup() """
        return (func(self, item, **kwargs) for item in args)

    return enc

def
cleanwrap
(
func
)
:
def
enc
(
self
,
*
args
,
*
*
kwargs
)
:
""" Send each item to _cleanup() """
return
(
func
(
self
,
item
,
*
*
kwargs
)
for
item
in
args
)
return
enc


def ss_wrap(func):
    """ ensure that a SavedSearch object exists """

    def wrapper(self, *args, **kwargs):
        if not self.savedsearch:
            self.savedsearch = SavedSearch(self)
        return func(self, *args, **kwargs)

    return wrapper

def ss_wrap ( func ) : def wrapper ( self , * args , * * kwargs ) : if not self . savedsearch : self . savedsearch = SavedSearch ( self ) return func ( self , * args , * * kwargs ) return wrapper


def error_handler(req):
    """ Error handler for HTTP requests
    """
    error_codes = {
        400: ze.UnsupportedParams,
        401: ze.UserNotAuthorised,
        403: ze.UserNotAuthorised,
        404: ze.ResourceNotFound,
        409: ze.Conflict,
        412: ze.PreConditionFailed,
        413: ze.RequestEntityTooLarge,
        428: ze.PreConditionRequired,
        429: ze.TooManyRequests,
    }

    def err_msg(req):
        """ Return a nicely-formatted error message
        """
        return "\nCode: %s\nURL: %s\nMethod: %s\nResponse: %s" % (
            req.status_code,
            # error.msg,
            req.url,
            req.request.method,
            req.text,
        )

    if error_codes.get(req.status_code):
        # check to see whether its 429
        if req.status_code == 429:
            # call our back-off function
            delay = backoff.delay
            if delay > 32:
                # we've waited a total of 62 seconds (2 + 4 … + 32), so give up
                backoff.reset()
                raise ze.TooManyRetries(
                    "Continuing to receive HTTP 429 \
responses after 62 seconds. You are being rate-limited, try again later"
                )
            time.sleep(delay)
            sess = requests.Session()
            new_req = sess.send(req.request)
            try:
                new_req.raise_for_status()
            except requests.exceptions.HTTPError:
                error_handler(new_req)
        else:
            raise error_codes.get(req.status_code)(err_msg(req))
    else:
        raise ze.HTTPError(err_msg(req))


def default_headers(self):
        """
        It's always OK to include these headers
        """
        _headers = {
            "User-Agent": "Pyzotero/%s" % __version__,
            "Zotero-API-Version": "%s" % __api_version__,
        }
        if self.api_key:
            _headers["Authorization"] = "Bearer %s" % self.api_key
        return _headers

def
default_headers
(
self
)
:
_headers
=
{
"User-Agent"
:
"Pyzotero/%s"
%
__version__
,
"Zotero-API-Version"
:
"%s"
%
__api_version__
,
}
if
self
.
api_key
:
_headers
[
"Authorization"
]
=
"Bearer %s"
%
self
.
api_key
return
_headers


def _cache(self, response, key):
        """
        Add a retrieved template to the cache for 304 checking
        accepts a dict and key name, adds the retrieval time, and adds both
        to self.templates as a new dict using the specified key
        """
        # cache template and retrieval time for subsequent calls
        thetime = datetime.datetime.utcnow().replace(tzinfo=pytz.timezone("GMT"))
        self.templates[key] = {"tmplt": response.json(), "updated": thetime}
        return copy.deepcopy(response.json())

Add a retrieved template to the cache for 304 checking accepts a dict and key name adds the retrieval time and adds both to self . templates as a new dict using the specified key


def _cleanup(self, to_clean, allow=()):
        """ Remove keys we added for internal use
        """
        # this item's been retrieved from the API, we only need the 'data'
        # entry
        if to_clean.keys() == ["links", "library", "version", "meta", "key", "data"]:
            to_clean = to_clean["data"]
        return dict(
            [
                [k, v]
                for k, v in list(to_clean.items())
                if (k in allow or k not in self.temp_keys)
            ]
        )


def _retrieve_data(self, request=None):
        """
        Retrieve Zotero items via the API
        Combine endpoint and request to access the specific resource
        Returns a JSON document
        """
        full_url = "%s%s" % (self.endpoint, request)
        # The API doesn't return this any more, so we have to cheat
        self.self_link = request
        self.request = requests.get(url=full_url, headers=self.default_headers())
        self.request.encoding = "utf-8"
        try:
            self.request.raise_for_status()
        except requests.exceptions.HTTPError:
            error_handler(self.request)
        return self.request


def _extract_links(self):
        """
        Extract self, first, next, last links from a request response
        """
        extracted = dict()
        try:
            for key, value in self.request.links.items():
                parsed = urlparse(value["url"])
                fragment = "{path}?{query}".format(path=parsed[2], query=parsed[4])
                extracted[key] = fragment
            # add a 'self' link
            parsed = list(urlparse(self.self_link))
            # strip 'format' query parameter
            stripped = "&".join(
                [
                    "%s=%s" % (p[0], p[1])
                    for p in parse_qsl(parsed[4])
                    if p[0] != "format"
                ]
            )
            # rebuild url fragment
            # this is a death march
            extracted["self"] = urlunparse(
                [parsed[0], parsed[1], parsed[2], parsed[3], stripped, parsed[5]]
            )
            return extracted
        except KeyError:
            # No links present, because it's a single item
            return None

Extract self first next last links from a request response


def _build_query(self, query_string, no_params=False):
        """
        Set request parameters. Will always add the user ID if it hasn't
        been specifically set by an API method
        """
        try:
            query = quote(query_string.format(u=self.library_id, t=self.library_type))
        except KeyError as err:
            raise ze.ParamNotPassed("There's a request parameter missing: %s" % err)
        # Add the URL parameters and the user key, if necessary
        if no_params is False:
            if not self.url_params:
                self.add_parameters()
            query = "%s?%s" % (query, self.url_params)
        return query

Set request parameters . Will always add the user ID if it hasn t been specifically set by an API method


def publications(self):
        """ Return the contents of My Publications
        """
        if self.library_type != "users":
            raise ze.CallDoesNotExist(
                "This API call does not exist for group libraries"
            )
        query_string = "/{t}/{u}/publications/items"
        return self._build_query(query_string)


def num_collectionitems(self, collection):
        """ Return the total number of items in the specified collection
        """
        query = "/{t}/{u}/collections/{c}/items".format(
            u=self.library_id, t=self.library_type, c=collection.upper()
        )
        return self._totals(query)

def num_collectionitems ( self , collection ) : query = "/{t}/{u}/collections/{c}/items" . format ( u = self . library_id , t = self . library_type , c = collection . upper ( ) ) return self . _totals ( query )


def num_tagitems(self, tag):
        """ Return the total number of items for the specified tag
        """
        query = "/{t}/{u}/tags/{ta}/items".format(
            u=self.library_id, t=self.library_type, ta=tag
        )
        return self._totals(query)

def num_tagitems ( self , tag ) : query = "/{t}/{u}/tags/{ta}/items" . format ( u = self . library_id , t = self . library_type , ta = tag ) return self . _totals ( query )


def _totals(self, query):
        """ General method for returning total counts
        """
        self.add_parameters(limit=1)
        query = self._build_query(query)
        self._retrieve_data(query)
        self.url_params = None
        # extract the 'total items' figure
        return int(self.request.headers["Total-Results"])

def
_totals
(
self
,
query
)
:
self
.
add_parameters
(
limit
=
1
)
query
=
self
.
_build_query
(
query
)
self
.
_retrieve_data
(
query
)
self
.
url_params
=
# extract the 'total items' figure
return
int
(
self
.
request
.
headers
[
"Total-Results"
]
)


def key_info(self, **kwargs):
        """
        Retrieve info about the permissions associated with the
        key associated to the given Zotero instance
        """
        query_string = "/keys/{k}".format(k=self.api_key)
        return self._build_query(query_string)

def key_info ( self , * * kwargs ) : query_string = "/keys/{k}" . format ( k = self . api_key ) return self . _build_query ( query_string )


def fulltext_item(self, itemkey, **kwargs):
        """ Get full-text content for an item"""
        query_string = "/{t}/{u}/items/{itemkey}/fulltext".format(
            t=self.library_type, u=self.library_id, itemkey=itemkey
        )
        return self._build_query(query_string)

def fulltext_item ( self , itemkey , * * kwargs ) : query_string = "/{t}/{u}/items/{itemkey}/fulltext" . format ( t = self . library_type , u = self . library_id , itemkey = itemkey ) return self . _build_query ( query_string )


def last_modified_version(self, **kwargs):
        """ Get the last modified version
        """
        self.items(**kwargs)
        return int(self.request.headers.get("last-modified-version", 0))

def last_modified_version ( self , * * kwargs ) : self . items ( * * kwargs ) return int ( self . request . headers . get ( "last-modified-version" , 0 ) )


def file(self, item, **kwargs):
        """ Get the file from an specific item
        """
        query_string = "/{t}/{u}/items/{i}/file".format(
            u=self.library_id, t=self.library_type, i=item.upper()
        )
        return self._build_query(query_string, no_params=True)

def file ( self , item , * * kwargs ) : query_string = "/{t}/{u}/items/{i}/file" . format ( u = self . library_id , t = self . library_type , i = item . upper ( ) ) return self . _build_query ( query_string , no_params = True )


def dump(self, itemkey, filename=None, path=None):
        """
        Dump a file attachment to disk, with optional filename and path
        """
        if not filename:
            filename = self.item(itemkey)["data"]["filename"]
        if path:
            pth = os.path.join(path, filename)
        else:
            pth = filename
        file = self.file(itemkey)
        if self.snapshot:
            self.snapshot = False
            pth = pth + ".zip"
        with open(pth, "wb") as f:
            f.write(file)

def dump ( self , itemkey , filename = None , path = None ) : if not filename : filename = self . item ( itemkey ) [ "data" ] [ "filename" ] if path : pth = os . path . join ( path , filename ) else : pth = filename file = self . file ( itemkey ) if self . snapshot : self . snapshot = False pth = pth + ".zip" with open ( pth , "wb" ) as f : f . write ( file )

Dump a file attachment to disk with optional filename and path


def all_collections(self, collid=None):
        """
        Retrieve all collections and subcollections. Works for top-level collections
        or for a specific collection. Works at all collection depths.
        """
        all_collections = []

        def subcoll(clct):
            """ recursively add collections to a flat master list """
            all_collections.append(clct)
            if clct["meta"].get("numCollections", 0) > 0:
                # add collection to master list & recur with all child
                # collections
                [
                    subcoll(c)
                    for c in self.everything(self.collections_sub(clct["data"]["key"]))
                ]

        # select all top-level collections or a specific collection and
        # children
        if collid:
            toplevel = [self.collection(collid)]
        else:
            toplevel = self.everything(self.collections_top())
        [subcoll(collection) for collection in toplevel]
        return all_collections

Retrieve all collections and subcollections . Works for top - level collections or for a specific collection . Works at all collection depths .


def collections_sub(self, collection, **kwargs):
        """ Get subcollections for a specific collection
        """
        query_string = "/{t}/{u}/collections/{c}/collections".format(
            u=self.library_id, t=self.library_type, c=collection.upper()
        )
        return self._build_query(query_string)

def collections_sub ( self , collection , * * kwargs ) : query_string = "/{t}/{u}/collections/{c}/collections" . format ( u = self . library_id , t = self . library_type , c = collection . upper ( ) ) return self . _build_query ( query_string )


def everything(self, query):
        """
        Retrieve all items in the library for a particular query
        This method will override the 'limit' parameter if it's been set
        """
        try:
            items = []
            items.extend(query)
            while self.links.get("next"):
                items.extend(self.follow())
        except TypeError:
            # we have a bibliography object ughh
            items = copy.deepcopy(query)
            while self.links.get("next"):
                items.entries.extend(self.follow().entries)
        return items

def
everything
(
self
,
query
)
:
try
:
items
=
[
]
items
.
extend
(
query
)
while
self
.
links
.
get
(
"next"
)
:
items
.
extend
(
self
.
follow
(
)
)
except
TypeError
:
# we have a bibliography object ughh
items
=
copy
.
deepcopy
(
query
)
while
self
.
links
.
get
(
"next"
)
:
items
.
entries
.
extend
(
self
.
follow
(
)
.
entries
)
return
items

Retrieve all items in the library for a particular query This method will override the limit parameter if it s been set


def _json_processor(self, retrieved):
        """ Format and return data from API calls which return Items
        """
        json_kwargs = {}
        if self.preserve_json_order:
            json_kwargs["object_pairs_hook"] = OrderedDict
        # send entries to _tags_data if there's no JSON
        try:
            items = [
                json.loads(e["content"][0]["value"], **json_kwargs)
                for e in retrieved.entries
            ]
        except KeyError:
            return self._tags_data(retrieved)
        return items


def _csljson_processor(self, retrieved):
        """ Return a list of dicts which are dumped CSL JSON
        """
        items = []
        json_kwargs = {}
        if self.preserve_json_order:
            json_kwargs["object_pairs_hook"] = OrderedDict
        for csl in retrieved.entries:
            items.append(json.loads(csl["content"][0]["value"], **json_kwargs))
        self.url_params = None
        return items

def _csljson_processor ( self , retrieved ) : items = [ ] json_kwargs = { } if self . preserve_json_order : json_kwargs [ "object_pairs_hook" ] = OrderedDict for csl in retrieved . entries : items . append ( json . loads ( csl [ "content" ] [ 0 ] [ "value" ] , * * json_kwargs ) ) self . url_params = None return items


def _bib_processor(self, retrieved):
        """ Return a list of strings formatted as HTML bibliography entries
        """
        items = []
        for bib in retrieved.entries:
            items.append(bib["content"][0]["value"])
        self.url_params = None
        return items

def _bib_processor ( self , retrieved ) : items = [ ] for bib in retrieved . entries : items . append ( bib [ "content" ] [ 0 ] [ "value" ] ) self . url_params = None return items


def _citation_processor(self, retrieved):
        """ Return a list of strings formatted as HTML citation entries
        """
        items = []
        for cit in retrieved.entries:
            items.append(cit["content"][0]["value"])
        self.url_params = None
        return items

def _citation_processor ( self , retrieved ) : items = [ ] for cit in retrieved . entries : items . append ( cit [ "content" ] [ 0 ] [ "value" ] ) self . url_params = None return items


def item_template(self, itemtype):
        """ Get a template for a new item
        """
        # if we have a template and it hasn't been updated since we stored it
        template_name = "item_template_" + itemtype
        query_string = "/items/new?itemType={i}".format(i=itemtype)
        if self.templates.get(template_name) and not self._updated(
            query_string, self.templates[template_name], template_name
        ):
            return copy.deepcopy(self.templates[template_name]["tmplt"])
        # otherwise perform a normal request and cache the response
        retrieved = self._retrieve_data(query_string)
        return self._cache(retrieved, template_name)


def _attachment(self, payload, parentid=None):
        """
        Create attachments
        accepts a list of one or more attachment template dicts
        and an optional parent Item ID. If this is specified,
        attachments are created under this ID
        """
        attachment = Zupload(self, payload, parentid)
        res = attachment.upload()
        return res

def _attachment ( self , payload , parentid = None ) : attachment = Zupload ( self , payload , parentid ) res = attachment . upload ( ) return res

Create attachments accepts a list of one or more attachment template dicts and an optional parent Item ID . If this is specified attachments are created under this ID


def show_condition_operators(self, condition):
        """ Show available operators for a given saved search condition """
        # dict keys of allowed operators for the current condition
        permitted_operators = self.savedsearch.conditions_operators.get(condition)
        # transform these into values
        permitted_operators_list = set(
            [self.savedsearch.operators.get(op) for op in permitted_operators]
        )
        return permitted_operators_list


def delete_saved_search(self, keys):
        """ Delete one or more saved searches by passing a list of one or more
        unique search keys
        """
        headers = {"Zotero-Write-Token": token()}
        headers.update(self.default_headers())
        req = requests.delete(
            url=self.endpoint
            + "/{t}/{u}/searches".format(t=self.library_type, u=self.library_id),
            headers=headers,
            params={"searchKey": ",".join(keys)},
        )
        self.request = req
        try:
            req.raise_for_status()
        except requests.exceptions.HTTPError:
            error_handler(req)
        return req.status_code

def delete_saved_search ( self , keys ) : headers = { "Zotero-Write-Token" : token ( ) } headers . update ( self . default_headers ( ) ) req = requests . delete ( url = self . endpoint + "/{t}/{u}/searches" . format ( t = self . library_type , u = self . library_id ) , headers = headers , params = { "searchKey" : "," . join ( keys ) } , ) self . request = req try : req . raise_for_status ( ) except requests . exceptions . HTTPError : error_handler ( req ) return req . status_code


def add_tags(self, item, *tags):
        """
        Add one or more tags to a retrieved item,
        then update it on the server
        Accepts a dict, and one or more tags to add to it
        Returns the updated item from the server
        """
        # Make sure there's a tags field, or add one
        try:
            assert item["data"]["tags"]
        except AssertionError:
            item["data"]["tags"] = list()
        for tag in tags:
            item["data"]["tags"].append({"tag": "%s" % tag})
        # make sure everything's OK
        assert self.check_items([item])
        return self.update_item(item)

Add one or more tags to a retrieved item then update it on the server Accepts a dict and one or more tags to add to it Returns the updated item from the server


def fields_types(self, tname, qstring, itemtype):
        """ Retrieve item fields or creator types
        """
        # check for a valid cached version
        template_name = tname + itemtype
        query_string = qstring.format(i=itemtype)
        if self.templates.get(template_name) and not self._updated(
            query_string, self.templates[template_name], template_name
        ):
            return self.templates[template_name]["tmplt"]
        # otherwise perform a normal request and cache the response
        retrieved = self._retrieve_data(query_string)
        return self._cache(retrieved, template_name)


def item_fields(self):
        """ Get all available item fields
        """
        # Check for a valid cached version
        if self.templates.get("item_fields") and not self._updated(
            "/itemFields", self.templates["item_fields"], "item_fields"
        ):
            return self.templates["item_fields"]["tmplt"]
        query_string = "/itemFields"
        # otherwise perform a normal request and cache the response
        retrieved = self._retrieve_data(query_string)
        return self._cache(retrieved, "item_fields")


def update_item(self, payload, last_modified=None):
        """
        Update an existing item
        Accepts one argument, a dict containing Item data
        """
        to_send = self.check_items([payload])[0]
        if last_modified is None:
            modified = payload["version"]
        else:
            modified = last_modified
        ident = payload["key"]
        headers = {"If-Unmodified-Since-Version": str(modified)}
        headers.update(self.default_headers())
        req = requests.patch(
            url=self.endpoint
            + "/{t}/{u}/items/{id}".format(
                t=self.library_type, u=self.library_id, id=ident
            ),
            headers=headers,
            data=json.dumps(to_send),
        )
        self.request = req
        try:
            req.raise_for_status()
        except requests.exceptions.HTTPError:
            error_handler(req)
        return True

def update_item ( self , payload , last_modified = None ) : to_send = self . check_items ( [ payload ] ) [ 0 ] if last_modified is None : modified = payload [ "version" ] else : modified = last_modified ident = payload [ "key" ] headers = { "If-Unmodified-Since-Version" : str ( modified ) } headers . update ( self . default_headers ( ) ) req = requests . patch ( url = self . endpoint + "/{t}/{u}/items/{id}" . format ( t = self . library_type , u = self . library_id , id = ident ) , headers = headers , data = json . dumps ( to_send ) , ) self . request = req try : req . raise_for_status ( ) except requests . exceptions . HTTPError : error_handler ( req ) return True

Update an existing item Accepts one argument a dict containing Item data


def update_items(self, payload):
        """
        Update existing items
        Accepts one argument, a list of dicts containing Item data
        """
        to_send = [self.check_items([p])[0] for p in payload]
        headers = {}
        headers.update(self.default_headers())
        # the API only accepts 50 items at a time, so we have to split
        # anything longer
        for chunk in chunks(to_send, 50):
            req = requests.post(
                url=self.endpoint
                + "/{t}/{u}/items/".format(t=self.library_type, u=self.library_id),
                headers=headers,
                data=json.dumps(chunk),
            )
            self.request = req
            try:
                req.raise_for_status()
            except requests.exceptions.HTTPError:
                error_handler(req)
            return True

Update existing items Accepts one argument a list of dicts containing Item data


def _validate(self, conditions):
        """ Validate saved search conditions, raising an error if any contain invalid operators """
        allowed_keys = set(self.searchkeys)
        operators_set = set(self.operators.keys())
        for condition in conditions:
            if set(condition.keys()) != allowed_keys:
                raise ze.ParamNotPassed(
                    "Keys must be all of: %s" % ", ".join(self.searchkeys)
                )
            if condition.get("operator") not in operators_set:
                raise ze.ParamNotPassed(
                    "You have specified an unknown operator: %s"
                    % condition.get("operator")
                )
            # dict keys of allowed operators for the current condition
            permitted_operators = self.conditions_operators.get(
                condition.get("condition")
            )
            # transform these into values
            permitted_operators_list = set(
                [self.operators.get(op) for op in permitted_operators]
            )
            if condition.get("operator") not in permitted_operators_list:
                raise ze.ParamNotPassed(
                    "You may not use the '%s' operator when selecting the '%s' condition. \nAllowed operators: %s"
                    % (
                        condition.get("operator"),
                        condition.get("condition"),
                        ", ".join(list(permitted_operators_list)),
                    )
                )

Validate saved search conditions raising an error if any contain invalid operators


def upload(self):
        """
        File upload functionality

        Goes through upload steps 0 - 3 (private class methods), and returns
        a dict noting success, failure, or unchanged
        (returning the payload entries with that property as a list for each status)
        """
        result = {"success": [], "failure": [], "unchanged": []}
        self._create_prelim()
        for item in self.payload:
            if "key" not in item:
                result["failure"].append(item)
                continue
            attach = str(self.basedir.joinpath(item["filename"]))
            authdata = self._get_auth(attach, item["key"], md5=item.get("md5", None))
            # no need to keep going if the file exists
            if authdata.get("exists"):
                result["unchanged"].append(item)
                continue
            self._upload_file(authdata, attach, item["key"])
            result["success"].append(item)
        return result


def split_multiline(value):
    """Split a multiline string into a list, excluding blank lines."""
    return [element for element in (line.strip() for line in value.split('\n'))
            if element]

def split_multiline ( value ) : return [ element for element in ( line . strip ( ) for line in value . split ( '\n' ) ) if element ]

Split a multiline string into a list excluding blank lines .


def split_elements(value):
    """Split a string with comma or space-separated elements into a list."""
    items = [v.strip() for v in value.split(',')]
    if len(items) == 1:
        items = value.split()
    return items

def split_elements ( value ) : items = [ v . strip ( ) for v in value . split ( ',' ) ] if len ( items ) == 1 : items = value . split ( ) return items

Split a string with comma or space - separated elements into a list .


def eval_environ(value):
    """Evaluate environment markers."""
    def eval_environ_str(value):
        parts = value.split(';')
        if len(parts) < 2:
            return value
        expr = parts[1].lstrip()
        if not re.match("^((\\w+(\\.\\w+)?|'.*?'|\".*?\")\\s+"
                        '(in|==|!=|not in)\\s+'
                        "(\\w+(\\.\\w+)?|'.*?'|\".*?\")"
                        '(\\s+(or|and)\\s+)?)+$', expr):
            raise ValueError('bad environment marker: %r' % expr)
        expr = re.sub(r"(platform\.\w+)", r"\1()", expr)
        return parts[0] if eval(expr) else ''

    if isinstance(value, list):
        new_value = []
        for element in value:
            element = eval_environ_str(element)
            if element:
                new_value.append(element)
    elif isinstance(value, str):
        new_value = eval_environ_str(value)
    else:
        new_value = value

    return new_value

def
eval_environ
(
value
)
:
def
eval_environ_str
(
value
)
:
parts
=
value
.
split
(
';'
)
if
len
(
parts
)
<
2
:
return
value
expr
=
parts
[
1
]
.
lstrip
(
)
if
not
re
.
match
(
"^((\\w+(\\.\\w+)?|'.*?'|\".*?\")\\s+"
'(in|==|!=|not in)\\s+'
"(\\w+(\\.\\w+)?|'.*?'|\".*?\")"
'(\\s+(or|and)\\s+)?)+$'
,
expr
)
:
raise
ValueError
(
'bad environment marker: %r'
%
expr
)
expr
=
re
.
sub
(
r"(platform\.\w+)"
,
r"\1()"
,
expr
)
return
parts
[
0
]
if
eval
(
expr
)
else
''
if
isinstance
(
value
,
list
)
:
new_value
=
[
]
for
element
in
value
:
element
=
eval_environ_str
(
element
)
if
element
:
new_value
.
append
(
element
)
elif
isinstance
(
value
,
str
)
:
new_value
=
eval_environ_str
(
value
)
else
:
new_value
=
value
return
new_value


def get_cfg_value(config, section, option):
    """Get configuration value."""
    try:
        value = config[section][option]
    except KeyError:
        if (section, option) in MULTI_OPTIONS:
            return []
        else:
            return ''
    if (section, option) in MULTI_OPTIONS:
        value = split_multiline(value)
    if (section, option) in ENVIRON_OPTIONS:
        value = eval_environ(value)
    return value

def get_cfg_value ( config , section , option ) : try : value = config [ section ] [ option ] except KeyError : if ( section , option ) in MULTI_OPTIONS : return [ ] else : return '' if ( section , option ) in MULTI_OPTIONS : value = split_multiline ( value ) if ( section , option ) in ENVIRON_OPTIONS : value = eval_environ ( value ) return value


def set_cfg_value(config, section, option, value):
    """Set configuration value."""
    if isinstance(value, list):
        value = '\n'.join(value)
    config[section][option] = value

def set_cfg_value ( config , section , option , value ) : if isinstance ( value , list ) : value = '\n' . join ( value ) config [ section ] [ option ] = value


def cfg_to_args(config):
    """Compatibility helper to use setup.cfg in setup.py."""
    kwargs = {}
    opts_to_args = {
        'metadata': [
            ('name', 'name'),
            ('author', 'author'),
            ('author-email', 'author_email'),
            ('maintainer', 'maintainer'),
            ('maintainer-email', 'maintainer_email'),
            ('home-page', 'url'),
            ('summary', 'description'),
            ('description', 'long_description'),
            ('download-url', 'download_url'),
            ('classifier', 'classifiers'),
            ('platform', 'platforms'),
            ('license', 'license'),
            ('keywords', 'keywords'),
        ],
        'files': [
            ('packages_root', 'package_dir'),
            ('packages', 'packages'),
            ('modules', 'py_modules'),
            ('scripts', 'scripts'),
            ('package_data', 'package_data'),
            ('data_files', 'data_files'),
        ],
    }

    opts_to_args['metadata'].append(('requires-dist', 'install_requires'))
    if IS_PY2K and not which('3to2'):
        kwargs['setup_requires'] = ['3to2']
    kwargs['zip_safe'] = False

    for section in opts_to_args:
        for option, argname in opts_to_args[section]:
            value = get_cfg_value(config, section, option)
            if value:
                kwargs[argname] = value

    if 'long_description' not in kwargs:
        kwargs['long_description'] = read_description_file(config)

    if 'package_dir' in kwargs:
        kwargs['package_dir'] = {'': kwargs['package_dir']}

    if 'keywords' in kwargs:
        kwargs['keywords'] = split_elements(kwargs['keywords'])

    if 'package_data' in kwargs:
        kwargs['package_data'] = get_package_data(kwargs['package_data'])

    if 'data_files' in kwargs:
        kwargs['data_files'] = get_data_files(kwargs['data_files'])

    kwargs['version'] = get_version()

    if not IS_PY2K:
        kwargs['test_suite'] = 'test'

    return kwargs

def cfg_to_args ( config ) : kwargs = { } opts_to_args = { 'metadata' : [ ( 'name' , 'name' ) , ( 'author' , 'author' ) , ( 'author-email' , 'author_email' ) , ( 'maintainer' , 'maintainer' ) , ( 'maintainer-email' , 'maintainer_email' ) , ( 'home-page' , 'url' ) , ( 'summary' , 'description' ) , ( 'description' , 'long_description' ) , ( 'download-url' , 'download_url' ) , ( 'classifier' , 'classifiers' ) , ( 'platform' , 'platforms' ) , ( 'license' , 'license' ) , ( 'keywords' , 'keywords' ) , ] , 'files' : [ ( 'packages_root' , 'package_dir' ) , ( 'packages' , 'packages' ) , ( 'modules' , 'py_modules' ) , ( 'scripts' , 'scripts' ) , ( 'package_data' , 'package_data' ) , ( 'data_files' , 'data_files' ) , ] , } opts_to_args [ 'metadata' ] . append ( ( 'requires-dist' , 'install_requires' ) ) if IS_PY2K and not which ( '3to2' ) : kwargs [ 'setup_requires' ] = [ '3to2' ] kwargs [ 'zip_safe' ] = False for section in opts_to_args : for option , argname in opts_to_args [ section ] : value = get_cfg_value ( config , section , option ) if value : kwargs [ argname ] = value if 'long_description' not in kwargs : kwargs [ 'long_description' ] = read_description_file ( config ) if 'package_dir' in kwargs : kwargs [ 'package_dir' ] = { '' : kwargs [ 'package_dir' ] } if 'keywords' in kwargs : kwargs [ 'keywords' ] = split_elements ( kwargs [ 'keywords' ] ) if 'package_data' in kwargs : kwargs [ 'package_data' ] = get_package_data ( kwargs [ 'package_data' ] ) if 'data_files' in kwargs : kwargs [ 'data_files' ] = get_data_files ( kwargs [ 'data_files' ] ) kwargs [ 'version' ] = get_version ( ) if not IS_PY2K : kwargs [ 'test_suite' ] = 'test' return kwargs

Compatibility helper to use setup . cfg in setup . py .


def run_3to2(args=None):
    """Convert Python files using lib3to2."""
    args = BASE_ARGS_3TO2 if args is None else BASE_ARGS_3TO2 + args
    try:
        proc = subprocess.Popen(['3to2'] + args, stderr=subprocess.PIPE)
    except OSError:
        for path in glob.glob('*.egg'):
            if os.path.isdir(path) and path not in sys.path:
                sys.path.append(path)
        try:
            from lib3to2.main import main as lib3to2_main
        except ImportError:
            raise OSError('3to2 script is unavailable.')
        else:
            if lib3to2_main('lib3to2.fixes', args):
                raise Exception('lib3to2 parsing error')
    else:
        # HACK: workaround for 3to2 never returning non-zero
        # when using the -j option.
        num_errors = 0
        while proc.poll() is None:
            line = proc.stderr.readline()
            sys.stderr.write(line)
            num_errors += line.count(': ParseError: ')
        if proc.returncode or num_errors:
            raise Exception('lib3to2 parsing error')


def write_py2k_header(file_list):
    """Write Python 2 shebang and add encoding cookie if needed."""
    if not isinstance(file_list, list):
        file_list = [file_list]

    python_re = re.compile(br"^(#!.*\bpython)(.*)([\r\n]+)$")
    coding_re = re.compile(br"coding[:=]\s*([-\w.]+)")
    new_line_re = re.compile(br"([\r\n]+)$")
    version_3 = LooseVersion('3')

    for file in file_list:
        if not os.path.getsize(file):
            continue

        rewrite_needed = False
        python_found = False
        coding_found = False
        lines = []

        f = open(file, 'rb')
        try:
            while len(lines) < 2:
                line = f.readline()
                match = python_re.match(line)
                if match:
                    python_found = True
                    version = LooseVersion(match.group(2).decode() or '2')
                    try:
                        version_test = version >= version_3
                    except TypeError:
                        version_test = True
                    if version_test:
                        line = python_re.sub(br"\g<1>2\g<3>", line)
                        rewrite_needed = True
                elif coding_re.search(line):
                    coding_found = True
                lines.append(line)
            if not coding_found:
                match = new_line_re.search(lines[0])
                newline = match.group(1) if match else b"\n"
                line = b"# -*- coding: utf-8 -*-" + newline
                lines.insert(1 if python_found else 0, line)
                rewrite_needed = True
            if rewrite_needed:
                lines += f.readlines()
        finally:
            f.close()

        if rewrite_needed:
            f = open(file, 'wb')
            try:
                f.writelines(lines)
            finally:
                f.close()

def
write_py2k_header
(
file_list
)
:
if
not
isinstance
(
file_list
,
list
)
:
file_list
=
[
file_list
]
python_re
=
re
.
compile
(
br"^(#!.*\bpython)(.*)([\r\n]+)$"
)
coding_re
=
re
.
compile
(
br"coding[:=]\s*([-\w.]+)"
)
new_line_re
=
re
.
compile
(
br"([\r\n]+)$"
)
version_3
=
LooseVersion
(
'3'
)
for
file
in
file_list
:
if
not
os
.
path
.
getsize
(
file
)
:
continue
rewrite_needed
=
False
python_found
=
False
coding_found
=
False
lines
=
[
]
f
=
open
(
file
,
'rb'
)
try
:
while
len
(
lines
)
<
2
:
line
=
f
.
readline
(
)
match
=
python_re
.
match
(
line
)
if
match
:
python_found
=
True
version
=
LooseVersion
(
match
.
group
(
2
)
.
decode
(
)
or
'2'
)
try
:
version_test
=
version
>=
version_3
except
TypeError
:
version_test
=
True
if
version_test
:
line
=
python_re
.
sub
(
br"\g<1>2\g<3>"
,
line
)
rewrite_needed
=
True
elif
coding_re
.
search
(
line
)
:
coding_found
=
True
lines
.
append
(
line
)
if
not
coding_found
:
match
=
new_line_re
.
search
(
lines
[
0
]
)
newline
=
match
.
group
(
1
)
if
match
else
b"\n"
line
=
b"# -*- coding: utf-8 -*-"
+
newline
lines
.
insert
(
1
if
python_found
else
0
,
line
)
rewrite_needed
=
True
if
rewrite_needed
:
lines
+=
f
.
readlines
(
)
finally
:
f
.
close
(
)
if
rewrite_needed
:
f
=
open
(
file
,
'wb'
)
try
:
f
.
writelines
(
lines
)
finally
:
f
.
close
(
)

Write Python 2 shebang and add encoding cookie if needed .


def get_version():
    """Get LanguageTool version."""
    version = _get_attrib().get('version')
    if not version:
        match = re.search(r"LanguageTool-?.*?(\S+)$", get_directory())
        if match:
            version = match.group(1)
    return version

def get_version ( ) : version = _get_attrib ( ) . get ( 'version' ) if not version : match = re . search ( r"LanguageTool-?.*?(\S+)$" , get_directory ( ) ) if match : version = match . group ( 1 ) return version


def get_languages() -> set:
    """Get supported languages."""
    try:
        languages = cache['languages']
    except KeyError:
        languages = LanguageTool._get_languages()
        cache['languages'] = languages
    return languages

def get_languages ( ) -> set : try : languages = cache [ 'languages' ] except KeyError : languages = LanguageTool . _get_languages ( ) cache [ 'languages' ] = languages return languages


def get_directory():
    """Get LanguageTool directory."""
    try:
        language_check_dir = cache['language_check_dir']
    except KeyError:
        def version_key(string):
            return [int(e) if e.isdigit() else e
                    for e in re.split(r"(\d+)", string)]

        def get_lt_dir(base_dir):
            paths = [
                path for path in
                glob.glob(os.path.join(base_dir, 'LanguageTool*'))
                if os.path.isdir(path)
            ]
            return max(paths, key=version_key) if paths else None

        base_dir = os.path.dirname(sys.argv[0])
        language_check_dir = get_lt_dir(base_dir)
        if not language_check_dir:
            try:
                base_dir = os.path.dirname(os.path.abspath(__file__))
            except NameError:
                pass
            else:
                language_check_dir = get_lt_dir(base_dir)
            if not language_check_dir:
                raise PathError("can't find LanguageTool directory in {!r}"
                                .format(base_dir))
        cache['language_check_dir'] = language_check_dir
    return language_check_dir


def set_directory(path=None):
    """Set LanguageTool directory."""
    old_path = get_directory()
    terminate_server()
    cache.clear()
    if path:
        cache['language_check_dir'] = path
        try:
            get_jar_info()
        except Error:
            cache['language_check_dir'] = old_path
            raise

def set_directory ( path = None ) : old_path = get_directory ( ) terminate_server ( ) cache . clear ( ) if path : cache [ 'language_check_dir' ] = path try : get_jar_info ( ) except Error : cache [ 'language_check_dir' ] = old_path raise


def check(self, text: str, srctext=None) -> [Match]:
        """Match text against enabled rules."""
        root = self._get_root(self._url, self._encode(text, srctext))
        return [Match(e.attrib) for e in root if e.tag == 'error']

def check ( self , text : str , srctext = None ) -> [ Match ] : root = self . _get_root ( self . _url , self . _encode ( text , srctext ) ) return [ Match ( e . attrib ) for e in root if e . tag == 'error' ]


def get_newest_possible_languagetool_version():
    """Return newest compatible version.

    >>> version = get_newest_possible_languagetool_version()
    >>> version in [JAVA_6_COMPATIBLE_VERSION,
    ...             JAVA_7_COMPATIBLE_VERSION,
    ...             LATEST_VERSION]
    True

    """
    java_path = find_executable('java')
    if not java_path:
        # Just ignore this and assume an old version of Java. It might not be
        # found because of a PATHEXT-related issue
        # (https://bugs.python.org/issue2200).
        return JAVA_6_COMPATIBLE_VERSION

    output = subprocess.check_output([java_path, '-version'],
                                     stderr=subprocess.STDOUT,
                                     universal_newlines=True)

    java_version = parse_java_version(output)

    if java_version >= (1, 8):
        return LATEST_VERSION
    elif java_version >= (1, 7):
        return JAVA_7_COMPATIBLE_VERSION
    elif java_version >= (1, 6):
        warn('language-check would be able to use a newer version of '
             'LanguageTool if you had Java 7 or newer installed')
        return JAVA_6_COMPATIBLE_VERSION
    else:
        raise SystemExit(
            'You need at least Java 6 to use language-check')


def get_common_prefix(z):
    """Get common directory in a zip file if any."""
    name_list = z.namelist()
    if name_list and all(n.startswith(name_list[0]) for n in name_list[1:]):
        return name_list[0]
    return None

def get_common_prefix ( z ) : name_list = z . namelist ( ) if name_list and all ( n . startswith ( name_list [ 0 ] ) for n in name_list [ 1 : ] ) : return name_list [ 0 ] return None

Get common directory in a zip file if any .


def _process_events(self, events):
        """Process events from proactor."""
        for f, callback, transferred, key, ov in events:
            try:
                self._logger.debug('Invoking event callback {}'.format(callback))
                value = callback(transferred, key, ov)
            except OSError:
                self._logger.warning('Event callback failed', exc_info=sys.exc_info())
            else:
                f.set_result(value)

def
_process_events
(
self
,
events
)
:
for
f
,
callback
,
transferred
,
key
,
ov
in
events
:
try
:
self
.
_logger
.
debug
(
'Invoking event callback {}'
.
format
(
callback
)
)
value
=
callback
(
transferred
,
key
,
ov
)
except
OSError
:
self
.
_logger
.
warning
(
'Event callback failed'
,
exc_info
=
sys
.
exc_info
(
)
)
else
:
f
.
set_result
(
value
)


def asyncClose(fn):
    """Allow to run async code before application is closed."""
    @functools.wraps(fn)
    def wrapper(*args, **kwargs):
        f = asyncio.ensure_future(fn(*args, **kwargs))
        while not f.done():
            QApplication.instance().processEvents()

    return wrapper

def asyncClose ( fn ) : @ functools . wraps ( fn ) def wrapper ( * args , * * kwargs ) : f = asyncio . ensure_future ( fn ( * args , * * kwargs ) ) while not f . done ( ) : QApplication . instance ( ) . processEvents ( ) return wrapper

Allow to run async code before application is closed .