def validate_submit_args_or_fail(job_descriptor, provider_name, input_providers,
                                 output_providers, logging_providers):
  """Validate that arguments passed to submit_job have valid file providers.

  This utility function takes resources and task data args from `submit_job`
  in the base provider. This function will fail with a value error if any of the
  parameters are not valid. See the following example;

  >>> job_resources = type('', (object,),
  ...    {"logging": job_model.LoggingParam('gs://logtemp', job_model.P_GCS)})()
  >>> job_params={'inputs': set(), 'outputs': set(), 'mounts': set()}
  >>> task_descriptors = [
  ...     job_model.TaskDescriptor(None, {
  ...       'inputs': {
  ...           job_model.FileParam('IN', uri='gs://in/*',
  ...                               file_provider=job_model.P_GCS)},
  ...       'outputs': set()}, None),
  ...     job_model.TaskDescriptor(None, {
  ...       'inputs': set(),
  ...       'outputs': {
  ...           job_model.FileParam('OUT', uri='gs://out/*',
  ...                               file_provider=job_model.P_GCS)}}, None)]
  ...
  >>> validate_submit_args_or_fail(job_model.JobDescriptor(None, job_params,
  ...                              job_resources, task_descriptors),
  ...                              provider_name='MYPROVIDER',
  ...                              input_providers=[job_model.P_GCS],
  ...                              output_providers=[job_model.P_GCS],
  ...                              logging_providers=[job_model.P_GCS])
  ...
  >>> validate_submit_args_or_fail(job_model.JobDescriptor(None, job_params,
  ...                              job_resources, task_descriptors),
  ...                              provider_name='MYPROVIDER',
  ...                              input_providers=[job_model.P_GCS],
  ...                              output_providers=[job_model.P_LOCAL],
  ...                              logging_providers=[job_model.P_GCS])
  Traceback (most recent call last):
       ...
  ValueError: Unsupported output path (gs://out/*) for provider 'MYPROVIDER'.

  Args:
    job_descriptor: instance of job_model.JobDescriptor.
    provider_name: (str) the name of the execution provider.
    input_providers: (string collection) whitelist of file providers for input.
    output_providers: (string collection) whitelist of providers for output.
    logging_providers: (string collection) whitelist of providers for logging.

  Raises:
    ValueError: if any file providers do not match the whitelists.
  """
  job_resources = job_descriptor.job_resources
  job_params = job_descriptor.job_params
  task_descriptors = job_descriptor.task_descriptors

  # Validate logging file provider.
  _validate_providers([job_resources.logging], 'logging', logging_providers,
                      provider_name)

  # Validate job input and output file providers
  _validate_providers(job_params['inputs'], 'input', input_providers,
                      provider_name)
  _validate_providers(job_params['outputs'], 'output', output_providers,
                      provider_name)

  # Validate input and output file providers.
  for task_descriptor in task_descriptors:
    _validate_providers(task_descriptor.task_params['inputs'], 'input',
                        input_providers, provider_name)
    _validate_providers(task_descriptor.task_params['outputs'], 'output',
                        output_providers, provider_name)

Validate that arguments passed to submit_job have valid file providers .


def _interval_to_seconds(interval, valid_units='smhdw'):
  """Convert the timeout duration to seconds.

  The value must be of the form "<integer><unit>" where supported
  units are s, m, h, d, w (seconds, minutes, hours, days, weeks).

  Args:
    interval: A "<integer><unit>" string.
    valid_units: A list of supported units.

  Returns:
    A string of the form "<integer>s" or None if timeout is empty.
  """
  if not interval:
    return None

  try:
    last_char = interval[-1]

    if last_char == 's' and 's' in valid_units:
      return str(float(interval[:-1])) + 's'
    elif last_char == 'm' and 'm' in valid_units:
      return str(float(interval[:-1]) * 60) + 's'
    elif last_char == 'h' and 'h' in valid_units:
      return str(float(interval[:-1]) * 60 * 60) + 's'
    elif last_char == 'd' and 'd' in valid_units:
      return str(float(interval[:-1]) * 60 * 60 * 24) + 's'
    elif last_char == 'w' and 'w' in valid_units:
      return str(float(interval[:-1]) * 60 * 60 * 24 * 7) + 's'
    else:
      raise ValueError(
          'Unsupported units in interval string %s: %s' % (interval, last_char))

  except (ValueError, OverflowError) as e:
    raise ValueError('Unable to parse interval string %s: %s' % (interval, e))

Convert the timeout duration to seconds .


def get_variable_name(self, name):
    """Produce a default variable name if none is specified."""
    if not name:
      name = '%s%s' % (self._auto_prefix, self._auto_index)
      self._auto_index += 1
    return name

def get_variable_name ( self , name ) : if not name : name = '%s%s' % ( self . _auto_prefix , self . _auto_index ) self . _auto_index += 1 return name

Produce a default variable name if none is specified .


def rewrite_uris(self, raw_uri, file_provider):
    """Accept a raw uri and return rewritten versions.

    This function returns a normalized URI and a docker path. The normalized
    URI may have minor alterations meant to disambiguate and prepare for use
    by shell utilities that may require a specific format.

    The docker rewriter makes substantial modifications to the raw URI when
    constructing a docker path, but modifications must follow these rules:
      1) System specific characters are not allowed (ex. indirect paths).
      2) The path, if it is a directory, must end in a forward slash.
      3) The path will begin with the value set in self._relative_path.
      4) The path will have an additional prefix (after self._relative_path) set
         by the file provider-specific rewriter.

    Rewrite output for the docker path:
      >>> out_util = FileParamUtil('AUTO_', 'output')
      >>> out_util.rewrite_uris('gs://mybucket/myfile.txt', job_model.P_GCS)[1]
      'output/gs/mybucket/myfile.txt'
      >>> out_util.rewrite_uris('./data/myfolder/', job_model.P_LOCAL)[1]
      'output/file/data/myfolder/'

    When normalizing the URI for cloud buckets, no rewrites are done. For local
    files, the user directory will be expanded and relative paths will be
    converted to absolute:
      >>> in_util = FileParamUtil('AUTO_', 'input')
      >>> in_util.rewrite_uris('gs://mybucket/gcs_dir/', job_model.P_GCS)[0]
      'gs://mybucket/gcs_dir/'
      >>> in_util.rewrite_uris('/data/./dir_a/../myfile.txt',
      ...   job_model.P_LOCAL)[0]
      '/data/myfile.txt'
      >>> in_util.rewrite_uris('file:///tmp/data/*.bam', job_model.P_LOCAL)[0]
      '/tmp/data/*.bam'

    Args:
      raw_uri: (str) the path component of the raw URI.
      file_provider: a valid provider (contained in job_model.FILE_PROVIDERS).

    Returns:
      normalized: a cleaned version of the uri provided by command line.
      docker_path: the uri rewritten in the format required for mounting inside
                   a docker worker.

    Raises:
      ValueError: if file_provider is not valid.
    """
    if file_provider == job_model.P_GCS:
      normalized, docker_path = _gcs_uri_rewriter(raw_uri)
    elif file_provider == job_model.P_LOCAL:
      normalized, docker_path = _local_uri_rewriter(raw_uri)
    else:
      raise ValueError('File provider not supported: %r' % file_provider)
    return normalized, os.path.join(self._relative_path, docker_path)

def
rewrite_uris
(
self
,
raw_uri
,
file_provider
)
:
if
file_provider
==
job_model
.
P_GCS
:
normalized
,
docker_path
=
_gcs_uri_rewriter
(
raw_uri
)
elif
file_provider
==
job_model
.
P_LOCAL
:
normalized
,
docker_path
=
_local_uri_rewriter
(
raw_uri
)
else
:
raise
ValueError
(
'File provider not supported: %r'
%
file_provider
)
return
normalized
,
os
.
path
.
join
(
self
.
_relative_path
,
docker_path
)

Accept a raw uri and return rewritten versions .


def parse_file_provider(uri):
    """Find the file provider for a URI."""
    providers = {'gs': job_model.P_GCS, 'file': job_model.P_LOCAL}
    # URI scheme detector uses a range up to 30 since none of the IANA
    # registered schemes are longer than this.
    provider_found = re.match(r'^([A-Za-z][A-Za-z0-9+.-]{0,29})://', uri)
    if provider_found:
      prefix = provider_found.group(1).lower()
    else:
      # If no provider is specified in the URI, assume that the local
      # filesystem is being used. Availability and validity of the local
      # file/directory will be checked later.
      prefix = 'file'
    if prefix in providers:
      return providers[prefix]
    else:
      raise ValueError('File prefix not supported: %s://' % prefix)


def _validate_paths_or_fail(uri, recursive):
    """Do basic validation of the uri, return the path and filename."""
    path, filename = os.path.split(uri)

    # dsub could support character ranges ([0-9]) with some more work, but for
    # now we assume that basic asterisk wildcards are sufficient. Reject any URI
    # that includes square brackets or question marks, since we know that
    # if they actually worked, it would be accidental.
    if '[' in uri or ']' in uri:
      raise ValueError(
          'Square bracket (character ranges) are not supported: %s' % uri)
    if '?' in uri:
      raise ValueError('Question mark wildcards are not supported: %s' % uri)

    # Only support file URIs and *filename* wildcards
    # Wildcards at the directory level or "**" syntax would require better
    # support from the Pipelines API *or* doing expansion here and
    # (potentially) producing a series of FileParams, instead of one.
    if '*' in path:
      raise ValueError(
          'Path wildcard (*) are only supported for files: %s' % uri)
    if '**' in filename:
      raise ValueError('Recursive wildcards ("**") not supported: %s' % uri)
    if filename in ('..', '.'):
      raise ValueError('Path characters ".." and "." not supported '
                       'for file names: %s' % uri)

    # Do not allow non-recursive IO to reference directories.
    if not recursive and not filename:
      raise ValueError('Input or output values that are not recursive must '
                       'reference a filename or wildcard: %s' % uri)

Do basic validation of the uri return the path and filename .


def parse_uri(self, raw_uri, recursive):
    """Return a valid docker_path, uri, and file provider from a flag value."""
    # Assume recursive URIs are directory paths.
    if recursive:
      raw_uri = directory_fmt(raw_uri)
    # Get the file provider, validate the raw URI, and rewrite the path
    # component of the URI for docker and remote.
    file_provider = self.parse_file_provider(raw_uri)
    self._validate_paths_or_fail(raw_uri, recursive)
    uri, docker_uri = self.rewrite_uris(raw_uri, file_provider)
    uri_parts = job_model.UriParts(
        directory_fmt(os.path.dirname(uri)), os.path.basename(uri))
    return docker_uri, uri_parts, file_provider

Return a valid docker_path uri and file provider from a flag value .


def _parse_image_uri(self, raw_uri):
    """Return a valid docker_path from a Google Persistent Disk url."""
    # The string replace is so we don't have colons and double slashes in the
    # mount path. The idea is the resulting mount path would look like:
    # /mnt/data/mount/http/www.googleapis.com/compute/v1/projects/...
    docker_uri = os.path.join(self._relative_path,
                              raw_uri.replace('https://', 'https/', 1))
    return docker_uri

Return a valid docker_path from a Google Persistent Disk url .


def _parse_local_mount_uri(self, raw_uri):
    """Return a valid docker_path for a local file path."""
    raw_uri = directory_fmt(raw_uri)
    _, docker_path = _local_uri_rewriter(raw_uri)
    local_path = docker_path[len('file'):]
    docker_uri = os.path.join(self._relative_path, docker_path)
    return local_path, docker_uri

def _parse_local_mount_uri ( self , raw_uri ) : raw_uri = directory_fmt ( raw_uri ) _ , docker_path = _local_uri_rewriter ( raw_uri ) local_path = docker_path [ len ( 'file' ) : ] docker_uri = os . path . join ( self . _relative_path , docker_path ) return local_path , docker_uri

Return a valid docker_path for a local file path .


def _parse_gcs_uri(self, raw_uri):
    """Return a valid docker_path for a GCS bucket."""
    # Assume URI is a directory path.
    raw_uri = directory_fmt(raw_uri)
    _, docker_path = _gcs_uri_rewriter(raw_uri)
    docker_uri = os.path.join(self._relative_path, docker_path)
    return docker_uri

def
_parse_gcs_uri
(
self
,
raw_uri
)
:
# Assume URI is a directory path.
raw_uri
=
directory_fmt
(
raw_uri
)
_
,
docker_path
=
_gcs_uri_rewriter
(
raw_uri
)
docker_uri
=
os
.
path
.
join
(
self
.
_relative_path
,
docker_path
)
return
docker_uri

Return a valid docker_path for a GCS bucket .


def make_param(self, name, raw_uri, disk_size):
    """Return a MountParam given a GCS bucket, disk image or local path."""
    if raw_uri.startswith('https://www.googleapis.com/compute'):
      # Full Image URI should look something like:
      # https://www.googleapis.com/compute/v1/projects/<project>/global/images/
      # But don't validate further, should the form of a valid image URI
      # change (v1->v2, for example)
      docker_path = self._parse_image_uri(raw_uri)
      return job_model.PersistentDiskMountParam(
          name, raw_uri, docker_path, disk_size, disk_type=None)
    elif raw_uri.startswith('file://'):
      local_path, docker_path = self._parse_local_mount_uri(raw_uri)
      return job_model.LocalMountParam(name, raw_uri, docker_path, local_path)
    elif raw_uri.startswith('gs://'):
      docker_path = self._parse_gcs_uri(raw_uri)
      return job_model.GCSMountParam(name, raw_uri, docker_path)
    else:
      raise ValueError(
          'Mount parameter {} must begin with valid prefix.'.format(raw_uri))

Return a MountParam given a GCS bucket disk image or local path .


def validate_param_name(name, param_type):
  """Validate that the name follows posix conventions for env variables."""
  # http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap03.html#tag_03_235
  #
  # 3.235 Name
  # In the shell command language, a word consisting solely of underscores,
  # digits, and alphabetics from the portable character set.
  if not re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*$', name):
    raise ValueError('Invalid %s: %s' % (param_type, name))

Validate that the name follows posix conventions for env variables .


def validate_bucket_name(bucket):
  """Validate that the name is a valid GCS bucket."""
  if not bucket.startswith('gs://'):
    raise ValueError(
        'Invalid bucket path "%s". Must start with "gs://".' % bucket)
  bucket_name = bucket[len('gs://'):]
  if not re.search(r'^\w[\w_\.-]{1,61}\w$', bucket_name):
    raise ValueError('Invalid bucket name: %s' % bucket)

Validate that the name is a valid GCS bucket .


def convert_to_label_chars(s):
  """Turn the specified name and value into a valid Google label."""

  # We want the results to be user-friendly, not just functional.
  # So we can't base-64 encode it.
  #   * If upper-case: lower-case it
  #   * If the char is not a standard letter or digit. make it a dash

  # March 2019 note: underscores are now allowed in labels.
  # However, removing the conversion of underscores to dashes here would
  # create inconsistencies between old jobs and new jobs.
  # With existing code, $USER "jane_doe" has a user-id label of "jane-doe".
  # If we remove the conversion, the user-id label for new jobs is "jane_doe".
  # This makes looking up old jobs more complicated.

  accepted_characters = string.ascii_lowercase + string.digits + '-'

  def label_char_transform(char):
    if char in accepted_characters:
      return char
    if char in string.ascii_uppercase:
      return char.lower()
    return '-'

  return ''.join(label_char_transform(c) for c in s)

Turn the specified name and value into a valid Google label .


def ensure_task_params_are_complete(task_descriptors):
  """For each task, ensure that each task param entry is not None."""
  for task_desc in task_descriptors:
    for param in [
        'labels', 'envs', 'inputs', 'outputs', 'input-recursives',
        'output-recursives'
    ]:
      if not task_desc.task_params.get(param):
        task_desc.task_params[param] = set()

def ensure_task_params_are_complete ( task_descriptors ) : for task_desc in task_descriptors : for param in [ 'labels' , 'envs' , 'inputs' , 'outputs' , 'input-recursives' , 'output-recursives' ] : if not task_desc . task_params . get ( param ) : task_desc . task_params [ param ] = set ( )

For each task ensure that each task param entry is not None .


def _remove_empty_items(d, required):
  """Return a new dict with any empty items removed.

  Note that this is not a deep check. If d contains a dictionary which
  itself contains empty items, those are never checked.

  This method exists to make to_serializable() functions cleaner.
  We could revisit this some day, but for now, the serialized objects are
  stripped of empty values to keep the output YAML more compact.

  Args:
    d: a dictionary
    required: list of required keys (for example, TaskDescriptors always emit
      the "task-id", even if None)

  Returns:
    A dictionary with empty items removed.
  """

  new_dict = {}
  for k, v in d.items():
    if k in required:
      new_dict[k] = v
    elif isinstance(v, int) or v:
      # "if v" would suppress emitting int(0)
      new_dict[k] = v

  return new_dict

def
_remove_empty_items
(
d
,
required
)
:
new_dict
=
{
}
for
k
,
v
in
d
.
items
(
)
:
if
k
in
required
:
new_dict
[
k
]
=
v
elif
isinstance
(
v
,
int
)
or
v
:
# "if v" would suppress emitting int(0)
new_dict
[
k
]
=
v
return
new_dict

Return a new dict with any empty items removed .


def task_view_generator(job_descriptor):
  """Generator that yields a task-specific view of the job.

  This generator exists to make it easy for callers to iterate over the tasks
  in a JobDescriptor. Each pass yields a new JobDescriptor with a single task.

  Args:
    job_descriptor: A JobDescriptor with 1 or more tasks.

  Yields:
    A JobDescriptor with a single task.
  """
  for task_descriptor in job_descriptor.task_descriptors:
    jd = JobDescriptor(job_descriptor.job_metadata, job_descriptor.job_params,
                       job_descriptor.job_resources, [task_descriptor])
    yield jd

Generator that yields a task - specific view of the job .


def numeric_task_id(task_id):
  """Converts a task-id to the numeric task-id.

  Args:
    task_id: task-id in either task-n or n format

  Returns:
    n
  """

  # This function exists to support the legacy "task-id" format in the "google"
  # provider. Google labels originally could not be numeric. When the google
  # provider is completely replaced by the google-v2 provider, this function can
  # go away.

  if task_id is not None:
    if task_id.startswith('task-'):
      return int(task_id[len('task-'):])
    else:
      return int(task_id)

Converts a task - id to the numeric task - id .


def _validate_label(cls, name, value):
    """Raise ValueError if the label is invalid."""
    # Rules for labels are described in:
    #  https://cloud.google.com/compute/docs/labeling-resources#restrictions

    # * Keys and values cannot be longer than 63 characters each.
    # * Keys and values can only contain lowercase letters, numeric characters,
    #   underscores, and dashes.
    # * International characters are allowed.
    # * Label keys must start with a lowercase letter and international
    #   characters are allowed.
    # * Label keys cannot be empty.
    cls._check_label_name(name)
    cls._check_label_value(value)

    # Ensure that reserved labels are not being used.
    if not cls._allow_reserved_keys and name in RESERVED_LABELS:
      raise ValueError('Label flag (%s=...) must not use reserved keys: %r' %
                       (name, list(RESERVED_LABELS)))

Raise ValueError if the label is invalid .


def _from_yaml_v0(cls, job):
    """Populate a JobDescriptor from the local provider's original meta.yaml.

    The local job provider had the first incarnation of a YAML file for each
    task. That idea was extended here in the JobDescriptor and the local
    provider adopted the JobDescriptor.to_yaml() call to write its meta.yaml.

    The JobDescriptor.from_yaml() detects if it receives a local provider's
    "v0" meta.yaml and calls this function.

    Args:
      job: an object produced from decoding meta.yaml.

    Returns:
      A JobDescriptor populated as best we can from the old meta.yaml.
    """

    # The v0 meta.yaml only contained:
    #   create-time, job-id, job-name, logging, task-id
    #   labels, envs, inputs, outputs
    # It did NOT contain user-id.
    # dsub-version might be there as a label.

    job_metadata = {}
    for key in ['job-id', 'job-name', 'create-time']:
      job_metadata[key] = job.get(key)

    # Make sure that create-time string is turned into a datetime
    job_metadata['create-time'] = dsub_util.replace_timezone(
        datetime.datetime.strptime(job['create-time'], '%Y-%m-%d %H:%M:%S.%f'),
        tzlocal())

    # The v0 meta.yaml contained a "logging" field which was the task-specific
    # logging path. It did not include the actual "--logging" value the user
    # specified.
    job_resources = Resources()

    # The v0 meta.yaml represented a single task.
    # It did not distinguish whether params were job params or task params.
    # We will treat them as either all job params or all task params, based on
    # whether the task-id is empty or an integer value.
    #
    # We also cannot distinguish whether inputs/outputs were recursive or not.
    # Just treat them all as non-recursive.
    params = {}

    # The dsub-version may be in the meta.yaml as a label. If so remove it
    # and set it as a top-level job metadata value.
    labels = job.get('labels', {})
    if 'dsub-version' in labels:
      job_metadata['dsub-version'] = labels['dsub-version']
      del labels['dsub-version']
    params['labels'] = cls._label_params_from_dict(labels)

    params['envs'] = cls._env_params_from_dict(job.get('envs', {}))
    params['inputs'] = cls._input_file_params_from_dict(
        job.get('inputs', {}), False)
    params['outputs'] = cls._output_file_params_from_dict(
        job.get('outputs', {}), False)

    if job.get('task-id') is None:
      job_params = params
      task_metadata = {'task-id': None}
      task_params = {}
    else:
      job_params = {}
      task_metadata = {'task-id': str(job.get('task-id'))}
      task_params = params

    task_resources = Resources(logging_path=job.get('logging'))

    task_descriptors = [
        TaskDescriptor.get_complete_descriptor(task_metadata, task_params,
                                               task_resources)
    ]

    return JobDescriptor.get_complete_descriptor(
        job_metadata, job_params, job_resources, task_descriptors)

Populate a JobDescriptor from the local provider s original meta . yaml .


def from_yaml(cls, yaml_string):
    """Populate and return a JobDescriptor from a YAML string."""
    try:
      job = yaml.full_load(yaml_string)
    except AttributeError:
      # For installations that cannot update their PyYAML version
      job = yaml.load(yaml_string)

    # If the YAML does not contain a top-level dsub version, then assume that
    # the string is coming from the local provider, reading an old version of
    # its meta.yaml.
    dsub_version = job.get('dsub-version')
    if not dsub_version:
      return cls._from_yaml_v0(job)

    job_metadata = {}
    for key in [
        'job-id', 'job-name', 'task-ids', 'user-id', 'dsub-version',
        'user-project', 'script-name'
    ]:
      if job.get(key) is not None:
        job_metadata[key] = job.get(key)

    # Make sure that create-time string is turned into a datetime
    job_metadata['create-time'] = dsub_util.replace_timezone(
        job.get('create-time'), pytz.utc)

    job_resources = Resources(logging=job.get('logging'))

    job_params = {}
    job_params['labels'] = cls._label_params_from_dict(job.get('labels', {}))
    job_params['envs'] = cls._env_params_from_dict(job.get('envs', {}))
    job_params['inputs'] = cls._input_file_params_from_dict(
        job.get('inputs', {}), False)
    job_params['input-recursives'] = cls._input_file_params_from_dict(
        job.get('input-recursives', {}), True)
    job_params['outputs'] = cls._output_file_params_from_dict(
        job.get('outputs', {}), False)
    job_params['output-recursives'] = cls._output_file_params_from_dict(
        job.get('output-recursives', {}), True)
    job_params['mounts'] = cls._mount_params_from_dict(job.get('mounts', {}))

    task_descriptors = []
    for task in job.get('tasks', []):
      task_metadata = {'task-id': task.get('task-id')}

      # Old instances of the meta.yaml do not have a task create time.
      create_time = task.get('create-time')
      if create_time:
        task_metadata['create-time'] = dsub_util.replace_timezone(
            create_time, pytz.utc)

      if task.get('task-attempt') is not None:
        task_metadata['task-attempt'] = task.get('task-attempt')

      task_params = {}
      task_params['labels'] = cls._label_params_from_dict(
          task.get('labels', {}))
      task_params['envs'] = cls._env_params_from_dict(task.get('envs', {}))
      task_params['inputs'] = cls._input_file_params_from_dict(
          task.get('inputs', {}), False)
      task_params['input-recursives'] = cls._input_file_params_from_dict(
          task.get('input-recursives', {}), True)
      task_params['outputs'] = cls._output_file_params_from_dict(
          task.get('outputs', {}), False)
      task_params['output-recursives'] = cls._output_file_params_from_dict(
          task.get('output-recursives', {}), True)

      task_resources = Resources(logging_path=task.get('logging-path'))

      task_descriptors.append(
          TaskDescriptor(task_metadata, task_params, task_resources))

    return JobDescriptor(job_metadata, job_params, job_resources,
                         task_descriptors)

Populate and return a JobDescriptor from a YAML string .


def find_task_descriptor(self, task_id):
    """Returns the task_descriptor corresponding to task_id."""

    # It is not guaranteed that the index will be task_id - 1 when --tasks is
    # used with a min/max range.
    for task_descriptor in self.task_descriptors:
      if task_descriptor.task_metadata.get('task-id') == task_id:
        return task_descriptor
    return None

Returns the task_descriptor corresponding to task_id .


def get_file_environment_variables(file_params):
  """Return a dictionary of environment variables for the user container."""
  env = {}
  for param in file_params:
    # We have no cases where the environment variable provided to user
    # scripts have a trailing slash, so be sure to always strip it.
    # The case that this is specifically handling is --input-recursive and
    # --output-recursive variables, which are directory values.
    env[param.name] = os.path.join(
        DATA_MOUNT_POINT, param.docker_path.rstrip('/')) if param.value else ''
  return env

Return a dictionary of environment variables for the user container .


def get_job_and_task_param(job_params, task_params, field):
  """Returns a dict combining the field for job and task params."""
  return job_params.get(field, set()) | task_params.get(field, set())

def get_job_and_task_param ( job_params , task_params , field ) : return job_params . get ( field , set ( ) ) | task_params . get ( field , set ( ) )

Returns a dict combining the field for job and task params .


def _emit_search_criteria(user_ids, job_ids, task_ids, labels):
  """Print the filters used to delete tasks. Use raw flags as arguments."""
  print('Delete running jobs:')
  print('  user:')
  print('    %s\n' % user_ids)
  print('  job-id:')
  print('    %s\n' % job_ids)
  if task_ids:
    print('  task-id:')
    print('    %s\n' % task_ids)
  # Labels are in a LabelParam namedtuple and must be reformated for printing.
  if labels:
    print('  labels:')
    print('    %s\n' % repr(labels))

Print the filters used to delete tasks . Use raw flags as arguments .


def ddel_tasks(provider,
               user_ids=None,
               job_ids=None,
               task_ids=None,
               labels=None,
               create_time_min=None,
               create_time_max=None):
  """Kill jobs or job tasks.

  This function separates ddel logic from flag parsing and user output. Users
  of ddel who intend to access the data programmatically should use this.

  Args:
    provider: an instantiated dsub provider.
    user_ids: a set of user ids who "own" the job(s) to delete.
    job_ids: a set of job ids to delete.
    task_ids: a set of task ids to delete.
    labels: a set of LabelParam, each must match the job(s) to be cancelled.
    create_time_min: a timezone-aware datetime value for the earliest create
                     time of a task, inclusive.
    create_time_max: a timezone-aware datetime value for the most recent create
                     time of a task, inclusive.

  Returns:
    list of job ids which were deleted.
  """
  # Delete the requested jobs
  deleted_tasks, error_messages = provider.delete_jobs(
      user_ids, job_ids, task_ids, labels, create_time_min, create_time_max)

  # Emit any errors canceling jobs
  for msg in error_messages:
    print(msg)

  return deleted_tasks

def
ddel_tasks
(
provider
,
user_ids
=
,
job_ids
=
,
task_ids
=
,
labels
=
,
create_time_min
=
,
create_time_max
=
)
:
# Delete the requested jobs
deleted_tasks
,
error_messages
=
provider
.
delete_jobs
(
user_ids
,
job_ids
,
task_ids
,
labels
,
create_time_min
,
create_time_max
)
# Emit any errors canceling jobs
for
msg
in
error_messages
:
print
(
msg
)
return
deleted_tasks


def get_action_by_id(op, action_id):
  """Return the operation's array of actions."""
  actions = get_actions(op)
  if actions and 1 <= action_id < len(actions):
    return actions[action_id - 1]

def get_action_by_id ( op , action_id ) : actions = get_actions ( op ) if actions and 1 <= action_id < len ( actions ) : return actions [ action_id - 1 ]

Return the operation s array of actions .


def _get_action_by_name(op, name):
  """Return the value for the specified action."""
  actions = get_actions(op)
  for action in actions:
    if action.get('name') == name:
      return action

def _get_action_by_name ( op , name ) : actions = get_actions ( op ) for action in actions : if action . get ( 'name' ) == name : return action

Return the value for the specified action .


def get_action_environment(op, name):
  """Return the environment for the operation."""
  action = _get_action_by_name(op, name)
  if action:
    return action.get('environment')

def get_action_environment ( op , name ) : action = _get_action_by_name ( op , name ) if action : return action . get ( 'environment' )

Return the environment for the operation .


def get_action_image(op, name):
  """Return the image for the operation."""
  action = _get_action_by_name(op, name)
  if action:
    return action.get('imageUri')

def get_action_image ( op , name ) : action = _get_action_by_name ( op , name ) if action : return action . get ( 'imageUri' )


def get_event_of_type(op, event_type):
  """Return all events of a particular type."""
  events = get_events(op)
  if not events:
    return None

  return [e for e in events if e.get('details', {}).get('@type') == event_type]

def get_event_of_type ( op , event_type ) : events = get_events ( op ) if not events : return None return [ e for e in events if e . get ( 'details' , { } ) . get ( '@type' ) == event_type ]

Return all events of a particular type .


def get_last_update(op):
  """Return the most recent timestamp in the operation."""
  last_update = get_end_time(op)

  if not last_update:
    last_event = get_last_event(op)
    if last_event:
      last_update = last_event['timestamp']

  if not last_update:
    last_update = get_create_time(op)

  return last_update

def get_last_update ( op ) : last_update = get_end_time ( op ) if not last_update : last_event = get_last_event ( op ) if last_event : last_update = last_event [ 'timestamp' ] if not last_update : last_update = get_create_time ( op ) return last_update

Return the most recent timestamp in the operation .


def _prepare_summary_table(rows):
  """Create a new table that is a summary of the input rows.

  All with the same (job-name or job-id, status) go together.

  Args:
    rows: the input rows, a list of dictionaries.
  Returns:
    A new row set of summary information.
  """
  if not rows:
    return []

  # We either group on the job-name (if present) or fall back to the job-id
  key_field = 'job-name'
  if key_field not in rows[0]:
    key_field = 'job-id'

  # Group each of the rows based on (job-name or job-id, status)
  grouped = collections.defaultdict(lambda: collections.defaultdict(lambda: []))
  for row in rows:
    grouped[row.get(key_field, '')][row.get('status', '')] += [row]

  # Now that we have the rows grouped, create a summary table.
  # Use the original table as the driver in order to preserve the order.
  new_rows = []
  for job_key in sorted(grouped.keys()):
    group = grouped.get(job_key, None)
    canonical_status = ['RUNNING', 'SUCCESS', 'FAILURE', 'CANCEL']
    # Written this way to ensure that if somehow a new status is introduced,
    # it shows up in our output.
    for status in canonical_status + sorted(group.keys()):
      if status not in group:
        continue
      task_count = len(group[status])
      del group[status]
      if task_count:
        summary_row = collections.OrderedDict()
        summary_row[key_field] = job_key
        summary_row['status'] = status
        summary_row['task-count'] = task_count
        new_rows.append(summary_row)

  return new_rows

Create a new table that is a summary of the input rows .


def lookup_job_tasks(provider,
                     statuses,
                     user_ids=None,
                     job_ids=None,
                     job_names=None,
                     task_ids=None,
                     task_attempts=None,
                     labels=None,
                     create_time_min=None,
                     create_time_max=None,
                     max_tasks=0,
                     page_size=0,
                     summary_output=False):
  """Generate formatted jobs individually, in order of create-time.

  Args:
    provider: an instantiated dsub provider.
    statuses: a set of status strings that eligible jobs may match.
    user_ids: a set of user strings that eligible jobs may match.
    job_ids: a set of job-id strings eligible jobs may match.
    job_names: a set of job-name strings eligible jobs may match.
    task_ids: a set of task-id strings eligible tasks may match.
    task_attempts: a set of task-attempt strings eligible tasks may match.
    labels: set of LabelParam that all tasks must match.
    create_time_min: a timezone-aware datetime value for the earliest create
                     time of a task, inclusive.
    create_time_max: a timezone-aware datetime value for the most recent create
                     time of a task, inclusive.
    max_tasks: (int) maximum number of tasks to return per dstat job lookup.
    page_size: the page size to use for each query to the backend. May be
               ignored by some provider implementations.
    summary_output: (bool) summarize the job list.

  Yields:
    Individual task dictionaries with associated metadata
  """
  tasks_generator = provider.lookup_job_tasks(
      statuses,
      user_ids=user_ids,
      job_ids=job_ids,
      job_names=job_names,
      task_ids=task_ids,
      task_attempts=task_attempts,
      labels=labels,
      create_time_min=create_time_min,
      create_time_max=create_time_max,
      max_tasks=max_tasks,
      page_size=page_size)

  # Yield formatted tasks.
  for task in tasks_generator:
    yield _prepare_row(task, True, summary_output)

Generate formatted jobs individually in order of create - time .


def prepare_output(self, row):
    """Convert types of task fields."""
    date_fields = ['last-update', 'create-time', 'start-time', 'end-time']
    int_fields = ['task-attempt']

    for col in date_fields:
      if col in row:
        row[col] = self.default_format_date(row[col])

    for col in int_fields:
      if col in row and row[col] is not None:
        row[col] = int(row[col])

    return row

def prepare_output ( self , row ) : date_fields = [ 'last-update' , 'create-time' , 'start-time' , 'end-time' ] int_fields = [ 'task-attempt' ] for col in date_fields : if col in row : row [ col ] = self . default_format_date ( row [ col ] ) for col in int_fields : if col in row and row [ col ] is not None : row [ col ] = int ( row [ col ] ) return row


def trim_display_field(self, value, max_length):
    """Return a value for display; if longer than max length, use ellipsis."""
    if not value:
      return ''
    if len(value) > max_length:
      return value[:max_length - 3] + '...'
    return value

def trim_display_field ( self , value , max_length ) : if not value : return '' if len ( value ) > max_length : return value [ : max_length - 3 ] + '...' return value

Return a value for display ; if longer than max length use ellipsis .


def format_pairs(self, values):
    """Returns a string of comma-delimited key=value pairs."""
    return ', '.join(
        '%s=%s' % (key, value) for key, value in sorted(values.items()))

def
format_pairs
(
self
,
values
)
:
return
', '
.
join
(
'%s=%s'
%
(
key
,
value
)
for
key
,
value
in
sorted
(
values
.
items
(
)
)
)

Returns a string of comma - delimited key = value pairs .


def string_presenter(self, dumper, data):
    """Presenter to force yaml.dump to use multi-line string style."""
    if '\n' in data:
      return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|')
    else:
      return dumper.represent_scalar('tag:yaml.org,2002:str', data)

def string_presenter ( self , dumper , data ) : if '\n' in data : return dumper . represent_scalar ( 'tag:yaml.org,2002:str' , data , style = '|' ) else : return dumper . represent_scalar ( 'tag:yaml.org,2002:str' , data )

Presenter to force yaml . dump to use multi - line string style .


def get_zones(input_list):
  """Returns a list of zones based on any wildcard input.

  This function is intended to provide an easy method for producing a list
  of desired zones for a pipeline to run in.

  The Pipelines API default zone list is "any zone". The problem with
  "any zone" is that it can lead to incurring Cloud Storage egress charges
  if the GCE zone selected is in a different region than the GCS bucket.
  See https://cloud.google.com/storage/pricing#network-egress.

  A user with a multi-region US bucket would want to pipelines to run in
  a "us-*" zone.
  A user with a regional bucket in US would want to restrict pipelines to
  run in a zone in that region.

  Rarely does the specific zone matter for a pipeline.

  This function allows for a simple short-hand such as:
     [ "us-*" ]
     [ "us-central1-*" ]
  These examples will expand out to the full list of US and us-central1 zones
  respectively.

  Args:
    input_list: list of zone names/patterns

  Returns:
    A list of zones, with any wildcard zone specifications expanded.
  """
  if not input_list:
    return []

  output_list = []

  for zone in input_list:
    if zone.endswith('*'):
      prefix = zone[:-1]
      output_list.extend([z for z in _ZONES if z.startswith(prefix)])
    else:
      output_list.append(zone)

  return output_list

def get_zones ( input_list ) : if not input_list : return [ ] output_list = [ ] for zone in input_list : if zone . endswith ( '*' ) : prefix = zone [ : - 1 ] output_list . extend ( [ z for z in _ZONES if z . startswith ( prefix ) ] ) else : output_list . append ( zone ) return output_list

Returns a list of zones based on any wildcard input .


def parse_rfc3339_utc_string(rfc3339_utc_string):
  """Converts a datestamp from RFC3339 UTC to a datetime.

  Args:
    rfc3339_utc_string: a datetime string in RFC3339 UTC "Zulu" format

  Returns:
    A datetime.
  """

  # The timestamp from the Google Operations are all in RFC3339 format, but
  # they are sometimes formatted to millisconds, microseconds, sometimes
  # nanoseconds, and sometimes only seconds:
  # * 2016-11-14T23:05:56Z
  # * 2016-11-14T23:05:56.010Z
  # * 2016-11-14T23:05:56.010429Z
  # * 2016-11-14T23:05:56.010429380Z
  m = re.match(r'(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2}).?(\d*)Z',
               rfc3339_utc_string)

  # It would be unexpected to get a different date format back from Google.
  # If we raise an exception here, we can break people completely.
  # Instead, let's just return None and people can report that some dates
  # are not showing up.
  # We might reconsider this approach in the future; it was originally
  # established when dates were only used for display.
  if not m:
    return None

  groups = m.groups()
  if len(groups[6]) not in (0, 3, 6, 9):
    return None

  # Create a UTC datestamp from parsed components
  # 1- Turn components 0-5 from strings to integers
  # 2- If the last component does not exist, set it to 0.
  #    If it does exist, make sure to interpret it as milliseconds.
  g = [int(val) for val in groups[:6]]

  fraction = groups[6]
  if not fraction:
    micros = 0
  elif len(fraction) == 3:
    micros = int(fraction) * 1000
  elif len(fraction) == 6:
    micros = int(fraction)
  elif len(fraction) == 9:
    # When nanoseconds are provided, we round
    micros = int(round(int(fraction) / 1000))
  else:
    assert False, 'Fraction length not 0, 6, or 9: {}'.len(fraction)

  try:
    return datetime(g[0], g[1], g[2], g[3], g[4], g[5], micros, tzinfo=pytz.utc)
  except ValueError as e:
    assert False, 'Could not parse RFC3339 datestring: {} exception: {}'.format(
        rfc3339_utc_string, e)

Converts a datestamp from RFC3339 UTC to a datetime .


def get_operation_full_job_id(op):
  """Returns the job-id or job-id.task-id for the operation."""
  job_id = op.get_field('job-id')
  task_id = op.get_field('task-id')
  if task_id:
    return '%s.%s' % (job_id, task_id)
  else:
    return job_id

def get_operation_full_job_id ( op ) : job_id = op . get_field ( 'job-id' ) task_id = op . get_field ( 'task-id' ) if task_id : return '%s.%s' % ( job_id , task_id ) else : return job_id

Returns the job - id or job - id . task - id for the operation .


def _cancel_batch(batch_fn, cancel_fn, ops):
  """Cancel a batch of operations.

  Args:
    batch_fn: API-specific batch function.
    cancel_fn: API-specific cancel function.
    ops: A list of operations to cancel.

  Returns:
    A list of operations canceled and a list of error messages.
  """

  # We define an inline callback which will populate a list of
  # successfully canceled operations as well as a list of operations
  # which were not successfully canceled.

  canceled = []
  failed = []

  def handle_cancel_response(request_id, response, exception):
    """Callback for the cancel response."""
    del response  # unused

    if exception:
      # We don't generally expect any failures here, except possibly trying
      # to cancel an operation that is already canceled or finished.
      #
      # If the operation is already finished, provide a clearer message than
      # "error 400: Bad Request".

      msg = 'error %s: %s' % (exception.resp.status, exception.resp.reason)
      if exception.resp.status == FAILED_PRECONDITION_CODE:
        detail = json.loads(exception.content)
        status = detail.get('error', {}).get('status')
        if status == FAILED_PRECONDITION_STATUS:
          msg = 'Not running'

      failed.append({'name': request_id, 'msg': msg})
    else:
      canceled.append({'name': request_id})

    return

  # Set up the batch object
  batch = batch_fn(callback=handle_cancel_response)

  # The callback gets a "request_id" which is the operation name.
  # Build a dict such that after the callback, we can lookup the operation
  # objects by name
  ops_by_name = {}
  for op in ops:
    op_name = op.get_field('internal-id')
    ops_by_name[op_name] = op
    batch.add(cancel_fn(name=op_name, body={}), request_id=op_name)

  # Cancel the operations
  batch.execute()

  # Iterate through the canceled and failed lists to build our return lists
  canceled_ops = [ops_by_name[op['name']] for op in canceled]
  error_messages = []
  for fail in failed:
    op = ops_by_name[fail['name']]
    error_messages.append("Error canceling '%s': %s" %
                          (get_operation_full_job_id(op), fail['msg']))

  return canceled_ops, error_messages


def cancel(batch_fn, cancel_fn, ops):
  """Cancel operations.

  Args:
    batch_fn: API-specific batch function.
    cancel_fn: API-specific cancel function.
    ops: A list of operations to cancel.

  Returns:
    A list of operations canceled and a list of error messages.
  """

  # Canceling many operations one-by-one can be slow.
  # The Pipelines API doesn't directly support a list of operations to cancel,
  # but the requests can be performed in batch.

  canceled_ops = []
  error_messages = []

  max_batch = 256
  total_ops = len(ops)
  for first_op in range(0, total_ops, max_batch):
    batch_canceled, batch_messages = _cancel_batch(
        batch_fn, cancel_fn, ops[first_op:first_op + max_batch])
    canceled_ops.extend(batch_canceled)
    error_messages.extend(batch_messages)

  return canceled_ops, error_messages


def retry_api_check(exception):
  """Return True if we should retry. False otherwise.

  Args:
    exception: An exception to test for transience.

  Returns:
    True if we should retry. False otherwise.
  """
  if isinstance(exception, apiclient.errors.HttpError):
    if exception.resp.status in TRANSIENT_HTTP_ERROR_CODES:
      _print_error('Retrying...')
      return True

  if isinstance(exception, socket.error):
    if exception.errno in TRANSIENT_SOCKET_ERROR_CODES:
      _print_error('Retrying...')
      return True

  if isinstance(exception, oauth2client.client.AccessTokenRefreshError):
    _print_error('Retrying...')
    return True

  # For a given installation, this could be a permanent error, but has only
  # been observed as transient.
  if isinstance(exception, SSLError):
    _print_error('Retrying...')
    return True

  # This has been observed as a transient error:
  #   ServerNotFoundError: Unable to find the server at genomics.googleapis.com
  if isinstance(exception, ServerNotFoundError):
    _print_error('Retrying...')
    return True

  return False

Return True if we should retry . False otherwise .


def retry_auth_check(exception):
  """Specific check for auth error codes.

  Return True if we should retry.

  False otherwise.
  Args:
    exception: An exception to test for transience.

  Returns:
    True if we should retry. False otherwise.
  """
  if isinstance(exception, apiclient.errors.HttpError):
    if exception.resp.status in HTTP_AUTH_ERROR_CODES:
      _print_error('Retrying...')
      return True

  return False

def retry_auth_check ( exception ) : if isinstance ( exception , apiclient . errors . HttpError ) : if exception . resp . status in HTTP_AUTH_ERROR_CODES : _print_error ( 'Retrying...' ) return True return False


def setup_service(api_name, api_version, credentials=None):
  """Configures genomics API client.

  Args:
    api_name: Name of the Google API (for example: "genomics")
    api_version: Version of the API (for example: "v2alpha1")
    credentials: Credentials to be used for the gcloud API calls.

  Returns:
    A configured Google Genomics API client with appropriate credentials.
  """
  if not credentials:
    credentials = oauth2client.client.GoogleCredentials.get_application_default(
    )
  return apiclient.discovery.build(
      api_name, api_version, credentials=credentials)

def setup_service ( api_name , api_version , credentials = None ) : if not credentials : credentials = oauth2client . client . GoogleCredentials . get_application_default ( ) return apiclient . discovery . build ( api_name , api_version , credentials = credentials )


def execute(api):
    """Executes operation.

    Args:
      api: The base API object

    Returns:
       A response body object
    """
    try:
      return api.execute()
    except Exception as exception:
      now = datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')
      _print_error('%s: Exception %s: %s' % (now, type(exception).__name__,
                                             str(exception)))
      # Re-raise exception to be handled by retry logic
      raise exception


def _eval_arg_type(arg_type, T=Any, arg=None, sig=None):
    """Returns a type from a snippit of python source. Should normally be
    something just like 'str' or 'Object'.

        arg_type            the source to be evaluated
        T                         the default type
        arg                     context of where this type was extracted
        sig                     context from where the arg was extracted

    Returns a type or a Type
    """
    try:
        T = eval(arg_type)
    except Exception as e:
        raise ValueError('The type of {0} could not be evaluated in {1} for {2}: {3}' \
            .format(arg_type, arg, sig, text_type(e)))
    else:
        if type(T) not in (type, Type):
            raise TypeError('{0} is not a valid type in {1} for {2}' \
                .format(repr(T), arg, sig))
        return T

Returns a type from a snippit of python source . Should normally be something just like str or Object .


def jsonify_status_code(status_code, *args, **kw):
    """Returns a jsonified response with the specified HTTP status code.

    The positional and keyword arguments are passed directly to the
    :func:`flask.jsonify` function which creates the response.
    """
    is_batch = kw.pop('is_batch', False)
    if is_batch:
        response = flask_make_response(json.dumps(*args, **kw))
        response.mimetype = 'application/json'
        response.status_code = status_code
        return response
    response = jsonify(*args, **kw)
    response.status_code = status_code
    return response

def jsonify_status_code ( status_code , * args , * * kw ) : is_batch = kw . pop ( 'is_batch' , False ) if is_batch : response = flask_make_response ( json . dumps ( * args , * * kw ) ) response . mimetype = 'application/json' response . status_code = status_code return response response = jsonify ( * args , * * kw ) response . status_code = status_code return response

Returns a jsonified response with the specified HTTP status code .


def send_payload(self, params):
        """Performs the actual sending action and returns the result
        """
        data = json.dumps({
            'jsonrpc': self.version,
            'method': self.service_name,
            'params': params,
            'id': text_type(uuid.uuid4())
        })
        data_binary = data.encode('utf-8')
        url_request = Request(self.service_url, data_binary, headers=self.headers)
        return urlopen(url_request).read()

def send_payload ( self , params ) : data = json . dumps ( { 'jsonrpc' : self . version , 'method' : self . service_name , 'params' : params , 'id' : text_type ( uuid . uuid4 ( ) ) } ) data_binary = data . encode ( 'utf-8' ) url_request = Request ( self . service_url , data_binary , headers = self . headers ) return urlopen ( url_request ) . read ( )


def json_rpc_format(self):
        """Return the Exception data in a format for JSON-RPC
        """

        error = {
            'name': text_type(self.__class__.__name__),
            'code': self.code,
            'message': '{0}'.format(text_type(self.message)),
            'data': self.data
        }

        if current_app.config['DEBUG']:
            import sys, traceback
            error['stack'] = traceback.format_exc()
            error['executable'] = sys.executable

        return error

def json_rpc_format ( self ) : error = { 'name' : text_type ( self . __class__ . __name__ ) , 'code' : self . code , 'message' : '{0}' . format ( text_type ( self . message ) ) , 'data' : self . data } if current_app . config [ 'DEBUG' ] : import sys , traceback error [ 'stack' ] = traceback . format_exc ( ) error [ 'executable' ] = sys . executable return error

Return the Exception data in a format for JSON - RPC


def from_file(cls, file):
        """Try loading given config file.

        :param str file: full path to the config file to load
        """
        if not os.path.exists(file):
            raise ValueError("Config file not found.")

        try:
            config_parser = configparser.ConfigParser()
            config_parser.read(file)

            configuration = cls(file, config_parser)
            if not configuration.check_config_sanity():
                raise ValueError("Error in config file.")
            else:
                return configuration
        except configparser.Error:
            raise ValueError("Config file is invalid.")

def
from_file
(
cls
,
file
)
:
if
not
os
.
path
.
exists
(
file
)
:
raise
ValueError
(
"Config file not found."
)
try
:
config_parser
=
configparser
.
ConfigParser
(
)
config_parser
.
read
(
file
)
configuration
=
cls
(
file
,
config_parser
)
if
not
configuration
.
check_config_sanity
(
)
:
raise
ValueError
(
"Error in config file."
)
else
:
return
configuration
except
configparser
.
Error
:
raise
ValueError
(
"Config file is invalid."
)


def discover(cls):
        """Make a guess about the config file location an try loading it."""
        file = os.path.join(Config.config_dir, Config.config_name)
        return cls.from_file(file)

def discover ( cls ) : file = os . path . join ( Config . config_dir , Config . config_name ) return cls . from_file ( file )

Make a guess about the config file location an try loading it .


def create_config(cls, cfgfile, nick, twtfile, twturl, disclose_identity, add_news):
        """Create a new config file at the default location.

        :param str cfgfile: path to the config file
        :param str nick: nickname to use for own tweets
        :param str twtfile: path to the local twtxt file
        :param str twturl: URL to the remote twtxt file
        :param bool disclose_identity: if true the users id will be disclosed
        :param bool add_news: if true follow twtxt news feed
        """
        cfgfile_dir = os.path.dirname(cfgfile)
        if not os.path.exists(cfgfile_dir):
            os.makedirs(cfgfile_dir)

        cfg = configparser.ConfigParser()

        cfg.add_section("twtxt")
        cfg.set("twtxt", "nick", nick)
        cfg.set("twtxt", "twtfile", twtfile)
        cfg.set("twtxt", "twturl", twturl)
        cfg.set("twtxt", "disclose_identity", str(disclose_identity))
        cfg.set("twtxt", "character_limit", "140")
        cfg.set("twtxt", "character_warning", "140")

        cfg.add_section("following")
        if add_news:
            cfg.set("following", "twtxt", "https://buckket.org/twtxt_news.txt")

        conf = cls(cfgfile, cfg)
        conf.write_config()
        return conf

def create_config ( cls , cfgfile , nick , twtfile , twturl , disclose_identity , add_news ) : cfgfile_dir = os . path . dirname ( cfgfile ) if not os . path . exists ( cfgfile_dir ) : os . makedirs ( cfgfile_dir ) cfg = configparser . ConfigParser ( ) cfg . add_section ( "twtxt" ) cfg . set ( "twtxt" , "nick" , nick ) cfg . set ( "twtxt" , "twtfile" , twtfile ) cfg . set ( "twtxt" , "twturl" , twturl ) cfg . set ( "twtxt" , "disclose_identity" , str ( disclose_identity ) ) cfg . set ( "twtxt" , "character_limit" , "140" ) cfg . set ( "twtxt" , "character_warning" , "140" ) cfg . add_section ( "following" ) if add_news : cfg . set ( "following" , "twtxt" , "https://buckket.org/twtxt_news.txt" ) conf = cls ( cfgfile , cfg ) conf . write_config ( ) return conf

Create a new config file at the default location .


def write_config(self):
        """Writes `self.cfg` to `self.config_file`."""
        with open(self.config_file, "w") as config_file:
            self.cfg.write(config_file)

def write_config ( self ) : with open ( self . config_file , "w" ) as config_file : self . cfg . write ( config_file )

Writes self . cfg to self . config_file .


def validate_config_key(ctx, param, value):
    """Validate a configuration key according to `section.item`."""
    if not value:
        return value

    try:
        section, item = value.split(".", 1)
    except ValueError:
        raise click.BadArgumentUsage("Given key does not contain a section name.")
    else:
        return section, item

Validate a configuration key according to section . item .


def expand_mentions(text, embed_names=True):
    """Searches the given text for mentions and expands them.

    For example:
    "@source.nick" will be expanded to "@<source.nick source.url>".
    """
    if embed_names:
        mention_format = "@<{name} {url}>"
    else:
        mention_format = "@<{url}>"

    def handle_mention(match):
        source = get_source_by_name(match.group(1))
        if source is None:
            return "@{0}".format(match.group(1))
        return mention_format.format(
            name=source.nick,
            url=source.url)

    return short_mention_re.sub(handle_mention, text)

def
expand_mentions
(
text
,
embed_names
=
True
)
:
if
embed_names
:
mention_format
=
"@<{name} {url}>"
else
:
mention_format
=
"@<{url}>"
def
handle_mention
(
match
)
:
source
=
get_source_by_name
(
match
.
group
(
1
)
)
if
source
is
:
return
"@{0}"
.
format
(
match
.
group
(
1
)
)
return
mention_format
.
format
(
name
=
source
.
nick
,
url
=
source
.
url
)
return
short_mention_re
.
sub
(
handle_mention
,
text
)

Searches the given text for mentions and expands them .


def make_aware(dt):
    """Appends tzinfo and assumes UTC, if datetime object has no tzinfo already."""
    return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc)

def make_aware ( dt ) : return dt if dt . tzinfo else dt . replace ( tzinfo = timezone . utc )

Appends tzinfo and assumes UTC if datetime object has no tzinfo already .


def from_file(cls, file, *args, **kwargs):
        """Try loading given cache file."""
        try:
            cache = shelve.open(file)
            return cls(file, cache, *args, **kwargs)
        except OSError as e:
            logger.debug("Loading {0} failed".format(file))
            raise e

def
from_file
(
cls
,
file
,
*
args
,
*
*
kwargs
)
:
try
:
cache
=
shelve
.
open
(
file
)
return
cls
(
file
,
cache
,
*
args
,
*
*
kwargs
)
except
OSError
as
e
:
logger
.
debug
(
"Loading {0} failed"
.
format
(
file
)
)
raise
e


def discover(cls, *args, **kwargs):
        """Make a guess about the cache file location an try loading it."""
        file = os.path.join(Cache.cache_dir, Cache.cache_name)
        return cls.from_file(file, *args, **kwargs)

def discover ( cls , * args , * * kwargs ) : file = os . path . join ( Cache . cache_dir , Cache . cache_name ) return cls . from_file ( file , * args , * * kwargs )

Make a guess about the cache file location an try loading it .


def is_cached(self, url):
        """Checks if specified URL is cached."""
        try:
            return True if url in self.cache else False
        except TypeError:
            return False

def is_cached ( self , url ) : try : return True if url in self . cache else False except TypeError : return False


def add_tweets(self, url, last_modified, tweets):
        """Adds new tweets to the cache."""
        try:
            self.cache[url] = {"last_modified": last_modified, "tweets": tweets}
            self.mark_updated()
            return True
        except TypeError:
            return False

def add_tweets ( self , url , last_modified , tweets ) : try : self . cache [ url ] = { "last_modified" : last_modified , "tweets" : tweets } self . mark_updated ( ) return True except TypeError : return False


def get_tweets(self, url, limit=None):
        """Retrieves tweets from the cache."""
        try:
            tweets = self.cache[url]["tweets"]
            self.mark_updated()
            return sorted(tweets, reverse=True)[:limit]
        except KeyError:
            return []

def get_tweets ( self , url , limit = None ) : try : tweets = self . cache [ url ] [ "tweets" ] self . mark_updated ( ) return sorted ( tweets , reverse = True ) [ : limit ] except KeyError : return [ ]


def remove_tweets(self, url):
        """Tries to remove cached tweets."""
        try:
            del self.cache[url]
            self.mark_updated()
            return True
        except KeyError:
            return False

def remove_tweets ( self , url ) : try : del self . cache [ url ] self . mark_updated ( ) return True except KeyError : return False


def timeline(ctx, pager, limit, twtfile, sorting, timeout, porcelain, source, cache, force_update):
    """Retrieve your personal timeline."""
    if source:
        source_obj = ctx.obj["conf"].get_source_by_nick(source)
        if not source_obj:
            logger.debug("Not following {0}, trying as URL".format(source))
            source_obj = Source(source, source)
        sources = [source_obj]
    else:
        sources = ctx.obj["conf"].following

    tweets = []

    if cache:
        try:
            with Cache.discover(update_interval=ctx.obj["conf"].timeline_update_interval) as cache:
                force_update = force_update or not cache.is_valid
                if force_update:
                    tweets = get_remote_tweets(sources, limit, timeout, cache)
                else:
                    logger.debug("Multiple calls to 'timeline' within {0} seconds. Skipping update".format(
                        cache.update_interval))
                    # Behold, almighty list comprehensions! (I might have gone overboard here…)
                    tweets = list(chain.from_iterable([cache.get_tweets(source.url) for source in sources]))
        except OSError as e:
            logger.debug(e)
            tweets = get_remote_tweets(sources, limit, timeout)
    else:
        tweets = get_remote_tweets(sources, limit, timeout)

    if twtfile and not source:
        source = Source(ctx.obj["conf"].nick, ctx.obj["conf"].twturl, file=twtfile)
        tweets.extend(get_local_tweets(source, limit))

    if not tweets:
        return

    tweets = sort_and_truncate_tweets(tweets, sorting, limit)

    if pager:
        click.echo_via_pager(style_timeline(tweets, porcelain))
    else:
        click.echo(style_timeline(tweets, porcelain))


def config(ctx, key, value, remove, edit):
    """Get or set config item."""
    conf = ctx.obj["conf"]

    if not edit and not key:
        raise click.BadArgumentUsage("You have to specify either a key or use --edit.")

    if edit:
        return click.edit(filename=conf.config_file)

    if remove:
        try:
            conf.cfg.remove_option(key[0], key[1])
        except Exception as e:
            logger.debug(e)
        else:
            conf.write_config()
        return

    if not value:
        try:
            click.echo(conf.cfg.get(key[0], key[1]))
        except Exception as e:
            logger.debug(e)
        return

    if not conf.cfg.has_section(key[0]):
        conf.cfg.add_section(key[0])

    conf.cfg.set(key[0], key[1], value)
    conf.write_config()


def relative_datetime(self):
        """Return human-readable relative time string."""
        now = datetime.now(timezone.utc)
        tense = "from now" if self.created_at > now else "ago"
        return "{0} {1}".format(humanize.naturaldelta(now - self.created_at), tense)

def
relative_datetime
(
self
)
:
now
=
datetime
.
now
(
timezone
.
utc
)
tense
=
"from now"
if
self
.
created_at
>
now
else
"ago"
return
"{0} {1}"
.
format
(
humanize
.
naturaldelta
(
now
self
.
created_at
)
,
tense
)

Return human - readable relative time string .


def save(url, *args, **kwargs):
    """ Parse the options, set defaults and then fire up PhantomJS. """

    device = heimdallDevice(kwargs.get('device', None))

    kwargs['width'] = kwargs.get('width', None) or device.width
    kwargs['height'] = kwargs.get('height', None) or device.height
    kwargs['user_agent'] = kwargs.get('user_agent', None) or device.user_agent

    screenshot_image = screenshot(url, **kwargs)

    if kwargs.get('optimize'):
        image = Image.open(screenshot_image.path)
        image.save(screenshot_image.path, optimize=True)

    return screenshot_image

def save ( url , * args , * * kwargs ) : device = heimdallDevice ( kwargs . get ( 'device' , None ) ) kwargs [ 'width' ] = kwargs . get ( 'width' , None ) or device . width kwargs [ 'height' ] = kwargs . get ( 'height' , None ) or device . height kwargs [ 'user_agent' ] = kwargs . get ( 'user_agent' , None ) or device . user_agent screenshot_image = screenshot ( url , * * kwargs ) if kwargs . get ( 'optimize' ) : image = Image . open ( screenshot_image . path ) image . save ( screenshot_image . path , optimize = True ) return screenshot_image

Parse the options set defaults and then fire up PhantomJS .


def screenshot(url, *args, **kwargs):
    """ Call PhantomJS with the specified flags and options. """

    phantomscript = os.path.join(os.path.dirname(__file__),
                                 'take_screenshot.js')

    directory = kwargs.get('save_dir', '/tmp')
    image_name = kwargs.get('image_name', None) or _image_name_from_url(url)
    ext = kwargs.get('format', 'png').lower()
    save_path = os.path.join(directory, image_name) + '.' + ext
    crop_to_visible = kwargs.get('crop_to_visible', False)

    cmd_args = [
        'phantomjs',
        '--ssl-protocol=any',
        phantomscript,
        url,
        '--width',
        str(kwargs['width']),
        '--height',
        str(kwargs['height']),
        '--useragent',
        str(kwargs['user_agent']),
        '--dir',
        directory,
        '--ext',
        ext,
        '--name',
        str(image_name),
    ]
    if crop_to_visible:
        cmd_args.append('--croptovisible')

    # TODO:
    # - quality
    # - renderafter
    # - maxexecutiontime
    # - resourcetimeout

    output = subprocess.Popen(cmd_args,
                              stdout=subprocess.PIPE).communicate()[0]

    return Screenshot(save_path, directory, image_name + '.' + ext, ext)

def
screenshot
(
url
,
*
args
,
*
*
kwargs
)
:
phantomscript
=
os
.
path
.
join
(
os
.
path
.
dirname
(
__file__
)
,
'take_screenshot.js'
)
directory
=
kwargs
.
get
(
'save_dir'
,
'/tmp'
)
image_name
=
kwargs
.
get
(
'image_name'
,
)
or
_image_name_from_url
(
url
)
ext
=
kwargs
.
get
(
'format'
,
'png'
)
.
lower
(
)
save_path
=
os
.
path
.
join
(
directory
,
image_name
)
+
'.'
+
ext
crop_to_visible
=
kwargs
.
get
(
'crop_to_visible'
,
False
)
cmd_args
=
[
'phantomjs'
,
'--ssl-protocol=any'
,
phantomscript
,
url
,
'--width'
,
str
(
kwargs
[
'width'
]
)
,
'--height'
,
str
(
kwargs
[
'height'
]
)
,
'--useragent'
,
str
(
kwargs
[
'user_agent'
]
)
,
'--dir'
,
directory
,
'--ext'
,
ext
,
'--name'
,
str
(
image_name
)
,
]
if
crop_to_visible
:
cmd_args
.
append
(
'--croptovisible'
)
# TODO:
# - quality
# - renderafter
# - maxexecutiontime
# - resourcetimeout
output
=
subprocess
.
Popen
(
cmd_args
,
stdout
=
subprocess
.
PIPE
)
.
communicate
(
)
[
0
]
return
Screenshot
(
save_path
,
directory
,
image_name
+
'.'
+
ext
,
ext
)

Call PhantomJS with the specified flags and options .


def _image_name_from_url(url):
    """ Create a nice image name from the url. """

    find = r'https?://|[^\w]'
    replace = '_'
    return re.sub(find, replace, url).strip('_')

def _image_name_from_url ( url ) : find = r'https?://|[^\w]' replace = '_' return re . sub ( find , replace , url ) . strip ( '_' )


def worker(f):
    """
    Decorator. Abortable worker. If wrapped task will be cancelled by
    dispatcher, decorator will send ftp codes of successful interrupt.

    ::

        >>> @worker
        ... async def worker(self, connection, rest):
        ...     ...

    """
    @functools.wraps(f)
    async def wrapper(cls, connection, rest):
        try:
            await f(cls, connection, rest)
        except asyncio.CancelledError:
            connection.response("426", "transfer aborted")
            connection.response("226", "abort successful")

    return wrapper

def
worker
(
f
)
:
@
functools
.
wraps
(
f
)
async
def
wrapper
(
cls
,
connection
,
rest
)
:
try
:
await
f
(
cls
,
connection
,
rest
)
except
asyncio
.
CancelledError
:
connection
.
response
(
"426"
,
"transfer aborted"
)
connection
.
response
(
"226"
,
"abort successful"
)
return
wrapper

Decorator . Abortable worker . If wrapped task will be cancelled by dispatcher decorator will send ftp codes of successful interrupt .


def get_permissions(self, path):
        """
        Return nearest parent permission for `path`.

        :param path: path which permission you want to know
        :type path: :py:class:`str` or :py:class:`pathlib.PurePosixPath`

        :rtype: :py:class:`aioftp.Permission`
        """
        path = pathlib.PurePosixPath(path)
        parents = filter(lambda p: p.is_parent(path), self.permissions)
        perm = min(
            parents,
            key=lambda p: len(path.relative_to(p.path).parts),
            default=Permission(),
        )
        return perm

def get_permissions ( self , path ) : path = pathlib . PurePosixPath ( path ) parents = filter ( lambda p : p . is_parent ( path ) , self . permissions ) perm = min ( parents , key = lambda p : len ( path . relative_to ( p . path ) . parts ) , default = Permission ( ) , ) return perm

Return nearest parent permission for path .


def release(self):
        """
        Release, incrementing the internal counter by one.
        """
        if self.value is not None:
            self.value += 1
            if self.value > self.maximum_value:
                raise ValueError("Too many releases")

def
release
(
self
)
:
if
self
.
value
is
not
:
self
.
value
+=
1
if
self
.
value
>
self
.
maximum_value
:
raise
ValueError
(
"Too many releases"
)

Release incrementing the internal counter by one .


def register_memory():
    """Register an approximation of memory used by FTP server process
    and all of its children.
    """
    # XXX How to get a reliable representation of memory being used is
    # not clear. (rss - shared) seems kind of ok but we might also use
    # the private working set via get_memory_maps().private*.
    def get_mem(proc):
        if os.name == 'posix':
            mem = proc.memory_info_ex()
            counter = mem.rss
            if 'shared' in mem._fields:
                counter -= mem.shared
            return counter
        else:
            # TODO figure out what to do on Windows
            return proc.get_memory_info().rss

    if SERVER_PROC is not None:
        mem = get_mem(SERVER_PROC)
        for child in SERVER_PROC.children():
            mem += get_mem(child)
        server_memory.append(bytes2human(mem))

Register an approximation of memory used by FTP server process and all of its children .


def connect():
    """Connect to FTP server, login and return an ftplib.FTP instance."""
    ftp_class = ftplib.FTP if not SSL else ftplib.FTP_TLS
    ftp = ftp_class(timeout=TIMEOUT)
    ftp.connect(HOST, PORT)
    ftp.login(USER, PASSWORD)
    if SSL:
        ftp.prot_p()  # secure data connection
    return ftp

def
connect
(
)
:
ftp_class
=
ftplib
.
FTP
if
not
SSL
else
ftplib
.
FTP_TLS
ftp
=
ftp_class
(
timeout
=
TIMEOUT
)
ftp
.
connect
(
HOST
,
PORT
)
ftp
.
login
(
USER
,
PASSWORD
)
if
SSL
:
ftp
.
prot_p
(
)
# secure data connection
return
ftp

Connect to FTP server login and return an ftplib . FTP instance .


def bytes_per_second(ftp, retr=True):
    """Return the number of bytes transmitted in 1 second."""
    tot_bytes = 0
    if retr:
        def request_file():
            ftp.voidcmd('TYPE I')
            conn = ftp.transfercmd("retr " + TESTFN)
            return conn

        with contextlib.closing(request_file()) as conn:
            register_memory()
            stop_at = time.time() + 1.0
            while stop_at > time.time():
                chunk = conn.recv(BUFFER_LEN)
                if not chunk:
                    a = time.time()
                    ftp.voidresp()
                    conn.close()
                    conn = request_file()
                    stop_at += time.time() - a
                tot_bytes += len(chunk)

        try:
            while chunk:
                chunk = conn.recv(BUFFER_LEN)
            ftp.voidresp()
            conn.close()
        except (ftplib.error_temp, ftplib.error_perm):
            pass
    else:
        ftp.voidcmd('TYPE I')
        with contextlib.closing(ftp.transfercmd("STOR " + TESTFN)) as conn:
            register_memory()
            chunk = b'x' * BUFFER_LEN
            stop_at = time.time() + 1
            while stop_at > time.time():
                tot_bytes += conn.send(chunk)
        ftp.voidresp()

    return tot_bytes

def
bytes_per_second
(
ftp
,
retr
=
True
)
:
tot_bytes
=
0
if
retr
:
def
request_file
(
)
:
ftp
.
voidcmd
(
'TYPE I'
)
conn
=
ftp
.
transfercmd
(
"retr "
+
TESTFN
)
return
conn
with
contextlib
.
closing
(
request_file
(
)
)
as
conn
:
register_memory
(
)
stop_at
=
time
.
time
(
)
+
1.0
while
stop_at
>
time
.
time
(
)
:
chunk
=
conn
.
recv
(
BUFFER_LEN
)
if
not
chunk
:
a
=
time
.
time
(
)
ftp
.
voidresp
(
)
conn
.
close
(
)
conn
=
request_file
(
)
stop_at
+=
time
.
time
(
)
a
tot_bytes
+=
len
(
chunk
)
try
:
while
chunk
:
chunk
=
conn
.
recv
(
BUFFER_LEN
)
ftp
.
voidresp
(
)
conn
.
close
(
)
except
(
ftplib
.
error_temp
,
ftplib
.
error_perm
)
:
pass
else
:
ftp
.
voidcmd
(
'TYPE I'
)
with
contextlib
.
closing
(
ftp
.
transfercmd
(
"STOR "
+
TESTFN
)
)
as
conn
:
register_memory
(
)
chunk
=
b'x'
*
BUFFER_LEN
stop_at
=
time
.
time
(
)
+
1
while
stop_at
>
time
.
time
(
)
:
tot_bytes
+=
conn
.
send
(
chunk
)
ftp
.
voidresp
(
)
return
tot_bytes

Return the number of bytes transmitted in 1 second .


def async_enterable(f):
    """
    Decorator. Bring coroutine result up, so it can be used as async context

    ::

        >>> async def foo():
        ...
        ...     ...
        ...     return AsyncContextInstance(...)
        ...
        ... ctx = await foo()
        ... async with ctx:
        ...
        ...     # do

    ::

        >>> @async_enterable
        ... async def foo():
        ...
        ...     ...
        ...     return AsyncContextInstance(...)
        ...
        ... async with foo() as ctx:
        ...
        ...     # do
        ...
        ... ctx = await foo()
        ... async with ctx:
        ...
        ...     # do

    """
    @functools.wraps(f)
    def wrapper(*args, **kwargs):

        class AsyncEnterableInstance:

            async def __aenter__(self):
                self.context = await f(*args, **kwargs)
                return await self.context.__aenter__()

            async def __aexit__(self, *args, **kwargs):
                await self.context.__aexit__(*args, **kwargs)

            def __await__(self):
                return f(*args, **kwargs).__await__()

        return AsyncEnterableInstance()

    return wrapper

def async_enterable ( f ) : @ functools . wraps ( f ) def wrapper ( * args , * * kwargs ) : class AsyncEnterableInstance : async def __aenter__ ( self ) : self . context = await f ( * args , * * kwargs ) return await self . context . __aenter__ ( ) async def __aexit__ ( self , * args , * * kwargs ) : await self . context . __aexit__ ( * args , * * kwargs ) def __await__ ( self ) : return f ( * args , * * kwargs ) . __await__ ( ) return AsyncEnterableInstance ( ) return wrapper

Decorator . Bring coroutine result up so it can be used as async context


def setlocale(name):
    """
    Context manager with threading lock for set locale on enter, and set it
    back to original state on exit.

    ::

        >>> with setlocale("C"):
        ...     ...
    """
    with LOCALE_LOCK:
        old_locale = locale.setlocale(locale.LC_ALL)
        try:
            yield locale.setlocale(locale.LC_ALL, name)
        finally:
            locale.setlocale(locale.LC_ALL, old_locale)

def setlocale ( name ) : with LOCALE_LOCK : old_locale = locale . setlocale ( locale . LC_ALL ) try : yield locale . setlocale ( locale . LC_ALL , name ) finally : locale . setlocale ( locale . LC_ALL , old_locale )

Context manager with threading lock for set locale on enter and set it back to original state on exit .


def append(self, data, start):
        """
        Count `data` for throttle

        :param data: bytes of data for count
        :type data: :py:class:`bytes`

        :param start: start of read/write time from
            :py:meth:`asyncio.BaseEventLoop.time`
        :type start: :py:class:`float`
        """
        if self._limit is not None and self._limit > 0:
            if self._start is None:
                self._start = start
            if start - self._start > self.reset_rate:
                self._sum -= round((start - self._start) * self._limit)
                self._start = start
            self._sum += len(data)

def append ( self , data , start ) : if self . _limit is not None and self . _limit > 0 : if self . _start is None : self . _start = start if start - self . _start > self . reset_rate : self . _sum -= round ( ( start - self . _start ) * self . _limit ) self . _start = start self . _sum += len ( data )


def limit(self, value):
        """
        Set throttle limit

        :param value: bytes per second
        :type value: :py:class:`int` or :py:class:`None`
        """
        self._limit = value
        self._start = None
        self._sum = 0

def limit ( self , value ) : self . _limit = value self . _start = None self . _sum = 0


def clone(self):
        """
        Clone throttles without memory
        """
        return StreamThrottle(
            read=self.read.clone(),
            write=self.write.clone()
        )

def clone ( self ) : return StreamThrottle ( read = self . read . clone ( ) , write = self . write . clone ( ) )


def append(self, name, data, start):
        """
        Update timeout for all throttles

        :param name: name of throttle to append to ("read" or "write")
        :type name: :py:class:`str`

        :param data: bytes of data for count
        :type data: :py:class:`bytes`

        :param start: start of read/write time from
            :py:meth:`asyncio.BaseEventLoop.time`
        :type start: :py:class:`float`
        """
        for throttle in self.throttles.values():
            getattr(throttle, name).append(data, start)

def append ( self , name , data , start ) : for throttle in self . throttles . values ( ) : getattr ( throttle , name ) . append ( data , start )


def check_codes(self, expected_codes, received_code, info):
        """
        Checks if any of expected matches received.

        :param expected_codes: tuple of expected codes
        :type expected_codes: :py:class:`tuple`

        :param received_code: received code for matching
        :type received_code: :py:class:`aioftp.Code`

        :param info: list of response lines from server
        :type info: :py:class:`list`

        :raises aioftp.StatusCodeError: if received code does not matches any
            expected code
        """
        if not any(map(received_code.matches, expected_codes)):
            raise errors.StatusCodeError(expected_codes, received_code, info)

def check_codes ( self , expected_codes , received_code , info ) : if not any ( map ( received_code . matches , expected_codes ) ) : raise errors . StatusCodeError ( expected_codes , received_code , info )

Checks if any of expected matches received .


def parse_directory_response(s):
        """
        Parsing directory server response.

        :param s: response line
        :type s: :py:class:`str`

        :rtype: :py:class:`pathlib.PurePosixPath`
        """
        seq_quotes = 0
        start = False
        directory = ""
        for ch in s:
            if not start:
                if ch == "\"":
                    start = True
            else:
                if ch == "\"":
                    seq_quotes += 1
                else:
                    if seq_quotes == 1:
                        break
                    elif seq_quotes == 2:
                        seq_quotes = 0
                        directory += '"'
                    directory += ch
        return pathlib.PurePosixPath(directory)

def parse_directory_response ( s ) : seq_quotes = 0 start = False directory = "" for ch in s : if not start : if ch == "\"" : start = True else : if ch == "\"" : seq_quotes += 1 else : if seq_quotes == 1 : break elif seq_quotes == 2 : seq_quotes = 0 directory += '"' directory += ch return pathlib . PurePosixPath ( directory )


def parse_list_line_windows(self, b):
        """
        Parsing Microsoft Windows `dir` output

        :param b: response line
        :type b: :py:class:`bytes` or :py:class:`str`

        :return: (path, info)
        :rtype: (:py:class:`pathlib.PurePosixPath`, :py:class:`dict`)
        """
        line = b.decode(encoding=self.encoding).rstrip("\r\n")
        date_time_end = line.index("M")
        date_time_str = line[:date_time_end + 1].strip().split(" ")
        date_time_str = " ".join([x for x in date_time_str if len(x) > 0])
        line = line[date_time_end + 1:].lstrip()
        with setlocale("C"):
            strptime = datetime.datetime.strptime
            date_time = strptime(date_time_str, "%m/%d/%Y %I:%M %p")
        info = {}
        info["modify"] = self.format_date_time(date_time)
        next_space = line.index(" ")
        if line.startswith("<DIR>"):
            info["type"] = "dir"
        else:
            info["type"] = "file"
            info["size"] = line[:next_space].replace(",", "")
            if not info["size"].isdigit():
                raise ValueError
        # This here could cause a problem if a filename started with
        # whitespace, but if we were to try to detect such a condition
        # we would have to make strong assumptions about the input format
        filename = line[next_space:].lstrip()
        if filename == "." or filename == "..":
            raise ValueError
        return pathlib.PurePosixPath(filename), info


def upload_stream(self, destination, *, offset=0):
        """
        Create stream for write data to `destination` file.

        :param destination: destination path of file on server side
        :type destination: :py:class:`str` or :py:class:`pathlib.PurePosixPath`

        :param offset: byte offset for stream start position
        :type offset: :py:class:`int`

        :rtype: :py:class:`aioftp.DataConnectionThrottleStreamIO`
        """
        return self.get_stream(
            "STOR " + str(destination),
            "1xx",
            offset=offset,
        )

def
upload_stream
(
self
,
destination
,
*
,
offset
=
0
)
:
return
self
.
get_stream
(
"STOR "
+
str
(
destination
)
,
"1xx"
,
offset
=
offset
,
)

Create stream for write data to destination file .


def jenks_breaks(values, nb_class):
    """
    Compute jenks natural breaks on a sequence of `values`, given `nb_class`,
    the number of desired class.

    Parameters
    ----------
    values : array-like
        The Iterable sequence of numbers (integer/float) to be used.
    nb_class : int
        The desired number of class (as some other functions requests
        a `k` value, `nb_class` is like `k` + 1). Have to be lesser than
        the length of `values` and greater than 2.

    Returns
    -------
    breaks : tuple of floats
        The computed break values, including minimum and maximum, in order
        to have all the bounds for building `nb_class` class,
        so the returned tuple has a length of `nb_class` + 1.

    Examples
    --------
    Using nb_class = 3, expecting 4 break values , including min and max :

    >>> jenks_breaks(
            [1.3, 7.1, 7.3, 2.3, 3.9, 4.1, 7.8, 1.2, 4.3, 7.3, 5.0, 4.3],
            nb_class = 3)  # Should output (1.2, 2.3, 5.0, 7.8)

    """

    if not isinstance(values, Iterable) or isinstance(values, (str, bytes)):
        raise TypeError("A sequence of numbers is expected")
    if isinstance(nb_class, float) and int(nb_class) == nb_class:
        nb_class = int(nb_class)
    if not isinstance(nb_class, int):
        raise TypeError(
            "Number of class have to be a positive integer: "
            "expected an instance of 'int' but found {}"
            .format(type(nb_class)))

    nb_values = len(values)
    if np and isinstance(values, np.ndarray):
        values = values[np.argwhere(np.isfinite(values)).reshape(-1)]
    else:
        values = [i for i in values if isfinite(i)]

    if len(values) != nb_values:
        warnings.warn('Invalid values encountered (NaN or Inf) were ignored')
        nb_values = len(values)

    if nb_class >= nb_values or nb_class < 2:
        raise ValueError("Number of class have to be an integer "
                         "greater than 2 and "
                         "smaller than the number of values to use")

    return jenks._jenks_breaks(values, nb_class)

Compute jenks natural breaks on a sequence of values given nb_class the number of desired class .


def grab(self, bbox=None):
        """Grabs an image directly to a buffer.

        :param bbox: Optional tuple or list containing (x1, y1, x2, y2) coordinates
            of sub-region to capture.
        :return: PIL RGB image
        :raises: ValueError, if image data does not have 3 channels (RGB), each with 8
            bits.
        :rtype: Image
        """
        w = Gdk.get_default_root_window()
        if bbox is not None:
            g = [bbox[0], bbox[1], bbox[2] - bbox[0], bbox[3] - bbox[1]]
        else:
            g = w.get_geometry()
        pb = Gdk.pixbuf_get_from_window(w, *g)
        if pb.get_bits_per_sample() != 8:
            raise ValueError('Expected 8 bits per pixel.')
        elif pb.get_n_channels() != 3:
            raise ValueError('Expected RGB image.')

        # Read the entire buffer into a python bytes object.
        # read_pixel_bytes: New in version 2.32.
        pixel_bytes = pb.read_pixel_bytes().get_data()  # type: bytes
        width, height = g[2], g[3]

        # Probably for SSE alignment reasons, the pixbuf has extra data in each line.
        # The args after "raw" help handle this; see
        # http://effbot.org/imagingbook/decoder.htm#the-raw-decoder
        return Image.frombytes(
            'RGB', (width, height), pixel_bytes, 'raw', 'RGB', pb.get_rowstride(), 1)


def grab(bbox=None, childprocess=None, backend=None):
    """Copy the contents of the screen to PIL image memory.

    :param bbox: optional bounding box (x1,y1,x2,y2)
    :param childprocess: pyscreenshot can cause an error,
            if it is used on more different virtual displays
            and back-end is not in different process.
            Some back-ends are always different processes: scrot, imagemagick
            The default is False if the program was started inside IDLE,
            otherwise it is True.
    :param backend: back-end can be forced if set (examples:scrot, wx,..),
                    otherwise back-end is automatic
    """
    if childprocess is None:
        childprocess = childprocess_default_value()
    return _grab(
        to_file=False, childprocess=childprocess, backend=backend, bbox=bbox)

def grab ( bbox = None , childprocess = None , backend = None ) : if childprocess is None : childprocess = childprocess_default_value ( ) return _grab ( to_file = False , childprocess = childprocess , backend = backend , bbox = bbox )

Copy the contents of the screen to PIL image memory .


def backend_version(backend, childprocess=None):
    """Back-end version.

    :param backend: back-end (examples:scrot, wx,..)
    :param childprocess: see :py:func:`grab`
    :return: version as string
    """
    if childprocess is None:
        childprocess = childprocess_default_value()
    if not childprocess:
        return _backend_version(backend)
    else:
        return run_in_childprocess(_backend_version, None, backend)

def backend_version ( backend , childprocess = None ) : if childprocess is None : childprocess = childprocess_default_value ( ) if not childprocess : return _backend_version ( backend ) else : return run_in_childprocess ( _backend_version , None , backend )


def open(
    config, mode="continue", zoom=None, bounds=None, single_input_file=None,
    with_cache=False, debug=False
):
    """
    Open a Mapchete process.

    Parameters
    ----------
    config : MapcheteConfig object, config dict or path to mapchete file
        Mapchete process configuration
    mode : string
        * ``memory``: Generate process output on demand without reading
          pre-existing data or writing new data.
        * ``readonly``: Just read data without processing new data.
        * ``continue``: (default) Don't overwrite existing output.
        * ``overwrite``: Overwrite existing output.
    zoom : list or integer
        process zoom level or a pair of minimum and maximum zoom level
    bounds : tuple
        left, bottom, right, top process boundaries in output pyramid
    single_input_file : string
        single input file if supported by process
    with_cache : bool
        process output data cached in memory

    Returns
    -------
    Mapchete
        a Mapchete process object
    """
    return Mapchete(
        MapcheteConfig(
            config, mode=mode, zoom=zoom, bounds=bounds,
            single_input_file=single_input_file, debug=debug),
        with_cache=with_cache)

def open ( config , mode = "continue" , zoom = None , bounds = None , single_input_file = None , with_cache = False , debug = False ) : return Mapchete ( MapcheteConfig ( config , mode = mode , zoom = zoom , bounds = bounds , single_input_file = single_input_file , debug = debug ) , with_cache = with_cache )


def _get_zoom_level(zoom, process):
    """Determine zoom levels."""
    if zoom is None:
        return reversed(process.config.zoom_levels)
    if isinstance(zoom, int):
        return [zoom]
    elif len(zoom) == 2:
        return reversed(range(min(zoom), max(zoom)+1))
    elif len(zoom) == 1:
        return zoom

def _get_zoom_level ( zoom , process ) : if zoom is None : return reversed ( process . config . zoom_levels ) if isinstance ( zoom , int ) : return [ zoom ] elif len ( zoom ) == 2 : return reversed ( range ( min ( zoom ) , max ( zoom ) + 1 ) ) elif len ( zoom ) == 1 : return zoom


def _process_worker(process, process_tile):
    """Worker function running the process."""
    logger.debug((process_tile.id, "running on %s" % current_process().name))

    # skip execution if overwrite is disabled and tile exists
    if (
        process.config.mode == "continue" and
        process.config.output.tiles_exist(process_tile)
    ):
        logger.debug((process_tile.id, "tile exists, skipping"))
        return ProcessInfo(
            tile=process_tile,
            processed=False,
            process_msg="output already exists",
            written=False,
            write_msg="nothing written"
        )

    # execute on process tile
    else:
        with Timer() as t:
            try:
                output = process.execute(process_tile, raise_nodata=True)
            except MapcheteNodataTile:
                output = None
        processor_message = "processed in %s" % t
        logger.debug((process_tile.id, processor_message))
        writer_info = process.write(process_tile, output)
        return ProcessInfo(
            tile=process_tile,
            processed=True,
            process_msg=processor_message,
            written=writer_info.written,
            write_msg=writer_info.write_msg
        )


def get_process_tiles(self, zoom=None):
        """
        Yield process tiles.

        Tiles intersecting with the input data bounding boxes as well as
        process bounds, if provided, are considered process tiles. This is to
        avoid iterating through empty tiles.

        Parameters
        ----------
        zoom : integer
            zoom level process tiles should be returned from; if none is given,
            return all process tiles

        yields
        ------
        BufferedTile objects
        """
        if zoom or zoom == 0:
            for tile in self.config.process_pyramid.tiles_from_geom(
                self.config.area_at_zoom(zoom), zoom
            ):
                yield tile
        else:
            for zoom in reversed(self.config.zoom_levels):
                for tile in self.config.process_pyramid.tiles_from_geom(
                    self.config.area_at_zoom(zoom), zoom
                ):
                    yield tile

def get_process_tiles ( self , zoom = None ) : if zoom or zoom == 0 : for tile in self . config . process_pyramid . tiles_from_geom ( self . config . area_at_zoom ( zoom ) , zoom ) : yield tile else : for zoom in reversed ( self . config . zoom_levels ) : for tile in self . config . process_pyramid . tiles_from_geom ( self . config . area_at_zoom ( zoom ) , zoom ) : yield tile


def batch_process(
        self, zoom=None, tile=None, multi=cpu_count(), max_chunksize=1
    ):
        """
        Process a large batch of tiles.

        Parameters
        ----------
        process : MapcheteProcess
            process to be run
        zoom : list or int
            either single zoom level or list of minimum and maximum zoom level;
            None processes all (default: None)
        tile : tuple
            zoom, row and column of tile to be processed (cannot be used with
            zoom)
        multi : int
            number of workers (default: number of CPU cores)
        max_chunksize : int
            maximum number of process tiles to be queued for each worker;
            (default: 1)
        """
        list(self.batch_processor(zoom, tile, multi, max_chunksize))

def batch_process ( self , zoom = None , tile = None , multi = cpu_count ( ) , max_chunksize = 1 ) : list ( self . batch_processor ( zoom , tile , multi , max_chunksize ) )


def batch_processor(
        self, zoom=None, tile=None, multi=cpu_count(), max_chunksize=1
    ):
        """
        Process a large batch of tiles and yield report messages per tile.

        Parameters
        ----------
        zoom : list or int
            either single zoom level or list of minimum and maximum zoom level;
            None processes all (default: None)
        tile : tuple
            zoom, row and column of tile to be processed (cannot be used with
            zoom)
        multi : int
            number of workers (default: number of CPU cores)
        max_chunksize : int
            maximum number of process tiles to be queued for each worker;
            (default: 1)
        """
        if zoom and tile:
            raise ValueError("use either zoom or tile")

        # run single tile
        if tile:
            yield _run_on_single_tile(self, tile)
        # run concurrently
        elif multi > 1:
            for process_info in _run_with_multiprocessing(
                self, list(_get_zoom_level(zoom, self)), multi, max_chunksize
            ):
                yield process_info
        # run sequentially
        elif multi == 1:
            for process_info in _run_without_multiprocessing(
                self, list(_get_zoom_level(zoom, self))
            ):
                yield process_info

def
batch_processor
(
self
,
zoom
=
,
tile
=
,
multi
=
cpu_count
(
)
,
max_chunksize
=
1
)
:
if
zoom
and
tile
:
raise
ValueError
(
"use either zoom or tile"
)
# run single tile
if
tile
:
yield
_run_on_single_tile
(
self
,
tile
)
# run concurrently
elif
multi
>
1
:
for
process_info
in
_run_with_multiprocessing
(
self
,
list
(
_get_zoom_level
(
zoom
,
self
)
)
,
multi
,
max_chunksize
)
:
yield
process_info
# run sequentially
elif
multi
==
1
:
for
process_info
in
_run_without_multiprocessing
(
self
,
list
(
_get_zoom_level
(
zoom
,
self
)
)
)
:
yield
process_info

Process a large batch of tiles and yield report messages per tile .


def execute(self, process_tile, raise_nodata=False):
        """
        Run the Mapchete process.

        Execute, write and return data.

        Parameters
        ----------
        process_tile : Tile or tile index tuple
            Member of the process tile pyramid (not necessarily the output
            pyramid, if output has a different metatiling setting)

        Returns
        -------
        data : NumPy array or features
            process output
        """
        if self.config.mode not in ["memory", "continue", "overwrite"]:
            raise ValueError("process mode must be memory, continue or overwrite")
        if isinstance(process_tile, tuple):
            process_tile = self.config.process_pyramid.tile(*process_tile)
        elif isinstance(process_tile, BufferedTile):
            pass
        else:
            raise TypeError("process_tile must be tuple or BufferedTile")

        if process_tile.zoom not in self.config.zoom_levels:
            return self.config.output.empty(process_tile)

        return self._execute(process_tile, raise_nodata=raise_nodata)


def read(self, output_tile):
        """
        Read from written process output.

        Parameters
        ----------
        output_tile : BufferedTile or tile index tuple
            Member of the output tile pyramid (not necessarily the process
            pyramid, if output has a different metatiling setting)

        Returns
        -------
        data : NumPy array or features
            process output
        """
        if self.config.mode not in ["readonly", "continue", "overwrite"]:
            raise ValueError("process mode must be readonly, continue or overwrite")
        if isinstance(output_tile, tuple):
            output_tile = self.config.output_pyramid.tile(*output_tile)
        elif isinstance(output_tile, BufferedTile):
            pass
        else:
            raise TypeError("output_tile must be tuple or BufferedTile")

        return self.config.output.read(output_tile)