def capture_stdout():
    """This ``Context Manager`` redirects STDOUT to a ``StringIO`` objects
    which is returned from the ``Context``.  On exit STDOUT is restored.

    Example:

    .. code-block:: python

        with capture_stdout() as capture:
            print('foo')

        # got here? => capture.getvalue() will now have "foo\\n"
    """
    stdout = sys.stdout
    try:
        capture_out = StringIO()
        sys.stdout = capture_out
        yield capture_out
    finally:
        sys.stdout = stdout

def capture_stdout ( ) : stdout = sys . stdout try : capture_out = StringIO ( ) sys . stdout = capture_out yield capture_out finally : sys . stdout = stdout

This Context Manager redirects STDOUT to a StringIO objects which is returned from the Context . On exit STDOUT is restored .


def capture_stderr():
    """This ``Context Manager`` redirects STDERR to a ``StringIO`` objects
    which is returned from the ``Context``.  On exit STDERR is restored.

    Example:

    .. code-block:: python

        with capture_stderr() as capture:
            print('foo')

        # got here? => capture.getvalue() will now have "foo\\n"
    """
    stderr = sys.stderr
    try:
        capture_out = StringIO()
        sys.stderr = capture_out
        yield capture_out
    finally:
        sys.stderr = stderr

def capture_stderr ( ) : stderr = sys . stderr try : capture_out = StringIO ( ) sys . stderr = capture_out yield capture_out finally : sys . stderr = stderr

This Context Manager redirects STDERR to a StringIO objects which is returned from the Context . On exit STDERR is restored .


def urls(cls):
        """Builds the URL configuration for this resource."""
        return urls.patterns('', urls.url(
            r'^{}(?:$|(?P<path>[/:(.].*))'.format(cls.meta.name),
            cls.view,
            name='armet-api-{}'.format(cls.meta.name),
            kwargs={'resource': cls.meta.name}))

def urls ( cls ) : return urls . patterns ( '' , urls . url ( r'^{}(?:$|(?P<path>[/:(.].*))' . format ( cls . meta . name ) , cls . view , name = 'armet-api-{}' . format ( cls . meta . name ) , kwargs = { 'resource' : cls . meta . name } ) )

Builds the URL configuration for this resource .


def dump(obj, fp, startindex=1, separator=DEFAULT, index_separator=DEFAULT):
    '''Dump an object in req format to the fp given.

    :param Mapping obj: The object to serialize.  Must have a keys method.
    :param fp: A writable that can accept all the types given.
    :param separator: The separator between key and value.  Defaults to u'|' or b'|', depending on the types.
    :param index_separator: The separator between key and index.  Defaults to u'_' or b'_', depending on the types.
    '''

    if startindex < 0:
        raise ValueError('startindex must be non-negative, but was {}'.format(startindex))

    try:
        firstkey = next(iter(obj.keys()))
    except StopIteration:
        return

    if isinstance(firstkey, six.text_type):
        converter = six.u
    else:
        converter = six.b

    default_separator = converter('|')
    default_index_separator = converter('_')
    newline = converter('\n')

    if separator is DEFAULT:
        separator = default_separator
    if index_separator is DEFAULT:
        index_separator = default_index_separator

    for key, value in six.iteritems(obj):
        if isinstance(value, (list, tuple, set)):
            for index, item in enumerate(value, start=startindex):
                fp.write(key)
                fp.write(index_separator)
                fp.write(converter(str(index)))
                fp.write(separator)
                fp.write(item)
                fp.write(newline)
        else:
            fp.write(key)
            fp.write(separator)
            fp.write(value)
            fp.write(newline)

Dump an object in req format to the fp given .


def dumps(obj, startindex=1, separator=DEFAULT, index_separator=DEFAULT):
    '''Dump an object in req format to a string.

    :param Mapping obj: The object to serialize.  Must have a keys method.
    :param separator: The separator between key and value.  Defaults to u'|' or b'|', depending on the types.
    :param index_separator: The separator between key and index.  Defaults to u'_' or b'_', depending on the types.
    '''

    try:
        firstkey = next(iter(obj.keys()))
    except StopIteration:
        return str()

    if isinstance(firstkey, six.text_type):
        io = StringIO()
    else:
        io = BytesIO()

    dump(
        obj=obj,
        fp=io,
        startindex=startindex,
        separator=separator,
        index_separator=index_separator,
        )
    return io.getvalue()

def dumps ( obj , startindex = 1 , separator = DEFAULT , index_separator = DEFAULT ) : try : firstkey = next ( iter ( obj . keys ( ) ) ) except StopIteration : return str ( ) if isinstance ( firstkey , six . text_type ) : io = StringIO ( ) else : io = BytesIO ( ) dump ( obj = obj , fp = io , startindex = startindex , separator = separator , index_separator = index_separator , ) return io . getvalue ( )

Dump an object in req format to a string .


def load(fp, separator=DEFAULT, index_separator=DEFAULT, cls=dict, list_cls=list):
    '''Load an object from the file pointer.

    :param fp: A readable filehandle.
    :param separator: The separator between key and value.  Defaults to u'|' or b'|', depending on the types.
    :param index_separator: The separator between key and index.  Defaults to u'_' or b'_', depending on the types.
    :param cls: A callable that returns a Mapping that is filled with pairs.  The most common alternate option would be OrderedDict.
    :param list_cls: A callable that takes an iterable and returns a sequence.
    '''

    converter = None

    output = cls()
    arraykeys = set()

    for line in fp:
        if converter is None:
            if isinstance(line, six.text_type):
                converter = six.u
            else:
                converter = six.b
            default_separator = converter('|')
            default_index_separator = converter('_')
            newline = converter('\n')

            if separator is DEFAULT:
                separator = default_separator
            if index_separator is DEFAULT:
                index_separator = default_index_separator

        key, value = line.strip().split(separator, 1)

        keyparts = key.split(index_separator)

        try:
            index = int(keyparts[-1])
            endwithint = True
        except ValueError:
            endwithint = False

        # We do everything in-place to ensure that we maintain order when using
        # an OrderedDict.
        if len(keyparts) > 1 and endwithint:
            # If this is an array key
            basekey = key.rsplit(index_separator, 1)[0]
            if basekey not in arraykeys:
                arraykeys.add(basekey)

            if basekey in output:
                # If key already exists as non-array, fix it
                if not isinstance(output[basekey], dict):
                    output[basekey] = {-1: output[basekey]}
            else:
                output[basekey] = {}

            output[basekey][index] = value

        else:
            if key in output and isinstance(output[key], dict):
                output[key][-1] = value
            else:
                output[key] = value

    # Convert array keys
    for key in arraykeys:
        output[key] = list_cls(pair[1] for pair in sorted(six.iteritems(output[key])))

    return output


def loads(s, separator=DEFAULT, index_separator=DEFAULT, cls=dict, list_cls=list):
    '''Loads an object from a string.

    :param s: An object to parse
    :type s: bytes or str
    :param separator: The separator between key and value.  Defaults to u'|' or b'|', depending on the types.
    :param index_separator: The separator between key and index.  Defaults to u'_' or b'_', depending on the types.
    :param cls: A callable that returns a Mapping that is filled with pairs.  The most common alternate option would be OrderedDict.
    :param list_cls: A callable that takes an iterable and returns a sequence.
    '''

    if isinstance(s, six.text_type):
        io = StringIO(s)
    else:
        io = BytesIO(s)

    return load(
        fp=io,
        separator=separator,
        index_separator=index_separator,
        cls=cls,
        list_cls=list_cls,
        )

def loads ( s , separator = DEFAULT , index_separator = DEFAULT , cls = dict , list_cls = list ) : if isinstance ( s , six . text_type ) : io = StringIO ( s ) else : io = BytesIO ( s ) return load ( fp = io , separator = separator , index_separator = index_separator , cls = cls , list_cls = list_cls , )


def reverse(self):
        """ Reverse all bumpers """
        if not self.test_drive and self.bumps:
            map(lambda b: b.reverse(), self.bumpers)

def reverse ( self ) : if not self . test_drive and self . bumps : map ( lambda b : b . reverse ( ) , self . bumpers )


def _expand_targets(self, targets, base_dir=None):
        """ Expand targets by looking for '-r' in targets. """
        all_targets = []

        for target in targets:
            target_dirs = [p for p in [base_dir, os.path.dirname(target)] if p]
            target_dir = target_dirs and os.path.join(*target_dirs) or ''
            target = os.path.basename(target)
            target_path = os.path.join(target_dir, target)

            if os.path.exists(target_path):
                all_targets.append(target_path)

                with open(target_path) as fp:
                    for line in fp:
                        if line.startswith('-r '):
                            _, new_target = line.split(' ', 1)
                            all_targets.extend(self._expand_targets([new_target.strip()], base_dir=target_dir))

        return all_targets

def
_expand_targets
(
self
,
targets
,
base_dir
=
)
:
all_targets
=
[
]
for
target
in
targets
:
target_dirs
=
[
p
for
p
in
[
base_dir
,
os
.
path
.
dirname
(
target
)
]
if
p
]
target_dir
=
target_dirs
and
os
.
path
.
join
(
*
target_dirs
)
or
''
target
=
os
.
path
.
basename
(
target
)
target_path
=
os
.
path
.
join
(
target_dir
,
target
)
if
os
.
path
.
exists
(
target_path
)
:
all_targets
.
append
(
target_path
)
with
open
(
target_path
)
as
fp
:
for
line
in
fp
:
if
line
.
startswith
(
'-r '
)
:
_
,
new_target
=
line
.
split
(
' '
,
1
)
all_targets
.
extend
(
self
.
_expand_targets
(
[
new_target
.
strip
(
)
]
,
base_dir
=
target_dir
)
)
return
all_targets

Expand targets by looking for - r in targets .


def get_nginx_config(self):
        """
        Gets the Nginx config for the project

        """
        if os.path.exists(self._nginx_config):
            return open(self._nginx_config, 'r').read()
        else:
            return None

def get_nginx_config ( self ) : if os . path . exists ( self . _nginx_config ) : return open ( self . _nginx_config , 'r' ) . read ( ) else : return None


def check_directories(self):
        """
        Creates base directories for app, virtualenv, and nginx

        """
        self.log.debug('Checking directories')
        if not os.path.exists(self._ve_dir):
            os.makedirs(self._ve_dir)
        if not os.path.exists(self._app_dir):
            os.makedirs(self._app_dir)
        if not os.path.exists(self._conf_dir):
            os.makedirs(self._conf_dir)
        if not os.path.exists(self._var_dir):
            os.makedirs(self._var_dir)
        if not os.path.exists(self._log_dir):
            os.makedirs(self._log_dir)
        if not os.path.exists(self._script_dir):
            os.makedirs(self._script_dir)

        # copy uswgi_params for nginx
        uwsgi_params = '/etc/nginx/uwsgi_params'
        if os.path.exists(uwsgi_params):
            shutil.copy(uwsgi_params, self._conf_dir)
        else:
            logging.warning('Unable to find Nginx uwsgi_params.  You must manually copy this to {0}.'.format(self._conf_dir))

        # copy mime.types for nginx
        mime_types = '/etc/nginx/mime.types'
        if os.path.exists(mime_types):
            shutil.copy(mime_types, self._conf_dir)
            self._include_mimetypes = True
        else:
            logging.warn('Unable to find mime.types for Nginx.  You must manually copy this to {0}.'.format(self._conf_dir))

Creates base directories for app virtualenv and nginx


def create_virtualenv(self):
        """
        Creates the virtualenv for the project

        """
        if check_command('virtualenv'):
            ve_dir = os.path.join(self._ve_dir, self._project_name)
            if os.path.exists(ve_dir):
                if self._force:
                    logging.warn('Removing existing virtualenv')
                    shutil.rmtree(ve_dir)
                else:
                    logging.warn('Found existing virtualenv; not creating (use --force to overwrite)')
                    return
            logging.info('Creating virtualenv')
            p = subprocess.Popen('virtualenv --no-site-packages {0} > /dev/null'.format(ve_dir), shell=True)
            os.waitpid(p.pid, 0)
            # install modules
            for m in self._modules:
                self.log.info('Installing module {0}'.format(m))
                p = subprocess.Popen('{0} install {1} > /dev/null'.format(os.path.join(self._ve_dir, \
                self._project_name) + os.sep + 'bin' + os.sep + 'pip', m), shell=True)
                os.waitpid(p.pid, 0)


def create_nginx_config(self):
        """
        Creates the Nginx configuration for the project

        """
        cfg = '# nginx config for {0}\n'.format(self._project_name)
        if not self._shared_hosting:
            # user
            if self._user:
                cfg += 'user {0};\n'.format(self._user)
            # misc nginx config
            cfg += 'worker_processes 1;\nerror_log {0}-errors.log;\n\
pid {1}_    nginx.pid;\n\n'.format(os.path.join(self._log_dir, \
                self._project_name), os.path.join(self._var_dir, self._project_name))
            cfg += 'events {\n\tworker_connections 32;\n}\n\n'
            # http section
            cfg += 'http {\n'
            if self._include_mimetypes:
                cfg += '\tinclude mime.types;\n'
            cfg += '\tdefault_type application/octet-stream;\n'
            cfg += '\tclient_max_body_size 1G;\n'
            cfg += '\tproxy_max_temp_file_size 0;\n'
            cfg += '\tproxy_buffering off;\n'
            cfg += '\taccess_log {0}-access.log;\n'.format(os.path.join \
                (self._log_dir, self._project_name))
            cfg += '\tsendfile on;\n'
            cfg += '\tkeepalive_timeout 65;\n'
            # server section
        cfg += '\tserver {\n'
        cfg += '\t\tlisten 0.0.0.0:{0};\n'.format(self._port)
        if self._server_name:
            cfg += '\t\tserver_name {0};\n'.format(self._server_name)
        # location section
        cfg += '\t\tlocation / {\n'
        cfg += '\t\t\tuwsgi_pass unix:///{0}.sock;\n'.format(\
            os.path.join(self._var_dir, self._project_name))
        cfg += '\t\t\tinclude uwsgi_params;\n'
        cfg += '\t\t}\n\n'
        # end location
        # error page templates
        cfg += '\t\terror_page 500 502 503 504 /50x.html;\n'
        cfg += '\t\tlocation = /50x.html {\n'
        cfg += '\t\t\troot html;\n'
        # end error page section
        cfg += '\t\t}\n'
        # end server section
        cfg += '\t}\n'
        if not self._shared_hosting:
            # end http section
            cfg += '}\n'

        # create conf
        f = open(self._nginx_config, 'w')
        f.write(cfg)
        f.close()


def create_manage_scripts(self):
        """
        Creates scripts to start and stop the application

        """
        # create start script
        start = '# start script for {0}\n\n'.format(self._project_name)
        # start uwsgi
        start += 'echo \'Starting uWSGI...\'\n'
        start += 'sh {0}.uwsgi\n'.format(os.path.join(self._conf_dir, self._project_name))
        start += 'sleep 1\n'
        # start nginx
        start += 'echo \'Starting Nginx...\'\n'
        start += 'nginx -c {0}_nginx.conf\n'.format(os.path.join(self._conf_dir, self._project_name))
        start += 'sleep 1\n'
        start += 'echo \'{0} started\'\n\n'.format(self._project_name)

        # stop script
        stop = '# stop script for {0}\n\n'.format(self._project_name)
        # stop nginx
        stop += 'if [ -e {0}_nginx.pid ]; then nginx -c {1}_nginx.conf -s stop ; fi\n'.format(os.path.join(self._var_dir, self._project_name), os.path.join(self._conf_dir, self._project_name))
        # stop uwsgi
        stop += 'if [ -e {0}_uwsgi.pid ]; then kill -9 `cat {0}_uwsgi.pid` ; rm {0}_uwsgi.pid 2>&1 > /dev/null ; fi\n'.format(os.path.join(self._var_dir, self._project_name))
        stop += 'echo \'{0} stopped\'\n'.format(self._project_name)

        # write scripts
        start_file = '{0}_start.sh'.format(os.path.join(self._script_dir, self._project_name))
        stop_file = '{0}_stop.sh'.format(os.path.join(self._script_dir, self._project_name))
        f = open(start_file, 'w')
        f.write(start)
        f.close()
        f = open(stop_file, 'w')
        f.write(stop)
        f.close()
        # make executable
        os.chmod(start_file, 0754)
        os.chmod(stop_file, 0754)


def create(self):
        """
        Creates the full project

        """
        # create virtualenv
        self.create_virtualenv()
        # create project
        self.create_project()
        # generate uwsgi script
        self.create_uwsgi_script()
        # generate nginx config
        self.create_nginx_config()
        # generate management scripts
        self.create_manage_scripts()
        logging.info('** Make sure to set proper permissions for the webserver user account on the var and log directories in the project root')


def dasherize(value):
    """Dasherizes the passed value."""
    value = value.strip()
    value = re.sub(r'([A-Z])', r'-\1', value)
    value = re.sub(r'[-_\s]+', r'-', value)
    value = re.sub(r'^-', r'', value)
    value = value.lower()
    return value

def dasherize ( value ) : value = value . strip ( ) value = re . sub ( r'([A-Z])' , r'-\1' , value ) value = re . sub ( r'[-_\s]+' , r'-' , value ) value = re . sub ( r'^-' , r'' , value ) value = value . lower ( ) return value


def redirect(cls, request, response):
        """Redirect to the canonical URI for this resource."""
        if cls.meta.legacy_redirect:
            if request.method in ('GET', 'HEAD',):
                # A SAFE request is allowed to redirect using a 301
                response.status = http.client.MOVED_PERMANENTLY

            else:
                # All other requests must use a 307
                response.status = http.client.TEMPORARY_REDIRECT

        else:
            # Modern redirects are allowed. Let's have some fun.
            # Hopefully you're client supports this.
            # The RFC explicitly discourages UserAgent sniffing.
            response.status = http.client.PERMANENT_REDIRECT

        # Terminate the connection.
        response.close()

Redirect to the canonical URI for this resource .


def parse(cls, path):
        """Parses out parameters and separates them out of the path.

        This uses one of the many defined patterns on the options class. But,
        it defaults to a no-op if there are no defined patterns.
        """
        # Iterate through the available patterns.
        for resource, pattern in cls.meta.patterns:
            # Attempt to match the path.
            match = re.match(pattern, path)
            if match is not None:
                # Found something.
                return resource, match.groupdict(), match.string[match.end():]

        # No patterns at all; return unsuccessful.
        return None if not cls.meta.patterns else False

Parses out parameters and separates them out of the path .


def traverse(cls, request, params=None):
        """Traverses down the path and determines the accessed resource.

        This makes use of the patterns array to implement simple traversal.
        This defaults to a no-op if there are no defined patterns.
        """
        # Attempt to parse the path using a pattern.
        result = cls.parse(request.path)
        if result is None:
            # No parsing was requested; no-op.
            return cls, {}

        elif not result:
            # Parsing failed; raise 404.
            raise http.exceptions.NotFound()

        # Partition out the result.
        resource, data, rest = result

        if params:
            # Append params to data.
            data.update(params)

        if resource is None:
            # No traversal; return parameters.
            return cls, data

        # Modify the path appropriately.
        if data.get('path') is not None:
            request.path = data.pop('path')

        elif rest is not None:
            request.path = rest

        # Send us through traversal again.
        result = resource.traverse(request, params=data)
        return result

Traverses down the path and determines the accessed resource .


def stream(cls, response, sequence):
        """
        Helper method used in conjunction with the view handler to
        stream responses to the client.
        """
        # Construct the iterator and run the sequence once in order
        # to capture any headers and status codes set.
        iterator = iter(sequence)
        data = {'chunk': next(iterator)}
        response.streaming = True

        def streamer():
            # Iterate through the iterator and yield its content
            while True:
                if response.asynchronous:
                    # Yield our current chunk.
                    yield data['chunk']

                else:
                    # Write the chunk to the response
                    response.send(data['chunk'])

                    # Yield its body
                    yield response.body

                    # Unset the body.
                    response.body = None

                try:
                    # Get the next chunk.
                    data['chunk'] = next(iterator)

                except StopIteration:
                    # Get out of the loop.
                    break

            if not response.asynchronous:
                # Close the response.
                response.close()

        # Return the streaming function.
        return streamer()

Helper method used in conjunction with the view handler to stream responses to the client .


def deserialize(self, request=None, text=None, format=None):
        """Deserializes the text using a determined deserializer.

        @param[in] request
            The request object to pull information from; normally used to
            determine the deserialization format (when `format` is
            not provided).

        @param[in] text
            The text to be deserialized. Can be left blank and the
            request will be read.

        @param[in] format
            A specific format to deserialize in; if provided, no detection is
            done. If not provided, the content-type header is looked at to
            determine an appropriate deserializer.

        @returns
            A tuple of the deserialized data and an instance of the
            deserializer used.
        """
        if isinstance(self, Resource):
            if not request:
                # Ensure we have a response object.
                request = self._request

        Deserializer = None
        if format:
            # An explicit format was given; do not attempt to auto-detect
            # a deserializer.
            Deserializer = self.meta.deserializers[format]

        if not Deserializer:
            # Determine an appropriate deserializer to use by
            # introspecting the request object and looking at
            # the `Content-Type` header.
            media_ranges = request.get('Content-Type')
            if media_ranges:
                # Parse the media ranges and determine the deserializer
                # that is the closest match.
                media_types = six.iterkeys(self._deserializer_map)
                media_type = mimeparse.best_match(media_types, media_ranges)
                if media_type:
                    format = self._deserializer_map[media_type]
                    Deserializer = self.meta.deserializers[format]

            else:
                # Client didn't provide a content-type; we're supposed
                # to auto-detect.
                # TODO: Implement this.
                pass

        if Deserializer:
            try:
                # Attempt to deserialize the data using the determined
                # deserializer.
                deserializer = Deserializer()
                data = deserializer.deserialize(request=request, text=text)
                return data, deserializer

            except ValueError:
                # Failed to deserialize the data.
                pass

        # Failed to determine a deserializer; or failed to deserialize.
        raise http.exceptions.UnsupportedMediaType()

Deserializes the text using a determined deserializer .


def serialize(self, data, response=None, request=None, format=None):
        """Serializes the data using a determined serializer.

        @param[in] data
            The data to be serialized.

        @param[in] response
            The response object to serialize the data to.
            If this method is invoked as an instance method, the response
            object can be omitted and it will be taken from the instance.

        @param[in] request
            The request object to pull information from; normally used to
            determine the serialization format (when `format` is not provided).
            May be used by some serializers as well to pull additional headers.
            If this method is invoked as an instance method, the request
            object can be omitted and it will be taken from the instance.

        @param[in] format
            A specific format to serialize in; if provided, no detection is
            done. If not provided, the accept header (as well as the URL
            extension) is looked at to determine an appropriate serializer.

        @returns
            A tuple of the serialized text and an instance of the
            serializer used.
        """
        if isinstance(self, Resource):
            if not request:
                # Ensure we have a response object.
                request = self._request

        Serializer = None
        if format:
            # An explicit format was given; do not attempt to auto-detect
            # a serializer.
            Serializer = self.meta.serializers[format]

        if not Serializer:
            # Determine an appropriate serializer to use by
            # introspecting the request object and looking at the `Accept`
            # header.
            media_ranges = (request.get('Accept') or '*/*').strip()
            if not media_ranges:
                # Default the media ranges to */*
                media_ranges = '*/*'

            if media_ranges != '*/*':
                # Parse the media ranges and determine the serializer
                # that is the closest match.
                media_types = six.iterkeys(self._serializer_map)
                media_type = mimeparse.best_match(media_types, media_ranges)
                if media_type:
                    format = self._serializer_map[media_type]
                    Serializer = self.meta.serializers[format]

            else:
                # Client indicated no preference; use the default.
                default = self.meta.default_serializer
                Serializer = self.meta.serializers[default]

        if Serializer:
            try:
                # Attempt to serialize the data using the determined
                # serializer.
                serializer = Serializer(request, response)
                return serializer.serialize(data), serializer

            except ValueError:
                # Failed to serialize the data.
                pass

        # Either failed to determine a serializer or failed to serialize
        # the data; construct a list of available and valid encoders.
        available = {}
        for name in self.meta.allowed_serializers:
            Serializer = self.meta.serializers[name]
            instance = Serializer(request, None)
            if instance.can_serialize(data):
                available[name] = Serializer.media_types[0]

        # Raise a Not Acceptable exception.
        raise http.exceptions.NotAcceptable(available)

Serializes the data using a determined serializer .


def dispatch(self, request, response):
        """Entry-point of the dispatch cycle for this resource.

        Performs common work such as authentication, decoding, etc. before
        handing complete control of the result to a function with the
        same name as the request method.
        """
        # Assert authentication and attempt to get a valid user object.
        self.require_authentication(request)

        # Assert accessibiltiy of the resource in question.
        self.require_accessibility(request.user, request.method)

        # Facilitate CORS by applying various headers.
        # This must be done on every request.
        # TODO: Provide cross_domain configuration that turns this off.
        self._process_cross_domain_request(request, response)

        # Route the HTTP/1.1 request to an appropriate method.
        return self.route(request, response)

Entry - point of the dispatch cycle for this resource .


def require_authentication(self, request):
        """Ensure we are authenticated."""
        request.user = user = None

        if request.method == 'OPTIONS':
            # Authentication should not be checked on an OPTIONS request.
            return

        for auth in self.meta.authentication:
            user = auth.authenticate(request)
            if user is False:
                # Authentication protocol failed to authenticate;
                # pass the baton.
                continue

            if user is None and not auth.allow_anonymous:
                # Authentication protocol determined the user is
                # unauthenticated.
                auth.unauthenticated()

            # Authentication protocol determined the user is indeed
            # authenticated (or not); Store the user for later reference.
            request.user = user
            return

        if not user and not auth.allow_anonymous:
            # No authenticated user found and protocol doesn't allow
            # anonymous users.
            auth.unauthenticated()


def require_accessibility(self, user, method):
        """Ensure we are allowed to access this resource."""
        if method == 'OPTIONS':
            # Authorization should not be checked on an OPTIONS request.
            return

        authz = self.meta.authorization
        if not authz.is_accessible(user, method, self):
            # User is not authorized; raise an appropriate message.
            authz.unaccessible()

Ensure we are allowed to access this resource .


def require_http_allowed_method(cls, request):
        """Ensure that we're allowed to use this HTTP method."""
        allowed = cls.meta.http_allowed_methods
        if request.method not in allowed:
            # The specified method is not allowed for the resource
            # identified by the request URI.
            # RFC 2616 § 10.4.6 — 405 Method Not Allowed
            raise http.exceptions.MethodNotAllowed(allowed)

Ensure that we re allowed to use this HTTP method .


def route(self, request, response):
        """Processes every request.

        Directs control flow to the appropriate HTTP/1.1 method.
        """
        # Ensure that we're allowed to use this HTTP method.
        self.require_http_allowed_method(request)

        # Retrieve the function corresponding to this HTTP method.
        function = getattr(self, request.method.lower(), None)
        if function is None:
            # Server is not capable of supporting it.
            raise http.exceptions.NotImplemented()

        # Delegate to the determined function to process the request.
        return function(request, response)


def options(self, request, response):
        """Process an `OPTIONS` request.

        Used to initiate a cross-origin request. All handling specific to
        CORS requests is done on every request however this method also
        returns a list of available methods.
        """
        # Gather a list available HTTP/1.1 methods for this URI.
        response['Allowed'] = ', '.join(self.meta.http_allowed_methods)

        # All CORS handling is done for every HTTP/1.1 method.
        # No more handling is neccesary; set the response to 200 and return.
        response.status = http.client.OK


def resource(**kwargs):
    """Wraps the decorated function in a lightweight resource."""
    def inner(function):
        name = kwargs.pop('name', None)
        if name is None:
            name = utils.dasherize(function.__name__)

        methods = kwargs.pop('methods', None)
        if isinstance(methods, six.string_types):
            # Tuple-ify the method if we got just a string.
            methods = methods,

        # Construct a handler.
        handler = (function, methods)

        if name not in _resources:
            # Initiate the handlers list.
            _handlers[name] = []

            # Construct a light-weight resource using the passed kwargs
            # as the arguments for the meta.
            from armet import resources
            kwargs['name'] = name

            class LightweightResource(resources.Resource):
                Meta = type(str('Meta'), (), kwargs)

                def route(self, request, response):
                    for handler, methods in _handlers[name]:
                        if methods is None or request.method in methods:
                            return handler(request, response)

                    resources.Resource.route(self)

            # Construct and add this resource.
            _resources[name] = LightweightResource

        # Add this to the handlers.
        _handlers[name].append(handler)

        # Return the resource.
        return _resources[name]

    # Return the inner method.
    return inner

Wraps the decorated function in a lightweight resource .


def render_to_string(self):
        """Render to cookie strings.
        """
        values = ''
        for key, value in self.items():
            values += '{}={};'.format(key, value)
        return values

def render_to_string ( self ) : values = '' for key , value in self . items ( ) : values += '{}={};' . format ( key , value ) return values


def from_cookie_string(self, cookie_string):
        """update self with cookie_string.
        """
        for key_value in cookie_string.split(';'):
            if '=' in key_value:
                key, value = key_value.split('=', 1)
            else:
                key = key_value
            strip_key = key.strip()
            if strip_key and strip_key.lower() not in COOKIE_ATTRIBUTE_NAMES:
                self[strip_key] = value.strip()

def from_cookie_string ( self , cookie_string ) : for key_value in cookie_string . split ( ';' ) : if '=' in key_value : key , value = key_value . split ( '=' , 1 ) else : key = key_value strip_key = key . strip ( ) if strip_key and strip_key . lower ( ) not in COOKIE_ATTRIBUTE_NAMES : self [ strip_key ] = value . strip ( )


def _add_method(self, effect, verb, resource, conditions):
        """
        Adds a method to the internal lists of allowed or denied methods.
        Each object in the internal list contains a resource ARN and a
        condition statement. The condition statement can be null.
        """
        if verb != '*' and not hasattr(HttpVerb, verb):
            raise NameError('Invalid HTTP verb ' + verb +
                            '. Allowed verbs in HttpVerb class')
        resource_pattern = re.compile(self.path_regex)
        if not resource_pattern.match(resource):
            raise NameError('Invalid resource path: ' + resource +
                            '. Path should match ' + self.path_regex)

        if resource[:1] == '/':
            resource = resource[1:]

        resource_arn = ('arn:aws:execute-api:' +
                        self.region + ':' +
                        self.aws_account_id + ':' +
                        self.rest_api_id + '/' +
                        self.stage + '/' +
                        verb + '/' +
                        resource)

        if effect.lower() == 'allow':
            self.allowMethods.append({
                'resource_arn': resource_arn,
                'conditions': conditions
            })
        elif effect.lower() == 'deny':
            self.denyMethods.append({
                'resource_arn': resource_arn,
                'conditions': conditions
            })

def
_add_method
(
self
,
effect
,
verb
,
resource
,
conditions
)
:
if
verb
!=
'*'
and
not
hasattr
(
HttpVerb
,
verb
)
:
raise
NameError
(
'Invalid HTTP verb '
+
verb
+
'. Allowed verbs in HttpVerb class'
)
resource_pattern
=
re
.
compile
(
self
.
path_regex
)
if
not
resource_pattern
.
match
(
resource
)
:
raise
NameError
(
'Invalid resource path: '
+
resource
+
'. Path should match '
+
self
.
path_regex
)
if
resource
[
:
1
]
==
'/'
:
resource
=
resource
[
1
:
]
resource_arn
=
(
'arn:aws:execute-api:'
+
self
.
region
+
':'
+
self
.
aws_account_id
+
':'
+
self
.
rest_api_id
+
'/'
+
self
.
stage
+
'/'
+
verb
+
'/'
+
resource
)
if
effect
.
lower
(
)
==
'allow'
:
self
.
allowMethods
.
append
(
{
'resource_arn'
:
resource_arn
,
'conditions'
:
conditions
}
)
elif
effect
.
lower
(
)
==
'deny'
:
self
.
denyMethods
.
append
(
{
'resource_arn'
:
resource_arn
,
'conditions'
:
conditions
}
)

Adds a method to the internal lists of allowed or denied methods . Each object in the internal list contains a resource ARN and a condition statement . The condition statement can be null .


def _get_effect_statement(self, effect, methods):
        """
        This function loops over an array of objects containing
        a resourceArn and conditions statement and generates
        the array of statements for the policy.
        """
        statements = []

        if len(methods) > 0:
            statement = self._get_empty_statement(effect)

            for method in methods:
                if (method['conditions'] is None or
                        len(method['conditions']) == 0):
                    statement['Resource'].append(method['resource_arn'])
                else:
                    cond_statement = self._get_empty_statement(effect)
                    cond_statement['Resource'].append(method['resource_arn'])
                    cond_statement['Condition'] = method['conditions']
                    statements.append(cond_statement)
            statements.append(statement)

        return statements

def _get_effect_statement ( self , effect , methods ) : statements = [ ] if len ( methods ) > 0 : statement = self . _get_empty_statement ( effect ) for method in methods : if ( method [ 'conditions' ] is None or len ( method [ 'conditions' ] ) == 0 ) : statement [ 'Resource' ] . append ( method [ 'resource_arn' ] ) else : cond_statement = self . _get_empty_statement ( effect ) cond_statement [ 'Resource' ] . append ( method [ 'resource_arn' ] ) cond_statement [ 'Condition' ] = method [ 'conditions' ] statements . append ( cond_statement ) statements . append ( statement ) return statements

This function loops over an array of objects containing a resourceArn and conditions statement and generates the array of statements for the policy .


def deref(self, data):
        """AWS doesn't quite have Swagger 2.0 validation right and will fail
        on some refs. So, we need to convert to deref before
        upload."""

        # We have to make a deepcopy here to create a proper JSON
        # compatible object, otherwise `json.dumps` fails when it
        # hits jsonref.JsonRef objects.
        deref = copy.deepcopy(jsonref.JsonRef.replace_refs(data))

        # Write out JSON version because we might want this.
        self.write_template(deref, filename='swagger.json')

        return deref

AWS doesn t quite have Swagger 2 . 0 validation right and will fail on some refs . So we need to convert to deref before upload .


def check_pre_requirements(pre_requirements):
    """Check all necessary system requirements to exist.

    :param pre_requirements:
        Sequence of pre-requirements to check by running
        ``where <pre_requirement>`` on Windows and ``which ...`` elsewhere.
    """
    pre_requirements = set(pre_requirements or [])
    pre_requirements.add('virtualenv')

    for requirement in pre_requirements:
        if not which(requirement):
            print_error('Requirement {0!r} is not found in system'.
                        format(requirement))
            return False

    return True

Check all necessary system requirements to exist .


def config_to_args(config):
    """Convert config dict to arguments list.

    :param config: Configuration dict.
    """
    result = []

    for key, value in iteritems(config):
        if value is False:
            continue

        key = '--{0}'.format(key.replace('_', '-'))

        if isinstance(value, (list, set, tuple)):
            for item in value:
                result.extend((key, smart_str(item)))
        elif value is not True:
            result.extend((key, smart_str(value)))
        else:
            result.append(key)

    return tuple(result)

def config_to_args ( config ) : result = [ ] for key , value in iteritems ( config ) : if value is False : continue key = '--{0}' . format ( key . replace ( '_' , '-' ) ) if isinstance ( value , ( list , set , tuple ) ) : for item in value : result . extend ( ( key , smart_str ( item ) ) ) elif value is not True : result . extend ( ( key , smart_str ( value ) ) ) else : result . append ( key ) return tuple ( result )


def create_env(env, args, recreate=False, ignore_activated=False, quiet=False):
    """Create virtual environment.

    :param env: Virtual environment name.
    :param args: Pass given arguments to ``virtualenv`` script.
    :param recerate: Recreate virtual environment? By default: False
    :param ignore_activated:
        Ignore already activated virtual environment and create new one. By
        default: False
    :param quiet: Do not output messages into terminal. By default: False
    """
    cmd = None
    result = True

    inside_env = hasattr(sys, 'real_prefix') or os.environ.get('VIRTUAL_ENV')
    env_exists = os.path.isdir(env)

    if not quiet:
        print_message('== Step 1. Create virtual environment ==')

    if (
        recreate or (not inside_env and not env_exists)
    ) or (
        ignore_activated and not env_exists
    ):
        cmd = ('virtualenv', ) + args + (env, )

    if not cmd and not quiet:
        if inside_env:
            message = 'Working inside of virtual environment, done...'
        else:
            message = 'Virtual environment {0!r} already created, done...'
        print_message(message.format(env))

    if cmd:
        with disable_error_handler():
            result = not run_cmd(cmd, echo=not quiet)

    if not quiet:
        print_message()

    return result


def error_handler(func):
    """Decorator to error handling."""
    @wraps(func)
    def wrapper(*args, **kwargs):
        """
        Run actual function and if exception catched and error handler enabled
        put traceback to log file
        """
        try:
            return func(*args, **kwargs)
        except BaseException as err:
            # Do not catch exceptions on testing
            if BOOTSTRAPPER_TEST_KEY in os.environ:
                raise
            # Fail silently if error handling disabled
            if ERROR_HANDLER_DISABLED:
                return True
            # Otherwise save traceback to log
            return save_traceback(err)
    return wrapper


def install(env, requirements, args, ignore_activated=False,
            install_dev_requirements=False, quiet=False):
    """Install library or project into virtual environment.

    :param env: Use given virtual environment name.
    :param requirements: Use given requirements file for pip.
    :param args: Pass given arguments to pip script.
    :param ignore_activated:
        Do not run pip inside already activated virtual environment. By
        default: False
    :param install_dev_requirements:
        When enabled install prefixed or suffixed dev requirements after
        original installation process completed. By default: False
    :param quiet: Do not output message to terminal. By default: False
    """
    if os.path.isfile(requirements):
        args += ('-r', requirements)
        label = 'project'
    else:
        args += ('-U', '-e', '.')
        label = 'library'

    # Attempt to install development requirements
    if install_dev_requirements:
        dev_requirements = None
        dirname = os.path.dirname(requirements)
        basename, ext = os.path.splitext(os.path.basename(requirements))

        # Possible dev requirements files:
        #
        # * <requirements>-dev.<ext>
        # * dev-<requirements>.<ext>
        # * <requirements>_dev.<ext>
        # * dev_<requirements>.<ext>
        # * <requirements>dev.<ext>
        # * dev<requirements>.<ext>
        #
        # Where <requirements> is basename of given requirements file to use
        # and <ext> is its extension.
        for delimiter in ('-', '_', ''):
            filename = os.path.join(
                dirname, ''.join((basename, delimiter, 'dev', ext))
            )
            if os.path.isfile(filename):
                dev_requirements = filename
                break

            filename = os.path.join(
                dirname, ''.join(('dev', delimiter, basename, ext))
            )
            if os.path.isfile(filename):
                dev_requirements = filename
                break

        # If at least one dev requirements file found, install dev requirements
        if dev_requirements:
            args += ('-r', dev_requirements)

    if not quiet:
        print_message('== Step 2. Install {0} =='.format(label))

    result = not pip_cmd(env,
                         ('install', ) + args,
                         ignore_activated,
                         echo=not quiet)

    if not quiet:
        print_message()

    return result

Install library or project into virtual environment .


def iteritems(data, **kwargs):
    """Iterate over dict items."""
    return iter(data.items(**kwargs)) if IS_PY3 else data.iteritems(**kwargs)

def iteritems ( data , * * kwargs ) : return iter ( data . items ( * * kwargs ) ) if IS_PY3 else data . iteritems ( * * kwargs )


def iterkeys(data, **kwargs):
    """Iterate over dict keys."""
    return iter(data.keys(**kwargs)) if IS_PY3 else data.iterkeys(**kwargs)

def iterkeys ( data , * * kwargs ) : return iter ( data . keys ( * * kwargs ) ) if IS_PY3 else data . iterkeys ( * * kwargs )


def main(*args):
    r"""Bootstrap Python projects and libraries with virtualenv and pip.

    Also check system requirements before bootstrap and run post bootstrap
    hook if any.

    :param \*args: Command line arguments list.
    """
    # Create parser, read arguments from direct input or command line
    with disable_error_handler():
        args = parse_args(args or sys.argv[1:])

    # Read current config from file and command line arguments
    config = read_config(args.config, args)
    if config is None:
        return True
    bootstrap = config[__script__]

    # Check pre-requirements
    if not check_pre_requirements(bootstrap['pre_requirements']):
        return True

    # Create virtual environment
    env_args = prepare_args(config['virtualenv'], bootstrap)
    if not create_env(
        bootstrap['env'],
        env_args,
        bootstrap['recreate'],
        bootstrap['ignore_activated'],
        bootstrap['quiet']
    ):
        # Exit if couldn't create virtual environment
        return True

    # And install library or project here
    pip_args = prepare_args(config['pip'], bootstrap)
    if not install(
        bootstrap['env'],
        bootstrap['requirements'],
        pip_args,
        bootstrap['ignore_activated'],
        bootstrap['install_dev_requirements'],
        bootstrap['quiet']
    ):
        # Exist if couldn't install requirements into venv
        return True

    # Run post-bootstrap hook
    run_hook(bootstrap['hook'], bootstrap, bootstrap['quiet'])

    # All OK!
    if not bootstrap['quiet']:
        print_message('All OK!')

    # False means everything went alright, exit code: 0
    return False

r Bootstrap Python projects and libraries with virtualenv and pip .


def parse_args(args):
    """
    Parse args from command line by creating argument parser instance and
    process it.

    :param args: Command line arguments list.
    """
    from argparse import ArgumentParser

    description = ('Bootstrap Python projects and libraries with virtualenv '
                   'and pip.')
    parser = ArgumentParser(description=description)
    parser.add_argument('--version', action='version', version=__version__)

    parser.add_argument(
        '-c', '--config', default=DEFAULT_CONFIG,
        help='Path to config file. By default: {0}'.format(DEFAULT_CONFIG)
    )
    parser.add_argument(
        '-p', '--pre-requirements', default=[], nargs='+',
        help='List of pre-requirements to check, separated by space.'
    )
    parser.add_argument(
        '-e', '--env',
        help='Virtual environment name. By default: {0}'.
             format(CONFIG[__script__]['env'])
    )
    parser.add_argument(
        '-r', '--requirements',
        help='Path to requirements file. By default: {0}'.
             format(CONFIG[__script__]['requirements'])
    )
    parser.add_argument(
        '-d', '--install-dev-requirements', action='store_true', default=None,
        help='Install prefixed or suffixed "dev" requirements after '
             'installation of original requirements file or library completed '
             'without errors.'
    )
    parser.add_argument(
        '-C', '--hook', help='Execute this hook after bootstrap process.'
    )
    parser.add_argument(
        '--ignore-activated', action='store_true', default=None,
        help='Ignore pre-activated virtualenv, like on Travis CI.'
    )
    parser.add_argument(
        '--recreate', action='store_true', default=None,
        help='Recreate virtualenv on every run.'
    )
    parser.add_argument(
        '-q', '--quiet', action='store_true', default=None,
        help='Minimize output, show only error messages.'
    )

    return parser.parse_args(args)

Parse args from command line by creating argument parser instance and process it .


def pip_cmd(env, cmd, ignore_activated=False, **kwargs):
    r"""Run pip command in given or activated virtual environment.

    :param env: Virtual environment name.
    :param cmd: Pip subcommand to run.
    :param ignore_activated:
        Ignore activated virtual environment and use given venv instead. By
        default: False
    :param \*\*kwargs:
        Additional keyword arguments to be passed to :func:`~run_cmd`
    """
    cmd = tuple(cmd)
    dirname = safe_path(env)

    if not ignore_activated:
        activated_env = os.environ.get('VIRTUAL_ENV')

        if hasattr(sys, 'real_prefix'):
            dirname = sys.prefix
        elif activated_env:
            dirname = activated_env

    pip_path = os.path.join(dirname, 'Scripts' if IS_WINDOWS else 'bin', 'pip')

    if kwargs.pop('return_path', False):
        return pip_path

    if not os.path.isfile(pip_path):
        raise OSError('No pip found at {0!r}'.format(pip_path))

    # Disable pip version check in tests
    if BOOTSTRAPPER_TEST_KEY in os.environ and cmd[0] == 'install':
        cmd = list(cmd)
        cmd.insert(1, '--disable-pip-version-check')
        cmd = tuple(cmd)

    with disable_error_handler():
        return run_cmd((pip_path, ) + cmd, **kwargs)

def
pip_cmd
(
env
,
cmd
,
ignore_activated
=
False
,
*
*
kwargs
)
:
cmd
=
tuple
(
cmd
)
dirname
=
safe_path
(
env
)
if
not
ignore_activated
:
activated_env
=
os
.
environ
.
get
(
'VIRTUAL_ENV'
)
if
hasattr
(
sys
,
'real_prefix'
)
:
dirname
=
sys
.
prefix
elif
activated_env
:
dirname
=
activated_env
pip_path
=
os
.
path
.
join
(
dirname
,
'Scripts'
if
IS_WINDOWS
else
'bin'
,
'pip'
)
if
kwargs
.
pop
(
'return_path'
,
False
)
:
return
pip_path
if
not
os
.
path
.
isfile
(
pip_path
)
:
raise
OSError
(
'No pip found at {0!r}'
.
format
(
pip_path
)
)
# Disable pip version check in tests
if
BOOTSTRAPPER_TEST_KEY
in
os
.
environ
and
cmd
[
0
]
==
'install'
:
cmd
=
list
(
cmd
)
cmd
.
insert
(
1
,
'--disable-pip-version-check'
)
cmd
=
tuple
(
cmd
)
with
disable_error_handler
(
)
:
return
run_cmd
(
(
pip_path
,
)
+
cmd
,
*
*
kwargs
)

r Run pip command in given or activated virtual environment .


def prepare_args(config, bootstrap):
    """Convert config dict to command line args line.

    :param config: Configuration dict.
    :param bootstrap: Bootstrapper configuration dict.
    """
    config = copy.deepcopy(config)
    environ = dict(copy.deepcopy(os.environ))

    data = {'env': bootstrap['env'],
            'pip': pip_cmd(bootstrap['env'], '', return_path=True),
            'requirements': bootstrap['requirements']}
    environ.update(data)

    if isinstance(config, string_types):
        return config.format(**environ)

    for key, value in iteritems(config):
        if not isinstance(value, string_types):
            continue
        config[key] = value.format(**environ)

    return config_to_args(config)

def prepare_args ( config , bootstrap ) : config = copy . deepcopy ( config ) environ = dict ( copy . deepcopy ( os . environ ) ) data = { 'env' : bootstrap [ 'env' ] , 'pip' : pip_cmd ( bootstrap [ 'env' ] , '' , return_path = True ) , 'requirements' : bootstrap [ 'requirements' ] } environ . update ( data ) if isinstance ( config , string_types ) : return config . format ( * * environ ) for key , value in iteritems ( config ) : if not isinstance ( value , string_types ) : continue config [ key ] = value . format ( * * environ ) return config_to_args ( config )

Convert config dict to command line args line .


def print_error(message, wrap=True):
    """Print error message to stderr, using ANSI-colors.

    :param message: Message to print
    :param wrap:
        Wrap message into ``ERROR: <message>. Exit...`` template. By default:
        True
    """
    if wrap:
        message = 'ERROR: {0}. Exit...'.format(message.rstrip('.'))

    colorizer = (_color_wrap(colorama.Fore.RED)
                 if colorama
                 else lambda message: message)
    return print(colorizer(message), file=sys.stderr)

def
print_error
(
message
,
wrap
=
True
)
:
if
wrap
:
message
=
'ERROR: {0}. Exit...'
.
format
(
message
.
rstrip
(
'.'
)
)
colorizer
=
(
_color_wrap
(
colorama
.
Fore
.
RED
)
if
colorama
else
lambda
message
:
message
)
return
print
(
colorizer
(
message
)
,
file
=
sys
.
stderr
)

Print error message to stderr using ANSI - colors .


def print_message(message=None):
    """Print message via ``subprocess.call`` function.

    This helps to ensure consistent output and avoid situations where print
    messages actually shown after messages from all inner threads.

    :param message: Text message to print.
    """
    kwargs = {'stdout': sys.stdout,
              'stderr': sys.stderr,
              'shell': True}
    return subprocess.call('echo "{0}"'.format(message or ''), **kwargs)

def
print_message
(
message
=
)
:
kwargs
=
{
'stdout'
:
sys
.
stdout
,
'stderr'
:
sys
.
stderr
,
'shell'
:
True
}
return
subprocess
.
call
(
'echo "{0}"'
.
format
(
message
or
''
)
,
*
*
kwargs
)

Print message via subprocess . call function .


def read_config(filename, args):
    """
    Read and parse configuration file. By default, ``filename`` is relative
    path to current work directory.

    If no config file found, default ``CONFIG`` would be used.

    :param filename: Read config from given filename.
    :param args: Parsed command line arguments.
    """
    # Initial vars
    config = defaultdict(dict)
    splitter = operator.methodcaller('split', ' ')

    converters = {
        __script__: {
            'env': safe_path,
            'pre_requirements': splitter,
        },
        'pip': {
            'allow_external': splitter,
            'allow_unverified': splitter,
        }
    }
    default = copy.deepcopy(CONFIG)
    sections = set(iterkeys(default))

    # Append download-cache for old pip versions
    if int(getattr(pip, '__version__', '1.x').split('.')[0]) < 6:
        default['pip']['download_cache'] = safe_path(os.path.expanduser(
            os.path.join('~', '.{0}'.format(__script__), 'pip-cache')
        ))

    # Expand user and environ vars in config filename
    is_default = filename == DEFAULT_CONFIG
    filename = os.path.expandvars(os.path.expanduser(filename))

    # Read config if it exists on disk
    if not is_default and not os.path.isfile(filename):
        print_error('Config file does not exist at {0!r}'.format(filename))
        return None

    parser = ConfigParser()

    try:
        parser.read(filename)
    except ConfigParserError:
        print_error('Cannot parse config file at {0!r}'.format(filename))
        return None

    # Apply config for each possible section
    for section in sections:
        if not parser.has_section(section):
            continue

        items = parser.items(section)

        # Make auto convert here for integers and boolean values
        for key, value in items:
            try:
                value = int(value)
            except (TypeError, ValueError):
                try:
                    value = bool(strtobool(value))
                except ValueError:
                    pass

            if section in converters and key in converters[section]:
                value = converters[section][key](value)

            config[section][key] = value

    # Update config with default values if necessary
    for section, data in iteritems(default):
        if section not in config:
            config[section] = data
        else:
            for key, value in iteritems(data):
                config[section].setdefault(key, value)

    # Update bootstrap config from parsed args
    keys = set((
        'env', 'hook', 'install_dev_requirements', 'ignore_activated',
        'pre_requirements', 'quiet', 'recreate', 'requirements'
    ))

    for key in keys:
        value = getattr(args, key)
        config[__script__].setdefault(key, value)

        if key == 'pre_requirements' and not value:
            continue

        if value is not None:
            config[__script__][key] = value

    return config

Read and parse configuration file . By default filename is relative path to current work directory .


def run_cmd(cmd, echo=False, fail_silently=False, **kwargs):
    r"""Call given command with ``subprocess.call`` function.

    :param cmd: Command to run.
    :type cmd: tuple or str
    :param echo:
        If enabled show command to call and its output in STDOUT, otherwise
        hide all output. By default: False
    :param fail_silently: Do not raise exception on error. By default: False
    :param \*\*kwargs:
        Additional keyword arguments to be passed to ``subprocess.call``
        function. STDOUT and STDERR streams would be setup inside of function
        to ensure hiding command output in case of disabling ``echo``.
    """
    out, err = None, None

    if echo:
        cmd_str = cmd if isinstance(cmd, string_types) else ' '.join(cmd)
        kwargs['stdout'], kwargs['stderr'] = sys.stdout, sys.stderr
        print_message('$ {0}'.format(cmd_str))
    else:
        out, err = get_temp_streams()
        kwargs['stdout'], kwargs['stderr'] = out, err

    try:
        retcode = subprocess.call(cmd, **kwargs)
    except subprocess.CalledProcessError as err:
        if fail_silently:
            return False
        print_error(str(err) if IS_PY3 else unicode(err))  # noqa
    finally:
        if out:
            out.close()
        if err:
            err.close()

    if retcode and echo and not fail_silently:
        print_error('Command {0!r} returned non-zero exit status {1}'.
                    format(cmd_str, retcode))

    return retcode

r Call given command with subprocess . call function .


def run_hook(hook, config, quiet=False):
    """Run post-bootstrap hook if any.

    :param hook: Hook to run.
    :param config: Configuration dict.
    :param quiet: Do not output messages to STDOUT/STDERR. By default: False
    """
    if not hook:
        return True

    if not quiet:
        print_message('== Step 3. Run post-bootstrap hook ==')

    result = not run_cmd(prepare_args(hook, config),
                         echo=not quiet,
                         fail_silently=True,
                         shell=True)

    if not quiet:
        print_message()

    return result

def
run_hook
(
hook
,
config
,
quiet
=
False
)
:
if
not
hook
:
return
True
if
not
quiet
:
print_message
(
'== Step 3. Run post-bootstrap hook =='
)
result
=
not
run_cmd
(
prepare_args
(
hook
,
config
)
,
echo
=
not
quiet
,
fail_silently
=
True
,
shell
=
True
)
if
not
quiet
:
print_message
(
)
return
result


def save_traceback(err):
    """Save error traceback to bootstrapper log file.

    :param err: Catched exception.
    """
    # Store logs to ~/.bootstrapper directory
    dirname = safe_path(os.path.expanduser(
        os.path.join('~', '.{0}'.format(__script__))
    ))

    # But ensure that directory exists
    if not os.path.isdir(dirname):
        os.mkdir(dirname)

    # Now we ready to put traceback to log file
    filename = os.path.join(dirname, '{0}.log'.format(__script__))

    with open(filename, 'a+') as handler:
        traceback.print_exc(file=handler)

    # And show colorized message
    message = ('User aborted workflow'
               if isinstance(err, KeyboardInterrupt)
               else 'Unexpected error catched')
    print_error(message)
    print_error('Full log stored to {0}'.format(filename), False)

    return True

Save error traceback to bootstrapper log file .


def smart_str(value, encoding='utf-8', errors='strict'):
    """Convert Python object to string.

    :param value: Python object to convert.
    :param encoding: Encoding to use if in Python 2 given object is unicode.
    :param errors: Errors mode to use if in Python 2 given object is unicode.
    """
    if not IS_PY3 and isinstance(value, unicode):  # noqa
        return value.encode(encoding, errors)
    return str(value)

def
smart_str
(
value
,
encoding
=
'utf-8'
,
errors
=
'strict'
)
:
if
not
IS_PY3
and
isinstance
(
value
,
unicode
)
:
# noqa
return
value
.
encode
(
encoding
,
errors
)
return
str
(
value
)


def copy_w_plus(src, dst):
    """Copy file from `src` path to `dst` path. If `dst` already exists, will add '+' characters
    to the end of the basename without extension.

    Parameters
    ----------
    src: str

    dst: str

    Returns
    -------
    dstpath: str
    """
    dst_ext = get_extension(dst)
    dst_pre = remove_ext   (dst)

    while op.exists(dst_pre + dst_ext):
        dst_pre += '+'

    shutil.copy(src, dst_pre + dst_ext)

    return dst_pre + dst_ext

def copy_w_plus ( src , dst ) : dst_ext = get_extension ( dst ) dst_pre = remove_ext ( dst ) while op . exists ( dst_pre + dst_ext ) : dst_pre += '+' shutil . copy ( src , dst_pre + dst_ext ) return dst_pre + dst_ext

Copy file from src path to dst path . If dst already exists will add + characters to the end of the basename without extension .


def get_abspath(folderpath):
    """Returns the absolute path of folderpath.
    If the path does not exist, will raise IOError.
    """
    if not op.exists(folderpath):
        raise FolderNotFound(folderpath)

    return op.abspath(folderpath)

def get_abspath ( folderpath ) : if not op . exists ( folderpath ) : raise FolderNotFound ( folderpath ) return op . abspath ( folderpath )

Returns the absolute path of folderpath . If the path does not exist will raise IOError .


def get_extension(filepath, check_if_exists=False, allowed_exts=ALLOWED_EXTS):
    """Return the extension of fpath.

    Parameters
    ----------
    fpath: string
    File name or path

    check_if_exists: bool

    allowed_exts: dict
    Dictionary of strings, where the key if the last part of a complex ('.' separated) extension
    and the value is the previous part.
    For example: for the '.nii.gz' extension I would have a dict as {'.gz': ['.nii',]}

    Returns
    -------
    str
    The extension of the file name or path
    """
    if check_if_exists:
        if not op.exists(filepath):
            raise IOError('File not found: ' + filepath)

    rest, ext = op.splitext(filepath)
    if ext in allowed_exts:
        alloweds = allowed_exts[ext]
        _, ext2 = op.splitext(rest)
        if ext2 in alloweds:
            ext = ext2 + ext

    return ext

def
get_extension
(
filepath
,
check_if_exists
=
False
,
allowed_exts
=
ALLOWED_EXTS
)
:
if
check_if_exists
:
if
not
op
.
exists
(
filepath
)
:
raise
IOError
(
'File not found: '
+
filepath
)
rest
,
ext
=
op
.
splitext
(
filepath
)
if
ext
in
allowed_exts
:
alloweds
=
allowed_exts
[
ext
]
_
,
ext2
=
op
.
splitext
(
rest
)
if
ext2
in
alloweds
:
ext
=
ext2
+
ext
return
ext


def add_extension_if_needed(filepath, ext, check_if_exists=False):
    """Add the extension ext to fpath if it doesn't have it.

    Parameters
    ----------
    filepath: str
    File name or path

    ext: str
    File extension

    check_if_exists: bool

    Returns
    -------
    File name or path with extension added, if needed.
    """
    if not filepath.endswith(ext):
        filepath += ext

    if check_if_exists:
        if not op.exists(filepath):
            raise IOError('File not found: ' + filepath)

    return filepath

def
add_extension_if_needed
(
filepath
,
ext
,
check_if_exists
=
False
)
:
if
not
filepath
.
endswith
(
ext
)
:
filepath
+=
ext
if
check_if_exists
:
if
not
op
.
exists
(
filepath
)
:
raise
IOError
(
'File not found: '
+
filepath
)
return
filepath

Add the extension ext to fpath if it doesn t have it .


def join_path_to_filelist(path, filelist):
    """Joins path to each line in filelist

    Parameters
    ----------
    path: str

    filelist: list of str

    Returns
    -------
    list of filepaths
    """
    return [op.join(path, str(item)) for item in filelist]

def join_path_to_filelist ( path , filelist ) : return [ op . join ( path , str ( item ) ) for item in filelist ]


def remove_all(filelist, folder=''):
    """Deletes all files in filelist

    Parameters
    ----------
    filelist: list of str
        List of the file paths to be removed

    folder: str
        Path to be used as common directory for all file paths in filelist
    """
    if not folder:
        for f in filelist:
            os.remove(f)
    else:
        for f in filelist:
            os.remove(op.join(folder, f))

def remove_all ( filelist , folder = '' ) : if not folder : for f in filelist : os . remove ( f ) else : for f in filelist : os . remove ( op . join ( folder , f ) )


def ux_file_len(filepath):
    """Returns the length of the file using the 'wc' GNU command

    Parameters
    ----------
    filepath: str

    Returns
    -------
    float
    """
    p = subprocess.Popen(['wc', '-l', filepath], stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)
    result, err = p.communicate()

    if p.returncode != 0:
        raise IOError(err)

    l = result.strip()
    l = int(l.split()[0])
    return l

def ux_file_len ( filepath ) : p = subprocess . Popen ( [ 'wc' , '-l' , filepath ] , stdout = subprocess . PIPE , stderr = subprocess . PIPE ) result , err = p . communicate ( ) if p . returncode != 0 : raise IOError ( err ) l = result . strip ( ) l = int ( l . split ( ) [ 0 ] ) return l

Returns the length of the file using the wc GNU command


def merge(dict_1, dict_2):
    """Merge two dictionaries.

    Values that evaluate to true take priority over falsy values.
    `dict_1` takes priority over `dict_2`.

    """
    return dict((str(key), dict_1.get(key) or dict_2.get(key))
                for key in set(dict_2) | set(dict_1))

def merge ( dict_1 , dict_2 ) : return dict ( ( str ( key ) , dict_1 . get ( key ) or dict_2 . get ( key ) ) for key in set ( dict_2 ) | set ( dict_1 ) )


def get_sys_path(rcpath, app_name, section_name=None):
    """Return a folder path if it exists.

    First will check if it is an existing system path, if it is, will return it
    expanded and absoluted.

    If this fails will look for the rcpath variable in the app_name rcfiles or
    exclusively within the given section_name, if given.

    Parameters
    ----------
    rcpath: str
        Existing folder path or variable name in app_name rcfile with an
        existing one.

    section_name: str
        Name of a section in the app_name rcfile to look exclusively there for
        variable names.

    app_name: str
        Name of the application to look for rcfile configuration files.

    Returns
    -------
    sys_path: str
        A expanded absolute file or folder path if the path exists.

    Raises
    ------
    IOError if the proposed sys_path does not exist.
    """
    # first check if it is an existing path
    if op.exists(rcpath):
        return op.realpath(op.expanduser(rcpath))

    # look for the rcfile
    try:
        settings = rcfile(app_name, section_name)
    except:
        raise

    # look for the variable within the rcfile configutarions
    try:
        sys_path = op.expanduser(settings[rcpath])
    except KeyError:
        raise IOError('Could not find an existing variable with name {0} in'
                      ' section {1} of {2}rc config setup. Maybe it is a '
                      ' folder that could not be found.'.format(rcpath,
                                                                section_name,
                                                                app_name))
    # found the variable, now check if it is an existing path
    else:
        if not op.exists(sys_path):
            raise IOError('Could not find the path {3} indicated by the '
                          'variable {0} in section {1} of {2}rc config '
                          'setup.'.format(rcpath, section_name, app_name,
                                          sys_path))

        # expand the path and return
        return op.realpath(op.expanduser(sys_path))


def rcfile(appname, section=None, args={}, strip_dashes=True):
    """Read environment variables and config files and return them merged with
    predefined list of arguments.

    Parameters
    ----------
    appname: str
        Application name, used for config files and environment variable
        names.

    section: str
        Name of the section to be read. If this is not set: appname.

    args:
        arguments from command line (optparse, docopt, etc).

    strip_dashes: bool
        Strip dashes prefixing key names from args dict.

    Returns
    --------
    dict
        containing the merged variables of environment variables, config
        files and args.

    Raises
    ------
    IOError
        In case the return value is empty.

    Notes
    -----
    Environment variables are read if they start with appname in uppercase
    with underscore, for example:

        TEST_VAR=1

    Config files compatible with ConfigParser are read and the section name
    appname is read, example:

        [appname]
        var=1

    We can also have host-dependent configuration values, which have
    priority over the default appname values.

        [appname]
        var=1

        [appname:mylinux]
        var=3

    For boolean flags do not try to use: 'True' or 'False',
                                         'on' or 'off',
                                         '1' or '0'.
    Unless you are willing to parse this values by yourself.
    We recommend commenting the variables out with '#' if you want to set a
    flag to False and check if it is in the rcfile cfg dict, i.e.:

        flag_value = 'flag_variable' in cfg

    Files are read from: /etc/appname/config,
                         /etc/appfilerc,
                         ~/.config/appname/config,
                         ~/.config/appname,
                         ~/.appname/config,
                         ~/.appnamerc,
                         appnamerc,
                         .appnamerc,
                         appnamerc file found in 'path' folder variable in args,
                         .appnamerc file found in 'path' folder variable in args,
                         file provided by 'config' variable in args.

    Example
    -------
        args = rcfile(__name__, docopt(__doc__, version=__version__))
    """
    if strip_dashes:
        for k in args.keys():
            args[k.lstrip('-')] = args.pop(k)

    environ = get_environment(appname)

    if section is None:
        section = appname

    config = get_config(appname,
                        section,
                        args.get('config', ''),
                        args.get('path', ''))
    config = merge(merge(args, config), environ)

    if not config:
        raise IOError('Could not find any rcfile for application '
                      '{}.'.format(appname))

    return config

Read environment variables and config files and return them merged with predefined list of arguments .


def get_rcfile_section(app_name, section_name):
    """ Return the dictionary containing the rcfile section configuration
    variables.

    Parameters
    ----------
    section_name: str
        Name of the section in the rcfiles.

    app_name: str
        Name of the application to look for its rcfiles.

    Returns
    -------
    settings: dict
        Dict with variable values
    """
    try:
        settings = rcfile(app_name, section_name)
    except IOError:
        raise
    except:
        raise KeyError('Error looking for section {} in {} '
                       ' rcfiles.'.format(section_name, app_name))
    else:
        return settings

def
get_rcfile_section
(
app_name
,
section_name
)
:
try
:
settings
=
rcfile
(
app_name
,
section_name
)
except
IOError
:
raise
except
:
raise
KeyError
(
'Error looking for section {} in {} '
' rcfiles.'
.
format
(
section_name
,
app_name
)
)
else
:
return
settings

Return the dictionary containing the rcfile section configuration variables .


def get_rcfile_variable_value(var_name, app_name, section_name=None):
    """ Return the value of the variable in the section_name section of the
    app_name rc file.

    Parameters
    ----------
    var_name: str
        Name of the variable to be searched for.

    section_name: str
        Name of the section in the rcfiles.

    app_name: str
        Name of the application to look for its rcfiles.

    Returns
    -------
    var_value: str
        The value of the variable with given var_name.
    """
    cfg = get_rcfile_section(app_name, section_name)

    if var_name in cfg:
        raise KeyError('Option {} not found in {} '
                       'section.'.format(var_name, section_name))

    return cfg[var_name]

def
get_rcfile_variable_value
(
var_name
,
app_name
,
section_name
=
)
:
cfg
=
get_rcfile_section
(
app_name
,
section_name
)
if
var_name
in
cfg
:
raise
KeyError
(
'Option {} not found in {} '
'section.'
.
format
(
var_name
,
section_name
)
)
return
cfg
[
var_name
]

Return the value of the variable in the section_name section of the app_name rc file .


def find_in_sections(var_name, app_name):
    """ Return the section and the value of the variable where the first
    var_name is found in the app_name rcfiles.

    Parameters
    ----------
    var_name: str
        Name of the variable to be searched for.

    app_name: str
        Name of the application to look for its rcfiles.

    Returns
    -------
    section_name: str
        Name of the section in the rcfiles where var_name was first found.

    var_value: str
        The value of the first variable with given var_name.
    """
    sections = get_sections(app_name)

    if not sections:
        raise ValueError('No sections found in {} rcfiles.'.format(app_name))

    for s in sections:
        try:
            var_value = get_rcfile_variable_value(var_name, section_name=s,
                                                  app_name=app_name)
        except:
            pass
        else:
            return s, var_value

    raise KeyError('No variable {} has been found in {} '
                   'rcfiles.'.format(var_name, app_name))

def
find_in_sections
(
var_name
,
app_name
)
:
sections
=
get_sections
(
app_name
)
if
not
sections
:
raise
ValueError
(
'No sections found in {} rcfiles.'
.
format
(
app_name
)
)
for
s
in
sections
:
try
:
var_value
=
get_rcfile_variable_value
(
var_name
,
section_name
=
s
,
app_name
=
app_name
)
except
:
pass
else
:
return
s
,
var_value
raise
KeyError
(
'No variable {} has been found in {} '
'rcfiles.'
.
format
(
var_name
,
app_name
)
)

Return the section and the value of the variable where the first var_name is found in the app_name rcfiles .


def filter_list(lst, pattern):
    """
    Filters the lst using pattern.
    If pattern starts with '(' it will be considered a re regular expression,
    otherwise it will use fnmatch filter.

    :param lst: list of strings

    :param pattern: string

    :return: list of strings
    Filtered list of strings
    """
    if is_fnmatch_regex(pattern) and not is_regex(pattern):
        #use fnmatch
        log.info('Using fnmatch for {0}'.format(pattern))
        filst = fnmatch.filter(lst, pattern)

    else:
        #use re
        log.info('Using regex match for {0}'.format(pattern))
        filst = match_list(lst, pattern)

    if filst:
        filst.sort()

    return filst

def
filter_list
(
lst
,
pattern
)
:
if
is_fnmatch_regex
(
pattern
)
and
not
is_regex
(
pattern
)
:
#use fnmatch
log
.
info
(
'Using fnmatch for {0}'
.
format
(
pattern
)
)
filst
=
fnmatch
.
filter
(
lst
,
pattern
)
else
:
#use re
log
.
info
(
'Using regex match for {0}'
.
format
(
pattern
)
)
filst
=
match_list
(
lst
,
pattern
)
if
filst
:
filst
.
sort
(
)
return
filst

Filters the lst using pattern . If pattern starts with ( it will be considered a re regular expression otherwise it will use fnmatch filter .


def get_subdict(adict, path, sep=os.sep):
    """
    Given a nested dictionary adict.
    This returns its childen just below the path.
    The path is a string composed of adict keys separated by sep.

    :param adict: nested dict

    :param path: str

    :param sep: str

    :return: dict or list or leaf of treemap

    """
    return reduce(adict.__class__.get, [p for p in op.split(sep) if p], adict)

def get_subdict ( adict , path , sep = os . sep ) : return reduce ( adict . __class__ . get , [ p for p in op . split ( sep ) if p ] , adict )

Given a nested dictionary adict . This returns its childen just below the path . The path is a string composed of adict keys separated by sep .


def get_dict_leaves(data):
    """
    Given a nested dictionary, this returns all its leave elements in a list.

    :param adict:

    :return: list
    """
    result = []
    if isinstance(data, dict):
        for item in data.values():
            result.extend(get_dict_leaves(item))
    elif isinstance(data, list):
        result.extend(data)
    else:
        result.append(data)

    return result

def get_dict_leaves ( data ) : result = [ ] if isinstance ( data , dict ) : for item in data . values ( ) : result . extend ( get_dict_leaves ( item ) ) elif isinstance ( data , list ) : result . extend ( data ) else : result . append ( data ) return result

Given a nested dictionary this returns all its leave elements in a list .


def get_possible_paths(base_path, path_regex):
    """
    Looks for path_regex within base_path. Each match is append
    in the returned list.
    path_regex may contain subfolder structure.
    If any part of the folder structure is a

    :param base_path: str

    :param path_regex: str

    :return list of strings
    """
    if not path_regex:
        return []

    if len(path_regex) < 1:
        return []

    if path_regex[0] == os.sep:
        path_regex = path_regex[1:]

    rest_files = ''
    if os.sep in path_regex:
        #split by os.sep
        node_names = path_regex.partition(os.sep)
        first_node = node_names[0]
        rest_nodes = node_names[2]

        folder_names = filter_list(os.listdir(base_path), first_node)

        for nom in folder_names:
            new_base = op.join(base_path, nom)
            if op.isdir(new_base):
                rest_files = get_possible_paths(new_base, rest_nodes)
    else:
        rest_files = filter_list(os.listdir(base_path), path_regex)

    files = []
    if rest_files:
        files = [op.join(base_path, f) for f in rest_files]

    return files

def
get_possible_paths
(
base_path
,
path_regex
)
:
if
not
path_regex
:
return
[
]
if
len
(
path_regex
)
<
1
:
return
[
]
if
path_regex
[
0
]
==
os
.
sep
:
path_regex
=
path_regex
[
1
:
]
rest_files
=
''
if
os
.
sep
in
path_regex
:
#split by os.sep
node_names
=
path_regex
.
partition
(
os
.
sep
)
first_node
=
node_names
[
0
]
rest_nodes
=
node_names
[
2
]
folder_names
=
filter_list
(
os
.
listdir
(
base_path
)
,
first_node
)
for
nom
in
folder_names
:
new_base
=
op
.
join
(
base_path
,
nom
)
if
op
.
isdir
(
new_base
)
:
rest_files
=
get_possible_paths
(
new_base
,
rest_nodes
)
else
:
rest_files
=
filter_list
(
os
.
listdir
(
base_path
)
,
path_regex
)
files
=
[
]
if
rest_files
:
files
=
[
op
.
join
(
base_path
,
f
)
for
f
in
rest_files
]
return
files

Looks for path_regex within base_path . Each match is append in the returned list . path_regex may contain subfolder structure . If any part of the folder structure is a


def create_folder(dirpath, overwrite=False):
        """ Will create dirpath folder. If dirpath already exists and overwrite is False,
        will append a '+' suffix to dirpath until dirpath does not exist."""
        if not overwrite:
            while op.exists(dirpath):
                dirpath += '+'

        os.makedirs(dirpath, exist_ok=overwrite)
        return dirpath

def create_folder ( dirpath , overwrite = False ) : if not overwrite : while op . exists ( dirpath ) : dirpath += '+' os . makedirs ( dirpath , exist_ok = overwrite ) return dirpath

Will create dirpath folder . If dirpath already exists and overwrite is False will append a + suffix to dirpath until dirpath does not exist .


def _import_config(filepath):
        """
        Imports filetree and root_path variable values from the filepath.

        :param filepath:
        :return: root_path and filetree
        """
        if not op.isfile(filepath):
            raise IOError('Data config file not found. '
                          'Got: {0}'.format(filepath))

        cfg = import_pyfile(filepath)

        if not hasattr(cfg, 'root_path'):
            raise KeyError('Config file root_path key not found.')

        if not hasattr(cfg, 'filetree'):
            raise KeyError('Config file filetree key not found.')

        return cfg.root_path, cfg.filetree

def
_import_config
(
filepath
)
:
if
not
op
.
isfile
(
filepath
)
:
raise
IOError
(
'Data config file not found. '
'Got: {0}'
.
format
(
filepath
)
)
cfg
=
import_pyfile
(
filepath
)
if
not
hasattr
(
cfg
,
'root_path'
)
:
raise
KeyError
(
'Config file root_path key not found.'
)
if
not
hasattr
(
cfg
,
'filetree'
)
:
raise
KeyError
(
'Config file filetree key not found.'
)
return
cfg
.
root_path
,
cfg
.
filetree

Imports filetree and root_path variable values from the filepath .


def remove_nodes(self, pattern, adict):
        """
        Remove the nodes that match the pattern.
        """
        mydict = self._filetree if adict is None else adict

        if isinstance(mydict, dict):
            for nom in mydict.keys():
                if isinstance(mydict[nom], dict):
                    matchs = filter_list(mydict[nom], pattern)
                    for nom in matchs:
                        mydict = self.remove_nodes(pattern, mydict[nom])
                        mydict.pop(nom)
                else:
                    mydict[nom] = filter_list(mydict[nom], pattern)
        else:
            matchs = set(filter_list(mydict, pattern))
            mydict = set(mydict) - matchs

        return mydict

def remove_nodes ( self , pattern , adict ) : mydict = self . _filetree if adict is None else adict if isinstance ( mydict , dict ) : for nom in mydict . keys ( ) : if isinstance ( mydict [ nom ] , dict ) : matchs = filter_list ( mydict [ nom ] , pattern ) for nom in matchs : mydict = self . remove_nodes ( pattern , mydict [ nom ] ) mydict . pop ( nom ) else : mydict [ nom ] = filter_list ( mydict [ nom ] , pattern ) else : matchs = set ( filter_list ( mydict , pattern ) ) mydict = set ( mydict ) - matchs return mydict

Remove the nodes that match the pattern .


def count_node_match(self, pattern, adict=None):
        """
        Return the number of nodes that match the pattern.

        :param pattern:

        :param adict:
        :return: int
        """
        mydict = self._filetree if adict is None else adict

        k = 0
        if isinstance(mydict, dict):
            names = mydict.keys()
            k += len(filter_list(names, pattern))
            for nom in names:
                k += self.count_node_match(pattern, mydict[nom])
        else:
            k = len(filter_list(mydict, pattern))

        return k

def count_node_match ( self , pattern , adict = None ) : mydict = self . _filetree if adict is None else adict k = 0 if isinstance ( mydict , dict ) : names = mydict . keys ( ) k += len ( filter_list ( names , pattern ) ) for nom in names : k += self . count_node_match ( pattern , mydict [ nom ] ) else : k = len ( filter_list ( mydict , pattern ) ) return k

Return the number of nodes that match the pattern .


def as_float_array(X, copy=True, force_all_finite=True):
    """Converts an array-like to an array of floats

    The new dtype will be np.float32 or np.float64, depending on the original
    type. The function can create a copy or modify the argument depending
    on the argument copy.

    Parameters
    ----------
    X : {array-like, sparse matrix}

    copy : bool, optional
        If True, a copy of X will be created. If False, a copy may still be
        returned if X's dtype is not a floating point type.

    Returns
    -------
    XT : {array, sparse matrix}
        An array of type np.float
    """
    if isinstance(X, np.matrix) or (not isinstance(X, np.ndarray)
                                    and not sp.issparse(X)):
        return check_array(X, ['csr', 'csc', 'coo'], dtype=np.float64,
                           copy=copy, force_all_finite=force_all_finite,
                           ensure_2d=False)
    elif sp.issparse(X) and X.dtype in [np.float32, np.float64]:
        return X.copy() if copy else X
    elif X.dtype in [np.float32, np.float64]:  # is numpy array
        return X.copy('F' if X.flags['F_CONTIGUOUS'] else 'C') if copy else X
    else:
        return X.astype(np.float32 if X.dtype == np.int32 else np.float64)

def
as_float_array
(
X
,
copy
=
True
,
force_all_finite
=
True
)
:
if
isinstance
(
X
,
np
.
matrix
)
or
(
not
isinstance
(
X
,
np
.
ndarray
)
and
not
sp
.
issparse
(
X
)
)
:
return
check_array
(
X
,
[
'csr'
,
'csc'
,
'coo'
]
,
dtype
=
np
.
float64
,
copy
=
copy
,
force_all_finite
=
force_all_finite
,
ensure_2d
=
False
)
elif
sp
.
issparse
(
X
)
and
X
.
dtype
in
[
np
.
float32
,
np
.
float64
]
:
return
X
.
copy
(
)
if
copy
else
X
elif
X
.
dtype
in
[
np
.
float32
,
np
.
float64
]
:
# is numpy array
return
X
.
copy
(
'F'
if
X
.
flags
[
'F_CONTIGUOUS'
]
else
'C'
)
if
copy
else
X
else
:
return
X
.
astype
(
np
.
float32
if
X
.
dtype
==
np
.
int32
else
np
.
float64
)

Converts an array - like to an array of floats


def indexable(*iterables):
    """Make arrays indexable for cross-validation.

    Checks consistent length, passes through None, and ensures that everything
    can be indexed by converting sparse matrices to csr and converting
    non-interable objects to arrays.

    Parameters
    ----------
    iterables : lists, dataframes, arrays, sparse matrices
        List of objects to ensure sliceability.
    """
    result = []
    for X in iterables:
        if sp.issparse(X):
            result.append(X.tocsr())
        elif hasattr(X, "__getitem__") or hasattr(X, "iloc"):
            result.append(X)
        elif X is None:
            result.append(X)
        else:
            result.append(np.array(X))
    check_consistent_length(*result)
    return result

def indexable ( * iterables ) : result = [ ] for X in iterables : if sp . issparse ( X ) : result . append ( X . tocsr ( ) ) elif hasattr ( X , "__getitem__" ) or hasattr ( X , "iloc" ) : result . append ( X ) elif X is None : result . append ( X ) else : result . append ( np . array ( X ) ) check_consistent_length ( * result ) return result

Make arrays indexable for cross - validation .


def check_array(array, accept_sparse=None, dtype=None, order=None, copy=False,
                force_all_finite=True, ensure_2d=True, allow_nd=False):
    """Input validation on an array, list, sparse matrix or similar.

    By default, the input is converted to an at least 2nd numpy array.

    Parameters
    ----------
    array : object
        Input object to check / convert.

    accept_sparse : string, list of string or None (default=None)
        String[s] representing allowed sparse matrix formats, such as 'csc',
        'csr', etc.  None means that sparse matrix input will raise an error.
        If the input is sparse but not in the allowed format, it will be
        converted to the first listed format.

    order : 'F', 'C' or None (default=None)
        Whether an array will be forced to be fortran or c-style.

    copy : boolean (default=False)
        Whether a forced copy will be triggered. If copy=False, a copy might
        be triggered by a conversion.

    force_all_finite : boolean (default=True)
        Whether to raise an error on np.inf and np.nan in X.

    ensure_2d : boolean (default=True)
        Whether to make X at least 2d.

    allow_nd : boolean (default=False)
        Whether to allow X.ndim > 2.

    Returns
    -------
    X_converted : object
        The converted and validated X.
    """
    if isinstance(accept_sparse, str):
        accept_sparse = [accept_sparse]

    if sp.issparse(array):
        array = _ensure_sparse_format(array, accept_sparse, dtype, order,
                                      copy, force_all_finite)
    else:
        if ensure_2d:
            array = np.atleast_2d(array)
        array = np.array(array, dtype=dtype, order=order, copy=copy)
        if not allow_nd and array.ndim >= 3:
            raise ValueError("Found array with dim %d. Expected <= 2" %
                             array.ndim)
        if force_all_finite:
            _assert_all_finite(array)

    return array

Input validation on an array list sparse matrix or similar .


def check_X_y(X, y, accept_sparse=None, dtype=None, order=None, copy=False,
              force_all_finite=True, ensure_2d=True, allow_nd=False,
              multi_output=False):
    """Input validation for standard estimators.

    Checks X and y for consistent length, enforces X 2d and y 1d.
    Standard input checks are only applied to y. For multi-label y,
    set multi_ouput=True to allow 2d and sparse y.

    Parameters
    ----------
    X : nd-array, list or sparse matrix
        Input data.

    y : nd-array, list or sparse matrix
        Labels.

    accept_sparse : string, list of string or None (default=None)
        String[s] representing allowed sparse matrix formats, such as 'csc',
        'csr', etc.  None means that sparse matrix input will raise an error.
        If the input is sparse but not in the allowed format, it will be
        converted to the first listed format.

    order : 'F', 'C' or None (default=None)
        Whether an array will be forced to be fortran or c-style.

    copy : boolean (default=False)
        Whether a forced copy will be triggered. If copy=False, a copy might
        be triggered by a conversion.

    force_all_finite : boolean (default=True)
        Whether to raise an error on np.inf and np.nan in X.

    ensure_2d : boolean (default=True)
        Whether to make X at least 2d.

    allow_nd : boolean (default=False)
        Whether to allow X.ndim > 2.

    Returns
    -------
    X_converted : object
        The converted and validated X.
    """
    X = check_array(X, accept_sparse, dtype, order, copy, force_all_finite,
                    ensure_2d, allow_nd)
    if multi_output:
        y = check_array(y, 'csr', force_all_finite=True, ensure_2d=False)
    else:
        y = column_or_1d(y, warn=True)
        _assert_all_finite(y)

    check_consistent_length(X, y)

    return X, y

def check_X_y ( X , y , accept_sparse = None , dtype = None , order = None , copy = False , force_all_finite = True , ensure_2d = True , allow_nd = False , multi_output = False ) : X = check_array ( X , accept_sparse , dtype , order , copy , force_all_finite , ensure_2d , allow_nd ) if multi_output : y = check_array ( y , 'csr' , force_all_finite = True , ensure_2d = False ) else : y = column_or_1d ( y , warn = True ) _assert_all_finite ( y ) check_consistent_length ( X , y ) return X , y

Input validation for standard estimators .


def column_or_1d(y, warn=False):
    """ Ravel column or 1d numpy array, else raises an error

    Parameters
    ----------
    y : array-like

    Returns
    -------
    y : array

    """
    shape = np.shape(y)
    if len(shape) == 1:
        return np.ravel(y)
    if len(shape) == 2 and shape[1] == 1:
        if warn:
            warnings.warn("A column-vector y was passed when a 1d array was"
                          " expected. Please change the shape of y to "
                          "(n_samples, ), for example using ravel().",
                          DataConversionWarning, stacklevel=2)
        return np.ravel(y)

    raise ValueError("bad input shape {0}".format(shape))

Ravel column or 1d numpy array else raises an error


def warn_if_not_float(X, estimator='This algorithm'):
    """Warning utility function to check that data type is floating point.

    Returns True if a warning was raised (i.e. the input is not float) and
    False otherwise, for easier input validation.
    """
    if not isinstance(estimator, str):
        estimator = estimator.__class__.__name__
    if X.dtype.kind != 'f':
        warnings.warn("%s assumes floating point values as input, "
                      "got %s" % (estimator, X.dtype))
        return True
    return False

Warning utility function to check that data type is floating point .


def as_ndarray(arr, copy=False, dtype=None, order='K'):
    """Convert an arbitrary array to numpy.ndarray.

    In the case of a memmap array, a copy is automatically made to break the
    link with the underlying file (whatever the value of the "copy" keyword).

    The purpose of this function is mainly to get rid of memmap objects, but
    it can be used for other purposes. In particular, combining copying and
    casting can lead to performance improvements in some cases, by avoiding
    unnecessary copies.

    If not specified, input array order is preserved, in all cases, even when
    a copy is requested.

    Caveat: this function does not copy during bool to/from 1-byte dtype
    conversions. This can lead to some surprising results in some rare cases.
    Example:

        a = numpy.asarray([0, 1, 2], dtype=numpy.int8)
        b = as_ndarray(a, dtype=bool)  # array([False, True, True], dtype=bool)
        c = as_ndarray(b, dtype=numpy.int8)  # array([0, 1, 2], dtype=numpy.int8)

    The usually expected result for the last line would be array([0, 1, 1])
    because True evaluates to 1. Since there is no copy made here, the original
    array is recovered.

    Parameters
    ----------
    arr: array-like
        input array. Any value accepted by numpy.asarray is valid.

    copy: bool
        if True, force a copy of the array. Always True when arr is a memmap.

    dtype: any numpy dtype
        dtype of the returned array. Performing copy and type conversion at the
        same time can in some cases avoid an additional copy.

    order: string
        gives the order of the returned array.
        Valid values are: "C", "F", "A", "K", None.
        default is "K". See ndarray.copy() for more information.

    Returns
    -------
    ret: np.ndarray
        Numpy array containing the same data as arr, always of class
        numpy.ndarray, and with no link to any underlying file.
    """
    if order not in ('C', 'F', 'A', 'K', None):
        raise ValueError("Invalid value for 'order': {}".format(str(order)))

    if isinstance(arr, np.memmap):
        if dtype is None:
            if order in ('K', 'A', None):
                ret = np.array(np.asarray(arr), copy=True)
            else:
                ret = np.array(np.asarray(arr), copy=True, order=order)
        else:
            if order in ('K', 'A', None):
                # always copy (even when dtype does not change)
                ret = np.asarray(arr).astype(dtype)
            else:
                # load data from disk without changing order
                # Changing order while reading through a memmap is incredibly
                # inefficient.
                ret = _asarray(np.array(arr, copy=True), dtype=dtype, order=order)

    elif isinstance(arr, np.ndarray):
        ret = _asarray(arr, dtype=dtype, order=order)
        # In the present cas, np.may_share_memory result is always reliable.
        if np.may_share_memory(ret, arr) and copy:
            # order-preserving copy
            ret = ret.T.copy().T if ret.flags['F_CONTIGUOUS'] else ret.copy()

    elif isinstance(arr, (list, tuple)):
        if order in ("A", "K"):
            ret = np.asarray(arr, dtype=dtype)
        else:
            ret = np.asarray(arr, dtype=dtype, order=order)

    else:
        raise ValueError("Type not handled: {}".format(arr.__class__))

    return ret

Convert an arbitrary array to numpy . ndarray .