def _parse_field_value(line):
    """ Parse the field and value from a line. """
    if line.startswith(':'):
        # Ignore the line
        return None, None

    if ':' not in line:
        # Treat the entire line as the field, use empty string as value
        return line, ''

    # Else field is before the ':' and value is after
    field, value = line.split(':', 1)

    # If value starts with a space, remove it.
    value = value[1:] if value.startswith(' ') else value

    return field, value


def dataReceived(self, data):
        """
        Translates bytes into lines, and calls lineReceived.

        Copied from ``twisted.protocols.basic.LineOnlyReceiver`` but using
        str.splitlines() to split on ``\r\n``, ``\n``, and ``\r``.
        """
        self.resetTimeout()
        lines = (self._buffer + data).splitlines()

        # str.splitlines() doesn't split the string after a trailing newline
        # character so we must check if there is a trailing newline and, if so,
        # clear the buffer as the line is "complete". Else, the line is
        # incomplete and we keep the last line in the buffer.
        if data.endswith(b'\n') or data.endswith(b'\r'):
            self._buffer = b''
        else:
            self._buffer = lines.pop(-1)

        for line in lines:
            if self.transport.disconnecting:
                # this is necessary because the transport may be told to lose
                # the connection by a line within a larger packet, and it is
                # important to disregard all the lines in that packet following
                # the one that told it to close.
                return
            if len(line) > self._max_length:
                self.lineLengthExceeded(line)
                return
            else:
                self.lineReceived(line)
        if len(self._buffer) > self._max_length:
            self.lineLengthExceeded(self._buffer)
            return

Translates bytes into lines and calls lineReceived .


def _handle_field_value(self, field, value):
        """ Handle the field, value pair. """
        if field == 'event':
            self._event = value
        elif field == 'data':
            self._data_lines.append(value)
        elif field == 'id':
            # Not implemented
            pass
        elif field == 'retry':
            # Not implemented
            pass

def
_handle_field_value
(
self
,
field
,
value
)
:
if
field
==
'event'
:
self
.
_event
=
value
elif
field
==
'data'
:
self
.
_data_lines
.
append
(
value
)
elif
field
==
'id'
:
# Not implemented
pass
elif
field
==
'retry'
:
# Not implemented
pass


def _dispatch_event(self):
        """
        Dispatch the event to the handler.
        """
        data = self._prepare_data()
        if data is not None:
            self._handler(self._event, data)

        self._reset_event_data()

def _dispatch_event ( self ) : data = self . _prepare_data ( ) if data is not None : self . _handler ( self . _event , data ) self . _reset_event_data ( )


def listen_events(self, reconnects=0):
        """
        Start listening for events from Marathon, running a sync when we first
        successfully subscribe and triggering a sync on API request events.
        """
        self.log.info('Listening for events from Marathon...')
        self._attached = False

        def on_finished(result, reconnects):
            # If the callback fires then the HTTP request to the event stream
            # went fine, but the persistent connection for the SSE stream was
            # dropped. Just reconnect for now- if we can't actually connect
            # then the errback will fire rather.
            self.log.warn('Connection lost listening for events, '
                          'reconnecting... ({reconnects} so far)',
                          reconnects=reconnects)
            reconnects += 1
            return self.listen_events(reconnects)

        def log_failure(failure):
            self.log.failure('Failed to listen for events', failure)
            return failure

        return self.marathon_client.get_events({
            'event_stream_attached': self._sync_on_event_stream_attached,
            'api_post_event': self._sync_on_api_post_event
        }).addCallbacks(on_finished, log_failure, callbackArgs=[reconnects])

Start listening for events from Marathon running a sync when we first successfully subscribe and triggering a sync on API request events .


def sync(self):
        """
        Fetch the list of apps from Marathon, find the domains that require
        certificates, and issue certificates for any domains that don't already
        have a certificate.
        """
        self.log.info('Starting a sync...')

        def log_success(result):
            self.log.info('Sync completed successfully')
            return result

        def log_failure(failure):
            self.log.failure('Sync failed', failure, LogLevel.error)
            return failure

        return (self.marathon_client.get_apps()
                .addCallback(self._apps_acme_domains)
                .addCallback(self._filter_new_domains)
                .addCallback(self._issue_certs)
                .addCallbacks(log_success, log_failure))

def
sync
(
self
)
:
self
.
log
.
info
(
'Starting a sync...'
)
def
log_success
(
result
)
:
self
.
log
.
info
(
'Sync completed successfully'
)
return
result
def
log_failure
(
failure
)
:
self
.
log
.
failure
(
'Sync failed'
,
failure
,
LogLevel
.
error
)
return
failure
return
(
self
.
marathon_client
.
get_apps
(
)
.
addCallback
(
self
.
_apps_acme_domains
)
.
addCallback
(
self
.
_filter_new_domains
)
.
addCallback
(
self
.
_issue_certs
)
.
addCallbacks
(
log_success
,
log_failure
)
)

Fetch the list of apps from Marathon find the domains that require certificates and issue certificates for any domains that don t already have a certificate .


def _issue_cert(self, domain):
        """
        Issue a certificate for the given domain.
        """
        def errback(failure):
            # Don't fail on some of the errors we could get from the ACME
            # server, rather just log an error so that we can continue with
            # other domains.
            failure.trap(txacme_ServerError)
            acme_error = failure.value.message

            if acme_error.code in ['rateLimited', 'serverInternal',
                                   'connection', 'unknownHost']:
                # TODO: Fire off an error to Sentry or something?
                self.log.error(
                    'Error ({code}) issuing certificate for "{domain}": '
                    '{detail}', code=acme_error.code, domain=domain,
                    detail=acme_error.detail)
            else:
                # There are more error codes but if they happen then something
                # serious has gone wrong-- carry on error-ing.
                return failure

        d = self.txacme_service.issue_cert(domain)
        return d.addErrback(errback)

Issue a certificate for the given domain .


def rm_fwd_refs(obj):
    """When removing an object, other objects with references to the current
    object should remove those references. This function identifies objects
    with forward references to the current object, then removes those
    references.

    :param obj: Object to which forward references should be removed

    """
    for stack, key in obj._backrefs_flat:

        # Unpack stack
        backref_key, parent_schema_name, parent_field_name = stack

        # Get parent info
        parent_schema = obj._collections[parent_schema_name]
        parent_key_store = parent_schema._pk_to_storage(key)
        parent_object = parent_schema.load(parent_key_store)
        if parent_object is None:
            continue

        # Remove forward references
        if parent_object._fields[parent_field_name]._list:
            getattr(parent_object, parent_field_name).remove(obj)
        else:
            parent_field_object = parent_object._fields[parent_field_name]
            setattr(parent_object, parent_field_name, parent_field_object._gen_default())

        # Save
        parent_object.save()

def
rm_fwd_refs
(
obj
)
:
for
stack
,
key
in
obj
.
_backrefs_flat
:
# Unpack stack
backref_key
,
parent_schema_name
,
parent_field_name
=
stack
# Get parent info
parent_schema
=
obj
.
_collections
[
parent_schema_name
]
parent_key_store
=
parent_schema
.
_pk_to_storage
(
key
)
parent_object
=
parent_schema
.
load
(
parent_key_store
)
if
parent_object
is
:
continue
# Remove forward references
if
parent_object
.
_fields
[
parent_field_name
]
.
_list
:
getattr
(
parent_object
,
parent_field_name
)
.
remove
(
obj
)
else
:
parent_field_object
=
parent_object
.
_fields
[
parent_field_name
]
setattr
(
parent_object
,
parent_field_name
,
parent_field_object
.
_gen_default
(
)
)
# Save
parent_object
.
save
(
)

When removing an object other objects with references to the current object should remove those references . This function identifies objects with forward references to the current object then removes those references .


def rm_back_refs(obj):
    """When removing an object with foreign fields, back-references from
    other objects to the current object should be deleted. This function
    identifies foreign fields of the specified object whose values are not
    None and which specify back-reference keys, then removes back-references
    from linked objects to the specified object.

    :param obj: Object for which back-references should be removed

    """
    for ref in _collect_refs(obj):
        ref['value']._remove_backref(
            ref['field_instance']._backref_field_name,
            obj,
            ref['field_name'],
            strict=False
        )

def rm_back_refs ( obj ) : for ref in _collect_refs ( obj ) : ref [ 'value' ] . _remove_backref ( ref [ 'field_instance' ] . _backref_field_name , obj , ref [ 'field_name' ] , strict = False )

When removing an object with foreign fields back - references from other objects to the current object should be deleted . This function identifies foreign fields of the specified object whose values are not None and which specify back - reference keys then removes back - references from linked objects to the specified object .


def ensure_backrefs(obj, fields=None):
    """Ensure that all forward references on the provided object have the
    appropriate backreferences.

    :param StoredObject obj: Database record
    :param list fields: Optional list of field names to check

    """
    for ref in _collect_refs(obj, fields):
        updated = ref['value']._update_backref(
            ref['field_instance']._backref_field_name,
            obj,
            ref['field_name'],
        )
        if updated:
            logging.debug('Updated reference {}:{}:{}:{}:{}'.format(
                obj._name, obj._primary_key, ref['field_name'],
                ref['value']._name, ref['value']._primary_key,
            ))

def
ensure_backrefs
(
obj
,
fields
=
)
:
for
ref
in
_collect_refs
(
obj
,
fields
)
:
updated
=
ref
[
'value'
]
.
_update_backref
(
ref
[
'field_instance'
]
.
_backref_field_name
,
obj
,
ref
[
'field_name'
]
,
)
if
updated
:
logging
.
debug
(
'Updated reference {}:{}:{}:{}:{}'
.
format
(
obj
.
_name
,
obj
.
_primary_key
,
ref
[
'field_name'
]
,
ref
[
'value'
]
.
_name
,
ref
[
'value'
]
.
_primary_key
,
)
)

Ensure that all forward references on the provided object have the appropriate backreferences .


def _remove_by_pk(self, key, flush=True):
        """Retrieve value from store.

        :param key: Key

        """
        try:
            del self.store[key]
        except Exception as error:
            pass
        if flush:
            self.flush()

def _remove_by_pk ( self , key , flush = True ) : try : del self . store [ key ] except Exception as error : pass if flush : self . flush ( )


def start(self):
        """Initializes the bot, plugins, and everything."""
        self.bot_start_time = datetime.now()
        self.webserver = Webserver(self.config['webserver']['host'], self.config['webserver']['port'])
        self.plugins.load()
        self.plugins.load_state()
        self._find_event_handlers()
        self.sc = ThreadedSlackClient(self.config['slack_token'])

        self.always_send_dm = ['_unauthorized_']
        if 'always_send_dm' in self.config:
            self.always_send_dm.extend(map(lambda x: '!' + x, self.config['always_send_dm']))

        # Rocket is very noisy at debug
        logging.getLogger('Rocket.Errors.ThreadPool').setLevel(logging.INFO)

        self.is_setup = True
        if self.test_mode:
            self.metrics['startup_time'] = (datetime.now() - self.bot_start_time).total_seconds() * 1000.0

def
start
(
self
)
:
self
.
bot_start_time
=
datetime
.
now
(
)
self
.
webserver
=
Webserver
(
self
.
config
[
'webserver'
]
[
'host'
]
,
self
.
config
[
'webserver'
]
[
'port'
]
)
self
.
plugins
.
load
(
)
self
.
plugins
.
load_state
(
)
self
.
_find_event_handlers
(
)
self
.
sc
=
ThreadedSlackClient
(
self
.
config
[
'slack_token'
]
)
self
.
always_send_dm
=
[
'_unauthorized_'
]
if
'always_send_dm'
in
self
.
config
:
self
.
always_send_dm
.
extend
(
map
(
lambda
x
:
'!'
+
x
,
self
.
config
[
'always_send_dm'
]
)
)
# Rocket is very noisy at debug
logging
.
getLogger
(
'Rocket.Errors.ThreadPool'
)
.
setLevel
(
logging
.
INFO
)
self
.
is_setup
=
True
if
self
.
test_mode
:
self
.
metrics
[
'startup_time'
]
=
(
datetime
.
now
(
)
self
.
bot_start_time
)
.
total_seconds
(
)
*
1000.0

Initializes the bot plugins and everything .


def run(self, start=True):
        """
        Connects to slack and enters the main loop.

        * start - If True, rtm.start API is used. Else rtm.connect API is used

        For more info, refer to
        https://python-slackclient.readthedocs.io/en/latest/real_time_messaging.html#rtm-start-vs-rtm-connect
        """
        # Fail out if setup wasn't run
        if not self.is_setup:
            raise NotSetupError

        # Start the web server
        self.webserver.start()

        first_connect = True

        try:
            while self.runnable:
                if self.reconnect_needed:
                    if not self.sc.rtm_connect(with_team_state=start):
                        return False
                    self.reconnect_needed = False
                    if first_connect:
                        first_connect = False
                        self.plugins.connect()

                # Get all waiting events - this always returns a list
                try:
                    events = self.sc.rtm_read()
                except AttributeError:
                    self.log.exception('Something has failed in the slack rtm library.  This is fatal.')
                    self.runnable = False
                    events = []
                except:
                    self.log.exception('Unhandled exception in rtm_read()')
                    self.reconnect_needed = True
                    events = []
                for e in events:
                    try:
                        self._handle_event(e)
                    except KeyboardInterrupt:
                        # Gracefully shutdown
                        self.runnable = False
                    except:
                        self.log.exception('Unhandled exception in event handler')
                sleep(0.1)
        except KeyboardInterrupt:
            # On ctrl-c, just exit
            pass
        except:
            self.log.exception('Unhandled exception')

Connects to slack and enters the main loop .


def stop(self):
        """Does cleanup of bot and plugins."""
        if self.webserver is not None:
            self.webserver.stop()
        if not self.test_mode:
            self.plugins.save_state()

def stop ( self ) : if self . webserver is not None : self . webserver . stop ( ) if not self . test_mode : self . plugins . save_state ( )


def send_message(self, channel, text, thread=None, reply_broadcast=None):
        """
        Sends a message to the specified channel

        * channel - The channel to send to.  This can be a SlackChannel object, a channel id, or a channel name
        (without the #)
        * text - String to send
        * thread - reply to the thread. See https://api.slack.com/docs/message-threading#threads_party
        * reply_broadcast - Set to true to indicate your reply is germane to all members of a channel
        """
        # This doesn't want the # in the channel name
        if isinstance(channel, SlackRoomIMBase):
            channel = channel.id
        self.log.debug("Trying to send to %s: %s", channel, text)
        self.sc.rtm_send_message(channel, text, thread=thread, reply_broadcast=reply_broadcast)


def send_im(self, user, text):
        """
        Sends a message to a user as an IM

        * user - The user to send to.  This can be a SlackUser object, a user id, or the username (without the @)
        * text - String to send
        """
        if isinstance(user, SlackUser):
            user = user.id
            channelid = self._find_im_channel(user)
        else:
            channelid = user.id
        self.send_message(channelid, text)

def send_im ( self , user , text ) : if isinstance ( user , SlackUser ) : user = user . id channelid = self . _find_im_channel ( user ) else : channelid = user . id self . send_message ( channelid , text )


def push(self, message):
        """
        Takes a SlackEvent, parses it for a command, and runs against registered plugin
        """
        if self._ignore_event(message):
            return None, None
        args = self._parse_message(message)
        self.log.debug("Searching for command using chunks: %s", args)
        cmd, msg_args = self._find_longest_prefix_command(args)
        if cmd is not None:
            if message.user is None:
                self.log.debug("Discarded message with no originating user: %s", message)
                return None, None
            sender = message.user.username
            if message.channel is not None:
                sender = "#%s/%s" % (message.channel.name, sender)
            self.log.info("Received from %s: %s, args %s", sender, cmd, msg_args)
            f = self._get_command(cmd, message.user)
            if f:
                if self._is_channel_ignored(f, message.channel):
                    self.log.info("Channel %s is ignored, discarding command %s", message.channel, cmd)
                    return '_ignored_', ""
                return cmd, f.execute(message, msg_args)
            return '_unauthorized_', "Sorry, you are not authorized to run %s" % cmd
        return None, None

Takes a SlackEvent parses it for a command and runs against registered plugin


def _ignore_event(self, message):
        """
        message_replied event is not truly a message event and does not have a message.text
        don't process such events

        commands may not be idempotent, so ignore message_changed events.
        """
        if hasattr(message, 'subtype') and message.subtype in self.ignored_events:
            return True
        return False

def _ignore_event ( self , message ) : if hasattr ( message , 'subtype' ) and message . subtype in self . ignored_events : return True return False

message_replied event is not truly a message event and does not have a message . text don t process such events


def acl_show(self, msg, args):
        """Show current allow and deny blocks for the given acl."""
        name = args[0] if len(args) > 0 else None
        if name is None:
            return "%s: The following ACLs are defined: %s" % (msg.user, ', '.join(self._acl.keys()))

        if name not in self._acl:
            return "Sorry, couldn't find an acl named '%s'" % name

        return '\n'.join([
            "%s: ACL '%s' is defined as follows:" % (msg.user, name),
            "allow: %s" % ', '.join(self._acl[name]['allow']),
            "deny: %s" % ', '.join(self._acl[name]['deny'])
        ])

Show current allow and deny blocks for the given acl .


def add_user_to_allow(self, name, user):
        """Add a user to the given acl allow block."""

        # Clear user from both allow and deny before adding
        if not self.remove_user_from_acl(name, user):
            return False

        if name not in self._acl:
            return False

        self._acl[name]['allow'].append(user)
        return True

Add a user to the given acl allow block .


def create_acl(self, name):
        """Create a new acl."""
        if name in self._acl:
            return False

        self._acl[name] = {
            'allow': [],
            'deny': []
        }
        return True

def create_acl ( self , name ) : if name in self . _acl : return False self . _acl [ name ] = { 'allow' : [ ] , 'deny' : [ ] } return True


def delete_acl(self, name):
        """Delete an acl."""
        if name not in self._acl:
            return False

        del self._acl[name]
        return True

def delete_acl ( self , name ) : if name not in self . _acl : return False del self . _acl [ name ] return True


def mongo(daemon=False, port=20771):
    '''Run the mongod process.
    '''
    cmd = "mongod --port {0}".format(port)
    if daemon:
        cmd += " --fork"
    run(cmd)

def
mongo
(
daemon
=
False
,
port
=
20771
)
:
cmd
=
"mongod --port {0}"
.
format
(
port
)
if
daemon
:
cmd
+=
" --fork"
run
(
cmd
)


def proxy_factory(BaseSchema, label, ProxiedClass, get_key):
    """Create a proxy to a class instance stored in ``proxies``.

    :param class BaseSchema: Base schema (e.g. ``StoredObject``)
    :param str label: Name of class variable to set
    :param class ProxiedClass: Class to get or create
    :param function get_key: Extension-specific key function; may return e.g.
        the current Flask request

    """
    def local():
        key = get_key()
        try:
            return proxies[BaseSchema][label][key]
        except KeyError:
            proxies[BaseSchema][label][key] = ProxiedClass()
            return proxies[BaseSchema][label][key]
    return LocalProxy(local)

def proxy_factory ( BaseSchema , label , ProxiedClass , get_key ) : def local ( ) : key = get_key ( ) try : return proxies [ BaseSchema ] [ label ] [ key ] except KeyError : proxies [ BaseSchema ] [ label ] [ key ] = ProxiedClass ( ) return proxies [ BaseSchema ] [ label ] [ key ] return LocalProxy ( local )

Create a proxy to a class instance stored in proxies .


def with_proxies(proxy_map, get_key):
    """Class decorator factory; adds proxy class variables to target class.

    :param dict proxy_map: Mapping between class variable labels and proxied
        classes
    :param function get_key: Extension-specific key function; may return e.g.
        the current Flask request

    """
    def wrapper(cls):
        for label, ProxiedClass in six.iteritems(proxy_map):
            proxy = proxy_factory(cls, label, ProxiedClass, get_key)
            setattr(cls, label, proxy)
        return cls
    return wrapper

def with_proxies ( proxy_map , get_key ) : def wrapper ( cls ) : for label , ProxiedClass in six . iteritems ( proxy_map ) : proxy = proxy_factory ( cls , label , ProxiedClass , get_key ) setattr ( cls , label , proxy ) return cls return wrapper

Class decorator factory ; adds proxy class variables to target class .


def _to_primary_key(self, value):
        """
        Return primary key; if value is StoredObject, verify
        that it is loaded.

        """
        if value is None:
            return None
        if isinstance(value, self.base_class):
            if not value._is_loaded:
                raise exceptions.DatabaseError('Record must be loaded.')
            return value._primary_key

        return self.base_class._to_primary_key(value)

def
_to_primary_key
(
self
,
value
)
:
if
value
is
:
return
if
isinstance
(
value
,
self
.
base_class
)
:
if
not
value
.
_is_loaded
:
raise
exceptions
.
DatabaseError
(
'Record must be loaded.'
)
return
value
.
_primary_key
return
self
.
base_class
.
_to_primary_key
(
value
)

Return primary key ; if value is StoredObject verify that it is loaded .


def set_nested(data, value, *keys):
    """Assign to a nested dictionary.

    :param dict data: Dictionary to mutate
    :param value: Value to set
    :param list *keys: List of nested keys

    >>> data = {}
    >>> set_nested(data, 'hi', 'k0', 'k1', 'k2')
    >>> data
    {'k0': {'k1': {'k2': 'hi'}}}

    """
    if len(keys) == 1:
        data[keys[0]] = value
    else:
        if keys[0] not in data:
            data[keys[0]] = {}
        set_nested(data[keys[0]], value, *keys[1:])

def set_nested ( data , value , * keys ) : if len ( keys ) == 1 : data [ keys [ 0 ] ] = value else : if keys [ 0 ] not in data : data [ keys [ 0 ] ] = { } set_nested ( data [ keys [ 0 ] ] , value , * keys [ 1 : ] )


def get_by_username(self, username):
        """Retrieve user by username"""
        res = filter(lambda x: x.username == username, self.users.values())
        if len(res) > 0:
            return res[0]
        return None

def get_by_username ( self , username ) : res = filter ( lambda x : x . username == username , self . users . values ( ) ) if len ( res ) > 0 : return res [ 0 ] return None


def set(self, user):
        """
        Adds a user object to the user manager

        user - a SlackUser object
        """

        self.log.info("Loading user information for %s/%s", user.id, user.username)
        self.load_user_info(user)
        self.log.info("Loading user rights for %s/%s", user.id, user.username)
        self.load_user_rights(user)
        self.log.info("Added user: %s/%s", user.id, user.username)
        self._add_user_to_cache(user)
        return user

def
set
(
self
,
user
)
:
self
.
log
.
info
(
"Loading user information for %s/%s"
,
user
.
id
,
user
.
username
)
self
.
load_user_info
(
user
)
self
.
log
.
info
(
"Loading user rights for %s/%s"
,
user
.
id
,
user
.
username
)
self
.
load_user_rights
(
user
)
self
.
log
.
info
(
"Added user: %s/%s"
,
user
.
id
,
user
.
username
)
self
.
_add_user_to_cache
(
user
)
return
user


def load_user_rights(self, user):
        """Sets permissions on user object"""
        if user.username in self.admins:
            user.is_admin = True
        elif not hasattr(user, 'is_admin'):
            user.is_admin = False

def load_user_rights ( self , user ) : if user . username in self . admins : user . is_admin = True elif not hasattr ( user , 'is_admin' ) : user . is_admin = False


def send_message(self, channel, text):
        """
        Used to send a message to the specified channel.

        * channel - can be a channel or user
        * text - message to send
        """
        if isinstance(channel, SlackIM) or isinstance(channel, SlackUser):
            self._bot.send_im(channel, text)
        elif isinstance(channel, SlackRoom):
            self._bot.send_message(channel, text)
        elif isinstance(channel, basestring):
            if channel[0] == '@':
                self._bot.send_im(channel[1:], text)
            elif channel[0] == '#':
                self._bot.send_message(channel[1:], text)
            else:
                self._bot.send_message(channel, text)
        else:
            self._bot.send_message(channel, text)

def send_message ( self , channel , text ) : if isinstance ( channel , SlackIM ) or isinstance ( channel , SlackUser ) : self . _bot . send_im ( channel , text ) elif isinstance ( channel , SlackRoom ) : self . _bot . send_message ( channel , text ) elif isinstance ( channel , basestring ) : if channel [ 0 ] == '@' : self . _bot . send_im ( channel [ 1 : ] , text ) elif channel [ 0 ] == '#' : self . _bot . send_message ( channel [ 1 : ] , text ) else : self . _bot . send_message ( channel , text ) else : self . _bot . send_message ( channel , text )

Used to send a message to the specified channel .


def start_timer(self, duration, func, *args):
        """
        Schedules a function to be called after some period of time.

        * duration - time in seconds to wait before firing
        * func - function to be called
        * args - arguments to pass to the function
        """
        t = threading.Timer(duration, self._timer_callback, (func, args))
        self._timer_callbacks[func] = t
        t.start()
        self.log.info("Scheduled call to %s in %ds", func.__name__, duration)

def
start_timer
(
self
,
duration
,
func
,
*
args
)
:
t
=
threading
.
Timer
(
duration
,
self
.
_timer_callback
,
(
func
,
args
)
)
self
.
_timer_callbacks
[
func
]
=
t
t
.
start
(
)
self
.
log
.
info
(
"Scheduled call to %s in %ds"
,
func
.
__name__
,
duration
)

Schedules a function to be called after some period of time .


def stop_timer(self, func):
        """
        Stops a timer if it hasn't fired yet

        * func - the function passed in start_timer
        """
        if func in self._timer_callbacks:
            t = self._timer_callbacks[func]
            t.cancel()
            del self._timer_callbacks[func]

def stop_timer ( self , func ) : if func in self . _timer_callbacks : t = self . _timer_callbacks [ func ] t . cancel ( ) del self . _timer_callbacks [ func ]


def get_user(self, username):
        """
        Utility function to query slack for a particular user

        :param username: The username of the user to lookup
        :return: SlackUser object or None
        """
        if hasattr(self._bot, 'user_manager'):
            user = self._bot.user_manager.get_by_username(username)
            if user:
                return user
            user = SlackUser.get_user(self._bot.sc, username)
            self._bot.user_manager.set(user)
            return user
        return SlackUser.get_user(self._bot.sc, username)

def get_user ( self , username ) : if hasattr ( self . _bot , 'user_manager' ) : user = self . _bot . user_manager . get_by_username ( username ) if user : return user user = SlackUser . get_user ( self . _bot . sc , username ) self . _bot . user_manager . set ( user ) return user return SlackUser . get_user ( self . _bot . sc , username )


def webhook(*args, **kwargs):
    """
    Decorator to mark plugin functions as entry points for web calls

    * route - web route to register, uses Flask syntax
    * method - GET/POST, defaults to POST
    """
    def wrapper(func):
        func.is_webhook = True
        func.route = args[0]
        func.form_params = kwargs.get('form_params', [])
        func.method = kwargs.get('method', 'POST')
        return func
    return wrapper

def webhook ( * args , * * kwargs ) : def wrapper ( func ) : func . is_webhook = True func . route = args [ 0 ] func . form_params = kwargs . get ( 'form_params' , [ ] ) func . method = kwargs . get ( 'method' , 'POST' ) return func return wrapper


def freeze(value):
    """ Cast value to its frozen counterpart. """
    if isinstance(value, list):
        return FrozenList(*value)
    if isinstance(value, dict):
        return FrozenDict(**value)
    return value

def freeze ( value ) : if isinstance ( value , list ) : return FrozenList ( * value ) if isinstance ( value , dict ) : return FrozenDict ( * * value ) return value


def help(self, msg, args):
        """Displays help for each command"""
        output = []
        if len(args) == 0:
            commands = sorted(self._bot.dispatcher.commands.items(), key=itemgetter(0))
            commands = filter(lambda x: x[1].is_subcmd is False, commands)
            # Filter commands if auth is enabled, hide_admin_commands is enabled, and user is not admin
            if self._should_filter_help_commands(msg.user):
                commands = filter(lambda x: x[1].admin_only is False, commands)
            for name, cmd in commands:
                output.append(self._get_short_help_for_command(name))
        else:
            name = '!' + args[0]
            output = [self._get_help_for_command(name)]
        return '\n'.join(output)


def save(self, msg, args):
        """Causes the bot to write its current state to backend."""
        self.send_message(msg.channel, "Saving current state...")
        self._bot.plugins.save_state()
        self.send_message(msg.channel, "Done.")

def
save
(
self
,
msg
,
args
)
:
self
.
send_message
(
msg
.
channel
,
"Saving current state..."
)
self
.
_bot
.
plugins
.
save_state
(
)
self
.
send_message
(
msg
.
channel
,
"Done."
)

Causes the bot to write its current state to backend .


def shutdown(self, msg, args):
        """Causes the bot to gracefully shutdown."""
        self.log.info("Received shutdown from %s", msg.user.username)
        self._bot.runnable = False
        return "Shutting down..."

def
shutdown
(
self
,
msg
,
args
)
:
self
.
log
.
info
(
"Received shutdown from %s"
,
msg
.
user
.
username
)
self
.
_bot
.
runnable
=
False
return
"Shutting down..."


def whoami(self, msg, args):
        """Prints information about the user and bot version."""
        output = ["Hello %s" % msg.user]
        if hasattr(self._bot.dispatcher, 'auth_manager') and msg.user.is_admin is True:
            output.append("You are a *bot admin*.")
        output.append("Bot version: %s-%s" % (self._bot.version, self._bot.commit))
        return '\n'.join(output)

def
whoami
(
self
,
msg
,
args
)
:
output
=
[
"Hello %s"
%
msg
.
user
]
if
hasattr
(
self
.
_bot
.
dispatcher
,
'auth_manager'
)
and
msg
.
user
.
is_admin
is
True
:
output
.
append
(
"You are a *bot admin*."
)
output
.
append
(
"Bot version: %s-%s"
%
(
self
.
_bot
.
version
,
self
.
_bot
.
commit
)
)
return
'\n'
.
join
(
output
)

Prints information about the user and bot version .


def sleep(self, channel):
        """Causes the bot to ignore all messages from the channel.

        Usage:
        !sleep [channel name] - ignore the specified channel (or current if none specified)
        """
        self.log.info('Sleeping in %s', channel)
        self._bot.dispatcher.ignore(channel)
        self.send_message(channel, 'Good night')

def
sleep
(
self
,
channel
)
:
self
.
log
.
info
(
'Sleeping in %s'
,
channel
)
self
.
_bot
.
dispatcher
.
ignore
(
channel
)
self
.
send_message
(
channel
,
'Good night'
)

Causes the bot to ignore all messages from the channel .


def wake(self, channel):
        """Causes the bot to resume operation in the channel.

        Usage:
        !wake [channel name] - unignore the specified channel (or current if none specified)
        """
        self.log.info('Waking up in %s', channel)
        self._bot.dispatcher.unignore(channel)
        self.send_message(channel, 'Hello, how may I be of service?')

def
wake
(
self
,
channel
)
:
self
.
log
.
info
(
'Waking up in %s'
,
channel
)
self
.
_bot
.
dispatcher
.
unignore
(
channel
)
self
.
send_message
(
channel
,
'Hello, how may I be of service?'
)

Causes the bot to resume operation in the channel .


def _sort_by(key):
    """
    High order function for sort methods.
    """

    @staticmethod
    def sort_by(p_list, reverse=False):
        return sorted(
            p_list,
            key=lambda p: getattr(p, key),
            reverse=reverse,
        )

    return sort_by

def _sort_by ( key ) : @ staticmethod def sort_by ( p_list , reverse = False ) : return sorted ( p_list , key = lambda p : getattr ( p , key ) , reverse = reverse , ) return sort_by


def select(self, filters=all_true, recursive=True):
        """Select path by criterion.

        :param filters: a lambda function that take a `pathlib.Path` as input,
          boolean as a output.
        :param recursive: include files in subfolder or not.

        **中文文档**

        根据filters中定义的条件选择路径。
        """
        self.assert_is_dir_and_exists()

        if recursive:
            for p in self.glob("**/*"):
                if filters(p):
                    yield p
        else:
            for p in self.iterdir():
                if filters(p):
                    yield p

def select ( self , filters = all_true , recursive = True ) : self . assert_is_dir_and_exists ( ) if recursive : for p in self . glob ( "**/*" ) : if filters ( p ) : yield p else : for p in self . iterdir ( ) : if filters ( p ) : yield p


def select_file(self, filters=all_true, recursive=True):
        """Select file path by criterion.

        **中文文档**

        根据filters中定义的条件选择文件。
        """
        for p in self.select(filters, recursive):
            if p.is_file():
                yield p

def select_file ( self , filters = all_true , recursive = True ) : for p in self . select ( filters , recursive ) : if p . is_file ( ) : yield p


def select_dir(self, filters=all_true, recursive=True):
        """Select dir path by criterion.

        **中文文档**

        根据filters中定义的条件选择文件夹。
        """
        for p in self.select(filters, recursive):
            if p.is_dir():
                yield p

def select_dir ( self , filters = all_true , recursive = True ) : for p in self . select ( filters , recursive ) : if p . is_dir ( ) : yield p


def n_file(self):
        """
        Count how many files in this directory. Including file in sub folder.
        """
        self.assert_is_dir_and_exists()
        n = 0
        for _ in self.select_file(recursive=True):
            n += 1
        return n

def n_file ( self ) : self . assert_is_dir_and_exists ( ) n = 0 for _ in self . select_file ( recursive = True ) : n += 1 return n

Count how many files in this directory . Including file in sub folder .


def n_dir(self):
        """
        Count how many folders in this directory. Including folder in sub folder.
        """
        self.assert_is_dir_and_exists()
        n = 0
        for _ in self.select_dir(recursive=True):
            n += 1
        return n

def n_dir ( self ) : self . assert_is_dir_and_exists ( ) n = 0 for _ in self . select_dir ( recursive = True ) : n += 1 return n

Count how many folders in this directory . Including folder in sub folder .


def select_by_ext(self, ext, recursive=True):
        """
        Select file path by extension.

        :param ext:

        **中文文档**

        选择与预定义的若干个扩展名匹配的文件。
        """
        ext = [ext.strip().lower() for ext in ensure_list(ext)]

        def filters(p): return p.suffix.lower() in ext

        return self.select_file(filters, recursive)

def select_by_ext ( self , ext , recursive = True ) : ext = [ ext . strip ( ) . lower ( ) for ext in ensure_list ( ext ) ] def filters ( p ) : return p . suffix . lower ( ) in ext return self . select_file ( filters , recursive )


def select_by_pattern_in_fname(self,
                                   pattern,
                                   recursive=True,
                                   case_sensitive=False):
        """
        Select file path by text pattern in file name.

        **中文文档**

        选择文件名中包含指定子字符串的文件。
        """
        if case_sensitive:
            def filters(p):
                return pattern in p.fname
        else:
            pattern = pattern.lower()

            def filters(p):
                return pattern in p.fname.lower()

        return self.select_file(filters, recursive)

def select_by_pattern_in_fname ( self , pattern , recursive = True , case_sensitive = False ) : if case_sensitive : def filters ( p ) : return pattern in p . fname else : pattern = pattern . lower ( ) def filters ( p ) : return pattern in p . fname . lower ( ) return self . select_file ( filters , recursive )

Select file path by text pattern in file name .


def select_by_pattern_in_abspath(self,
                                     pattern,
                                     recursive=True,
                                     case_sensitive=False):
        """
        Select file path by text pattern in absolute path.

        **中文文档**

        选择绝对路径中包含指定子字符串的文件。
        """
        if case_sensitive:
            def filters(p):
                return pattern in p.abspath
        else:
            pattern = pattern.lower()

            def filters(p):
                return pattern in p.abspath.lower()

        return self.select_file(filters, recursive)

def select_by_pattern_in_abspath ( self , pattern , recursive = True , case_sensitive = False ) : if case_sensitive : def filters ( p ) : return pattern in p . abspath else : pattern = pattern . lower ( ) def filters ( p ) : return pattern in p . abspath . lower ( ) return self . select_file ( filters , recursive )

Select file path by text pattern in absolute path .


def select_by_size(self, min_size=0, max_size=1 << 40, recursive=True):
        """
        Select file path by size.

        **中文文档**

        选择所有文件大小在一定范围内的文件。
        """

        def filters(p): return min_size <= p.size <= max_size

        return self.select_file(filters, recursive)

def select_by_size ( self , min_size = 0 , max_size = 1 << 40 , recursive = True ) : def filters ( p ) : return min_size <= p . size <= max_size return self . select_file ( filters , recursive )


def select_by_mtime(self, min_time=0, max_time=ts_2100,
                        recursive=True):
        """
        Select file path by modify time.

        :param min_time: lower bound timestamp
        :param max_time: upper bound timestamp

        **中文文档**

        选择所有 :attr:`pathlib_mate.pathlib2.Path.mtime` 在一定范围内的文件。
        """

        def filters(p): return min_time <= p.mtime <= max_time

        return self.select_file(filters, recursive)

def select_by_mtime ( self , min_time = 0 , max_time = ts_2100 , recursive = True ) : def filters ( p ) : return min_time <= p . mtime <= max_time return self . select_file ( filters , recursive )


def select_by_atime(self, min_time=0, max_time=ts_2100, recursive=True):
        """
        Select file path by access time.

        :param min_time: lower bound timestamp
        :param max_time: upper bound timestamp

        **中文文档**

        选择所有 :attr:`pathlib_mate.pathlib2.Path.atime` 在一定范围内的文件。
        """

        def filters(p): return min_time <= p.atime <= max_time

        return self.select_file(filters, recursive)

def select_by_atime ( self , min_time = 0 , max_time = ts_2100 , recursive = True ) : def filters ( p ) : return min_time <= p . atime <= max_time return self . select_file ( filters , recursive )


def select_by_ctime(self, min_time=0, max_time=ts_2100,
                        recursive=True):
        """
        Select file path by create time.

        :param min_time: lower bound timestamp
        :param max_time: upper bound timestamp

        **中文文档**

        选择所有 :attr:`pathlib_mate.pathlib2.Path.ctime` 在一定范围内的文件。
        """

        def filters(p): return min_time <= p.ctime <= max_time

        return self.select_file(filters, recursive)

def select_by_ctime ( self , min_time = 0 , max_time = ts_2100 , recursive = True ) : def filters ( p ) : return min_time <= p . ctime <= max_time return self . select_file ( filters , recursive )


def make_zip_archive(self,
                         dst=None,
                         filters=all_true,
                         compress=True,
                         overwrite=False,
                         makedirs=False,
                         verbose=False):  # pragma: no cover
        """
        Make a zip archive.

        :param dst: output file path. if not given, will be automatically assigned.
        :param filters: custom path filter. By default it allows any file.
        :param compress: compress or not.
        :param overwrite: overwrite exists or not.
        :param verbose: display log or not.
        :return:
        """
        self.assert_exists()

        if dst is None:
            dst = self._auto_zip_archive_dst()
        else:
            dst = self.change(new_abspath=dst)

        if not dst.basename.lower().endswith(".zip"):
            raise ValueError("zip archive name has to be endswith '.zip'!")

        if dst.exists():
            if not overwrite:
                raise IOError("'%s' already exists!" % dst)

        if compress:
            compression = ZIP_DEFLATED
        else:
            compression = ZIP_STORED

        if not dst.parent.exists():
            if makedirs:
                os.makedirs(dst.parent.abspath)

        if verbose:
            msg = "Making zip archive for '%s' ..." % self
            print(msg)

        current_dir = os.getcwd()

        if self.is_dir():
            total_size = 0
            selected = list()
            for p in self.glob("**/*"):
                if filters(p):
                    selected.append(p)
                    total_size += p.size

            if verbose:
                msg = "Got {} files, total size is {}, compressing ...".format(
                    len(selected), repr_data_size(total_size),
                )
                print(msg)

            with ZipFile(dst.abspath, "w", compression) as f:
                os.chdir(self.abspath)
                for p in selected:
                    relpath = p.relative_to(self).__str__()
                    f.write(relpath)

        elif self.is_file():
            with ZipFile(dst.abspath, "w", compression) as f:
                os.chdir(self.parent.abspath)
                f.write(self.basename)

        os.chdir(current_dir)

        if verbose:
            msg = "Complete! Archive size is {}.".format(dst.size_in_text)
            print(msg)


def backup(self,
               dst=None,
               ignore=None,
               ignore_ext=None,
               ignore_pattern=None,
               ignore_size_smaller_than=None,
               ignore_size_larger_than=None,
               case_sensitive=False):  # pragma: no cover
        """
        Create a compressed zip archive backup for a directory.

        :param dst: the output file path.
        :param ignore: file or directory defined in this list will be ignored.
        :param ignore_ext: file with extensions defined in this list will be ignored.
        :param ignore_pattern: any file or directory that contains this pattern
            will be ignored.
        :param ignore_size_smaller_than: any file size smaller than this
            will be ignored.
        :param ignore_size_larger_than: any file size larger than this
            will be ignored.

        **中文文档**

        为一个目录创建一个备份压缩包。可以通过过滤器选择你要备份的文件。
        """

        def preprocess_arg(arg):  # pragma: no cover
            if arg is None:
                return []

            if isinstance(arg, (tuple, list)):
                return list(arg)
            else:
                return [arg, ]

        self.assert_is_dir_and_exists()

        ignore = preprocess_arg(ignore)
        for i in ignore:
            if i.startswith("/") or i.startswith("\\"):
                raise ValueError

        ignore_ext = preprocess_arg(ignore_ext)
        for ext in ignore_ext:
            if not ext.startswith("."):
                raise ValueError

        ignore_pattern = preprocess_arg(ignore_pattern)

        if case_sensitive:
            pass
        else:
            ignore = [i.lower() for i in ignore]
            ignore_ext = [i.lower() for i in ignore_ext]
            ignore_pattern = [i.lower() for i in ignore_pattern]

        def filters(p):
            relpath = p.relative_to(self).abspath
            if not case_sensitive:
                relpath = relpath.lower()

            # ignore
            for i in ignore:
                if relpath.startswith(i):
                    return False

            # ignore_ext
            if case_sensitive:
                ext = p.ext
            else:
                ext = p.ext.lower()

            if ext in ignore_ext:
                return False

            # ignore_pattern
            for pattern in ignore_pattern:
                if pattern in relpath:
                    return False

            # ignore_size_smaller_than
            if ignore_size_smaller_than:
                if p.size < ignore_size_smaller_than:
                    return False

            # ignore_size_larger_than
            if ignore_size_larger_than:
                if p.size > ignore_size_larger_than:
                    return False

            return True

        self.make_zip_archive(
            dst=dst, filters=filters, compress=True, overwrite=False, verbose=True,
        )

def
backup
(
self
,
dst
=
,
ignore
=
,
ignore_ext
=
,
ignore_pattern
=
,
ignore_size_smaller_than
=
,
ignore_size_larger_than
=
,
case_sensitive
=
False
)
:
# pragma: no cover
def
preprocess_arg
(
arg
)
:
# pragma: no cover
if
arg
is
:
return
[
]
if
isinstance
(
arg
,
(
tuple
,
list
)
)
:
return
list
(
arg
)
else
:
return
[
arg
,
]
self
.
assert_is_dir_and_exists
(
)
ignore
=
preprocess_arg
(
ignore
)
for
i
in
ignore
:
if
i
.
startswith
(
"/"
)
or
i
.
startswith
(
"\\"
)
:
raise
ValueError
ignore_ext
=
preprocess_arg
(
ignore_ext
)
for
ext
in
ignore_ext
:
if
not
ext
.
startswith
(
"."
)
:
raise
ValueError
ignore_pattern
=
preprocess_arg
(
ignore_pattern
)
if
case_sensitive
:
pass
else
:
ignore
=
[
i
.
lower
(
)
for
i
in
ignore
]
ignore_ext
=
[
i
.
lower
(
)
for
i
in
ignore_ext
]
ignore_pattern
=
[
i
.
lower
(
)
for
i
in
ignore_pattern
]
def
filters
(
p
)
:
relpath
=
p
.
relative_to
(
self
)
.
abspath
if
not
case_sensitive
:
relpath
=
relpath
.
lower
(
)
# ignore
for
i
in
ignore
:
if
relpath
.
startswith
(
i
)
:
return
False
# ignore_ext
if
case_sensitive
:
ext
=
p
.
ext
else
:
ext
=
p
.
ext
.
lower
(
)
if
ext
in
ignore_ext
:
return
False
# ignore_pattern
for
pattern
in
ignore_pattern
:
if
pattern
in
relpath
:
return
False
# ignore_size_smaller_than
if
ignore_size_smaller_than
:
if
p
.
size
<
ignore_size_smaller_than
:
return
False
# ignore_size_larger_than
if
ignore_size_larger_than
:
if
p
.
size
>
ignore_size_larger_than
:
return
False
return
True
self
.
make_zip_archive
(
dst
=
dst
,
filters
=
filters
,
compress
=
True
,
overwrite
=
False
,
verbose
=
True
,
)

Create a compressed zip archive backup for a directory .


def acquire_lock(func):
    """Decorate methods when locking repository is required."""
    @wraps(func)
    def wrapper(self, *args, **kwargs):
        with self.locker as r:
            # get the result
            acquired, code, _  = r
            if acquired:
                try:
                    r = func(self, *args, **kwargs)
                except Exception as err:
                    e = str(err)
                else:
                    e = None
            else:
                warnings.warn("code %s. Unable to aquire the lock when calling '%s'. You may try again!"%(code,func.__name__) )
                e = None
                r = None
        # raise error after exiting with statement and releasing the lock!
        if e is not None:
            traceback.print_stack()
            raise Exception(e)
        return r
    return wrapper

Decorate methods when locking repository is required .


def sync_required(func):
    """Decorate methods when synchronizing repository is required."""
    @wraps(func)
    def wrapper(self, *args, **kwargs):
        if not self._keepSynchronized:
            r = func(self, *args, **kwargs)
        else:
            state = self._load_state()
            #print("----------->  ",state, self.state)
            if state is None:
                r = func(self, *args, **kwargs)
            elif state == self.state:
                r = func(self, *args, **kwargs)
            else:
                warnings.warn("Repository at '%s' is out of date. Need to load it again to avoid conflict."%self.path)
                r = None
        return r
    return wrapper

Decorate methods when synchronizing repository is required .


def get_pickling_errors(obj, seen=None):
    """Investigate pickling errors."""
    if seen == None:
        seen = []
    if hasattr(obj, "__getstate__"):
        state = obj.__getstate__()
    #elif hasattr(obj, "__dict__"):
    #    state = obj.__dict__
    else:
        return None
    #try:
    #    state = obj.__getstate__()
    #except AttributeError as e:
    #    #state = obj.__dict__
    #    return str(e)
    if state == None:
        return 'object state is None'
    if isinstance(state,tuple):
        if not isinstance(state[0], dict):
            state=state[1]
        else:
            state=state[0].update(state[1])
    result = {}
    for i in state:
        try:
            pickle.dumps(state[i], protocol=2)
        except pickle.PicklingError as e:
            if not state[i] in seen:
                seen.append(state[i])
                result[i]=get_pickling_errors(state[i],seen)
    return result

def
get_pickling_errors
(
obj
,
seen
=
)
:
if
seen
==
:
seen
=
[
]
if
hasattr
(
obj
,
"__getstate__"
)
:
state
=
obj
.
__getstate__
(
)
#elif hasattr(obj, "__dict__"):
#    state = obj.__dict__
else
:
return
#try:
#    state = obj.__getstate__()
#except AttributeError as e:
#    #state = obj.__dict__
#    return str(e)
if
state
==
:
return
'object state is None'
if
isinstance
(
state
,
tuple
)
:
if
not
isinstance
(
state
[
0
]
,
dict
)
:
state
=
state
[
1
]
else
:
state
=
state
[
0
]
.
update
(
state
[
1
]
)
result
=
{
}
for
i
in
state
:
try
:
pickle
.
dumps
(
state
[
i
]
,
protocol
=
2
)
except
pickle
.
PicklingError
as
e
:
if
not
state
[
i
]
in
seen
:
seen
.
append
(
state
[
i
]
)
result
[
i
]
=
get_pickling_errors
(
state
[
i
]
,
seen
)
return
result


def walk_files_relative_path(self, relativePath=""):
        """
        Walk the repository and yield all found files relative path joined with file name.

        :parameters:
            #. relativePath (str): The relative path from which start the walk.
        """
        def walk_files(directory, relativePath):
            directories = dict.__getitem__(directory, 'directories')
            files       = dict.__getitem__(directory, 'files')
            for f in sorted(files):
                yield os.path.join(relativePath, f)
            for k in sorted(dict.keys(directories)):
                path = os.path.join(relativePath, k)
                dir  = directories.__getitem__(k)
                for e in walk_files(dir, path):
                    yield e
        dir, errorMessage = self.get_directory_info(relativePath)
        assert dir is not None, errorMessage
        return walk_files(dir, relativePath='')

def walk_files_relative_path ( self , relativePath = "" ) : def walk_files ( directory , relativePath ) : directories = dict . __getitem__ ( directory , 'directories' ) files = dict . __getitem__ ( directory , 'files' ) for f in sorted ( files ) : yield os . path . join ( relativePath , f ) for k in sorted ( dict . keys ( directories ) ) : path = os . path . join ( relativePath , k ) dir = directories . __getitem__ ( k ) for e in walk_files ( dir , path ) : yield e dir , errorMessage = self . get_directory_info ( relativePath ) assert dir is not None , errorMessage return walk_files ( dir , relativePath = '' )

Walk the repository and yield all found files relative path joined with file name .


def walk_directories_relative_path(self, relativePath=""):
        """
        Walk repository and yield all found directories relative path

        :parameters:
            #. relativePath (str): The relative path from which start the walk.
        """
        def walk_directories(directory, relativePath):
            directories = dict.__getitem__(directory, 'directories')
            dirNames = dict.keys(directories)
            for d in sorted(dirNames):
                yield os.path.join(relativePath, d)
            for k in sorted(dict.keys(directories)):
                path = os.path.join(relativePath, k)
                dir  = dict.__getitem__(directories, k)
                for e in walk_directories(dir, path):
                    yield e
        dir, errorMessage = self.get_directory_info(relativePath)
        assert dir is not None, errorMessage
        return walk_directories(dir, relativePath='')

def walk_directories_relative_path ( self , relativePath = "" ) : def walk_directories ( directory , relativePath ) : directories = dict . __getitem__ ( directory , 'directories' ) dirNames = dict . keys ( directories ) for d in sorted ( dirNames ) : yield os . path . join ( relativePath , d ) for k in sorted ( dict . keys ( directories ) ) : path = os . path . join ( relativePath , k ) dir = dict . __getitem__ ( directories , k ) for e in walk_directories ( dir , path ) : yield e dir , errorMessage = self . get_directory_info ( relativePath ) assert dir is not None , errorMessage return walk_directories ( dir , relativePath = '' )


def walk_directories_info(self, relativePath=""):
        """
        Walk repository and yield all found directories relative path.

        :parameters:
            #. relativePath (str): The relative path from which start the walk.
        """
        def walk_directories(directory, relativePath):
            directories = dict.__getitem__(directory, 'directories')
            for fname in sorted(directories):
                info = dict.__getitem__(directories,fname)
                yield os.path.join(relativePath, fname), info
            for k in sorted(dict.keys(directories)):
                path = os.path.join(relativePath, k)
                dir  = dict.__getitem__(directories, k)
                for e in walk_directories(dir, path):
                    yield e
        dir, errorMessage = self.get_directory_info(relativePath)
        assert dir is not None, errorMessage
        return walk_directories(dir, relativePath='')

def walk_directories_info ( self , relativePath = "" ) : def walk_directories ( directory , relativePath ) : directories = dict . __getitem__ ( directory , 'directories' ) for fname in sorted ( directories ) : info = dict . __getitem__ ( directories , fname ) yield os . path . join ( relativePath , fname ) , info for k in sorted ( dict . keys ( directories ) ) : path = os . path . join ( relativePath , k ) dir = dict . __getitem__ ( directories , k ) for e in walk_directories ( dir , path ) : yield e dir , errorMessage = self . get_directory_info ( relativePath ) assert dir is not None , errorMessage return walk_directories ( dir , relativePath = '' )

Walk repository and yield all found directories relative path .


def walk_directory_directories_relative_path(self, relativePath=""):
        """
        Walk a certain directory in repository and yield all found directories relative path.

        :parameters:
            #. relativePath (str): The relative path of the directory.
        """
        # get directory info dict
        errorMessage = ""
        relativePath = os.path.normpath(relativePath)
        dirInfoDict, errorMessage = self.get_directory_info(relativePath)
        assert dirInfoDict is not None, errorMessage
        for dname in dict.__getitem__(dirInfoDict, "directories"):
            yield os.path.join(relativePath, dname)

Walk a certain directory in repository and yield all found directories relative path .


def synchronize(self, verbose=False):
        """
        Synchronizes the Repository information with the directory.
        All registered but missing files and directories in the directory,
        will be automatically removed from the Repository.

        :parameters:
            #. verbose (boolean): Whether to be warn and inform about any abnormalities.
        """
        if self.__path is None:
            return
        # walk directories
        for dirPath in sorted(list(self.walk_directories_relative_path())):
            realPath = os.path.join(self.__path, dirPath)
            # if directory exist
            if os.path.isdir(realPath):
                continue
            if verbose: warnings.warn("%s directory is missing"%realPath)
            # loop to get dirInfoDict
            keys = dirPath.split(os.sep)
            dirInfoDict = self
            for idx in range(len(keys)-1):
                dirs = dict.get(dirInfoDict, 'directories', None)
                if dirs is None: break
                dirInfoDict = dict.get(dirs, keys[idx], None)
                if dirInfoDict is None: break
            # remove dirInfoDict directory if existing
            if dirInfoDict is not None:
                dirs = dict.get(dirInfoDict, 'directories', None)
                if dirs is not None:
                    dict.pop( dirs, keys[-1], None )
        # walk files
        for filePath in sorted(list(self.walk_files_relative_path())):
            realPath = os.path.join(self.__path, filePath)
            # if file exists
            if os.path.isfile( realPath ):
                continue
            if verbose: warnings.warn("%s file is missing"%realPath)
            # loop to get dirInfoDict
            keys = filePath.split(os.sep)
            dirInfoDict = self
            for idx in range(len(keys)-1):
                dirs = dict.get(dirInfoDict, 'directories', None)
                if dirs is None: break
                dirInfoDict = dict.get(dirs, keys[idx], None)
                if dirInfoDict is None: break
            # remove dirInfoDict file if existing
            if dirInfoDict is not None:
                files = dict.get(dirInfoDict, 'files', None)
                if files is not None:
                    dict.pop( files, keys[-1], None )

Synchronizes the Repository information with the directory . All registered but missing files and directories in the directory will be automatically removed from the Repository .


def load_repository(self, path):
        """
        Load repository from a directory path and update the current instance.

        :Parameters:
            #. path (string): The path of the directory from where to load the repository.
               If '.' or an empty string is passed, the current working directory will be used.

        :Returns:
             #. repository (pyrep.Repository): returns self repository with loaded data.
        """
        # try to open
        if path.strip() in ('','.'):
            path = os.getcwd()
        repoPath = os.path.realpath( os.path.expanduser(path) )
        if not self.is_repository(repoPath):
            raise Exception("no repository found in '%s'"%str(repoPath))
        # get pyrepinfo path
        repoInfoPath = os.path.join(repoPath, ".pyrepinfo")
        try:
            fd = open(repoInfoPath, 'rb')
        except Exception as e:
            raise Exception("unable to open repository file(%s)"%e)
        # before doing anything try to lock repository
        # can't decorate with @acquire_lock because this will point to old repository
        # path or to current working directory which might not be the path anyways
        L =  Locker(filePath=None, lockPass=str(uuid.uuid1()), lockPath=os.path.join(repoPath, ".pyreplock"))
        acquired, code = L.acquire_lock()
        # check if acquired.
        if not acquired:
            warnings.warn("code %s. Unable to aquire the lock when calling 'load_repository'. You may try again!"%(code,) )
            return
        try:
            # unpickle file
            try:
                repo = pickle.load( fd )
            except Exception as e:
                fd.close()
                raise Exception("unable to pickle load repository (%s)"%e)
            finally:
                fd.close()
            # check if it's a PyrepInfo instance
            if not isinstance(repo, Repository):
                raise Exception(".pyrepinfo in '%s' is not a repository instance."%s)
            else:
                # update info path
                self.__reset_repository()
                self.__update_repository(repo)
                self.__path = repoPath
            # set timestamp
            self.__state = self._get_or_create_state()
        except Exception as e:
            L.release_lock()
            raise Exception(e)
        finally:
            L.release_lock()
        # set loaded repo locker path to L because repository have been moved to another directory
        self.__locker = L
        # return
        return self

Load repository from a directory path and update the current instance .


def get_repository(self, path, info=None, verbose=True):
        """
        Create a repository at given real path or load any existing one.
        This method insures the creation of the directory in the system if it is missing.\n
        Unlike create_repository, this method doesn't erase any existing repository
        in the path but loads it instead.

        **N.B. On some systems and some paths, creating a directory may requires root permissions.**

        :Parameters:
            #. path (string): The real absolute path where to create the Repository.
               If '.' or an empty string is passed, the current working directory will be used.
            #. info (None, object): Any information that can identify the repository.
            #. verbose (boolean): Whether to be warn and informed about any abnormalities.
        """
        # get real path
        if path.strip() in ('','.'):
            path = os.getcwd()
        realPath = os.path.realpath( os.path.expanduser(path) )
        # create directory if not existing
        if not os.path.isdir(realPath):
            os.makedirs(realPath)
        # create Repository
        if not self.is_repository(realPath):
            self.create_repository(realPath, info=info, verbose=verbose)
        else:
            self.load_repository(realPath)

def
get_repository
(
self
,
path
,
info
=
,
verbose
=
True
)
:
# get real path
if
path
.
strip
(
)
in
(
''
,
'.'
)
:
path
=
os
.
getcwd
(
)
realPath
=
os
.
path
.
realpath
(
os
.
path
.
expanduser
(
path
)
)
# create directory if not existing
if
not
os
.
path
.
isdir
(
realPath
)
:
os
.
makedirs
(
realPath
)
# create Repository
if
not
self
.
is_repository
(
realPath
)
:
self
.
create_repository
(
realPath
,
info
=
info
,
verbose
=
verbose
)
else
:
self
.
load_repository
(
realPath
)

Create a repository at given real path or load any existing one . This method insures the creation of the directory in the system if it is missing . \ n Unlike create_repository this method doesn t erase any existing repository in the path but loads it instead .


def remove_repository(self, path=None, relatedFiles=False, relatedFolders=False, verbose=True):
        """
        Remove .pyrepinfo file from path if exists and related files and directories
        when respective flags are set to True.

        :Parameters:
            #. path (None, string): The path of the directory where to remove an existing repository.
               If None, current repository is removed if initialized.
            #. relatedFiles (boolean): Whether to also remove all related files from system as well.
            #. relatedFolders (boolean): Whether to also remove all related directories from system as well.
               Directories will be removed only if they are left empty after removing the files.
            #. verbose (boolean): Whether to be warn and informed about any abnormalities.
        """
        if path is not None:
            realPath = os.path.realpath( os.path.expanduser(path) )
        else:
            realPath = self.__path
        if realPath is None:
            if verbose: warnings.warn('path is None and current Repository is not initialized!')
            return
        if not self.is_repository(realPath):
            if verbose: warnings.warn("No repository found in '%s'!"%realPath)
            return
        # check for security
        if realPath == os.path.realpath('/..') :
            if verbose: warnings.warn('You are about to wipe out your system !!! action aboarded')
            return
        # get repo
        if path is not None:
            repo = Repository()
            repo.load_repository(realPath)
        else:
            repo = self
        # delete files
        if relatedFiles:
            for relativePath in repo.walk_files_relative_path():
                realPath = os.path.join(repo.path, relativePath)
                if not os.path.isfile(realPath):
                    continue
                if not os.path.exists(realPath):
                    continue
                os.remove( realPath )
        # delete directories
        if relatedFolders:
            for relativePath in reversed(list(repo.walk_directories_relative_path())):
                realPath = os.path.join(repo.path, relativePath)
                # protect from wiping out the system
                if not os.path.isdir(realPath):
                    continue
                if not os.path.exists(realPath):
                    continue
                if not len(os.listdir(realPath)):
                    os.rmdir( realPath )
        # delete repository
        os.remove( os.path.join(repo.path, ".pyrepinfo" ) )
        for fname in (".pyrepstate", ".pyreplock"):
            p = os.path.join(repo.path, fname )
            if os.path.exists( p ):
                os.remove( p )
        # remove main directory if empty
        if os.path.isdir(repo.path):
            if not len(os.listdir(repo.path)):
                os.rmdir( repo.path )
        # reset repository
        repo.__reset_repository()

Remove . pyrepinfo file from path if exists and related files and directories when respective flags are set to True .


def save(self):
        """ Save repository .pyrepinfo to disk. """
        # open file
        repoInfoPath = os.path.join(self.__path, ".pyrepinfo")
        try:
            fdinfo = open(repoInfoPath, 'wb')
        except Exception as e:
            raise Exception("unable to open repository info for saving (%s)"%e)
        # save repository
        try:
            pickle.dump( self, fdinfo, protocol=2 )
        except Exception as e:
            fdinfo.flush()
            os.fsync(fdinfo.fileno())
            fdinfo.close()
            raise Exception( "Unable to save repository info (%s)"%e )
        finally:
            fdinfo.flush()
            os.fsync(fdinfo.fileno())
            fdinfo.close()
        # save timestamp
        repoTimePath = os.path.join(self.__path, ".pyrepstate")
        try:
            self.__state = ("%.6f"%time.time()).encode()
            with open(repoTimePath, 'wb') as fdtime:
                fdtime.write( self.__state )
                fdtime.flush()
                os.fsync(fdtime.fileno())
        except Exception as e:
            raise Exception("unable to open repository time stamp for saving (%s)"%e)


def create_package(self, path=None, name=None, mode=None):
        """
        Create a tar file package of all the repository files and directories.
        Only files and directories that are stored in the repository info
        are stored in the package tar file.

        **N.B. On some systems packaging requires root permissions.**

        :Parameters:
            #. path (None, string): The real absolute path where to create the package.
               If None, it will be created in the same directory as the repository
               If '.' or an empty string is passed, the current working directory will be used.
            #. name (None, string): The name to give to the package file
               If None, the package directory name will be used with the appropriate extension added.
            #. mode (None, string): The writing mode of the tarfile.
               If None, automatically the best compression mode will be chose.
               Available modes are ('w', 'w:', 'w:gz', 'w:bz2')
        """
        # check mode
        assert mode in (None, 'w', 'w:', 'w:gz', 'w:bz2'), 'unkown archive mode %s'%str(mode)
        if mode is None:
            mode = 'w:bz2'
            mode = 'w:'
        # get root
        if path is None:
            root = os.path.split(self.__path)[0]
        elif path.strip() in ('','.'):
            root = os.getcwd()
        else:
            root = os.path.realpath( os.path.expanduser(path) )
        assert os.path.isdir(root), 'absolute path %s is not a valid directory'%path
        # get name
        if name is None:
            ext = mode.split(":")
            if len(ext) == 2:
                if len(ext[1]):
                    ext = "."+ext[1]
                else:
                    ext = '.tar'
            else:
                ext = '.tar'
            name = os.path.split(self.__path)[1]+ext
        # save repository
        self.save()
        # create tar file
        tarfilePath = os.path.join(root, name)
        try:
            tarHandler = tarfile.TarFile.open(tarfilePath, mode=mode)
        except Exception as e:
            raise Exception("Unable to create package (%s)"%e)
        # walk directory and create empty directories
        for directory in sorted(list(self.walk_directories_relative_path())):
            t = tarfile.TarInfo( directory )
            t.type = tarfile.DIRTYPE
            tarHandler.addfile(t)
        # walk files and add to tar
        for file in self.walk_files_relative_path():
            tarHandler.add(os.path.join(self.__path,file), arcname=file)
        # save repository .pyrepinfo
        tarHandler.add(os.path.join(self.__path,".pyrepinfo"), arcname=".pyrepinfo")
        # close tar file
        tarHandler.close()

Create a tar file package of all the repository files and directories . Only files and directories that are stored in the repository info are stored in the package tar file .


def get_directory_info(self, relativePath):
        """
        get directory info from the Repository.

        :Parameters:
            #. relativePath (string): The relative to the repository path of the directory.

        :Returns:
            #. info (None, dictionary): The directory information dictionary.
               If None, it means an error has occurred.
            #. error (string): The error message if any error occurred.
        """
        relativePath = os.path.normpath(relativePath)
        # if root directory
        if relativePath in ('','.'):
            return self, ""
        currentDir  = self.__path
        dirInfoDict = self
        for dir in relativePath.split(os.sep):
            dirInfoDict = dict.__getitem__(dirInfoDict, "directories")
            currentDir = os.path.join(currentDir, dir)
            # check if path exists
            if not os.path.exists(currentDir):
                return None,  "directory '%s' is not found"%currentDir
            val = dirInfoDict.get(dir, None)
            # check if directory is registered in repository
            if val is None:
                return None,  "directory '%s' is not registered in PyrepInfo"%currentDir
            dirInfoDict = val
        return dirInfoDict, ""

get directory info from the Repository .


def get_parent_directory_info(self, relativePath):
        """
        get parent directory info of a file or directory from the Repository.

        :Parameters:
            #. relativePath (string): The relative to the repository path of the file or directory of which the parent directory info is requested.

        :Returns:
            #. info (None, dictionary): The directory information dictionary.
               If None, it means an error has occurred.
            #. error (string): The error message if any error occurred.
        """
        relativePath = os.path.normpath(relativePath)
        # if root directory
        if relativePath in ('','.'):
            return self, "relativePath is empty pointing to the repostitory itself."
        # split path
        parentDirPath, _ = os.path.split(relativePath)
        # get parent directory info
        return self.get_directory_info(parentDirPath)

get parent directory info of a file or directory from the Repository .


def get_file_info(self, relativePath, name=None):
        """
        get file information dict from the repository given its relative path and name.

        :Parameters:
            #. relativePath (string): The relative to the repository path of the directory where the file is.
            #. name (string): The file name.
               If None is given, name will be split from relativePath.

        :Returns:
            #. info (None, dictionary): The file information dictionary.
               If None, it means an error has occurred.
            #. errorMessage (string): The error message if any error occurred.
        """
        # normalize relative path and name
        relativePath = os.path.normpath(relativePath)
        if relativePath == '.':
            relativePath = ''
            assert name != '.pyrepinfo', "'.pyrepinfo' can't be a file name."
        if name is None:
            assert len(relativePath), "name must be given when relative path is given as empty string or as a simple dot '.'"
            relativePath,name = os.path.split(relativePath)
        # initialize message
        errorMessage = ""
        # get directory info
        dirInfoDict, errorMessage = self.get_directory_info(relativePath)
        if dirInfoDict is None:
            return None, errorMessage
        # get file info
        fileInfo = dict.__getitem__(dirInfoDict, "files").get(name, None)
        if fileInfo is None:
            errorMessage = "file %s does not exist in relative path '%s'"%(name, relativePath)
        return fileInfo, errorMessage

get file information dict from the repository given its relative path and name .


def get_file_relative_path_by_id(self, id):
        """
        Given an id, get the corresponding file info relative path joined with file name.

        Parameters:
            #. id (string): The file unique id string.

        :Returns:
            #. relativePath (string): The file relative path joined with file name.
               If None, it means file was not found.
        """
        for path, info in self.walk_files_info():
            if info['id']==id:
                return path
        # none was found
        return None

def
get_file_relative_path_by_id
(
self
,
id
)
:
for
path
,
info
in
self
.
walk_files_info
(
)
:
if
info
[
'id'
]
==
id
:
return
path
# none was found
return

Given an id get the corresponding file info relative path joined with file name .


def get_file_relative_path_by_name(self, name, skip=0):
        """
        Get file relative path given the file name. If file name is redundant in different
        directories in the repository, this method ensures to return all or some of the
        files according to skip value.

        Parameters:
            #. name (string): The file name.
            #. skip (None, integer): As file names can be identical, skip determines
               the number of satisfying files name to skip before returning.\n
               If None is given, a list of all files relative path will be returned.

        :Returns:
            #. relativePath (string, list): The file relative path.
               If None, it means file was not found.\n
               If skip is None a list of all found files relative paths will be returned.
        """
        if skip is None:
            paths = []
        else:
            paths = None
        for path, info in self.walk_files_info():
            _, n = os.path.split(path)
            if n==name:
                if skip is None:
                    paths.append(path)
                elif skip>0:
                    skip -= 1
                else:
                    paths = path
                    break
        return paths

def get_file_relative_path_by_name ( self , name , skip = 0 ) : if skip is None : paths = [ ] else : paths = None for path , info in self . walk_files_info ( ) : _ , n = os . path . split ( path ) if n == name : if skip is None : paths . append ( path ) elif skip > 0 : skip -= 1 else : paths = path break return paths

Get file relative path given the file name . If file name is redundant in different directories in the repository this method ensures to return all or some of the files according to skip value .


def add_directory(self, relativePath, info=None):
        """
        Adds a directory in the repository and creates its
        attribute in the Repository with utc timestamp.
        It insures adding all the missing directories in the path.

        :Parameters:
            #. relativePath (string): The relative to the repository path of the directory to add in the repository.
            #. info (None, string, pickable object): Any random info about the folder.

        :Returns:
            #. info (dict): The directory info dict.
        """
        path = os.path.normpath(relativePath)
        # create directories
        currentDir  = self.path
        currentDict = self
        if path in ("","."):
            return currentDict
        save = False
        for dir in path.split(os.sep):
            dirPath = os.path.join(currentDir, dir)
            # create directory
            if not os.path.exists(dirPath):
                 os.mkdir(dirPath)
            # create dictionary key
            currentDict = dict.__getitem__(currentDict, "directories")
            if currentDict.get(dir, None) is None:
                save = True
                currentDict[dir] = {"directories":{}, "files":{},
                                    "timestamp":datetime.utcnow(),
                                    "id":str(uuid.uuid1()),
                                    "info": info} # INFO MUST BE SET ONLY FOR THE LAST DIRECTORY

            currentDict = currentDict[dir]
            currentDir  = dirPath
        # save repository
        if save:
            self.save()
        # return currentDict
        return currentDict

Adds a directory in the repository and creates its attribute in the Repository with utc timestamp . It insures adding all the missing directories in the path .


def remove_directory(self, relativePath, removeFromSystem=False):
        """
        Remove directory from repository.

        :Parameters:
            #. relativePath (string): The relative to the repository path of the directory to remove from the repository.
            #. removeFromSystem (boolean): Whether to also remove directory and all files from the system.\n
               Only files saved in the repository will be removed and empty left directories.
        """
        # get parent directory info
        relativePath = os.path.normpath(relativePath)
        parentDirInfoDict, errorMessage = self.get_parent_directory_info(relativePath)
        assert parentDirInfoDict is not None, errorMessage
        # split path
        path, name = os.path.split(relativePath)
        if dict.__getitem__(parentDirInfoDict, 'directories').get(name, None) is None:
            raise Exception("'%s' is not a registered directory in repository relative path '%s'"%(name, path))
        # remove from system
        if removeFromSystem:
            # remove files
            for rp in self.walk_files_relative_path(relativePath=relativePath):
                ap = os.path.join(self.__path, relativePath, rp)
                if not os.path.isfile(ap):
                    continue
                if not os.path.exists(ap):
                    continue
                if os.path.isfile(ap):
                    os.remove( ap )
            # remove directories
            for rp in self.walk_directories_relative_path(relativePath=relativePath):
                ap = os.path.join(self.__path, relativePath, rp)
                if not os.path.isdir(ap):
                    continue
                if not os.path.exists(ap):
                    continue
                if not len(os.listdir(ap)):
                    os.rmdir(ap)
        # pop directory from repo
        dict.__getitem__(parentDirInfoDict, 'directories').pop(name, None)
        ap = os.path.join(self.__path, relativePath)
        if not os.path.isdir(ap):
            if not len(os.listdir(ap)):
                os.rmdir(ap)
        # save repository
        self.save()


def move_directory(self, relativePath, relativeDestination, replace=False, verbose=True):
        """
        Move a directory in the repository from one place to another. It insures moving all the
        files and subdirectories in the system.

        :Parameters:
            #. relativePath (string): The relative to the repository path of the directory to be moved.
            #. relativeDestination (string): The new relative to the repository path of the directory.
            #. replace (boolean): Whether to replace existing files with the same name in the new created directory.
            #. verbose (boolean): Whether to be warn and informed about any abnormalities.
        """
        # normalize path
        relativePath    = os.path.normpath(relativePath)
        relativeDestination = os.path.normpath(relativeDestination)
        # get files and directories
        filesInfo = list( self.walk_files_info(relativePath=relativePath) )
        dirsPath  = list( self.walk_directories_relative_path(relativePath=relativePath) )
        dirInfoDict, errorMessage = self.get_directory_info(relativePath)
        assert dirInfoDict is not None, errorMessage
        # remove directory info only
        self.remove_directory(relativePath=relativePath, removeFromSystem=False)
        # create new relative path
        self.add_directory(relativeDestination)
        # move files
        for RP, info in filesInfo:
            source      = os.path.join(self.__path, relativePath, RP)
            destination = os.path.join(self.__path, relativeDestination, RP)
            # add directory
            newDirRP, fileName = os.path.split(os.path.join(relativeDestination, RP))
            dirInfoDict = self.add_directory( newDirRP )
            # move file
            if os.path.isfile(destination):
                if replace:
                    os.remove(destination)
                    if verbose:
                        warnings.warn("file '%s' is copied replacing existing one in destination '%s'."%(fileName, newDirRP))
                else:
                    if verbose:
                        warnings.warn("file '%s' is not copied because the same file exists in destination '%s'."%(fileName,destination))
                    continue
            os.rename(source, destination)
            # set file information
            dict.__getitem__(dirInfoDict, "files")[fileName] = info
        # save repository
        self.save()

Move a directory in the repository from one place to another . It insures moving all the files and subdirectories in the system .


def rename_file(self, relativePath, name, newName, replace=False, verbose=True):
        """
        Rename a directory in the repository. It insures renaming the file in the system.

        :Parameters:
            #. relativePath (string): The relative to the repository path of the directory where the file is located.
            #. name (string): The file name.
            #. newName (string): The file new name.
            #. replace (boolean): Whether to force renaming when new folder name exists in the system.
               It fails when new folder name is registered in repository.
            #. verbose (boolean): Whether to be warn and informed about any abnormalities.
        """
        # normalize path
        relativePath = os.path.normpath(relativePath)
        if relativePath == '.':
            relativePath = ''
        dirInfoDict, errorMessage = self.get_directory_info(relativePath)
        assert dirInfoDict is not None, errorMessage
        # check directory in repository
        assert name in dict.__getitem__(dirInfoDict, "files"), "file '%s' is not found in repository relative path '%s'"%(name, relativePath)
        # get real path
        realPath = os.path.join(self.__path, relativePath, name)
        assert os.path.isfile(realPath), "file '%s' is not found in system"%realPath
        # assert directory new name doesn't exist in repository
        assert newName not in dict.__getitem__(dirInfoDict, "files"), "file '%s' already exists in repository relative path '%s'"%(newName, relativePath)
        # check new directory in system
        newRealPath = os.path.join(self.__path, relativePath, newName)
        if os.path.isfile( newRealPath ):
            if replace:
                os.remove(newRealPath)
                if verbose:
                    warnings.warn( "file '%s' already exists found in system, it is now replaced by '%s' because 'replace' flag is True."%(newRealPath,realPath) )
            else:
                raise Exception( "file '%s' already exists in system but not registered in repository."%newRealPath )
        # rename file
        os.rename(realPath, newRealPath)
        dict.__setitem__( dict.__getitem__(dirInfoDict, "files"),
                          newName,
                          dict.__getitem__(dirInfoDict, "files").pop(name) )
        # save repository
        self.save()

Rename a directory in the repository . It insures renaming the file in the system .


def dump_copy(self, path, relativePath, name=None,
                        description=None,
                        replace=False, verbose=False):
        """
        Copy an exisitng system file to the repository.
        attribute in the Repository with utc timestamp.

        :Parameters:
            #. path (str): The full path of the file to copy into the repository.
            #. relativePath (str): The relative to the repository path of the directory where the file should be dumped.
               If relativePath does not exist, it will be created automatically.
            #. name (string): The file name.
               If None is given, name will be split from path.
            #. description (None, string, pickable object): Any random description about the file.
            #. replace (boolean): Whether to replace any existing file with the same name if existing.
            #. verbose (boolean): Whether to be warn and informed about any abnormalities.
        """
        relativePath = os.path.normpath(relativePath)
        if relativePath == '.':
            relativePath = ''
        if name is None:
            _,name = os.path.split(path)
        # ensure directory added
        self.add_directory(relativePath)
        # ger real path
        realPath = os.path.join(self.__path, relativePath)
        # get directory info dict
        dirInfoDict, errorMessage = self.get_directory_info(relativePath)
        assert dirInfoDict is not None, errorMessage
        if name in dict.__getitem__(dirInfoDict, "files"):
            if not replace:
                if verbose:
                    warnings.warn("a file with the name '%s' is already defined in repository dictionary info. Set replace flag to True if you want to replace the existing file"%(name))
                return
        # convert dump and pull methods to strings
        dump = "raise Exception(\"dump is ambiguous for copied file '$FILE_PATH' \")"
        pull = "raise Exception(\"pull is ambiguous for copied file '$FILE_PATH' \")"
        # dump file
        try:
            shutil.copyfile(path, os.path.join(realPath,name))
        except Exception as e:
            if verbose:
                warnings.warn(e)
            return
        # set info
        klass = None
        # save the new file to the repository
        dict.__getitem__(dirInfoDict, "files")[name] = {"dump":dump,
                                                        "pull":pull,
                                                        "timestamp":datetime.utcnow(),
                                                        "id":str(uuid.uuid1()),
                                                        "class": klass,
                                                        "description":description}
        # save repository
        self.save()

Copy an exisitng system file to the repository . attribute in the Repository with utc timestamp .


def update_file(self, value, relativePath, name=None,
                          description=False, klass=False,
                          dump=False, pull=False,
                          ACID=None, verbose=False):
        """
        Update the value and the utc timestamp of a file that is already in the Repository.\n
        If file is not registered in repository, and error will be thrown.\n
        If file is missing in the system, it will be regenerated as dump method is called.

        :Parameters:
            #. value (object): The value of the file to update. It is any python object or a file.
            #. relativePath (str): The relative to the repository path of the directory where the file should be dumped.
            #. name (None, string): The file name.
               If None is given, name will be split from relativePath.
            #. description (False, string, pickable object): Any random description about the file.
               If False is given, the description info won't be updated,
               otherwise it will be update to what description argument value is.
            #. klass (False, class): The dumped object class. If False is given,
               the class info won't be updated, otherwise it will be update to what klass argument value is.
            #. dump (False, string): The new dump method. If False is given, the old one will be used.
            #. pull (False, string): The new pull method. If False is given, the old one will be used.
            #. ACID (boolean): Whether to ensure the ACID (Atomicity, Consistency, Isolation, Durability)
               properties of the repository upon dumping a file. This is ensured by dumping the file in
               a temporary path first and then moving it to the desired path.
               If None is given, repository ACID property will be used.
            #. verbose (boolean): Whether to be warn and informed about any abnormalities.
        """
        # check ACID
        if ACID is None:
            ACID = self.__ACID
        assert isinstance(ACID, bool), "ACID must be boolean"
        # get relative path normalized
        relativePath = os.path.normpath(relativePath)
        if relativePath == '.':
            relativePath = ''
            assert name != '.pyrepinfo', "'.pyrepinfo' is not allowed as file name in main repository directory"
            assert name != '.pyrepstate', "'.pyrepstate' is not allowed as file name in main repository directory"
            assert name != '.pyreplock', "'.pyreplock' is not allowed as file name in main repository directory"
        if name is None:
            assert len(relativePath), "name must be given when relative path is given as empty string or as a simple dot '.'"
            relativePath,name = os.path.split(relativePath)
        # get file info dict
        fileInfoDict, errorMessage = self.get_file_info(relativePath, name)
        assert fileInfoDict is not None, errorMessage
        # get real path
        realPath = os.path.join(self.__path, relativePath)
        # check if file exists
        if verbose:
            if not os.path.isfile( os.path.join(realPath, name) ):
                warnings.warn("file '%s' is in repository but does not exist in the system. It is therefore being recreated."%os.path.join(realPath, name))
        # convert dump and pull methods to strings
        if not dump:
            dump = fileInfoDict["dump"]
        if not pull:
            pull = fileInfoDict["pull"]
        # get savePath
        if ACID:
            savePath = os.path.join(tempfile.gettempdir(), name)
        else:
            savePath = os.path.join(realPath,name)
        # dump file
        try:
            exec( dump.replace("$FILE_PATH", str(savePath)) )
        except Exception as e:
            message = "unable to dump the file (%s)"%e
            if 'pickle.dump(' in dump:
                message += '\nmore info: %s'%str(get_pickling_errors(value))
            raise Exception( message )
        # copy if ACID
        if ACID:
            try:
                shutil.copyfile(savePath, os.path.join(realPath,name))
            except Exception as e:
                os.remove(savePath)
                if verbose:
                    warnings.warn(e)
                return
            os.remove(savePath)
        # update timestamp
        fileInfoDict["timestamp"] = datetime.utcnow()
        if description is not False:
            fileInfoDict["description"] = description
        if klass is not False:
            assert inspect.isclass(klass), "klass must be a class definition"
            fileInfoDict["class"] = klass
        # save repository
        self.save()

Update the value and the utc timestamp of a file that is already in the Repository . \ n If file is not registered in repository and error will be thrown . \ n If file is missing in the system it will be regenerated as dump method is called .


def ensure_str(value):
    """
    Ensure value is string.
    """
    if isinstance(value, six.string_types):
        return value
    else:
        return six.text_type(value)

def ensure_str ( value ) : if isinstance ( value , six . string_types ) : return value else : return six . text_type ( value )


def stats(cls, traces):
        """Build per minute stats for each key"""

        data = {}
        stats = {}
        # Group traces by key and minute
        for trace in traces:
            key = trace['key']
            if key not in data:
                data[key] = []
                stats[key] = {}

            data[key].append(trace['total_time'])
            cls._traces.pop(trace['id'])

        for key in data:
            times = data[key]
            stats[key] = dict(
                count=len(times),
                max=max(times),
                min=min(times),
                avg=sum(times)/len(times)
            )

        return stats

def
stats
(
cls
,
traces
)
:
data
=
{
}
stats
=
{
}
# Group traces by key and minute
for
trace
in
traces
:
key
=
trace
[
'key'
]
if
key
not
in
data
:
data
[
key
]
=
[
]
stats
[
key
]
=
{
}
data
[
key
]
.
append
(
trace
[
'total_time'
]
)
cls
.
_traces
.
pop
(
trace
[
'id'
]
)
for
key
in
data
:
times
=
data
[
key
]
stats
[
key
]
=
dict
(
count
=
len
(
times
)
,
max
=
max
(
times
)
,
min
=
min
(
times
)
,
avg
=
sum
(
times
)
/
len
(
times
)
)
return
stats


def start(self):
        """
        Monitors data kept in files in the predefined directory in a new thread.

        Note: Due to the underlying library, it may take a few milliseconds after this method is started for changes to
        start to being noticed.
        """
        with self._status_lock:
            if self._running:
                raise RuntimeError("Already running")
            self._running = True

        # Cannot re-use Observer after stopped
        self._observer = Observer()
        self._observer.schedule(self._event_handler, self._directory_location, recursive=True)
        self._observer.start()

        # Load all in directory afterwards to ensure no undetected changes between loading all and observing
        self._origin_mapped_data = self._load_all_in_directory()

Monitors data kept in files in the predefined directory in a new thread .


def stop(self):
        """
        Stops monitoring the predefined directory.
        """
        with self._status_lock:
            if self._running:
                assert self._observer is not None
                self._observer.stop()
                self._running = False
                self._origin_mapped_data = dict()

def stop ( self ) : with self . _status_lock : if self . _running : assert self . _observer is not None self . _observer . stop ( ) self . _running = False self . _origin_mapped_data = dict ( )

Stops monitoring the predefined directory .