def _on_file_moved(self, event: FileSystemMovedEvent):
        """
        Called when a file in the monitored directory has been moved.

        Breaks move down into a delete and a create (which it is sometimes detected as!).
        :param event: the file system event
        """
        if not event.is_directory and self.is_data_file(event.src_path):
            delete_event = FileSystemEvent(event.src_path)
            delete_event.event_type = EVENT_TYPE_DELETED
            self._on_file_deleted(delete_event)

            create_event = FileSystemEvent(event.dest_path)
            create_event.event_type = EVENT_TYPE_CREATED
            self._on_file_created(create_event)

def _on_file_moved ( self , event : FileSystemMovedEvent ) : if not event . is_directory and self . is_data_file ( event . src_path ) : delete_event = FileSystemEvent ( event . src_path ) delete_event . event_type = EVENT_TYPE_DELETED self . _on_file_deleted ( delete_event ) create_event = FileSystemEvent ( event . dest_path ) create_event . event_type = EVENT_TYPE_CREATED self . _on_file_created ( create_event )

Called when a file in the monitored directory has been moved .


def tear_down(self):
        """
        Tears down all temp files and directories.
        """
        while len(self._temp_directories) > 0:
            directory = self._temp_directories.pop()
            shutil.rmtree(directory, ignore_errors=True)
        while len(self._temp_files) > 0:
            file = self._temp_files.pop()
            try:
                os.remove(file)
            except OSError:
                pass

def tear_down ( self ) : while len ( self . _temp_directories ) > 0 : directory = self . _temp_directories . pop ( ) shutil . rmtree ( directory , ignore_errors = True ) while len ( self . _temp_files ) > 0 : file = self . _temp_files . pop ( ) try : os . remove ( file ) except OSError : pass

Tears down all temp files and directories .


def is_not_exist_or_allow_overwrite(self, overwrite=False):
        """
        Test whether a file target is not exists or it exists but allow
        overwrite.
        """
        if self.exists() and overwrite is False:
            return False
        else:  # pragma: no cover
            return True

def
is_not_exist_or_allow_overwrite
(
self
,
overwrite
=
False
)
:
if
self
.
exists
(
)
and
overwrite
is
False
:
return
False
else
:
# pragma: no cover
return
True

Test whether a file target is not exists or it exists but allow overwrite .


def copyto(self,
               new_abspath=None,
               new_dirpath=None,
               new_dirname=None,
               new_basename=None,
               new_fname=None,
               new_ext=None,
               overwrite=False,
               makedirs=False):
        """
        Copy this file to other place.
        """
        self.assert_exists()

        p = self.change(
            new_abspath=new_abspath,
            new_dirpath=new_dirpath,
            new_dirname=new_dirname,
            new_basename=new_basename,
            new_fname=new_fname,
            new_ext=new_ext,
        )

        if p.is_not_exist_or_allow_overwrite(overwrite=overwrite):
            # 如果两个路径不同, 才进行copy
            if self.abspath != p.abspath:
                try:
                    shutil.copy(self.abspath, p.abspath)
                except IOError as e:
                    if makedirs:
                        os.makedirs(p.parent.abspath)
                        shutil.copy(self.abspath, p.abspath)
                    else:
                        raise e
        return p

def
copyto
(
self
,
new_abspath
=
,
new_dirpath
=
,
new_dirname
=
,
new_basename
=
,
new_fname
=
,
new_ext
=
,
overwrite
=
False
,
makedirs
=
False
)
:
self
.
assert_exists
(
)
p
=
self
.
change
(
new_abspath
=
new_abspath
,
new_dirpath
=
new_dirpath
,
new_dirname
=
new_dirname
,
new_basename
=
new_basename
,
new_fname
=
new_fname
,
new_ext
=
new_ext
,
)
if
p
.
is_not_exist_or_allow_overwrite
(
overwrite
=
overwrite
)
:
# 如果两个路径不同, 才进行copy
if
self
.
abspath
!=
p
.
abspath
:
try
:
shutil
.
copy
(
self
.
abspath
,
p
.
abspath
)
except
IOError
as
e
:
if
makedirs
:
os
.
makedirs
(
p
.
parent
.
abspath
)
shutil
.
copy
(
self
.
abspath
,
p
.
abspath
)
else
:
raise
e
return
p


def create_client() -> APIClient:
    """
    Clients a Docker client.

    Will raise a `ConnectionError` if the Docker daemon is not accessible.
    :return: the Docker client
    """
    global _client
    client = _client()
    if client is None:
        # First try looking at the environment variables for specification of the daemon's location
        docker_environment = kwargs_from_env(assert_hostname=False)
        if "base_url" in docker_environment:
            client = _create_client(docker_environment.get("base_url"), docker_environment.get("tls"))
            if client is None:
                raise ConnectionError(
                    "Could not connect to the Docker daemon specified by the `DOCKER_X` environment variables: %s"
                    % docker_environment)
            else:
                logging.info("Connected to Docker daemon specified by the environment variables")
        else:
            # Let's see if the Docker daemon is accessible via the UNIX socket
            client = _create_client("unix://var/run/docker.sock")
            if client is not None:
                logging.info("Connected to Docker daemon running on UNIX socket")
            else:
                raise ConnectionError(
                    "Cannot connect to Docker - is the Docker daemon running? `$DOCKER_HOST` should be set or the "
                    "daemon should be accessible via the standard UNIX socket.")
        _client = weakref.ref(client)
    assert isinstance(client, APIClient)
    return client


def path_required(func):
    """Decorate methods when repository path is required."""
    @wraps(func)
    def wrapper(self, *args, **kwargs):
        if self.path is None:
            warnings.warn('Must load (Repository.load_repository) or initialize (Repository.create_repository) the repository first !')
            return
        return func(self, *args, **kwargs)
    return wrapper

Decorate methods when repository path is required .


def __clean_before_after(self, stateBefore, stateAfter, keepNoneEmptyDirectory=True):
        """clean repository given before and after states"""
        # prepare after for faster search
        errors    = []
        afterDict = {}
        [afterDict.setdefault(list(aitem)[0],[]).append(aitem) for aitem in stateAfter]
        # loop before
        for bitem in reversed(stateBefore):
            relaPath = list(bitem)[0]
            basename = os.path.basename(relaPath)
            btype    = bitem[relaPath]['type']
            alist    = afterDict.get(relaPath, [])
            aitem    = [a for a in alist if a[relaPath]['type']==btype]
            if len(aitem)>1:
                errors.append("Multiple '%s' of type '%s' where found in '%s', this should never had happened. Please report issue"%(basename,btype,relaPath))
                continue
            if not len(aitem):
                removeDirs  = []
                removeFiles = []
                if btype == 'dir':
                    if not len(relaPath):
                        errors.append("Removing main repository directory is not allowed")
                        continue
                    removeDirs.append(os.path.join(self.__path,relaPath))
                    removeFiles.append(os.path.join(self.__path,relaPath,self.__dirInfo))
                    removeFiles.append(os.path.join(self.__path,relaPath,self.__dirLock))
                elif btype == 'file':
                    removeFiles.append(os.path.join(self.__path,relaPath))
                    removeFiles.append(os.path.join(self.__path,relaPath,self.__fileInfo%basename))
                    removeFiles.append(os.path.join(self.__path,relaPath,self.__fileLock%basename))
                else:
                    ### MUST VERIFY THAT ONCE pyrepobjectdir IS IMPLEMENTED
                    removeDirs.append(os.path.join(self.__path,relaPath))
                    removeFiles.append(os.path.join(self.__path,relaPath,self.__fileInfo%basename))
                # remove files
                for fpath in removeFiles:
                    if os.path.isfile(fpath):
                        try:
                            os.remove(fpath)
                        except Exception as err:
                            errors.append("Unable to clean file '%s' (%s)"%(fpath, str(err)))
                # remove directories
                for dpath in removeDirs:
                    if os.path.isdir(dpath):
                        if keepNoneEmptyDirectory or not len(os.listdir(dpath)):
                            try:
                                shutil.rmtree(dpath)
                            except Exception as err:
                                errors.append("Unable to clean directory '%s' (%s)"%(fpath, str(err)))
        # return result and errors list
        return len(errors)==0, errors


def get_stats(self):
        """
        Get repository descriptive stats

        :Returns:
            #. numberOfDirectories (integer): Number of diretories in repository
            #. numberOfFiles (integer): Number of files in repository
        """
        if self.__path is None:
            return 0,0
        nfiles = 0
        ndirs  = 0
        for fdict in self.get_repository_state():
            fdname = list(fdict)[0]
            if fdname == '':
                continue
            if fdict[fdname].get('pyrepfileinfo', False):
                nfiles += 1
            elif fdict[fdname].get('pyrepdirinfo', False):
                ndirs += 1
            else:
                raise Exception('Not sure what to do next. Please report issue')
        return ndirs,nfiles


def reset(self):
        """Reset repository instance.
        """
        self.__path   = None
        self.__repo   = {'repository_unique_name': str(uuid.uuid1()),
                         'create_utctime': time.time(),
                         'last_update_utctime': None,
                         'pyrep_version': str(__version__),
                         'repository_information': '',
                         'walk_repo': []}

def reset ( self ) : self . __path = None self . __repo = { 'repository_unique_name' : str ( uuid . uuid1 ( ) ) , 'create_utctime' : time . time ( ) , 'last_update_utctime' : None , 'pyrep_version' : str ( __version__ ) , 'repository_information' : '' , 'walk_repo' : [ ] }


def load_repository(self, path, verbose=True, ntrials=3):
        """
        Load repository from a directory path and update the current instance.
        First, new repository still will be loaded. If failed, then old
        style repository load will be tried.

        :Parameters:
            #. path (string): The path of the directory from where to load
               the repository from. If '.' or an empty string is passed,
               the current working directory will be used.
            #. verbose (boolean): Whether to be verbose about abnormalities
            #. ntrials (int): After aquiring all locks, ntrials is the maximum
               number of trials allowed before failing.
               In rare cases, when multiple processes
               are accessing the same repository components, different processes
               can alter repository components between successive lock releases
               of some other process. Bigger number of trials lowers the
               likelyhood of failure due to multiple processes same time
               alteration.

        :Returns:
             #. repository (pyrep.Repository): returns self repository with loaded data.
        """
        assert isinstance(ntrials, int), "ntrials must be integer"
        assert ntrials>0, "ntrials must be >0"
        repo = None
        for _trial in range(ntrials):
            try:
                self.__load_repository(path=path, verbose=True)
            except Exception as err1:
                try:
                    from .OldRepository import Repository
                    REP = Repository(path)
                except Exception as err2:
                    #traceback.print_exc()
                    error = "Unable to load repository using neiher new style (%s) nor old style (%s)"%(err1, err2)
                    if self.DEBUG_PRINT_FAILED_TRIALS: print("Trial %i failed in Repository.%s (%s). Set Repository.DEBUG_PRINT_FAILED_TRIALS to False to mute"%(_trial, inspect.stack()[1][3], str(error)))
                else:
                    error = None
                    repo  = REP
                    break
            else:
                error = None
                repo  = self
                break
        # check and return
        assert error is None, error
        return repo

Load repository from a directory path and update the current instance . First new repository still will be loaded . If failed then old style repository load will be tried .


def remove_repository(self, path=None, removeEmptyDirs=True):
        """
        Remove all repository from path along with all repository tracked files.

        :Parameters:
            #. path (None, string): The path the repository to remove.
            #. removeEmptyDirs (boolean): Whether to remove remaining empty
               directories.
        """
        assert isinstance(removeEmptyDirs, bool), "removeEmptyDirs must be boolean"
        if path is not None:
            if path != self.__path:
                repo = Repository()
                repo.load_repository(path)
            else:
                repo = self
        else:
            repo = self
        assert repo.path is not None, "path is not given and repository is not initialized"
        # remove repo files and directories
        for fdict in reversed(repo.get_repository_state()):
            relaPath   = list(fdict)[0]
            realPath   = os.path.join(repo.path, relaPath)
            path, name = os.path.split(realPath)
            if fdict[relaPath]['type'] == 'file':
                if os.path.isfile(realPath):
                    os.remove(realPath)
                if os.path.isfile(os.path.join(repo.path,path,self.__fileInfo%name)):
                    os.remove(os.path.join(repo.path,path,self.__fileInfo%name))
                if os.path.isfile(os.path.join(repo.path,path,self.__fileLock%name)):
                    os.remove(os.path.join(repo.path,path,self.__fileLock%name))
                if os.path.isfile(os.path.join(repo.path,path,self.__fileClass%name)):
                    os.remove(os.path.join(repo.path,path,self.__fileClass%name))
            elif fdict[relaPath]['type'] == 'dir':
                if os.path.isfile(os.path.join(realPath,self.__dirInfo)):
                    os.remove(os.path.join(realPath,self.__dirInfo))
                if os.path.isfile(os.path.join(realPath,self.__dirLock)):
                    os.remove(os.path.join(realPath,self.__dirLock))
                if not len(os.listdir(realPath)) and removeEmptyDirs:
                    shutil.rmtree( realPath )
        # remove repo information file
        if os.path.isfile(os.path.join(repo.path,self.__repoFile)):
            os.remove(os.path.join(repo.path,self.__repoFile))
        if os.path.isfile(os.path.join(repo.path,self.__repoLock)):
            os.remove(os.path.join(repo.path,self.__repoLock))

Remove all repository from path along with all repository tracked files .


def is_name_allowed(self, path):
        """
        Get whether creating a file or a directory from the basenane of the given
        path is allowed

        :Parameters:
            #. path (str): The absolute or relative path or simply the file
               or directory name.

        :Returns:
            #. allowed (bool): Whether name is allowed.
            #. message (None, str): Reason for the name to be forbidden.
        """
        assert isinstance(path, basestring), "given path must be a string"
        name = os.path.basename(path)
        if not len(name):
            return False, "empty name is not allowed"
        # exact match
        for em in [self.__repoLock,self.__repoFile,self.__dirInfo,self.__dirLock]:
            if name == em:
                return False, "name '%s' is reserved for pyrep internal usage"%em
        # pattern match
        for pm in [self.__fileInfo,self.__fileLock]:#,self.__objectDir]:
            if name == pm or (name.endswith(pm[3:]) and name.startswith('.')):
                return False, "name pattern '%s' is not allowed as result may be reserved for pyrep internal usage"%pm
        # name is ok
        return True, None


def to_repo_relative_path(self, path, split=False):
        """
        Given a path, return relative path to diretory

        :Parameters:
            #. path (str): Path as a string
            #. split (boolean): Whether to split path to its components

        :Returns:
            #. relativePath (str, list): Relative path as a string or as a list
               of components if split is True
        """
        path = os.path.normpath(path)
        if path == '.':
            path = ''
        path = path.split(self.__path)[-1].strip(os.sep)
        if split:
            return path.split(os.sep)
        else:
            return path

def to_repo_relative_path ( self , path , split = False ) : path = os . path . normpath ( path ) if path == '.' : path = '' path = path . split ( self . __path ) [ - 1 ] . strip ( os . sep ) if split : return path . split ( os . sep ) else : return path

Given a path return relative path to diretory


def get_repository_state(self, relaPath=None):
        """
        Get a list representation of repository state along with useful
        information. List state is ordered relativeley to directories level

        :Parameters:
            #. relaPath (None, str): relative directory path from where to
               start. If None all repository representation is returned.

        :Returns:
            #. state (list): List representation of the repository.
               List items are all dictionaries. Every dictionary has a single
               key which is the file or the directory name and the value is a
               dictionary of information including:

                   * 'type': the type of the tracked whether it's file, dir, or objectdir
                   * 'exists': whether file or directory actually exists on disk
                   * 'pyrepfileinfo': In case of a file or an objectdir whether .%s_pyrepfileinfo exists
                   * 'pyrepdirinfo': In case of a directory whether .pyrepdirinfo exists
        """
        state = []
        def _walk_dir(relaPath, dirList):
            dirDict = {'type':'dir',
                       'exists':os.path.isdir(os.path.join(self.__path,relaPath)),
                       'pyrepdirinfo':os.path.isfile(os.path.join(self.__path,relaPath,self.__dirInfo)),
                      }
            state.append({relaPath:dirDict})
            # loop files and dirobjects
            for fname in sorted([f for f in dirList if isinstance(f, basestring)]):
                relaFilePath = os.path.join(relaPath,fname)
                realFilePath = os.path.join(self.__path,relaFilePath)
                #if os.path.isdir(realFilePath) and df.startswith('.') and df.endswith(self.__objectDir[3:]):
                #    fileDict = {'type':'objectdir',
                #                'exists':True,
                #                'pyrepfileinfo':os.path.isfile(os.path.join(self.__path,relaPath,self.__fileInfo%fname)),
                #               }
                #else:
                #    fileDict = {'type':'file',
                #                'exists':os.path.isfile(realFilePath),
                #                'pyrepfileinfo':os.path.isfile(os.path.join(self.__path,relaPath,self.__fileInfo%fname)),
                #               }
                fileDict = {'type':'file',
                            'exists':os.path.isfile(realFilePath),
                            'pyrepfileinfo':os.path.isfile(os.path.join(self.__path,relaPath,self.__fileInfo%fname)),
                           }
                state.append({relaFilePath:fileDict})
            # loop directories
            #for ddict in sorted([d for d in dirList if isinstance(d, dict) and len(d)], key=lambda k: list(k)[0]):
            for ddict in sorted([d for d in dirList if isinstance(d, dict)], key=lambda k: list(k)[0]):
                dirname = list(ddict)[0]
                _walk_dir(relaPath=os.path.join(relaPath,dirname), dirList=ddict[dirname])
        # call recursive _walk_dir
        if relaPath is None:
            _walk_dir(relaPath='', dirList=self.__repo['walk_repo'])
        else:
            assert isinstance(relaPath, basestring), "relaPath must be None or a str"
            relaPath = self.to_repo_relative_path(path=relaPath, split=False)
            spath    = relaPath.split(os.sep)
            dirList  = self.__repo['walk_repo']
            while len(spath):
                dirname = spath.pop(0)
                dList   = [d for d in dirList if isinstance(d, dict)]
                if not len(dList):
                    dirList = None
                    break
                cDict = [d for d in dList if dirname in d]
                if not len(cDict):
                    dirList = None
                    break
                dirList = cDict[0][dirname]
            if dirList is not None:
                _walk_dir(relaPath=relaPath, dirList=dirList)
        # return state list
        return state

Get a list representation of repository state along with useful information . List state is ordered relativeley to directories level


def get_file_info(self, relativePath):
        """
        Get file information dict from the repository given its relative path.

        :Parameters:
            #. relativePath (string): The relative to the repository path of
               the file.

        :Returns:
            #. info (None, dictionary): The file information dictionary.
               If None, it means an error has occurred.
            #. errorMessage (string): The error message if any error occurred.
        """
        relativePath = self.to_repo_relative_path(path=relativePath, split=False)
        fileName     = os.path.basename(relativePath)
        isRepoFile,fileOnDisk, infoOnDisk, classOnDisk = self.is_repository_file(relativePath)
        if not isRepoFile:
            return None, "file is not a registered repository file."
        if not infoOnDisk:
            return None, "file is a registered repository file but info file missing"
        fileInfoPath = os.path.join(self.__path,os.path.dirname(relativePath),self.__fileInfo%fileName)
        try:
            with open(fileInfoPath, 'rb') as fd:
                info = pickle.load(fd)
        except Exception as err:
            return None, "Unable to read file info from disk (%s)"%str(err)
        return info, ''

Get file information dict from the repository given its relative path .


def is_repository_file(self, relativePath):
        """
        Check whether a given relative path is a repository file path

        :Parameters:
            #. relativePath (string): File relative path

        :Returns:
            #. isRepoFile (boolean): Whether file is a repository file.
            #. isFileOnDisk (boolean): Whether file is found on disk.
            #. isFileInfoOnDisk (boolean): Whether file info is found on disk.
            #. isFileClassOnDisk (boolean): Whether file class is found on disk.
        """
        relativePath  = self.to_repo_relative_path(path=relativePath, split=False)
        if relativePath == '':
            return False, False, False, False
        relaDir, name = os.path.split(relativePath)
        fileOnDisk    = os.path.isfile(os.path.join(self.__path, relativePath))
        infoOnDisk    = os.path.isfile(os.path.join(self.__path,os.path.dirname(relativePath),self.__fileInfo%name))
        classOnDisk   = os.path.isfile(os.path.join(self.__path,os.path.dirname(relativePath),self.__fileClass%name))
        cDir          = self.__repo['walk_repo']
        if len(relaDir):
            for dirname in relaDir.split(os.sep):
                dList = [d for d in cDir if isinstance(d, dict)]
                if not len(dList):
                    cDir = None
                    break
                cDict = [d for d in dList if dirname in d]
                if not len(cDict):
                    cDir = None
                    break
                cDir = cDict[0][dirname]
        if cDir is None:
            return False, fileOnDisk, infoOnDisk, classOnDisk
        #if name not in cDir:
        if str(name) not in [str(i) for i in cDir]:
            return False, fileOnDisk, infoOnDisk, classOnDisk
        # this is a repository registered file. check whether all is on disk
        return True, fileOnDisk, infoOnDisk, classOnDisk


def create_package(self, path=None, name=None, mode=None):
        """
        Create a tar file package of all the repository files and directories.
        Only files and directories that are tracked in the repository
        are stored in the package tar file.

        **N.B. On some systems packaging requires root permissions.**

        :Parameters:
            #. path (None, string): The real absolute path where to create the
               package. If None, it will be created in the same directory as
               the repository. If '.' or an empty string is passed, the current
               working directory will be used.
            #. name (None, string): The name to give to the package file
               If None, the package directory name will be used with the
               appropriate extension added.
            #. mode (None, string): The writing mode of the tarfile.
               If None, automatically the best compression mode will be chose.
               Available modes are ('w', 'w:', 'w:gz', 'w:bz2')
        """
        # check mode
        assert mode in (None, 'w', 'w:', 'w:gz', 'w:bz2'), 'unkown archive mode %s'%str(mode)
        if mode is None:
            #mode = 'w:bz2'
            mode = 'w:'
        # get root
        if path is None:
            root = os.path.split(self.__path)[0]
        elif path.strip() in ('','.'):
            root = os.getcwd()
        else:
            root = os.path.realpath( os.path.expanduser(path) )
        assert os.path.isdir(root), 'absolute path %s is not a valid directory'%path
        # get name
        if name is None:
            ext = mode.split(":")
            if len(ext) == 2:
                if len(ext[1]):
                    ext = "."+ext[1]
                else:
                    ext = '.tar'
            else:
                ext = '.tar'
            name = os.path.split(self.__path)[1]+ext
        # create tar file
        tarfilePath = os.path.join(root, name)
        try:
            tarHandler = tarfile.TarFile.open(tarfilePath, mode=mode)
        except Exception as e:
            raise Exception("Unable to create package (%s)"%e)
        # walk directory and create empty directories
        for dpath in sorted(list(self.walk_directories_path(recursive=True))):
            t = tarfile.TarInfo( dpath )
            t.type = tarfile.DIRTYPE
            tarHandler.addfile(t)
            tarHandler.add(os.path.join(self.__path,dpath,self.__dirInfo), arcname=self.__dirInfo)
        # walk files and add to tar
        for fpath in self.walk_files_path(recursive=True):
            relaPath, fname = os.path.split(fpath)
            tarHandler.add(os.path.join(self.__path,fpath), arcname=fname)
            tarHandler.add(os.path.join(self.__path,relaPath,self.__fileInfo%fname), arcname=self.__fileInfo%fname)
            tarHandler.add(os.path.join(self.__path,relaPath,self.__fileClass%fname), arcname=self.__fileClass%fname)
        # save repository .pyrepinfo
        tarHandler.add(os.path.join(self.__path,self.__repoFile), arcname=".pyrepinfo")
        # close tar file
        tarHandler.close()

Create a tar file package of all the repository files and directories . Only files and directories that are tracked in the repository are stored in the package tar file .


def rename(self, key: Any, new_key: Any):
        """
        Renames an item in this collection as a transaction.

        Will override if new key name already exists.
        :param key: the current name of the item
        :param new_key: the new name that the item should have
        """
        if new_key == key:
            return

        required_locks = [self._key_locks[key], self._key_locks[new_key]]
        ordered_required_locks = sorted(required_locks, key=lambda x: id(x))
        for lock in ordered_required_locks:
            lock.acquire()

        try:
            if key not in self._data:
                raise KeyError("Attribute to rename \"%s\" does not exist" % key)
            self._data[new_key] = self[key]
            del self._data[key]
        finally:
            for lock in required_locks:
                lock.release()

Renames an item in this collection as a transaction .


def get_text_fingerprint(text, hash_meth, encoding="utf-8"):  # pragma: no cover
    """
    Use default hash method to return hash value of a piece of string
    default setting use 'utf-8' encoding.
    """
    m = hash_meth()
    m.update(text.encode(encoding))
    return m.hexdigest()

def
get_text_fingerprint
(
text
,
hash_meth
,
encoding
=
"utf-8"
)
:
# pragma: no cover
m
=
hash_meth
(
)
m
.
update
(
text
.
encode
(
encoding
)
)
return
m
.
hexdigest
(
)

Use default hash method to return hash value of a piece of string default setting use utf - 8 encoding .


def md5file(abspath, nbytes=0, chunk_size=DEFAULT_CHUNK_SIZE):
    """
    Return md5 hash value of a piece of a file

    Estimate processing time on:

    :param abspath: the absolute path to the file
    :param nbytes: only has first N bytes of the file. if 0 or None,
      hash all file

    CPU = i7-4600U 2.10GHz - 2.70GHz, RAM = 8.00 GB
    1 second can process 0.25GB data

    - 0.59G - 2.43 sec
    - 1.3G - 5.68 sec
    - 1.9G - 7.72 sec
    - 2.5G - 10.32 sec
    - 3.9G - 16.0 sec
    """
    return get_file_fingerprint(abspath, hashlib.md5, nbytes=nbytes, chunk_size=chunk_size)

def md5file ( abspath , nbytes = 0 , chunk_size = DEFAULT_CHUNK_SIZE ) : return get_file_fingerprint ( abspath , hashlib . md5 , nbytes = nbytes , chunk_size = chunk_size )


def sha256file(abspath, nbytes=0, chunk_size=DEFAULT_CHUNK_SIZE):
    """
    Return sha256 hash value of a piece of a file

    Estimate processing time on:

    :param abspath: the absolute path to the file
    :param nbytes: only has first N bytes of the file. if 0 or None,
      hash all file
    """
    return get_file_fingerprint(abspath, hashlib.sha256, nbytes=nbytes, chunk_size=chunk_size)

def sha256file ( abspath , nbytes = 0 , chunk_size = DEFAULT_CHUNK_SIZE ) : return get_file_fingerprint ( abspath , hashlib . sha256 , nbytes = nbytes , chunk_size = chunk_size )


def sha512file(abspath, nbytes=0, chunk_size=DEFAULT_CHUNK_SIZE):
    """
    Return sha512 hash value of a piece of a file

    Estimate processing time on:

    :param abspath: the absolute path to the file
    :param nbytes: only has first N bytes of the file. if 0 or None,
      hash all file
    """
    return get_file_fingerprint(abspath, hashlib.sha512, nbytes=nbytes, chunk_size=chunk_size)

def sha512file ( abspath , nbytes = 0 , chunk_size = DEFAULT_CHUNK_SIZE ) : return get_file_fingerprint ( abspath , hashlib . sha512 , nbytes = nbytes , chunk_size = chunk_size )


def auto_complete_choices(self, case_sensitive=False):
        """
        A command line auto complete similar behavior. Find all item with same
        prefix of this one.

        :param case_sensitive: toggle if it is case sensitive.
        :return: list of :class:`pathlib_mate.pathlib2.Path`.
        """
        self_basename = self.basename
        self_basename_lower = self.basename.lower()
        if case_sensitive:  # pragma: no cover
            def match(basename):
                return basename.startswith(self_basename)
        else:
            def match(basename):
                return basename.lower().startswith(self_basename_lower)

        choices = list()
        if self.is_dir():
            choices.append(self)
            for p in self.sort_by_abspath(self.select(recursive=False)):
                choices.append(p)
        else:
            p_parent = self.parent
            if p_parent.is_dir():
                for p in self.sort_by_abspath(p_parent.select(recursive=False)):
                    if match(p.basename):
                        choices.append(p)
            else:  # pragma: no cover
                raise ValueError("'%s' directory does not exist!" % p_parent)
        return choices

def
auto_complete_choices
(
self
,
case_sensitive
=
False
)
:
self_basename
=
self
.
basename
self_basename_lower
=
self
.
basename
.
lower
(
)
if
case_sensitive
:
# pragma: no cover
def
match
(
basename
)
:
return
basename
.
startswith
(
self_basename
)
else
:
def
match
(
basename
)
:
return
basename
.
lower
(
)
.
startswith
(
self_basename_lower
)
choices
=
list
(
)
if
self
.
is_dir
(
)
:
choices
.
append
(
self
)
for
p
in
self
.
sort_by_abspath
(
self
.
select
(
recursive
=
False
)
)
:
choices
.
append
(
p
)
else
:
p_parent
=
self
.
parent
if
p_parent
.
is_dir
(
)
:
for
p
in
self
.
sort_by_abspath
(
p_parent
.
select
(
recursive
=
False
)
)
:
if
match
(
p
.
basename
)
:
choices
.
append
(
p
)
else
:
# pragma: no cover
raise
ValueError
(
"'%s' directory does not exist!"
%
p_parent
)
return
choices

A command line auto complete similar behavior . Find all item with same prefix of this one .


def print_big_dir(self, top_n=5):
        """
        Print ``top_n`` big dir in this dir.
        """
        self.assert_is_dir_and_exists()

        size_table = sorted(
            [(p, p.dirsize) for p in self.select_dir(recursive=False)],
            key=lambda x: x[1],
            reverse=True,
        )
        for p, size in size_table[:top_n]:
            print("{:<9}    {:<9}".format(repr_data_size(size), p.abspath))

def
print_big_dir
(
self
,
top_n
=
5
)
:
self
.
assert_is_dir_and_exists
(
)
size_table
=
sorted
(
[
(
p
,
p
.
dirsize
)
for
p
in
self
.
select_dir
(
recursive
=
False
)
]
,
key
=
lambda
x
:
x
[
1
]
,
reverse
=
True
,
)
for
p
,
size
in
size_table
[
:
top_n
]
:
print
(
"{:<9}    {:<9}"
.
format
(
repr_data_size
(
size
)
,
p
.
abspath
)
)


def print_big_file(self, top_n=5):
        """
        Print ``top_n`` big file in this dir.
        """
        self.assert_is_dir_and_exists()

        size_table = sorted(
            [(p, p.size) for p in self.select_file(recursive=True)],
            key=lambda x: x[1],
            reverse=True,
        )
        for p, size in size_table[:top_n]:
            print("{:<9}    {:<9}".format(repr_data_size(size), p.abspath))

def
print_big_file
(
self
,
top_n
=
5
)
:
self
.
assert_is_dir_and_exists
(
)
size_table
=
sorted
(
[
(
p
,
p
.
size
)
for
p
in
self
.
select_file
(
recursive
=
True
)
]
,
key
=
lambda
x
:
x
[
1
]
,
reverse
=
True
,
)
for
p
,
size
in
size_table
[
:
top_n
]
:
print
(
"{:<9}    {:<9}"
.
format
(
repr_data_size
(
size
)
,
p
.
abspath
)
)


def print_big_dir_and_big_file(self, top_n=5):
        """Print ``top_n`` big dir and ``top_n`` big file in each dir.
        """
        self.assert_is_dir_and_exists()

        size_table1 = sorted(
            [(p, p.dirsize) for p in self.select_dir(recursive=False)],
            key=lambda x: x[1],
            reverse=True,
        )
        for p1, size1 in size_table1[:top_n]:
            print("{:<9}    {:<9}".format(repr_data_size(size1), p1.abspath))
            size_table2 = sorted(
                [(p, p.size) for p in p1.select_file(recursive=True)],
                key=lambda x: x[1],
                reverse=True,
            )
            for p2, size2 in size_table2[:top_n]:
                print("    {:<9}    {:<9}".format(
                    repr_data_size(size2), p2.abspath))

def
print_big_dir_and_big_file
(
self
,
top_n
=
5
)
:
self
.
assert_is_dir_and_exists
(
)
size_table1
=
sorted
(
[
(
p
,
p
.
dirsize
)
for
p
in
self
.
select_dir
(
recursive
=
False
)
]
,
key
=
lambda
x
:
x
[
1
]
,
reverse
=
True
,
)
for
p1
,
size1
in
size_table1
[
:
top_n
]
:
print
(
"{:<9}    {:<9}"
.
format
(
repr_data_size
(
size1
)
,
p1
.
abspath
)
)
size_table2
=
sorted
(
[
(
p
,
p
.
size
)
for
p
in
p1
.
select_file
(
recursive
=
True
)
]
,
key
=
lambda
x
:
x
[
1
]
,
reverse
=
True
,
)
for
p2
,
size2
in
size_table2
[
:
top_n
]
:
print
(
"    {:<9}    {:<9}"
.
format
(
repr_data_size
(
size2
)
,
p2
.
abspath
)
)

Print top_n big dir and top_n big file in each dir .


def mirror_to(self, dst):  # pragma: no cover
        """
        Create a new folder having exactly same structure with this directory.
        However, all files are just empty file with same file name.

        :param dst: destination directory. The directory can't exists before
        you execute this.

        **中文文档**

        创建一个目录的镜像拷贝, 与拷贝操作不同的是, 文件的副本只是在文件名上
        与原件一致, 但是是空文件, 完全没有内容, 文件大小为0。
        """
        self.assert_is_dir_and_exists()

        src = self.abspath
        dst = os.path.abspath(dst)
        if os.path.exists(dst):  # pragma: no cover
            raise Exception("distination already exist!")

        folder_to_create = list()
        file_to_create = list()

        for current_folder, _, file_list in os.walk(self.abspath):
            current_folder = current_folder.replace(src, dst)
            try:
                os.mkdir(current_folder)
            except:  # pragma: no cover
                pass
            for basename in file_list:
                abspath = os.path.join(current_folder, basename)
                with open(abspath, "wb") as _:
                    pass

def
mirror_to
(
self
,
dst
)
:
# pragma: no cover
self
.
assert_is_dir_and_exists
(
)
src
=
self
.
abspath
dst
=
os
.
path
.
abspath
(
dst
)
if
os
.
path
.
exists
(
dst
)
:
# pragma: no cover
raise
Exception
(
"distination already exist!"
)
folder_to_create
=
list
(
)
file_to_create
=
list
(
)
for
current_folder
,
_
,
file_list
in
os
.
walk
(
self
.
abspath
)
:
current_folder
=
current_folder
.
replace
(
src
,
dst
)
try
:
os
.
mkdir
(
current_folder
)
except
:
# pragma: no cover
pass
for
basename
in
file_list
:
abspath
=
os
.
path
.
join
(
current_folder
,
basename
)
with
open
(
abspath
,
"wb"
)
as
_
:
pass

Create a new folder having exactly same structure with this directory . However all files are just empty file with same file name .


def execute_pyfile(self, py_exe=None):  # pragma: no cover
        """
        Execute every ``.py`` file as main script.

        :param py_exe: str, python command or python executable path.

        **中文文档**

        将目录下的所有Python文件作为主脚本用当前解释器运行。
        """
        import subprocess

        self.assert_is_dir_and_exists()

        if py_exe is None:
            if six.PY2:
                py_exe = "python2"
            elif six.PY3:
                py_exe = "python3"

        for p in self.select_by_ext(".py"):
            subprocess.Popen('%s "%s"' % (py_exe, p.abspath))

def
execute_pyfile
(
self
,
py_exe
=
)
:
# pragma: no cover
import
subprocess
self
.
assert_is_dir_and_exists
(
)
if
py_exe
is
:
if
six
.
PY2
:
py_exe
=
"python2"
elif
six
.
PY3
:
py_exe
=
"python3"
for
p
in
self
.
select_by_ext
(
".py"
)
:
subprocess
.
Popen
(
'%s "%s"'
%
(
py_exe
,
p
.
abspath
)
)

Execute every . py file as main script .


def trail_space(self, filters=lambda p: p.ext == ".py"):  # pragma: no cover
        """
        Trail white space at end of each line for every ``.py`` file.

        **中文文档**

        将目录下的所有被选择的文件中行末的空格删除。
        """
        self.assert_is_dir_and_exists()

        for p in self.select_file(filters):
            try:
                with open(p.abspath, "rb") as f:
                    lines = list()
                    for line in f:
                        lines.append(line.decode("utf-8").rstrip())

                with open(p.abspath, "wb") as f:
                    f.write("\n".join(lines).encode("utf-8"))

            except Exception as e:  # pragma: no cover
                raise e

def
trail_space
(
self
,
filters
=
lambda
p
:
p
.
ext
==
".py"
)
:
# pragma: no cover
self
.
assert_is_dir_and_exists
(
)
for
p
in
self
.
select_file
(
filters
)
:
try
:
with
open
(
p
.
abspath
,
"rb"
)
as
f
:
lines
=
list
(
)
for
line
in
f
:
lines
.
append
(
line
.
decode
(
"utf-8"
)
.
rstrip
(
)
)
with
open
(
p
.
abspath
,
"wb"
)
as
f
:
f
.
write
(
"\n"
.
join
(
lines
)
.
encode
(
"utf-8"
)
)
except
Exception
as
e
:
# pragma: no cover
raise
e

Trail white space at end of each line for every . py file .


def autopep8(self, **kwargs):  # pragma: no cover
        """
        Auto convert your python code in a directory to pep8 styled code.

        :param kwargs: arguments for ``autopep8.fix_code`` method.

        **中文文档**

        将目录下的所有Python文件用pep8风格格式化。增加其可读性和规范性。
        """
        self.assert_is_dir_and_exists()

        for p in self.select_by_ext(".py"):
            with open(p.abspath, "rb") as f:
                code = f.read().decode("utf-8")

            formatted_code = autopep8.fix_code(code, **kwargs)

            with open(p.abspath, "wb") as f:
                f.write(formatted_code.encode("utf-8"))

def
autopep8
(
self
,
*
*
kwargs
)
:
# pragma: no cover
self
.
assert_is_dir_and_exists
(
)
for
p
in
self
.
select_by_ext
(
".py"
)
:
with
open
(
p
.
abspath
,
"rb"
)
as
f
:
code
=
f
.
read
(
)
.
decode
(
"utf-8"
)
formatted_code
=
autopep8
.
fix_code
(
code
,
*
*
kwargs
)
with
open
(
p
.
abspath
,
"wb"
)
as
f
:
f
.
write
(
formatted_code
.
encode
(
"utf-8"
)
)

Auto convert your python code in a directory to pep8 styled code .


def size(self):
        """
        File size in bytes.
        """
        try:
            return self._stat.st_size
        except:  # pragma: no cover
            self._stat = self.stat()
            return self.size

def
size
(
self
)
:
try
:
return
self
.
_stat
.
st_size
except
:
# pragma: no cover
self
.
_stat
=
self
.
stat
(
)
return
self
.
size


def mtime(self):
        """
        Get most recent modify time in timestamp.
        """
        try:
            return self._stat.st_mtime
        except:  # pragma: no cover
            self._stat = self.stat()
            return self.mtime

def
mtime
(
self
)
:
try
:
return
self
.
_stat
.
st_mtime
except
:
# pragma: no cover
self
.
_stat
=
self
.
stat
(
)
return
self
.
mtime

Get most recent modify time in timestamp .


def atime(self):
        """
        Get most recent access time in timestamp.
        """
        try:
            return self._stat.st_atime
        except:  # pragma: no cover
            self._stat = self.stat()
            return self.atime

def
atime
(
self
)
:
try
:
return
self
.
_stat
.
st_atime
except
:
# pragma: no cover
self
.
_stat
=
self
.
stat
(
)
return
self
.
atime

Get most recent access time in timestamp .


def ctime(self):
        """
        Get most recent create time in timestamp.
        """
        try:
            return self._stat.st_ctime
        except:  # pragma: no cover
            self._stat = self.stat()
            return self.ctime

def
ctime
(
self
)
:
try
:
return
self
.
_stat
.
st_ctime
except
:
# pragma: no cover
self
.
_stat
=
self
.
stat
(
)
return
self
.
ctime

Get most recent create time in timestamp .


def unusedoptions(self, sections):
        """Lists options that have not been used to format other values in
        their sections.

        Good for finding out if the user has misspelled any of the options.
        """
        unused = set([])
        for section in _list(sections):
            if not self.has_section(section):
                continue
            options = self.options(section)
            raw_values = [self.get(section, option, raw=True) for option in options]
            for option in options:
                formatter = "%(" + option + ")s"
                for raw_value in raw_values:
                    if formatter in raw_value:
                        break
                else:
                    unused.add(option)
            return list(unused)

def unusedoptions ( self , sections ) : unused = set ( [ ] ) for section in _list ( sections ) : if not self . has_section ( section ) : continue options = self . options ( section ) raw_values = [ self . get ( section , option , raw = True ) for option in options ] for option in options : formatter = "%(" + option + ")s" for raw_value in raw_values : if formatter in raw_value : break else : unused . add ( option ) return list ( unused )

Lists options that have not been used to format other values in their sections . Good for finding out if the user has misspelled any of the options .


def keys(self):
        """List names of options and positional arguments."""
        return self.options.keys() + [p.name for p in self.positional_args]

def keys ( self ) : return self . options . keys ( ) + [ p . name for p in self . positional_args ]

List names of options and positional arguments .


def _add_option(self, option):
        """Add an Option object to the user interface."""
        if option.name in self.options:
            raise ValueError('name already in use')
        if option.abbreviation in self.abbreviations:
            raise ValueError('abbreviation already in use')
        if option.name in [arg.name for arg in self.positional_args]:
            raise ValueError('name already in use by a positional argument')
        self.options[option.name] = option
        if option.abbreviation:
            self.abbreviations[option.abbreviation] = option
        self.option_order.append(option.name)

Add an Option object to the user interface .


def _add_positional_argument(self, posarg):
        """Append a positional argument to the user interface.

        Optional positional arguments must be added after the required ones.
        The user interface can have at most one recurring positional argument,
        and if present, that argument must be the last one.
        """
        if self.positional_args:
            if self.positional_args[-1].recurring:
                raise ValueError("recurring positional arguments must be last")
            if self.positional_args[-1].optional and not posarg.optional:
                raise ValueError("required positional arguments must precede optional ones")
        self.positional_args.append(posarg)

Append a positional argument to the user interface .


def read_docs(self, docsfiles):
        """Read program documentation from a DocParser compatible file.

        docsfiles is a list of paths to potential docsfiles: parse if present.
        A string is taken as a list of one item.
        """
        updates = DocParser()
        for docsfile in _list(docsfiles):
            if os.path.isfile(docsfile):
                updates.parse(docsfile)
        self.docs.update((k, _docs(updates[k], self.docvars)) for k in self.docs if updates.blocks[k])
        for name, text in updates['parameters'].items():
            if name in self:
                self.getparam(name).docs = text[0] % self.docvars
            elif name not in self.ignore:
                raise ValueError("parameter %r does not exist" % name)

def
read_docs
(
self
,
docsfiles
)
:
updates
=
DocParser
(
)
for
docsfile
in
_list
(
docsfiles
)
:
if
os
.
path
.
isfile
(
docsfile
)
:
updates
.
parse
(
docsfile
)
self
.
docs
.
update
(
(
k
,
_docs
(
updates
[
k
]
,
self
.
docvars
)
)
for
k
in
self
.
docs
if
updates
.
blocks
[
k
]
)
for
name
,
text
in
updates
[
'parameters'
]
.
items
(
)
:
if
name
in
self
:
self
.
getparam
(
name
)
.
docs
=
text
[
0
]
%
self
.
docvars
elif
name
not
in
self
.
ignore
:
raise
ValueError
(
"parameter %r does not exist"
%
name
)

Read program documentation from a DocParser compatible file .


def optionhelp(self, indent=0, maxindent=25, width=79):
        """Return user friendly help on program options."""
        def makelabels(option):
            labels = '%*s--%s' % (indent, ' ', option.name)
            if option.abbreviation:
                labels += ', -' + option.abbreviation
            return labels + ': '
        docs = []
        helpindent = _autoindent([makelabels(o) for o in self.options.values()], indent, maxindent)
        for name in self.option_order:
            option = self.options[name]
            labels = makelabels(option)
            helpstring = "%s(%s). %s" % (option.formatname, option.strvalue, option.docs)
            wrapped = self._wrap_labelled(labels, helpstring, helpindent, width)
            docs.extend(wrapped)
        return '\n'.join(docs)

def
optionhelp
(
self
,
indent
=
0
,
maxindent
=
25
,
width
=
79
)
:
def
makelabels
(
option
)
:
labels
=
'%*s--%s'
%
(
indent
,
' '
,
option
.
name
)
if
option
.
abbreviation
:
labels
+=
', -'
+
option
.
abbreviation
return
labels
+
': '
docs
=
[
]
helpindent
=
_autoindent
(
[
makelabels
(
o
)
for
o
in
self
.
options
.
values
(
)
]
,
indent
,
maxindent
)
for
name
in
self
.
option_order
:
option
=
self
.
options
[
name
]
labels
=
makelabels
(
option
)
helpstring
=
"%s(%s). %s"
%
(
option
.
formatname
,
option
.
strvalue
,
option
.
docs
)
wrapped
=
self
.
_wrap_labelled
(
labels
,
helpstring
,
helpindent
,
width
)
docs
.
extend
(
wrapped
)
return
'\n'
.
join
(
docs
)

Return user friendly help on program options .


def posarghelp(self, indent=0, maxindent=25, width=79):
        """Return user friendly help on positional arguments in the program."""
        docs = []
        makelabel = lambda posarg: ' ' * indent + posarg.displayname + ': '
        helpindent = _autoindent([makelabel(p) for p in self.positional_args], indent, maxindent)
        for posarg in self.positional_args:
            label = makelabel(posarg)
            text = posarg.formatname + '. ' + posarg.docs
            wrapped = self._wrap_labelled(label, text, helpindent, width)
            docs.extend(wrapped)
        return '\n'.join(docs)

def
posarghelp
(
self
,
indent
=
0
,
maxindent
=
25
,
width
=
79
)
:
docs
=
[
]
makelabel
=
lambda
posarg
:
' '
*
indent
+
posarg
.
displayname
+
': '
helpindent
=
_autoindent
(
[
makelabel
(
p
)
for
p
in
self
.
positional_args
]
,
indent
,
maxindent
)
for
posarg
in
self
.
positional_args
:
label
=
makelabel
(
posarg
)
text
=
posarg
.
formatname
+
'. '
+
posarg
.
docs
wrapped
=
self
.
_wrap_labelled
(
label
,
text
,
helpindent
,
width
)
docs
.
extend
(
wrapped
)
return
'\n'
.
join
(
docs
)

Return user friendly help on positional arguments in the program .


def strsettings(self, indent=0, maxindent=25, width=0):
        """Return user friendly help on positional arguments.

        indent is the number of spaces preceeding the text on each line.

        The indent of the documentation is dependent on the length of the
        longest label that is shorter than maxindent. A label longer than
        maxindent will be printed on its own line.

        width is maximum allowed page width, use self.width if 0.
        """
        out = []
        makelabel = lambda name: ' ' * indent + name + ': '
        settingsindent = _autoindent([makelabel(s) for s in self.options], indent, maxindent)
        for name in self.option_order:
            option = self.options[name]
            label = makelabel(name)
            settingshelp = "%s(%s): %s" % (option.formatname, option.strvalue, option.location)
            wrapped = self._wrap_labelled(label, settingshelp, settingsindent, width)
            out.extend(wrapped)
        return '\n'.join(out)

def
strsettings
(
self
,
indent
=
0
,
maxindent
=
25
,
width
=
0
)
:
out
=
[
]
makelabel
=
lambda
name
:
' '
*
indent
+
name
+
': '
settingsindent
=
_autoindent
(
[
makelabel
(
s
)
for
s
in
self
.
options
]
,
indent
,
maxindent
)
for
name
in
self
.
option_order
:
option
=
self
.
options
[
name
]
label
=
makelabel
(
name
)
settingshelp
=
"%s(%s): %s"
%
(
option
.
formatname
,
option
.
strvalue
,
option
.
location
)
wrapped
=
self
.
_wrap_labelled
(
label
,
settingshelp
,
settingsindent
,
width
)
out
.
extend
(
wrapped
)
return
'\n'
.
join
(
out
)

Return user friendly help on positional arguments .


def settingshelp(self, width=0):
        """Return a summary of program options, their values and origins.

        width is maximum allowed page width, use self.width if 0.
        """
        out = []
        out.append(self._wrap(self.docs['title'], width=width))
        if self.docs['description']:
            out.append(self._wrap(self.docs['description'], indent=2, width=width))
        out.append('')
        out.append('SETTINGS:')
        out.append(self.strsettings(indent=2, width=width))
        out.append('')
        return '\n'.join(out)

def settingshelp ( self , width = 0 ) : out = [ ] out . append ( self . _wrap ( self . docs [ 'title' ] , width = width ) ) if self . docs [ 'description' ] : out . append ( self . _wrap ( self . docs [ 'description' ] , indent = 2 , width = width ) ) out . append ( '' ) out . append ( 'SETTINGS:' ) out . append ( self . strsettings ( indent = 2 , width = width ) ) out . append ( '' ) return '\n' . join ( out )

Return a summary of program options their values and origins . width is maximum allowed page width use self . width if 0 .


def parse(self, file):
        """Parse text blocks from a file."""
        if isinstance(file, basestring):
            file = open(file)
        line_number = 0
        label = None
        block = self.untagged
        for line in file:
            line_number += 1
            line = line.rstrip('\n')
            if self.tabsize > 0:
                line = line.replace('\t', ' ' * self.tabsize)
            if self.decommenter:
                line = self.decommenter.decomment(line)
                if line is None:
                    continue
            tag = line.split(':', 1)[0].strip()
            # Still in the same block?
            if tag not in self.names:
                if block is None:
                    if line and not line.isspace():
                        raise ParseError(file.name, line, "garbage before first block: %r" % line)
                    continue
                block.addline(line)
                continue
            # Open a new block.
            name = self.names[tag]
            label = line.split(':',1)[1].strip()
            if name in self.labelled_classes:
                if not label:
                    raise ParseError(file.name, line, "missing label for %r block" % name)
                block = self.blocks[name].setdefault(label, self.labelled_classes[name]())
            else:
                if label:
                    msg = "label %r present for unlabelled block %r" % (label, name)
                    raise ParseError(file.name, line_number, msg)
                block = self.blocks[name]
            block.startblock()


def parse(self, argv):
        """Pop, parse and return the first self.nargs items from args.

        if self.nargs > 1 a list of parsed values will be returned.

        Raise BadNumberOfArguments or BadArgument on errors.

        NOTE: argv may be modified in place by this method.
        """
        if len(argv) < self.nargs:
            raise BadNumberOfArguments(self.nargs, len(argv))
        if self.nargs == 1:
            return self.parse_argument(argv.pop(0))
        return [self.parse_argument(argv.pop(0)) for tmp in range(self.nargs)]

def parse ( self , argv ) : if len ( argv ) < self . nargs : raise BadNumberOfArguments ( self . nargs , len ( argv ) ) if self . nargs == 1 : return self . parse_argument ( argv . pop ( 0 ) ) return [ self . parse_argument ( argv . pop ( 0 ) ) for tmp in range ( self . nargs ) ]

Pop parse and return the first self . nargs items from args .


def parsestr(self, argstr):
        """Parse arguments found in settings files.

        Use the values in self.true for True in settings files, or those in
        self.false for False, case insensitive.
        """
        argv = shlex.split(argstr, comments=True)
        if len(argv) != 1:
            raise BadNumberOfArguments(1, len(argv))
        arg = argv[0]
        lower = arg.lower()
        if lower in self.true:
            return True
        if lower in self.false:
            return False
        raise BadArgument(arg, "Allowed values are " + self.allowed + '.')

def
parsestr
(
self
,
argstr
)
:
argv
=
shlex
.
split
(
argstr
,
comments
=
True
)
if
len
(
argv
)
!=
1
:
raise
BadNumberOfArguments
(
1
,
len
(
argv
)
)
arg
=
argv
[
0
]
lower
=
arg
.
lower
(
)
if
lower
in
self
.
true
:
return
True
if
lower
in
self
.
false
:
return
False
raise
BadArgument
(
arg
,
"Allowed values are "
+
self
.
allowed
+
'.'
)

Parse arguments found in settings files . Use the values in self . true for True in settings files or those in self . false for False case insensitive .


def get_separator(self, i):
        """Return the separator that preceding format i, or '' for i == 0."""
        return i and self.separator[min(i - 1, len(self.separator) - 1)] or ''

def get_separator ( self , i ) : return i and self . separator [ min ( i - 1 , len ( self . separator ) - 1 ) ] or ''

Return the separator that preceding format i or for i == 0 .


def authorize_url(self):
        """
        Return a URL to redirect the user to for OAuth authentication.
        """
        auth_url = OAUTH_ROOT + '/authorize'
        params = {
            'client_id': self.client_id,
            'redirect_uri': self.redirect_uri,
        }
        return "{}?{}".format(auth_url, urlencode(params))

def authorize_url ( self ) : auth_url = OAUTH_ROOT + '/authorize' params = { 'client_id' : self . client_id , 'redirect_uri' : self . redirect_uri , } return "{}?{}" . format ( auth_url , urlencode ( params ) )

Return a URL to redirect the user to for OAuth authentication .


def exchange_token(self, code):
        """
        Exchange the authorization code for an access token.
        """
        access_token_url = OAUTH_ROOT + '/access_token'
        params = {
            'client_id': self.client_id,
            'client_secret': self.client_secret,
            'redirect_uri': self.redirect_uri,
            'code': code,
        }
        resp = requests.get(access_token_url, params=params)
        if not resp.ok:
            raise MixcloudOauthError("Could not get access token.")
        return resp.json()['access_token']

def
exchange_token
(
self
,
code
)
:
access_token_url
=
OAUTH_ROOT
+
'/access_token'
params
=
{
'client_id'
:
self
.
client_id
,
'client_secret'
:
self
.
client_secret
,
'redirect_uri'
:
self
.
redirect_uri
,
'code'
:
code
,
}
resp
=
requests
.
get
(
access_token_url
,
params
=
params
)
if
not
resp
.
ok
:
raise
MixcloudOauthError
(
"Could not get access token."
)
return
resp
.
json
(
)
[
'access_token'
]

Exchange the authorization code for an access token .


def acquire(self, *args, **kwargs):
        """ Wraps Lock.acquire """
        with self._stat_lock:
            self._waiting += 1

        self._lock.acquire(*args, **kwargs)

        with self._stat_lock:
            self._locked = True
            self._waiting -= 1

def acquire ( self , * args , * * kwargs ) : with self . _stat_lock : self . _waiting += 1 self . _lock . acquire ( * args , * * kwargs ) with self . _stat_lock : self . _locked = True self . _waiting -= 1


def release(self):
        """ Wraps Lock.release """
        self._lock.release()

        with self._stat_lock:
            self._locked = False
            self._last_released = datetime.now()

def release ( self ) : self . _lock . release ( ) with self . _stat_lock : self . _locked = False self . _last_released = datetime . now ( )


def default_decoder(self, obj):
        """Handle a dict that might contain a wrapped state for a custom type."""
        typename, marshalled_state = self.unwrap_callback(obj)
        if typename is None:
            return obj

        try:
            cls, unmarshaller = self.serializer.unmarshallers[typename]
        except KeyError:
            raise LookupError('no unmarshaller found for type "{}"'.format(typename)) from None

        if cls is not None:
            instance = cls.__new__(cls)
            unmarshaller(instance, marshalled_state)
            return instance
        else:
            return unmarshaller(marshalled_state)

def
default_decoder
(
self
,
obj
)
:
typename
,
marshalled_state
=
self
.
unwrap_callback
(
obj
)
if
typename
is
:
return
obj
try
:
cls
,
unmarshaller
=
self
.
serializer
.
unmarshallers
[
typename
]
except
KeyError
:
raise
LookupError
(
'no unmarshaller found for type "{}"'
.
format
(
typename
)
)
from
if
cls
is
not
:
instance
=
cls
.
__new__
(
cls
)
unmarshaller
(
instance
,
marshalled_state
)
return
instance
else
:
return
unmarshaller
(
marshalled_state
)

Handle a dict that might contain a wrapped state for a custom type .


def wrap_state_dict(self, typename: str, state) -> Dict[str, Any]:
        """
        Wrap the marshalled state in a dictionary.

        The returned dictionary has two keys, corresponding to the ``type_key`` and ``state_key``
        options. The former holds the type name and the latter holds the marshalled state.

        :param typename: registered name of the custom type
        :param state: the marshalled state of the object
        :return: an object serializable by the serializer

        """
        return {self.type_key: typename, self.state_key: state}

def wrap_state_dict ( self , typename : str , state ) -> Dict [ str , Any ] : return { self . type_key : typename , self . state_key : state }

Wrap the marshalled state in a dictionary .


def publish(quiet, dataset_uri):
    """Enable HTTP access to a dataset.

    This only works on datasets in some systems. For example, datasets stored
    in AWS S3 object storage and Microsoft Azure Storage can be published as
    datasets accessible over HTTP. A published dataset is world readable.
    """
    access_uri = http_publish(dataset_uri)
    if not quiet:
        click.secho("Dataset accessible at ", nl=False, fg="green")
    click.secho(access_uri)

def
publish
(
quiet
,
dataset_uri
)
:
access_uri
=
http_publish
(
dataset_uri
)
if
not
quiet
:
click
.
secho
(
"Dataset accessible at "
,
nl
=
False
,
fg
=
"green"
)
click
.
secho
(
access_uri
)


def _prompt_for_values(d):
    """Update the descriptive metadata interactively.

    Uses values entered by the user. Note that the function keeps recursing
    whenever a value is another ``CommentedMap`` or a ``list``. The
    function works as passing dictionaries and lists into a function edits
    the values in place.
    """
    for key, value in d.items():
        if isinstance(value, CommentedMap):
            _prompt_for_values(value)
        elif isinstance(value, list):
            for item in value:
                _prompt_for_values(item)
        else:
            typ = type(value)

            if isinstance(value, ScalarFloat):  # Deal with ruamel.yaml floats.
                typ = float

            new_value = click.prompt(key, type=typ, default=value)
            d[key] = new_value
    return d

def
_prompt_for_values
(
d
)
:
for
key
,
value
in
d
.
items
(
)
:
if
isinstance
(
value
,
CommentedMap
)
:
_prompt_for_values
(
value
)
elif
isinstance
(
value
,
list
)
:
for
item
in
value
:
_prompt_for_values
(
item
)
else
:
typ
=
type
(
value
)
if
isinstance
(
value
,
ScalarFloat
)
:
# Deal with ruamel.yaml floats.
typ
=
float
new_value
=
click
.
prompt
(
key
,
type
=
typ
,
default
=
value
)
d
[
key
]
=
new_value
return
d

Update the descriptive metadata interactively .


def create(quiet, name, base_uri, symlink_path):
    """Create a proto dataset."""
    _validate_name(name)

    admin_metadata = dtoolcore.generate_admin_metadata(name)
    parsed_base_uri = dtoolcore.utils.generous_parse_uri(base_uri)

    if parsed_base_uri.scheme == "symlink":
        if symlink_path is None:
            raise click.UsageError("Need to specify symlink path using the -s/--symlink-path option")  # NOQA

    if symlink_path:
        base_uri = dtoolcore.utils.sanitise_uri(
            "symlink:" + parsed_base_uri.path
        )
        parsed_base_uri = dtoolcore.utils.generous_parse_uri(base_uri)

    # Create the dataset.
    proto_dataset = dtoolcore.generate_proto_dataset(
        admin_metadata=admin_metadata,
        base_uri=dtoolcore.utils.urlunparse(parsed_base_uri),
        config_path=CONFIG_PATH)

    # If we are creating a symlink dataset we need to set the symlink_path
    # attribute on the storage broker.
    if symlink_path:
        symlink_abspath = os.path.abspath(symlink_path)
        proto_dataset._storage_broker.symlink_path = symlink_abspath
    try:
        proto_dataset.create()
    except dtoolcore.storagebroker.StorageBrokerOSError as err:
        raise click.UsageError(str(err))

    proto_dataset.put_readme("")

    if quiet:
        click.secho(proto_dataset.uri)
    else:
        # Give the user some feedback and hints on what to do next.
        click.secho("Created proto dataset ", nl=False, fg="green")
        click.secho(proto_dataset.uri)
        click.secho("Next steps: ")

        step = 1

        if parsed_base_uri.scheme != "symlink":
            click.secho("{}. Add raw data, eg:".format(step))
            click.secho(
                "   dtool add item my_file.txt {}".format(proto_dataset.uri),
                fg="cyan")

            if parsed_base_uri.scheme == "file":
                # Find the abspath of the data directory for user feedback.
                data_path = proto_dataset._storage_broker._data_abspath
                click.secho("   Or use your system commands, e.g: ")
                click.secho(
                    "   mv my_data_directory {}/".format(data_path),
                    fg="cyan"
                )
            step = step + 1

        click.secho("{}. Add descriptive metadata, e.g: ".format(step))
        click.secho(
            "   dtool readme interactive {}".format(proto_dataset.uri),
            fg="cyan")
        step = step + 1

        click.secho(
            "{}. Convert the proto dataset into a dataset: ".format(step)
        )
        click.secho("   dtool freeze {}".format(proto_dataset.uri), fg="cyan")


def interactive(proto_dataset_uri):
    """Interactive prompting to populate the readme."""
    proto_dataset = dtoolcore.ProtoDataSet.from_uri(
        uri=proto_dataset_uri,
        config_path=CONFIG_PATH)

    # Create an CommentedMap representation of the yaml readme template.
    readme_template = _get_readme_template()
    yaml = YAML()
    yaml.explicit_start = True
    yaml.indent(mapping=2, sequence=4, offset=2)
    descriptive_metadata = yaml.load(readme_template)

    descriptive_metadata = _prompt_for_values(descriptive_metadata)

    # Write out the descriptive metadata to the readme file.
    stream = StringIO()

    yaml.dump(descriptive_metadata, stream)

    proto_dataset.put_readme(stream.getvalue())

    click.secho("Updated readme ", fg="green")
    click.secho("To edit the readme using your default editor:")
    click.secho(
        "dtool readme edit {}".format(proto_dataset_uri),
        fg="cyan")

Interactive prompting to populate the readme .


def edit(dataset_uri):
    """Default editor updating of readme content.
    """
    try:
        dataset = dtoolcore.ProtoDataSet.from_uri(
            uri=dataset_uri,
            config_path=CONFIG_PATH
        )
    except dtoolcore.DtoolCoreTypeError:
        dataset = dtoolcore.DataSet.from_uri(
            uri=dataset_uri,
            config_path=CONFIG_PATH
        )
    readme_content = dataset.get_readme_content()

    try:
        # Python2 compatibility.
        readme_content = unicode(readme_content, "utf-8")
    except NameError:
        pass

    edited_content = click.edit(readme_content)
    if edited_content is not None:
        _validate_and_put_readme(dataset, edited_content)
        click.secho("Updated readme ", nl=False, fg="green")
    else:
        click.secho("Did not update readme ", nl=False, fg="red")
    click.secho(dataset_uri)

def
edit
(
dataset_uri
)
:
try
:
dataset
=
dtoolcore
.
ProtoDataSet
.
from_uri
(
uri
=
dataset_uri
,
config_path
=
CONFIG_PATH
)
except
dtoolcore
.
DtoolCoreTypeError
:
dataset
=
dtoolcore
.
DataSet
.
from_uri
(
uri
=
dataset_uri
,
config_path
=
CONFIG_PATH
)
readme_content
=
dataset
.
get_readme_content
(
)
try
:
# Python2 compatibility.
readme_content
=
unicode
(
readme_content
,
"utf-8"
)
except
NameError
:
pass
edited_content
=
click
.
edit
(
readme_content
)
if
edited_content
is
not
:
_validate_and_put_readme
(
dataset
,
edited_content
)
click
.
secho
(
"Updated readme "
,
nl
=
False
,
fg
=
"green"
)
else
:
click
.
secho
(
"Did not update readme "
,
nl
=
False
,
fg
=
"red"
)
click
.
secho
(
dataset_uri
)

Default editor updating of readme content .


def show(dataset_uri):
    """Show the descriptive metadata in the readme."""
    try:
        dataset = dtoolcore.ProtoDataSet.from_uri(
            uri=dataset_uri,
            config_path=CONFIG_PATH
        )
    except dtoolcore.DtoolCoreTypeError:
        dataset = dtoolcore.DataSet.from_uri(
            uri=dataset_uri,
            config_path=CONFIG_PATH
        )
    readme_content = dataset.get_readme_content()
    click.secho(readme_content)

def show ( dataset_uri ) : try : dataset = dtoolcore . ProtoDataSet . from_uri ( uri = dataset_uri , config_path = CONFIG_PATH ) except dtoolcore . DtoolCoreTypeError : dataset = dtoolcore . DataSet . from_uri ( uri = dataset_uri , config_path = CONFIG_PATH ) readme_content = dataset . get_readme_content ( ) click . secho ( readme_content )

Show the descriptive metadata in the readme .


def write(proto_dataset_uri, input):
    """Use YAML from a file or stdin to populate the readme.

    To stream content from stdin use "-", e.g.

    echo "desc: my data" | dtool readme write <DS_URI> -
    """
    proto_dataset = dtoolcore.ProtoDataSet.from_uri(
        uri=proto_dataset_uri
    )
    _validate_and_put_readme(proto_dataset, input.read())

def write ( proto_dataset_uri , input ) : proto_dataset = dtoolcore . ProtoDataSet . from_uri ( uri = proto_dataset_uri ) _validate_and_put_readme ( proto_dataset , input . read ( ) )

Use YAML from a file or stdin to populate the readme .


def item(proto_dataset_uri, input_file, relpath_in_dataset):
    """Add a file to the proto dataset."""
    proto_dataset = dtoolcore.ProtoDataSet.from_uri(
        proto_dataset_uri,
        config_path=CONFIG_PATH)
    if relpath_in_dataset == "":
        relpath_in_dataset = os.path.basename(input_file)
    proto_dataset.put_item(input_file, relpath_in_dataset)

def item ( proto_dataset_uri , input_file , relpath_in_dataset ) : proto_dataset = dtoolcore . ProtoDataSet . from_uri ( proto_dataset_uri , config_path = CONFIG_PATH ) if relpath_in_dataset == "" : relpath_in_dataset = os . path . basename ( input_file ) proto_dataset . put_item ( input_file , relpath_in_dataset )


def metadata(proto_dataset_uri, relpath_in_dataset, key, value):
    """Add metadata to a file in the proto dataset."""
    proto_dataset = dtoolcore.ProtoDataSet.from_uri(
        uri=proto_dataset_uri,
        config_path=CONFIG_PATH)
    proto_dataset.add_item_metadata(
        handle=relpath_in_dataset,
        key=key,
        value=value)

def metadata ( proto_dataset_uri , relpath_in_dataset , key , value ) : proto_dataset = dtoolcore . ProtoDataSet . from_uri ( uri = proto_dataset_uri , config_path = CONFIG_PATH ) proto_dataset . add_item_metadata ( handle = relpath_in_dataset , key = key , value = value )

Add metadata to a file in the proto dataset .


def freeze(proto_dataset_uri):
    """Convert a proto dataset into a dataset.

    This step is carried out after all files have been added to the dataset.
    Freezing a dataset finalizes it with a stamp marking it as frozen.
    """
    proto_dataset = dtoolcore.ProtoDataSet.from_uri(
        uri=proto_dataset_uri,
        config_path=CONFIG_PATH
    )

    num_items = len(list(proto_dataset._identifiers()))
    max_files_limit = int(dtoolcore.utils.get_config_value(
        "DTOOL_MAX_FILES_LIMIT",
        CONFIG_PATH,
        10000
    ))
    assert isinstance(max_files_limit, int)
    if num_items > max_files_limit:
        click.secho(
            "Too many items ({} > {}) in proto dataset".format(
                num_items,
                max_files_limit
            ),
            fg="red"
        )
        click.secho("1. Consider splitting the dataset into smaller datasets")
        click.secho("2. Consider packaging small files using tar")
        click.secho("3. Increase the limit using the DTOOL_MAX_FILES_LIMIT")
        click.secho("   environment variable")
        sys.exit(2)

    handles = [h for h in proto_dataset._storage_broker.iter_item_handles()]
    for h in handles:
        if not valid_handle(h):
            click.secho(
                "Invalid item name: {}".format(h),
                fg="red"
            )
            click.secho("1. Consider renaming the item")
            click.secho("2. Consider removing the item")
            sys.exit(3)

    with click.progressbar(length=len(list(proto_dataset._identifiers())),
                           label="Generating manifest") as progressbar:
        try:
            proto_dataset.freeze(progressbar=progressbar)
        except dtoolcore.storagebroker.DiskStorageBrokerValidationWarning as e:
            click.secho("")
            click.secho(str(e), fg="red", nl=False)
            sys.exit(4)

    click.secho("Dataset frozen ", nl=False, fg="green")
    click.secho(proto_dataset_uri)

Convert a proto dataset into a dataset .


def cp(resume, quiet, dataset_uri, dest_base_uri):
    """Copy a dataset to a different location."""
    _copy(resume, quiet, dataset_uri, dest_base_uri)

def cp ( resume , quiet , dataset_uri , dest_base_uri ) : _copy ( resume , quiet , dataset_uri , dest_base_uri )

Copy a dataset to a different location .


def compress(obj, level=6, return_type="bytes"):
    """Compress anything to bytes or string.

    :params obj:
    :params level:
    :params return_type: if bytes, then return bytes; if str, then return
      base64.b64encode bytes in utf-8 string.
    """
    if isinstance(obj, binary_type):
        b = zlib.compress(obj, level)
    elif isinstance(obj, string_types):
        b = zlib.compress(obj.encode("utf-8"), level)
    else:
        b = zlib.compress(pickle.dumps(obj, protocol=2), level)

    if return_type == "bytes":
        return b
    elif return_type == "str":
        return base64.b64encode(b).decode("utf-8")
    else:
        raise ValueError("'return_type' has to be one of 'bytes', 'str'!")


def find_probable_year_index(self, tokens):
        """
        attempt to deduce if a pre 100 year was lost
         due to padded zeros being taken off
        """
        for index, token in enumerate(self):
            potential_year_tokens = _ymd.find_potential_year_tokens(
                token, tokens)
            if len(potential_year_tokens) == 1 and len(potential_year_tokens[0]) > 2:
                return index

def find_probable_year_index ( self , tokens ) : for index , token in enumerate ( self ) : potential_year_tokens = _ymd . find_potential_year_tokens ( token , tokens ) if len ( potential_year_tokens ) == 1 and len ( potential_year_tokens [ 0 ] ) > 2 : return index


def tzname_in_python2(namefunc):
    """Change unicode output into bytestrings in Python 2

    tzname() API changed in Python 3. It used to return bytes, but was changed
    to unicode strings
    """
    def adjust_encoding(*args, **kwargs):
        name = namefunc(*args, **kwargs)
        if name is not None and not PY3:
            name = name.encode()

        return name

    return adjust_encoding

def tzname_in_python2 ( namefunc ) : def adjust_encoding ( * args , * * kwargs ) : name = namefunc ( * args , * * kwargs ) if name is not None and not PY3 : name = name . encode ( ) return name return adjust_encoding


def _validate_fromutc_inputs(f):
    """
    The CPython version of ``fromutc`` checks that the input is a ``datetime``
    object and that ``self`` is attached as its ``tzinfo``.
    """
    @wraps(f)
    def fromutc(self, dt):
        if not isinstance(dt, datetime):
            raise TypeError("fromutc() requires a datetime argument")
        if dt.tzinfo is not self:
            raise ValueError("dt.tzinfo is not self")

        return f(self, dt)

    return fromutc

The CPython version of fromutc checks that the input is a datetime object and that self is attached as its tzinfo .


def fromutc(self, dt):
        """ Given a datetime in UTC, return local time """
        if not isinstance(dt, datetime):
            raise TypeError("fromutc() requires a datetime argument")

        if dt.tzinfo is not self:
            raise ValueError("dt.tzinfo is not self")

        # Get transitions - if there are none, fixed offset
        transitions = self.transitions(dt.year)
        if transitions is None:
            return dt + self.utcoffset(dt)

        # Get the transition times in UTC
        dston, dstoff = transitions

        dston -= self._std_offset
        dstoff -= self._std_offset

        utc_transitions = (dston, dstoff)
        dt_utc = dt.replace(tzinfo=None)

        isdst = self._naive_isdst(dt_utc, utc_transitions)

        if isdst:
            dt_wall = dt + self._dst_offset
        else:
            dt_wall = dt + self._std_offset

        _fold = int(not isdst and self.is_ambiguous(dt_wall))

        return enfold(dt_wall, fold=_fold)

Given a datetime in UTC return local time


def strip_comment_line_with_symbol(line, start):
    """Strip comments from line string.
    """
    parts = line.split(start)
    counts = [len(findall(r'(?:^|[^"\\]|(?:\\\\|\\")+)(")', part))
              for part in parts]
    total = 0
    for nr, count in enumerate(counts):
        total += count
        if total % 2 == 0:
            return start.join(parts[:nr + 1]).rstrip()
    else:  # pragma: no cover
        return line.rstrip()

def
strip_comment_line_with_symbol
(
line
,
start
)
:
parts
=
line
.
split
(
start
)
counts
=
[
len
(
findall
(
r'(?:^|[^"\\]|(?:\\\\|\\")+)(")'
,
part
)
)
for
part
in
parts
]
total
=
0
for
nr
,
count
in
enumerate
(
counts
)
:
total
+=
count
if
total
%
2
==
0
:
return
start
.
join
(
parts
[
:
nr
+
1
]
)
.
rstrip
(
)
else
:
# pragma: no cover
return
line
.
rstrip
(
)


def strip_comments(string, comment_symbols=frozenset(('#', '//'))):
    """Strip comments from json string.

    :param string: A string containing json with comments started by comment_symbols.
    :param comment_symbols: Iterable of symbols that start a line comment (default # or //).
    :return: The string with the comments removed.
    """
    lines = string.splitlines()
    for k in range(len(lines)):
        for symbol in comment_symbols:
            lines[k] = strip_comment_line_with_symbol(lines[k], start=symbol)
    return '\n'.join(lines)

def strip_comments ( string , comment_symbols = frozenset ( ( '#' , '//' ) ) ) : lines = string . splitlines ( ) for k in range ( len ( lines ) ) : for symbol in comment_symbols : lines [ k ] = strip_comment_line_with_symbol ( lines [ k ] , start = symbol ) return '\n' . join ( lines )


def picknthweekday(year, month, dayofweek, hour, minute, whichweek):
    """ dayofweek == 0 means Sunday, whichweek 5 means last instance """
    first = datetime.datetime(year, month, 1, hour, minute)

    # This will work if dayofweek is ISO weekday (1-7) or Microsoft-style (0-6),
    # Because 7 % 7 = 0
    weekdayone = first.replace(day=((dayofweek - first.isoweekday()) % 7) + 1)
    wd = weekdayone + ((whichweek - 1) * ONEWEEK)
    if (wd.month != month):
        wd -= ONEWEEK

    return wd

dayofweek == 0 means Sunday whichweek 5 means last instance


def valuestodict(key):
    """Convert a registry key's values to a dictionary."""
    dout = {}
    size = winreg.QueryInfoKey(key)[1]
    tz_res = None

    for i in range(size):
        key_name, value, dtype = winreg.EnumValue(key, i)
        if dtype == winreg.REG_DWORD or dtype == winreg.REG_DWORD_LITTLE_ENDIAN:
            # If it's a DWORD (32-bit integer), it's stored as unsigned - convert
            # that to a proper signed integer
            if value & (1 << 31):
                value = value - (1 << 32)
        elif dtype == winreg.REG_SZ:
            # If it's a reference to the tzres DLL, load the actual string
            if value.startswith('@tzres'):
                tz_res = tz_res or tzres()
                value = tz_res.name_from_string(value)

            value = value.rstrip('\x00')    # Remove trailing nulls

        dout[key_name] = value

    return dout

Convert a registry key s values to a dictionary .


def name_from_string(self, tzname_str):
        """
        Parse strings as returned from the Windows registry into the time zone
        name as defined in the registry.

        >>> from dateutil.tzwin import tzres
        >>> tzr = tzres()
        >>> print(tzr.name_from_string('@tzres.dll,-251'))
        'Dateline Daylight Time'
        >>> print(tzr.name_from_string('Eastern Standard Time'))
        'Eastern Standard Time'

        :param tzname_str:
            A timezone name string as returned from a Windows registry key.

        :return:
            Returns the localized timezone string from tzres.dll if the string
            is of the form `@tzres.dll,-offset`, else returns the input string.
        """
        if not tzname_str.startswith('@'):
            return tzname_str

        name_splt = tzname_str.split(',-')
        try:
            offset = int(name_splt[1])
        except:
            raise ValueError("Malformed timezone string.")

        return self.load_name(offset)

def
name_from_string
(
self
,
tzname_str
)
:
if
not
tzname_str
.
startswith
(
'@'
)
:
return
tzname_str
name_splt
=
tzname_str
.
split
(
',-'
)
try
:
offset
=
int
(
name_splt
[
1
]
)
except
:
raise
ValueError
(
"Malformed timezone string."
)
return
self
.
load_name
(
offset
)

Parse strings as returned from the Windows registry into the time zone name as defined in the registry .


def gettz(name):
    """
    This retrieves a time zone from the local zoneinfo tarball that is packaged
    with dateutil.

    :param name:
        An IANA-style time zone name, as found in the zoneinfo file.

    :return:
        Returns a :class:`dateutil.tz.tzfile` time zone object.

    .. warning::
        It is generally inadvisable to use this function, and it is only
        provided for API compatibility with earlier versions. This is *not*
        equivalent to ``dateutil.tz.gettz()``, which selects an appropriate
        time zone based on the inputs, favoring system zoneinfo. This is ONLY
        for accessing the dateutil-specific zoneinfo (which may be out of
        date compared to the system zoneinfo).

    .. deprecated:: 2.6
        If you need to use a specific zoneinfofile over the system zoneinfo,
        instantiate a :class:`dateutil.zoneinfo.ZoneInfoFile` object and call
        :func:`dateutil.zoneinfo.ZoneInfoFile.get(name)` instead.

        Use :func:`get_zonefile_instance` to retrieve an instance of the
        dateutil-provided zoneinfo.
    """
    warnings.warn("zoneinfo.gettz() will be removed in future versions, "
                  "to use the dateutil-provided zoneinfo files, instantiate a "
                  "ZoneInfoFile object and use ZoneInfoFile.zones.get() "
                  "instead. See the documentation for details.",
                  DeprecationWarning)

    if len(_CLASS_ZONE_INSTANCE) == 0:
        _CLASS_ZONE_INSTANCE.append(ZoneInfoFile(getzoneinfofile_stream()))
    return _CLASS_ZONE_INSTANCE[0].zones.get(name)

This retrieves a time zone from the local zoneinfo tarball that is packaged with dateutil .


def gettz_db_metadata():
    """ Get the zonefile metadata

    See `zonefile_metadata`_

    :returns:
        A dictionary with the database metadata

    .. deprecated:: 2.6
        See deprecation warning in :func:`zoneinfo.gettz`. To get metadata,
        query the attribute ``zoneinfo.ZoneInfoFile.metadata``.
    """
    warnings.warn("zoneinfo.gettz_db_metadata() will be removed in future "
                  "versions, to use the dateutil-provided zoneinfo files, "
                  "ZoneInfoFile object and query the 'metadata' attribute "
                  "instead. See the documentation for details.",
                  DeprecationWarning)

    if len(_CLASS_ZONE_INSTANCE) == 0:
        _CLASS_ZONE_INSTANCE.append(ZoneInfoFile(getzoneinfofile_stream()))
    return _CLASS_ZONE_INSTANCE[0].metadata


def get_config(jid):
    """Get the configuration for the given JID based on XMPP_HTTP_UPLOAD_ACCESS.

    If the JID does not match any rule, ``False`` is returned.
    """

    acls = getattr(settings, 'XMPP_HTTP_UPLOAD_ACCESS', (('.*', False), ))

    for regex, config in acls:
        if isinstance(regex, six.string_types):
            regex = [regex]

        for subex in regex:
            if re.search(subex, jid):
                return config

    return False

def get_config ( jid ) : acls = getattr ( settings , 'XMPP_HTTP_UPLOAD_ACCESS' , ( ( '.*' , False ) , ) ) for regex , config in acls : if isinstance ( regex , six . string_types ) : regex = [ regex ] for subex in regex : if re . search ( subex , jid ) : return config return False

Get the configuration for the given JID based on XMPP_HTTP_UPLOAD_ACCESS .


def datetime_exists(dt, tz=None):
    """
    Given a datetime and a time zone, determine whether or not a given datetime
    would fall in a gap.

    :param dt:
        A :class:`datetime.datetime` (whose time zone will be ignored if ``tz``
        is provided.)

    :param tz:
        A :class:`datetime.tzinfo` with support for the ``fold`` attribute. If
        ``None`` or not provided, the datetime's own time zone will be used.

    :return:
        Returns a boolean value whether or not the "wall time" exists in ``tz``.
    """
    if tz is None:
        if dt.tzinfo is None:
            raise ValueError('Datetime is naive and no time zone provided.')
        tz = dt.tzinfo

    dt = dt.replace(tzinfo=None)

    # This is essentially a test of whether or not the datetime can survive
    # a round trip to UTC.
    dt_rt = dt.replace(tzinfo=tz).astimezone(tzutc()).astimezone(tz)
    dt_rt = dt_rt.replace(tzinfo=None)

    return dt == dt_rt

Given a datetime and a time zone determine whether or not a given datetime would fall in a gap .


def _set_tzdata(self, tzobj):
        """ Set the time zone data of this object from a _tzfile object """
        # Copy the relevant attributes over as private attributes
        for attr in _tzfile.attrs:
            setattr(self, '_' + attr, getattr(tzobj, attr))


def normalized(self):
        """
        Return a version of this object represented entirely using integer
        values for the relative attributes.

        >>> relativedelta(days=1.5, hours=2).normalized()
        relativedelta(days=1, hours=14)

        :return:
            Returns a :class:`dateutil.relativedelta.relativedelta` object.
        """
        # Cascade remainders down (rounding each to roughly nearest
        # microsecond)
        days = int(self.days)

        hours_f = round(self.hours + 24 * (self.days - days), 11)
        hours = int(hours_f)

        minutes_f = round(self.minutes + 60 * (hours_f - hours), 10)
        minutes = int(minutes_f)

        seconds_f = round(self.seconds + 60 * (minutes_f - minutes), 8)
        seconds = int(seconds_f)

        microseconds = round(self.microseconds + 1e6 * (seconds_f - seconds))

        # Constructor carries overflow back up with call to _fix()
        return self.__class__(years=self.years, months=self.months,
                              days=days, hours=hours, minutes=minutes,
                              seconds=seconds, microseconds=microseconds,
                              leapdays=self.leapdays, year=self.year,
                              month=self.month, day=self.day,
                              weekday=self.weekday, hour=self.hour,
                              minute=self.minute, second=self.second,
                              microsecond=self.microsecond)

Return a version of this object represented entirely using integer values for the relative attributes .


def _hash(secret: bytes, data: bytes, alg: str) -> bytes:
    """
    Create a new HMAC hash.

    :param secret: The secret used when hashing data.
    :type secret: bytes
    :param data: The data to hash.
    :type data: bytes
    :param alg: The algorithm to use when hashing `data`.
    :type alg: str
    :return: New HMAC hash.
    :rtype: bytes
    """
    algorithm = get_algorithm(alg)
    return hmac \
        .new(secret, msg=data, digestmod=algorithm) \
        .digest()

def _hash ( secret : bytes , data : bytes , alg : str ) -> bytes : algorithm = get_algorithm ( alg ) return hmac . new ( secret , msg = data , digestmod = algorithm ) . digest ( )


def decode(secret: Union[str, bytes], token: Union[str, bytes],
           alg: str = default_alg) -> Tuple[dict, dict]:
    """
    Decodes the given token's header and payload and validates the signature.

    :param secret: The secret used to decode the token. Must match the
        secret used when creating the token.
    :type secret: Union[str, bytes]
    :param token: The token to decode.
    :type token: Union[str, bytes]
    :param alg: The algorithm used to decode the token. Must match the
        algorithm used when creating the token.
    :type alg: str
    :return: The decoded header and payload.
    :rtype: Tuple[dict, dict]
    """
    secret = util.to_bytes(secret)
    token = util.to_bytes(token)
    pre_signature, signature_segment = token.rsplit(b'.', 1)
    header_b64, payload_b64 = pre_signature.split(b'.')
    try:
        header_json = util.b64_decode(header_b64)
        header = json.loads(util.from_bytes(header_json))
    except (json.decoder.JSONDecodeError, UnicodeDecodeError, ValueError):
        raise InvalidHeaderError('Invalid header')
    try:
        payload_json = util.b64_decode(payload_b64)
        payload = json.loads(util.from_bytes(payload_json))
    except (json.decoder.JSONDecodeError, UnicodeDecodeError, ValueError):
        raise InvalidPayloadError('Invalid payload')

    if not isinstance(header, dict):
        raise InvalidHeaderError('Invalid header: {}'.format(header))
    if not isinstance(payload, dict):
        raise InvalidPayloadError('Invalid payload: {}'.format(payload))

    signature = util.b64_decode(signature_segment)
    calculated_signature = _hash(secret, pre_signature, alg)

    if not compare_signature(signature, calculated_signature):
        raise InvalidSignatureError('Invalid signature')
    return header, payload

def
decode
(
secret
:
Union
[
str
,
bytes
]
,
token
:
Union
[
str
,
bytes
]
,
alg
:
str
=
default_alg
)
->
Tuple
[
dict
,
dict
]
:
secret
=
util
.
to_bytes
(
secret
)
token
=
util
.
to_bytes
(
token
)
pre_signature
,
signature_segment
=
token
.
rsplit
(
b'.'
,
1
)
header_b64
,
payload_b64
=
pre_signature
.
split
(
b'.'
)
try
:
header_json
=
util
.
b64_decode
(
header_b64
)
header
=
json
.
loads
(
util
.
from_bytes
(
header_json
)
)
except
(
json
.
decoder
.
JSONDecodeError
,
UnicodeDecodeError
,
ValueError
)
:
raise
InvalidHeaderError
(
'Invalid header'
)
try
:
payload_json
=
util
.
b64_decode
(
payload_b64
)
payload
=
json
.
loads
(
util
.
from_bytes
(
payload_json
)
)
except
(
json
.
decoder
.
JSONDecodeError
,
UnicodeDecodeError
,
ValueError
)
:
raise
InvalidPayloadError
(
'Invalid payload'
)
if
not
isinstance
(
header
,
dict
)
:
raise
InvalidHeaderError
(
'Invalid header: {}'
.
format
(
header
)
)
if
not
isinstance
(
payload
,
dict
)
:
raise
InvalidPayloadError
(
'Invalid payload: {}'
.
format
(
payload
)
)
signature
=
util
.
b64_decode
(
signature_segment
)
calculated_signature
=
_hash
(
secret
,
pre_signature
,
alg
)
if
not
compare_signature
(
signature
,
calculated_signature
)
:
raise
InvalidSignatureError
(
'Invalid signature'
)
return
header
,
payload

Decodes the given token s header and payload and validates the signature .


def compare_signature(expected: Union[str, bytes],
                      actual: Union[str, bytes]) -> bool:
    """
    Compares the given signatures.

    :param expected: The expected signature.
    :type expected: Union[str, bytes]
    :param actual: The actual signature.
    :type actual: Union[str, bytes]
    :return: Do the signatures match?
    :rtype: bool
    """
    expected = util.to_bytes(expected)
    actual = util.to_bytes(actual)
    return hmac.compare_digest(expected, actual)

def compare_signature ( expected : Union [ str , bytes ] , actual : Union [ str , bytes ] ) -> bool : expected = util . to_bytes ( expected ) actual = util . to_bytes ( actual ) return hmac . compare_digest ( expected , actual )


def compare_token(expected: Union[str, bytes],
                  actual: Union[str, bytes]) -> bool:
    """
    Compares the given tokens.

    :param expected: The expected token.
    :type expected: Union[str, bytes]
    :param actual: The actual token.
    :type actual: Union[str, bytes]
    :return: Do the tokens match?
    :rtype: bool
    """
    expected = util.to_bytes(expected)
    actual = util.to_bytes(actual)
    _, expected_sig_seg = expected.rsplit(b'.', 1)
    _, actual_sig_seg = actual.rsplit(b'.', 1)
    expected_sig = util.b64_decode(expected_sig_seg)
    actual_sig = util.b64_decode(actual_sig_seg)
    return compare_signature(expected_sig, actual_sig)

def compare_token ( expected : Union [ str , bytes ] , actual : Union [ str , bytes ] ) -> bool : expected = util . to_bytes ( expected ) actual = util . to_bytes ( actual ) _ , expected_sig_seg = expected . rsplit ( b'.' , 1 ) _ , actual_sig_seg = actual . rsplit ( b'.' , 1 ) expected_sig = util . b64_decode ( expected_sig_seg ) actual_sig = util . b64_decode ( actual_sig_seg ) return compare_signature ( expected_sig , actual_sig )


def valid(self, time: int = None) -> bool:
        """
        Is the token valid? This method only checks the timestamps within the
        token and compares them against the current time if none is provided.

        :param time: The timestamp to validate against
        :type time: Union[int, None]
        :return: The validity of the token.
        :rtype: bool
        """
        if time is None:
            epoch = datetime(1970, 1, 1, 0, 0, 0)
            now = datetime.utcnow()
            time = int((now - epoch).total_seconds())
        if isinstance(self.valid_from, int) and time < self.valid_from:
            return False
        if isinstance(self.valid_to, int) and time > self.valid_to:
            return False
        return True

def valid ( self , time : int = None ) -> bool : if time is None : epoch = datetime ( 1970 , 1 , 1 , 0 , 0 , 0 ) now = datetime . utcnow ( ) time = int ( ( now - epoch ) . total_seconds ( ) ) if isinstance ( self . valid_from , int ) and time < self . valid_from : return False if isinstance ( self . valid_to , int ) and time > self . valid_to : return False return True

Is the token valid? This method only checks the timestamps within the token and compares them against the current time if none is provided .


def _pop_claims_from_payload(self):
        """
        Check for registered claims in the payload and move them to the
        registered_claims property, overwriting any extant claims.
        """
        claims_in_payload = [k for k in self.payload.keys() if
                             k in registered_claims.values()]
        for name in claims_in_payload:
            self.registered_claims[name] = self.payload.pop(name)

def _pop_claims_from_payload ( self ) : claims_in_payload = [ k for k in self . payload . keys ( ) if k in registered_claims . values ( ) ] for name in claims_in_payload : self . registered_claims [ name ] = self . payload . pop ( name )

Check for registered claims in the payload and move them to the registered_claims property overwriting any extant claims .


def encode(self) -> str:
        """
        Create a token based on the data held in the class.

        :return: A new token
        :rtype: str
        """
        payload = {}
        payload.update(self.registered_claims)
        payload.update(self.payload)
        return encode(self.secret, payload, self.alg, self.header)

def encode ( self ) -> str : payload = { } payload . update ( self . registered_claims ) payload . update ( self . payload ) return encode ( self . secret , payload , self . alg , self . header )

Create a token based on the data held in the class .


def decode(secret: Union[str, bytes], token: Union[str, bytes],
               alg: str = default_alg) -> 'Jwt':
        """
        Decodes the given token into an instance of `Jwt`.

        :param secret: The secret used to decode the token. Must match the
            secret used when creating the token.
        :type secret: Union[str, bytes]
        :param token: The token to decode.
        :type token: Union[str, bytes]
        :param alg: The algorithm used to decode the token. Must match the
            algorithm used when creating the token.
        :type alg: str
        :return: The decoded token.
        :rtype: `Jwt`
        """
        header, payload = decode(secret, token, alg)
        return Jwt(secret, payload, alg, header)

def decode ( secret : Union [ str , bytes ] , token : Union [ str , bytes ] , alg : str = default_alg ) -> 'Jwt' : header , payload = decode ( secret , token , alg ) return Jwt ( secret , payload , alg , header )

Decodes the given token into an instance of Jwt .