#!/usr/bin/python 

# This is a version of DCC Snapshot 2.0 that runs under Python 2.4. 
# It no longer prints out the handy help text specified by 
# later-supported epilog param of optparse. Since this is meant to 
# run in an automated fashion and is quite verbose with its error  
# messages, I feel this change is worthwhile.

# Last revision: February 22nd, 2011 by <NAME>

import os
import sys
import re
import shutil
import stat
import optparse
import ConfigParser
import logging

log = logging.getLogger("dcc_snapshot")

class IsFileError(IOError):
    def __init__ (self, boguspath):
        self.fault = boguspath
    def __str__ (self):
        return "IsFileError implies that this \
is a file when I wanted a directory: " + self.fault
    def __repr__(self):
        return "IsFileError(" + repr(self.fault) +")"

class DestinationClobberError(IOError):
    def __init__ (self, boguspath):
        self.fault = boguspath
    def __str__ (self):
        return "DestinationClobberError implies that this is probably not\
 something you wanted me to delete: " + self.fault
    def __repr__(self):
        return "DestinationClobberError(" + repr(self.fault) +")"

class PluginManager(object):
    def __init__ (self, plugcfg = None):
        self.plugins = set()
        self.plugcfg = plugcfg
    
    def directory_linked(self, src, dest, isNew):
        for plugin in self.plugins:
            try:
                plugin.directory_linked(src, dest, isNew)
            except AttributeError:
                log.debug(str(plugin) + " has no directory_linked")

    def done_linking_directories(self):
        for plugin in self.plugins:
            try:
                plugin.done_linking_directories()
            except (AttributeError):
                log.debug("That plugin doesn't care if we're done")

    def sweeping(self, old, new):
        for plugin in self.plugins:
            try:
                plugin.sweeping(old, new)
            except AttributeError:
                log.debug(str(plugin) + " has no sweeping")

    def done_sweeping(self):
        for plugin in self.plugins:
            try:
                plugin.done_sweeping()
            except (AttributeError):
                log.debug("That plugin doesn't care if we're done")

    def directory_complete(self, path):
        for plugin in self.plugins:
            try:
                plugin.directory_complete(path)
            except (AttributeError):
                log.debug("That plugin has no directory_complete")

    def add_plugin_path(self, path):
        path = path.strip()
        if len(path) == 0:
            log.warning("add_plugin_path called with blank path")
            return
        sys.path.insert(0, path)
        
    def add_plugin(self, candidate_module):
        candidate_module = candidate_module.strip()
        if len(candidate_module) == 0:
            log.warning("add_plugin called with blank path")
            return
        try:
            plugin_mod = __import__(candidate_module)
            if plugin_mod not in plugins:
                plugins.add(plugin_mod)
                plugin_mod.initialize_plugin(self.plugcfg)
        except ImportError:
            if(candidate_module.endswith((".py", ".PY"))):
                log.info("Removing trailing .py from plugin name: " + candidate_module)
                self.add_plugin(candidate_module[:-3])
                return
            log.error("Could not load requested plugin module at all: "
                    + candidate_module)
        except AttributeError:
            log.debug("Plugin had no initialize_plugin, probably")
        #let TypeError be an error

level_regex = re.compile(r"(?P<key>.*\.Level_\d+\.\d+\.)"
                        r"(?P<revision>\d+)\.(?P<build>\d+)$"
                        )
magetab_regex = re.compile(r"(?P<key>.*\.mage-tab\.\d+\.)"
                        r"(?P<revision>\d+)\.(?P<build>\d+)$"
                        )
plugs = PluginManager()

def max_shove(dic, key, value):
    """Update a key in the dictionary if the previous value was lower.
    
    dic -- dictionary to insert into
    key -- key to query
    value -- proposed new value
    
    The dictionary is queried for the key. If the key is not in the
    dictionary at all, it is inserted with the specified value.
    If it is in the dictionary, its corresponding value is compared
    to the provided value. If the new value is higher, the value in
    the dictionary is updated and the function returns True (same
    as if there is no key); if it is not, the dictionary is unchanged
    and the function returns False."""
    if (key not in dic) or value > dic[key]:
        dic[key] = value
        return True
    return False

def slay_the_weaker(dic, key, value):
    """Delete the less-up-to-date directory by the same name, if any.
    
    Parameters:
    dic -- Dictionary of absolute paths minus version numbers to
            version numbers
    key -- String representing the absolute path minus version number
            to check against
    value -- Version number of candidate for new maximum or deletion
    
    If the key does not exist in the table, it is created and this
    function returns False. If the key does exist, values are compared.
    The directory corresponding to the lower value is recursively
    deleted, and the higher value is inserted into the table. If the
    values are equal, the function simply returns, although this case
    shouldn't happen in the context of how the function was initially
    written.
    
    Calls on-sweep plugins before the sweep."""
    
    if not(key in dic):
        dic[key] = value
        log.debug("No competitor for directory " + key +"."+str(value))
        return False
    log.debug("Comparing two directories: " + key + " of " + str(dic[key]) +
              " and " + str(value))
    if value > dic[key]:
        low = key + str(dic[key][0]) + "." + str(dic[key][1])
        high = key + str(value[0]) + "." + str(value[1])
        dic[key] = value
    elif value == dic[key]:
        log.error("slay_the_weaker found a clone: " +key+"."+str(value))
        return False
    else:
        low = key + str(value[0]) + "." + str(value[1])
        high = key + str(dic[key][0]) + "." + str(dic[key][1])
    log.info("Sweeping out of date directory: " + low)
    plugs.sweeping(low, high)
    shutil.rmtree(low)
    return True

def dont_kill_the_repository(path, recursive = True):
    """Throw an exception if the specified path is not safe to write.
    
    Parameters:
    path -- Path to check for safety
    recursive -- Whether or not to check subdirectories for safety,
                 as opposed to blindly assuming it
    Throws:
        A DestinationClobberError if it looks like the specified directory
        is anything other than a snapshot directory.
    A snapshot directory should contain only symlinks and subdirectories
    that are themselves valid snapshot directories. Anything else
    might be real data and is not safe to blindly overwrite with this
    specific tool, which should only be writing into a new directory or
    overwriting a previous snapshot directory because its write behavior
    is highly destructive."""
    log.debug("Making sure destroying " + path + " isn't ruinous")
    if not os.path.isdir(path):
        log.critical(path + ", slated for destruction, is not a directory. " + 
                    "abort in dont_kill_the_repository")
        raise DestinationClobberError(path)
    for name in os.listdir(path):
        checkpath = os.path.join(path, name)
        if not os.path.islink(checkpath):
            if os.path.isdir(checkpath):
                if recursive:
                    dont_kill_the_repository(checkpath)
            else:
                log.critical(name + " is a real file in " +
                        "condemned directory " + path + ". abort in " +
                        "dont_kill_the_repository")
                raise DestinationClobberError(checkpath)
    log.debug("dont_kill_the_repository says " + path + " is okay")
    return True

def recursive_symlinkify(src, dest):
    """Create a recursive symlink copy in dest of src.
    
    Parameters:
    src -- Path to make a shallow symlink copy of
    dest -- Where to put it
    If dest is already populated, it is verified to be safe to overwrite
    using a recursive call to dont_kill_the_repository. It is then
    deleted and simply rewritten; no attempt is made to "update" such
    a directory. A directory is safe to overwrite if and only if it
    contains only symlinks and subdirectories that are, by the same
    rule, safe to overwrite."""
    log.debug("recursively shadowing " + src + " into " + dest)
    if not os.path.isdir(src):
        log.critical("recursive_symlinkify can't recurse into " +
                    src + " because it isn't a directory. abort")
        raise IsFileError(src)
    if os.path.exists(dest):
        if os.path.isfile(dest):
            log.critical(dest + " exists as a file and " +
                    "recursive_symlinkify is trying to write a " +
                    "directory over it, which is probably bad. abort")
            raise DestinationClobberError(dest)
        if os.path.isdir(dest):
            dont_kill_the_repository(dest)
            shutil.rmtree(dest)
        else:
            log.critical(dest+" exists, but isn't a file or directory" +
                    " and recursive_symlinkify has no idea what to do" +
                    " with that kind of thing. abort")
            raise IOError
    os.mkdir(dest)
    for name in os.listdir(src):
        srcpath = os.path.join(src, name)
        destpath = os.path.join(dest, name)
        if os.path.isdir(srcpath):
            recursive_symlinkify(srcpath, destpath)
        elif os.path.isfile(srcpath):
            log.debug("writing symlink: " + srcpath + "  ==>  " +
                        destpath)
            os.symlink(srcpath, destpath)

def sweep(path):
    """Delete everything but the most up-to-date revisions."""
    log.info("Cleaning up old directories in " + path)
    dont_kill_the_repository(path, False) #sweep is called depth-first
                        #so setting recurse to true will cause d^2 perf
    matchtable = {}
    for entry in os.listdir(path):
        fullpath = os.path.join(path, entry)
        if not os.path.isdir(fullpath):
            continue
        match = level_regex.match(entry)
        if match is None:
            match = magetab_regex.match(entry)
        if match is None:
            log.warning("Non-snapshot directory found in sweep target: "
                    + fullpath)
            continue
        #now it's one of the types
        myrevision = (int(match.group("revision")),
                      int(match.group("build")))
        slay_the_weaker(matchtable, 
                        os.path.join(path, match.group("key")),
                        myrevision)

def recursive_latest(src, dest):
    """Create a snapshot of the latest data in the directory.
    
    Recurses on all subdirectories. Latest is defined as, for each
    major version of a MAGE_TAB directory, the highest-numbered minor
    version; for a leve_n directory, for each major.minor version,
    the highest-numbered revision. (Assumes major.minor.revis.build)"""
    
    #destination directory-making postponed: only happens if
    #there's anything in this entire tree that we care about

    print "         in recursive_latest ", src, dest
    
    matchtable = {}
    log.info("Scanning " + src)
    for entry in os.listdir(src):
        srcpath = os.path.join(src, entry)
        destpath = os.path.join(dest, entry)
        if os.path.isfile(srcpath):
            continue
        log.debug("at " + src + " processing " + entry)
        match = level_regex.match(entry)
        if match is None:
            match = magetab_regex.match(entry)
        #now if it's none, it's neither type
        if match is not None:
            log.debug("Comparing/inserting " + entry)
            myrevision = (int(match.group("revision")), 
                          int(match.group("build")))
            max_shove(matchtable, match.group("key"), myrevision)
        else:
            recursive_latest(srcpath, destpath)
            
    #matchtable now contains the highest edition for each
    log.debug("table of " + src + ": " + str(matchtable))
    if len(matchtable):
        log.info("Found objects in " + src + ", mirroring into " + dest)
        if not os.path.exists(dest):
            os.makedirs(dest)
        elif os.path.isfile(dest):
            raise IsFileError(dest)
    for primary, edition in matchtable.iteritems():
        name = primary + str(edition[0]) + "." + str(edition[1])
        srcpath = os.path.join(src, name)
        destpath = os.path.join(dest, name)
        is_new = not os.path.exists(destpath)
        recursive_symlinkify(os.path.abspath(srcpath), destpath)
        plugs.directory_linked(srcpath, destpath, is_new)
    
    if len(matchtable):
        sweep(dest)
        plugs.directory_complete(dest)
    
    return len(matchtable)

def recursive_chmod(path, value):
    """Recursively sets directory permissions."""
    for root, dirs, files in os.walk(path):
        for directory in dirs:
            log.debug("Chmodding " + root + "/" + directory)
            try:
                os.chmod(os.path.join(root, directory), value)
            except OSError:
                log.error("Couldn't chmod"+os.path.join(root, directory) 
                      + "-- moving on")

if __name__ == "__main__":
    detailtext = """Creates a snapshot of the latest data in a dcc tree.
Data is identified as a directory ending in Level_[n].[version]
or mage-tab.[version], where a Level directory is expected
to have a four-part version and a mage-tab expected to have
a three-part version.

The "latest" data is the highest-revision Level data for each
uniquely-named level_* root and each major.minor version, and
the highest-minor-version mage-tab data for each uniquely-named
mage-tab root.

The snapshot is a clone of the directory structure
stripped down to directories that contain data,
and data directories themselves only have the latest, which
is a symlink to the real data directory.

SRC: the directory to make a snapshot of.
DEST: the directory to create the snapshot in. If the name does
not contain "snapshot" somewhere on the path, it will be
rejected unless the -f/--force option is used.
CONFIGFILE: Instead of using SRC/DEST, a configuration file can
be specified instead. Using a configuration file does not imply
-f, so if the config file specifies a destination without
the text "snapshot" in its path, -f or --force must be used."""
    parser = optparse.OptionParser(
                                usage="%prog [options] SRC DEST\n"
                                "\t%prog [options] CONFIGFILE",
                                version="%prog 2.1")
    parser.add_option("-l", "--logfile", dest="logfile", default=None,
            help="Specify a log file- default is console")
    parser.add_option("-v", "--verbosity", dest="verbosity",
            default="INFO", type="string",
            help="Logging noise level; chooose from "
                 "ERROR, WARN, INFO, or DEBUG. "
                 "Please be aware that DEBUG will produce "
                 "extremely large logs, as every symlink is recorded. "
                 "Defualt is INFO.")
    parser.add_option("-o", "--override", dest="override",
            action="store_true", default=False,
            help="Override a specified configuration file. "
                 "Can only be used when a configuration file is used. "
                 "Causes -v and -l settings to override settings in "
                 "specified configuration file.")
    parser.add_option("-f", "--force", dest="force", 
            action="store_true", default=False,
            help="Suppress check that dest contains 'snapshot'")
    parser.add_option("-d", "--plugin-directories", dest="plug_path",
            default=None, help="Path to search for plugins")
    parser.add_option("-p", "--plugins", dest="plugins", default=None,
            help="Plugin scripts to use after copies/before removals")
    parser.add_option("-c", "--plugcfg", dest="plugcfg", default=None,
            help="Configuration file for all plugins (it's up to plugins to use separate sections)")
            
    flags, args = parser.parse_args()
    src = dest = config = None
    if len(args) == 0:
        parser.error("No configuration file or SRC/DEST specified")
    elif len(args)==1:
        # config mode
        config = ConfigParser.SafeConfigParser(
                {
                    "verbosity":"INFO",
                    "logfile":None,
                    "plugins":None,
                    "plugin-directories":None,
                    "plugcfg":None
                }
            )
        config.read(args[0])
        try:
            src = config.get("dcc-snapshot", "src")
            dest = config.get("dcc-snapshot", "dest")
        except IOError:
            if src is None:
                parser.error("Configuration file did not contain 'src'")
            if dest is None:
                parser.error("Configuration file did not contain 'dest'")
    elif len(args) == 2:
        #src dest mode
        src = args[0]
        dest = args[1]
    else:
        parser.error("Too many arguments.")
    
    if not flags.force:
        if dest.find("snapshot") == -1:
            parser.error("Destination argument " + dest +
                         " may not be a snapshot directory (failed check"
                         " for 'snapshot' in the name). Specify a "
                         "snapshot directory instead, or use the -f "
                         "option to override this check.")
    
    logger = logging.getLogger("dcc_snapshot")
    handler = vosity = None
    if config is None or (flags.override and flags.logfile is not None):
        if flags.logfile is None:
            handler = logging.StreamHandler()
        else:
            handler = logging.FileHandler(flags.logfile)
    else:
        lfile = config.get("dcc-snapshot", "logfile")
        if lfile is None:
            lfile = flags.logfile
            
        if lfile is None or lfile == "":
            handler = logging.StreamHandler()
        else:
            handler = logging.FileHandler(lfile)
        
    formatter = logging.Formatter("%(asctime)s - %(name)s - "
                                         "%(levelname)s - %(message)s")
    vosity = flags.verbosity.upper()
    if config is not None and not flags.override:
        temp = config.get("dcc-snapshot", "verbosity")
        if temp is not None:
            vosity = temp

    if vosity == "CRITICAL":
        handler.setLevel(logging.CRITICAL)
        logger.setLevel(logging.CRITICAL)
    elif vosity == "ERROR":
        handler.setLevel(logging.ERROR)
        logger.setLevel(logging.ERROR)
    elif vosity == "WARNING" or vosity == "WARN":
        handler.setLevel(logging.WARNING)
        logger.setLevel(logging.WARNING)
    elif vosity == "INFO" :
        handler.setLevel(logging.INFO)
        logger.setLevel(logging.INFO)
    elif vosity == "DEBUG":
        handler.setLevel(logging.DEBUG)
        logger.setLevel(logging.DEBUG)
    else:
        parser.error("verbosity: Invalid value")
    logger.addHandler(handler)
    logger.debug("Logger initialized.")

    if flags.override:
        if flags.plugcfg:
            plugs.plugcfg = ConfigParser.SafeConfigParser()
            plugs.plugcfg.read(flags.plugcfg)
        elif config:
            plugcfg = config.get("dcc-snapshot", "plugcfg")
            if plugcfg:
                plugs.plugcfg = ConfigParser.SafeConfigParser()
                plugs.plugcfg.read(plugcfg)
    else:
        if config:
            plugcfg = config.get("dcc-snapshot", "plugcfg")
            if plugcfg:
                plugs.plugcfg = ConfigParser.SafeConfigParser()
                plugs.plugcfg.read(plugcfg)
        elif flags.plugcfg:
            plugs.plugcfg = ConfigParser.SafeConfigParser()
            plugs.plugcfg.read(flags.plugcfg)


    if flags.plug_path:
        for path in flags.plug_path.split(";"):
            plugs.add_plugin_path(path)

    if config:
        paths = config.get("dcc-snapshot", "plugin-directories")
        if paths is not None:
            for path in paths.split(";"):
                plugs.add_plugin_path(path)

    if flags.plugins:
        for plug in flags.plugins.split(";"):
            plugs.add_plugin(plug)

    if config:
        plug_group = config.get("dcc-snapshot", "plugins")
        if plug_group is not None:
            for plug in plug_group.split(";"):
                plugs.add_plugin(plug)

    recursive_latest(src, dest)
    plugs.done_linking_directories()
    plugs.done_sweeping()
    ## this seems to be causing problems ??? SMR 6-dec-12 
    ## recursive_chmod(dest, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) #world-writable: 777


#!/usr/bin/python 

# This is a version of DCC Snapshot 2.0 that runs under Python 2.4. 
# It no longer prints out the handy help text specified by 
# later-supported epilog param of optparse. Since this is meant to 
# run in an automated fashion and is quite verbose with its error  
# messages, I feel this change is worthwhile.

# Last revision: February 22nd, 2011 by <NAME>

import os
import sys
import re
import shutil
import stat
import optparse
import ConfigParser
import logging

log = logging.getLogger("dcc_snapshot")

class IsFileError(IOError):
    def __init__ (self, boguspath):
        self.fault = boguspath
    def __str__ (self):
        return "IsFileError implies that this \
is a file when I wanted a directory: " + self.fault
    def __repr__(self):
        return "IsFileError(" + repr(self.fault) +")"

class DestinationClobberError(IOError):
    def __init__ (self, boguspath):
        self.fault = boguspath
    def __str__ (self):
        return "DestinationClobberError implies that this is probably not\
 something you wanted me to delete: " + self.fault
    def __repr__(self):
        return "DestinationClobberError(" + repr(self.fault) +")"

class PluginManager(object):
    def __init__ (self, plugcfg = None):
        self.plugins = set()
        self.plugcfg = plugcfg
    
    def directory_linked(self, src, dest, isNew):
        for plugin in self.plugins:
            try:
                plugin.directory_linked(src, dest, isNew)
            except AttributeError:
                log.debug(str(plugin) + " has no directory_linked")

    def done_linking_directories(self):
        for plugin in self.plugins:
            try:
                plugin.done_linking_directories()
            except (AttributeError):
                log.debug("That plugin doesn't care if we're done")

    def sweeping(self, old, new):
        for plugin in self.plugins:
            try:
                plugin.sweeping(old, new)
            except AttributeError:
                log.debug(str(plugin) + " has no sweeping")

    def done_sweeping(self):
        for plugin in self.plugins:
            try:
                plugin.done_sweeping()
            except (AttributeError):
                log.debug("That plugin doesn't care if we're done")

    def directory_complete(self, path):
        for plugin in self.plugins:
            try:
                plugin.directory_complete(path)
            except (AttributeError):
                log.debug("That plugin has no directory_complete")

    def add_plugin_path(self, path):
        path = path.strip()
        if len(path) == 0:
            log.warning("add_plugin_path called with blank path")
            return
        sys.path.insert(0, path)
        
    def add_plugin(self, candidate_module):
        candidate_module = candidate_module.strip()
        if len(candidate_module) == 0:
            log.warning("add_plugin called with blank path")
            return
        try:
            plugin_mod = __import__(candidate_module)
            if plugin_mod not in plugins:
                plugins.add(plugin_mod)
                plugin_mod.initialize_plugin(self.plugcfg)
        except ImportError:
            if(candidate_module.endswith((".py", ".PY"))):
                log.info("Removing trailing .py from plugin name: " + candidate_module)
                self.add_plugin(candidate_module[:-3])
                return
            log.error("Could not load requested plugin module at all: "
                    + candidate_module)
        except AttributeError:
            log.debug("Plugin had no initialize_plugin, probably")
        #let TypeError be an error

level_regex = re.compile(r"(?P<key>.*\.Level_\d+\.\d+\.)"
                        r"(?P<revision>\d+)\.(?P<build>\d+)$"
                        )
magetab_regex = re.compile(r"(?P<key>.*\.mage-tab\.\d+\.)"
                        r"(?P<revision>\d+)\.(?P<build>\d+)$"
                        )
plugs = PluginManager()

def max_shove(dic, key, value):
    """Update a key in the dictionary if the previous value was lower.
    
    dic -- dictionary to insert into
    key -- key to query
    value -- proposed new value
    
    The dictionary is queried for the key. If the key is not in the
    dictionary at all, it is inserted with the specified value.
    If it is in the dictionary, its corresponding value is compared
    to the provided value. If the new value is higher, the value in
    the dictionary is updated and the function returns True (same
    as if there is no key); if it is not, the dictionary is unchanged
    and the function returns False."""
    if (key not in dic) or value > dic[key]:
        dic[key] = value
        return True
    return False

def slay_the_weaker(dic, key, value):
    """Delete the less-up-to-date directory by the same name, if any.
    
    Parameters:
    dic -- Dictionary of absolute paths minus version numbers to
            version numbers
    key -- String representing the absolute path minus version number
            to check against
    value -- Version number of candidate for new maximum or deletion
    
    If the key does not exist in the table, it is created and this
    function returns False. If the key does exist, values are compared.
    The directory corresponding to the lower value is recursively
    deleted, and the higher value is inserted into the table. If the
    values are equal, the function simply returns, although this case
    shouldn't happen in the context of how the function was initially
    written.
    
    Calls on-sweep plugins before the sweep."""
    
    if not(key in dic):
        dic[key] = value
        log.debug("No competitor for directory " + key +"."+str(value))
        return False
    log.debug("Comparing two directories: " + key + " of " + str(dic[key]) +
              " and " + str(value))
    if value > dic[key]:
        low = key + str(dic[key][0]) + "." + str(dic[key][1])
        high = key + str(value[0]) + "." + str(value[1])
        dic[key] = value
    elif value == dic[key]:
        log.error("slay_the_weaker found a clone: " +key+"."+str(value))
        return False
    else:
        low = key + str(value[0]) + "." + str(value[1])
        high = key + str(dic[key][0]) + "." + str(dic[key][1])
    log.info("Sweeping out of date directory: " + low)
    plugs.sweeping(low, high)
    shutil.rmtree(low)
    return True

def dont_kill_the_repository(path, recursive = True):
    """Throw an exception if the specified path is not safe to write.
    
    Parameters:
    path -- Path to check for safety
    recursive -- Whether or not to check subdirectories for safety,
                 as opposed to blindly assuming it
    Throws:
        A DestinationClobberError if it looks like the specified directory
        is anything other than a snapshot directory.
    A snapshot directory should contain only symlinks and subdirectories
    that are themselves valid snapshot directories. Anything else
    might be real data and is not safe to blindly overwrite with this
    specific tool, which should only be writing into a new directory or
    overwriting a previous snapshot directory because its write behavior
    is highly destructive."""
    log.debug("Making sure destroying " + path + " isn't ruinous")
    if not os.path.isdir(path):
        log.critical(path + ", slated for destruction, is not a directory. " + 
                    "abort in dont_kill_the_repository")
        raise DestinationClobberError(path)
    for name in os.listdir(path):
        checkpath = os.path.join(path, name)
        if not os.path.islink(checkpath):
            if os.path.isdir(checkpath):
                if recursive:
                    dont_kill_the_repository(checkpath)
            else:
                log.critical(name + " is a real file in " +
                        "condemned directory " + path + ". abort in " +
                        "dont_kill_the_repository")
                raise DestinationClobberError(checkpath)
    log.debug("dont_kill_the_repository says " + path + " is okay")
    return True

def recursive_symlinkify(src, dest):
    """Create a recursive symlink copy in dest of src.
    
    Parameters:
    src -- Path to make a shallow symlink copy of
    dest -- Where to put it
    If dest is already populated, it is verified to be safe to overwrite
    using a recursive call to dont_kill_the_repository. It is then
    deleted and simply rewritten; no attempt is made to "update" such
    a directory. A directory is safe to overwrite if and only if it
    contains only symlinks and subdirectories that are, by the same
    rule, safe to overwrite."""
    log.debug("recursively shadowing " + src + " into " + dest)
    if not os.path.isdir(src):
        log.critical("recursive_symlinkify can't recurse into " +
                    src + " because it isn't a directory. abort")
        raise IsFileError(src)
    if os.path.exists(dest):
        if os.path.isfile(dest):
            log.critical(dest + " exists as a file and " +
                    "recursive_symlinkify is trying to write a " +
                    "directory over it, which is probably bad. abort")
            raise DestinationClobberError(dest)
        if os.path.isdir(dest):
            dont_kill_the_repository(dest)
            shutil.rmtree(dest)
        else:
            log.critical(dest+" exists, but isn't a file or directory" +
                    " and recursive_symlinkify has no idea what to do" +
                    " with that kind of thing. abort")
            raise IOError
    os.mkdir(dest)
    for name in os.listdir(src):
        srcpath = os.path.join(src, name)
        destpath = os.path.join(dest, name)
        if os.path.isdir(srcpath):
            recursive_symlinkify(srcpath, destpath)
        elif os.path.isfile(srcpath):
            log.debug("writing symlink: " + srcpath + "  ==>  " +
                        destpath)
            os.symlink(srcpath, destpath)

def sweep(path):
    """Delete everything but the most up-to-date revisions."""
    log.info("Cleaning up old directories in " + path)
    dont_kill_the_repository(path, False) #sweep is called depth-first
                        #so setting recurse to true will cause d^2 perf
    matchtable = {}
    for entry in os.listdir(path):
        fullpath = os.path.join(path, entry)
        if not os.path.isdir(fullpath):
            continue
        match = level_regex.match(entry)
        if match is None:
            match = magetab_regex.match(entry)
        if match is None:
            log.warning("Non-snapshot directory found in sweep target: "
                    + fullpath)
            continue
        #now it's one of the types
        myrevision = (int(match.group("revision")),
                      int(match.group("build")))
        slay_the_weaker(matchtable, 
                        os.path.join(path, match.group("key")),
                        myrevision)

def recursive_latest(src, dest):
    """Create a snapshot of the latest data in the directory.
    
    Recurses on all subdirectories. Latest is defined as, for each
    major version of a MAGE_TAB directory, the highest-numbered minor
    version; for a leve_n directory, for each major.minor version,
    the highest-numbered revision. (Assumes major.minor.revis.build)"""
    
    #destination directory-making postponed: only happens if
    #there's anything in this entire tree that we care about

    print "         in recursive_latest ", src, dest
    
    matchtable = {}
    log.info("Scanning " + src)
    for entry in os.listdir(src):
        srcpath = os.path.join(src, entry)
        destpath = os.path.join(dest, entry)
        if os.path.isfile(srcpath):
            continue
        log.debug("at " + src + " processing " + entry)
        match = level_regex.match(entry)
        if match is None:
            match = magetab_regex.match(entry)
        #now if it's none, it's neither type
        if match is not None:
            log.debug("Comparing/inserting " + entry)
            myrevision = (int(match.group("revision")), 
                          int(match.group("build")))
            max_shove(matchtable, match.group("key"), myrevision)
        else:
            recursive_latest(srcpath, destpath)
            
    #matchtable now contains the highest edition for each
    log.debug("table of " + src + ": " + str(matchtable))
    if len(matchtable):
        log.info("Found objects in " + src + ", mirroring into " + dest)
        if not os.path.exists(dest):
            os.makedirs(dest)
        elif os.path.isfile(dest):
            raise IsFileError(dest)
    for primary, edition in matchtable.iteritems():
        name = primary + str(edition[0]) + "." + str(edition[1])
        srcpath = os.path.join(src, name)
        destpath = os.path.join(dest, name)
        is_new = not os.path.exists(destpath)
        recursive_symlinkify(os.path.abspath(srcpath), destpath)
        plugs.directory_linked(srcpath, destpath, is_new)
    
    if len(matchtable):
        sweep(dest)
        plugs.directory_complete(dest)
    
    return len(matchtable)

def recursive_chmod(path, value):
    """Recursively sets directory permissions."""
    for root, dirs, files in os.walk(path):
        for directory in dirs:
            log.debug("Chmodding " + root + "/" + directory)
            try:
                os.chmod(os.path.join(root, directory), value)
            except OSError:
                log.error("Couldn't chmod"+os.path.join(root, directory) 
                      + "-- moving on")

if __name__ == "__main__":
    detailtext = """Creates a snapshot of the latest data in a dcc tree.
Data is identified as a directory ending in Level_[n].[version]
or mage-tab.[version], where a Level directory is expected
to have a four-part version and a mage-tab expected to have
a three-part version.

The "latest" data is the highest-revision Level data for each
uniquely-named level_* root and each major.minor version, and
the highest-minor-version mage-tab data for each uniquely-named
mage-tab root.

The snapshot is a clone of the directory structure
stripped down to directories that contain data,
and data directories themselves only have the latest, which
is a symlink to the real data directory.

SRC: the directory to make a snapshot of.
DEST: the directory to create the snapshot in. If the name does
not contain "snapshot" somewhere on the path, it will be
rejected unless the -f/--force option is used.
CONFIGFILE: Instead of using SRC/DEST, a configuration file can
be specified instead. Using a configuration file does not imply
-f, so if the config file specifies a destination without
the text "snapshot" in its path, -f or --force must be used."""
    parser = optparse.OptionParser(
                                usage="%prog [options] SRC DEST\n"
                                "\t%prog [options] CONFIGFILE",
                                version="%prog 2.1")
    parser.add_option("-l", "--logfile", dest="logfile", default=None,
            help="Specify a log file- default is console")
    parser.add_option("-v", "--verbosity", dest="verbosity",
            default="INFO", type="string",
            help="Logging noise level; chooose from "
                 "ERROR, WARN, INFO, or DEBUG. "
                 "Please be aware that DEBUG will produce "
                 "extremely large logs, as every symlink is recorded. "
                 "Defualt is INFO.")
    parser.add_option("-o", "--override", dest="override",
            action="store_true", default=False,
            help="Override a specified configuration file. "
                 "Can only be used when a configuration file is used. "
                 "Causes -v and -l settings to override settings in "
                 "specified configuration file.")
    parser.add_option("-f", "--force", dest="force", 
            action="store_true", default=False,
            help="Suppress check that dest contains 'snapshot'")
    parser.add_option("-d", "--plugin-directories", dest="plug_path",
            default=None, help="Path to search for plugins")
    parser.add_option("-p", "--plugins", dest="plugins", default=None,
            help="Plugin scripts to use after copies/before removals")
    parser.add_option("-c", "--plugcfg", dest="plugcfg", default=None,
            help="Configuration file for all plugins (it's up to plugins to use separate sections)")
            
    flags, args = parser.parse_args()
    src = dest = config = None
    if len(args) == 0:
        parser.error("No configuration file or SRC/DEST specified")
    elif len(args)==1:
        # config mode
        config = ConfigParser.SafeConfigParser(
                {
                    "verbosity":"INFO",
                    "logfile":None,
                    "plugins":None,
                    "plugin-directories":None,
                    "plugcfg":None
                }
            )
        config.read(args[0])
        try:
            src = config.get("dcc-snapshot", "src")
            dest = config.get("dcc-snapshot", "dest")
        except IOError:
            if src is None:
                parser.error("Configuration file did not contain 'src'")
            if dest is None:
                parser.error("Configuration file did not contain 'dest'")
    elif len(args) == 2:
        #src dest mode
        src = args[0]
        dest = args[1]
    else:
        parser.error("Too many arguments.")
    
    if not flags.force:
        if dest.find("snapshot") == -1:
            parser.error("Destination argument " + dest +
                         " may not be a snapshot directory (failed check"
                         " for 'snapshot' in the name). Specify a "
                         "snapshot directory instead, or use the -f "
                         "option to override this check.")
    
    logger = logging.getLogger("dcc_snapshot")
    handler = vosity = None
    if config is None or (flags.override and flags.logfile is not None):
        if flags.logfile is None:
            handler = logging.StreamHandler()
        else:
            handler = logging.FileHandler(flags.logfile)
    else:
        lfile = config.get("dcc-snapshot", "logfile")
        if lfile is None:
            lfile = flags.logfile
            
        if lfile is None or lfile == "":
            handler = logging.StreamHandler()
        else:
            handler = logging.FileHandler(lfile)
        
    formatter = logging.Formatter("%(asctime)s - %(name)s - "
                                         "%(levelname)s - %(message)s")
    vosity = flags.verbosity.upper()
    if config is not None and not flags.override:
        temp = config.get("dcc-snapshot", "verbosity")
        if temp is not None:
            vosity = temp

    if vosity == "CRITICAL":
        handler.setLevel(logging.CRITICAL)
        logger.setLevel(logging.CRITICAL)
    elif vosity == "ERROR":
        handler.setLevel(logging.ERROR)
        logger.setLevel(logging.ERROR)
    elif vosity == "WARNING" or vosity == "WARN":
        handler.setLevel(logging.WARNING)
        logger.setLevel(logging.WARNING)
    elif vosity == "INFO" :
        handler.setLevel(logging.INFO)
        logger.setLevel(logging.INFO)
    elif vosity == "DEBUG":
        handler.setLevel(logging.DEBUG)
        logger.setLevel(logging.DEBUG)
    else:
        parser.error("verbosity: Invalid value")
    logger.addHandler(handler)
    logger.debug("Logger initialized.")

    if flags.override:
        if flags.plugcfg:
            plugs.plugcfg = ConfigParser.SafeConfigParser()
            plugs.plugcfg.read(flags.plugcfg)
        elif config:
            plugcfg = config.get("dcc-snapshot", "plugcfg")
            if plugcfg:
                plugs.plugcfg = ConfigParser.SafeConfigParser()
                plugs.plugcfg.read(plugcfg)
    else:
        if config:
            plugcfg = config.get("dcc-snapshot", "plugcfg")
            if plugcfg:
                plugs.plugcfg = ConfigParser.SafeConfigParser()
                plugs.plugcfg.read(plugcfg)
        elif flags.plugcfg:
            plugs.plugcfg = ConfigParser.SafeConfigParser()
            plugs.plugcfg.read(flags.plugcfg)


    if flags.plug_path:
        for path in flags.plug_path.split(";"):
            plugs.add_plugin_path(path)

    if config:
        paths = config.get("dcc-snapshot", "plugin-directories")
        if paths is not None:
            for path in paths.split(";"):
                plugs.add_plugin_path(path)

    if flags.plugins:
        for plug in flags.plugins.split(";"):
            plugs.add_plugin(plug)

    if config:
        plug_group = config.get("dcc-snapshot", "plugins")
        if plug_group is not None:
            for plug in plug_group.split(";"):
                plugs.add_plugin(plug)

    recursive_latest(src, dest)
    plugs.done_linking_directories()
    plugs.done_sweeping()
    ## this seems to be causing problems ??? SMR 6-dec-12 
    ## recursive_chmod(dest, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) #world-writable: 777


#!/usr/bin/python # This is a version of DCC Snapshot 2.0 that runs under Python 2.4. # It no longer prints out the handy help text specified by # later-supported epilog param of optparse. Since this is meant to # run in an automated fashion and is quite verbose with its error # messages, I feel this change is worthwhile. # Last revision: February 22nd, 2011 by <NAME> #let TypeError be an error Update a key in the dictionary if the previous value was lower.          dic -- dictionary to insert into     key -- key to query     value -- proposed new value          The dictionary is queried for the key. If the key is not in the     dictionary at all, it is inserted with the specified value.     If it is in the dictionary, its corresponding value is compared     to the provided value. If the new value is higher, the value in     the dictionary is updated and the function returns True (same     as if there is no key); if it is not, the dictionary is unchanged     and the function returns False. Delete the less-up-to-date directory by the same name, if any.          Parameters:     dic -- Dictionary of absolute paths minus version numbers to             version numbers     key -- String representing the absolute path minus version number             to check against     value -- Version number of candidate for new maximum or deletion          If the key does not exist in the table, it is created and this     function returns False. If the key does exist, values are compared.     The directory corresponding to the lower value is recursively     deleted, and the higher value is inserted into the table. If the     values are equal, the function simply returns, although this case     shouldn't happen in the context of how the function was initially     written.          Calls on-sweep plugins before the sweep. Throw an exception if the specified path is not safe to write.          Parameters:     path -- Path to check for safety     recursive -- Whether or not to check subdirectories for safety,                  as opposed to blindly assuming it     Throws:         A DestinationClobberError if it looks like the specified directory         is anything other than a snapshot directory.     A snapshot directory should contain only symlinks and subdirectories     that are themselves valid snapshot directories. Anything else     might be real data and is not safe to blindly overwrite with this     specific tool, which should only be writing into a new directory or     overwriting a previous snapshot directory because its write behavior     is highly destructive. Create a recursive symlink copy in dest of src.          Parameters:     src -- Path to make a shallow symlink copy of     dest -- Where to put it     If dest is already populated, it is verified to be safe to overwrite     using a recursive call to dont_kill_the_repository. It is then     deleted and simply rewritten; no attempt is made to "update" such     a directory. A directory is safe to overwrite if and only if it     contains only symlinks and subdirectories that are, by the same     rule, safe to overwrite. Delete everything but the most up-to-date revisions. #sweep is called depth-first #so setting recurse to true will cause d^2 perf #now it's one of the types Create a snapshot of the latest data in the directory.          Recurses on all subdirectories. Latest is defined as, for each     major version of a MAGE_TAB directory, the highest-numbered minor     version; for a leve_n directory, for each major.minor version,     the highest-numbered revision. (Assumes major.minor.revis.build) #destination directory-making postponed: only happens if #there's anything in this entire tree that we care about #now if it's none, it's neither type #matchtable now contains the highest edition for each Recursively sets directory permissions. Creates a snapshot of the latest data in a dcc tree. Data is identified as a directory ending in Level_[n].[version] or mage-tab.[version], where a Level directory is expected to have a four-part version and a mage-tab expected to have a three-part version.  The "latest" data is the highest-revision Level data for each uniquely-named level_* root and each major.minor version, and the highest-minor-version mage-tab data for each uniquely-named mage-tab root.  The snapshot is a clone of the directory structure stripped down to directories that contain data, and data directories themselves only have the latest, which is a symlink to the real data directory.  SRC: the directory to make a snapshot of. DEST: the directory to create the snapshot in. If the name does not contain "snapshot" somewhere on the path, it will be rejected unless the -f/--force option is used. CONFIGFILE: Instead of using SRC/DEST, a configuration file can be specified instead. Using a configuration file does not imply -f, so if the config file specifies a destination without the text "snapshot" in its path, -f or --force must be used. # config mode #src dest mode ## this seems to be causing problems ??? SMR 6-dec-12 ## recursive_chmod(dest, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) #world-writable: 777