class etree(ebase):
    def __init__(self, fname):
        if fname is not None:
            parser = XMLParser(huge_tree=True, remove_comments=False)
            et = parse(fname, parser=parser)
        else:
            et = ElementTree(file=None)

        ebase.__init__(self, et)

    def write(self, fname, encoding=None):
        # Make sure, that we end with a newline
        self.et.getroot().tail = '\n'
        self.et.write(fname, encoding=encoding)

    def tostring(self):
        return self.et.tostring()

    def ensure_child(self, tag):
        retval = self.et.find("./" + tag)
        if retval is not None:
            return elem(retval)
        return elem(SubElement(self.et.getroot(), tag))

    def set_child_position(self, child, pos):
        root = self.et.getroot()
        root.remove(child.et)
        root.insert(pos, child.et)

    def setroot(self, tag):
        retval = elem(Element(tag))
        # pylint: disable=protected-access
        self.et._setroot(retval.et)
        return retval

    @property
    def root(self):
        return elem(self.et.getroot()) def validate_xml(fname):
    if os.path.getsize(fname) > (1 << 30):
        return ["%s is greater than 1 GiB. "
                "Elbe does not support files of this size." % fname]

    schema_file = "https://www.linutronix.de/projects/Elbe/dbsfed.xsd"
    parser = XMLParser(huge_tree=True)
    schema_tree = etree.parse(schema_file)
    schema = etree.XMLSchema(schema_tree)

    try:
        xml = parse(fname, parser=parser)

        if schema.validate(xml):
            return validate_xml_content(xml)
    except etree.XMLSyntaxError:
        return ["XML Parse error\n" + str(sys.exc_info()[1])]
    except BaseException:
        return ["Unknown Exception during validation\n" +
                str(sys.exc_info()[1])]

    # We have errors, return them in string form...
    return error_log_to_strings(schema.error_log) def copy_kinitrd(prj, target_dir):

    suite = prj.text("suite")

    try:
        tmp = TmpdirFilesystem()
        if prj.has("mirror/cdrom"):
            system('7z x -o%s "%s" initrd-cdrom.gz vmlinuz' %
                   (tmp.fname('/'), prj.text("mirror/cdrom")))

            # initrd.gz needs to be cdrom version !
            copyfile(tmp.fname("initrd-cdrom.gz"),
                     os.path.join(target_dir, "initrd.gz"))
        else:
            mirror = get_primary_mirror(prj)
            download_kinitrd(tmp, suite, mirror, prj.has("noauth"))

            copyfile(tmp.fname("initrd.gz"),
                     os.path.join(target_dir, "initrd.gz"))

        copyfile(tmp.fname("initrd-cdrom.gz"),
                 os.path.join(target_dir, "initrd-cdrom.gz"))

        copyfile(tmp.fname("vmlinuz"),
                 os.path.join(target_dir, "vmlinuz"))

    except IOError as e:
        raise NoKinitrdException('IoError %s' % str(e))
    except InvalidSignature as e:
        raise NoKinitrdException('InvalidSignature %s' % str(e))
    except HashValidationFailed as e:
        raise NoKinitrdException('HashValidationFailed %s' % str(e)) class NoKinitrdException(Exception):
    pass class ElbeDefaults:

    def __init__(self, build_type):

        assert build_type in defaults, ("Invalid buildtype %s\n"
                                        "Valid buildtypes are:\n  - %s" %
                                        (build_type, "\n  - ".join(defaults.keys())))

        self.defaults = defaults[build_type]
        self.defaults["nicmac"] = get_random_mac()

        self.generic_defaults = archindep_defaults

    def __getitem__(self, key):
        if key in self.defaults:
            return self.defaults[key]
        if key in self.generic_defaults:
            return self.generic_defaults[key]

        return None def write_template(outname, fname, d, linebreak=False):
    outfile = open(outname, "w")
    outfile.write(template(fname, d, linebreak))
    outfile.close() def get_initvm_preseed(xml):
    def_xml = etree(default_preseed_fname)

    preseed = {}
    for c in def_xml.node("/preseed"):
        k = (c.et.attrib["owner"], c.et.attrib["key"])
        v = (c.et.attrib["type"], c.et.attrib["value"])

        preseed[k] = v

    if not xml.has("./initvm/preseed"):
        return preseed

    for c in xml.node("/initvm/preseed"):
        k = (c.et.attrib["owner"], c.et.attrib["key"])
        v = (c.et.attrib["type"], c.et.attrib["value"])

        preseed[k] = v

    return preseed def init_directories(elbe_relpath):
def get_cmdlist(): class Config(dict):
    def __init__(self): def command_out(cmd, stdin=None, output=PIPE, env_add=None):
    """command_out() - Execute cmd in a shell.

    Returns a tuple with the exitcode and the output of cmd.

    --

    >>> command_out("true")
    (0, '')

    >>> command_out("$TRUE && true", env_add={"TRUE":"true"})
    (0, '')

    >>> command_out("cat -", stdin=b"ELBE")
    (0, 'ELBE')

    >>> command_out("2>&1 cat -", stdin=b"ELBE")
    (0, 'ELBE')

    >>> command_out("2>&1 cat -", stdin="ELBE")
    (0, 'ELBE')

    >>> command_out("false")
    (1, '')

    """
    new_env = os.environ.copy()
    if env_add:
        new_env.update(env_add)

    if isinstance(stdin, str):
        stdin = stdin.encode()

    if stdin is None:
        p = Popen(cmd, shell=True,
                  stdout=output, stderr=STDOUT, env=new_env)
        out, _ = p.communicate()
    else:
        p = Popen(cmd, shell=True,
                  stdout=output, stderr=STDOUT, stdin=PIPE, env=new_env)
        out, _ = p.communicate(input=stdin)

    out = TextIOWrapper(BytesIO(out), encoding='utf-8', errors='replace').read()

    return p.returncode, out def system(cmd, allow_fail=False, env_add=None):
    """system() - Execute cmd in a shell.

    Throws a CommandError if cmd returns none-zero and allow_fail=False

    --

    >>> system("true")

    >>> system("false", allow_fail=True)

    >>> system("$FALSE", env_add={"FALSE":"false"}) # doctest: +ELLIPSIS
    Traceback (most recent call last):
    ...
    elbepack.shellhelper.CommandError: ...

    """
    new_env = os.environ.copy()
    if env_add:
        new_env.update(env_add)

    ret = call(cmd, shell=True, env=new_env)

    if ret != 0:
        if not allow_fail:
            raise CommandError(cmd, ret) def do(cmd, allow_fail=False, stdin=None, env_add=None):
    """do() - Execute cmd in a shell and redirect outputs to logging.

    Throws a CommandError if cmd failed with allow_Fail=False.

    --

    Let's redirect the loggers to current stdout
    >>> import sys
    >>> from elbepack.log import open_logging
    >>> open_logging({"streams":sys.stdout})

    >>> do("true")
    [CMD] true

    >>> do("false", allow_fail=True)
    [CMD] false

    >>> do("cat -", stdin=b"ELBE")
    [CMD] cat -

    >>> do("cat - && false", stdin=b"ELBE") # doctest: +ELLIPSIS
    Traceback (most recent call last):
    ...
    elbepack.shellhelper.CommandError: ...

    >>> do("false") # doctest: +ELLIPSIS
    Traceback (most recent call last):
    ...
    elbepack.shellhelper.CommandError: ...
    """

    new_env = os.environ.copy()
    if env_add:
        new_env.update(env_add)

    if isinstance(stdin, str):
        stdin = stdin.encode()

    logging.info(cmd, extra={"context":"[CMD] "})

    r, w = os.pipe()

    if stdin is None:
        p = Popen(cmd, shell=True, stdout=w, stderr=STDOUT, env=new_env)
    else:
        p = Popen(cmd, shell=True, stdin=PIPE, stdout=w, stderr=STDOUT, env=new_env)

    async_logging(r, w, soap, log)
    p.communicate(input=stdin)

    if p.returncode and not allow_fail:
        raise CommandError(cmd, p.returncode) def system_out(cmd, stdin=None, allow_fail=False, env_add=None):
    """system_out() - Wrapper around command_out().

    On failure, raises an exception if allow_fail=False, on success,
    returns the output of cmd.

    --

    >>> system_out("false") # doctest: +ELLIPSIS
    Traceback (most recent call last):
    ...
    elbepack.shellhelper.CommandError: ...

    >>> system_out("false", allow_fail=True)
    ''

    """
    code, out = command_out(cmd, stdin=stdin, env_add=env_add)

    if code != 0:
        if not allow_fail:
            raise CommandError(cmd, code)

    return out @contextmanager
def elbe_logging(*args, **kwargs):
    try:
        open_logging(*args, **kwargs)
        yield
    finally:
        close_logging() class Filesystem:

    # pylint: disable=too-many-public-methods

    def __init__(self, path, clean=False):
        """
        >>> os.path.isdir(this.path)
        True
        """

        self.path = os.path.abspath(path)

        if clean:
            shutil.rmtree(self.path, True)
            os.makedirs(self.path)

    def fname(self, path):
        """
        >>> expect = os.path.join(this.path, "fname")
        >>> this.fname("/fname") == expect == this.fname("fname")
        True
        """
        if path.startswith('/'):
            path = path[1:]
        return os.path.join(self.path, path)

    def open(self, path, mode="r"):
        """
        >>> this.open("open") # doctest: +ELLIPSIS
        Traceback (most recent call last):
        ...
        FileNotFoundError: [Errno 2] ...

        >>> this.open("open", mode="w") # doctest: +ELLIPSIS
        <_io.TextIOWrapper ...>

        >>> _.close()
        """
        return open(self.fname(path), mode)

    def open_gz(self, path, mode="r"):
        """
        >>> this.open_gz("open_gz") # doctest: +ELLIPSIS
        Traceback (most recent call last):
        ...
        FileNotFoundError: [Errno 2] ...

        >>> this.open_gz("open_gz", mode="w") # doctest: +ELLIPSIS
        <gzip _io.BufferedWriter ...>

        >>> _.close()
        """
        return gzip.open(self.fname(path), mode)

    def isdir(self, path):
        """
        >>> this.isdir("isdir")
        False

        >>> os.makedirs(this.fname("isdir"))
        >>> this.isdir("isdir")
        True
        """
        return os.path.isdir(self.fname(path))

    def islink(self, path):
        """
        >>> this.islink("islink")
        False

        >>> os.symlink("target", this.fname("islink"))
        >>> this.islink("islink")
        True
        """
        return os.path.islink(self.fname(path))

    def isfile(self, path):
        """
        >>> this.isfile("isfile")
        False

        >>> open(this.fname("isfile"), mode="w").close()
        >>> this.isfile("isfile")
        True
        """
        return os.path.isfile(self.fname(path))

    def exists(self, path):
        """
        >>> this.exists("exists")
        False

        >>> os.symlink("broken", this.fname("exixsts-broken-link"))
        >>> this.exists("exists-broken-link")
        False

        >>> open(this.fname("exists"), mode="w").close()
        >>> this.exists("exists")
        True
        """
        return os.path.exists(self.fname(path))

    def lexists(self, path):
        """
        >>> this.lexists("lexists")
        False

        >>> os.symlink("target", this.fname("lexists"))
        >>> os.path.lexists(this.fname("lexists"))
        True
        """
        return os.path.lexists(self.fname(path))

    def mkdir(self, path):
        """
        >>> os.path.isdir(this.fname("mkdir"))
        False
        >>> this.mkdir("mkdir")
        >>> os.path.isdir(this.fname("mkdir"))
        True
        """
        os.makedirs(self.realpath(path))

    def readlink(self, path):
        """
        >>> this.symlink("target", "readlink")
        >>> this.readlink("readlink")
        'target'
        """
        return os.readlink(self.fname(path))

    def realpath(self, path):
        """
        >>> this.realpath(".") == this.path
        True

        >>> this.realpath("..") == this.path
        True

        >>> this.realpath("/realpath") == this.fname("/realpath")
        True

        >>> this.symlink("/target", "realpath-abs-symlink")
        >>> this.realpath("realpath-abs-symlink") == this.fname("/target")
        True

        >>> this.symlink("target", "/realpath-symlink")
        >>> this.realpath("realpath-symlink") == this.fname("/target")
        True

        >>> this.symlink(".././realpath-symlink", "realpath-multi-symlink")
        >>> this.realpath("realpath-multi-symlink") == this.fname("/target")
        True

        >>> this.symlink("realpath-loop-A", "realpath-loop-B")
        >>> this.symlink("realpath-loop-B", "realpath-loop-A")
        >>> this.realpath("realpath-loop-A") == this.fname("/realpath-loop-A")
        True

        >>> this.realpath("realpath-loop-B") == this.fname("realpath-loop-B")
        True
        """
        path = path.split(os.sep)
        path.reverse()
        following = []
        real_path = [self.path]

        while path:
            candidate = path.pop()

            # Don't care
            if candidate in ('', os.curdir):
                continue

            # Can't go out of RFS
            if candidate == os.pardir:
                if following:
                    following.pop()
                if len(real_path) > 1:
                    real_path.pop()
                continue

            parent = os.sep.join(real_path)
            new_path = os.path.join(parent, candidate)
            if not os.path.islink(new_path):
                if following:
                    following.pop()
                real_path.append(candidate)
                continue

            # Circular loop; Don't follow it
            if new_path in following:
                real_path.append(candidate)
                continue

            following.append(new_path)
            link = os.readlink(new_path)

            # Reset root for absolute link
            if os.path.isabs(link):
                real_path = [self.path]

            for element in reversed(link.split(os.sep)):
                path.append(element)

        return os.sep.join(real_path)

    def symlink(self, src, path, allow_exists=False):
        """
        >>> this.symlink("target", "symlink-link")
        >>> os.readlink(this.fname("symlink-link"))
        'target'

        >>> this.symlink("target", "symlink-link") # doctest: +ELLIPSIS
        Traceback (most recent call last):
        ...
        FileExistsError: [Errno 17] ...

        >>> os.readlink(this.fname("not-a-link")) # doctest: +ELLIPSIS
        Traceback (most recent call last):
        ...
        FileNotFoundError: [Errno 2] ...

        >>> this.symlink("target", "symlink-link", allow_exists=True)
        """
        try:
            os.symlink(src, self.fname(path))
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise
            if not allow_exists:
                raise

    def stat(self, path):
        return os.stat(self.fname(path))

    def chown(self, path, uid, gid):
        os.chown(self.fname(path), uid, gid)

    def chmod(self, path, mode):
        os.chmod(self.fname(path), mode)

    def utime(self, path, times=None):
        os.utime(self.fname(path), times)

    def cat_file(self, inf):
        content = []
        try:
            f = self.open(inf)
            content = f.readlines()
            f.close()
        except IOError:
            pass
        return content

    def remove(self, path, noerr=False):
        """
        >>> this.remove("remove") # doctest: +ELLIPSIS
        Traceback (most recent call last):
        ...
        FileNotFoundError: [Errno 2] ...

        >>> this.remove("remove", noerr=True)

        >>> open(this.fname("remove"), "w").close()
        >>> this.remove("remove")

        >>> this.lexists("remove")
        False
        """
        try:
            return os.remove(self.fname(path))
        except BaseException:
            if not noerr:
                raise

    def rmtree(self, path):
        """
        >>> this.mkdir("/rmtree/rmtree")
        >>> this.rmtree("/rmtree")
        >>> this.lexists("/rmtree/rmtree")
        False
        >>> this.lexists("/rmtree")
        False
        """
        shutil.rmtree(self.fname(path))

    def listdir(self, path='', ignore=None, skiplinks=False):
        if not ignore:
            ignore = []

        retval = [
            os.path.join(
                self.path,
                path,
                x) for x in os.listdir(
                self.fname(path)) if x not in ignore]

        if skiplinks:
            retval = [
                x for x in retval if (
                    not os.path.islink(x)) and os.path.isdir(x)]

        return retval

    def glob(self, path):
        flist = glob(self.fname(path))
        for i in flist:
            if not i.startswith(self.path):
                raise IOError("Broken glob '%s'" % path)

        return flist

    @staticmethod
    def _write_file(path, f, cont, mode):
        f.write(cont)
        f.close()
        if mode is not None:
            os.chmod(path, mode)

    def write_file(self, path, mode, cont):
        path = self.realpath(path)
        self._write_file(path, open(path, "w"), cont, mode)

    def append_file(self, path, cont, mode=None):
        path = self.realpath(path)
        self._write_file(path, open(path, "a"), cont, mode)

    def read_file(self, path, gz=False):
        path = self.realpath(path)
        if gz:
            fp = gzip.open(path, "r")
        else:
            fp = open(path, "r")

        with fp:
            retval = fp.read()

        return retval

    def mkdir_p(self, newdir, mode=0o755):
        """works the way a good mkdir -p would...
                - already exists, silently complete
                - regular file in the way, raise an exception
                - parent directory(ies) does not exist, make them as well

        --
        >>> this.mkdir_p("mkdir_p/foo/bar")
        >>> this.isdir("mkdir_p")
        True

        >>> this.isdir("mkdir_p/foo")
        True

        >>> this.isdir("mkdir_p/foo/bar")
        True

        >>> open(this.fname("mkdir_p/foo/bar/baz"), "w").close()

        >>> this.mkdir_p("mkdir_p/foo/bar/baz") # doctest: +ELLIPSIS
        Traceback (most recent call last):
        ...
        OSError: a file with the same name as the desired dir, ...
        """
        if self.isdir(newdir):
            pass
        elif self.isfile(newdir):
            raise OSError(
                "a file with the same name as the desired "
                "dir, '%s', already exists. in RFS %s" %
                (newdir, self.path))
        else:
            self.mkdir(newdir)
            self.chmod(newdir, mode)

    def touch_file(self, fname):
        if self.exists(fname):
            self.utime(fname)
        else:
            fp = self.open(fname, "w")
            fp.close()

    def walk_files(self, directory='', exclude_dirs=None):
        # pylint: disable=too-many-locals
        if not exclude_dirs:
            exclude_dirs = []

        dirname = self.fname(directory)
        if dirname == "/":
            striplen = 0
        else:
            striplen = len(dirname)
        for dirpath, dirnames, filenames in os.walk(dirname):
            subpath = dirpath[striplen:]
            if not subpath:
                subpath = "/"

            deldirs = []
            for d in dirnames:
                dpath = os.path.join(subpath, d)
                if dpath in exclude_dirs:
                    deldirs.append(d)

            for d in deldirs:
                dirnames.remove(d)

            for f in filenames:
                fpath = os.path.join(subpath, f)
                realpath = os.path.join(dirpath, f)
                yield "/" + fpath, realpath

    def mtime_snap(self, dirname='', exclude_dirs=None):
        if not exclude_dirs:
            exclude_dirs = []
        mtime_index = {}

        for fpath, realpath in self.walk_files(dirname, exclude_dirs):
            stat = os.lstat(realpath)
            mtime_index[fpath] = stat.st_mtime

        return mtime_index

    def __disk_usage(self, directory):
        size = os.path.getsize(directory)

        for i in os.listdir(directory):
            full = os.path.join(directory, i)
            if os.path.isfile(full):
                size += os.path.getsize(full)
            elif os.path.isdir(full):
                size += self.__disk_usage(full)

        return size

    def disk_usage(self, dirname=''):
        directory = self.fname(dirname)
        return self.__disk_usage(directory)
import os
import sys
import shutil
import logging
from optparse import OptionParser
from elbepack.treeutils import etree
from elbepack.validate import validate_xml
from elbepack.debinstaller import copy_kinitrd, NoKinitrdException
from elbepack.xmldefaults import ElbeDefaults
from elbepack.version import elbe_version
from elbepack.templates import write_template, get_initvm_preseed
from elbepack.directories import init_template_dir, elbe_dir
from elbepack.config import cfg
from elbepack.shellhelper import command_out, system, do, system_out
from elbepack.log import elbe_logging
from elbepack.filesystem import Filesystem
            machine_type = 'pc-i440fx-2.6'
        else:
            machine_type = 'pc'

        try:
            os.makedirs(opt.directory)
        except OSError as e:
            logging.error("unable to create project directory: %s (%s)",
                          opt.directory,
                          e.strerror)
            sys.exit(30)

        out_path = os.path.join(opt.directory, ".elbe-in")
        try:
            os.makedirs(out_path)
        except OSError as e:
            logging.error("unable to create subdirectory: %s (%s)",
                          out_path,
                          e.strerror)
            sys.exit(30)

        initvm_http_proxy = http_proxy.replace('http://localhost:',
                                               'http://10.0.2.2:')
        d = {"elbe_version": elbe_version,
             "defs": defs,
             "opt": opt,
             "xml": xml,
             "prj": xml.node("/initvm"),
             "http_proxy": initvm_http_proxy,
             "pkgs": xml.node("/initvm/pkg-list") or [],
             "preseed": get_initvm_preseed(xml),
             "machine_type": machine_type,
             "cfg": cfg}

        if http_proxy != "":
            os.putenv("http_proxy", http_proxy)
            os.putenv("https_proxy", http_proxy)
            os.putenv("no_proxy", "localhost,127.0.0.1")

        try:
            copy_kinitrd(xml.node("/initvm"), out_path)
        except NoKinitrdException as e:
            msg = str(e)
            logging.error("Failure to download kernel/initrd debian Package:")
            logging.error("")
            logging.error(msg)
            logging.error("")
            logging.error("Check Mirror configuration")
            if 'SHA256SUMS' in msg:
                logging.error("If you use debmirror please read "
                              "https://github.com/Linutronix/elbe/issues/188 "
                              "on how to work around the issue")
            sys.exit(20)

        templates = os.listdir(init_template_dir)

        make_executable = ["init-elbe.sh.mako",
                           "preseed.cfg.mako"]

        for t in templates:
            o = t.replace(".mako", "")

            if t in ("Makefile.mako", "libvirt.xml.mako"):
                write_template(
                    os.path.join(
                        opt.directory, o), os.path.join(
                        init_template_dir, t), d, linebreak=True)
            else:
                write_template(
                    os.path.join(
                        out_path, o), os.path.join(
                        init_template_dir, t), d, linebreak=False)

            if t in make_executable:
                os.chmod(os.path.join(out_path, o), 0o755)

        shutil.copyfile(args[0],
                        os.path.join(out_path, "source.xml"))


        keys = []
        for key in xml.all(".//initvm/mirror/url-list/url/raw-key"):
            keys.append(key.et.text)

        if opt.cdrom:
            cmd = '7z x -so "%s" repo.pub' % opt.cdrom
            keys.append(system_out(cmd))

        import_keyring = os.path.join(out_path, "elbe-keyring")

        do('gpg --no-options \
                --no-default-keyring \
                --keyring %s --import' % import_keyring,
           stdin="".join(keys).encode('ascii'),
           allow_fail=True,
           env_add={'GNUPGHOME': out_path})

        export_keyring = import_keyring + ".gpg"

        # No need to set GNUPGHOME because both input and output
        # keyring files are specified.

        do('gpg --no-options \
                --no-default-keyring \
                --keyring %s \
                --export \
                --output %s' % (import_keyring, export_keyring))

        if opt.devel:
            out_real = os.path.realpath(out_path)
            opts = []
            if out_real.startswith(elbe_dir + os.sep):
                opts.append('--exclude "%s"' %
                            os.path.relpath(out_path, start=elbe_dir))

            opts.append("--exclude-vcs")
            opts.append("--exclude-vcs-ignores")
            opts.append("--exclude='elbe-build*'")
            opts.append("--exclude='docs/*'")
            tar_fname = os.path.join(out_path, "elbe-devel.tar.bz2")

            system('tar cfj "%s" %s -C "%s" .' % (tar_fname,