def get_versioned_filename(name, version, suffix):
    return quote(name, ' ') + '_' + quote(version, ' ') + suffix def dump_fullpkgs(xml, rfs, cache):
    xml.clear_full_pkglist()

    instpkgs = cache.get_installed_pkgs()
    for p in instpkgs:
        xml.append_full_pkg(p)

    sources_list = xml.xml.ensure_child('sources_list')
    slist = rfs.read_file("etc/apt/sources.list")
    sources_list.set_text(slist)

    try:
        preferences = xml.xml.ensure_child('apt_prefs')
        prefs = rfs.read_file("etc/apt/preferences")
        preferences.set_text(prefs)
    except IOError:
        pass def gen_update_pkg(project, xml_filename, upd_filename,
                   override_buildtype=None, skip_validate=False, debug=False,
                   cmd_dir=None, cfg_dir=None):

    # pylint: disable=too-many-arguments
    # pylint: disable=too-many-locals
    # pylint: disable=too-many-statements
    # pylint: disable=too-many-branches

    if xml_filename:
        xml = ElbeXML(xml_filename, buildtype=override_buildtype,
                      skip_validate=skip_validate)

        if not xml.has("fullpkgs"):
            raise MissingData("Xml does not have fullpkgs list")

        if not project.xml.has("fullpkgs"):
            raise MissingData("Source Xml does not have fullpkgs list")

        if not project.buildenv.rfs:
            raise MissingData("Target does not have a build environment")

        cache = project.get_rpcaptcache()

        instpkgs = cache.get_installed_pkgs()
        instindex = {}

        for p in instpkgs:
            instindex[p.name] = p

        xmlpkgs = xml.node("/fullpkgs")
        xmlindex = {}

        fnamelist = []

        for p in xmlpkgs:
            name = p.et.text
            ver = p.et.get('version')

            xmlindex[name] = p

            if name not in instindex:
                logging.info("Package removed: %s", name)
                continue

            ipkg = instindex[name]
            comp = cache.compare_versions(ipkg.installed_version, ver)

            pfname = ipkg.installed_deb

            if comp == 0:
                logging.info("Package ok: %s-%s", name, ipkg.installed_version)
                if debug:
                    fnamelist.append(pfname)
                continue

            if comp > 0:
                logging.info("Package upgrade: %s", pfname)
                fnamelist.append(pfname)
            else:
                logging.info("Package downgrade: %s-%s",
                             name, ipkg.installed_version)

        for p in instpkgs:
            if p.name in xmlindex:
                continue

            logging.info("Package %s newly installed", p.name)
            pfname = p.installed_deb
            fnamelist.append(pfname)

    update = os.path.join(project.builddir, "update")

    if os.path.exists(update):
        rmtree(update)

    system('mkdir -p %s' % update)

    if xml_filename:
        repodir = os.path.join(update, "repo")

        repo = UpdateRepo(xml, repodir)

        for fname in fnamelist:
            path = os.path.join(
                project.chrootpath,
                "var/cache/apt/archives",
                fname)
            repo.includedeb(path)

        repo.finalize()

        dump_fullpkgs(project.xml, project.buildenv.rfs, cache)

        project.xml.xml.write(os.path.join(update, "new.xml"))
        system("cp %s %s" % (xml_filename,
                             os.path.join(update, "base.xml")))
    else:
        system("cp source.xml update/new.xml")

    if project.presh_file:
        copyfile(project.presh_file, update + '/pre.sh')
        os.chmod(update + '/pre.sh', 0o755)

    if project.postsh_file:
        copyfile(project.postsh_file, update + '/post.sh')
        os.chmod(update + '/post.sh', 0o755)

    if cmd_dir:
        inlucdedir(update, 'cmd', cmd_dir, mode=0o755)

    if cfg_dir:
        inlucdedir(update, 'conf', cfg_dir)

    create_zip_archive(upd_filename, update, ".")

    if project.postbuild_file:
        logging.info("Postbuild script")
        cmd = ' "%s %s %s"' % (upd_filename,
                               project.xml.text("project/version"),
                               project.xml.text("project/name"))
        do(project.postbuild_file + cmd, allow_fail=True) def gen_binpkg_archive(ep, repodir):
    repopath = path.join(ep.builddir, repodir)

    try:
        rmtree(repopath)
    except OSError as e:
        if e.errno != errno.ENOENT:
            raise

    # Create archive directory for packages we have to download
    ep.buildenv.rfs.mkdir_p('/var/cache/elbe/pkgarchive')

    try:
        # Repository containing all packages currently installed
        repo = ArchiveRepo(ep.xml, repopath, "Elbe",
                           "Elbe package archive", ["main"])

        c = ep.get_rpcaptcache()
        pkglist = c.get_installed_pkgs()

        for pkg in pkglist:
            # Use package from local APT archive, if the file exists
            filename = pkg.installed_deb
            rel_path = path.join('var/cache/apt/archives', filename)
            abs_path = ep.buildenv.rfs.fname(rel_path)

            if not path.isfile(abs_path):
                # Package file does not exist, download it and adjust path name
                logging.warning('Package file "%s" not found in var/cache/apt/archives, downloading it',
                                filename)
                abs_path = ep.buildenv.rfs.fname(rel_path)
                pkg_id = "%s-%s" % (pkg.name, pkg.installed_version)
                try:
                    abs_path = c.download_binary(pkg.name,
                                                 '/var/cache/elbe/pkgarchive',
                                                 pkg.installed_version)
                except ValueError:
                    logging.error('No package "%s"', pkg_id)
                    raise
                except FetchError:
                    logging.error('Package "%s" could not be downloaded', pkg_id)
                    raise
                except TypeError:
                    logging.error('Package "%s" missing name or version', pkg_id)
                    raise

            # Add package to repository
            # XXX Use correct component
            repo.includedeb(abs_path, "main")

        repo.finalize()

    finally:
        rmtree(ep.buildenv.rfs.fname('var/cache/elbe/pkgarchive'))
        repo.finalize() def checkout_binpkg_archive(ep, repodir):

    # pylint: disable=too-many-locals
    # pylint: disable=too-many-statements
    # pylint: disable=too-many-branches

    repopath = path.join(ep.builddir, repodir)
    sources_list = ep.buildenv.rfs.fname('etc/apt/sources.list')
    sources_list_d = ep.buildenv.rfs.fname('etc/apt/sources.list.d')
    sources_list_backup = path.join(ep.builddir, 'sources.list.orig')
    sources_list_d_backup = path.join(ep.builddir, 'sources.list.d.orig')
    pkgarchive = ep.buildenv.rfs.fname('var/cache/elbe/pkgarchive')

    with ep.buildenv:
        try:
            # Copy the package archive into the buildenv,
            # so the RPCAptCache can access it
            logging.info("Copying package archive into build environment")
            copytree(repopath, pkgarchive)

            # Move original etc/apt/sources.list and etc/apt/sources.list.d out
            # of the way
            logging.info("Moving original APT configuration out of the way")
            if path.isfile(sources_list):
                move(sources_list, sources_list_backup)
            if path.isdir(sources_list_d):
                move(sources_list_d, sources_list_d_backup)

            # Now create our own, with the package archive being the only
            # source
            logging.info("Creating new /etc/apt/sources.list")
            deb = "deb file:///var/cache/elbe/pkgarchive "
            deb += ep.xml.text("/project/suite")
            deb += " main"
            with open(sources_list, 'w') as f:
                f.write(deb)

            # We need to update the APT cache to apply the changed package
            # source
            logging.info("Updating APT cache to use package archive")
            ep.drop_rpcaptcache()
            c = ep.get_rpcaptcache()
            c.update()

            # Iterate over all packages, and mark them for installation or
            # deletion, using the same logic as in commands/updated.py
            logging.info("Calculating packages to install/remove")
            fpl = ep.xml.node("fullpkgs")
            pkgs = c.get_pkglist('all')

            for p in pkgs:
                marked = False
                for fpi in fpl:
                    if p.name == fpi.et.text:
                        version = fpi.et.get('version')
                        logging.info('Install "%s-%s"', p.name, version)
                        c.mark_install(p.name, version,
                                       from_user=not fpi.et.get('auto'),
                                       nodeps=True)
                        marked = True

                if not marked:
                    logging.info('Delete "%s-%s"', p.name, version)
                    c.mark_delete(p.name)

            # Now commit the changes
            logging.info("Commiting package changes")
            c.commit()
        finally:
            # If we changed the package sources, move back the backup
            if path.isdir(sources_list_d_backup) or \
                    path.isfile(sources_list_backup):
                logging.info("Moving back original APT configuration")
                update_needed = True
            else:
                update_needed = False

            if path.isdir(sources_list_d_backup):
                move(sources_list_d_backup, sources_list_d)

            if path.isfile(sources_list_backup):
                if path.isfile(sources_list):
                    remove(sources_list)
                move(sources_list_backup, sources_list)

            # Remove the package archive from the buildenv
            if path.isdir(pkgarchive):
                logging.info("Removing package archive from build environment")
                rmtree(pkgarchive)

            # Update APT cache, if we modified the package sources
            if update_needed:
                logging.info("Updating APT cache to use original package sources")
                ep.drop_rpcaptcache()
                ep.get_rpcaptcache().update() class DebootstrapException (Exception):
    def __init__(self):
        Exception.__init__(self, "Debootstrap Failed") class AptCacheCommitError(Exception):
    def __init__(self, msg=''):
        Exception.__init__(self, "Error Committing rpcaptcache %s" % msg) class AptCacheUpdateError(Exception):
    def __init__(self, e):
        Exception.__init__(self, "Error Updating rpcaptcache: " + str(e)) def do(cmd, allow_fail=False, stdin=None, env_add=None):
    """do() - Execute cmd in a shell and redirect outputs to logging.

    Throws a CommandError if cmd failed with allow_Fail=False.

    --

    Let's redirect the loggers to current stdout
    >>> import sys
    >>> from elbepack.log import open_logging
    >>> open_logging({"streams":sys.stdout})

    >>> do("true")
    [CMD] true

    >>> do("false", allow_fail=True)
    [CMD] false

    >>> do("cat -", stdin=b"ELBE")
    [CMD] cat -

    >>> do("cat - && false", stdin=b"ELBE") # doctest: +ELLIPSIS
    Traceback (most recent call last):
    ...
    elbepack.shellhelper.CommandError: ...

    >>> do("false") # doctest: +ELLIPSIS
    Traceback (most recent call last):
    ...
    elbepack.shellhelper.CommandError: ...
    """

    new_env = os.environ.copy()
    if env_add:
        new_env.update(env_add)

    if isinstance(stdin, str):
        stdin = stdin.encode()

    logging.info(cmd, extra={"context":"[CMD] "})

    r, w = os.pipe()

    if stdin is None:
        p = Popen(cmd, shell=True, stdout=w, stderr=STDOUT, env=new_env)
    else:
        p = Popen(cmd, shell=True, stdin=PIPE, stdout=w, stderr=STDOUT, env=new_env)

    async_logging(r, w, soap, log)
    p.communicate(input=stdin)

    if p.returncode and not allow_fail:
        raise CommandError(cmd, p.returncode) @contextmanager
def elbe_logging(*args, **kwargs):
    try:
        open_logging(*args, **kwargs)
        yield
    finally:
        close_logging() def read_maxlevel(proj):
    return QHandler.max_level(proj) @classmethod
def reset_level(cls, target):
    try:
        cls.queues[target].max_level = logging.NOTSET
    except (IndexError, KeyError):
        pass
from threading import Thread
from queue import Queue
from os import path, getcwd, chdir
from contextlib import contextmanager
from urllib.parse import quote
from elbepack.db import get_versioned_filename
from elbepack.dump import dump_fullpkgs
from elbepack.updatepkg import gen_update_pkg
from elbepack.pkgarchive import gen_binpkg_archive, checkout_binpkg_archive
from elbepack.rfs import DebootstrapException
from elbepack.elbeproject import AptCacheCommitError, AptCacheUpdateError
from elbepack.shellhelper import do
from elbepack.log import elbe_logging, read_maxlevel, reset_level
import logging
# ELBE - Debian Based Embedded Rootfilesystem Builder
# Copyright (c) 2014-2015 Torben Hohn <torben.hohn@linutronix.de>
# Copyright (c) 2014 Stefan Gast <stefan.gast@linutronix.de>
# Copyright (c) 2015-2018 Manuel Traut <manut@linutronix.de>
#
# SPDX-License-Identifier: GPL-3.0-or-later




class AsyncWorkerJob:

    build_done   = "build_done"
    build_failed = "build_failed"
    has_changes  = "has_changes"

    def __init__(self, project):
        self.project = project

    def enqueue(self, queue, _db):
        reset_level(self.project.builddir)
        queue.put(self)

    def execute(self, _db):
        pass


class BuildSysrootJob(AsyncWorkerJob):
    def __init__(self, project):
        AsyncWorkerJob.__init__(self, project)

    def enqueue(self, queue, db):
        db.set_busy(self.project.builddir,
                    ["empty_project", "needs_build", "has_changes",
                     "build_done", "build_failed"])
        logging.info("Enqueueing project for building sysroot")
        AsyncWorkerJob.enqueue(self, queue, db)

    def execute(self, db):
        success = self.build_failed
        try:
            logging.info("Build sysroot started")
            self.project.build_sysroot()
            db.update_project_files(self.project)
        # pylint: disable=broad-except
        except Exception:
            logging.exception("Build sysroot failed")
        else:

            if read_maxlevel(self.project.builddir) >= logging.ERROR: