Make a package repository and optionally sign it and packages present

    Given the repodir (directory to create repository in), create a Debian
    repository and optionally sign it and packages present. This state is
    best used with onchanges linked to your package building states.

    repodir
        The directory to find packages that will be in the repository.

    keyid
        .. versionchanged:: 2016.3.0

        Optional Key ID to use in signing packages and repository.
        This consists of the last 8 hex digits of the GPG key ID.

        Utilizes Public and Private keys associated with keyid which have
        been loaded into the minion's Pillar data. Leverages gpg-agent and
        gpg-preset-passphrase for caching keys, etc.
        These pillar values are assumed to be filenames which are present
        in ``gnupghome``. The pillar keys shown below have to match exactly.

        For example, contents from a Pillar data file with named Public
        and Private keys as follows:

        .. code-block:: yaml

            gpg_pkg_priv_keyname: gpg_pkg_key.pem
            gpg_pkg_pub_keyname: gpg_pkg_key.pub

    env
        .. versionchanged:: 2016.3.0

        A dictionary of environment variables to be utilized in creating the
        repository.

    use_passphrase : False
        .. versionadded:: 2016.3.0

        Use a passphrase with the signing key presented in ``keyid``.
        Passphrase is received from Pillar data which could be passed on the
        command line with ``pillar`` parameter. For example:

        .. code-block:: bash

            pillar='{ "gpg_passphrase" : "my_passphrase" }'

    gnupghome : /etc/salt/gpgkeys
        .. versionadded:: 2016.3.0

        Location where GPG related files are stored, used with ``keyid``.

    runas : root
        .. versionadded:: 2016.3.0

        User to create the repository as, and optionally sign packages.

        .. note::

            Ensure the user has correct permissions to any files and
            directories which are to be utilized.

    timeout : 15.0
        .. versionadded:: 2016.3.4

        Timeout in seconds to wait for the prompt for inputting the passphrase.

    CLI Example:

    .. code-block:: bash

        salt '*' pkgbuild.make_repo /var/www/html

def make_repo(repodir,
              keyid=None,
              env=None,
              use_passphrase=False,
              gnupghome='/etc/salt/gpgkeys',
              runas='root',
              timeout=15.0):
    '''
    Make a package repository and optionally sign it and packages present

    Given the repodir (directory to create repository in), create a Debian
    repository and optionally sign it and packages present. This state is
    best used with onchanges linked to your package building states.

    repodir
        The directory to find packages that will be in the repository.

    keyid
        .. versionchanged:: 2016.3.0

        Optional Key ID to use in signing packages and repository.
        This consists of the last 8 hex digits of the GPG key ID.

        Utilizes Public and Private keys associated with keyid which have
        been loaded into the minion's Pillar data. Leverages gpg-agent and
        gpg-preset-passphrase for caching keys, etc.
        These pillar values are assumed to be filenames which are present
        in ``gnupghome``. The pillar keys shown below have to match exactly.

        For example, contents from a Pillar data file with named Public
        and Private keys as follows:

        .. code-block:: yaml

            gpg_pkg_priv_keyname: gpg_pkg_key.pem
            gpg_pkg_pub_keyname: gpg_pkg_key.pub

    env
        .. versionchanged:: 2016.3.0

        A dictionary of environment variables to be utilized in creating the
        repository.

    use_passphrase : False
        .. versionadded:: 2016.3.0

        Use a passphrase with the signing key presented in ``keyid``.
        Passphrase is received from Pillar data which could be passed on the
        command line with ``pillar`` parameter. For example:

        .. code-block:: bash

            pillar='{ "gpg_passphrase" : "my_passphrase" }'

    gnupghome : /etc/salt/gpgkeys
        .. versionadded:: 2016.3.0

        Location where GPG related files are stored, used with ``keyid``.

    runas : root
        .. versionadded:: 2016.3.0

        User to create the repository as, and optionally sign packages.

        .. note::

            Ensure the user has correct permissions to any files and
            directories which are to be utilized.

    timeout : 15.0
        .. versionadded:: 2016.3.4

        Timeout in seconds to wait for the prompt for inputting the passphrase.

    CLI Example:

    .. code-block:: bash

        salt '*' pkgbuild.make_repo /var/www/html

    '''
    res = {
            'retcode': 1,
            'stdout': '',
            'stderr': 'initialization value'
            }

    retrc = 0

    if gnupghome and env is None:
        env = {}
        env['GNUPGHOME'] = gnupghome

    repoconf = os.path.join(repodir, 'conf')
    if not os.path.isdir(repoconf):
        os.makedirs(repoconf)

    codename, repocfg_dists = _get_repo_dists_env(env)
    repoconfdist = os.path.join(repoconf, 'distributions')
    with salt.utils.files.fopen(repoconfdist, 'w') as fow:
        fow.write(salt.utils.stringutils.to_str(repocfg_dists))

    repocfg_opts = _get_repo_options_env(env)
    repoconfopts = os.path.join(repoconf, 'options')
    with salt.utils.files.fopen(repoconfopts, 'w') as fow:
        fow.write(salt.utils.stringutils.to_str(repocfg_opts))

    cmd = 'chown {0}:{0} -R {1}'.format(runas, repoconf)
    retrc = __salt__['cmd.retcode'](cmd, runas='root')
    if retrc != 0:
        raise SaltInvocationError(
             'failed to ensure rights to repoconf directory, error {0}, '
             'check logs for further details'.format(retrc)
        )

    local_keygrip_to_use = None
    local_key_fingerprint = None
    local_keyid = None
    phrase = ''

    # preset passphase and interaction with gpg-agent
    gpg_info_file = '{0}/gpg-agent-info-salt'.format(gnupghome)
    gpg_tty_info_file = '{0}/gpg-tty-info-salt'.format(gnupghome)

    # if using older than gnupg 2.1, then env file exists
    older_gnupg = __salt__['file.file_exists'](gpg_info_file)

    if keyid is not None:
        with salt.utils.files.fopen(repoconfdist, 'a') as fow:
            fow.write(salt.utils.stringutils.to_str('SignWith: {0}\n'.format(keyid)))

        # import_keys
        pkg_pub_key_file = '{0}/{1}'.format(gnupghome, __salt__['pillar.get']('gpg_pkg_pub_keyname', None))
        pkg_priv_key_file = '{0}/{1}'.format(gnupghome, __salt__['pillar.get']('gpg_pkg_priv_keyname', None))

        if pkg_pub_key_file is None or pkg_priv_key_file is None:
            raise SaltInvocationError(
                'Pillar data should contain Public and Private keys associated with \'keyid\''
            )
        try:
            __salt__['gpg.import_key'](user=runas, filename=pkg_pub_key_file, gnupghome=gnupghome)
            __salt__['gpg.import_key'](user=runas, filename=pkg_priv_key_file, gnupghome=gnupghome)

        except SaltInvocationError:
            raise SaltInvocationError(
                'Public and Private key files associated with Pillar data and \'keyid\' '
                '{0} could not be found'
                .format(keyid)
            )

        # gpg keys should have been loaded as part of setup
        # retrieve specified key, obtain fingerprint and preset passphrase
        local_keys = __salt__['gpg.list_keys'](user=runas, gnupghome=gnupghome)
        for gpg_key in local_keys:
            if keyid == gpg_key['keyid'][8:]:
                local_keygrip_to_use = gpg_key['fingerprint']
                local_key_fingerprint = gpg_key['fingerprint']
                local_keyid = gpg_key['keyid']
                break

        if not older_gnupg:
            try:
                _check_repo_sign_utils_support('gpg2')
                cmd = 'gpg2 --with-keygrip --list-secret-keys'
            except CommandExecutionError:
                # later gpg versions have dispensed with gpg2 - Ubuntu 18.04
                cmd = 'gpg --with-keygrip --list-secret-keys'
            local_keys2_keygrip = __salt__['cmd.run'](cmd, runas=runas, env=env)
            local_keys2 = iter(local_keys2_keygrip.splitlines())
            try:
                for line in local_keys2:
                    if line.startswith('sec'):
                        line_fingerprint = next(local_keys2).lstrip().rstrip()
                        if local_key_fingerprint == line_fingerprint:
                            lkeygrip = next(local_keys2).split('=')
                            local_keygrip_to_use = lkeygrip[1].lstrip().rstrip()
                            break
            except StopIteration:
                raise SaltInvocationError(
                    'unable to find keygrip associated with fingerprint \'{0}\' for keyid \'{1}\''
                    .format(local_key_fingerprint, local_keyid)
                )

        if local_keyid is None:
            raise SaltInvocationError(
                'The key ID \'{0}\' was not found in GnuPG keyring at \'{1}\''
                .format(keyid, gnupghome)
            )

        _check_repo_sign_utils_support('debsign')

        if older_gnupg:
            with salt.utils.files.fopen(gpg_info_file, 'r') as fow:
                gpg_raw_info = fow.readlines()

            for gpg_info_line in gpg_raw_info:
                gpg_info_line = salt.utils.stringutils.to_unicode(gpg_info_line)
                gpg_info = gpg_info_line.split('=')
                env[gpg_info[0]] = gpg_info[1]
                break
        else:
            with salt.utils.files.fopen(gpg_tty_info_file, 'r') as fow:
                gpg_raw_info = fow.readlines()

            for gpg_tty_info_line in gpg_raw_info:
                gpg_tty_info_line = salt.utils.stringutils.to_unicode(gpg_tty_info_line)
                gpg_tty_info = gpg_tty_info_line.split('=')
                env[gpg_tty_info[0]] = gpg_tty_info[1]
                break

        if use_passphrase:
            _check_repo_gpg_phrase_utils()
            phrase = __salt__['pillar.get']('gpg_passphrase')
            cmd = '/usr/lib/gnupg2/gpg-preset-passphrase --verbose --preset --passphrase "{0}" {1}'.format(
                    phrase,
                    local_keygrip_to_use)
            retrc |= __salt__['cmd.retcode'](cmd, runas=runas, env=env)

    for debfile in os.listdir(repodir):
        abs_file = os.path.join(repodir, debfile)
        if debfile.endswith('.changes'):
            os.remove(abs_file)

        if debfile.endswith('.dsc'):
            # sign_it_here
            if older_gnupg:
                if local_keyid is not None:
                    cmd = 'debsign --re-sign -k {0} {1}'.format(keyid, abs_file)
                    retrc |= __salt__['cmd.retcode'](cmd, runas=runas, cwd=repodir, use_vt=True, env=env)

                cmd = 'reprepro --ignore=wrongdistribution --component=main -Vb . includedsc {0} {1}'.format(
                        codename,
                        abs_file)
                retrc |= __salt__['cmd.retcode'](cmd, runas=runas, cwd=repodir, use_vt=True, env=env)
            else:
                # interval of 0.125 is really too fast on some systems
                interval = 0.5
                if local_keyid is not None:
                    number_retries = timeout / interval
                    times_looped = 0
                    error_msg = 'Failed to debsign file {0}'.format(abs_file)
                    if ((__grains__['os'] in ['Ubuntu'] and __grains__['osmajorrelease'] < 18)
                         or (__grains__['os'] in ['Debian'] and __grains__['osmajorrelease'] <= 8)):
                        cmd = 'debsign --re-sign -k {0} {1}'.format(keyid, abs_file)
                        try:
                            proc = salt.utils.vt.Terminal(
                                cmd,
                                env=env,
                                shell=True,
                                stream_stdout=True,
                                stream_stderr=True
                            )
                            while proc.has_unread_data:
                                stdout, _ = proc.recv()
                                if stdout and SIGN_PROMPT_RE.search(stdout):
                                    # have the prompt for inputting the passphrase
                                    proc.sendline(phrase)
                                else:
                                    times_looped += 1

                                if times_looped > number_retries:
                                    raise SaltInvocationError(
                                        'Attempting to sign file {0} failed, timed out after {1} seconds'.format(
                                            abs_file,
                                            int(times_looped * interval))
                                    )
                                time.sleep(interval)

                            proc_exitstatus = proc.exitstatus
                            if proc_exitstatus != 0:
                                raise SaltInvocationError(
                                    'Signing file {0} failed with proc.status {1}'.format(
                                        abs_file,
                                        proc_exitstatus)
                                )
                        except salt.utils.vt.TerminalException as err:
                            trace = traceback.format_exc()
                            log.error(error_msg, err, trace)
                            res = {
                                    'retcode': 1,
                                    'stdout': '',
                                    'stderr': trace}
                        finally:
                            proc.close(terminate=True, kill=True)
                    else:
                        cmd = 'debsign --re-sign -k {0} {1}'.format(local_key_fingerprint, abs_file)
                        retrc |= __salt__['cmd.retcode'](cmd, runas=runas, cwd=repodir, use_vt=True, env=env)

                number_retries = timeout / interval
                times_looped = 0
                error_msg = 'Failed to reprepro includedsc file {0}'.format(abs_file)
                cmd = 'reprepro --ignore=wrongdistribution --component=main -Vb . includedsc {0} {1}'.format(
                        codename,
                        abs_file)
                if ((__grains__['os'] in ['Ubuntu'] and __grains__['osmajorrelease'] < 18)
                    or (__grains__['os'] in ['Debian'] and __grains__['osmajorrelease'] <= 8)):
                    try:
                        proc = salt.utils.vt.Terminal(
                                cmd,
                                env=env,
                                shell=True,
                                cwd=repodir,
                                stream_stdout=True,
                                stream_stderr=True
                                )
                        while proc.has_unread_data:
                            stdout, _ = proc.recv()
                            if stdout and REPREPRO_SIGN_PROMPT_RE.search(stdout):
                                # have the prompt for inputting the passphrase
                                proc.sendline(phrase)
                            else:
                                times_looped += 1

                            if times_looped > number_retries:
                                raise SaltInvocationError(
                                        'Attempting to reprepro includedsc for file {0} failed, timed out after {1} loops'
                                        .format(abs_file, times_looped)
                                 )
                            time.sleep(interval)

                        proc_exitstatus = proc.exitstatus
                        if proc_exitstatus != 0:
                            raise SaltInvocationError(
                                 'Reprepro includedsc for codename {0} and file {1} failed with proc.status {2}'.format(
                                    codename,
                                    abs_file,
                                    proc_exitstatus)
                            )
                    except salt.utils.vt.TerminalException as err:
                        trace = traceback.format_exc()
                        log.error(error_msg, err, trace)
                        res = {
                                'retcode': 1,
                                'stdout': '',
                                'stderr': trace
                                }
                    finally:
                        proc.close(terminate=True, kill=True)
                else:
                    retrc |= __salt__['cmd.retcode'](cmd, runas=runas, cwd=repodir, use_vt=True, env=env)

        if retrc != 0:
            raise SaltInvocationError(
                 'Making a repo encountered errors, return error {0}, check logs for further details'.format(retrc))

        if debfile.endswith('.deb'):
            cmd = 'reprepro --ignore=wrongdistribution --component=main -Vb . includedeb {0} {1}'.format(
                    codename,
                    abs_file)
            res = __salt__['cmd.run_all'](cmd, runas=runas, cwd=repodir, use_vt=True, env=env)

    return res