def handle_make_arguments(
        input_make_args,
        force_single_threaded_when_running_tests=False):
    """Special handling for make arguments.

    If force_single_threaded_when_running_tests is True, jobs flags are
    replaced with -j1, because tests cannot handle parallelization.

    If no job flags are present and there are none in the MAKEFLAGS environment
    variable, then make flags are set to the cpu_count, e.g. -j4 -l4.

    :param input_make_args: list of make arguments to be handled
    :type input_make_args: list
    :param force_single_threaded_when_running_tests: self explanatory
    :type force_single_threaded_when_running_tests: bool
    :returns: copied list of make arguments, potentially with some modifications
    :rtype: list
    """
    make_args = list(input_make_args)

    # Get the values for the jobs flags which may be in the make args
    jobs_dict = extract_jobs_flags_values(' '.join(make_args))
    jobs_args = extract_jobs_flags(' '.join(make_args))
    if jobs_args:
        # Remove jobs flags from cli args if they're present
        make_args = re.sub(' '.join(jobs_args), '', ' '.join(make_args)).split()

    if force_single_threaded_when_running_tests:
        # force single threaded execution when running test since rostest does not support multiple parallel runs
        run_tests = [a for a in make_args if a.startswith('run_tests')]
        if run_tests:
            wide_log('Forcing "-j1" for running unit tests.')
            jobs_dict['jobs'] = 1

    if job_server.gnu_make_enabled():
        make_args.extend(job_server.gnu_make_args())
    else:
        if 'jobs' in jobs_dict:
            make_args.append('-j{0}'.format(jobs_dict['jobs']))
        if 'load-average' in jobs_dict:
            make_args.append('-l{0}'.format(jobs_dict['load-average']))

    return make_args def mkdir_p(path):
    """Equivalent to UNIX mkdir -p"""
    if os.path.exists(path):
        return
    try:
        return os.makedirs(path)
    except OSError as exc:  # Python >2.5
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise CMAKE_EXEC = which('cmake') CMAKE_INSTALL_MANIFEST_FILENAME = 'install_manifest.txt' class CMakeIOBufferProtocol(IOBufferProtocol):

    """An asyncio protocol that collects stdout and stderr.

    This class also generates `stdout` and `stderr` events.

    Since the underlying asyncio API constructs the actual protocols, this
    class provides a factory method to inject the job and stage information
    into the created protocol.
    """

    def abspath(self, groups):
        """Group filter that turns source-relative paths into absolute paths."""
        return (groups[0] if groups[0].startswith(os.sep) else os.path.join(self.source_path, groups[0]),) + groups[1:]

    def __init__(self, label, job_id, stage_label, event_queue, log_path, source_path, suppress_stdout, *args,
                 **kwargs):
        super(CMakeIOBufferProtocol, self).__init__(label, job_id, stage_label, event_queue, log_path, *args, **kwargs)
        self.source_path = source_path
        self.suppress_stdout = suppress_stdout

        # These are buffers for incomplete lines that we want to wait to parse
        # until we have received them completely
        self.stdout_tail = b''
        self.stderr_tail = b''

        # Line formatting filters
        # Each is a 3-tuple:
        #  - regular expression (with captured groups)
        #  - output formatting line (subs captured groups)
        #  - functor which filters captured groups
        filters = [
            (r'^-- :(.+)', '@{cf}--@| :@{yf}{}@|', None),
            (r'^-- (.+)', '@{cf}--@| {}', None),
            (r'CMake Error at (.+):(.+)', '@{rf}@!CMake Error@| at {}:{}', self.abspath),
            (r'CMake Warning at (.+):(.+)', '@{yf}@!CMake Warning@| at {}:{}', self.abspath),
            (r'CMake Warning (dev) at (.+):(.+)', '@{yf}@!CMake Warning (dev)@| at {}:{}', self.abspath),
            (r'(?i)(warning.*)', '@{yf}{}@|', None),
            (r'(?i)ERROR:(.*)', '@!@{rf}ERROR:@|{}@|', None),
            (r'Call Stack \(most recent call first\):(.*)', '@{cf}Call Stack (most recent call first):@|{}', None),
        ]

        self.filters = [(re.compile(p), r, f) for (p, r, f) in filters]

    def on_stdout_received(self, data):
        if not self.suppress_stdout:
            data_head, self.stdout_tail = split_to_last_line_break(self.stdout_tail + data)
            colored = self.color_lines(data_head)
            super(CMakeIOBufferProtocol, self).on_stdout_received(colored)

    def on_stderr_received(self, data):
        data_head, self.stderr_tail = split_to_last_line_break(self.stderr_tail + data)
        colored = self.color_lines(data_head)
        super(CMakeIOBufferProtocol, self).on_stderr_received(colored)

    def close(self):
        # Make sure tail buffers are flushed
        self.flush_tails()
        super(CMakeIOBufferProtocol, self).close()

    def flush_tails(self):
        """Write out any unprocessed tail buffers."""

        colored = self.color_lines(self.stdout_tail)
        super(CMakeIOBufferProtocol, self).on_stdout_received(colored)
        self.stdout_tail = b''

        colored = self.color_lines(self.stderr_tail)
        super(CMakeIOBufferProtocol, self).on_stderr_received(colored)
        self.stderr_tail = b''

    def color_lines(self, data):
        """Apply colorization rules to each line in data"""
        decoded_data = self._decode(data)
        # TODO: This will only work if all lines are received at once. Instead
        # of directly splitting lines, we should buffer the data lines until
        # the last character is a line break
        lines = decoded_data.splitlines(True)  # Keep line breaks
        colored_lines = [self.colorize_cmake(line) for line in lines]
        colored_data = ''.join(colored_lines)
        encoded_data = self._encode(colored_data)
        return encoded_data

    @classmethod
    def factory_factory(cls, source_path, suppress_stdout=False):
        """Factory factory for constructing protocols that know the source path for this CMake package."""
        def factory(label, job_id, stage_label, event_queue, log_path):
            # factory is called by catkin_tools executor
            def init_proxy(*args, **kwargs):
                # init_proxy is called by asyncio
                return cls(label, job_id, stage_label, event_queue, log_path, source_path, suppress_stdout, *args,
                           **kwargs)
            return init_proxy
        return factory

    def colorize_cmake(self, line):
        """Colorizes output from CMake

        This also prepends the source path to the locations of warnings and errors.

        :param line: one, new line terminated, line from `cmake` which needs coloring.
        :type line: str
        """
        # return line
        cline = sanitize(line).rstrip()

        if len(cline.strip()) > 0:
            for p, r, f in self.filters:
                match = p.match(cline)
                if match is not None:
                    cline = fmt(r, reset=False)
                    if f is not None:
                        cline = cline.format(*f(match.groups()))
                    else:
                        cline = cline.format(*match.groups())
                    break

        return cline + '\n' class CMakeMakeIOBufferProtocol(IOBufferProtocol):

    """An IOBufferProtocol which parses CMake's progress prefixes and emits corresponding STAGE_PROGRESS events."""

    def __init__(self, label, job_id, stage_label, event_queue, log_path, *args, **kwargs):
        super(CMakeMakeIOBufferProtocol, self).__init__(
            label, job_id, stage_label, event_queue, log_path, *args, **kwargs)

    def on_stdout_received(self, data):
        super(CMakeMakeIOBufferProtocol, self).on_stdout_received(data)
        self.send_progress(data)

    def send_progress(self, data):
        """Parse CMake Make completion progress"""
        progress_matches = re.match(r'\[\s*([0-9]+)%\]', self._decode(data))
        if progress_matches is not None:
            self.event_queue.put(ExecutionEvent(
                'STAGE_PROGRESS',
                job_id=self.job_id,
                stage_label=self.stage_label,
                percent=str(progress_matches.groups()[0]))) def get_installed_files(path):
    """Get a set of files installed by a CMake package as specified by an
    install_manifest.txt in a given directory."""

    install_manifest_path = os.path.join(
        path,
        CMAKE_INSTALL_MANIFEST_FILENAME)
    installed_files = set()
    if os.path.exists(install_manifest_path):
        with open(install_manifest_path) as f:
            installed_files = set([line.strip() for line in f.readlines()])
    return installed_files def copyfiles(logger, event_queue, source_paths, dest_path):
    """FunctionStage functor that copies one or more files"""
    for source_path in source_paths:
        shutil.copy(source_path, dest_path)
    return 0 def loadenv(logger, event_queue, job_env, package, context, verbose=True):
    # Get the paths to the env loaders
    env_loader_paths = get_env_loaders(package, context)
    # If DESTDIR is set, set _CATKIN_SETUP_DIR as well
    if context.destdir is not None:
        job_env['_CATKIN_SETUP_DIR'] = context.package_dest_path(package)

    envs = []
    for env_loader_path in env_loader_paths:
        if logger and verbose:
            logger.out('Loading environment from: {}'.format(env_loader_path))
        envs.append(get_resultspace_environment(
            os.path.split(env_loader_path)[0],
            base_env=job_env,
            quiet=True,
            cached=context.use_env_cache,
            strict=False))

    # Avoid using merge logic if not required (in the non-isolated resultspace
    # case. It has corner cases which may trip up the unwary, so having the
    # option to switch to a merged resultspace is a good fallback.
    if len(envs) > 1:
        merge_envs(job_env, envs)
    elif len(envs) == 1:
        job_env.update(envs[0])
    return 0 def makedirs(logger, event_queue, path):
    """FunctionStage functor that makes a path of directories."""
    mkdir_p(path)
    return 0 def require_command(name, which):
    if not which:
        raise CommandMissing(name) def rmfiles(logger, event_queue, paths, dry_run, remove_empty=False, empty_root='/'):
    """FunctionStage functor that removes a list of files and directories.

    If remove_empty is True, then this will also remove directories which
    become empty after deleting the files in `paths`. It will delete files up
    to the path specified by `empty_root`.
    """

    # Determine empty directories
    if remove_empty:
        # First get a list of directories to check
        dirs_to_check = set()

        for path in paths:
            # Make sure the file is given by an absolute path and it exists
            if not os.path.isabs(path) or not os.path.exists(path):
                continue

            # Only look in the devel space
            while empty_root.find(path) != 0:
                # Pop up a directory
                path, dirname = os.path.split(path)

                # Skip if this path isn't a directory
                if not os.path.isdir(path):
                    continue

                dirs_to_check.add(path)

        # For each directory which may be empty after cleaning, visit them
        # depth-first and count their descendants
        dir_descendants = dict()
        for path in sorted(dirs_to_check, key=lambda k: -len(k.split(os.path.sep))):
            # Get the absolute path to all the files currently in this directory
            files = [os.path.join(path, f) for f in os.listdir(path)]
            # Filter out the files which we intend to remove
            files = [f for f in files if f not in paths]
            # Compute the minimum number of files potentially contained in this path
            dir_descendants[path] = sum([
                (dir_descendants.get(f, 1) if os.path.isdir(f) else 1)
                for f in files
            ])

            # Schedule the directory for removal if removal of the given files will make it empty
            if dir_descendants[path] == 0:
                paths.append(path)

    # Remove the paths
    for index, path in enumerate(paths):

        # Remove the path
        if os.path.exists(path):
            if os.path.isdir(path):
                logger.out('Removing directory: {}'.format(path))
                if not dry_run:
                    shutil.rmtree(path)
            else:
                logger.out('     Removing file: {}'.format(path))
                if not dry_run:
                    os.remove(path)
        else:
            logger.err('Warning: File {} could not be deleted because it does not exist.'.format(path))

        # Report progress
        event_queue.put(ExecutionEvent(
            'STAGE_PROGRESS',
            job_id=logger.job_id,
            stage_label=logger.stage_label,
            percent=str(index / float(len(paths)))))

    return 0 class IOBufferProtocol(IOBufferContainer, AsyncSubprocessProtocol):

    """An asyncio protocol that collects stdout and stderr.

    This class also generates `stdout` and `stderr` events.

    Since the underlying asyncio API constructs the actual protocols, this
    class provides a factory method to inject the job and stage information
    into the created protocol.
    """

    def __init__(self, label, job_id, stage_label, event_queue, log_path, *args, **kwargs):
        IOBufferContainer.__init__(self, label, job_id, stage_label, event_queue, log_path)
        AsyncSubprocessProtocol.__init__(self, *args, **kwargs)

        self.intermediate_stdout_buffer = b''
        self.intermediate_stderr_buffer = b''

    @staticmethod
    def _split(data):
        try:
            last_break = data.rindex(b'\n') + 1
            return data[0:last_break], data[last_break:]
        except ValueError:
            return b'', data

    def on_stdout_received(self, data):
        """
        :type data: encoded bytes
        """

        data, self.intermediate_stdout_buffer = self._split(self.intermediate_stdout_buffer + data)

        self.stdout_buffer += data
        self.interleaved_buffer += data
        self.log_file.write(data)

        # Get the decoded Python str
        decoded_data = self._decode(data)

        # Emit event with decoded Python str
        self.event_queue.put(ExecutionEvent(
            'STDOUT',
            job_id=self.job_id,
            stage_label=self.stage_label,
            data=decoded_data))

    def on_stderr_received(self, data):
        """
        :type data: encoded bytes
        """

        data, self.intermediate_stderr_buffer = self._split(self.intermediate_stderr_buffer + data)

        self.stderr_buffer += data
        self.interleaved_buffer += data
        self.log_file.write(data)

        # Get the decoded Python str
        decoded_data = self._decode(data)

        # Emit event with decoded Python str
        self.event_queue.put(ExecutionEvent(
            'STDERR',
            job_id=self.job_id,
            stage_label=self.stage_label,
            data=decoded_data))

    def on_process_exited2(self, returncode):
        """
        Dump anything remaining in the intermediate buffers.
        """

        if len(self.intermediate_stdout_buffer) > 0:
            self.on_stdout_received(self.intermediate_stdout_buffer + b'\n')
        if len(self.intermediate_stderr_buffer) > 0:
            self.on_stderr_received(self.intermediate_stderr_buffer + b'\n') class Job(object):

    """A Job is a series of operations, each of which is considered a "stage" of the job."""

    def __init__(self, jid, deps, env, stages, continue_on_failure=True):
        """
        jid: Unique job identifier
        deps: Dependencies (in terms of other jid's)
        stages: List of stages to be run in order

        """
        self.jid = jid
        self.deps = deps
        self.env = env
        self.stages = stages
        self.continue_on_failure = continue_on_failure

    def all_deps_completed(self, completed_jobs):
        """Return True if all dependencies have been completed."""
        return all([dep_id in completed_jobs for dep_id in self.deps])

    def all_deps_succeeded(self, completed_jobs):
        """Return True if all dependencies have been completed and succeeded."""
        return all([completed_jobs.get(dep_id, False) for dep_id in self.deps])

    def any_deps_failed(self, completed_jobs):
        """Return True if any dependencies which have been completed have failed."""
        return any([not completed_jobs.get(dep_id, True) for dep_id in self.deps]) class CommandStage(Stage):

    """Job stage that describes a system command.
    """

    def __init__(
            self,
            label,
            cmd,
            cwd=None,
            env=None,
            env_overrides=None,
            shell=False,
            emulate_tty=True,
            stderr_to_stdout=False,
            occupy_job=True,
            locked_resource=None,
            logger_factory=IOBufferProtocol.factory,
            early_termination_retcode=None,
            success_retcodes=(0,)):
        """
        :param label: The label for the stage
        :param cmd: The command to run
        :param cwd: The directory in which to run the command (default: os.getcwd())
        :param env: The base environment. (default: {})
        :param env_overrides: The variables that override the base environment. (default: {})
        :param shell: Whether to run the command in "shell" mode. (default: False)
        :param emulate_tty: Support colored output (default: True)
        :param stderr_to_stdout: Pipe stderr to stdout (default: False)

        :param occupy_job: Whether this stage should wait for a worker from the job server (default: True)
        :param logger_factory: The factory to use to construct a logger (default: IOBufferProtocol.factory)
        :param early_termination_retcode: A return code that ends the whole job successfully (default: None)
        :param success_retcodes: A tuple of return codes that mean successful termination of the command (default: (0,))
        """

        if not type(cmd) in [list, tuple] or not all([isinstance(s, str) for s in cmd]):
            raise ValueError('Command stage must be a list of strings: {}'.format(cmd))
        super(CommandStage, self).__init__(label, logger_factory, occupy_job, locked_resource,
                                           early_termination_retcode, success_retcodes)

        # Store environment overrides
        self.env_overrides = env_overrides or {}

        # Override base environment
        env = env or {}
        env.update(self.env_overrides)

        self.async_execute_process_kwargs = {
            'cmd': cmd,
            'cwd': cwd or os.getcwd(),
            'env': env,
            'shell': shell,
            # Emulate tty for cli colors
            'emulate_tty': emulate_tty,
            # Capture stderr and stdout separately
            'stderr_to_stdout': stderr_to_stdout,
        }

    def update_env(self, base_env):
        """Update the environment for this stage with the env args passed to the constructor."""
        self.async_execute_process_kwargs['env'].update(base_env)
        self.async_execute_process_kwargs['env'].update(self.env_overrides)

    def get_reproduction_cmd(self, verb, jid):
        """Get a command line to reproduce this stage with the proper environment."""

        # Define the base env command
        get_env_cmd = 'catkin {} --get-env {}'.format(verb, jid)

        # Add additional env args
        env_overrides_formatted = ' '.join([
            '{}={}'.format(k, cmd_quote(v))
            for k, v in self.env_overrides.items()
        ])

        # Define the actual command to reproduce
        cmd_str = ' '.join([cmd_quote(t) for t in self.async_execute_process_kwargs['cmd']])

        # Define the command to run the subcommand
        env_cmd = 'catkin env -si {} {}'.format(
            env_overrides_formatted,
            cmd_str)

        # Return the full command
        return 'cd {}; {} | {}; cd -'.format(
            self.async_execute_process_kwargs['cwd'],
            get_env_cmd,
            env_cmd) class FunctionStage(Stage):

    """Job stage that describes a python function.

    :param label: The label for the stage
    :param function: A python function which returns 0 on success

    Functions must take the arguments:
        - logger
        - event_queue
    """

    def __init__(
            self,
            label,
            function,
            logger_factory=IOBufferLogger.factory,
            occupy_job=True,
            locked_resource=None,
            early_termination_retcode=None,
            success_retcodes=(0,),
            *args,
            **kwargs):
        if not callable(function):
            raise ValueError('Function stage must be callable.')
        super(FunctionStage, self).__init__(label, logger_factory, occupy_job, locked_resource,
                                            early_termination_retcode, success_retcodes)

        self.args = args
        self.kwargs = kwargs

        def function_proxy(logger, event_queue):
            try:
                return function(logger, event_queue, *args, **kwargs)
            except:  # noqa: E722
                # Silencing E722 here since we immediately re-raise the exception.
                logger.err(str(traceback.format_exc()))
                raise
        self.function = function_proxy class ColorMapper(object):

    """This class encapsulates a set of human-readable format strings and the
    functionality to convert them into colorized version.
    """

    # This map translates more human readable format strings into colorized versions
    default_color_translation_map = {
        # 'output': 'colorized_output'
        '': fmt('@!' + sanitize('') + '@|')
    }

    def __init__(self, color_map={}):
        """Create a color mapper with a given map.

        :param color_map: A dictionary of format strings and colorized versions
        :type color_map: dict
        """
        self.color_map = ColorMapper.default_color_translation_map
        self.color_map.update(color_map)

    def clr(self, key):
        """Returns a colorized version of the string given.

        This is accomplished by either returning a hit from the color translation
        map or by calling :py:func:`fmt` on the string and returning it.

        :param key: string to be colorized
        :type key: str
        """
        global _color_on
        if not _color_on:
            return fmt(key)
        val = self.color_map.get(key, None)
        if val is None:
            return fmt(key)
        return val
import os
import shutil
import subprocess
import sys
import tempfile
from catkin_tools.argument_parsing import handle_make_arguments
from catkin_tools.common import mkdir_p
from .commands.cmake import CMAKE_EXEC
from .commands.cmake import CMAKE_INSTALL_MANIFEST_FILENAME
from .commands.cmake import CMakeIOBufferProtocol
from .commands.cmake import CMakeMakeIOBufferProtocol
from .commands.cmake import get_installed_files
from .commands.make import MAKE_EXEC
from .utils import copyfiles
from .utils import loadenv
from .utils import makedirs
from .utils import require_command
from .utils import rmfiles
from catkin_tools.execution.io import IOBufferProtocol
from catkin_tools.execution.jobs import Job
from catkin_tools.execution.stages import CommandStage
from catkin_tools.execution.stages import FunctionStage
from catkin_tools.terminal_color import ColorMapper


def generate_setup_file(logger, event_queue, context, install_target):

    # Create full path to setup file
    setup_file_path = os.path.join(install_target, 'setup.sh')

    # Check if the setup file needs to be generated
    if context.install:
        # Create the setup file in the install space
        setup_file_needed = context.isolate_install or not os.path.exists(setup_file_path)
    else:
        # Do not replace existing setup.sh if devel space is merged
        setup_file_needed = not context.link_devel and context.isolate_devel or not os.path.exists(setup_file_path)

    if not setup_file_needed:
        logger.out("Setup file does not need to be generated.")
        return 0
    else:
        logger.out(clr("Generating setup file: @!@{yf}{}@|").format(setup_file_path))

    # Create the setup file that dependent packages will source
    arch = get_multiarch()
    subs = {}
    subs['cmake_prefix_path'] = install_target + ":"
    subs['ld_path'] = os.path.join(install_target, 'lib') + ":"
    pythonpath = os.path.join(install_target, get_python_install_dir(context))
    subs['pythonpath'] = pythonpath + ':'
    subs['pkgcfg_path'] = os.path.join(install_target, 'lib', 'pkgconfig') + ":"
    subs['path'] = os.path.join(install_target, 'bin') + ":"
    subs['cpath'] = os.path.join(install_target, 'include') + ":"
    subs['library_path'] = os.path.join(install_target, 'lib') + ":"
    if arch:
        subs['ld_path'] += os.path.join(install_target, 'lib', arch) + ":"
        subs['pkgcfg_path'] += os.path.join(install_target, 'lib', arch, 'pkgconfig') + ":"
    setup_file_directory = os.path.dirname(setup_file_path)
    if not os.path.exists(setup_file_directory):
        os.makedirs(setup_file_directory)

    # Create a temporary file in the setup_file_directory, so os.rename cannot fail
    tmp_dst_handle, tmp_dst_path = tempfile.mkstemp(
        dir=setup_file_directory,
        prefix=os.path.basename(setup_file_path) + '.')

    # Write the filled template to the file
    data = SETUP_FILE_TEMPLATE.format(**subs)
    os.write(tmp_dst_handle, data.encode('utf-8'))
    os.close(tmp_dst_handle)

    # Do an atomic rename with os.rename
    os.rename(tmp_dst_path, setup_file_path)

    return 0


def create_cmake_build_job(context, package, package_path, dependencies, force_cmake, pre_clean):

    # Package source space path
    pkg_dir = os.path.join(context.source_space_abs, package_path)

    # Package build space path
    build_space = context.package_build_space(package)
    # Package metadata path
    metadata_path = context.package_metadata_path(package)
    # Environment dictionary for the job, which will be built
    # up by the executions in the loadenv stage.
    job_env = dict(os.environ)

    # Get actual staging path
    dest_path = context.package_dest_path(package)
    final_path = context.package_final_path(package)

    # Create job stages
    stages = []

    # Load environment for job.
    stages.append(FunctionStage(
        'loadenv',
        loadenv,
        locked_resource='installspace',
        job_env=job_env,
        package=package,
        context=context
    ))

    # Create package build space
    stages.append(FunctionStage(
        'mkdir',
        makedirs,
        path=build_space
    ))

    # Create package metadata dir
    stages.append(FunctionStage(
        'mkdir',
        makedirs,
        path=metadata_path
    ))

    # Copy source manifest
    stages.append(FunctionStage(
        'cache-manifest',
        copyfiles,
        source_paths=[os.path.join(context.source_space_abs, package_path, 'package.xml')],
        dest_path=os.path.join(metadata_path, 'package.xml')
    ))

    require_command('cmake', CMAKE_EXEC)

    # CMake command
    makefile_path = os.path.join(build_space, 'Makefile')
    if not os.path.isfile(makefile_path) or force_cmake:
        stages.append(CommandStage(
            'cmake',
            ([CMAKE_EXEC,
              pkg_dir,
              '--no-warn-unused-cli',
              '-DCMAKE_INSTALL_PREFIX=' + final_path] +
             context.cmake_args),
            cwd=build_space,

            logger_factory=CMakeIOBufferProtocol.factory_factory(pkg_dir)