def add_libname_arguments(parser): add_libname_arguments_gen(parser, passing='') def update_env_with_libnames(env, args, default=(('.', 'Top'), ), include_passing=False, use_default=True, use_passing_default=True):
    all_keys = ('libnames', 'non_recursive_libnames', 'ocaml_dirnames', '_CoqProject', '_CoqProject_v_files', '_CoqProject_unknown')
    passing_opts = ('', ) if not include_passing else ('', 'passing_', 'nonpassing_')
    for passing in passing_opts:
        for key in all_keys:
            env[passing + key] = []
        if getattr(args, passing + 'CoqProjectFile'):
            for f in getattr(args, passing + 'CoqProjectFile'):
                env[passing + '_CoqProject'] = f.read()
                f.close()
        process_CoqProject(env, env[passing + '_CoqProject'], passing=passing)
        env[passing + 'libnames'].extend(getattr(args, passing + 'libnames'))
        env[passing + 'non_recursive_libnames'].extend(getattr(args, passing + 'non_recursive_libnames'))
        env[passing + 'ocaml_dirnames'].extend(getattr(args, passing + 'ocaml_dirnames'))
    if include_passing:
        # The nonpassing_ prefix is actually temporary; the semantics
        # are that, e.g., libnames = libnames + nonpassing_libnames,
        # while passing_libnames = libnames + passing_libnames
        for key in all_keys:
            env['passing_' + key] = env[key] + env['passing_' + key]
            env[key] = env[key] + env['nonpassing_' + key]
            del env['nonpassing_' + key]
    if use_default:
        update_env_with_libname_default(env, args, default=default)
    if use_passing_default and include_passing:
        update_env_with_libname_default(env, args, default=default, passing='passing_') def add_logging_arguments(parser):
    parser.add_argument('--verbose', '-v', dest='verbose',
                        action='count',
                        help='display some extra information by default (does not impact --verbose-log-file)')
    parser.add_argument('--quiet', '-q', dest='quiet',
                        action='count',
                        help='the inverse of --verbose')
    parser.add_argument('--log-file', '-l', dest='log_files', nargs='+', type=argparse.FileType('w'),
                        default=[sys.stderr],
                        help='The files to log output to.  Use - for stdout.')
    parser.add_argument('--verbose-log-file', dest='verbose_log_files', nargs='+', type=TupleType(int, argparse.FileType('w')),
                        default=[],
                        help=(('The files to log output to at verbosity other than the default verbosity (%d if no -v/-q arguments are passed); ' +
                               'each argument should be a pair of a verbosity level and a file name. '
                               'Use - for stdout.') % DEFAULT_VERBOSITY)) def process_logging_arguments(args):
    if args.verbose is None: args.verbose = DEFAULT_VERBOSITY
    if args.quiet is None: args.quiet = 0
    args.verbose -= args.quiet
    args.log = make_logger([(args.verbose, f) for f in args.log_files]
                           + args.verbose_log_files)
    del args.quiet
    del args.verbose
    return args LOG_ALWAYS=None DEFAULT_VERBOSITY=1 class UnsupportedCoqVersionError(Exception):
    pass DEFAULT_LIBNAMES=(('.', 'Top'), )
IMPORT_ABSOLUTIZE_TUPLE = ('lib', 'mod')
ALL_ABSOLUTIZE_TUPLE = ('lib', 'proj', 'rec', 'ind', 'constr', 'def', 'syndef', 'class', 'thm', 'lem', 'prf', 'ax', 'inst', 'prfax', 'coind', 'scheme', 'vardef', 'mod')# , 'modtype')
IMPORT_LINE_REG = re.compile(r'^\s*(?:Require\s+Import|Require\s+Export|Require|Load\s+Verbose|Load)\s+(.*?)\.(?:\s|$)', re.MULTILINE | re.DOTALL)
REQUIRE, REQUIRE_IMPORT, REQUIRE_EXPORT, EXPORT, IMPORT = 'REQUIRE', 'REQUIRE IMPORT', 'REQUIRE EXPORT', 'EXPORT', 'IMPORT'
def warning(*objs):
def error(*objs):
def fill_kwargs(kwargs, for_makefile=False):
def safe_kwargs(kwargs):
def fix_path(filename):
def absolutize_has_all_constants(absolutize_tuple):
def libname_with_dot(logical_name):
def clear_libimport_cache(libname):
def os_walk(top, topdown=True, onerror=None, followlinks=False):
def os_path_isfile(filename):
def filenames_of_lib_helper(lib, libnames, non_recursive_libnames, ext):
def local_filenames_of_lib_helper(lib, libnames, non_recursive_libnames, ext):
def filename_of_lib_helper(lib, libnames, non_recursive_libnames, ext):
def filename_of_lib(lib, ext='.v', **kwargs):
def lib_of_filename_helper(filename, libnames, non_recursive_libnames, exts):
def lib_of_filename(filename, exts=('.v', '.glob'), **kwargs):
def is_local_import(libname, **kwargs):
def get_raw_file_as_bytes(filename, **kwargs):
def get_raw_file(*args, **kwargs):
def list_endswith(ls, suffix):
def constr_name_endswith(code, endswith):
def move_strings_once(before, after, possibility, relaxed=False):
def move_strings_pre(before, after, possibility):
def move_function(before, after, get_len):
def move_strings(before, after, *possibilities):
def move_space(before, after):
def remove_from_require_before(contents, location):
def get_references_from_globs(globs):
def is_absolutizable_mod_reference(contents, reference):
def update_one_with_glob(contents, ref, absolutize, libname, transform_base=(lambda x: x), **kwargs):
def update_with_glob(contents, globs, *args, **kwargs):
def get_all_v_files(directory, exclude=tuple()):
def get_maybe_passing_arg(kwargs, key):
def run_coq_makefile_and_make(v_files, targets, **kwargs):
def make_one_glob_file(v_file, **kwargs):
def make_globs(logical_names, **kwargs):
def get_glob_file_for(filename, update_globs=False, **kwargs):
def get_byte_references_for(filename, types=None, appends=None, **kwargs):
def get_file_as_bytes(filename, absolutize=('lib',), update_globs=False, mod_remap=dict(), transform_base=(lambda x: x), **kwargs):
def get_file(*args, **kwargs):
def insert_references(contents, ranges, references, types=('lib',), appends=('<>',), **kwargs):
def get_file_statements_insert_references(filename, **kwargs):
def remove_after_first_range(text_as_bytes, ranges):
def classify_require_kind(text_as_bytes, ranges):
def split_requires_of_statements(annotated_contents, types=('lib',), appends=('<>',), **kwargs):
def get_require_dict(lib, update_globs=True, **kwargs):
def get_require_names(lib, **kwargs):
def get_require_locations(lib, **kwargs):
def transitively_close(d, make_new_value=(lambda x: tuple()), reflexive=True):
def get_recursive_requires(*libnames, **kwargs):
def get_recursive_require_names(libname, **kwargs):
def sort_files_by_dependency(filenames, reverse=True, **kwargs):
    def fcmp(f1, f2):
def get_imports(lib, fast=False, **kwargs):
def norm_libname(lib, **kwargs):
def merge_imports(imports, **kwargs):
def internal_recursively_get_imports(lib, **kwargs):
def recursively_get_imports(lib, **kwargs):
def run_recursively_get_imports(lib, recur=recursively_get_imports, fast=False, **kwargs): def write_to_file(file_name, contents, *args, **kwargs):
    return write_bytes_to_file(file_name, contents.replace('\n', os.linesep).encode('utf-8'), *args, **kwargs) def memoize(f):
    """ Memoization decorator for a function taking one or more arguments. """
    class memodict(dict):
        def __getitem__(self, *key, **kwkey):
            return dict.__getitem__(self, (tuple(key), tuple((k, kwkey[k]) for k in sorted(kwkey.keys()))))

        def __missing__(self, key):
            args, kwargs = key
            self[key] = ret = f(*args, **dict(kwargs))
            return ret

    return memodict().__getitem__ def run_binary_search(initial_state, check_state, step_state, save_good_state, valid_nondefault_actions):

    """
    Runs a binary search on initial_state to find the best final state.

    - initial_state : State
        Starting point.  Must be pure (no mutation necessary)

    - check_state : State -> bool
        returns True if the state is a good state, and False if it is bad.
        If run_binary_search returns any non-none value and
        check_state is pure, then check_state is guaranteed to return
        a truthy value on the returned state.

    - step_state : State -> Maybe Action -> State
        Takes a step based on the action, or returns None if no such step
        is possible.  The action is either None, or one from
        valid_nondefault_actions.  If no step is possible, step_state
        must return None, rather than its input state (otherwise
        run_binary_search may loop).  This argument must be a pure function.

        Requirement: check_state(st) == check_state(step_state(st, None))
        assuming step_state(st, None) != None

    - save_good_state : State -> ()
        Called after every discovery of a state that check_state is happy with

    - valid_nondefault_actions : [State]
        A list or tuple of actions.  Earlier actions are assumed to be better.

    Returns the state that is "best", as determined by a lexicographic
    ordering on the actions taken, where earlier actions in
    valid_nondefault_actions are better than later ones.
    """
    valid_actions = list(valid_nondefault_actions) + [None]

    @memoize
    def is_good(state):
        return check_state(state)

    def step_none(st):
        return step_state(st, None)

    def step_default(st):
        for action in valid_actions:
            try:
                return step_state(st, action)
            except ValueError:
                pass
        return step_none(st)

    last_good = None
    cur = initial_state
    while is_good(cur) and last_good != cur:
        last_good = cur
        current_states = list(make_states(cur, step_default))
        if current_states and is_good(current_states[0]):
            idx = binary_search(is_good, current_states)
            cur = current_states[idx]
            save_good_state(cur)
        else:
            for action in valid_actions:
                try:
                    next_st = step_state(cur, action)
                except ValueError:
                    continue
                if next_st and is_good(next_st):
                    cur = next_st
                    save_good_state(cur)
                    break
    return last_good
import shutil, os, os.path, sys, re
import diagnose_error
import util
from argparse_compat import argparse
from custom_arguments import add_libname_arguments, update_env_with_libnames, add_logging_arguments, process_logging_arguments, LOG_ALWAYS, DEFAULT_VERBOSITY
from split_file import UnsupportedCoqVersionError
from import_util import get_file_statements_insert_references, sort_files_by_dependency, classify_require_kind, EXPORT, REQUIRE_EXPORT
from file_util import write_to_file
from memoize import memoize
from minimizer_drivers import run_binary_search
    return (before.encode('utf-8'), after.encode('utf-8'))

def step_state(state, action):
    ret = []
    state = list(state)
    while len(state) > 0:
        (text_as_bytes, references, force_keep), state = state[0], state[1:]
        if references:
            (start, end, _loc, _append, _ty), new_references = references[0], tuple(references[1:])
            if action == SKIP or action is None:
                ret.append((text_as_bytes, new_references, force_keep))
            elif action == REMOVE and not force_keep:
                if new_references: # still other imports, safe to just remove
                    pre_text_as_bytes, post_text_as_bytes = gobble_whitespace(text_as_bytes[:start], text_as_bytes[end:])
                    ret.append((pre_text_as_bytes + post_text_as_bytes, new_references, force_keep))
                else: # no other imports; remove this line completely
                    if ret and state:
                        prev_text_as_bytes, post_text_as_bytes = gobble_whitespace(ret[-1][0], state[0][0])
                        ret[-1] = (prev_text_as_bytes,) + ret[-1][1:]
                        state[0] = (post_text_as_bytes,) + state[0][1:]
                    elif ret:
                        ret[-1] = (ret[-1][0].rstrip(),) + ret[-1][1:]
                    elif state:
                        state[0] = (state[0][0].lstrip(),) + state[0][1:]
            else:
                raise ValueError
            ret.extend(state)
            return tuple(ret)
        else:
            ret.append((text_as_bytes, references, force_keep))
    return None

def state_to_contents(state):
    return ''.join(reversed([v[0].decode('utf-8') for v in state]))

# the higher verbose_base, the less verbose we are, since we start at a higher level by default
def make_check_state(original_contents, verbose_base=4-DEFAULT_VERBOSITY, **kwargs):
    # original_contents is str
    expected_output, orig_cmds, orig_retcode, runtime = diagnose_error.get_coq_output(kwargs['coqc'], kwargs['coqc_args'], original_contents, kwargs['timeout'], verbose_base=2, **kwargs)
    @memoize
    def check_contents(contents):
        output, cmds, retcode, runtime = diagnose_error.get_coq_output(kwargs['coqc'], kwargs['coqc_args'], contents, kwargs['timeout'], verbose_base=2, **kwargs)
        # TODO: Should we be checking the error message and the retcode and the output, or just the retcode?
        retval = (diagnose_error.has_error(output) or output != expected_output or retcode != orig_retcode)
        if retval:
            kwargs['log']('Failed change.  Error when running "%s":\n%s' % ('" "'.join(cmds), output), level=verbose_base-1)
        else:
            kwargs['log']('Successful change', level=verbose_base)
            kwargs['log']('New contents:\n"""\n%s\n"""' % contents, level=verbose_base+1)
        return not retval

    def check_state(state):
        return check_contents(state_to_contents(state))

    return check_state

def make_save_state(filename, **kwargs):
    def save_state(state, final=False):
        contents = state_to_contents(state)
        if kwargs['inplace']:
            do_backup = kwargs['suffix'] is not None and len(kwargs['suffix']) > 0
            write_to_file(filename, contents, do_backup=do_backup, backup_ext=kwargs['suffix'])
        elif final:
            print(contents)
    return save_state

if __name__ == '__main__':
    args = process_logging_arguments(parser.parse_args())
    env = {
        'log': args.log,
        'keep_exports': args.keep_exports,
        'keep_going': args.keep_going,
        'coqc': (args.coqc if args.coqbin == '' else os.path.join(args.coqbin, args.coqc)),
        'coqc_args': (args.coq_args if args.coq_args else tuple()),
        'timeout': args.timeout,
        'inplace': args.suffix != '', # it's None if they passed no argument, and '' if they didn't pass -i
        'suffix': args.suffix,
        'input_files': tuple(f.name for f in args.input_files),
        }
    update_env_with_libnames(env, args)

    for f in args.input_files: f.close()

    if args.update_all:
        if env['_CoqProject'] is None:
            parser.error('--update-all given without -f')
            sys.exit(1)
        else:
            env['inplace'] = True
            if env['suffix'] == '': env['suffix'] = None
            env['input_files'] = tuple(list(env['input_files']) + list(env['_CoqProject_v_files']))
        if len(env['input_files']) == 0:
            parser.error('no .v files listed in %s' % args.CoqProjectFile)
    elif len(env['input_files']) == 0:
        parser.error('not enough arguments (-f COQPROJECTFILE with .v files is required if no .v files are given)')

    for dirname, libname in env['libnames']:
        env['coqc_args'] = tuple(list(env['coqc_args']) + ['-R', dirname, libname])
    for dirname, libname in env['non_recursive_libnames']:
        env['coqc_args'] = tuple(list(env['coqc_args']) + ['-Q', dirname, libname])
    for dirname in env['ocaml_dirnames']:
        env['coqc_args'] = tuple(list(env['coqc_args']) + ['-I', dirname])

    env['input_files'] = sort_files_by_dependency(env['input_files'], update_globs=True, **env)

    try:
        failed = []
        for name in env['input_files']:
            try:
                annotated_contents = get_file_statements_insert_references(name, absolutize=(('lib',) if args.absolutize else tuple()), update_globs=True, types=('lib',), appends=('<>',), **env)
                if annotated_contents is None:
                    env['log']('ERROR: Failed to get references for %s' % name, level=LOG_ALWAYS)
                    failed.append((name, 'failed to get references'))
                    if env['keep_going']:
                        continue
                    else:
                        break
                annotated_contents = mark_exports(tuple(reversed(annotated_contents)), env['keep_exports'])
                save_state = make_save_state(name, **env)
                check_state = make_check_state(state_to_contents(annotated_contents), **env)

                verbose_check_state = make_check_state(state_to_contents(annotated_contents), verbose_base=DEFAULT_VERBOSITY, **env)