<filename>src/r2b2/cli.py<gh_stars>1-10
"""
R2B2's command line interface offers significant out-of-the-box functionality with
respect to executing audits and generating audit data without requiring the user to
write a single line of Python.

Note:
    Why does this file exist, and why not put this in __main__?

      You might be tempted to import things from __main__ later, but that will cause
      problems: the code will get executed twice:

      - When you run `python -m r2b2` python will execute
        ``__main__.py`` as a script. That means there won't be any
        ``r2b2.__main__`` in ``sys.modules``.
      - When you import __main__ it will get executed again (as a module) because
        there's no ``r2b2.__main__`` in ``sys.modules``.

      Also see (1) from http://click.pocoo.org/5/setuptools/#setuptools-integration
"""
import math

import click
import pkg_resources

from r2b2.athena import Athena
from r2b2.audit import Audit
from r2b2.brla import BayesianRLA as BRLA
from r2b2.contest import Contest
from r2b2.contest import ContestType
from r2b2.election import Election
from r2b2.minerva import Minerva
from r2b2.tests import util


# Class for parsing space separated list of integers from command line
class IntList(click.ParamType):
    name = 'integer list'

    def convert(self, value, param, ctx):
        try:
            str_list = value.split(' ')
            return [int(i) for i in str_list]
        except TypeError:
            self.fail('Expected space separated list of integers')


INT_LIST = IntList()

# Audit type choices
# TODO: add new audit types when they become available
audit_types = click.Choice(['brla', 'minerva', 'athena'], case_sensitive=False)
# Contest type choice
contest_types = click.Choice(['PLURALITY', 'MAJORITY'])


@click.group()
def cli():
    click.echo('\nWelcome to the R2B2 auditing tool!\n')
    pass


@cli.command('interactive', short_help='Interactive audit execution.')
@click.option('-e',
              '--election-mode',
              default=False,
              is_flag=True,
              flag_value=True,
              show_default=True,
              help='Use election mode instead of single-contest mode.')
@click.option('--election-file', type=str, help='Pass election data as JSON file.')  # TODO: provide format
@click.option('--contest-file', type=str, help='Pass contest data as JSON file.')  # TODO: provide format
@click.option('-a', '--audit-type', type=audit_types, prompt='Select an audit type', help='Type of audit to execute on given contest.')
@click.option('-r',
              '--risk-limit',
              type=click.FloatRange(0.0, 1.0),
              prompt='Enter desired risk limit (e.g. use 0.1 for 10%)',
              help='Risk limit (alpha) of audit. Should be value between 0 and 1.')
@click.option('-m',
              '--max-fraction-to-draw',
              type=click.FloatRange(0.0, 1.0),
              prompt='Enter maximum fraction of ballots to draw during audit',
              help='Maximum fraction of total contest ballots that could be drawn during the audit.')
@click.option('-v',
              '--verbose',
              default=False,
              is_flag=True,
              flag_value=True,
              show_default=False,
              help='Provides risk and stopping probability schedule of previous rounds, minimum and  maximum sample size.')
def interactive(election_mode, election_file, contest_file, audit_type, risk_limit, max_fraction_to_draw, verbose):
    """Executes an audit round by round.

    Depending on what options are passed to the interactive command, users may be prompted for
    contest results, audit type, risk limit, and/or maximum fraction of contest ballots to draw
    when initializing the contest and audit to run.

    During execution, users will enter each round
    size and results of the round's sample and subsequently receive information about the current
    state of the audit. The process continues until either the stopping conditions are met or the
    audit reaches the maximum sample size.
    \f

    For information on each option run ::

        $ r2b2 interactive --help

    Example:
        Contest results can be passed as a JSON file rather than entering the data through the
        prompt::

            $ r2b2 interactive --contest-file example_contest.json

    Tip:
        To generate a template contest JSON file run::

            $ r2b2 template contest

    Example:
        Audit parameters can be passed in as options rather than entering through the prompt::

            $ r2b2 interactive --audit-type brla --risk-limit 0.1 --max-fraction-to-draw 0.2
            $ r2b2 interactive -a brla -r 0.1 -m 0.2    // Shortened equivalent

    Example:
        Election mode allows users to enter all the results from an election then select a contest
        from the election to audit::

            $ r2b2 interactive -e
            $ r2b2 interactive -e --election-file // pass election results as JSON file.

    Warning:
        Election mode simply allows you to enter an entire election's data, then select one
        one contest from that election to run. Auditing multiple contests from an election
        concurrently is not implemented.
    """

    if election_mode:
        if election_file is not None:
            election = util.parse_election(str(election_file))
        else:
            election = input_election()

        contest_choices = click.Choice(election.contests.keys(), case_sensitive=True)
        click.echo('\n')
        click.echo(election)
        contest_name = click.prompt('Select a contest from the above election', type=contest_choices)
        contest = election.contests[contest_name]

    # Check contest file if provided, otherwise request contest input
    elif contest_file is not None:
        contest = util.parse_contest(str(contest_file))
    else:
        contest = input_contest()

    # Confirm contest, if incorrect get new input
    click.echo('\n')
    if election_mode:
        click.echo(contest_name)
    click.echo(contest)
    while not click.confirm('\nUse the above contest data?'):
        contest = input_contest()
        click.echo(contest)

    # Create audit from prompted input
    audit = input_audit(contest, risk_limit, max_fraction_to_draw, audit_type)
    click.echo(audit)
    if audit_type == 'athena':
        click.echo('Delta: ' + str(audit.delta))
    # Confirm audit, if incorrect get new audit
    while not click.confirm('\nAre the audit parameters correct?'):
        audit = input_audit(contest)
        click.echo(audit)

    # Run audit
    audit.run(verbose)


@cli.command('bulk', short_help='Generate audit data for given round sizes.')
@click.option('-v',
              '--verbose',
              default=False,
              is_flag=True,
              flag_value=True,
              show_default=False,
              help='Provides risk and stopping probability schedule of previous rounds, minimum and  maximum sample size.')
@click.option('-o', '--output', type=click.File('w'), default=None, help='Write output into given file.')
@click.option('-r', '--round-list', type=INT_LIST, default=None, help='Provide a list of round sizes to generate stopping sizes for.')
@click.option('-f', '--full-audit-limit', type=int, default=None, help='Set maximum size for ballot by ballot output data.')
@click.option('-p', '--pair', type=str, default=None, help='Select candidate pair to generate data for.')
@click.argument('contest_file', type=str)
@click.argument('audit_type', type=audit_types)
@click.argument('risk_limit', type=click.FloatRange(0.0, 1.0))
@click.argument('max_fraction_to_draw', type=click.FloatRange(0.0, 1.0))
def bulk(audit_type, risk_limit, max_fraction_to_draw, contest_file, output, round_list, full_audit_limit, pair, verbose):
    """Bulk auditing mode generates stopping sizes for a given fixed round schedule.

    Either provide a list of round sizes for which to generate stopping sizes or
    generate a ballot by ballot list of stopping sizes from the minimum valid sample
    size to the default maximum sample size or a specified maximum sample size.
    \f

    Args:
        contest_file: Contest results as JSON file.
        audit_type: Which audit type to use to generate stopping sizes.
        risk_limit: Risk limit (alpha) of audit.
        max_fraction_to_draw: Maximum fraction of contest ballots that could be drawm during
            the audit. Sets the default maximum size of the ballot by ballot output.

    Tip:
        To generate a template contest JSON file, run::

            $ r2b2 template contest

    Returns:
        Formatted list of rounds and their associated stopping sizes. Default execution is
        ballot by ballot from minimum valid sample size to the maximum sample size of audit.

    Example:
        To generate stopping sizes for a specific set of round sizes, provide the round sizes
        as a space separated list of integers enclosed by quotes using the round list option::

            $ r2b2 bulk -l '100 200 300' contest.json brla 0.1 0.5

    Example:
        To generate a ballot by ballot result from the minimum valid sample size to a specific
        maximum (i.e. not the maximum fraction to draw of the audit), run::

            $ r2b2 bulk -f 221 contest.json brla 0.1 0.5

    Example:
        To write the results to a file instead of to stdout, run::

            $ r2b2 bulk -o output.txt contest.json brla 0.1 0.5

    Tip:
        Generating large or compute heavy data sets can take some time. To estimate run times,
        use the verbose flag to display a progress bar::

            $ r2b2 bulk -v contest.json brla 0.1 0.5

    """
    # Parse audit and contest from arguments
    contest = util.parse_contest(contest_file)

    if audit_type == 'brla':
        audit = BRLA(risk_limit, max_fraction_to_draw, contest)
    else:
        raise click.BadArgumentUsage('No valid audit type found.')

    # Get loser for pairwise audit
    if pair is not None and pair not in audit.sub_audits.keys():
        click.echo('pair [{}] is not valid.'.format(pair))
        pair = None
    if pair is None:
        pairs = click.Choice(audit.sub_audits.keys())
        pair = click.prompt('Select a pair to audit', type=pairs)

    out = '\n{:^20}|{:^20}\n'.format('Round Sizes', 'Stopping Sizes')
    out += '--------------------|--------------------\n'
    if round_list is not None:
        kmins = audit.compute_min_winner_ballots(audit.sub_audits[pair], round_list, progress=verbose)
        for i in range(len(kmins)):
            out += '{:^20}|{:^20}\n'.format(round_list[i], kmins[i])
    elif full_audit_limit is not None:
        kmins = audit.compute_all_min_winner_ballots(audit.sub_audits[pair], full_audit_limit, progress=verbose)
        for r in range(audit.sub_audits[pair].min_sample_size, full_audit_limit + 1):
            out += '{:^20}|{:^20}\n'.format(r, kmins[r - audit.sub_audits[pair].min_sample_size])
    else:
        kmins = audit.compute_all_min_winner_ballots(audit.sub_audits[pair], progress=verbose)
        for r in range(
                audit.sub_audits[pair].min_sample_size,
                min(audit.sub_audits[pair].sub_contest.contest_ballots,
                    math.ceil(max_fraction_to_draw * contest.contest_ballots) + 1)):
            out += '{:^20}|{:^20}\n'.format(r, kmins[r - audit.sub_audits[pair].min_sample_size])

    # Write or print output
    if output is not None:
        output.write(out)
    else:
        click.echo(out)


@cli.command('template', short_help='Generate template JSON input files')
@click.argument('style', type=click.Choice(['contest', 'election']))
@click.option('-o', '--output', type=click.File('wb'), default=None, help='Write output into given file.')
def template(style, output):
    """Generate JSON templates for possible input formats.
    \f

    Example:
        To create a contest results JSON file, first generate the template as a new JSON file::

            $ r2b2 template -o my_contest.json contest

        Now the file my_contest.json will be created and contain::

            {
                "contest_ballots" : 100,
                "tally" : {
                        "CandidateA" : 50,
                        "CandidateB" : 50
                },
                "num_winners" : 1,
                "reported_winners" : ["CandidateA"],
                "contest_type" : "PLURALITY"
            }

        Simply repopulate the fields with your contest results.
    """
    # TODO: docstring
    if style == 'contest':
        template = pkg_resources.resource_string(__name__, 'tests/data/single_contest_template.json')
    elif style == 'election':
        template = pkg_resources.resource_string(__name__, 'tests/data/election_template.json')
    else:
        raise click.BadArgumentUsage('No valid template style found')
    if output is not None:
        output.write(template)
        click.echo('Template written to {}'.format(output.name))
    else:
        click.echo(template)


def input_audit(contest: Contest,
                alpha: float = None,
                max_fraction_to_draw: float = None,
                audit_type: str = None,
                delta: float = None) -> Audit:
    # Create an audit from user-input.
    click.echo('\nCreate a new Audit')
    click.echo('==================\n')

    if alpha is None:
        alpha = click.prompt('Enter the desired risk limit', type=click.FloatRange(0.0, 1.0))
    if max_fraction_to_draw is None:
        max_fraction_to_draw = click.prompt('Enter the maximum fraction of contest ballots to draw', type=click.FloatRange(0.0, 1.0))
    if audit_type is None:
        audit_type = click.prompt('Select an audit type', type=audit_types)
    if delta is None and audit_type == 'athena':
        delta = click.prompt('Enter the Athena delta value', type=click.FloatRange(0.0))

    if audit_type == 'brla':
        return BRLA(alpha, max_fraction_to_draw, contest)
    elif audit_type == 'minerva':
        return Minerva(alpha, max_fraction_to_draw, contest)
    elif audit_type == 'athena':
        return Athena(alpha, delta, max_fraction_to_draw, contest)
    # TODO: add creation for other types of audits.
    return None


def input_contest() -> Contest:
    # Creates a contest from user-input.
    click.echo('\nCreate a new Contest')
    click.echo('====================\n')

    contest_ballots = click.prompt('Enter number of contest ballots', type=click.IntRange(min=1))
    num_candidates = click.prompt('Enter number of candidates', type=click.IntRange(min=2))

    tally = {}
    while len(tally) == 0:
        running_total = 0
        for i in range(num_candidates):
            candidate = click.prompt('Enter candidate name', type=str)
            candidate_votes = click.prompt('Enter number of votes reported for {}'.format(candidate),
                                           type=click.IntRange(0, contest_ballots))
            running_total += candidate_votes
            if running_total > contest_ballots:
                input_warning('Exceeded total ballots cast in contest.')
                click.echo('Restarting tally process...\n')
                tally.clear()
                break
            tally[candidate] = candidate_votes

    num_winners = click.prompt('Enter number of winners', type=click.IntRange(1, num_candidates - 1))
    reported_winners = []
    candidates = list(tally.keys())
    for i in range(num_winners):
        winner = click.prompt('Enter winner name', type=click.Choice(candidates, case_sensitive=False))
        candidates.remove(winner)
        reported_winners.append(winner)

    contest_type_in = click.prompt('Select contest type', type=contest_types)
    contest_type = ContestType[contest_type_in]

    return Contest(contest_ballots, tally, num_winners, reported_winners, contest_type)


def input_election() -> Election:
    # Creates an election from user input
    click.echo('\nCreate a new Election')
    click.echo('=====================\n')

    name = click.prompt('Enter election name', type=str)
    total_ballots = click.prompt('Enter total ballots cast in election', type=click.IntRange(min=1))
    contests = {}

    while True:
        contest_name = click.prompt('\nEnter new contest name', type=str)
        contest = input_contest()
        contests[contest_name] = contest
        if not click.confirm('\nWould you like to enter another contest? '):
            break

    return Election(name, total_ballots, contests)


def input_warning(msg):
    click.echo('\nINVALID INPUT: ' + msg)

<filename>src/r2b2/cli.py<gh_stars>1-10
"""
R2B2's command line interface offers significant out-of-the-box functionality with
respect to executing audits and generating audit data without requiring the user to
write a single line of Python.

Note:
    Why does this file exist, and why not put this in __main__?

      You might be tempted to import things from __main__ later, but that will cause
      problems: the code will get executed twice:

      - When you run `python -m r2b2` python will execute
        ``__main__.py`` as a script. That means there won't be any
        ``r2b2.__main__`` in ``sys.modules``.
      - When you import __main__ it will get executed again (as a module) because
        there's no ``r2b2.__main__`` in ``sys.modules``.

      Also see (1) from http://click.pocoo.org/5/setuptools/#setuptools-integration
"""
import math

import click
import pkg_resources

from r2b2.athena import Athena
from r2b2.audit import Audit
from r2b2.brla import BayesianRLA as BRLA
from r2b2.contest import Contest
from r2b2.contest import ContestType
from r2b2.election import Election
from r2b2.minerva import Minerva
from r2b2.tests import util


# Class for parsing space separated list of integers from command line
class IntList(click.ParamType):
    name = 'integer list'

    def convert(self, value, param, ctx):
        try:
            str_list = value.split(' ')
            return [int(i) for i in str_list]
        except TypeError:
            self.fail('Expected space separated list of integers')


INT_LIST = IntList()

# Audit type choices
# TODO: add new audit types when they become available
audit_types = click.Choice(['brla', 'minerva', 'athena'], case_sensitive=False)
# Contest type choice
contest_types = click.Choice(['PLURALITY', 'MAJORITY'])


@click.group()
def cli():
    click.echo('\nWelcome to the R2B2 auditing tool!\n')
    pass


@cli.command('interactive', short_help='Interactive audit execution.')
@click.option('-e',
              '--election-mode',
              default=False,
              is_flag=True,
              flag_value=True,
              show_default=True,
              help='Use election mode instead of single-contest mode.')
@click.option('--election-file', type=str, help='Pass election data as JSON file.')  # TODO: provide format
@click.option('--contest-file', type=str, help='Pass contest data as JSON file.')  # TODO: provide format
@click.option('-a', '--audit-type', type=audit_types, prompt='Select an audit type', help='Type of audit to execute on given contest.')
@click.option('-r',
              '--risk-limit',
              type=click.FloatRange(0.0, 1.0),
              prompt='Enter desired risk limit (e.g. use 0.1 for 10%)',
              help='Risk limit (alpha) of audit. Should be value between 0 and 1.')
@click.option('-m',
              '--max-fraction-to-draw',
              type=click.FloatRange(0.0, 1.0),
              prompt='Enter maximum fraction of ballots to draw during audit',
              help='Maximum fraction of total contest ballots that could be drawn during the audit.')
@click.option('-v',
              '--verbose',
              default=False,
              is_flag=True,
              flag_value=True,
              show_default=False,
              help='Provides risk and stopping probability schedule of previous rounds, minimum and  maximum sample size.')
def interactive(election_mode, election_file, contest_file, audit_type, risk_limit, max_fraction_to_draw, verbose):
    """Executes an audit round by round.

    Depending on what options are passed to the interactive command, users may be prompted for
    contest results, audit type, risk limit, and/or maximum fraction of contest ballots to draw
    when initializing the contest and audit to run.

    During execution, users will enter each round
    size and results of the round's sample and subsequently receive information about the current
    state of the audit. The process continues until either the stopping conditions are met or the
    audit reaches the maximum sample size.
    \f

    For information on each option run ::

        $ r2b2 interactive --help

    Example:
        Contest results can be passed as a JSON file rather than entering the data through the
        prompt::

            $ r2b2 interactive --contest-file example_contest.json

    Tip:
        To generate a template contest JSON file run::

            $ r2b2 template contest

    Example:
        Audit parameters can be passed in as options rather than entering through the prompt::

            $ r2b2 interactive --audit-type brla --risk-limit 0.1 --max-fraction-to-draw 0.2
            $ r2b2 interactive -a brla -r 0.1 -m 0.2    // Shortened equivalent

    Example:
        Election mode allows users to enter all the results from an election then select a contest
        from the election to audit::

            $ r2b2 interactive -e
            $ r2b2 interactive -e --election-file // pass election results as JSON file.

    Warning:
        Election mode simply allows you to enter an entire election's data, then select one
        one contest from that election to run. Auditing multiple contests from an election
        concurrently is not implemented.
    """

    if election_mode:
        if election_file is not None:
            election = util.parse_election(str(election_file))
        else:
            election = input_election()

        contest_choices = click.Choice(election.contests.keys(), case_sensitive=True)
        click.echo('\n')
        click.echo(election)
        contest_name = click.prompt('Select a contest from the above election', type=contest_choices)
        contest = election.contests[contest_name]

    # Check contest file if provided, otherwise request contest input
    elif contest_file is not None:
        contest = util.parse_contest(str(contest_file))
    else:
        contest = input_contest()

    # Confirm contest, if incorrect get new input
    click.echo('\n')
    if election_mode:
        click.echo(contest_name)
    click.echo(contest)
    while not click.confirm('\nUse the above contest data?'):
        contest = input_contest()
        click.echo(contest)

    # Create audit from prompted input
    audit = input_audit(contest, risk_limit, max_fraction_to_draw, audit_type)
    click.echo(audit)
    if audit_type == 'athena':
        click.echo('Delta: ' + str(audit.delta))
    # Confirm audit, if incorrect get new audit
    while not click.confirm('\nAre the audit parameters correct?'):
        audit = input_audit(contest)
        click.echo(audit)

    # Run audit
    audit.run(verbose)


@cli.command('bulk', short_help='Generate audit data for given round sizes.')
@click.option('-v',
              '--verbose',
              default=False,
              is_flag=True,
              flag_value=True,
              show_default=False,
              help='Provides risk and stopping probability schedule of previous rounds, minimum and  maximum sample size.')
@click.option('-o', '--output', type=click.File('w'), default=None, help='Write output into given file.')
@click.option('-r', '--round-list', type=INT_LIST, default=None, help='Provide a list of round sizes to generate stopping sizes for.')
@click.option('-f', '--full-audit-limit', type=int, default=None, help='Set maximum size for ballot by ballot output data.')
@click.option('-p', '--pair', type=str, default=None, help='Select candidate pair to generate data for.')
@click.argument('contest_file', type=str)
@click.argument('audit_type', type=audit_types)
@click.argument('risk_limit', type=click.FloatRange(0.0, 1.0))
@click.argument('max_fraction_to_draw', type=click.FloatRange(0.0, 1.0))
def bulk(audit_type, risk_limit, max_fraction_to_draw, contest_file, output, round_list, full_audit_limit, pair, verbose):
    """Bulk auditing mode generates stopping sizes for a given fixed round schedule.

    Either provide a list of round sizes for which to generate stopping sizes or
    generate a ballot by ballot list of stopping sizes from the minimum valid sample
    size to the default maximum sample size or a specified maximum sample size.
    \f

    Args:
        contest_file: Contest results as JSON file.
        audit_type: Which audit type to use to generate stopping sizes.
        risk_limit: Risk limit (alpha) of audit.
        max_fraction_to_draw: Maximum fraction of contest ballots that could be drawm during
            the audit. Sets the default maximum size of the ballot by ballot output.

    Tip:
        To generate a template contest JSON file, run::

            $ r2b2 template contest

    Returns:
        Formatted list of rounds and their associated stopping sizes. Default execution is
        ballot by ballot from minimum valid sample size to the maximum sample size of audit.

    Example:
        To generate stopping sizes for a specific set of round sizes, provide the round sizes
        as a space separated list of integers enclosed by quotes using the round list option::

            $ r2b2 bulk -l '100 200 300' contest.json brla 0.1 0.5

    Example:
        To generate a ballot by ballot result from the minimum valid sample size to a specific
        maximum (i.e. not the maximum fraction to draw of the audit), run::

            $ r2b2 bulk -f 221 contest.json brla 0.1 0.5

    Example:
        To write the results to a file instead of to stdout, run::

            $ r2b2 bulk -o output.txt contest.json brla 0.1 0.5

    Tip:
        Generating large or compute heavy data sets can take some time. To estimate run times,
        use the verbose flag to display a progress bar::

            $ r2b2 bulk -v contest.json brla 0.1 0.5

    """
    # Parse audit and contest from arguments
    contest = util.parse_contest(contest_file)

    if audit_type == 'brla':
        audit = BRLA(risk_limit, max_fraction_to_draw, contest)
    else:
        raise click.BadArgumentUsage('No valid audit type found.')

    # Get loser for pairwise audit
    if pair is not None and pair not in audit.sub_audits.keys():
        click.echo('pair [{}] is not valid.'.format(pair))
        pair = None
    if pair is None:
        pairs = click.Choice(audit.sub_audits.keys())
        pair = click.prompt('Select a pair to audit', type=pairs)

    out = '\n{:^20}|{:^20}\n'.format('Round Sizes', 'Stopping Sizes')
    out += '--------------------|--------------------\n'
    if round_list is not None:
        kmins = audit.compute_min_winner_ballots(audit.sub_audits[pair], round_list, progress=verbose)
        for i in range(len(kmins)):
            out += '{:^20}|{:^20}\n'.format(round_list[i], kmins[i])
    elif full_audit_limit is not None:
        kmins = audit.compute_all_min_winner_ballots(audit.sub_audits[pair], full_audit_limit, progress=verbose)
        for r in range(audit.sub_audits[pair].min_sample_size, full_audit_limit + 1):
            out += '{:^20}|{:^20}\n'.format(r, kmins[r - audit.sub_audits[pair].min_sample_size])
    else:
        kmins = audit.compute_all_min_winner_ballots(audit.sub_audits[pair], progress=verbose)
        for r in range(
                audit.sub_audits[pair].min_sample_size,
                min(audit.sub_audits[pair].sub_contest.contest_ballots,
                    math.ceil(max_fraction_to_draw * contest.contest_ballots) + 1)):
            out += '{:^20}|{:^20}\n'.format(r, kmins[r - audit.sub_audits[pair].min_sample_size])

    # Write or print output
    if output is not None:
        output.write(out)
    else:
        click.echo(out)


@cli.command('template', short_help='Generate template JSON input files')
@click.argument('style', type=click.Choice(['contest', 'election']))
@click.option('-o', '--output', type=click.File('wb'), default=None, help='Write output into given file.')
def template(style, output):
    """Generate JSON templates for possible input formats.
    \f

    Example:
        To create a contest results JSON file, first generate the template as a new JSON file::

            $ r2b2 template -o my_contest.json contest

        Now the file my_contest.json will be created and contain::

            {
                "contest_ballots" : 100,
                "tally" : {
                        "CandidateA" : 50,
                        "CandidateB" : 50
                },
                "num_winners" : 1,
                "reported_winners" : ["CandidateA"],
                "contest_type" : "PLURALITY"
            }

        Simply repopulate the fields with your contest results.
    """
    # TODO: docstring
    if style == 'contest':
        template = pkg_resources.resource_string(__name__, 'tests/data/single_contest_template.json')
    elif style == 'election':
        template = pkg_resources.resource_string(__name__, 'tests/data/election_template.json')
    else:
        raise click.BadArgumentUsage('No valid template style found')
    if output is not None:
        output.write(template)
        click.echo('Template written to {}'.format(output.name))
    else:
        click.echo(template)


def input_audit(contest: Contest,
                alpha: float = None,
                max_fraction_to_draw: float = None,
                audit_type: str = None,
                delta: float = None) -> Audit:
    # Create an audit from user-input.
    click.echo('\nCreate a new Audit')
    click.echo('==================\n')

    if alpha is None:
        alpha = click.prompt('Enter the desired risk limit', type=click.FloatRange(0.0, 1.0))
    if max_fraction_to_draw is None:
        max_fraction_to_draw = click.prompt('Enter the maximum fraction of contest ballots to draw', type=click.FloatRange(0.0, 1.0))
    if audit_type is None:
        audit_type = click.prompt('Select an audit type', type=audit_types)
    if delta is None and audit_type == 'athena':
        delta = click.prompt('Enter the Athena delta value', type=click.FloatRange(0.0))

    if audit_type == 'brla':
        return BRLA(alpha, max_fraction_to_draw, contest)
    elif audit_type == 'minerva':
        return Minerva(alpha, max_fraction_to_draw, contest)
    elif audit_type == 'athena':
        return Athena(alpha, delta, max_fraction_to_draw, contest)
    # TODO: add creation for other types of audits.
    return None


def input_contest() -> Contest:
    # Creates a contest from user-input.
    click.echo('\nCreate a new Contest')
    click.echo('====================\n')

    contest_ballots = click.prompt('Enter number of contest ballots', type=click.IntRange(min=1))
    num_candidates = click.prompt('Enter number of candidates', type=click.IntRange(min=2))

    tally = {}
    while len(tally) == 0:
        running_total = 0
        for i in range(num_candidates):
            candidate = click.prompt('Enter candidate name', type=str)
            candidate_votes = click.prompt('Enter number of votes reported for {}'.format(candidate),
                                           type=click.IntRange(0, contest_ballots))
            running_total += candidate_votes
            if running_total > contest_ballots:
                input_warning('Exceeded total ballots cast in contest.')
                click.echo('Restarting tally process...\n')
                tally.clear()
                break
            tally[candidate] = candidate_votes

    num_winners = click.prompt('Enter number of winners', type=click.IntRange(1, num_candidates - 1))
    reported_winners = []
    candidates = list(tally.keys())
    for i in range(num_winners):
        winner = click.prompt('Enter winner name', type=click.Choice(candidates, case_sensitive=False))
        candidates.remove(winner)
        reported_winners.append(winner)

    contest_type_in = click.prompt('Select contest type', type=contest_types)
    contest_type = ContestType[contest_type_in]

    return Contest(contest_ballots, tally, num_winners, reported_winners, contest_type)


def input_election() -> Election:
    # Creates an election from user input
    click.echo('\nCreate a new Election')
    click.echo('=====================\n')

    name = click.prompt('Enter election name', type=str)
    total_ballots = click.prompt('Enter total ballots cast in election', type=click.IntRange(min=1))
    contests = {}

    while True:
        contest_name = click.prompt('\nEnter new contest name', type=str)
        contest = input_contest()
        contests[contest_name] = contest
        if not click.confirm('\nWould you like to enter another contest? '):
            break

    return Election(name, total_ballots, contests)


def input_warning(msg):
    click.echo('\nINVALID INPUT: ' + msg)

R2B2's command line interface offers significant out-of-the-box functionality with respect to executing audits and generating audit data without requiring the user to write a single line of Python.  Note:     Why does this file exist, and why not put this in __main__?        You might be tempted to import things from __main__ later, but that will cause       problems: the code will get executed twice:        - When you run `python -m r2b2` python will execute         ``__main__.py`` as a script. That means there won't be any         ``r2b2.__main__`` in ``sys.modules``.       - When you import __main__ it will get executed again (as a module) because         there's no ``r2b2.__main__`` in ``sys.modules``.        Also see (1) from http://click.pocoo.org/5/setuptools/#setuptools-integration # Class for parsing space separated list of integers from command line # Audit type choices # TODO: add new audit types when they become available # Contest type choice # TODO: provide format # TODO: provide format Executes an audit round by round.      Depending on what options are passed to the interactive command, users may be prompted for     contest results, audit type, risk limit, and/or maximum fraction of contest ballots to draw     when initializing the contest and audit to run.      During execution, users will enter each round     size and results of the round's sample and subsequently receive information about the current     state of the audit. The process continues until either the stopping conditions are met or the     audit reaches the maximum sample size.     \f      For information on each option run ::          $ r2b2 interactive --help      Example:         Contest results can be passed as a JSON file rather than entering the data through the         prompt::              $ r2b2 interactive --contest-file example_contest.json      Tip:         To generate a template contest JSON file run::              $ r2b2 template contest      Example:         Audit parameters can be passed in as options rather than entering through the prompt::              $ r2b2 interactive --audit-type brla --risk-limit 0.1 --max-fraction-to-draw 0.2             $ r2b2 interactive -a brla -r 0.1 -m 0.2    // Shortened equivalent      Example:         Election mode allows users to enter all the results from an election then select a contest         from the election to audit::              $ r2b2 interactive -e             $ r2b2 interactive -e --election-file // pass election results as JSON file.      Warning:         Election mode simply allows you to enter an entire election's data, then select one         one contest from that election to run. Auditing multiple contests from an election         concurrently is not implemented. # Check contest file if provided, otherwise request contest input # Confirm contest, if incorrect get new input # Create audit from prompted input # Confirm audit, if incorrect get new audit # Run audit Bulk auditing mode generates stopping sizes for a given fixed round schedule.      Either provide a list of round sizes for which to generate stopping sizes or     generate a ballot by ballot list of stopping sizes from the minimum valid sample     size to the default maximum sample size or a specified maximum sample size.     \f      Args:         contest_file: Contest results as JSON file.         audit_type: Which audit type to use to generate stopping sizes.         risk_limit: Risk limit (alpha) of audit.         max_fraction_to_draw: Maximum fraction of contest ballots that could be drawm during             the audit. Sets the default maximum size of the ballot by ballot output.      Tip:         To generate a template contest JSON file, run::              $ r2b2 template contest      Returns:         Formatted list of rounds and their associated stopping sizes. Default execution is         ballot by ballot from minimum valid sample size to the maximum sample size of audit.      Example:         To generate stopping sizes for a specific set of round sizes, provide the round sizes         as a space separated list of integers enclosed by quotes using the round list option::              $ r2b2 bulk -l '100 200 300' contest.json brla 0.1 0.5      Example:         To generate a ballot by ballot result from the minimum valid sample size to a specific         maximum (i.e. not the maximum fraction to draw of the audit), run::              $ r2b2 bulk -f 221 contest.json brla 0.1 0.5      Example:         To write the results to a file instead of to stdout, run::              $ r2b2 bulk -o output.txt contest.json brla 0.1 0.5      Tip:         Generating large or compute heavy data sets can take some time. To estimate run times,         use the verbose flag to display a progress bar::              $ r2b2 bulk -v contest.json brla 0.1 0.5 # Parse audit and contest from arguments # Get loser for pairwise audit # Write or print output Generate JSON templates for possible input formats.     \f      Example:         To create a contest results JSON file, first generate the template as a new JSON file::              $ r2b2 template -o my_contest.json contest          Now the file my_contest.json will be created and contain::              {                 "contest_ballots" : 100,                 "tally" : {                         "CandidateA" : 50,                         "CandidateB" : 50                 },                 "num_winners" : 1,                 "reported_winners" : ["CandidateA"],                 "contest_type" : "PLURALITY"             }          Simply repopulate the fields with your contest results. # TODO: docstring # Create an audit from user-input. # TODO: add creation for other types of audits. # Creates a contest from user-input. # Creates an election from user input