def first(coll): # rewrite to use frozen dict
def last(coll): # rewrite to use frozen dict
def rest(coll): # rewrite to use frozen dict
def none(*args, **kwargs):
def is_seq(x):
def fmap(f, coll):
def walk(inner, outer, data):
    def process_node(inner, k, v):
def cons(x, seq):
def thread(x, form):
def threadfirst(x, form, *more):
def compose(*funcs):
    def compose2(f, g):
def threadlast(x, *funcs):
def thread_first(x, form, *more):
def thread_last(x, *funcs):
def memoize(f):
    def wrapper(*args):
def frozendict(*keyvals):
def zipdict(keys, vals):
def get(fdict, key, default=None):
def contains(fdict, key):
def find(fdict, key):
def keys(fdict):
def vals(fdict):
def merge(*fdicts):
    def items(fdict):
def walk_replace(smap, data):
    def replace_at(k, v):
    def process_node(k, v):
def union(*sets):
def intersection(x, y):
def dict_invert(dict):
def flatten(dict):
def assoc(fdict, key, val, *kvs):
def dissoc(fdict, key, *ks):
def hash_map(*keyvals):
def format(fmt, *args, **kwargs):
def select_keys(fdict, keys):
def destructure(f):
    def wrapper(*args, **kwargs): class NL(models.Model):
    """
    Natural language command.
    """
    str = models.TextField(primary_key=True) class Command(models.Model):
    """
    Command line.
    """
    str = models.TextField(primary_key=True)
    template = models.TextField(default='')
    language = models.TextField(default='bash')
    tags = models.ManyToManyField('Tag') class Comment(models.Model):
    """
    Each record is a commend submitted by a user (either an annotator or a
    judger) to an annotation.
    """
    user = models.ForeignKey(User, on_delete=models.CASCADE)
    reply = models.ForeignKey('self', null=True, on_delete=models.CASCADE)
    str = models.TextField()
    submission_time = models.DateTimeField(default=timezone.now) class URL(models.Model):
    """
    URL.

    :member str: url address.
    :member html_content: snapshot of the URL content at the time of annotation.
    :member commands: commands in the URL (automatically extracted)
    :member tags: tags of the URL (assigned based on user annotations)
    """
    str = models.TextField(primary_key=True)
    html_content = models.TextField(default='')
    commands = models.ManyToManyField('Command')
    tags = models.ManyToManyField('Tag') class User(models.Model):
    """
    Each record stores the information of a user.

    :member is_annotator: the annotator is responsible for collecting new data
        pairs
    :member is_judger: the judger is responsible for judging if a command pair
        is correct and add modifications and comments.
    """
    access_code = models.TextField(default='')
    ip_address = models.TextField(default='')
    first_name = models.TextField(default='anonymous')
    last_name = models.TextField(default='anonymous')
    organization = models.TextField(default='--')
    city = models.TextField(default='--')
    region = models.TextField(default='--')
    country = models.TextField(default='--')
    is_annotator = models.BooleanField(default=False)
    is_judger = models.BooleanField(default=False)
    time_logged = models.FloatField(null=True, blank=True) class URLTag(models.Model):
    """
    Each record stores a (url, tag) pair.
    """
    url = models.ForeignKey(URL, on_delete=models.CASCADE)
    tag = models.TextField() class Annotation(models.Model):
    """
    Each record is a natural language <-> code translation annotated by a
    programmer.
    """
    url = models.ForeignKey(URL, on_delete=models.CASCADE)
    nl = models.ForeignKey(NL, on_delete=models.CASCADE)
    cmd = models.ForeignKey(Command, on_delete=models.CASCADE)
    annotator = models.ForeignKey(User, on_delete=models.CASCADE)
    submission_time = models.DateTimeField(default=timezone.now) class AnnotationUpdate(models.Model):
    """
    Each record is an update of an annotation submitted by a judger.
    """
    annotation = models.ForeignKey(Annotation, on_delete=models.CASCADE)
    judger = models.ForeignKey(User, on_delete=models.CASCADE)
    update_str = models.TextField()
    update_type = models.TextField(default='nl')
    comment = models.ForeignKey(Comment, on_delete=models.CASCADE)
    submission_time = models.DateTimeField(default=timezone.now)
    status = models.TextField(default='open') class AnnotationProgress(models.Model):
    """
    Each record stores a user's annotation progress on a particular URL.
    """
    url = models.ForeignKey(URL, on_delete=models.CASCADE)
    tag = models.ForeignKey(Tag, null=True, on_delete=models.CASCADE)
    annotator = models.ForeignKey(User, on_delete=models.CASCADE)
    status = models.TextField() class Notification(models.Model):
    """
    Each record is a certain type of message issued from one annotator to
    another.
    """
    sender = models.ForeignKey(User, related_name='notification_sender',
                               on_delete=models.CASCADE)
    receiver = models.ForeignKey(User, related_name='notification_receiver',
                                 on_delete=models.CASCADE)
    type = models.TextField(default='comment')
    annotation_update = models.ForeignKey('AnnotationUpdate', null=True,
                                          on_delete=models.CASCADE)
    comment = models.ForeignKey('Comment', null=True, on_delete=models.CASCADE)
    url = models.ForeignKey('URL', on_delete=models.CASCADE)
    creation_time = models.DateTimeField(default=timezone.now)
    status = models.TextField(default='issued') def json_response(d={}, status='SUCCESS'):
    d.update({'status': status})
    resp = JsonResponse(d)
    return resp def get_nl(nl_str):
    nl, _ = NL.objects.get_or_create(str=nl_str.strip())
    return nl def get_command(command_str):
    command_str=command_str.strip()
    if Command.objects.filter(str=command_str).exists():
        cmd = Command.objects.get(str=command_str)
    else:
        cmd = Command.objects.create(str=command_str)
        ast = data_tools.bash_parser(command_str)
        for utility in data_tools.get_utilities(ast):
            cmd.tags.add(get_tag(utility))    
        template = data_tools.ast2template(
	        ast, loose_constraints=True)
        cmd.template = template
        cmd.save()
    return cmd def get_url(url_str):
    url_str = url_str.strip()
    try:
        url = URL.objects.get(str=url_str)
    except ObjectDoesNotExist:
        html = extract_html(url_str)
        if html is None:
            url = URL.objects.create(str=url_str)
        else:
            url = URL.objects.create(str=url_str, html_content=html)
    return url def get_tag(tag_str):
    tag, _ = Tag.objects.get_or_create(str=tag_str.strip())
    return tag
from django.core.exceptions import ObjectDoesNotExist
from django.db.models import Count
from django.http import HttpResponse
from django.template import loader
from website import functions
from website.models import NL, Command, Comment, URL, User, URLTag, \
    Annotation, AnnotationUpdate, AnnotationProgress, Notification
from website.utils import json_response
from website.utils import get_nl, get_command, get_url, get_tag


WHITE_LIST = {'find', 'xargs'}
BLACK_LIST = {'cpp', 'g++', 'java', 'perl', 'python', 'ruby', 'nano', 'emacs',
              'vim'}

GREY_LIST = {'alias', 'unalias', 'set', 'unset', 'screen', 'apt-get', 'brew',
             'yum', 'export', 'shift', 'exit', 'logout'}


def access_code_required(f):
    @functions.wraps(f)
    def g(request, *args, **kwargs):
        try:
            access_code = request.COOKIES['access_code']
        except KeyError:
            return login(request)
        return f(request, *args, access_code=access_code, **kwargs)
    return g


def safe_get_user(access_code):
    try:
        user = User.objects.get(access_code=access_code)
        return user
    except ObjectDoesNotExist:
        print('User {} does not exist!'.format(access_code))
        return None


@access_code_required
def collect_page(request, access_code):
    """
    Collection Interface.
    """
    template = loader.get_template('annotator/collect_page.html')
    user = safe_get_user(access_code)

    utility = request.GET.get('utility')
    tag = get_tag(utility)
    url = get_url(request.GET.get('url'))

    # search for existing annotations
    annotation_dict = {}
    command_list = []

    if user.is_judger:
        # judger sees the annotations of a web page by all other users

        annotation_list = Annotation.objects.filter(url=url)