MODELS = collections.OrderedDict() def predict_winner(clinton_stat, trump_stat, threshold, is_percent=False):
    difference = clinton_stat - trump_stat
    abs_difference = abs(difference)
    if difference == 0:
        color = 'grey'
        basic_class = ''
        winner = None
    else:
        if difference >= 0:
            color = 'blue'
            winner = 'clinton'
        else:
            color = 'red'
            winner = 'trump'

        max_stat = max(clinton_stat, trump_stat)
        if abs_difference > threshold or abs_difference == max_stat:
            basic_class = ''
        else:
            basic_class = ' basic'

    return {
        'color': color,
        'winner': winner,
        'basic': basic_class,
        'diff': abs(difference),
        'clinton': clinton_stat,
        'trump': trump_stat,
        'suffix': '%' if is_percent else '',
    } class EndorsementForm(forms.Form):
    position = forms.ModelChoiceField(Position.objects.all())
    quote = forms.CharField(
        widget=forms.Textarea(attrs={'rows': 1}),
        required=False,
    )
    context = forms.CharField(
        widget=forms.Textarea(attrs={
            'rows': 1,
            'placeholder': 'e.g., "In an editorial endorsement" (leave blank if no context is required)'
        }),
        required=False,
    )
    date = forms.DateField(widget=Html5DateInput)
    source_url = forms.URLField(
        widget=forms.TextInput(attrs={'placeholder': 'http://example.com'})
    )
    source_name = forms.CharField(
        widget=forms.TextInput(attrs={'placeholder': 'e.g., Politico'})
    )
    event = forms.ModelChoiceField(Event.objects.all(), required=False) class SourceForm(forms.ModelForm):
    class Meta:
        model = Source
        exclude = ('name',)
        widgets = {
            'date': Html5DateInput(),
        } class EndorsementFormWithoutPosition(EndorsementForm):
    position = None
    source_name = forms.CharField(required=False)
    date = forms.DateField(widget=Html5DateInput, required=False)
    source_url = forms.URLField(required=False) class PersonalTagForm(forms.Form):
    tag = forms.ModelChoiceField(
        Tag.objects.filter(is_personal=True)
    ) class EndorserForm(forms.Form):
    name = forms.CharField(max_length=100)
    description = forms.CharField(
        widget=forms.Textarea(attrs={'rows': 4}),
        required=False,
    )
    url = forms.URLField(required=False)
    twitter_username_1 = forms.CharField(
        max_length=15,
        widget=forms.TextInput(attrs={'placeholder': '@username1'}),
        required=False
    )
    twitter_username_2 = forms.CharField(
        max_length=15,
        widget=forms.TextInput(attrs={'placeholder': '@username1'}),
        required=False
    )
    tags = forms.ModelMultipleChoiceField(
        Tag.objects.all(), required=False
    )
    is_personal = forms.BooleanField(required=False) class OrganizationTagForm(forms.Form):
    tag = forms.ModelChoiceField(
        Tag.objects.filter(is_personal=False)
    ) class TagFilterForm(forms.Form):
    filter_tags_show = forms.ModelMultipleChoiceField(
        Tag.objects.all(), required=False
    )
    filter_tags_hide = forms.ModelMultipleChoiceField(
        Tag.objects.all(), required=False
    ) class Account(models.Model):
    objects = AccountManager()

    twitter_id = models.CharField(max_length=20, primary_key=True)

    screen_name = models.CharField(max_length=50)
    name = models.CharField(max_length=100)
    description = models.TextField(blank=True)
    location = models.CharField(max_length=100, blank=True)

    protected = models.BooleanField()
    verified = models.BooleanField()

    profile_image_url = models.URLField(max_length=300, null=True)
    url = models.URLField(max_length=300, blank=True, null=True)

    followers_count = models.PositiveIntegerField()
    friends_count = models.PositiveIntegerField()
    statuses_count = models.PositiveIntegerField()

    last_updated = models.DateTimeField(auto_now=True)

    endorser = models.ForeignKey(Endorser)

    class Meta:
        ordering = ['-followers_count']

    def __unicode__(self):
        return '@{username} ({name})'.format(
            username=self.screen_name,
            name=self.name
        )

    def get_large_image(self):
        return self.profile_image_url.replace('normal', '400x400')

    def get_absolute_url(self):
        return "https://twitter.com/{username}".format(
            username=self.screen_name
        )

    def get_profile_image(self):
        return '<img src="%s" height="100" />' % self.profile_image_url
    get_profile_image.allow_tags = True
    get_profile_image.short_description = 'Profile image'

    def save_image_to_s3(self):
        """TODO"""
        import boto
        s3_connection = boto.connect_s3()
        bucket = s3_connection.get_bucket('endorsementdb.com')

        url = self.get_large_image()

        response = requests.get(url, stream=True)
        with open('/tmp/profile_image.png', 'wb') as out_file:
            shutil.copyfileobj(response.raw, out_file)
            del response

        key = bucket.new_key('images/endorsers/%d.png' % self.endorser.pk)
        key.set_contents_from_filename(out_file.name)
        key.make_public() class Endorser(models.Model):
    name = models.CharField(max_length=100)
    description = models.TextField(null=True, blank=True)
    url = models.URLField(null=True, blank=True)
    is_personal = models.BooleanField(default=True)
    max_followers = models.PositiveIntegerField(default=0)
    tags = models.ManyToManyField(Tag, blank=True)
    missing_image = models.BooleanField(default=True)
    current_position = models.ForeignKey('Position', blank=True, null=True)

    class Meta:
        ordering = ['-max_followers']

    def __unicode__(self):
        return self.name

    def has_url(self):
        return bool(self.url)
    has_url.boolean = True

    def get_tags(self):
        return ' / '.join(tag.name for tag in self.tags.all())

    def get_current_endorsement(self):
        try:
            return self.endorsement_set.latest('quote')
        except Endorsement.DoesNotExist:
            pass

    def get_image(self):
        return '<img src="{url}" width="100" />'.format(
            url=self.get_image_url()
        )
    get_image.allow_tags = True

    def get_image_url(self):
        image_name = 'missing' if self.missing_image else self.pk
        return 'https://s3.amazonaws.com/endorsementdb.com/images/endorsers/%s.png' % image_name

    def needs_quotes(self):
        current_endorsement = self.get_current_endorsement()
        return not current_endorsement.quote.text
    needs_quotes.boolean = True

    def get_absolute_url(self):
        return reverse('view-endorser', args=[self.pk]) class Candidate(models.Model):
    endorser_link = models.OneToOneField(Endorser)
    name = models.CharField(max_length=50)
    description = models.TextField()
    color = models.CharField(max_length=6)
    rgb = models.CharField(max_length=13)
    still_running = models.BooleanField(default=False)
    position = models.OneToOneField('Position', blank=True, null=True)

    def __unicode__(self):
        return self.name class Source(models.Model):
    date = models.DateField(null=True, blank=True)
    url = models.URLField(unique=True, blank=True, max_length=300)
    name = models.CharField(max_length=100, default='')

    class Meta:
        ordering = ['url', '-date']

    def get_date_display(self):
        if self.date:
            return self.date.strftime('%b %d, %Y')

    def __unicode__(self):
        return "{url} on {date}".format(
            url=self.url,
            date=self.date,
        ) class Quote(models.Model):
    context = models.TextField(null=True, blank=True)
    text = models.TextField(blank=True, null=True)
    source = models.ForeignKey(Source)
    # Defaults to the date of the source, but can be overridden.
    date = models.DateField(null=True, blank=True)
    event = models.ForeignKey(Event, null=True, blank=True)

    def get_date_display(self):
        if self.date:
            return self.date.strftime('%b %d, %Y')

    def get_source_display(self):
        return '<a href="{url}">{name}</a>'.format(
            url=self.source.url,
            name=self.source.name or self.source.url[:30],
        )
    get_source_display.allow_tags = True

    def __unicode__(self):
        return self.text[:150]

    class Meta:
        ordering = ['date']

    def get_event_context(self):
        event = self.event
        if event:
            if self.date >= event.end_date:
                return 'After'
            elif self.date <= event.start_date:
                return 'Before'
            else:
                return 'During'
    get_event_context.short_description = 'Timing'

    def get_display(self):
        if self.context:
            if len(self.context) > 100:
                context = self.context[:100] + '...<br />'
            else:
                context = self.context + '<br />'
        else:
            context = ''

        if self.text:
            if len(self.text) > 100:
                text = self.text[:100] + '...'
            else:
                text = self.text
        else:
            text = ''

        return context + text
    get_display.short_description = 'Context + text'
    get_display.allow_tags = True class Tag(models.Model):
    name = models.CharField(max_length=50)
    description = models.TextField(blank=True)
    is_personal = models.BooleanField(default=True)
    category = models.ForeignKey(Category, null=True, blank=True)

    def __unicode__(self):
        return self.name

    class Meta:
        ordering = ['name']

    def get_mode(self):
        allow_personal = self.category.allow_personal
        allow_org = self.category.allow_org
        if allow_personal and allow_org:
            return 'none'
        elif allow_personal:
            return 'personal'
        else:
            return 'organization' class Endorsement(models.Model):
    endorser = models.ForeignKey(Endorser)
    quote = models.ForeignKey(Quote)
    position = models.ForeignKey(Position)
    confirmed = models.BooleanField(default=True)
    notes = models.TextField(null=True, blank=True)

    class Meta:
        ordering = ['-quote']

    def get_date(self):
        return self.quote.date

    def __unicode__(self):
        return unicode(self.position)

    def get_truncated_quote(self):
        if len(self.quote.text) > 100:
            return self.quote.text[:100] + '...'
        else:
            return self.quote.text class Category(models.Model):
    is_exclusive = models.BooleanField(default=False)
    name = models.CharField(max_length=30)
    allow_personal = models.BooleanField(default=True)
    allow_org = models.BooleanField(default=True)

    def __unicode__(self):
        return self.name

    class Meta:
        verbose_name_plural = 'categories' class Position(models.Model):
    colour = models.CharField(max_length=20, blank=True)
    past_tense_prefix = models.CharField(max_length=10, blank=True)
    present_tense_prefix = models.CharField(max_length=10, blank=True)
    suffix = models.CharField(max_length=30)
    slug = models.SlugField()
    show_on_load = models.BooleanField(default=False)

    def __unicode__(self):
        return self.get_name_display()

    def get_name_display(self):
        if self.present_tense_prefix:
            return "{present} {suffix}".format(
                present=self.present_tense_prefix,
                suffix=self.suffix
            )
        else:
            return self.suffix

    def get_present_display(self):
        if self.present_tense_prefix:
            return self.present_tense_prefix + ' ' + self.suffix
        else:
            return self.suffix

    def get_past_display(self):
        if self.past_tense_prefix:
            return self.past_tense_prefix + ' ' + self.suffix
        else:
            return self.suffix @register.filter(name='shorten')
def shorten(n):
    """Takes in an integer (over a thousand) and shortens it by using K or M.
    Examples:
    * 1,234,567 becomes 1.2M
    * 12,345 becomes 12K
    * 99,999,999 becomes 99M
    """
    if n > 10000000:
        return '%dM' % round(n / 1000000)
    elif n > 1000000:
        # It's between 1 and 10 million. Include the decimal point.
        return '%.1fM' % round(n / 1000000., 2)
    elif n > 1000:
        pre_decimal = int(round(n / 1000))
        return "%dK" % pre_decimal
    else:
        return str(n) class BulkImport(models.Model):
    slug = models.SlugField(max_length=255)
    created_at = models.DateTimeField(auto_now_add=True)
    text = models.TextField()

    def __unicode__(self):
        return str(self.created_at) class ImportedEndorsement(models.Model):
    bulk_import = models.ForeignKey(BulkImport)
    raw_text = models.TextField()
    imported_endorser = models.ForeignKey(
        ImportedEndorser, blank=True, null=True
    )
    confirmed_endorser = models.ForeignKey(Endorser, blank=True, null=True)
    sections = models.TextField(blank=True, null=True)
    notes = models.TextField(blank=True, null=True)

    def parse_text(self):
        return utils.parse_wiki_text(self.raw_text)

    def get_likely_endorser(self):
        name = self.parse_text()['endorser_name']
        if not name:
            return

        name = name.lower()

        query = Endorser.objects.filter(
            name__iexact=name,
        )
        if query.exists():
            return query.latest('pk')

        # See if putting "The" in front (or removing it) helps.
        if name.startswith('the '):
            query = Endorser.objects.filter(
                name__iexact=name[4:],
            )
            if query.exists():
                return query.first()
        else:
            query = Endorser.objects.filter(
                name__iexact='the ' + name,
            )
            if query.exists():
                return query.first()

        # See if the last name and part of the first name match.
        if ' ' in name:
            split_name = name.split(' ')
            first_name_start = split_name[0][:3]
            last_name = split_name[-1]
            if len(last_name) > 3:
                query = Endorser.objects.filter(
                    name__iendswith=last_name,
                    name__istartswith=first_name_start,
                )
                if query.exists():
                    return query.first()

    def __unicode__(self):
        return self.raw_text NEWSPAPER_SLUG = 'Newspaper_endorsements_in_the_United_States_presidential_election,_2016' class ImportedNewspaper(models.Model):
    bulk_import = models.ForeignKey(BulkImport)
    confirmed_endorser = models.ForeignKey(Endorser, blank=True, null=True)
    section = models.PositiveSmallIntegerField(choices=NEWSPAPER_SECTIONS)
    name = models.CharField(max_length=100)
    endorsement_2016 = models.CharField(max_length=15)
    endorsement_2012 = models.CharField(max_length=15, blank=True, null=True)
    circulation = models.PositiveIntegerField(blank=True, null=True)
    date = models.DateField(blank=True, null=True)
    city = models.CharField(max_length=20, blank=True, null=True)
    state = models.CharField(max_length=20, blank=True, null=True)
    url = models.URLField(blank=True, null=True, max_length=255)

    def __unicode__(self):
        return self.name

    def get_likely_endorser(self):
        name = self.name.lower()

        query = Endorser.objects.filter(
            name__iexact=name,
        )
        if query.exists():
            return query.first()

        # See if putting "The" in front (or removing it) helps.
        if name.startswith('the '):
            query = Endorser.objects.filter(
                name__iexact=name[4:],
            )
            if query.exists():
                return query.first()
        else:
            query = Endorser.objects.filter(
                name__iexact='the ' + name,
            )
            if query.exists():
                return query.first() class ImportedResult(models.Model):
    bulk_import = models.ForeignKey(BulkImport)
    tag = models.ForeignKey(Tag)
    candidate = models.ForeignKey(Candidate)
    count = models.PositiveIntegerField(default=0)
    percent = models.DecimalField(max_digits=4, decimal_places=2)

    class Meta:
        unique_together = ('tag', 'candidate') class ImportedRepresentative(models.Model):
    bulk_import = models.ForeignKey(BulkImport)
    state = models.ForeignKey(
        Tag,
        limit_choices_to={
            'category': 8,
        },
        related_name='+',
    )
    confirmed_endorser = models.ForeignKey(Endorser, blank=True, null=True)
    party = models.ForeignKey(
        Tag,
        limit_choices_to={
            'category': 2
        },
        related_name='+',
    )
    name = models.CharField(max_length=50)

    def get_likely_endorser(self):
        name = self.name.lower()

        query = Endorser.objects.filter(
            name__iexact=name,
        )
        if query.exists():
            return query.first()

        if ' ' in name:
            split_name = name.split(' ')
            first_name_start = split_name[0][:3]
            last_name = split_name[-1]
            if len(last_name) > 3:
                query = Endorser.objects.filter(
                    name__iendswith=last_name,
                    name__istartswith=first_name_start,
                )
                if query.exists():
                    return query.first()

        query = Endorser.objects.filter(
            name__icontains=name,
        )
        if query.exists():
            return query.first() class ElectoralVotes(models.Model):
    """Technically not imported from Wikipedia (entered manually), but this
    model fits better in the Wikipedia app."""
    state = models.OneToOneField(
        Tag,
        limit_choices_to={
            'category': 8,
        },
    )
    count = models.PositiveSmallIntegerField()

    def __unicode__(self):
        return "{state} - {count}".format(
            state=self.state,
            count=self.count
        )
import collections
import json
import random
from django.contrib import messages
from django.contrib.admin.views.decorators import staff_member_required
from django.core.cache import cache
from django.db.models import Count, Max, Min
from django.http import Http404, JsonResponse
from django.shortcuts import redirect, render, get_object_or_404
from django.urls import reverse
from django.views.decorators.cache import never_cache
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_POST
from election.predictions import MODELS
from election.utils import predict_winner
from endorsements.forms import EndorsementForm, SourceForm, \
                               EndorsementFormWithoutPosition, \
                               PersonalTagForm, EndorserForm, \
                               OrganizationTagForm, \
                               TagFilterForm
from endorsements.models import Account, Endorser, Candidate, Source, Quote, \
                                Tag, Endorsement, Category, Position
from endorsements.templatetags.endorsement_extras import shorten
from wikipedia.models import BulkImport, ImportedEndorsement, NEWSPAPER_SLUG, \
                             ImportedNewspaper, ImportedResult, \
                             ImportedRepresentative, ElectoralVotes
        num_endorsers = Endorser.objects.count()
        category_endorsers = Endorser.objects.filter(tags__category=category)
        num_tagged = category_endorsers.count()
        percent_reporting = num_tagged / float(num_endorsers) * 100

        categories.append({
            'name': category.name,
            'candidates': category_candidates,
            'tags': category_tags,
            'num_tagged': num_tagged,
            'percent_reporting': percent_reporting,
        })

    context = {
        'candidates': candidates,
        'categories': categories,
    }

    return render(request, 'stats/tags.html', context)


def charts(request):
    context = {}
    return render(request, 'stats/charts.html', context)


SLUG_MAPPING = {
    'List_of_Gary_Johnson_presidential_campaign_endorsements,_2016': 'johnson',
    'List_of_Jill_Stein_presidential_campaign_endorsements,_2016': 'stein',
    'List_of_Evan_McMullin_presidential_campaign_endorsements,_2016': 'mcmullin',
    'List_of_Donald_Trump_presidential_campaign_endorsements,_2016': 'trump',
    'List_of_Hillary_Clinton_presidential_campaign_endorsements,_2016': 'clinton',
}
def progress_wikipedia(request):
    slug_counts = collections.defaultdict(collections.Counter)
    for e in ImportedEndorsement.objects.all().prefetch_related('bulk_import'):
        slug = e.bulk_import.slug
        if e.confirmed_endorser_id:
            slug_counts[slug]['confirmed'] += 1
        else:
            if e.notes:
                slug_counts[slug]['unverifiable'] += 1
            else:
                slug_counts[slug]['unconfirmed'] += 1
        slug_counts[slug]['imported'] += 1

    positions = []
    for slug in slug_counts:
        position_slug = SLUG_MAPPING.get(slug)
        if position_slug:
            position = Position.objects.get(slug=position_slug)
            name = position.get_present_display()
            endorsers = Endorser.objects.filter(endorsement__position=position)
            missing_endorsers = endorsers.filter(importedendorsement=None)
            num_endorsers = endorsers.count()
            num_missing = missing_endorsers.count()
        else:
            position = None
            name = '--'
            num_endorsers = 0
            num_missing = 0

        num_unconfirmed = slug_counts[slug]['unconfirmed']
        num_imported = slug_counts[slug]['imported']
        if num_unconfirmed > 0:
            progress = int(
                (num_imported - num_unconfirmed) / float(num_imported) * 100
            )
        else:
            progress = 100

        last_import = BulkImport.objects.filter(slug=slug).latest('created_at')

        positions.append({
            'name': name,
            'last_checked': last_import.created_at,
            'slug': slug,
            'num_confirmed': slug_counts[slug]['confirmed'],
            'num_unconfirmed': num_unconfirmed,
            'num_unverifiable': slug_counts[slug]['unverifiable'],
            'num_imported': num_imported,
            'num_missing': num_missing,
            'num_endorsements': num_endorsers,
            'progress': progress,
        })

    num_newspapers = ImportedNewspaper.objects.count()
    num_confirmed = ImportedNewspaper.objects.filter(
        confirmed_endorser__isnull=False
    ).count()
    last_newspaper_import = BulkImport.objects.filter(
        importednewspaper__isnull=False
    ).latest('pk')

    num_missing = Endorser.objects.filter(
        importednewspaper=None,
        tags=Tag.objects.get(name='Publication')
    ).count()

    progress = num_confirmed / float(num_newspapers) * 100
    positions.append({
        'name': 'Newspaper endorsements',
        'last_checked': last_newspaper_import.created_at,
        'slug': last_newspaper_import.slug,
        'num_imported': num_newspapers,
        'num_confirmed': num_confirmed,
        'num_unconfirmed': num_newspapers - num_confirmed,
        'num_unverifiable': 0,
        'num_missing': num_missing,
        'progress': progress,
    })

    context = {
        'positions': positions,
    }
    return render(request, 'progress/wikipedia.html', context)


def progress_wikipedia_missing(request, slug):
    if slug == NEWSPAPER_SLUG:

        endorsements = Endorsement.objects.filter(