class Distribution(models.Model):
    """Model that stores the names of distributions that are available in the
    elasticsearch index.

    The distributions are part of the metadata of a query.
    """
    id = models.CharField(max_length=15, primary_key=True)
    name = models.CharField(max_length=35)

    def __unicode__(self):
        return self.id class ArticleType(models.Model):
    """Model to store the names of article types that are available in the
    elasticsearch index.

    The article types are part of the metadata of a query.
    """
    id = models.CharField(max_length=15, primary_key=True)
    name = models.CharField(max_length=35)

    def __unicode__(self):
        return self.id class Query(models.Model):
    """Model to store a User's queries.
    """
    title = models.CharField(max_length=100)
    comment = models.TextField(blank=True)
    query = models.TextField()
    nr_results = models.PositiveIntegerField(null=True)

    exclude_article_types = models.ManyToManyField(ArticleType, blank=True)
    exclude_distributions = models.ManyToManyField(Distribution, blank=True)
    selected_pillars = models.ManyToManyField(Pillar, blank=True)
    user = models.ForeignKey(User)

    date_created = models.DateTimeField(auto_now_add=True)
    date_modified = models.DateTimeField(auto_now=True, null=True)

    class Meta:
        """Make sure that Query titles are unique for a User"""
        unique_together = ('user', 'title')
        # Custom permissions
        permissions = (
            (
                'download_documents',
                'Can download results of a query',
            ),
            (
                'download_many_documents',
                'Can download larger numbers of results',
            ),
        )
        verbose_name_plural = 'Queries'

    def get_query_dict(self):
        """Returns a JSON serializable representation of the query object, that
        contains all relevant data and metadata.
        """
        periods = Period.objects.filter(query=self)
        selected_dateranges = [{'lower': p.date_lower.isoformat(), 'upper': p.date_upper.isoformat()} for p in periods]
        excl_art_types = [a.id for a in self.exclude_article_types.all()]
        excl_distr = [d.id for d in self.exclude_distributions.all()]
        selected_pillars = [p.id for p in self.selected_pillars.all()]
        selected_pillar_names = [p.name for p in self.selected_pillars.all()]

        return {
            'pk': self.pk,
            'title': self.title,
            'query': self.query,
            'comment': self.comment,
            'nr_results': self.nr_results,
            'date_created': self.date_created.isoformat(),
            'dates': selected_dateranges,
            'exclude_article_types': excl_art_types,
            'exclude_distributions': excl_distr,
            'selected_pillars': selected_pillars,
            'selected_pillar_names': selected_pillar_names
        }

    def __unicode__(self):
        return self.query class DayStatistic(models.Model):
    """DayStatistic is used to generate timeline graphs. Data for the
    DayStatistic table is gathered with the 'gatherstatistics' management
    command.
    """
    date = models.DateField()
    distribution = models.ForeignKey(Distribution, blank=True, null=True)
    article_type = models.ForeignKey(ArticleType, blank=True, null=True)
    count = models.IntegerField()
    checked = models.DateTimeField(auto_now=True)

    class Meta:
        unique_together = (('date', 'distribution', 'article_type'),)

    def __unicode__(self):
        return '{}: {}'.format(str(self.date), self.count) class StopWord(models.Model):
    """Model to store stopwords.

    Stopwords can be:

    - specific for a Query (applicable to a single User's Query, query/user != None)
    - specific for a User (applicable for all of a User's Queries, query == None, user != None)
    - non-specific/default (applicable to all Queries for all Users, user/query == None)
    """
    user = models.ForeignKey(User, null=True, blank=True)
    query = models.ForeignKey(Query, null=True, blank=True)
    word = models.CharField(max_length=100)

    def __unicode__(self):
        return self.word

    def get_stopword_dict(self):
        if self.user:
            user_name = self.user.username
        else:
            user_name = ''

        if self.query:
            query = self.query.title
        else:
            query = ''

        return {
            'id': self.id,
            'user': user_name,
            'query': query,
            'word': self.word
        } class Pillar(models.Model):
    """Model that allows to store a categorization of Newspapers along Pillars."""
    name = models.CharField(max_length=200, unique=True)

    def __unicode__(self):
        return self.name class Newspaper(models.Model):
    """Model that stores the available Newspapers"""
    id = models.CharField(max_length=9, primary_key=True)
    title = models.CharField(max_length=500)
    start_date = models.DateField()
    end_date = models.DateField()
    editions = models.PositiveIntegerField()
    pillar = models.ForeignKey(Pillar, null=True)

    def __unicode__(self):
        return self.title class Period(models.Model):
    """Model to store the Periods for a Query"""
    date_lower = models.DateField()
    date_upper = models.DateField()
    query = models.ForeignKey(Query) class Term(models.Model):
    """Model to store frequencies and inverse document frequencies per term"""
    PRE_WWII = 'pre'
    WWII = 'WWII'
    POST_WWII = 'post'
    TIMEFRAME_CHOICES = (
        (PRE_WWII, 'pre-WWII: 1900-1940'),
        (WWII, 'WWII: 1940-1945'),
        (POST_WWII, 'post-WWII: 1945-1990'),
    )
    timeframe = models.CharField(max_length=4, choices=TIMEFRAME_CHOICES)
    word = models.CharField(max_length=200)
    count = models.PositiveIntegerField()
    idf = models.DecimalField(max_digits=7, decimal_places=4)

    class Meta:
        unique_together = (('timeframe', 'word'),) def get_query_object(query_id):
    """
    Returns the query object stored for a query id and an appropriate error
    message if the query cannot be retrieved.
    """
    query = None
    response = None

    try:
        query = Query.objects.get(pk=query_id)
    except Query.DoesNotExist:
        msg = 'Query with id %s cannot be found.' % query_id
        response = json_response_message('error', msg)
    except DatabaseError:
        response = json_response_message(
            'error',
            'Database error while retrieving query.')

    if not query and not response:
        response = json_response_message('error', 'No query found.')

    return query, response def query2docidsdate(query, resolution):
    """
    Get the document ids plus their date for the query.
    """
    if settings.DEBUG:
        print >> stderr, "query2docidsdate()"

    query_dict = query.get_query_dict()

    return get_document_ids(settings.ES_INDEX,
                            settings.ES_DOCTYPE,
                            query_dict['query'],
                            query_dict['dates'],
                            resolution,
                            query_dict['exclude_distributions'],
                            query_dict['exclude_article_types'],
                            query_dict['selected_pillars']) def count_results(query):
    """Returns the number of results for a Query"""
    params = query.get_query_dict()
    result = count_search_results(settings.ES_INDEX,
                                  settings.ES_DOCTYPE,
                                  params['query'],
                                  params['dates'],
                                  params['exclude_distributions'],
                                  params['exclude_article_types'],
                                  params['selected_pillars'])
    return result.get('count') def get_search_parameters(req_dict):
    """Return a tuple of search parameters extracted from a dictionary

    Parameters:
        req_dict : dict
            A Django request dictionary

    Returns:
        dict : dict
            A dictionary that contains query metadata
    """
    query_str = req_dict.get('query', None)

    start = int(req_dict.get('startRecord', 1))

    result_size = int(req_dict.get('maximumRecords', 20))

    date_range_str = req_dict.get('dateRange', settings.TEXCAVATOR_DATE_RANGE)
    dates = daterange2dates(date_range_str)

    distributions = []
    for ds in _KB_DISTRIBUTION_VALUES.keys():
        use_ds = json.loads(req_dict.get(ds, "true"))
        if not use_ds:
            distributions.append(ds)

    article_types = []
    for typ in _KB_ARTICLE_TYPE_VALUES:
        use_type = json.loads(req_dict.get(typ, "true"))
        if not use_type:
            article_types.append(typ)

    pillars = [int(x) for x in req_dict.getlist('pillars')]
    collection = req_dict.get('collection', settings.ES_INDEX)
    sort_order = req_dict.get('sort_order', '_score')

    return {
        'query': query_str,
        'start': start,
        'result_size': result_size,
        'dates': dates,
        'distributions': distributions,
        'article_types': article_types,
        'pillars': pillars,
        'collection': collection,
        'sort_order': sort_order
    }
import csv
import logging
import os
from collections import defaultdict
from datetime import datetime
from sys import stderr
from urllib import quote_plus, unquote_plus
from urlparse import urljoin
from django.conf import settings
from django.contrib.auth.decorators import login_required, permission_required
from django.core.servers.basehttp import FileWrapper
from django.core.validators import validate_email
from django.db import IntegrityError
from django.db.models import Q, Min, Max, Sum
from django.http import HttpResponse
from django.shortcuts import get_object_or_404
from django.template.loader import render_to_string
from django.views.decorators.csrf import csrf_exempt
from .models import Distribution, ArticleType, Query, DayStatistic, \
    StopWord, Pillar, Newspaper, Period, Term
from .utils import get_query_object, query2docidsdate, count_results
from .download import create_zipname, execute
from services.es import get_search_parameters
from texcavator.utils import json_response_message
    try:
        query.query = params['query']
        query.title = request.POST.get('title')
        query.comment = request.POST.get('comment')
        query.save()

        Period.objects.filter(query__pk=query_id).delete()
        for date_range in params['dates']:
            date_lower = datetime.strptime(date_range['lower'], '%Y-%m-%d')
            date_upper = datetime.strptime(date_range['upper'], '%Y-%m-%d')
            p = Period(query=query, date_lower=date_lower, date_upper=date_upper)
            p.save()

        query.exclude_distributions.clear()
        for distr in Distribution.objects.all():
            if distr.id in params['distributions']:
                query.exclude_distributions.add(distr)

        query.exclude_article_types.clear()
        for art_type in ArticleType.objects.all():
            if art_type.id in params['article_types']:
                query.exclude_article_types.add(art_type)

        query.selected_pillars.clear()
        for pillar in Pillar.objects.all():
            if pillar.id in params['pillars']:
                query.selected_pillars.add(pillar)

        query.nr_results = count_results(query)
        query.save()
    except Exception as e:
        return json_response_message('ERROR', str(e))

    return json_response_message('SUCCESS', 'Query updated.')


@login_required
@csrf_exempt
def update_nr_results(request, query_id):
    query, response = get_query_object(query_id)

    if not query:
        return response

    query.nr_results = count_results(query)
    query.save()

    return json_response_message('SUCCESS', 'Number of results updated.', {'count': query.nr_results})


@login_required
def timeline(request, query_id, resolution):
    """
    Generates a timeline for a query.
    TODO: the timeline view should be moved to a separate app
    """
    logger.info('query/timeline/ - user: {}'.format(request.user.username))

    normalize = request.GET.get('normalize') == '1'

    if settings.DEBUG:
        print >> stderr, "query/bursts() query_id:", query_id
        print >> stderr, "resolution:", resolution
        print >> stderr, "query_id:", query_id
        print >> stderr, "normalize:", normalize
        print >> stderr, "resolution:", resolution

    query = get_object_or_404(Query, pk=query_id)

    # Retrieve the min/max date
    periods = Period.objects.filter(query=query)
    _, begindate = periods.aggregate(Min('date_lower')).popitem()
    _, enddate = periods.aggregate(Max('date_upper')).popitem()

    # Retrieve a dictionary with date and corresponding documents for this Query
    date2doc = query2docidsdate(query, resolution)

    # Retrieve normalization values (if necessary)
    if normalize:
        values = DayStatistic.objects.\
            filter(date__range=(begindate, enddate)).\
            exclude(article_type__in=query.exclude_article_types.all()).\
            exclude(distribution__in=query.exclude_distributions.all()).\
            values('date').\
            annotate(count=Sum('count'))
        date2normalize = defaultdict(int)
        for v in values:
            y = v['date'].year
            m = v['date'].month if resolution != 'year' else 1
            d = v['date'].day if resolution == 'day' else 1
            date2normalize[datetime(y, m, d)] += v['count']

    # Calculate relative frequencies
    result = dict()
    for d, docs in date2doc.items():
        value = len(docs)
        if normalize:
            value = round(value / float(date2normalize[d]), 10)
        result[d.isoformat()] = (value, len(docs), docs)

    # Return the result
    return json_response_message('SUCCESS', 'Timeline retrieved', {'result': result})


@csrf_exempt
@login_required
def add_stopword(request):
    """
    Adds a Stopword.
    """
    query_id = request.POST.get('query_id')
    word = request.POST.get('stopword')

    # Retrieve the Query, set to None if not found
    q = None
    try:
        q = Query.objects.get(pk=query_id)
    except Query.DoesNotExist:
        pass


    StopWord.objects.create(user=request.user, query=q, word=word)