class URLField(models.TextField):
    description = _("URL")

    def formfield(self, **kwargs):
        defaults = {
            'form_class': forms.URLField,
            'widget': forms.TextInput,
        }
        defaults.update(kwargs)
        return super().formfield(**defaults) def ensure_subscribed(topic_url, hub_url):
    """Makes sure the PubSubHubbub subscription is verified"""
    if settings.TESTS:
        if str(type(requests.post)) != "<class 'unittest.mock.MagicMock'>":
            raise ValueError("Not Mocked")

    if hub_url is None:
        return

    log = logger.bind(topic_url=topic_url, hub_url=hub_url)

    call, args = None, ()
    try:
        s = Subscription.objects.get(topic=topic_url, hub=hub_url)
    except Subscription.DoesNotExist:
        log.info("subscribing")
        call = Subscription.objects.subscribe
        args = topic_url, hub_url
    else:
        if (
            not s.verified or
            s.lease_expiration < timezone.now() + timedelta(days=1)
        ):
            log.info("renewing subscription", subscription=s.pk)
            call = s.subscribe
    if call is not None:
        try:
            call(*args)
        except SubscriptionError as e:
            log.info("subscription error", exc_info=e)
        except MissingSchema:
            pass def store_entries(feed_url, entries):
    from .models import Entry, Feed

    feeds = Feed.objects.select_related('user').filter(
        url=feed_url, user__is_suspended=False).values('pk', 'user_id',
                                                       'category_id',
                                                       'user__ttl')

    guids = set([entry['guid'] for entry in entries])

    es_query = [{'or': [{'term': {'feed': feed['pk']}} for feed in feeds]}]

    # When we have dates, filter the query to avoid returning the whole dataset
    date_generated = any([e.pop('date_generated') for e in entries])
    if not date_generated:
        earliest = min([entry['date'] for entry in entries])
        limit = earliest - timedelta(days=1)
        es_query.append({'range': {'timestamp': {'gt': limit}}})

    filter_by_title = len(guids) == 1 and len(entries) > 1
    if filter_by_title:
        # All items have the same guid. Query by title instead.
        titles = set([entry['title'] for entry in entries])
        es_query.append({'or': [{'term': {'raw_title': t}} for t in titles]})
    else:
        es_query.append({'or': [{'term': {'guid': g}} for g in guids]})

    existing = None

    indices = []
    for feed in feeds:
        indices.append(es.user_alias(feed['user_id']))

    if indices:
        es.wait_for_yellow()
        # Make sure guid and raw_title are not analyzed before querying
        # anything. Otherwise existing entries are never matched and things
        # keep being inserted.
        mappings = es.client.indices.get_field_mapping(index=",".join(indices),
                                                       doc_type='entries',
                                                       field='guid,raw_title')
        for mapping in mappings.values():
            mapping = mapping['mappings']['entries']
            for f in ['raw_title', 'guid']:
                assert mapping[f]['mapping'][f]['index'] == 'not_analyzed'
        existing_es = es.client.search(
            index=",".join(indices),
            doc_type='entries',
            body={
                'aggs': {
                    'existing': {
                        'filter': {'and': es_query},
                        'aggs': {
                            'feeds': {
                                'terms': {'field': 'feed', 'size': 0},
                                'aggs': {
                                    'guids': {'terms': {'field': 'guid',
                                                        'size': 0}},
                                    'titles': {'terms': {'field': 'raw_title',
                                                         'size': 0}},
                                },
                            },
                        },
                    },
                },
            },
        )
        existing_es = existing_es[
            'aggregations']['existing']['feeds']['buckets']
    else:
        existing_es = []

    existing_guids = defaultdict(set)
    existing_titles = defaultdict(set)
    if existing is not None:
        for entry in existing:
            existing_guids[entry['feed_id']].add(entry['guid'])
            if filter_by_title:
                existing_titles[entry['feed_id']].add(entry['title'])

    existing_es_guids = defaultdict(set)
    existing_es_titles = defaultdict(set)
    for bucket in existing_es:
        for sub in bucket['guids']['buckets']:
            existing_es_guids[bucket['key']].add(sub['key'])
        if filter_by_title:
            for sub in bucket['titles']['buckets']:
                existing_es_titles[bucket['key']].add(sub['key'])

    ops = []
    refresh_updates = defaultdict(list)
    for feed in feeds:
        seen_guids = set()
        seen_titles = set()
        for entry in entries:
            if (
                not filter_by_title and
                entry['guid'] in existing_es_guids[feed['pk']]
            ):
                continue
            if (
                filter_by_title and
                entry['title'] in existing_es_titles[feed['pk']]
            ):
                continue
            if (
                feed['user__ttl'] and
                should_skip(entry['date'], feed['user__ttl'])
            ):
                continue

            if filter_by_title and entry['title'] in seen_titles:
                continue
            seen_titles.add(entry['title'])

            if not filter_by_title and entry['guid'] in seen_guids:
                continue
            seen_guids.add(entry['guid'])

            data = Entry(**entry).serialize()
            data['category'] = feed['category_id']
            data['feed'] = feed['pk']
            data['_id'] = es.next_id()
            data['id'] = data['_id']
            data['_type'] = 'entries'
            data['user'] = feed['user_id']
            data['_index'] = settings.ES_INDEX
            ops.append(data)
            refresh_updates[feed['user_id']].append(entry['date'])

    if ops:
        es.bulk(ops, raise_on_error=True)

        if settings.TESTS:
            # Indices are refreshed asynchronously. Refresh immediately
            # during tests.
            indices = ",".join(set([doc['_index'] for doc in ops]))
            es.client.indices.refresh(indices)

    redis = get_redis_connection()
    for user_id, dates in refresh_updates.items():
        user = User(pk=user_id)
        new_score = float(max(dates).strftime('%s'))
        current_score = redis.zscore(user.last_update_key, feed_url) or 0
        if new_score > current_score:
            redis.zadd(user.last_update_key, feed_url, new_score) def update_favicon(feed_url, force_update=False):
    from .models import Favicon
    Favicon.objects.update_favicon(feed_url, force_update=force_update) def update_feed(url, etag=None, modified=None, subscribers=1,
                request_timeout=10, backoff_factor=1, error=None, link=None,
                title=None, hub=None):
    from .models import UniqueFeed
    try:
        UniqueFeed.objects.update_feed(
            url, etag=etag, last_modified=modified, subscribers=subscribers,
            backoff_factor=backoff_factor, previous_error=error, link=link,
            title=title, hub=hub)
    except JobTimeoutException:
        backoff_factor = min(UniqueFeed.MAX_BACKOFF,
                             backoff_factor + 1)
        logger.info("job timed out, backing off",
                    url=url, backoff_factor=backoff_factor)
        schedule_job(url, schedule_in=UniqueFeed.delay(backoff_factor),
                     backoff_factor=backoff_factor,
                     connection=get_redis_connection())
    except BaseException as e:
        logger.info("fatal job exception", url=url, exc_info=e)
        raise def epoch_to_utc(value):
    """Converts epoch (in seconds) values to a timezone-aware datetime."""
    return timezone.make_aware(
        datetime.datetime.fromtimestamp(value), timezone.utc) FAVICON_FETCHER = USER_AGENT % 'favicon fetcher' def get_job(name):
    redis = get_redis_connection()
    key = job_key(name)
    if not redis.exists(key):
        raise JobNotFound
    return job_details(name, connection=redis) def is_feed(parsed):
    return hasattr(parsed.feed, 'title') class JobNotFound(Exception):
    pass def remove_utm_tags(guid):
    parts = list(urlsplit(guid))
    qs = parse_qs(parts[3])  # [3] is query component
    filtered = sorted([(k, v) for k, v in qs.items()
                       if not k.startswith('utm_')])
    parts[3] = urlencode(filtered, doseq=True)
    return urlunsplit(parts) def resolve_url(url):
    if settings.TESTS:
        if str(type(requests.head)) != "<class 'unittest.mock.MagicMock'>":
            raise ValueError("Not mocked")
    cache_key = 'resolve_url:{0}'.format(url)
    resolved = cache.get(cache_key)
    if resolved is None:
        resolved = url
        try:
            response = requests.head(url, headers={'User-Agent': LINK_CHECKER},
                                     allow_redirects=True, timeout=3)
        except (ConnectionError, ConnectTimeout, InvalidSchema, MissingSchema,
                ReadTimeout, TooManyRedirects, InvalidURL, LocationValueError):
            pass
        else:
            if response.status_code == 200:
                resolved = response.url
        cache.set(cache_key, resolved, 3600 * 24 * 5)
    return resolved USER_AGENT = (
    'FeedHQ/%s (https://github.com/feedhq/feedhq; %%s; https://github.com/'
    'feedhq/feedhq/wiki/fetcher; like FeedFetcher-Google)'
) % __version__ class OverwritingStorage(FileSystemStorage):
    """
    File storage that allows overwriting of stored files.
    """

    def get_available_name(self, name, max_length=None):
        return name

    def _save(self, name, content):
        """
        Lifted partially from django/core/files/storage.py
        """
        full_path = self.path(name)

        directory = os.path.dirname(full_path)
        if not os.path.exists(directory):
            try:
                os.makedirs(directory)
            except OSError as e:
                if e.errno != errno.EEXIST:
                    raise
        if not os.path.isdir(directory):
            raise IOError("%s exists and is not a directory." % directory)

        # This file has a file path that we can move.
        if hasattr(content, 'temporary_file_path'):
            temp_data_location = content.temporary_file_path()
        else:
            tmp_prefix = "tmp_%s" % (get_valid_filename(name), )
            temp_data_location = tempfile.mktemp(prefix=tmp_prefix,
                                                 dir=self.location)
            try:
                # This is a normal uploadedfile that we can stream.
                # This fun binary flag incantation makes os.open throw an
                # OSError if the file already exists before we open it.
                fd = os.open(temp_data_location,
                             os.O_WRONLY | os.O_CREAT |
                             os.O_EXCL | getattr(os, 'O_BINARY', 0))
                locks.lock(fd, locks.LOCK_EX)
                for chunk in content.chunks():
                    os.write(fd, chunk)
                locks.unlock(fd)
                os.close(fd)
            except Exception:
                if os.path.exists(temp_data_location):
                    os.remove(temp_data_location)
                raise

        file_move_safe(temp_data_location, full_path, allow_overwrite=True)
        content.close()

        if settings.FILE_UPLOAD_PERMISSIONS is not None:
            os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
        return name def enqueue(function, args=None, kwargs=None, timeout=None, queue='default'):
    async = not settings.RQ_EAGER

    if args is None:
        args = ()
    if kwargs is None:
        kwargs = {}

    conn = get_redis_connection()
    queue = rq.Queue(queue, connection=conn, async=async)
    return queue.enqueue_call(func=function, args=tuple(args), kwargs=kwargs,
                              timeout=timeout) def get_redis_connection():
    """
    Helper used for obtain a raw redis client.
    """
    from redis_cache.cache import pool
    client = redis.Redis(**settings.REDIS)
    client.connection_pool = pool.get_connection_pool(
        client,
        parser_class=redis.connection.HiredisParser,
        connection_pool_class=redis.ConnectionPool,
        connection_pool_class_kwargs={},
        **settings.REDIS)
    return client
import base64
import datetime
import hashlib
import json
import random
import socket
import struct
import time
import bleach
import feedparser
import lxml.html
import magic
import pytz
import requests
import six
import structlog
from django.conf import settings
from django.core.cache import cache
from django.core.files.base import ContentFile
from django.core.urlresolvers import reverse, reverse_lazy
from django.db import models
from django.template.defaultfilters import slugify
from django.utils import timezone
from django.utils.dateparse import parse_datetime
from django.utils.encoding import force_bytes
from django.utils.html import format_html
from django.utils.text import unescape_entities
from django.utils.translation import string_concat, ugettext_lazy as _
from django_push.subscriber.signals import updated
from lxml.etree import ParserError
from model_utils import FieldTracker
from rache import delete_job, schedule_job
from requests.exceptions import ConnectionError
from requests.packages.urllib3.exceptions import (DecodeError,
                                                  LocationParseError)
from requests_oauthlib import OAuth1
from six.moves.http_client import IncompleteRead
from six.moves.urllib import parse as urlparse
from urlobject import URLObject
from .fields import URLField
from .tasks import (ensure_subscribed, store_entries, update_favicon,
                    update_feed)
from .utils import (epoch_to_utc, FAVICON_FETCHER, get_job, is_feed,
                    JobNotFound, remove_utm_tags, resolve_url, USER_AGENT)
from .. import es
from ..storage import OverwritingStorage
from ..tasks import enqueue
from ..utils import get_redis_connection
            'starred': self.starred,
            'broadcast': self.broadcast,
            'tags': [],
        }
        if self.user_id:
            data['user'] = self.user_id
        if self.feed_id:
            data['feed'] = self.feed_id
        if self.feed and self.feed.category_id:
            data['category'] = self.feed.category_id
        return data

    def index(self):
        name = es.user_alias(self.user_id)
        data = es.client.index(name, doc_type='entries', body=self.serialize(),
                               id=self.pk, params={'refresh': True})
        data['_source'] = self.serialize()
        data['_id'] = int(data['_id'])
        return EsEntry(data)


class EsEntry(BaseEntry):
    __slots__ = (
        'feed', 'category', 'guid', 'tags', 'read', 'timestamp', 'author',
        'broadcast', 'date', 'link', 'title', 'starred',
        'read_later_url', 'pk', 'subtitle', '_content', 'user',
    )

    def __repr__(self):
        return '<{0}: {1}>'.format(self.__class__.__name__, self.pk)

    def __init__(self, entry):
        if isinstance(entry['_id'], six.string_types):
            assert entry['_id'].isdigit()
            entry['_id'] = int(entry['_id'])

        self.pk = entry['_id']
        self.subtitle = entry['_source'].pop('content', None)
        for key, value in entry['_source'].items():
            setattr(self, key, value)
        if hasattr(self, 'timestamp'):
            if isinstance(self.timestamp, six.string_types):
                date = parse_datetime(self.timestamp)
            else:
                date = self.timestamp
            self.date = date

    def __str__(self):
        return u'%s' % self.title

    def update(self, refresh=False, **attrs):
        for key, value in attrs.items():
            setattr(self, key, value)
        es.client.update(es.user_alias(self.user.pk), doc_type='entries',
                         id=self.pk, body={'doc': attrs},
                         params={'refresh': refresh})

    def delete(self):
        es.client.delete(es.user_alias(self.user.pk), doc_type='entries',
                         id=self.pk)


def pubsubhubbub_update(notification, request, links, **kwargs):
    url = None
    # Try the header links first
    if links is not None:
        for link in links:
            if link['rel'] == 'self':
                url = link['url']
                break

    notification = feedparser.parse(notification)

    # Fallback to feed links if no header link found
    if url is None:
        for link in notification.feed.get('links', []):
            if link['rel'] == 'self':
                url = link['href']
                break

    if url is None:
        return

    entries = list(filter(
        None,
        [UniqueFeedManager.entry_data(
            entry, notification) for entry in notification.entries]
    ))
    if len(entries):
        enqueue(store_entries, args=[url, entries], queue='store')


updated.connect(pubsubhubbub_update)


class FaviconManager(models.Manager):
    def update_favicon(self, url, force_update=False):
        if not url:
            return
        parsed = list(urlparse.urlparse(url))
        if not parsed[0].startswith('http'):
            return
        favicon, created = self.get_or_create(url=url)
        feeds = Feed.objects.filter(url=url, favicon='')
        if (not created and not force_update) and favicon.favicon:
            # Still, add to existing
            favicon_urls = list(self.filter(url=url).exclude(
                favicon='').values_list('favicon', flat=True))
            if not favicon_urls:
                return favicon

            if not feeds.exists():
                return

            feeds.update(favicon=favicon_urls[0])
            return favicon

        ua = {'User-Agent': FAVICON_FETCHER}

        try:

            link = get_job(url).get('link')