class CollectionSet(models.Model):
    collection_set_id = models.CharField(max_length=32, unique=True, default=default_uuid)
    group = models.ForeignKey(Group, related_name='collection_sets', on_delete=models.CASCADE)
    name = models.CharField(max_length=255, blank=False,
                            verbose_name='Collection set name')
    description = models.TextField(blank=True)
    is_visible = models.BooleanField(default=True)
    date_added = models.DateTimeField(default=timezone.now)
    date_updated = models.DateTimeField(auto_now=True)
    history = HistoricalRecords(bases=[CollectionSetHistoryModel])
    history_note = models.TextField(blank=True)

    objects = CollectionSetManager()

    class Meta:
        diff_fields = ("group", "name", "description")

    def __str__(self):
        return '<Collection Set %s "%s">' % (self.id, self.name)

    def save(self, *args, **kw):
        return history_save(self, *args, **kw)

    def natural_key(self):
        return self.collection_set_id,

    def stats(self):
        """
        Returns a dict of items to count.
        """
        return _item_counts_to_dict(
            HarvestStat.objects.filter(harvest__collection__collection_set=self).values("item").annotate(
                count=models.Sum("count")))

    def stats_items(self):
        """
        Returns a list of items type that have been harvested for this collection set.
        """
        return list(
            HarvestStat.objects.filter(harvest__collection__collection_set=self).values_list("item",
                                                                                             flat=True).distinct())

    def item_stats(self, item, days=7, end_date=None):
        """
        Gets count of items harvested by date.

        If there are no items for a day, a date, count (of 0) pair is still returned.

        :param item: name of the item to get count for, e.g., tweet.
        :param days: backwards from end_datetime, the number of days to retrieve.
        :param end_date: the date to start backfrom from. Default is today.
        :return: List of date, count pairs.
        """
        if end_date is None:
            end_date = datetime.date.today()

        if days:
            start_date = end_date - datetime.timedelta(days=days - 1)
            date_counts = HarvestStat.objects.filter(harvest__collection__collection_set=self, item=item,
                                                     harvest_date__gte=start_date).order_by(
                "harvest_date").values("harvest_date").annotate(count=models.Sum("count"))
        else:
            date_counts = HarvestStat.objects.filter(harvest__collection__collection_set=self, item=item).order_by(
                "harvest_date").values("harvest_date").annotate(count=models.Sum("count"))
            if len(date_counts) > 0:
                days = (end_date - date_counts[0]["harvest_date"]).days + 1
            else:
                days = 1
            start_date = end_date - datetime.timedelta(days=days - 1)

        date_counts_dict = {}
        for date_count in date_counts:
            date_counts_dict[date_count["harvest_date"]] = date_count["count"]

        stats = []
        for i in range(days):
            date = start_date + datetime.timedelta(days=i)
            stats.append((date, date_counts_dict.get(date, 0)))
        return stats

    def warcs_count(self):
        """
        Returns total number of WARC files harvested for this collection set.
        """
        return Harvest.objects.filter(collection__collection_set=self).aggregate(count=models.Sum("warcs_count"))[
                   "count"] or 0

    def warcs_bytes(self):
        """
        Returns total number of WARC bytes harvested for this collection set.
        """
        return Harvest.objects.filter(collection__collection_set=self).aggregate(total=models.Sum("warcs_bytes"))[
            "total"]

    def get_collection_set(self):
        return self

    def is_active(self):
        """
        Returns True is has any active collections
        """
        return Collection.objects.filter(collection_set=self).filter(is_active=True).exists() class User(AbstractUser):
    DAILY = "daily"
    WEEKLY = "weekly"
    MONTHLY = "monthly"
    NONE = "none"
    EMAIL_FREQUENCY_CHOICES = [
        (DAILY, 'Daily'),
        (WEEKLY, 'Weekly'),
        (MONTHLY, 'Monthly'),
        (NONE, "None")
    ]
    local_id = models.CharField(max_length=255, blank=True, default='',
                                help_text='Local identifier')
    email_frequency = models.CharField(max_length=10, choices=EMAIL_FREQUENCY_CHOICES, default=DAILY)
    harvest_notifications = models.BooleanField(default=True)

    def get_user(self):
        return self class Credential(models.Model):
    UPDATE_VIEW = "updateView"
    CREATE_VIEW = "createView"
    TWITTER = "twitter"
    FLICKR = "flickr"
    WEIBO = "weibo"
    TUMBLR = "tumblr"
    PLATFORM_CHOICES = [
        (TWITTER, 'Twitter'),
        (FLICKR, 'Flickr'),
        (WEIBO, 'Weibo'),
        (TUMBLR, "Tumblr")
    ]
    credential_id = models.CharField(max_length=32, unique=True, default=default_uuid)
    name = models.CharField(max_length=255, verbose_name='Credential name')
    user = models.ForeignKey(User, related_name='credentials', on_delete=models.CASCADE)
    platform = models.CharField(max_length=255, help_text='Platform name', choices=PLATFORM_CHOICES)
    token = models.TextField(blank=True, unique=True)
    is_active = models.BooleanField(default=True)
    date_added = models.DateTimeField(default=timezone.now)
    date_updated = models.DateTimeField(auto_now=True)
    history_note = models.TextField(blank=True)
    history = HistoricalRecords(bases=[CredentialHistoryModel])

    objects = CredentialManager()

    class Meta:
        diff_fields = ("name", "platform", "token", "is_active")

    def __str__(self):
        return '<Credential %s "%s">' % (self.id, self.platform)

    def natural_key(self):
        return self.credential_id,

    def save(self, *args, **kw):
        return history_save(self, *args, **kw)

    def get_user(self):
        return self.user class Seed(models.Model):
    UPDATE_VIEW = "updateView"
    CREATE_VIEW = "createView"
    collection = models.ForeignKey(Collection, related_name='seeds', on_delete=models.CASCADE)
    seed_id = models.CharField(max_length=32, unique=True, default=default_uuid)
    token = models.TextField(blank=True)
    uid = models.TextField(blank=True)
    is_active = models.BooleanField(default=True)
    is_valid = models.BooleanField(default=True)
    date_added = models.DateTimeField(default=timezone.now)
    date_updated = models.DateTimeField(auto_now=True)
    history = HistoricalRecords(bases=[SeedHistoryModel])
    history_note = models.TextField(blank=True)

    objects = SeedManager()

    class Meta:
        diff_fields = ("token", "uid", "is_active")
        unique_together = ("collection", "uid", "token")

    def social_url(self):
        twitter_user_url = "https://twitter.com/"
        flickr_user_url = 'https://www.flickr.com/photos/'
        tumblr_blog_url = '.tumblr.com'
        weibo_topic_url = 'http://huati.weibo.com/k/'
        if self.collection.harvest_type == Collection.TWITTER_USER_TIMELINE and self.token:
            return twitter_user_url + self.token
        elif self.collection.harvest_type == Collection.FLICKR_USER and self.token:
            return flickr_user_url + self.token
        elif self.collection.harvest_type == Collection.TUMBLR_BLOG_POSTS and self.uid:
            return 'https://' + self.uid + tumblr_blog_url
        elif self.collection.harvest_type == Collection.WEIBO_SEARCH and self.token:
            return weibo_topic_url + self.token
        return None

    def __str__(self):
        return '<Seed %s "%s">' % (self.id, self.token)

    def save(self, *args, **kw):
        return history_save(self, *args, **kw)

    def natural_key(self):
        return self.seed_id,

    def label(self):
        labels = []
        if self.token:
            try:
                j = json.loads(self.token)
                for key, value in j.items():
                    labels.append(u"{}: {}".format(key.title(), value))
            except (AttributeError, ValueError):
                labels.append(u"{}".format(self.token))
        if self.uid:
            labels.append(u"{}".format(self.uid))
        return u", ".join(labels)

    def get_collection_set(self):
        return self.collection.collection_set

    def get_collection(self):
        return self.collection class Collection(models.Model):
    TWITTER_SEARCH = 'twitter_search'
    TWITTER_FILTER = "twitter_filter"
    TWITTER_USER_TIMELINE = 'twitter_user_timeline'
    TWITTER_SAMPLE = 'twitter_sample'
    FLICKR_USER = 'flickr_user'
    WEIBO_TIMELINE = 'weibo_timeline'
    WEIBO_SEARCH = 'weibo_search'
    TUMBLR_BLOG_POSTS = 'tumblr_blog_posts'
    SCHEDULE_CHOICES = [
        (1, 'One time harvest'),
        (30, 'Every 30 minutes'),
        (60, 'Every hour'),
        (60 * 4, 'Every 4 hours'),
        (60 * 12, 'Every 12 hours'),
        (60 * 24, 'Every day'),
        (60 * 24 * 7, 'Every week'),
        (60 * 24 * 7 * 4, 'Every 4 weeks')
    ]
    HARVEST_CHOICES = [
        (TWITTER_USER_TIMELINE, 'Twitter user timeline'),
        (TWITTER_SEARCH, 'Twitter search'),
        (TWITTER_FILTER, 'Twitter filter'),
        (TWITTER_SAMPLE, 'Twitter sample'),
        (TUMBLR_BLOG_POSTS, 'Tumblr blog posts'),
        (FLICKR_USER, 'Flickr user'),
        (WEIBO_TIMELINE, 'Weibo timeline')
    ]
    HARVEST_DESCRIPTION = {
        TWITTER_SEARCH: 'Recent tweets matching a query',
        TWITTER_FILTER: 'Tweets in real time matching filter criteria',
        TWITTER_USER_TIMELINE: 'Tweets from specific accounts',
        TWITTER_SAMPLE: 'A subset of all tweets in real time',
        FLICKR_USER: 'Posts and photos from specific accounts',
        WEIBO_TIMELINE: "Posts from a user and the user's friends",
        TUMBLR_BLOG_POSTS: 'Blog posts from specific blogs'
    }
    HARVEST_FIELDS = {
        TWITTER_SEARCH: {"link": None, "token": "Search query", "uid": None},
        TWITTER_FILTER: {"link": None, "token": "Filter criteria", "uid": None},
        TWITTER_USER_TIMELINE: {"link": "Link", "token": "Twitter accounts", "uid": "User ID"},
        TWITTER_SAMPLE: None,
        FLICKR_USER: {"link": "Link", "token": "Flickr users", "uid": "NSID"},
        WEIBO_TIMELINE: None,
        TUMBLR_BLOG_POSTS: {"link": "Link", "token": None, "uid": "Blog name"}
    }
    REQUIRED_SEED_COUNTS = {
        TWITTER_FILTER: 1,
        TWITTER_SEARCH: 1,
        WEIBO_SEARCH: 1,
        TWITTER_SAMPLE: 0,
        WEIBO_TIMELINE: 0
    }
    HARVEST_TYPES_TO_PLATFORM = {
        TWITTER_SEARCH: Credential.TWITTER,
        TWITTER_FILTER: Credential.TWITTER,
        TWITTER_USER_TIMELINE: Credential.TWITTER,
        TWITTER_SAMPLE: Credential.TWITTER,
        FLICKR_USER: Credential.FLICKR,
        WEIBO_TIMELINE: Credential.WEIBO,
        WEIBO_SEARCH: Credential.WEIBO,
        TUMBLR_BLOG_POSTS: Credential.TUMBLR
    }
    STREAMING_HARVEST_TYPES = (TWITTER_SAMPLE, TWITTER_FILTER)
    RATE_LIMITED_HARVEST_TYPES = (TWITTER_USER_TIMELINE, TWITTER_SEARCH)
    DEFAULT_VISIBILITY = 'default'
    LOCAL_VISIBILITY = 'local'
    VISIBILITY_CHOICES = [
        (DEFAULT_VISIBILITY, 'Group only'),
        # That is, with other SFM users.
        (LOCAL_VISIBILITY, 'All other users')
        # This can be expanded with additional options
    ]
    collection_id = models.CharField(max_length=32, unique=True, default=default_uuid)
    collection_set = models.ForeignKey(CollectionSet, related_name='collections', on_delete=models.CASCADE)
    credential = models.ForeignKey(Credential, related_name='collections', on_delete=models.CASCADE)
    harvest_type = models.CharField(max_length=255, choices=HARVEST_CHOICES)
    name = models.CharField(max_length=255, verbose_name='Collection name')
    description = models.TextField(blank=True)
    link = models.CharField(max_length=512, blank=True, verbose_name='Public link')
    is_on = models.BooleanField(default=False)
    is_active = models.BooleanField(default=True)
    visibility = models.CharField(max_length=255, choices=VISIBILITY_CHOICES, default=DEFAULT_VISIBILITY,
                                  help_text='Who else can view and export from this collection. Select "All other '
                                            'users" to share with all Social Feed Manager users.')
    schedule_minutes = models.PositiveIntegerField(default=60 * 24 * 7, choices=SCHEDULE_CHOICES,
                                                   verbose_name="schedule", null=True)
    harvest_options = models.TextField(blank=True)
    date_added = models.DateTimeField(default=timezone.now)
    date_updated = models.DateTimeField(auto_now=True)
    end_date = models.DateTimeField(blank=True,
                                    null=True,
                                    help_text="If blank, will continue until stopped.")
    history = HistoricalRecords(bases=[CollectionHistoryModel])
    history_note = models.TextField(blank=True)

    objects = CollectionManager()

    class Meta:
        diff_fields = (
            "collection_set", "credential", "harvest_type", "name", "description", "is_on", "is_active", "visibility",
            "schedule_minutes", "harvest_options", "end_date", "link")

    def __str__(self):
        return '<Collection %s "%s">' % (self.id, self.name)

    def natural_key(self):
        return self.collection_id,

    def required_seed_count(self):
        """
        Returns the number of seeds that are required for this harvest type.

        If None, then 1 or more is required.
        """
        return self.REQUIRED_SEED_COUNTS.get(str(self.harvest_type))

    def seed_count(self):
        return self.seeds.filter(collection=self).count()

    def deleted_seed_count(self):
        deleted_seed_count = self.seeds.filter(collection=self).count() - self.seeds.filter(is_active=True).count()
        return deleted_seed_count

    def active_seed_count(self):
        """
        Returns the number of active seeds.
        """
        return self.seeds.filter(is_active=True).count()

    def last_harvest(self, include_skipped=False, include_web_harvests=False):
        """
        Returns the most recent harvest or None if no harvests.
        """
        harvests = self.harvests
        if not include_web_harvests:
            harvests = harvests.exclude(harvest_type="web")
        if not include_skipped:
            harvests = harvests.exclude(status=Harvest.SKIPPED)

        return harvests.order_by("-date_requested").first()

    def is_streaming(self):
        """
        Returns True if a streaming harvest type.
        """
        return self.harvest_type in Collection.STREAMING_HARVEST_TYPES

    def stats(self):
        """
        Returns a dict of items to count.
        """
        return _item_counts_to_dict(
            HarvestStat.objects.filter(harvest__collection=self).values("item").annotate(count=models.Sum("count")))

    def warcs_count(self):
        """
        Returns total number of WARC files harvested for this collection.
        """
        return Harvest.objects.filter(collection=self).aggregate(count=models.Sum("warcs_count"))["count"] or 0

    def warcs_bytes(self):
        """
        Returns total number of WARC bytes harvested for this collection.
        """
        return Harvest.objects.filter(collection=self).aggregate(total=models.Sum("warcs_bytes"))["total"]

    def save(self, *args, **kw):
        return history_save(self, *args, **kw)

    def get_collection_set(self):
        return self.collection_set

    def get_collection(self):
        return self class Export(models.Model):
    NOT_REQUESTED = "not requested"
    REQUESTED = "requested"
    SUCCESS = "completed success"
    FAILURE = "completed failure"
    RUNNING = "running"
    STATUS_CHOICES = (
        (NOT_REQUESTED, "Not requested"),
        (REQUESTED, "Requested"),
        (RUNNING, "Running"),
        (SUCCESS, "Success"),
        (FAILURE, "Failure")
    )
    FORMAT_CHOICES = (
        ("xlsx", "Excel (XLSX)"),
        ("csv", "Comma separated values (CSV)"),
        ("tsv", "Tab separated values (TSV)"),
        ("json_full", "Full JSON"),
        ("json", "JSON of limited fields"),
        ("dehydrate", "Text file of identifiers (dehydrate)")
    )
    SEGMENT_CHOICES = [
        (100000, "100,000"),
        (250000, "250,000"),
        (500000, "500,000"),
        (1000000, "1,000,000"),
        (None, "Single file"),
    ]
    user = models.ForeignKey(User, related_name='exports', on_delete=models.CASCADE)
    collection = models.ForeignKey(Collection, blank=True, null=True, on_delete=models.CASCADE)
    seeds = models.ManyToManyField(Seed, blank=True)
    export_id = models.CharField(max_length=32, unique=True, default=default_uuid)
    export_type = models.CharField(max_length=255)
    export_format = models.CharField(max_length=10, choices=FORMAT_CHOICES, default="xlsx")
    export_segment_size = models.BigIntegerField(choices=SEGMENT_CHOICES, default=250000,
                                                 null=True, blank=True)
    status = models.CharField(max_length=20, choices=STATUS_CHOICES, default=NOT_REQUESTED)
    path = models.TextField(blank=True)
    date_requested = models.DateTimeField(blank=True, null=True)
    date_started = models.DateTimeField(blank=True, null=True)
    date_ended = models.DateTimeField(blank=True, null=True)
    date_updated = models.DateTimeField(auto_now=True)
    dedupe = models.BooleanField(blank=False, default=False)
    item_date_start = models.DateTimeField(blank=True, null=True)
    item_date_end = models.DateTimeField(blank=True, null=True)
    harvest_date_start = models.DateTimeField(blank=True, null=True)
    harvest_date_end = models.DateTimeField(blank=True, null=True)
    infos = JSONField(blank=True, default=[])
    warnings = JSONField(blank=True, default=[])
    errors = JSONField(blank=True, default=[])
    # These identify who is doing the harvest
    service = models.CharField(max_length=255, null=True)
    host = models.CharField(max_length=255, null=True)
    # Since a host may have multiple instances of a harvester, this identifies which. Might be a PID.
    instance = models.CharField(max_length=255, null=True)

    def save(self, *args, **kwargs):
        self.path = "{}/export/{}".format(settings.SFM_EXPORT_DATA_DIR, self.export_id)
        super(Export, self).save(*args, **kwargs)

    def __str__(self):
        return '<Export %s "%s">' % (self.id, self.export_id)

    def get_collection_set(self):
        # Export can have a collection or seeds
        if self.collection:
            return self.collection.collection_set
        seed = self.seeds.first()
        if seed:
            return seed.get_collection_set()
        return None

    def get_collection(self):
        # Export can have a collection or seeds
        if self.collection:
            return self.collection
        seed = self.seeds.first()
        if seed:
            return seed.get_collection()
        return None class CollectionSetListView(LoginRequiredMixin, ListView):
    model = CollectionSet
    template_name = 'ui/collection_set_list.html'
    paginate_by = 20
    allow_empty = True
    paginate_orphans = 0

    def get_context_data(self, **kwargs):
        context = super(CollectionSetListView, self).get_context_data(**kwargs)
        # collection set identity, collection set type Name, collection set data
        collection_sets_lists = []
        active_collection_sets, inactive_collection_sets = split_collection_sets(
            CollectionSet.objects.filter(group__in=self.request.user.groups.all()).annotate(
                num_collections=Count('collections')).order_by('name'))
        collection_sets_lists.append(('acs', "Active", active_collection_sets))
        collection_sets_lists.append(('iacs', "Inactive", inactive_collection_sets))
        collection_sets_lists.append(('scs', "Shared", list(
            CollectionSet.objects.filter(collections__visibility=Collection.LOCAL_VISIBILITY).annotate(
                num_collections=Count('collections')).order_by('name'))))
        if self.request.user.is_superuser or self.request.user.is_staff:
            other_active_collection_sets, other_inactive_collection_sets = split_collection_sets(
                CollectionSet.objects.exclude(group__in=self.request.user.groups.all()).annotate(
                    num_collections=Count('collections')).order_by('name'))
            collection_sets_lists.append(('oacs', "Other Active", other_active_collection_sets))
            collection_sets_lists.append(('oics', "Other Inactive", other_inactive_collection_sets))
        context['collection_sets_lists'] = collection_sets_lists
        return context class CollectionSetDetailView(LoginRequiredMixin, CollectionSetOrCollectionVisibilityOrSuperuserOrStaffPermissionMixin,
                              DetailView):
    model = CollectionSet
    template_name = 'ui/collection_set_detail.html'
    context_object_name = 'collection_set'

    def get_context_data(self, **kwargs):
        context = super(CollectionSetDetailView, self).get_context_data(**kwargs)
        collections = []
        for collection in self.object.collections.all():
            if has_collection_set_based_permission(collection, self.request.user, allow_superuser=True,
                                                   allow_staff=True,
                                                   allow_collection_visibility=True):
                collections.append(collection)
        context['collection_list'] = collections
        context["diffs"], context["diffs_len"] = top_object_diffs(self.object)
        context["harvest_types"] = Collection.HARVEST_CHOICES
        context["harvest_description"] = Collection.HARVEST_DESCRIPTION
        context["item_id"] = self.object.id
        context["model_name"] = "collection_set"
        context["can_edit"] = has_collection_set_based_permission(self.object, self.request.user)
        return context class CollectionSetUpdateView(LoginRequiredMixin, CollectionSetOrSuperuserPermissionMixin, UpdateView):
    model = CollectionSet
    form_class = CollectionSetForm
    template_name = 'ui/collection_set_update.html'
    initial = {'history_note': ''}
    context_object_name = 'collection_set'

    def get_context_data(self, **kwargs):
        context = super(CollectionSetUpdateView, self).get_context_data(**kwargs)
        context['collection_list'] = Collection.objects.filter(
            collection_set=self.object.pk).annotate(num_seeds=Count('seeds')).order_by('name')
        return context

    def get_form_kwargs(self):
        kwargs = super(CollectionSetUpdateView, self).get_form_kwargs()
        kwargs['request'] = self.request
        return kwargs

    def get_success_url(self):
        return reverse("collection_set_detail", args=(self.object.pk,)) class CollectionCreateView(LoginRequiredMixin, CollectionSetOrSuperuserPermissionMixin, SuccessMessageMixin,
                           CreateView):
    model = Collection
    template_name = 'ui/collection_create.html'

    def get_initial(self):
        initial = super(CollectionCreateView, self).get_initial()
        initial["collection_set"] = CollectionSet.objects.get(pk=self.kwargs["collection_set_pk"])
        return initial

    def get_context_data(self, **kwargs):
        context = super(CollectionCreateView, self).get_context_data(**kwargs)
        context["collection_set"] = CollectionSet.objects.get(pk=self.kwargs["collection_set_pk"])
        harvest_type = self.kwargs["harvest_type"]
        context["harvest_type_name"] = _get_harvest_type_name(harvest_type)
        credentials = _get_credential_list(self.kwargs["collection_set_pk"], harvest_type)
        context["credentials"] = credentials
        context["credential_use_map"] = _get_credential_use_map(credentials, harvest_type)
        context["platform"] = Collection.HARVEST_TYPES_TO_PLATFORM[self.kwargs["harvest_type"]]
        return context

    def get_form_kwargs(self):
        kwargs = super(CollectionCreateView, self).get_form_kwargs()
        kwargs["coll"] = self.kwargs["collection_set_pk"]
        kwargs['credential_list'] = _get_credential_list(self.kwargs["collection_set_pk"], self.kwargs["harvest_type"])
        return kwargs

    def get_form_class(self):
        return getattr(forms, _get_collection_form_class(self.kwargs["harvest_type"]))

    def get_success_url(self):
        return reverse('collection_detail', args=(self.object.pk,))

    def get_success_message(self, cleaned_data):
        if self.object.required_seed_count() != 0:
            return "New collection added. You can now add seeds."
        return "New collection added." class SeedUpdateView(LoginRequiredMixin, CollectionSetOrSuperuserPermissionMixin, UpdateView):
    model = Seed
    template_name = 'ui/seed_update.html'
    initial = {'history_note': ''}

    def get_form_kwargs(self):
        kwargs = super(SeedUpdateView, self).get_form_kwargs()
        kwargs["collection"] = self.object.collection.pk
        kwargs["view_type"] = Seed.UPDATE_VIEW
        kwargs["entry"] = self.get_object()
        return kwargs

    def get_context_data(self, **kwargs):
        context = super(SeedUpdateView, self).get_context_data(**kwargs)
        context["collection_set"] = CollectionSet.objects.get(id=self.object.collection.collection_set.id)
        return context

    def get_form_class(self):
        return getattr(forms, _get_seed_form_class(self.object.collection.harvest_type))

    def get_success_url(self):
        return reverse("seed_detail", args=(self.object.pk,)) class SeedCreateView(LoginRequiredMixin, CollectionSetOrSuperuserPermissionMixin, SuccessMessageMixin, CreateView):
    model = Seed
    template_name = 'ui/seed_create.html'

    def get_initial(self):
        initial = super(SeedCreateView, self).get_initial()
        collection = Collection.objects.get(pk=self.kwargs["collection_pk"])
        initial["collection"] = collection
        initial["collection_set"] = collection.collection_set
        return initial

    def get_context_data(self, **kwargs):
        context = super(SeedCreateView, self).get_context_data(**kwargs)
        collection = Collection.objects.get(pk=self.kwargs["collection_pk"])
        context["collection"] = collection
        context["collection_set"] = context["collection"].collection_set
        context["harvest_type_name"] = _get_harvest_type_name(collection.harvest_type)
        return context

    def get_form_kwargs(self):
        kwargs = super(SeedCreateView, self).get_form_kwargs()
        kwargs["collection"] = self.kwargs["collection_pk"]
        kwargs["view_type"] = Seed.CREATE_VIEW
        return kwargs

    def get_form_class(self):
        return getattr(forms,
                       _get_seed_form_class(Collection.objects.get(pk=self.kwargs["collection_pk"]).harvest_type))

    def get_success_url(self):
        return reverse("collection_detail", args=(self.kwargs["collection_pk"],))

    def get_success_message(self, cleaned_data):
        if self.object.collection.required_seed_count() == 1:
            return "New seed added."
        return "New seed added." class SeedDetailView(LoginRequiredMixin, CollectionSetOrCollectionVisibilityOrSuperuserOrStaffPermissionMixin,
                     DetailView):
    model = Seed
    template_name = 'ui/seed_detail.html'

    def get_context_data(self, **kwargs):
        # Call the base implementation first to get a context
        context = super(SeedDetailView, self).get_context_data(**kwargs)
        context["diffs"], context["diffs_len"] = top_object_diffs(self.object)
        context["collection_set"] = CollectionSet.objects.get(id=self.object.collection.collection_set.id)
        context["item_id"] = self.object.id
        context["model_name"] = "seed"
        can_edit = has_collection_set_based_permission(self.object,
                                                       self.request.user) and self.object.collection.is_active
        if self.object.collection.is_streaming():
            can_edit = can_edit and not self.object.collection.is_on
        context["can_edit"] = can_edit
        return context class ExportDetailView(LoginRequiredMixin, CollectionSetOrCollectionVisibilityOrSuperuserOrStaffPermissionMixin,
                       DetailView):
    model = Export
    template_name = 'ui/export_detail.html'

    def get_context_data(self, **kwargs):
        context = super(ExportDetailView, self).get_context_data(**kwargs)
        seeds = list(self.object.seeds.all())
        collection = seeds[0].collection if seeds else self.object.collection
        context["collection_set"] = collection.collection_set
        context["collection"] = collection
        context["fileinfos"] = _get_fileinfos(self.object.path) if self.object.status == Export.SUCCESS else ()
        return context def export_file(request, pk, file_name):
    """
    Allows authorized user to export a file.

    Adapted from https://github.com/ASKBOT/django-directory
    """
    export = get_object_or_404(Export, pk=pk)
    if (request.user == export.user) or request.user.is_superuser:
        filepath = os.path.join(export.path, file_name)
        log.debug("Exporting %s", filepath)
        if os.path.exists(filepath):
            response = StreamingHttpResponse()
            response['Content-Disposition'] = 'attachment; filename=%s' % file_name
            response['Content-Type'] = 'application/octet-stream'
            file_obj = open(filepath, 'rb')
            response.streaming_content = _read_file_chunkwise(file_obj)
            return response
        else:
            raise Http404
    else:
        raise PermissionDenied class ChangeLogView(LoginRequiredMixin, ListView):
    context_object_name = "history_list"
    template_name = "ui/change_log.html"
    paginate_by = 15

    def get_queryset(self):
        item_id = self.kwargs["item_id"]
        model_name = self.kwargs["model"].replace("_", "")
        ModelName = apps.get_model(app_label="ui", model_name=model_name)
        item = ModelName.objects.get(pk=item_id)
        # Check permissions
        check_collection_set_based_permission(item, self.request.user, allow_staff=True)
        return item.history.all()

    def get_context_data(self, **kwargs):
        context = super(ChangeLogView, self).get_context_data(**kwargs)

        item_id = self.kwargs["item_id"]
        model_name = self.kwargs["model"].replace("_", "")
        ModelName = apps.get_model(app_label="ui", model_name=model_name)
        item = ModelName.objects.get(pk=item_id)

        context["item"] = item
        context["model_name"] = self.kwargs["model"].replace("_", " ")
        try:
            context["name"] = item.name
        except Exception:
            context["name"] = item.token

        diff_list = list()
        for historical_obj in context['history_list']:
            diff_list.append(diff_historical_object(historical_obj))
        context['diff_list'] = diff_list
        return context class UserProfileDetailView(LoginRequiredMixin, DetailView):
    model = User
    template_name = 'ui/profile_details.html'

    def get_context_data(self, **kwargs):
        context = super(UserProfileDetailView, self).get_context_data(**kwargs)
        context["user_groups"] = ','.join([g.name for g in self.request.user.groups.all()])
        superusers = User.objects.filter(is_superuser=True)
        # get the first super user email information
        context["email_info"] = 'mailto:' + superusers[0].email if superusers else ''
        return context

    def get_object(self, queryset=None):
        return self.request.user class SeedsJSONAPIView(LoginRequiredMixin, BaseDatatableView):
    columns = ['link', 'token', 'uid', 'messages']
    # need to define the order columns, also need to match the Datatables setting in columnDefs
    order_columns = ['', 'token', 'uid', '']
    seed_infos = {}
    seed_warnings = {}
    seed_errors = {}
    last_harvest = None
    collection = None
    status = None

    # set max limit of records returned, this is used to protect our site if someone tries to attack our site
    # and make it return huge amount of data
    # max_display_length = 1200

    def get_initial_queryset(self):
        # return queryset used as base for futher sorting/filtering
        # these are simply objects displayed in datatable
        # You should not filter data returned here by any filter values entered by user. This is because
        # we need some base queryset to count total number of records.
        self.collection = get_object_or_404(Collection, pk=self.kwargs['pk'])
        # Check permissions to download
        check_collection_set_based_permission(self.collection, self.request.user, allow_staff=True,
                                              allow_collection_visibility=True)
        self.status = self.kwargs['status']
        self.last_harvest = self.collection.last_harvest()
        self.seed_infos = _get_seed_msg_map(self.last_harvest.infos) if self.last_harvest else {}
        seed_warnings = _get_seed_msg_map(self.last_harvest.warnings) if self.last_harvest else {}
        _add_duplicate_seed_warnings(self.collection, self.seed_warnings)
        self.seed_warnings = seed_warnings
        self.seed_errors = _get_seed_msg_map(self.last_harvest.errors) if self.last_harvest else {}

        # get rid of the exclude field
        self.columns = ['link', 'token', 'uid', 'messages']
        exclude_fields = []
        if not Collection.HARVEST_FIELDS[self.collection.harvest_type]['link']:
            exclude_fields.append('link')
        if not Collection.HARVEST_FIELDS[self.collection.harvest_type]['uid']:
            exclude_fields.append('uid')
        if not Collection.HARVEST_FIELDS[self.collection.harvest_type]['token']:
            exclude_fields.append('token')

        for exclusion in exclude_fields:
            if exclusion in self.columns:
                self.columns.remove(exclusion)
        # Need to update this property here, because it's used by the get_context_data() method of BaseDataTableView and is set PRIOR to the call to this method
        self._columns = self.columns

        return Seed.objects.filter(collection=self.kwargs['pk'], is_active=self.status == 'active').order_by('token',
                                                                                                             'uid')

    def filter_queryset(self, qs):
        # use request parameters to filter queryset
        search = self.request.GET.get(u'search[value]', None)
        if search:
            qs = qs.filter(Q(token__icontains=search) | Q(uid__icontains=search))
        return qs

    def render_column(self, row, column):
        # We want to render user as a custom column
        if column == 'link':
            platform_type = self.collection.harvest_type.split('_')[0]
            return mark_safe(u'<a target="_blank" href="{0}"> <img src="{1}" alt="Link to {2} account for {3}" height="35" width="35"/></a>'.format(
                row.social_url(), static('ui/img/{}_logo.png'.format(platform_type)), platform_type, row.token))
        elif column == 'messages':
            msg_seed = ""
            for msg in self.seed_infos.get(row.seed_id, []):
                msg_seed += u'<li><p>{}</p></li>'.format(msg)
            for msg in self.seed_warnings.get(row.seed_id, []):
                msg_seed += u'<li><p>{}</p></li>'.format(msg)
            for msg in self.seed_errors.get(row.seed_id, []):
                msg_seed += u'<li><p>{}</p></li>'.format(msg)
            return mark_safe(u'<ul>{}</ul>'.format(msg_seed)) if msg_seed else ""

        elif column == 'uid':
            return mark_safe(u'<a href="{}">{}</a>'.format(reverse('seed_detail', args=[row.pk]),
                                                           row.uid)) if row.uid else ""
        elif column == 'token':
            return mark_safe(u'<a href="{}">{}</a>'.format(reverse('seed_detail', args=[row.pk]),
                                                           ui_extras.json_list(row.token))) if row.token else ""
from django.contrib.auth.models import Group
from django.urls import reverse
from django.test import RequestFactory, TestCase, Client
from django.conf import settings
from django.core.exceptions import PermissionDenied
from .models import CollectionSet, User, Credential, Seed, Collection, Export
from .views import CollectionSetListView, CollectionSetDetailView, CollectionSetUpdateView, \
    CollectionCreateView, SeedUpdateView, SeedCreateView, SeedDetailView, ExportDetailView, export_file, \
    ChangeLogView, UserProfileDetailView, SeedsJSONAPIView
import os
import shutil
                                                    platform='test platform',
                                                    token="{'key': '1'}")
        self.collection = Collection.objects.create(collection_set=self.collection_set,
                                                    credential=self.credential,
                                                    harvest_type=Collection.TWITTER_USER_TIMELINE,
                                                    name='Test collection one',
                                                    )
        self.seed = Seed.objects.create(collection=self.collection,
                                        token="test token",
                                        uid="123",
                                        )
        self.factory = RequestFactory()

    def test_seed_form_collection_collection_set(self):
        """
        test that collection and collection_set are loaded with seed form view
        """
        request = self.factory.get(reverse("seed_create",
                                           args=[self.collection.pk]))
        request.user = self.user
        response = SeedCreateView.as_view()(request, collection_pk=self.collection.pk)
        self.assertEqual(self.collection, response.context_data["form"].initial["collection"])
        self.assertEqual(self.collection, response.context_data["collection"])
        self.assertEqual(self.collection_set, response.context_data["collection_set"])


class SeedTestsMixin:
    def setUp(self):
        self.group = Group.objects.create(name='testgroup1')
        self.user = User.objects.create_user('testuser', 'testuser@example.com',
                                             'password')
        self.user.groups.add(self.group)
        self.user.save()
        self.collection_set = CollectionSet.objects.create(name='Test Collection Set One',
                                                           group=self.group)
        self.credential = Credential.objects.create(user=self.user,
                                                    platform='test platform',
                                                    token="{'key': '1'}")
        self.collection = Collection.objects.create(collection_set=self.collection_set,
                                                    credential=self.credential,
                                                    harvest_type=Collection.TWITTER_USER_TIMELINE,
                                                    name='Test collection one',
                                                    )
        self.seed = Seed.objects.create(collection=self.collection,
                                        token="test",
                                        uid="123",
                                        )
        self.factory = RequestFactory()


class SeedUpdateViewTests(SeedTestsMixin, TestCase):
    def test_seed_update_collection_set(self):
        """
        test that collection_set loaded into seed update view
        """
        request = self.factory.get(reverse("seed_update", args=[self.seed.pk]))
        request.user = self.user
        response = SeedUpdateView.as_view()(request, pk=self.seed.pk)
        self.assertEqual(self.collection_set, response.context_data["collection_set"])


class SeedDetailViewTests(SeedTestsMixin, TestCase):
    def test_seed_detail_collection_set(self):
        """
        test that collection_set loaded into seed detail view
        """
        request = self.factory.get(reverse("seed_detail", args=[self.seed.pk]))
        request.user = self.user
        response = SeedDetailView.as_view()(request, pk=self.seed.pk)
        self.assertEqual(self.collection_set, response.context_data["collection_set"])


class SeedBulkCreateViewTests(SeedTestsMixin, TestCase):
    def setUp(self):
        SeedTestsMixin.setUp(self)
        self.client = Client()
        self.assertTrue(self.client.login(username=self.user.username, password='password'))

    def test_get(self):
        response = self.client.get(reverse("bulk_seed_create", args=[self.collection.pk]))
        self.assertTrue(response.context["form"])
        self.assertEqual(self.collection, response.context["collection"])
        self.assertEqual(self.collection_set, response.context["collection_set"])
        self.assertEqual("Twitter user timeline", response.context["harvest_type_name"])

    def test_post(self):
        response = self.client.post(reverse("bulk_seed_create", args=[self.collection.pk]),
                                    {'seeds_type': 'token', 'tokens': """
        test

        test2
          @test3
        null
        """})
        self.assertEqual(3, Seed.objects.filter(collection=self.collection).count())
        self.assertTrue(Seed.objects.filter(collection=self.collection, token='test3').exists())
        self.assertTrue(response.url.endswith('/ui/collections/1/'))

    def test_post_uid(self):
        response = self.client.post(reverse("bulk_seed_create", args=[self.collection.pk]),
                                    {'seeds_type': 'uid', 'tokens': """
        123

        45367
          9087
        """})
        self.assertEqual(3, Seed.objects.filter(collection=self.collection).count())
        self.assertSetEqual({'123', '45367', '9087'}, {s.uid for s in Seed.objects.filter(collection=self.collection)})
        self.assertTrue(response.url.endswith('/ui/collections/1/'))


class ExportDetailViewTests(TestCase):
    def setUp(self):
        self.group = Group.objects.create(name='testgroup1')
        self.user = User.objects.create_user('testuser', 'testuser@example.com',
                                             'password')
        self.user.groups.add(self.group)
        self.user.save()
        self.collection_set = CollectionSet.objects.create(name='Test Collection Set One',
                                                           group=self.group)

        self.credential = Credential.objects.create(user=self.user,