class Contact(models.Model):
    """
    A contact in RapidPro
    """

    DISPLAY_NAME = "name"
    DISPLAY_URNS = "urns"
    DISPLAY_ANON = "uuid"

    SAVE_GROUPS_ATTR = "__data__groups"

    org = models.ForeignKey(Org, verbose_name=_("Organization"), related_name="contacts", on_delete=models.PROTECT)

    uuid = models.CharField(max_length=36, unique=True, null=True)

    name = models.CharField(
        verbose_name=_("Full name"), max_length=128, null=True, blank=True, help_text=_("The name of this contact")
    )

    groups = models.ManyToManyField(Group, related_name="contacts")

    fields = HStoreField(null=True)

    language = models.CharField(
        max_length=3, verbose_name=_("Language"), null=True, blank=True, help_text=_("Language for this contact")
    )

    is_active = models.BooleanField(default=True, help_text="Whether this contact is active")

    is_blocked = models.BooleanField(default=False, help_text="Whether this contact is blocked")

    is_stopped = models.BooleanField(default=False, help_text="Whether this contact opted out of receiving messages")

    is_stub = models.BooleanField(default=False, help_text="Whether this contact is just a stub")

    suspended_groups = models.ManyToManyField(Group, help_text=_("Groups this contact has been suspended from"))

    created_on = models.DateTimeField(auto_now_add=True, help_text=_("When this contact was created"))

    urns = ArrayField(
        models.CharField(max_length=255), default=list, help_text=_("List of URNs of the format 'scheme:path'")
    )

    def __init__(self, *args, **kwargs):
        if self.SAVE_GROUPS_ATTR in kwargs:
            setattr(self, self.SAVE_GROUPS_ATTR, kwargs.pop(self.SAVE_GROUPS_ATTR))

        super(Contact, self).__init__(*args, **kwargs)

    @classmethod
    def get_or_create(cls, org, uuid, name=None):
        """
        Gets an existing contact or creates a stub contact. Used when receiving messages where the contact might not
        have been synced yet
        """
        with cls.lock(org, uuid):
            contact = cls.objects.filter(org=org, uuid=uuid).first()
            if not contact:
                contact = cls.objects.create(org=org, uuid=uuid, name=name, is_stub=True)

            return contact

    @classmethod
    def get_or_create_from_urn(cls, org, urn, name=None):
        """
        Gets an existing contact or creates a new contact. Used when opening a case without an initial message
        """
        normalized_urn = URN.normalize(urn)
        contact = cls.objects.filter(urns__contains=[normalized_urn]).first()
        if not contact:
            URN.validate(normalized_urn)
            uuid = org.get_backend().resolve_urn(org, normalized_urn)
            contact = cls.objects.filter(uuid=uuid)

            if not contact:
                contact = cls.objects.create(org=org, name=name, urns=[normalized_urn], is_stub=False, uuid=uuid)
        return contact

    @classmethod
    def lock(cls, org, uuid):
        return get_redis_connection().lock(CONTACT_LOCK_KEY % (org.pk, uuid), timeout=60)

    def get_display(self):
        """
        Gets the display of this contact. If the site uses anonymous contacts this is generated from the backend UUID.
        If the display setting is recognised and set then that field is returned, otherwise the name is returned.
        If no name is set an empty string is returned.
        """
        display_format = getattr(settings, "SITE_CONTACT_DISPLAY", self.DISPLAY_NAME)

        if display_format == self.DISPLAY_ANON and self.uuid:
            return self.uuid[:6].upper()
        elif display_format == self.DISPLAY_URNS and self.urns:
            _scheme, path = URN.to_parts(self.urns[0])
            return path
        elif display_format == self.DISPLAY_NAME and self.name:
            return self.name

        return "---"

    def get_fields(self, visible=None):
        fields = self.fields if self.fields else {}

        if visible:
            keys = Field.get_all(self.org, visible=True).values_list("key", flat=True)
            return {k: fields.get(k) for k in keys}
        else:
            return fields

    def get_language(self):
        if self.language:
            return {"code": self.language, "name": get_language_name(self.language)}
        else:
            return None

    def prepare_for_case(self):
        """
        Prepares this contact to be put in a case
        """
        if self.is_stub:  # pragma: no cover
            raise ValueError("Can't create a case for a stub contact")

        # suspend contact from groups while case is open
        self.suspend_groups()

        # expire any active flow runs they have
        self.expire_flows()

        # labelling task may have picked up messages whilst case was closed. Those now need to be archived.
        self.archive_messages()

    def suspend_groups(self):
        with self.lock(self.org, self.uuid):
            if self.suspended_groups.all():  # pragma: no cover
                raise ValueError("Can't suspend from groups as contact is already suspended from groups")

            cur_groups = list(self.groups.all())
            suspend_groups = set(Group.get_suspend_from(self.org))

            for group in cur_groups:
                if group in suspend_groups:
                    self.groups.remove(group)
                    self.suspended_groups.add(group)

                    self.org.get_backend().remove_from_group(self.org, self, group)

    def restore_groups(self):
        with self.lock(self.org, self.uuid):
            for group in list(self.suspended_groups.all()):
                if not group.is_dynamic:
                    self.groups.add(group)
                    self.org.get_backend().add_to_group(self.org, self, group)

                self.suspended_groups.remove(group)

    def expire_flows(self):
        self.org.get_backend().stop_runs(self.org, self)

    def archive_messages(self):
        self.incoming_messages.update(is_archived=True)

        self.org.get_backend().archive_contact_messages(self.org, self)

    def release(self):
        """
        Deletes this contact, removing them from any groups they were part of
        """
        self.groups.clear()

        # mark all messages as inactive and handled
        self.incoming_messages.update(is_handled=True, is_active=False)

        self.is_active = False
        self.save(update_fields=("is_active",))

    def as_json(self, full=True):
        """
        Prepares a contact for JSON serialization
        """
        result = {"id": self.pk, "display": self.get_display()}

        if full:
            hidden_fields = getattr(settings, "SITE_HIDE_CONTACT_FIELDS", [])
            result["urns"] = self.urns if "urns" not in hidden_fields else []
            result["name"] = self.name if "name" not in hidden_fields else None
            result["groups"] = [g.as_json(full=False) for g in self.groups.all()]
            result["fields"] = self.get_fields(visible=True)
            result["language"] = self.get_language()
            result["blocked"] = self.is_blocked
            result["stopped"] = self.is_stopped

        return result

    def __str__(self):
        return self.get_display() RESPONSE_DELAY_WARN_SECONDS = 24 * 60 * 60  # show response delays > 1 day as warning
def override_start(self, org):  # pragma: no cover
        def get_form_kwargs(self):
        def derive_initial(self):
        def save(self, obj):
        def get_success_url(self):
        def get_form_kwargs(self):
        def post_save(self, obj):
        def get_queryset(self):
        def get_context_data(self, **kwargs):
        def post(self, request, *args, **kwargs):
        def get(self, request, *args, **kwargs):
            def as_json(label):
        def post(self, request, *args, **kwargs):
        def post(self, request, *args, **kwargs):
    def derive_search(self):
        def get_messages(self, search, last_refresh=None):
        def get_context_data(self, **kwargs):
        def render_to_response(self, context, **response_kwargs):
        def derive_url_pattern(cls, path, action):
        def post(self, request, *args, **kwargs):
        def derive_url_pattern(cls, path, action):
        def post(self, request, *args, **kwargs):
        def derive_url_pattern(cls, path, action):
        def post(self, request, *args, **kwargs):
        def derive_url_pattern(cls, path, action):
        def post(self, request, *args, **kwargs):
        def derive_url_pattern(cls, path, action):
        def post(self, request, *args, **kwargs):
        def derive_url_pattern(cls, path, action):
        def get(self, request, *args, **kwargs):
        def post(self, request, *args, **kwargs):
    def derive_search(self):
        def derive_search(self):
        def get_context_data(self, **kwargs):
        def render_to_response(self, context, **response_kwargs):
        def get(self, request, *args, **kwargs):
            def as_json(msg):
        def post(self, request, *args, **kwargs):
    def derive_search(self):
        def derive_queryset(self, **kwargs):
        def get_form_kwargs(self):
        def save(self, obj):
        def derive_queryset(self, **kwargs):
        def get_context_data(self, **kwargs):
        def get_form_kwargs(self):
        def derive_initial(self):
        def derive_fields(self):
        def post(self, request, *args, **kwargs):
        def get_context_data(self, **kwargs):
        def render_to_response(self, context, **response_kwargs):
        def get_form_class(self):
        def post_save(self, task):
        def get_context_data(self, **kwargs):
        def render_to_response(self, context, **response_kwargs):
class LabelCRUDL(SmartCRUDL):
    class Create(RuleFormMixin, OrgPermsMixin, SmartCreateView):
    class Update(RuleFormMixin, OrgObjPermsMixin, SmartUpdateView):
    class Read(OrgObjPermsMixin, SmartReadView):
    class Delete(OrgObjPermsMixin, SmartDeleteView):
    class List(OrgPermsMixin, SmartListView):
    class Watch(OrgObjPermsMixin, SmartReadView):
    class Unwatch(OrgObjPermsMixin, SmartReadView):
class MessageSearchMixin(object):
class MessageCRUDL(SmartCRUDL):
    class Search(OrgPermsMixin, MessageSearchMixin, SmartTemplateView):
    class Lock(OrgPermsMixin, SmartTemplateView):
    class Action(OrgPermsMixin, SmartTemplateView):
    class Label(OrgPermsMixin, SmartTemplateView):
    class BulkReply(OrgPermsMixin, SmartTemplateView):
    class Forward(OrgPermsMixin, SmartTemplateView):
    class History(OrgPermsMixin, SmartTemplateView):
class MessageExportCRUDL(SmartCRUDL):
    class Create(NonAtomicMixin, OrgPermsMixin, MessageSearchMixin, SmartCreateView):
    class Read(BaseDownloadView):
class ReplySearchMixin(object):
class OutgoingCRUDL(SmartCRUDL):
    class Search(OrgPermsMixin, SmartTemplateView):
    class SearchReplies(OrgPermsMixin, ReplySearchMixin, SmartTemplateView):
class ReplyExportCRUDL(SmartCRUDL):
    class Create(NonAtomicMixin, OrgPermsMixin, ReplySearchMixin, SmartCreateView):
    class Read(BaseDownloadView):
class FaqSearchMixin(object):
class FaqCRUDL(SmartCRUDL):
    class List(OrgPermsMixin, SmartListView):
    class Create(OrgPermsMixin, SmartCreateView):
    class Read(OrgPermsMixin, SmartReadView):
    class Update(OrgPermsMixin, SmartUpdateView):
    class Delete(OrgPermsMixin, SmartDeleteView):
    class Search(OrgPermsMixin, FaqSearchMixin, SmartTemplateView):
    class Import(OrgPermsMixin, SmartCSVImportView):
        class Form(forms.ModelForm):
            class Meta:
    class Languages(OrgPermsMixin, SmartTemplateView): class Notification(models.Model):
    """
    A notification sent to a user
    """

    TYPE_MESSAGE_LABELLING = "L"
    TYPE_CASE_ASSIGNMENT = "C"
    TYPE_CASE_ACTION = "A"
    TYPE_CASE_REPLY = "R"

    TYPE_NAME = {
        TYPE_MESSAGE_LABELLING: "message_labelling",
        TYPE_CASE_ASSIGNMENT: "case_assignment",
        TYPE_CASE_ACTION: "case_action",
        TYPE_CASE_REPLY: "case_reply",
    }

    org = models.ForeignKey(Org, on_delete=models.PROTECT)

    user = models.ForeignKey(User, related_name="notifications", on_delete=models.PROTECT)

    type = models.CharField(max_length=1)

    message = models.ForeignKey(Message, null=True, on_delete=models.PROTECT)

    case_action = models.ForeignKey(CaseAction, null=True, on_delete=models.PROTECT)

    is_sent = models.BooleanField(default=False)

    created_on = models.DateTimeField(default=timezone.now)

    @classmethod
    def new_message_labelling(cls, org, user, message):
        return cls.objects.get_or_create(org=org, user=user, type=cls.TYPE_MESSAGE_LABELLING, message=message)

    @classmethod
    def new_case_assignment(cls, org, user, case_action):
        return cls.objects.get_or_create(org=org, user=user, type=cls.TYPE_CASE_ASSIGNMENT, case_action=case_action)

    @classmethod
    def new_case_action(cls, org, user, case_action):
        return cls.objects.get_or_create(org=org, user=user, type=cls.TYPE_CASE_ACTION, case_action=case_action)

    @classmethod
    def new_case_reply(cls, org, user, message):
        return cls.objects.get_or_create(org=org, user=user, type=cls.TYPE_CASE_REPLY, message=message)

    @classmethod
    def send_all(cls):
        unsent = cls.objects.filter(is_sent=False)
        unsent = unsent.select_related("org", "user", "message", "case_action").order_by("created_on")

        for notification in unsent:
            type_name = cls.TYPE_NAME[notification.type]
            subject, template, context = getattr(notification, "_build_%s_email" % type_name)()
            template_path = "profiles/email/%s" % template

            send_email([notification.user], str(subject), template_path, context)

        unsent.update(is_sent=True)

    def _build_message_labelling_email(self):
        context = {
            "labels": set(self.user.watched_labels.all()).intersection(self.message.labels.all()),
            "inbox_url": self.org.make_absolute_url(reverse("cases.inbox")),
        }
        return _("New labelled message"), "message_labelling", context

    def _build_case_assignment_email(self):
        case = self.case_action.case
        context = {
            "user": self.case_action.created_by,
            "case_url": self.org.make_absolute_url(reverse("cases.case_read", args=[case.pk])),
        }
        return _("New case assignment #%d") % case.pk, "case_assignment", context

    def _build_case_action_email(self):
        case = self.case_action.case
        context = {
            "user": self.case_action.created_by,
            "note": self.case_action.note,
            "assignee": self.case_action.assignee,
            "case_url": self.org.make_absolute_url(reverse("cases.case_read", args=[case.pk])),
        }

        if self.case_action.action == CaseAction.ADD_NOTE:
            subject = _("New note in case #%d") % case.pk
            template = "case_new_note"
        elif self.case_action.action == CaseAction.CLOSE:
            subject = _("Case #%d was closed") % case.pk
            template = "case_closed"
        elif self.case_action.action == CaseAction.REOPEN:
            subject = _("Case #%d was reopened") % case.pk
            template = "case_reopened"
        elif self.case_action.action == CaseAction.REASSIGN:
            subject = _("Case #%d was reassigned") % case.pk
            template = "case_reassigned"
        else:  # pragma: no cover
            raise ValueError("Notifications not supported for case action type %s" % self.case_action.action)

        return subject, template, context

    def _build_case_reply_email(self):
        case = self.message.case
        context = {"case_url": self.org.make_absolute_url(reverse("cases.case_read", args=[case.pk]))}
        return _("New reply in case #%d") % case.pk, "case_reply", context class ContainsTest(Test):
    """
    Test that returns whether the message text contains or doesn't contain the given keywords
    """

    TYPE = "contains"

    def __init__(self, keywords, quantifier):
        self.keywords = [normalize(word) for word in keywords]
        self.quantifier = quantifier

    @classmethod
    def from_json(cls, json_obj, context):
        return cls(json_obj["keywords"], Quantifier.from_json(json_obj["quantifier"]))

    def to_json(self):
        return {"type": self.TYPE, "keywords": self.keywords, "quantifier": self.quantifier.to_json()}

    def get_description(self):
        quoted_keywords = ['"%s"' % w for w in self.keywords]
        return "message contains %s %s" % (str(self.quantifier), ", ".join(quoted_keywords))

    def matches(self, message):
        text = normalize(message.text)

        def keyword_check(w):
            return lambda: bool(regex.search(r"\b" + w + r"\b", text, flags=regex.UNICODE | regex.V0))

        checks = [keyword_check(keyword) for keyword in self.keywords]

        return self.quantifier.evaluate(checks)

    @classmethod
    def is_valid_keyword(cls, keyword):
        return KEYWORD_REGEX.match(keyword)

    def __eq__(self, other):
        return (
            other
            and self.TYPE == other.TYPE
            and self.keywords == other.keywords
            and self.quantifier == other.quantifier
        ) class FieldTest(Test):
    """
    Test that returns whether the message was sent from a contact with the given field value
    """

    TYPE = "field"

    def __init__(self, key, values):
        self.key = key
        self.values = [normalize(v) for v in values]

    @classmethod
    def from_json(cls, json_obj, context):
        return cls(json_obj["key"], json_obj["values"])

    def to_json(self):
        return {"type": self.TYPE, "key": self.key, "values": self.values}

    def get_description(self):
        quoted_values = ['"%s"' % v for v in self.values]
        return "contact.%s is %s %s" % (self.key, Quantifier.ANY, ", ".join(quoted_values))

    def matches(self, message):
        if message.contact.fields:
            contact_value = normalize(message.contact.fields.get(self.key, ""))

            for value in self.values:
                if value == contact_value:
                    return True
        return False

    def __eq__(self, other):
        return other and self.TYPE == other.TYPE and self.key == other.key and self.values == other.values class GroupsTest(Test):
    """
    Test that returns whether the message was sent from the given contact groups
    """

    TYPE = "groups"

    def __init__(self, groups, quantifier):
        self.groups = groups
        self.quantifier = quantifier

    @classmethod
    def from_json(cls, json_obj, context):
        groups = list(Group.objects.filter(org=context.org, pk__in=json_obj["groups"]).order_by("pk"))
        return cls(groups, Quantifier.from_json(json_obj["quantifier"]))

    def to_json(self):
        return {"type": self.TYPE, "groups": [g.pk for g in self.groups], "quantifier": self.quantifier.to_json()}

    def get_description(self):
        group_names = [g.name for g in self.groups]
        return "contact belongs to %s %s" % (str(self.quantifier), ", ".join(group_names))

    def matches(self, message):
        contact_groups = set(message.contact.groups.all())

        def group_check(g):
            return lambda: g in contact_groups

        checks = [group_check(group) for group in self.groups]

        return self.quantifier.evaluate(checks)

    def __eq__(self, other):
        return (
            other and self.TYPE == other.TYPE and self.groups == other.groups and self.quantifier == other.quantifier
        ) class Quantifier(Enum):
    """
    Tests are typically composed of multiple conditions, e.g. contains ANY of X, Y or Z.
    """

    NONE = (1, _("none of"))
    ANY = (2, _("any of"))
    ALL = (3, _("all of"))

    def __init__(self, val, text):
        self.val = val
        self.text = text

    @classmethod
    def from_json(cls, val):
        return cls[val.upper()]

    def to_json(self):
        return self.name.lower()

    def evaluate(self, condition_callables):
        if self == Quantifier.NONE:
            for condition in condition_callables:
                if condition():
                    return False
            return True
        elif self == Quantifier.ANY:
            for condition in condition_callables:
                if condition():
                    return True
            return False
        elif self == Quantifier.ALL:
            for condition in condition_callables:
                if not condition():
                    return False
            return True

    def __str__(self):
        return str(self.text) class WordCountTest(Test):
    """
    Test that returns whether the message text contains at least the given number of words
    """

    TYPE = "words"

    def __init__(self, minimum):
        self.minimum = minimum

    @classmethod
    def from_json(cls, json_obj, context):
        return cls(json_obj["minimum"])

    def to_json(self):
        return {"type": self.TYPE, "minimum": self.minimum}

    def get_description(self):
        return "message has at least %d words" % self.minimum

    def matches(self, message):
        num_words = len(regex.findall(r"\w+", message.text, flags=regex.UNICODE | regex.V0))
        return num_words >= self.minimum

    def __eq__(self, other):
        return other and self.TYPE == other.TYPE and self.minimum == other.minimum @shared_task
def squash_counts():
    """
    Task to squash all daily counts
    """
    from .models import DailyCount, DailySecondTotalCount, TotalCount

    TotalCount.squash()
    DailyCount.squash()
    DailySecondTotalCount.squash() class BaseCasesTest(DashTest):
    """
    Base class for all test cases
    """

    def setUp(self):
        super(BaseCasesTest, self).setUp()

        settings.SITE_BACKEND = "casepro.test.TestBackend"
        settings.SITE_ORGS_STORAGE_ROOT = "test_orgs"

        # some orgs
        self.unicef = self.create_org("UNICEF", timezone=zoneinfo.ZoneInfo("Africa/Kampala"), subdomain="unicef")
        self.nyaruka = self.create_org("Nyaruka", timezone=zoneinfo.ZoneInfo("Africa/Kigali"), subdomain="nyaruka")

        # some admins for those orgs
        self.admin = self.create_admin(self.unicef, "Kidus", "kidus@unicef.org")
        self.norbert = self.create_admin(self.nyaruka, "Norbert Kwizera", "norbert@nyaruka.com")

        # some message labels
        self.aids = self.create_label(self.unicef, "L-001", "AIDS", "Messages about AIDS", ["aids", "hiv"])
        self.pregnancy = self.create_label(
            self.unicef, "L-002", "Pregnancy", "Messages about pregnancy", ["pregnant", "pregnancy"]
        )
        self.tea = self.create_label(self.unicef, None, "Tea", "Messages about tea", ["tea", "chai"], is_synced=False)
        self.code = self.create_label(self.nyaruka, "L-101", "Code", "Messages about code", ["java", "python", "go"])

        # some message faqs
        self.preg_faq1_eng = self.create_faq(
            self.unicef, "How do I know I'm pregnant?", "Do a pregnancy test.", "eng", None, [self.pregnancy]
        )
        self.preg_faq1_zul = self.create_faq(
            self.unicef, "ZUL How do I know I'm pregnant?", "ZUL Do a pregnancy test.", "zul", self.preg_faq1_eng, []
        )
        self.preg_faq1_lug = self.create_faq(
            self.unicef, "LUG How do I know I'm pregnant?", "LUG Do a pregnancy test.", "lug", self.preg_faq1_eng, []
        )
        self.preg_faq2_eng = self.create_faq(
            self.unicef,
            "How do I prevent HIV transfer to my baby?",
            "Take ARVs.",
            "eng",
            None,
            [self.pregnancy, self.aids],
        )
        self.tea_faq1_eng = self.create_faq(
            self.unicef, "Does tea contain caffeine?", "It varies - black tea does.", "eng", None, [self.tea]
        )

        # some partners
        self.moh = self.create_partner(self.unicef, "MOH", "Ministry of Health", None, [self.aids, self.pregnancy])
        self.who = self.create_partner(self.unicef, "WHO", "World Health Organisation", None, [self.aids])
        self.klab = self.create_partner(self.nyaruka, "kLab", "Kigali Lab", None, [self.code])

        # some users in those partners
        self.user1 = self.create_user(self.unicef, self.moh, ROLE_MANAGER, "Evan", "evan@unicef.org")
        self.user2 = self.create_user(self.unicef, self.moh, ROLE_ANALYST, "Rick", "rick@unicef.org")
        self.user3 = self.create_user(self.unicef, self.who, ROLE_MANAGER, "Carol", "carol@unicef.org")
        self.user4 = self.create_user(self.nyaruka, self.klab, ROLE_ANALYST, "Bosco", "bosco@klab.rw")

        # some groups
        self.males = self.create_group(self.unicef, "G-001", "Males")
        self.females = self.create_group(self.unicef, "G-002", "Females")
        self.reporters = self.create_group(self.unicef, "G-003", "Reporters", suspend_from=True, is_visible=False)
        self.registered = self.create_group(self.unicef, "G-004", "Registered (Dynamic)", is_dynamic=True)
        self.coders = self.create_group(self.nyaruka, "G-005", "Coders")

        # some fields
        self.nickname = self.create_field(self.unicef, "nickname", "Nickname", value_type="T")
        self.age = self.create_field(self.unicef, "age", "Age", value_type="N")
        self.state = self.create_field(self.unicef, "state", "State", value_type="S", is_visible=False)
        self.motorbike = self.create_field(self.nyaruka, "motorbike", "Moto", value_type="T")

    def create_org(self, name, timezone, subdomain):
        return Org.objects.create(
            name=name, timezone=timezone, subdomain=subdomain, created_by=self.superuser, modified_by=self.superuser
        )

    def create_partner(self, org, name, description, primary_contact, labels=(), restricted=True):
        return Partner.create(org, name, description, primary_contact, restricted, labels, None)

    def create_admin(self, org, name, email):
        return Profile.create_org_user(org, name, email, email)

    def create_user(self, org, partner, role, name, email):
        return Profile.create_partner_user(org, partner, role, name, email, email)

    def create_label(self, org, uuid, name, description, keywords=(), **kwargs):
        label = Label.objects.create(org=org, uuid=uuid, name=name, description=description, **kwargs)

        if keywords:
            rule = Rule.create(org, [ContainsTest(keywords, Quantifier.ANY)], [LabelAction(label)])
            label.rule = rule
            label.save(update_fields=("rule",))

        return label

    def create_rule(self, org, tests, actions):
        return Rule.create(org, tests, actions)

    def create_faq(self, org, question, answer, language, parent, labels=(), **kwargs):
        faq = FAQ.create(org, question, answer, language, parent, labels)
        return faq

    def create_contact(self, org, uuid, name, groups=(), fields=None, is_stub=False):
        contact = Contact.objects.create(org=org, uuid=uuid, name=name, is_stub=is_stub, fields=fields, language="eng")
        contact.groups.add(*groups)
        return contact

    def create_group(self, org, uuid, name, count=None, is_dynamic=False, is_visible=True, suspend_from=False):
        return Group.objects.create(
            org=org,
            uuid=uuid,
            name=name,
            count=count,
            is_dynamic=is_dynamic,
            is_visible=is_visible,
            suspend_from=suspend_from,
        )

    def create_field(self, org, key, label, value_type="T", is_visible=True):
        return Field.objects.create(org=org, key=key, label=label, value_type=value_type, is_visible=is_visible)

    def create_message(self, org, backend_id, contact, text, labels=(), **kwargs):
        if "type" not in kwargs:
            kwargs["type"] = "I"
        if "created_on" not in kwargs:
            kwargs["created_on"] = now()

        msg = Message.objects.create(org=org, backend_id=backend_id, contact=contact, text=text, **kwargs)
        msg.label(*labels)
        return msg

    def create_outgoing(self, org, user, broadcast_id, activity, text, contact, **kwargs):
        return Outgoing.objects.create(
            org=org,
            partner=user.get_partner(org),
            backend_broadcast_id=broadcast_id,
            activity=activity,
            text=text,
            contact=contact,
            created_by=user,
            **kwargs,
        )

    def create_case(self, org, contact, assignee, message, labels=(), user_assignee=None, **kwargs):
        case = Case.objects.create(
            org=org,
            contact=contact,
            assignee=assignee,
            user_assignee=user_assignee,
            initial_message=message,
            **kwargs,
        )
        case.labels.add(*labels)

        if "opened_on" in kwargs:  # uses auto_now_add
            case.opened_on = kwargs["opened_on"]
            case.save(update_fields=("opened_on",))

        if message:
            message.case = case
            message.save(update_fields=("case",))

        return case

    def url_post_json(self, subdomain, url, data):
        return self.url_post(subdomain, url, json.dumps(data), content_type="application/json")

    def assertLoginRedirect(self, response, next_url):
        self.assertRedirects(response, "/users/login/?next=%s" % next_url)

    def assertNotCalled(self, mock):
        """
        Because mock.assert_not_called doesn't exist in Python 2
        """
        self.assertEqual(len(mock.mock_calls), 0, "Expected no calls, called %d times" % len(mock.mock_calls))

    def openWorkbook(self, filename):
        return open_workbook("%s/%s" % (settings.MEDIA_ROOT, filename), "rb")

    def assertExcelRow(self, sheet, row_num, values, tz=None):
        """
        Asserts the cell values in the given worksheet row. Date values are converted using the provided timezone.
        """
        expected_values = []
        for expected in values:
            # if expected value is datetime, localize and remove microseconds
            if isinstance(expected, datetime):
                expected = expected.astimezone(tz).replace(microsecond=0, tzinfo=None)
            elif isinstance(expected, date):
                expected = datetime.combine(expected, time(0, 0))

            expected_values.append(expected)

        actual_values = []
        for c in range(0, sheet.ncols):
            cell = sheet.cell(row_num, c)
            actual = cell.value

            if cell.ctype == XL_CELL_DATE:
                actual = datetime(*xldate_as_tuple(actual, sheet.book.datemode))

            actual_values.append(actual)

        self.assertEqual(actual_values, expected_values)

    def assertSentMail(self, recipients, reset_outbox=True):
        self.assertEqual([e.to[0] for e in mail.outbox], recipients)

        if reset_outbox:
            mail.outbox = []

    def login(self, user):
        password = "root" if user == self.superuser else user.username
        result = self.client.login(username=user.username, password=password)
        self.assertTrue(result, "Couldn't login as %(user)s / %(user)s" % dict(user=user.username)) class FAQ(models.Model):
    """
    Pre-approved questions and answers to be used when replying to a message.
    """

    org = models.ForeignKey(Org, verbose_name=_("Organization"), related_name="faqs", on_delete=models.PROTECT)

    question = models.CharField(max_length=255)

    answer = models.TextField()

    language = models.CharField(
        max_length=3, verbose_name=_("Language"), null=True, blank=True, help_text=_("Language for this FAQ")
    )

    parent = models.ForeignKey("self", null=True, blank=True, related_name="translations", on_delete=models.PROTECT)

    labels = models.ManyToManyField(Label, help_text=_("Labels assigned to this FAQ"), related_name="faqs")

    @classmethod
    def create(cls, org, question, answer, language, parent, labels=(), **kwargs):
        """
        A helper for creating FAQs since labels (many-to-many) needs to be added after initial creation
        """
        faq = cls.objects.create(org=org, question=question, answer=answer, language=language, parent=parent, **kwargs)

        if labels:
            faq.labels.add(*labels)

        return faq

    @classmethod
    def search(cls, org, user, search):
        """
        Search for FAQs
        """
        language = search.get("language")
        label_id = search.get("label")
        text = search.get("text")

        queryset = cls.objects.filter(org=org)

        # Language filtering
        if language:
            queryset = queryset.filter(language=language)

        # Label filtering
        labels = Label.get_all(org, user)

        if label_id:
            labels = labels.filter(pk=label_id)
        else:
            # if not filtering by a single label, need distinct to avoid duplicates
            queryset = queryset.distinct()

        queryset = queryset.filter(Q(labels__in=list(labels)) | Q(parent__labels__in=list(labels)))

        # Text filtering
        if text:
            queryset = queryset.filter(Q(question__icontains=text) | Q(answer__icontains=text))

        queryset = queryset.prefetch_related(
            Prefetch("labels", Label.objects.filter(is_active=True).order_by("id")), "parent__labels"
        )

        return queryset.order_by("question")

    @classmethod
    def get_all(cls, org, label=None):
        queryset = cls.objects.filter(org=org)

        if label:
            queryset = queryset.filter(Q(labels=label) | Q(parent__labels=label))

        return queryset

    @staticmethod
    def get_language_from_code(code):
        return {"code": code, "name": get_language_name(code)}

    @classmethod
    def get_all_languages(cls, org):
        queryset = cls.objects.filter(org=org)
        return queryset.values("language").order_by("language").distinct()

    def get_language(self):
        if self.language:
            return self.get_language_from_code(self.language)
        else:
            return None

    def release(self):
        for child in self.translations.all():
            child.release()
        self.delete()

    def as_json(self, full=True):
        result = {"id": self.pk, "question": self.question}
        if full:
            if not self.parent:
                parent_json = None
                result["labels"] = [l.as_json() for l in self.labels.all()]
            else:
                parent_json = self.parent.id
                result["labels"] = [l.as_json() for l in self.parent.labels.all()]

            result["answer"] = self.answer
            result["language"] = self.get_language()
            result["parent"] = parent_json

        return result

    def __str__(self):
        return self.question class Label(models.Model):
    """
    Corresponds to a message label in RapidPro. Used for determining visibility of messages to different partners.
    """

    org = models.ForeignKey(Org, related_name="labels", on_delete=models.PROTECT)

    uuid = models.CharField(max_length=36, unique=True, null=True)

    name = models.CharField(verbose_name=_("Name"), max_length=64, help_text=_("Name of this label"))

    description = models.CharField(verbose_name=_("Description"), null=True, max_length=255)

    rule = models.OneToOneField("rules.Rule", null=True, on_delete=models.PROTECT)

    is_synced = models.BooleanField(default=True, help_text="Whether this label should be synced with the backend")

    watchers = models.ManyToManyField(
        User, related_name="watched_labels", help_text="Users to be notified when label is applied to a message"
    )

    is_active = models.BooleanField(default=True)

    INBOX_COUNT_CACHE_ATTR = "_inbox_count"
    ARCHIVED_COUNT_CACHE_ATTR = "_archived_count"

    MAX_NAME_LEN = 64

    @classmethod
    def create(cls, org, name, description, tests, is_synced):
        label = cls.objects.create(org=org, name=name, description=description, is_synced=is_synced)
        label.update_tests(tests)
        return label

    @classmethod
    def get_all(cls, org, user=None):
        if user:
            user_partner = user.get_partner(org)
            if user_partner and user_partner.is_restricted:
                return user_partner.get_labels()

        return org.labels.filter(is_active=True)

    def update_tests(self, tests):
        from casepro.rules.models import LabelAction, Rule

        if tests:
            if self.rule:
                self.rule.tests = json_encode(tests)
                self.rule.save(update_fields=("tests",))
            else:
                self.rule = Rule.create(self.org, tests, [LabelAction(self)])
                self.save(update_fields=("rule",))
        else:
            if self.rule:
                rule = self.rule
                self.rule = None
                self.save(update_fields=("rule",))

                rule.delete()

    def get_tests(self):
        return self.rule.get_tests() if self.rule else []

    def get_inbox_count(self, recalculate=False):
        """
        Number of inbox (non-archived) messages with this label
        """
        return get_obj_cacheable(self, self.INBOX_COUNT_CACHE_ATTR, lambda: self._get_inbox_count(), recalculate)

    def _get_inbox_count(self):
        from casepro.statistics.models import TotalCount

        return TotalCount.get_by_label([self], TotalCount.TYPE_INBOX).scope_totals()[self]

    def get_archived_count(self, recalculate=False):
        """
        Number of archived messages with this label
        """
        return get_obj_cacheable(self, self.ARCHIVED_COUNT_CACHE_ATTR, lambda: self._get_archived_count(), recalculate)

    def _get_archived_count(self):
        from casepro.statistics.models import TotalCount

        return TotalCount.get_by_label([self], TotalCount.TYPE_ARCHIVED).scope_totals()[self]

    @classmethod
    def bulk_cache_initialize(cls, labels):
        """
        Pre-loads cached counts on a set of labels to avoid fetching counts individually for each label
        """
        from casepro.statistics.models import TotalCount

        inbox_by_label = TotalCount.get_by_label(labels, TotalCount.TYPE_INBOX).scope_totals()
        archived_by_label = TotalCount.get_by_label(labels, TotalCount.TYPE_ARCHIVED).scope_totals()

        for label in labels:
            setattr(label, cls.INBOX_COUNT_CACHE_ATTR, inbox_by_label[label])
            setattr(label, cls.ARCHIVED_COUNT_CACHE_ATTR, archived_by_label[label])

    @classmethod
    def lock(cls, org, uuid):
        return get_redis_connection().lock(LABEL_LOCK_KEY % (org.pk, uuid), timeout=60)

    def watch(self, user):
        if not Label.get_all(self.org, user).filter(pk=self.pk).exists():
            raise PermissionDenied()

        self.watchers.add(user)

    def unwatch(self, user):
        self.watchers.remove(user)

    def is_watched_by(self, user):
        return user in self.watchers.all()

    def release(self):
        rule = self.rule

        self.rule = None
        self.is_active = False
        self.save(update_fields=("rule", "is_active"))

        if rule:
            rule.delete()

    def as_json(self, full=True):
        result = {"id": self.pk, "name": self.name}

        if full:
            result["description"] = self.description
            result["synced"] = self.is_synced
            result["counts"] = {"inbox": self.get_inbox_count(), "archived": self.get_archived_count()}

        return result

    def __str__(self):
        return self.name class Labelling(models.Model):
    """
    An application of a label to a message
    """

    label = models.ForeignKey(Label, on_delete=models.CASCADE)

    message = models.ForeignKey("msgs.Message", on_delete=models.CASCADE)

    message_is_archived = models.BooleanField()

    message_is_flagged = models.BooleanField()

    message_created_on = models.DateTimeField()

    @classmethod
    def create(cls, label, message):
        return cls(
            label=label,
            message=message,
            message_created_on=message.created_on,
            message_is_flagged=message.is_flagged,
            message_is_archived=message.is_archived,
        )

    class Meta:
        db_table = "msgs_message_labels"
        unique_together = ("message", "label")
        indexes = (
            Index(
                name="labelling_inbox", fields=("label", "-message_created_on"), condition=Q(message_is_archived=False)
            ),
            Index(
                name="labelling_archived",
                fields=("label", "-message_created_on"),
                condition=Q(message_is_archived=True),
            ),
            Index(
                name="labelling_flagged",
                fields=("label", "-message_created_on"),
                condition=Q(message_is_archived=False, message_is_flagged=True),
            ),
            Index(
                name="labelling_flagged_w_archived",
                fields=("label", "-message_created_on"),
                condition=Q(message_is_flagged=True),
            ),
        ) class Message(models.Model):
    """
    A incoming message from the backend
    """

    TYPE_INBOX = "I"
    TYPE_FLOW = "F"

    TYPE_CHOICES = ((TYPE_INBOX, "Inbox"), (TYPE_FLOW, "Flow"))

    SAVE_CONTACT_ATTR = "__data__contact"
    SAVE_LABELS_ATTR = "__data__labels"

    TIMELINE_TYPE = "I"

    SEARCH_BY_TEXT_DAYS = 90

    org = models.ForeignKey(Org, related_name="incoming_messages", on_delete=models.PROTECT)

    # identifier of the message on the backend
    backend_id = models.IntegerField(unique=True)

    contact = models.ForeignKey(Contact, related_name="incoming_messages", on_delete=models.PROTECT)

    type = models.CharField(max_length=1)

    text = models.TextField(max_length=640)

    labels = models.ManyToManyField(Label, through=Labelling, related_name="messages")

    has_labels = models.BooleanField(default=False)  # maintained via db triggers

    is_flagged = models.BooleanField(default=False)

    is_archived = models.BooleanField(default=False)

    created_on = models.DateTimeField()

    modified_on = models.DateTimeField(null=True, default=now)

    is_handled = models.BooleanField(default=False)

    is_active = models.BooleanField(default=True)

    case = models.ForeignKey("cases.Case", null=True, related_name="incoming_messages", on_delete=models.PROTECT)

    # when this message was last locked by a user
    locked_on = models.DateTimeField(null=True)

    # which user locked it
    locked_by = models.ForeignKey(User, null=True, related_name="actioned_messages", on_delete=models.PROTECT)

    def __init__(self, *args, **kwargs):
        if self.SAVE_CONTACT_ATTR in kwargs:
            setattr(self, self.SAVE_CONTACT_ATTR, kwargs.pop(self.SAVE_CONTACT_ATTR))
        if self.SAVE_LABELS_ATTR in kwargs:
            setattr(self, self.SAVE_LABELS_ATTR, kwargs.pop(self.SAVE_LABELS_ATTR))

        super(Message, self).__init__(*args, **kwargs)

    @classmethod
    def get_unhandled(cls, org):
        return cls.objects.filter(org=org, is_handled=False)

    @classmethod
    def lock(cls, org, backend_id):
        return get_redis_connection().lock(MESSAGE_LOCK_KEY % (org.pk, backend_id), timeout=60)

    @classmethod
    def search(cls, org, user, search, modified_after=None, all=False):
        """
        Search for messages
        """
        folder = search.get("folder")
        label_id = search.get("label")
        text = search.get("text")
        contact_id = search.get("contact")
        after = search.get("after")
        before = search.get("before")

        is_admin = user.can_administer(org)
        partner = user.get_partner(org)
        all_label_access = is_admin or (partner and not partner.is_restricted)

        assert folder != MessageFolder.unlabelled or is_admin, "non-admin access to unlabelled messages"

        # track what we need to filter on by where is can be found in the database
        msg_filtering = {}
        lbl_filtering = {}
        ordering = ("-created_on",)

        # if this is a refresh we want everything with new actions and locks
        if modified_after:
            msg_filtering["modified_on__gt"] = modified_after

            # these are sorted by the UI again so we use the optimal ordering for the index here
            ordering = ("-modified_on", "-created_on")
        else:
            if folder == MessageFolder.inbox or folder == MessageFolder.unlabelled:
                lbl_filtering["is_archived"] = False
            elif folder == MessageFolder.flagged:
                lbl_filtering["is_flagged"] = True
                lbl_filtering["is_archived"] = False
            elif folder == MessageFolder.flagged_with_archived:
                lbl_filtering["is_flagged"] = True
            elif folder == MessageFolder.archived:
                lbl_filtering["is_archived"] = True

        if text:
            msg_filtering["text__icontains"] = text
            msg_filtering["created_on__gt"] = now() - timedelta(days=cls.SEARCH_BY_TEXT_DAYS)
        if contact_id:
            msg_filtering["contact__id"] = contact_id
        if after and not modified_after:
            lbl_filtering["created_on__gt"] = after
        if before:
            lbl_filtering["created_on__lt"] = before

        # only show non-deleted handled messages
        msgs = org.incoming_messages.filter(is_active=True, is_handled=True).order_by(*ordering)

        # handle views that don't require filtering by any labels
        if folder == MessageFolder.unlabelled:
            return (
                msgs.filter(type=Message.TYPE_INBOX, has_labels=False).filter(**msg_filtering).filter(**lbl_filtering)
            )
        if all_label_access and not label_id:
            if folder == MessageFolder.inbox:
                return msgs.filter(has_labels=True).filter(**msg_filtering).filter(**lbl_filtering)
            else:
                return msgs.filter(**msg_filtering).filter(**lbl_filtering)

        # we either want messages with a specific label or any of the labels this user has access to
        labels = Label.get_all(org, user)
        if label_id:
            labels = labels.filter(id=label_id)

        # if we're only filtering on things on the labelling table..
        if not msg_filtering and not all:
            lbl_filtering = {f"message_{k}": v for k, v in lbl_filtering.items()}
            message_ids = set()

            for label in labels:
                message_ids.update(
                    Labelling.objects.filter(label=label, **lbl_filtering)
                    .order_by("-message_created_on")
                    .values_list("message_id", flat=True)[:1000]
                )

            return Message.objects.filter(id__in=message_ids, is_handled=True).order_by("-created_on")

        msg_filtering["has_labels"] = True
        msgs = msgs.filter(labels__in=list(labels))

        return msgs.filter(**msg_filtering).filter(**lbl_filtering)

    def get_history(self):
        """
        Gets the actions for this message in reverse chronological order
        :return: the actions
        """
        return self.actions.select_related("created_by", "label").order_by("-pk")

    def get_lock(self, user):
        if self.locked_by_id and self.locked_on and self.locked_on > (now() - timedelta(seconds=MESSAGE_LOCK_SECONDS)):
            if self.locked_by_id != user.id:
                diff = (self.locked_on + timedelta(seconds=MESSAGE_LOCK_SECONDS)) - now()
                return diff.seconds

        return False

    def release(self):
        """
        Deletes this message, removing it from any labels (only callable by sync)
        """
        self.labels.clear()

        self.is_active = False
        self.modified_on = now()
        self.save(update_fields=("is_active", "modified_on"))

    def label(self, *labels):
        """
        Adds the given labels to this message
        """
        from casepro.profiles.models import Notification
        from casepro.statistics.models import DailyCount, datetime_to_date

        existing_label_ids = Labelling.objects.filter(message=self, label__in=labels).values_list("label", flat=True)
        add_labels = [l for l in labels if l.id not in existing_label_ids]
        new_labellings = [Labelling.create(l, self) for l in add_labels]
        Labelling.objects.bulk_create(new_labellings)

        day = datetime_to_date(self.created_on, self.org)
        for label in add_labels:
            DailyCount.record_item(day, DailyCount.TYPE_INCOMING, label)

        # notify all users who watch these labels
        for watcher in set(User.objects.filter(watched_labels__in=labels)):
            Notification.new_message_labelling(self.org, watcher, self)

    def unlabel(self, *labels):
        """
        Removes the given labels from this message
        """
        from casepro.statistics.models import DailyCount, datetime_to_date

        existing_labellings = Labelling.objects.filter(message=self, label__in=labels).select_related("label")

        day = datetime_to_date(self.created_on, self.org)
        for labelling in existing_labellings:
            DailyCount.record_removal(day, DailyCount.TYPE_INCOMING, labelling.label)

        Labelling.objects.filter(id__in=[l.id for l in existing_labellings]).delete()

    def clear_labels(self):
        """
        Removes all labels from this message
        """
        from casepro.statistics.models import DailyCount, datetime_to_date

        day = datetime_to_date(self.created_on, self.org)
        for label in self.labels.all():
            DailyCount.record_removal(day, DailyCount.TYPE_INCOMING, label)

        Labelling.objects.filter(message=self).delete()

    def update_labels(self, user, labels):
        """
        Updates this message's labels to match the given set, creating label and unlabel actions as necessary
        """
        with self.lock(self.org, self.backend_id):
            current_labels = set(self.labels.all())

            add_labels = [l for l in labels if l not in current_labels]
            rem_labels = [l for l in current_labels if l not in labels]

            for label in add_labels:
                self.bulk_label(self.org, user, [self], label)
            for label in rem_labels:
                self.bulk_unlabel(self.org, user, [self], label)

    def user_lock(self, user):
        """
        Marks this message as locked by a user
        """
        self.locked_on = now()
        self.locked_by = user
        self.modified_on = now()
        self.save(update_fields=("locked_on", "locked_by", "modified_on"))

    def user_unlock(self):
        """
        Marks this message as no longer locked by a user
        """
        self.locked_on = None
        self.locked_by = None
        self.modified_on = now()
        self.save(update_fields=("locked_on", "locked_by", "modified_on"))

    @staticmethod
    def bulk_flag(org, user, messages):
        messages = list(messages)
        if messages:
            org.incoming_messages.filter(org=org, pk__in=[m.pk for m in messages]).update(
                is_flagged=True, modified_on=now()
            )

            org.get_backend().flag_messages(org, messages)

            MessageAction.create(org, user, messages, MessageAction.FLAG)

    @staticmethod
    def bulk_unflag(org, user, messages):
        messages = list(messages)
        if messages:
            org.incoming_messages.filter(org=org, pk__in=[m.pk for m in messages]).update(
                is_flagged=False, modified_on=now()
            )

            org.get_backend().unflag_messages(org, messages)

            MessageAction.create(org, user, messages, MessageAction.UNFLAG)

    @staticmethod
    def bulk_label(org, user, messages, label):
        messages = list(messages)
        if messages:
            for msg in messages:
                msg.label(label)

            org.incoming_messages.filter(org=org, pk__in=[m.pk for m in messages]).update(modified_on=now())

            if label.is_synced:
                org.get_backend().label_messages(org, messages, label)

            MessageAction.create(org, user, messages, MessageAction.LABEL, label)

    @staticmethod
    def bulk_unlabel(org, user, messages, label):
        messages = list(messages)
        if messages:
            for msg in messages:
                msg.unlabel(label)

            org.incoming_messages.filter(org=org, pk__in=[m.pk for m in messages]).update(modified_on=now())

            if label.is_synced:
                org.get_backend().unlabel_messages(org, messages, label)

            MessageAction.create(org, user, messages, MessageAction.UNLABEL, label)

    @staticmethod
    def bulk_archive(org, user, messages):
        messages = list(messages)
        if messages:
            org.incoming_messages.filter(org=org, pk__in=[m.pk for m in messages]).update(
                is_archived=True, modified_on=now()
            )

            org.get_backend().archive_messages(org, messages)

            MessageAction.create(org, user, messages, MessageAction.ARCHIVE)

    @staticmethod
    def bulk_restore(org, user, messages):
        messages = list(messages)
        if messages:
            org.incoming_messages.filter(org=org, pk__in=[m.pk for m in messages]).update(
                is_archived=False, modified_on=now()
            )

            org.get_backend().restore_messages(org, messages)

            MessageAction.create(org, user, messages, MessageAction.RESTORE)

    def as_json(self):
        """
        Prepares this message for JSON serialization
        """
        return {
            "id": self.backend_id,
            "contact": self.contact.as_json(full=False),
            "text": self.text,
            "time": self.created_on,
            "labels": [l.as_json(full=False) for l in self.labels.all()],
            "flagged": self.is_flagged,
            "archived": self.is_archived,
            "flow": self.type == self.TYPE_FLOW,
            "case": self.case.as_json(full=False) if self.case else None,
        }

    def __str__(self):
        return self.text if self.text else self.pk class MessageAction(models.Model):
    """
    An action performed on a set of messages
    """

    FLAG = "F"
    UNFLAG = "N"
    LABEL = "L"
    UNLABEL = "U"
    ARCHIVE = "A"
    RESTORE = "R"

    ACTION_CHOICES = (
        (FLAG, _("Flag")),
        (UNFLAG, _("Un-flag")),
        (LABEL, _("Label")),
        (UNLABEL, _("Remove Label")),
        (ARCHIVE, _("Archive")),
        (RESTORE, _("Restore")),
    )

    org = models.ForeignKey(
        Org, verbose_name=_("Organization"), related_name="message_actions", on_delete=models.PROTECT
    )

    messages = models.ManyToManyField(Message, related_name="actions")

    action = models.CharField(max_length=1, choices=ACTION_CHOICES)

    created_by = models.ForeignKey(User, related_name="message_actions", on_delete=models.PROTECT)

    created_on = models.DateTimeField(auto_now_add=True)

    label = models.ForeignKey(Label, null=True, on_delete=models.PROTECT)

    @classmethod
    def create(cls, org, user, messages, action, label=None):
        action_obj = MessageAction.objects.create(org=org, action=action, created_by=user, label=label)
        action_obj.messages.add(*messages)
        return action_obj

    def as_json(self):
        return {
            "id": self.pk,
            "action": self.action,
            "created_by": self.created_by.as_json(full=False),
            "created_on": self.created_on,
            "label": self.label.as_json() if self.label else None,
        } class MessageExport(BaseSearchExport):
    """
    An export of messages
    """

    directory = "message_exports"
    download_view = "msgs.messageexport_read"

    def get_search(self):
        search = super(MessageExport, self).get_search()
        search["folder"] = MessageFolder[search["folder"]]
        return search

    def render_search(self, book, search):
        from casepro.contacts.models import Field

        base_fields = ["Time", "Message ID", "Flagged", "Labels", "Text", "Contact"]
        contact_fields = Field.get_all(self.org, visible=True)
        all_fields = base_fields + [f.label for f in contact_fields]

        # load all messages to be exported
        items = Message.search(self.org, self.created_by, search, all=True).prefetch_related(
            "contact", "labels", "case__assignee", "case__user_assignee"
        )

        def add_sheet(num):
            sheet = book.add_sheet(str(_("Messages %d" % num)))
            self.write_row(sheet, 0, all_fields)
            return sheet

        sheet = None
        sheet_number = 0
        row = 1
        for item in items:
            if not sheet or row > self.MAX_SHEET_ROWS:
                sheet_number += 1
                sheet = add_sheet(sheet_number)
                row = 1

            values = [
                item.created_on,
                item.backend_id,
                item.is_flagged,
                ", ".join([l.name for l in item.labels.all()]),
                item.text[:32767],  # can't save cell content longer than this limit
                item.contact.uuid,
            ]

            fields = item.contact.get_fields()
            for field in contact_fields:
                values.append(fields.get(field.key, ""))

            self.write_row(sheet, row, values)
            row += 1 class MessageFolder(Enum):
    inbox = 1
    flagged = 2
    flagged_with_archived = 3
    archived = 4
    unlabelled = 5 class Outgoing(models.Model):
    """
    An outgoing message (i.e. broadcast) sent by a user
    """

    BULK_REPLY = "B"
    CASE_REPLY = "C"
    FORWARD = "F"

    ACTIVITY_CHOICES = ((BULK_REPLY, "Bulk Reply"), (CASE_REPLY, "Case Reply"), (FORWARD, "Forward"))
    REPLY_ACTIVITIES = (BULK_REPLY, CASE_REPLY)

    TIMELINE_TYPE = "O"

    SEARCH_BY_TEXT_DAYS = 90

    org = models.ForeignKey(Org, related_name="outgoing_messages", on_delete=models.PROTECT)

    partner = models.ForeignKey("cases.Partner", null=True, related_name="outgoing_messages", on_delete=models.PROTECT)

    activity = models.CharField(max_length=1, choices=ACTIVITY_CHOICES)

    text = models.TextField(max_length=800)

    backend_broadcast_id = models.IntegerField(null=True)

    contact = models.ForeignKey(
        Contact, null=True, related_name="outgoing_messages", on_delete=models.PROTECT
    )  # used for case and bulk replies

    urn = models.CharField(max_length=255, null=True)  # used for forwards

    reply_to = models.ForeignKey(Message, null=True, related_name="replies", on_delete=models.PROTECT)

    case = models.ForeignKey("cases.Case", null=True, related_name="outgoing_messages", on_delete=models.PROTECT)

    created_by = models.ForeignKey(User, related_name="outgoing_messages", on_delete=models.PROTECT)

    created_on = models.DateTimeField(default=now)

    @classmethod
    def create_bulk_replies(cls, org, user, text, messages):
        if not messages:
            raise ValueError("Must specify at least one message to reply to")

        replies = []
        for incoming in messages:
            reply = cls._create(org, user, cls.BULK_REPLY, text, incoming, contact=incoming.contact, push=False)
            replies.append(reply)

        # push together as a single broadcast
        org.get_backend().push_outgoing(org, replies, as_broadcast=True)

        return replies

    @classmethod
    def create_case_reply(cls, org, user, text, case):
        # will be a reply to the last message from the contact
        last_incoming = case.incoming_messages.order_by("-created_on").first()

        return cls._create(org, user, cls.CASE_REPLY, text, last_incoming, contact=case.contact, case=case)

    @classmethod
    def create_forwards(cls, org, user, text, urns, original_message):
        forwards = []
        for urn in urns:
            forwards.append(cls._create(org, user, cls.FORWARD, text, original_message, urn=urn, push=False))

        # push together as a single broadcast
        org.get_backend().push_outgoing(org, forwards, as_broadcast=True)

        return forwards

    @classmethod
    def _create(cls, org, user, activity, text, reply_to, contact=None, urn=None, case=None, push=True):
        if not text:
            raise ValueError("Message text cannot be empty")
        if not contact and not urn:  # pragma: no cover
            raise ValueError("Message must have a recipient")

        msg = cls.objects.create(
            org=org,
            partner=user.get_partner(org),
            activity=activity,
            text=text,
            contact=contact,
            urn=urn,
            reply_to=reply_to,
            case=case,
            created_by=user,
        )

        if push:
            org.get_backend().push_outgoing(org, [msg])

        return msg

    @classmethod
    def get_replies(cls, org):
        return org.outgoing_messages.filter(activity__in=cls.REPLY_ACTIVITIES)

    @classmethod
    def search(cls, org, user, search):
        text = search.get("text")
        contact_id = search.get("contact")

        queryset = org.outgoing_messages.all()

        partner = user.get_partner(org)
        if partner:
            queryset = queryset.filter(partner=partner)

        if text:
            queryset = queryset.filter(
                text__icontains=text, created_on__gt=now() - timedelta(days=cls.SEARCH_BY_TEXT_DAYS)
            )

        if contact_id:
            queryset = queryset.filter(contact__pk=contact_id)

        queryset = queryset.prefetch_related("partner", "contact", "case__assignee", "created_by__profile")

        return queryset.order_by("-created_on")

    @classmethod
    def search_replies(cls, org, user, search):
        partner_id = search.get("partner")
        after = search.get("after")
        before = search.get("before")

        queryset = cls.get_replies(org)

        user_partner = user.get_partner(org)
        if user_partner:
            queryset = queryset.filter(partner=user_partner)

        if partner_id:
            queryset = queryset.filter(partner__pk=partner_id)

        if after:
            queryset = queryset.filter(created_on__gte=after)
        if before:
            queryset = queryset.filter(created_on__lte=before)

        queryset = queryset.select_related("contact", "case__assignee", "created_by__profile")
        queryset = queryset.prefetch_related("reply_to__labels")

        return queryset.order_by("-created_on")

    def is_reply(self):
        return self.activity in self.REPLY_ACTIVITIES

    def get_sender(self):
        """
        Convenience method for accessing created_by since it can be null on transient instances returned from
        Backend.fetch_contact_messages
        """
        try:
            return self.created_by
        except User.DoesNotExist:
            return None

    def as_json(self):
        """
        Prepares this outgoing message for JSON serialization
        """
        return {
            "id": self.pk,
            "contact": self.contact.as_json(full=False) if self.contact else None,
            "urn": self.urn,
            "text": self.text,
            "time": self.created_on,
            "case": self.case.as_json(full=False) if self.case else None,
            "sender": self.get_sender().as_json(full=False) if self.get_sender() else None,
        }

    def __str__(self):
        return self.text class OutgoingFolder(Enum):
    sent = 1 class ReplyExport(BaseSearchExport):
    """
    An export of replies
    """

    directory = "reply_exports"
    download_view = "msgs.replyexport_read"

    def render_search(self, book, search):
        base_fields = [
            "Sent On",
            "User",
            "Message",
            "Delay",
            "Reply to",
            "Flagged",
            "Case Assignee",
            "Labels",
            "Contact",
        ]
        contact_fields = Field.get_all(self.org, visible=True)
        all_fields = base_fields + [f.label for f in contact_fields]

        # load all messages to be exported
        items = Outgoing.search_replies(self.org, self.created_by, search)

        def add_sheet(num):
            sheet = book.add_sheet(str(_("Replies %d" % num)))
            self.write_row(sheet, 0, all_fields)
            return sheet

        sheet = None
        sheet_number = 0
        row = 1
        for item in items:
            if not sheet or row > self.MAX_SHEET_ROWS:
                sheet_number += 1
                sheet = add_sheet(sheet_number)
                row = 1

            values = [
                item.created_on,
                item.created_by.email,
                item.text,
                timesince(item.reply_to.created_on, now=item.created_on),
                item.reply_to.text,
                item.reply_to.is_flagged,
                item.case.assignee.name if item.case else "",
                ", ".join([l.name for l in item.reply_to.labels.all()]),
                item.contact.uuid,
            ]

            fields = item.contact.get_fields()
            for field in contact_fields:
                values.append(fields.get(field.key, ""))

            self.write_row(sheet, row, values)
            row += 1 @shared_task
def faq_csv_import(org_id, task_id):  # pragma: no cover
    task = ImportTask.objects.get(id=task_id)

    org = Org.objects.get(id=org_id)

    task.task_id = faq_csv_import.request.id
    task.log("Started import at %s" % timezone.now())
    task.log("--------------------------------")
    task.save()

    try:
        with transaction.atomic() and open(task.csv_file.path) as csv_file:  # transaction prevents partial csv import
            # Load csv into Dict
            records = csv.DictReader(csv_file)
            lines = 0

            for line in records:
                lines += 1
                # Get or create parent Language object
                parent_lang = line["Parent Language"]

                # Get label objects
                labels = get_labels(task, org, line["Labels"])

                # Create parent FAQ
                parent_faq = FAQ.create(org, line["Parent Question"], line["Parent Answer"], parent_lang, None, labels)

                # Start creation of translation FAQs
                # get a list of the csv keys
                keys = list(line)
                # remove non-translation keys
                parent_keys = ["Parent Question", "Parent Language", "Parent Answer", "Parent ID", "Labels"]
                [keys.remove(parent_key) for parent_key in parent_keys]
                # get a set of unique translation language codes
                lang_codes = set()
                for key in keys:
                    lang_code, name = key.split(" ")
                    lang_codes.add(lang_code)
                # Loop through for each translation
                for lang_code in lang_codes:
                    # Create translation FAQ
                    FAQ.create(
                        org,
                        line["%s Question" % lang_code],
                        line["%s Answer" % lang_code],
                        lang_code,
                        parent_faq,
                        labels,
                    )

            task.save()
            task.log("Import finished at %s" % timezone.now())
            task.log("%d FAQ(s) added." % lines)

    except Exception as e:
        if not settings.TESTING:
            traceback.print_exc(e)

        task.log("\nError: %s\n" % e)

        raise e

    return task @org_task("message-handle", lock_timeout=12 * 60 * 60)
def handle_messages(org):
    backend = org.get_backend()

    case_replies = []
    num_rules_matched = 0

    # fetch all unhandled messages who now have full contacts
    unhandled = Message.get_unhandled(org).filter(contact__is_stub=False)
    unhandled = list(unhandled.select_related("contact").prefetch_related("contact__groups"))

    if unhandled:
        rules = Rule.get_all(org)
        rule_processor = Rule.BatchProcessor(org, rules)

        for msg in unhandled:
            open_case = Case.get_open_for_contact_on(org, msg.contact, msg.created_on)

            # only apply rules if there isn't a currently open case for this contact
            if open_case:
                open_case.add_reply(msg)

                case_replies.append(msg)
            else:
                rules_matched, actions_deferred = rule_processor.include_messages(msg)
                num_rules_matched += rules_matched

        # archive messages which are case replies on the backend
        if case_replies:
            backend.archive_messages(org, case_replies)

        rule_processor.apply_actions()

        # mark all of these messages as handled
        Message.objects.filter(pk__in=[m.pk for m in unhandled]).update(is_handled=True, modified_on=timezone.now())

    return {"handled": len(unhandled), "rules_matched": num_rules_matched, "case_replies": len(case_replies)} @org_task("message-pull", lock_timeout=2 * 60 * 60)
def pull_messages(org, since, until, prev_results):
    """
    Pulls new unsolicited messages for an org
    """
    backend = org.get_backend()

    # if we're running for the first time, then we'll fetch back to 1 hour ago
    if not since:
        since = until - timedelta(hours=1)

    labels_created, labels_updated, labels_deleted, ignored = backend.pull_labels(org)

    def progress(num):  # pragma: no cover
        logger.debug(f" > Synced {num} messages for org #{org.id}")

    cursor = None

    if prev_results:
        prev_resume = prev_results["messages"].get("resume")
        prev_until = prev_results["messages"].get("until")

        if prev_resume:
            cursor = prev_resume["cursor"]
            since = iso8601.parse_date(prev_resume["since"])
            until = iso8601.parse_date(prev_resume["until"])

            logger.warn(f"Resuming previous incomplete sync for #{org.id}")
        elif prev_until:
            since = iso8601.parse_date(prev_until)

    msgs_created, msgs_updated, msgs_deleted, _, new_cursor = backend.pull_messages(
        org, since, until, progress_callback=progress, resume_cursor=cursor
    )

    msgs_results = {"created": msgs_created, "updated": msgs_updated, "deleted": msgs_deleted}

    if new_cursor:
        msgs_results["resume"] = {"cursor": new_cursor, "since": since.isoformat(), "until": until.isoformat()}
    else:
        msgs_results["until"] = until.isoformat()

    return {
        "labels": {"created": labels_created, "updated": labels_updated, "deleted": labels_deleted},
        "messages": msgs_results,
    } @shared_task
def trim_old_messages():
    """
    Task to delete old messages which haven't been used in a case or labelled
    """

    # if setting has been configured, don't delete anything
    if not settings.TRIM_OLD_MESSAGES_DAYS:
        return

    trim_older = timezone.now() - timedelta(days=settings.TRIM_OLD_MESSAGES_DAYS)
    start = timezone.now()
    num_deleted = 0

    while True:
        msg_ids = list(
            Message.objects.filter(has_labels=False, is_flagged=False, case=None, created_on__lt=trim_older)
            .values_list("id", flat=True)
            .order_by("id")[:1000]
        )

        if not msg_ids:  # nothing left to trim
            break

        with transaction.atomic():
            # clear any reply-to foreign keys on outgoing messages
            Outgoing.objects.filter(reply_to_id__in=msg_ids).update(reply_to=None)

            # delete any notifications for these messages
            Notification.objects.filter(message_id__in=msg_ids).delete()

            # delete any references to these messages in message actions
            MessageAction.messages.through.objects.filter(message_id__in=msg_ids).delete()

            # finally delete the actual messages
            Message.objects.filter(id__in=msg_ids).delete()

        num_deleted += len(msg_ids)

        # task is only allowed to run for a certain amount of time
        if (timezone.now() - start).total_seconds() > TRIM_TASK_MAX_SECONDS:  # pragma: no cover
            break

    logger.info(f"Trimmed {num_deleted} messages older than {trim_older.isoformat()}")
from datetime import datetime, timedelta
from unittest.mock import ANY, call, patch
from dash.orgs.models import TaskState
from dateutil.relativedelta import relativedelta
from temba_client.utils import format_iso8601
from django.core.files.uploadedfile import SimpleUploadedFile
from django.test.utils import override_settings
from django.urls import reverse
from django.utils import timezone
from django.utils.timezone import now
from casepro.contacts.models import Contact
from casepro.msgs.views import ImportTask
from casepro.profiles.models import Notification
from casepro.rules.models import ContainsTest, FieldTest, GroupsTest, Quantifier, WordCountTest
from casepro.statistics.tasks import squash_counts
from casepro.test import BaseCasesTest
from .models import (
    FAQ,
    Label,
    Labelling,
    Message,
    MessageAction,
    MessageExport,
    MessageFolder,
    Outgoing,
    OutgoingFolder,
    ReplyExport,
)
from .tasks import faq_csv_import, handle_messages, pull_messages, trim_old_messages





faq_good_import = b"""Parent ID,Parent Language,Parent Question,Parent Answer,Labels,afr ID,afr Question,afr Answer,bla ID,bla Question,bla Answer
,eng,Can I drink tea while pregnant?,"Yes, but avoid too much caffeine","Tea, Pregnancy",,Kan ek tee drink tydens swangerskap?,"Ja, maar beperk jou kaffein inname",,Xtea Xpregnant?,Xyes
,eng,What is Aids?,Acquired immune deficiency syndrome,AIDS,,Wat is Vigs?,Verworwe immuniteitsgebreksindroom,,Xaids?,Xaids
,eng,Do you like tea?,Yes,Tea,,Hou jy van tee?,Ja,,Xtea?,Xyes
"""  # noqa


class LabelTest(BaseCasesTest):
    @patch("casepro.test.TestBackend.push_label")
    def test_save(self, mock_push_label):
        # create un-synced label

        tests = [ContainsTest(["ebola", "fever"], Quantifier.ALL), GroupsTest([self.reporters], Quantifier.ANY)]