class GlossSearchForm(forms.ModelForm):
    # Translators: GlossSearchForm label
    dataset = forms.ModelMultipleChoiceField(label=_('Dataset'), queryset=Dataset.objects.all(), required=False)
    # Translators: GlossSearchForm label
    signlanguage = forms.ModelMultipleChoiceField(label=_('Sign language'), queryset=SignLanguage.objects.all(),
                                                  required=False)

    search = forms.CharField(label=_("Search"))
    # Translators: GlossSearchForm label
    gloss = forms.CharField(label=_("Gloss"))
    # Translators: GlossSearchForm label
    idgloss_en = forms.CharField(label=_("Gloss in English"))
    # Translators: GlossSearchForm label
    keyword = forms.CharField(label=_('Translations'))
    # Translators: GlossSearchForm label
    trans_lang = forms.ModelChoiceField(required=False, empty_label=_('Choose language'), queryset=Language.objects.all())

    try:
        qs = AllowedTags.objects.get(content_type=ContentType.objects.get_for_model(Gloss)).allowed_tags.all()
    except (ObjectDoesNotExist, OperationalError, ProgrammingError):
        qs = Tag.objects.all()
    tags = forms.ModelMultipleChoiceField(queryset=qs, required=False)
    nottags = forms.ModelMultipleChoiceField(queryset=qs)

    published = forms.BooleanField(label=_('Is published'), required=False)

    # Translators: GlossSearchForm label
    hasvideo = forms.BooleanField(label=_('Has videos'), required=False)
    hasnovideo = forms.BooleanField(label=_('No videos'), required=False)
    multiplevideos = forms.BooleanField(label=_('Multiple videos'), required=False)

    # These have been disabled until they are later needed
    # TODO: To enable these, uncomment them.
    """
    # Translators: GlossSearchForm label
    relation = forms.CharField(
        label=_('Search for gloss of related signs'), widget=forms.TextInput(attrs=ATTRS_FOR_FORMS))
    # Translators: GlossSearchForm label
    relationToForeignSign = forms.CharField(
        label=_('Search for gloss of foreign signs'), widget=forms.TextInput(attrs=ATTRS_FOR_FORMS))
    # Translators: GlossSearchForm label
    morpheme = forms.CharField(
        label=_('Search for gloss with this as morpheme'), widget=forms.TextInput(attrs=ATTRS_FOR_FORMS))

    # Translators: GlossSearchForm label
    phonology_other = forms.CharField(
        label=_('Phonology other'), widget=forms.TextInput())

    # Translators: GlossSearchForm label
    hasRelationToForeignSign = forms.ChoiceField(label=_('Related to foreign sign or not'), choices=[
        (0, '---------'), (1, _('Yes')), (2, _('No'))], widget=forms.Select(attrs=ATTRS_FOR_FORMS))
    # Translators: GlossSearchForm label
    hasRelation = forms.ChoiceField(
        label=_('Type of relation'), choices=RELATION_ROLE_CHOICES, widget=forms.Select(attrs=ATTRS_FOR_FORMS))
    # Translators: GlossSearchForm label
    hasMorphemeOfType = forms.ChoiceField(
        label=_('Has morpheme type'), choices=MORPHEME_ROLE_CHOICES, widget=forms.Select(attrs=ATTRS_FOR_FORMS))

    # Translators: GlossSearchForm label
    repeated_movement = forms.ChoiceField(
        label=_('Repeating Movement'), choices=NULLBOOLEANCHOICES)
    # ,widget=forms.Select(attrs=ATTRS_FOR_FORMS));
    # Translators: GlossSearchForm label
    alternating_movement = forms.ChoiceField(
        label=_('Alternating Movement'), choices=NULLBOOLEANCHOICES)
    """

    class Meta:
        ATTRS_FOR_FORMS = {'class': 'form-control'}

        model = Gloss
        fields = ('idgloss', 'idgloss_en', 'dialect', 'strong_handshape', 'weak_handshape', 'location',
                  'handedness', 'notes', 'relation_between_articulators', 'absolute_orientation_palm',
                  'absolute_orientation_fingers', 'relative_orientation_movement', 'relative_orientation_location',
                  'orientation_change', 'handshape_change', 'repeated_movement', 'alternating_movement',
                  'movement_shape', 'movement_direction', 'movement_manner', 'contact_type', 'mouth_gesture',
                  'mouthing', 'phonetic_variation', 'iconic_image', 'named_entity', 'semantic_field',
                  'number_of_occurences',) class TagsAddForm(forms.Form):
    """Form to add a new tags to a gloss"""
    try:
        qs = AllowedTags.objects.get(content_type=ContentType.objects.get_for_model(Gloss)).allowed_tags.all()
    except (ObjectDoesNotExist, OperationalError, ProgrammingError):
        qs = Tag.objects.all()
    tags = forms.ModelMultipleChoiceField(label=_('Tags'), queryset=qs, to_field_name='name') class GlossRelationForm(forms.Form):
    dataset = forms.ModelChoiceField(label=_("Lexicon"), queryset=Dataset.objects.all(), empty_label=None)
    source = forms.CharField(widget=forms.HiddenInput())
    target = forms.CharField(label=_("Gloss"), widget=forms.TextInput(attrs={'class': 'glossrelation-autocomplete'}))
    try:
        qs = AllowedTags.objects.get(content_type=ContentType.objects.get_for_model(GlossRelation)).allowed_tags.all()
    except (ObjectDoesNotExist, OperationalError, ProgrammingError):
        qs = Tag.objects.all()
    tag = forms.ModelChoiceField(label=_("Relation type:"),
                                 queryset=qs,
                                 required=True, to_field_name='name',
                                 widget=forms.Select(attrs={'class': 'form-control'}))
    delete = forms.IntegerField(required=False, widget=forms.HiddenInput()) class RelationForm(forms.ModelForm):
    # Translators: RelationForm label
    sourceid = forms.CharField(label=_('Source Gloss'))
    # Translators: RelationForm label
    targetid = forms.CharField(label=_('Target Gloss'))
    # Note that to_field_name has to be unique!
    role = forms.ModelChoiceField(label=_('Type'), queryset=FieldChoice.objects.filter(field='MorphologyType'),
                                  to_field_name='machine_value', empty_label=None,
                                  widget=forms.Select(attrs=ATTRS_FOR_FORMS))

    class Meta:
        model = Relation
        fields = ['role']
        widgets = {
            'role': forms.Select(attrs={'class': 'form-control'}),
        } class MorphologyForm(forms.ModelForm):
    # Translators: MorphologyForm label
    parent_gloss_id = forms.CharField(label=_('Parent Gloss'))
    # role = forms.ChoiceField(label=_('Type'), choices=build_choice_list(
    #    'MorphologyType'), widget=forms.Select(attrs=ATTRS_FOR_FORMS))

    # Note that to_field_name has to be unique!
    role = forms.ModelChoiceField(label=_('Type'), queryset=FieldChoice.objects.filter(field='MorphologyType'),
                                  to_field_name='machine_value', empty_label=None,
                                  widget=forms.Select(attrs=ATTRS_FOR_FORMS))

    # role = forms.ChoiceField(label=_('Type'), widget=forms.Select(attrs=ATTRS_FOR_FORMS))
    # Translators: MorphologyForm label
    morpheme_id = forms.CharField(label=_('Morpheme'))

    class Meta:
        model = MorphologyDefinition
        fields = ['role'] class GlossRelationSearchForm(forms.Form):
    # Translators: GlossSearchForm label
    dataset = forms.ModelMultipleChoiceField(label=_('Dataset'), queryset=Dataset.objects.all(), required=False)
    search = forms.CharField(label=_("Search"))
    # Translators: GlossSearchForm label
    source = forms.CharField(label=_("Source Gloss"))
    # Translators: GlossSearchForm label
    target = forms.CharField(label=_("Target Gloss"))

    try:
        qs = AllowedTags.objects.get(content_type=ContentType.objects.get_for_model(GlossRelation)).allowed_tags.all()
    except (ObjectDoesNotExist, OperationalError, ProgrammingError):
        qs = Tag.objects.all()
    tags = forms.ModelMultipleChoiceField(queryset=qs, required=False, label=_("Relation type"))

    class Meta:
        ATTRS_FOR_FORMS = {'class': 'form-control'} class Gloss(models.Model):
    class Meta:
        unique_together = (("idgloss", "dataset"),)
        verbose_name = _('Gloss')
        verbose_name_plural = _('Glosses')
        ordering = ['idgloss']
        permissions = (
            # Translators: Gloss permissions
            ('update_video', _("Can Update Video")),
            # Translators: Gloss permissions
            ('search_gloss', _('Can Search/View Full Gloss Details')),
            # Translators: Gloss permissions
            ('export_csv', _('Can export sign details as CSV')),
            ('import_csv', _('Can import glosses from a CSV file')),
            # Translators: Gloss permissions
            ('view_advanced_properties', _('Include all properties in sign detail view')),
            # Translators: Gloss permissions
            ('publish_gloss', _('Can publish and unpublish Glosses')),
        )
    # ### Fields ###
    #: Boolean: Is this Gloss published in the public interface?
    published = models.BooleanField(_("Published"), default=False,
                                    help_text=_("Publish this gloss in the public gloss list"))
    #: Boolean: Exclude this gloss from all ELAN externally controlled vocabularies (ECV)?
    exclude_from_ecv = models.BooleanField(_("Exclude from ECV"), default=False,
                                           help_text=_("Exclude from ELAN externally controlled vocabularies (ECV)"))
    #: The Dataset (Lexicon) this Gloss is part of.
    dataset = models.ForeignKey("Dataset", verbose_name=_("Glosses dataset"),
                                help_text=_("Dataset a gloss is part of"), on_delete=models.PROTECT)
    # Translators: Gloss field: idgloss, verbose name
    #: Gloss in Finnish. This is the unique identifying name of the Gloss.
    idgloss = models.CharField(_("Gloss"), max_length=60,
                               # Translators: Help text for Gloss field: idgloss
                               help_text=_("""This is the unique identifying name of a Gloss."""))
    # Translators: Gloss field: idgloss_en (english), verbose name
    #: Gloss in English. This is the English name of the Gloss.
    idgloss_en = models.CharField(_("Gloss in English"), blank=True, max_length=60,
                                  # Translators: Help text for Gloss field: idgloss_en (english)
                                  help_text=_("""This is the English name for the Gloss"""))
    # Translators: Gloss models field: notes, verbose name. Notes/Further information about a Gloss.
    #: Notes about the Gloss.
    notes = models.TextField(_("Notes"), blank=True)

    #: One or more regional dialects that this Gloss is used in.
    dialect = models.ManyToManyField(Dialect, blank=True)

    #: The DateTime when the Gloss was created.
    created_at = models.DateTimeField(auto_now_add=True)
    #: The User who created the Gloss.
    created_by = models.ForeignKey(User, related_name='created_by_user', null=True, on_delete=models.SET_NULL)
    #: The DateTime when the Glosses information was last updated.
    updated_at = models.DateTimeField(auto_now=True)
    #: The User who last updated the Glosses information.
    updated_by = models.ForeignKey(User, related_name='updated_by_user', null=True, on_delete=models.SET_NULL)

    # ### Phonology fields ###
    # Translators: Gloss models field: handedness, verbose name
    handedness = models.ForeignKey('FieldChoice', verbose_name=_("Handedness"), to_field='machine_value',
                                   db_column='handedness', limit_choices_to={'field': 'handedness'},
                                   related_name="handedness", blank=True, null=True, on_delete=models.SET_NULL)
    # Translators: Gloss models field: strong_handshape, verbose name
    strong_handshape = models.ForeignKey('FieldChoice', verbose_name=_("Strong Hand"), to_field='machine_value',
                                         db_column='strong_handshape', limit_choices_to={'field': 'strong_handshape'},
                                         related_name="strong_handshape", blank=True, null=True, on_delete=models.SET_NULL)

    # Translators: Gloss models field: weak_handshape, verbose name
    weak_handshape = models.ForeignKey('FieldChoice', verbose_name=_("Weak Hand"), to_field='machine_value',
                                       db_column='weak_handshape', limit_choices_to={'field': 'weak_handshape'},
                                       related_name="weak_handshape", blank=True, null=True, on_delete=models.SET_NULL)

    # Translators: Gloss models field: location, verbose name
    location = models.ForeignKey('FieldChoice', verbose_name=_("Location"), to_field='machine_value',
                                 db_column='location', limit_choices_to={'field': 'location'}, related_name="location",
                                 blank=True, null=True, on_delete=models.SET_NULL)

    # Translators: Gloss models field: relation_between_articulators, verbose name
    relation_between_articulators = models.ForeignKey('FieldChoice', verbose_name=_("Relation Between Articulators"),
                                                      to_field='machine_value',
                                                      db_column='relation_between_articulators',
                                                      limit_choices_to={'field': 'relation_between_articulators'},
                                                      related_name="relation_between_articulators", blank=True,
                                                      null=True, on_delete=models.SET_NULL)

    # Translators: Gloss models field: absolute_orientation_palm, verbose name
    absolute_orientation_palm = models.ForeignKey('FieldChoice', verbose_name=_("Absolute Orientation: Palm"),
                                                  to_field='machine_value', db_column='absolute_orientation_palm',
                                                  limit_choices_to={'field': 'absolute_orientation_palm'},
                                                  related_name="absolute_orientation_palm", blank=True, null=True,
                                                  on_delete=models.SET_NULL)
    # Translators: Gloss models field: absolute_orientation_fingers, verbose name
    absolute_orientation_fingers = models.ForeignKey('FieldChoice', verbose_name=_("Absolute Orientation: Fingers"),
                                                     to_field='machine_value', db_column='absolute_orientation_fingers',
                                                     limit_choices_to={'field': 'absolute_orientation_fingers'},
                                                     related_name="absolute_orientation_fingers", blank=True, null=True,
                                                     on_delete=models.SET_NULL)

    # Translators: Gloss models field: relative_orientation_movement, verbose name
    relative_orientation_movement = models.ForeignKey('FieldChoice', verbose_name=_("Relative Orientation: Movement"),
                                                      to_field='machine_value',
                                                      db_column='relative_orientation_movement',
                                                      limit_choices_to={'field': 'relative_orientation_movement'},
                                                      related_name="relative_orientation_movement", blank=True,
                                                      null=True, on_delete=models.SET_NULL)
    # Translators: Gloss models field: relative_orientation_location, verbose name
    relative_orientation_location = models.ForeignKey('FieldChoice', verbose_name=_("Relative Orientation: Location"),
                                                      to_field='machine_value',
                                                      db_column='relative_orientation_location',
                                                      limit_choices_to={'field': 'relative_orientation_location'},
                                                      related_name="relative_orientation_location", blank=True,
                                                      null=True, on_delete=models.SET_NULL)
    # Translators: Gloss models field: orientation_change, verbose name
    orientation_change = models.ForeignKey('FieldChoice', verbose_name=_("Orientation Change"),
                                           to_field='machine_value', db_column='orientation_change',
                                           limit_choices_to={'field': 'orientation_change'},
                                           related_name="orientation_change", blank=True, null=True, on_delete=models.SET_NULL)

    # Translators: Gloss models field: handshape_change, verbose name
    handshape_change = models.ForeignKey('FieldChoice', verbose_name=_("Handshape Change"), to_field='machine_value',
                                         db_column='handshape_change', limit_choices_to={'field': 'handshape_change'},
                                         related_name="handshape_change", blank=True, null=True, on_delete=models.SET_NULL)

    # Translators: Gloss models field: repeated_movement, verbose name
    repeated_movement = models.BooleanField(_("Repeated Movement"), null=True, default=False)
    # Translators: Gloss models field: alternating_movement, verbose name
    alternating_movement = models.BooleanField(_("Alternating Movement"), null=True, default=False)

    # Translators: Gloss models field: movement_shape, verbose name
    movement_shape = models.ForeignKey('FieldChoice', verbose_name=_("Movement Shape"), to_field='machine_value',
                                       db_column='movement_shape', limit_choices_to={'field': 'movement_shape'},
                                       related_name="movement_shape", blank=True, null=True, on_delete=models.SET_NULL)
    # Translators: Gloss models field: movement_direction, verbose name
    movement_direction = models.ForeignKey('FieldChoice', verbose_name=_("Movement Direction"),
                                           to_field='machine_value', db_column='movement_direction',
                                           limit_choices_to={'field': 'movement_direction'},
                                           related_name="movement_direction", blank=True, null=True, on_delete=models.SET_NULL)
    # Translators: Gloss models field: movement_manner, verbose name
    movement_manner = models.ForeignKey('FieldChoice', verbose_name=_("Movement Manner"), to_field='machine_value',
                                        db_column='movement_manner', limit_choices_to={'field': 'movement_manner'},
                                        related_name="movement_manner", blank=True, null=True, on_delete=models.SET_NULL)
    # Translators: Gloss models field: contact_type, verbose name
    contact_type = models.ForeignKey('FieldChoice', verbose_name=_("Contact Type"), to_field='machine_value',
                                     db_column='contact_type', limit_choices_to={'field': 'contact_type'},
                                     related_name="contact_type", blank=True, null=True, on_delete=models.SET_NULL)

    # Translators: Gloss models field: phonology_other verbose name
    phonology_other = models.TextField(_("Phonology Other"), null=True, blank=True)

    # Translators: Gloss models field: mouth_gesture, verbose name
    mouth_gesture = models.CharField(_("Mouth Gesture"), max_length=50, blank=True)
    # Translators: Gloss models field: mouthing, verbose name
    mouthing = models.CharField(_("Mouthing"), max_length=50, blank=True)
    # Translators: Gloss models field: phonetic_variation, verbose name
    phonetic_variation = models.CharField(_("Phonetic Variation"), max_length=50, blank=True, )

    # ### Semantic fields
    # Translators: Gloss models field: iconic_image, verbose name
    iconic_image = models.CharField(_("Iconic Image"), max_length=50, blank=True)
    # Translators: Gloss models field: named_entity, verbose name
    named_entity = models.ForeignKey('FieldChoice', verbose_name=_("Named Entity"), to_field='machine_value',
                                     db_column='named_entity', limit_choices_to={'field': 'named_entity'},
                                     related_name="named_entity", blank=True, null=True, on_delete=models.SET_NULL)
    # Translators: Gloss models field: semantic_field, verbose name
    semantic_field = models.ForeignKey('FieldChoice', verbose_name=_("Semantic Field"), to_field='machine_value',
                                       db_column='semantic_field', limit_choices_to={'field': 'semantic_field'},
                                       related_name="semantic_field", blank=True, null=True, on_delete=models.SET_NULL)

    # ### Frequency fields
    # Translators: Gloss models field_ number_of_occurences, verbose name
    number_of_occurences = models.IntegerField(
        _("Number of Occurrences"), null=True, blank=True,
        # Translators: Help text for Gloss models field: number_of_occurences
        help_text=_("Number of occurences in annotation materials"))

    def __str__(self):
        return self.idgloss

    def get_absolute_url(self):
        return self.get_admin_absolute_url()

    def get_admin_absolute_url(self):
        return reverse('dictionary:admin_gloss_view', args=[str(self.id)])

    def get_public_absolute_url(self):
        return reverse('dictionary:public_gloss_view', args=[str(self.id)])

    def field_labels(self):
        """Return the dictionary of field labels for use in a template"""
        d = dict()
        for f in self._meta.fields:
            d[f.name] = self._meta.get_field(f.name).verbose_name

        return d

    def get_translation_languages(self):
        """Returns translation languages that are set for the Dataset of the Gloss."""
        return self.dataset.translation_languages.all()

    def get_translations_for_translation_languages(self):
        """Returns a zipped list of translation languages and translations."""
        translation_list = []
        translation_languages = self.get_translation_languages()
        for language in translation_languages:
            try:
                # Get translations from GlossTranslation object, if it exists for gloss+language.
                translation_list.append(GlossTranslations.objects.get(gloss=self, language=language))
            except GlossTranslations.DoesNotExist:
                # If it doesn't exist, try to get translations from Translation objects.
                translations = Translation.objects.filter(gloss=self, language=language)
                kwd_str = ""
                first = True
                for trans in translations:
                    if first:
                        kwd_str += str(trans.keyword)
                        first = False
                    else:
                        kwd_str += ", " + trans.keyword.text
                translation_list.append(kwd_str)

        return list(zip(translation_languages, translation_list))

    def get_fields(self):
        return [(field.name, field.value_to_string(self)) for field in Gloss._meta.fields]

    @staticmethod
    def get_choice_lists():
        """Return FieldChoices for selected fields in JSON, grouped by field, key=machine_value, value=english_name"""
        # The fields we want to generate choice lists for
        fields = ['handedness', 'location', 'strong_handshape', 'weak_handshape',
                  'relation_between_articulators', 'absolute_orientation_palm', 'absolute_orientation_fingers',
                  'relative_orientation_movement', 'relative_orientation_location', 'handshape_change',
                  'repeated_movement', 'alternating_movement', 'movement_shape', 'movement_direction',
                  'movement_manner', 'contact_type', 'named_entity', 'orientation_change', 'semantic_field']

        qs = FieldChoice.objects.filter(field__in=fields).values('field', 'machine_value', 'english_name')
        # TODO: How about other fields like Morphology? Should we just get all the fields?
        # Group the values by 'field'
        fields_grouped = {k: list(v) for k, v in groupby(qs, key=lambda x: x["field"])}
        field_choices = dict()
        # Construct a dict that has 'machine_value' as key and 'english_name' as value.
        for k, v in fields_grouped.items():
            field_choices[k] = {"_"+str(x['machine_value']): str(x['english_name']) for x in v}
        # Return results in JSON
        return json.dumps(field_choices) class Dataset(models.Model):
    """Dataset/Lexicon of which Glosses are part of."""
    #: A private name for the Dataset. Can include abbrevations not recognizable by the general users.
    name = models.CharField(_("Name"), unique=True, blank=False, null=False, max_length=60)
    #: Public name for the Dataset, intended for users of the public interface.
    public_name = models.CharField(_("Public name"), max_length=60)
    #: Boolean defining whether to show this Dataset in the public interface.
    is_public = models.BooleanField(_("Is public"), default=False, help_text=_("Is this dataset is public or private?"))
    #: Gloss language, which spoken language is used for the gloss name, used in ELAN external controlled vocabulary
    glosslanguage = models.ForeignKey("Language", verbose_name=_("Gloss language"), related_name="glosslanguage",
        on_delete=models.PROTECT, help_text=_("Language that is used for gloss names"))
    #: The Sign Language of the Glosses in this Dataset.
    signlanguage = models.ForeignKey("SignLanguage", verbose_name=_("Sign language"), on_delete=models.PROTECT)
    #: The translation equivalent languages that should be available to the Glosses of this Dataset.
    translation_languages = models.ManyToManyField("Language", verbose_name=_("Translation equivalent languages"),
                                                   help_text=_("These languages are options for translation equivalents."))
    #: A description of the Dataset: who maintains it, what is its purpose, etc.
    description = models.TextField(_("Description"))
    #: The copyright statement for the data in this Dataset, the license used for the videos etc.
    copyright = models.TextField(_("Copyright"))
    #: The admins of this Dataset. Admins receive notifications when a user applies for permissins for the Dataset.
    admins = models.ManyToManyField(User, verbose_name=_("Admins"))

    class Meta:
        permissions = (
            ('access_dataset', _('Access dataset')),
        )
        verbose_name = _('Lexicon')
        verbose_name_plural = _('Lexicons')
        ordering = ['id',]

    def __str__(self):
        return self.name class Translation(models.Model):
    """A translation equivalent of a sign in selected language."""
    #: The Gloss to translate.
    gloss = models.ForeignKey("Gloss", verbose_name=_("Gloss"), on_delete=models.CASCADE)
    #: The written/spoken Language of the translation.
    language = models.ForeignKey("Language", verbose_name=_("Language"), on_delete=models.CASCADE)
    #: The Keyword of the translation, the textual form.
    keyword = models.ForeignKey("Keyword", verbose_name=_("Keyword"), on_delete=models.PROTECT)
    #: The order number of the Translation within a Glosses Translations.
    order = models.IntegerField("Order")

    class Meta:
        unique_together = (("gloss", "language", "keyword"),)
        ordering = ['gloss', 'language', 'order']
        verbose_name = _('Translation equivalent')
        verbose_name_plural = _('Translation equivalents')

    def __str__(self):
        return self.keyword.text class GlossTranslations(models.Model):
    """Store a string representation of translation equivalents of certain Language for a Gloss."""
    #: The Gloss to translate
    gloss = models.ForeignKey("Gloss", verbose_name=_("Gloss"), on_delete=models.CASCADE)
    #: The written/spoken Language of the translations.
    language = models.ForeignKey("Language", verbose_name=_("Language"), on_delete=models.CASCADE)
    #: The fields that contains the translations, a text field.
    translations = models.TextField(blank=True)

    class Meta:
        unique_together = (("gloss", "language"),)
        verbose_name = _('Gloss translation field')
        verbose_name_plural = _('Gloss translation fields')
        ordering = ['language']

    def save(self, *args, **kwargs):
        # Is the object being created
        creating = self._state.adding
        # Remove duplicates and keep the order.
        keywords = self.get_keywords_unique()

        # Get Translation objects for GlossTranslation.gloss, filter according to GlossTranslation.language
        translations = self.gloss.translation_set.filter(language=self.language)
        # Keep the translation objects that have the Keywords that remain.
        translations_to_keep = translations.filter(keyword__text__in=keywords, language=self.language)
        # Delete translations that no longer exist in field GlossTranslations.translations.
        translations.exclude(pk__in=translations_to_keep).delete()

        if len(keywords) < 2 and keywords[0].strip() == "":
            # If the to be saved object has no 'translations'
            if not creating:
                # If the object is being updated
                self.delete()
            # If object is being created with empty 'translations', don't save.
            return

        existing_keywords = Keyword.objects.filter(text__in=keywords)
        for i, keyword_text in enumerate(keywords):
            (keyword, created) = existing_keywords.get_or_create(text=keyword_text)
            try:
                translation = translations_to_keep.get(gloss=self.gloss, language=self.language, keyword=keyword)
            except Translation.DoesNotExist:
                translation = Translation(gloss=self.gloss, language=self.language, keyword=keyword)
            translation.order = i
            translation.save()

        super(GlossTranslations, self).save(*args, **kwargs)

    def get_keywords(self):
        """Returns keywords parsed from self.translations."""
        # Remove number(s) that end with a dot (e.g. '1.') from the 'value'.
        translations_cleaned = re.sub('\d\.', '', str(self.translations))
        # Splitting the remaining string on comma, dot or semicolon. Then strip spaces around the keyword(s).
        keywords = [k.strip() for k in re.split('[,.;]', translations_cleaned)]
        return keywords

    def get_keywords_unique(self):
        """Returns only unique keywords from get_keywords()"""
        return list(OrderedDict.fromkeys(self.get_keywords()))

    def has_duplicates(self):
        keywords_str = self.get_keywords()
        return len(keywords_str) != (len(set(keywords_str)))

    def __str__(self):
        return self.translations class GlossURL(models.Model):
    """URL's for gloss"""
    #: The Gloss the URL belongs to.
    gloss = models.ForeignKey("Gloss", verbose_name=_("Gloss"), on_delete=models.CASCADE)
    #: The URL, a websites address.
    url = models.URLField(max_length=200)

    class Meta:
        verbose_name = _('Gloss URL')
        verbose_name_plural = _('Gloss URLs')

    def __str__(self):
        return self.gloss.idgloss + " - " + self.url class GlossRelation(models.Model):
    """Relation between two glosses"""
    #: The source Gloss of the Relation.
    source = models.ForeignKey(Gloss, related_name="glossrelation_source", on_delete=models.CASCADE)
    #: The target Gloss of the Relation, the Gloss to which the source Gloss related to.
    target = models.ForeignKey(Gloss, related_name="glossrelation_target", on_delete=models.CASCADE)

    def tag(self):
        """The type of the Relation, a Tag."""
        return list(Tag.objects.get_for_object(self))
    tag.short_description = 'Relation type'

    class Meta:
        ordering = ['source']
        verbose_name = _('Gloss relation')
        verbose_name_plural = _('Gloss relations')

    def __str__(self):
        return str(self.target) class RelationToForeignSign(models.Model):
    """Defines a relationship to another sign in another language (often a loan)"""
    #: The source Gloss of the relation.
    gloss = models.ForeignKey("Gloss", on_delete=models.CASCADE)
    # Translators: RelationToForeignSign field verbose name
    #: Boolean: Is this a loan sign?
    loan = models.BooleanField(_("Loan Sign"), default=False)
    # Translators: RelationToForeignSign field verbose name
    #: The language of the related sign.
    other_lang = models.CharField(_("Related Language"), max_length=20)
    # Translators: RelationToForeignSign field verbose name
    #: The name of the Gloss in the related language.
    other_lang_gloss = models.CharField(
        _("Gloss in related language"), max_length=50)

    class Meta:
        ordering = ['gloss', 'loan', 'other_lang', 'other_lang_gloss']
        verbose_name = _('Relation to Foreign Sign')
        verbose_name_plural = _('Relations to Foreign Signs')

    class Admin:
        list_display = ['gloss', 'loan', 'other_lang', 'other_lang_gloss']
        list_filter = ['other_lang']
        search_fields = ['gloss__idgloss']

    def __str__(self):
        return str(self.gloss) + "/" + str(self.other_lang) + ',' + str(self.other_lang_gloss) class Relation(models.Model):  # TODO: Remove
    """A relation between two glosses"""
    source = models.ForeignKey(Gloss, related_name="relation_sources", on_delete=models.CASCADE)
    target = models.ForeignKey(Gloss, related_name="relation_targets", on_delete=models.CASCADE)
    # role = models.CharField(max_length=20, choices=build_choice_list('MorphologyType'))
    role = models.ForeignKey('FieldChoice', to_field='machine_value', db_column='MorphologyType',
                             limit_choices_to={'field': 'MorphologyType'}, blank=True, on_delete=models.CASCADE)
    # antonym, synonym, cf (what's this? - see also), var[b-f]
    # (what's this - variant (XXXa is the stem, XXXb is a variant)

    class Admin:
        list_display = ['source', 'role', 'target']
        search_fields = ['source__idgloss', 'target__idgloss']

    class Meta:
        ordering = ['source']
        verbose_name = _('Relation')
        verbose_name_plural = _('Relations')

    def __str__(self):
        return str(self.source)+' -> ' + str(self.target) class MorphologyDefinition(models.Model):
    """Tells something about morphology of a gloss"""
    parent_gloss = models.ForeignKey(Gloss, related_name="parent_glosses", on_delete=models.CASCADE)
    # role = models.CharField(max_length=5, choices=(build_choice_list('MorphologyType')))
    role = models.ForeignKey('FieldChoice', to_field='machine_value', db_column='MorphologyType',
                             limit_choices_to={'field': 'MorphologyType'}, blank=True, on_delete=models.CASCADE)
    morpheme = models.ForeignKey(Gloss, related_name="morphemes", on_delete=models.CASCADE)

    class Meta:
        verbose_name = _('Morphology definition')
        verbose_name_plural = _('Morphology definitions')

    def __str__(self):
        return str(self.morpheme.idgloss) + ' is ' + str(self.role) + ' of ' + str(self.parent_gloss.idgloss) class GlossVideoForGlossForm(forms.ModelForm):
    """Form for GlossVideo with Gloss."""
    redirect = forms.CharField(widget=forms.HiddenInput, required=False)
    webcam = forms.BooleanField(required=False)

    class Meta:
        model = GlossVideo
        fields = ["title", "videofile", "gloss"] class GlossVideo(models.Model):
    """A video that represents a particular idgloss"""
    #: Descriptive title of the GlossVideo.
    title = models.CharField(_("Title"), blank=True, unique=False, max_length=100,
                             help_text=_("Descriptive name of the video."))
    #: Video file of the GlossVideo.
    videofile = models.FileField(_("Video file"), storage=GlossVideoStorage(),
                                 help_text=_("Video file."))
    #: Poster image of the GlossVideo.
    posterfile = models.FileField(_("Poster file"), upload_to=os.path.join("posters"),
                                  storage=GlossVideoStorage(), blank=True,
                                  help_text=_("Still image representation of the video."), default="")
    #: Boolean: Is this GlossVideo public? Do you want to show it in the public interface, for a published Gloss?
    is_public = models.BooleanField(_("Public"), default=True, help_text="Is this video is public or private?")
    #: The Gloss this GlossVideo belongs to.
    gloss = models.ForeignKey('dictionary.Gloss', verbose_name=_("Gloss"), null=True,
                              help_text=_("The gloss this GlossVideo is related to."), on_delete=models.CASCADE)
    #: The Dataset/Lexicon this GlossVideo is part of.
    dataset = models.ForeignKey('dictionary.Dataset', verbose_name=_("Glossvideo dataset"), null=True,
                                help_text=_("Dataset of a GlossVideo, derived from gloss (if any) or chosen when video "
                                            "was uploaded."), on_delete=models.PROTECT)
    # Translators: GlossVideo: version
    #: Version number of the GlossVideo within Glosses videos.
    version = models.IntegerField(_("Version"), default=0,
                                  help_text=_("A number that represents the order of the Glossvideo's in "
                                              "a gloss page. Smaller number means higher priority."))

    class Meta:
        ordering = ['version']
        verbose_name = _('Gloss video')
        verbose_name_plural = _('Gloss videos')

    def save(self, *args, **kwargs):
        creating = self._state.adding
        if creating:
            # If no title is set, use the filename of the uploaded file.
            if not self.title:
                self.title = self.videofile.name
            # Set version, one higher than highest version.
            self.version = self.next_version()
            super(GlossVideo, self).save(*args, **kwargs)

        # Rename the videofile if object has gloss set, now that the object has a pk.
        if self.gloss:
            # Make sure glossvideo has the same dataset as gloss.
            self.dataset = self.gloss.dataset
            # Rename videofile.
            self.rename_video()
            # Save without args and kwargs.
            super(GlossVideo, self).save()

    def next_version(self):
        """Return a next suitable version number."""
        try:
            return self.gloss.glossvideo_set.order_by('version').last().version + 1
        except AttributeError:
            # If no GlossVideo.gloss, we can set version to 0.
            return 0

    def get_glosses_videos(self):
        """Returns queryset of glosses GlossVideos."""
        try:
            return self.gloss.glossvideo_set.order_by('version')
        except AttributeError:
            return GlossVideo.objects.none()

    def correct_duplicate_versions(self):
        """If glosses glossvideos have duplicate version numbers, reset version numbers."""
        qs = self.get_glosses_videos()
        version_list = qs.order_by('version').values_list('version', flat=True)
        # Check if version_list has duplicates.
        has_duplicates = len(version_list) != len(set(version_list))
        if has_duplicates:
            # If duplicates, set new version numbers.
            for i, vid in enumerate(qs):
                vid.version = i
                vid.save()
        return

    def move_video_version(self, direction):
        """Move video back or forth in glosses videos."""
        qs = self.get_glosses_videos()
        # First correct possible duplicate version numbers.
        self.correct_duplicate_versions()
        # Exclude self from the queryset.
        glosses_videos = qs.exclude(pk=self.pk)
        if direction == "up" and self.version > 0:
            # Move video "up", make its version lower by swapping with video before it.
            swap_video = glosses_videos.filter(version__lte=self.version).last()
            self.version, swap_video.version = swap_video.version, self.version
            self.save(), swap_video.save()
        if direction == "down" and self.version < glosses_videos.last().version:
            # Move video "down", make its version higher by swapping with video after it.
            swap_video = glosses_videos.filter(version__gte=self.version).first()
            self.version, swap_video.version = swap_video.version, self.version
            self.save(), swap_video.save()
        return

    def get_absolute_url(self):
        return self.videofile.url

    def rename_video(self):
        """Rename the video and move the video to correct path if the glossvideo object has a foreignkey to a gloss."""
        storage = self.videofile.storage
        # Do not rename the file if glossvideo doesn't have a gloss.
        if hasattr(self, 'gloss') and self.gloss is not None:
            # Store the old file path, needed for removal later.
            old_file_path = self.videofile.path
            # Create the base filename.
            new_filename = self.create_filename()
            # Get the relative path in media folder.
            full_new_path = storage.get_valid_name(new_filename)
            # Proceed to change the file path if the new path is not equal to old path.
            if not old_file_path == os.path.join(storage.base_location, full_new_path):
                # Save the file into the new path.
                saved_file_path = storage.save(full_new_path, self.videofile)
                # Set the actual file path to videofile.
                self.videofile =  saved_file_path
                if os.path.isfile(old_file_path):
                    # Remove the file from the old path.
                    os.remove(old_file_path)

    def create_filename(self):
        """Returns a correctly named filename"""
        return "{idgloss}-{glosspk}_vid{videopk}{ext}".format(
            idgloss=self.gloss.idgloss, glosspk=self.gloss.pk, videopk=self.pk, ext=self.get_extension()
        )

    def create_poster_filename(self, ext):
        """Returns a preferred filename of posterfile. Ext is the file extension without the dot."""
        if self.gloss:
            return "{idgloss}-{glosspk}_vid{videopk}_poster.{ext}".format(
                idgloss=self.gloss.idgloss, glosspk=self.gloss.pk, videopk=self.pk, ext=ext
            )
        return "{videofilename}.{ext}".format(videofilename=self.videofile.name, ext=ext)

    @staticmethod
    def rename_glosses_videos(gloss):
        """Renames the filenames of selected Glosses videos to match the Gloss name"""
        glossvideos = gloss.glossvideo_set.all().order_by('version')
        for glossvideo in glossvideos:
            glossvideo.save()

    def get_extension(self):
        """Returns videofiles extension."""
        return os.path.splitext(self.videofile.path)[1]

    def has_poster(self):
        """Returns true if the glossvideo has a poster file."""
        if self.posterfile:
            return True
        return False

    def get_videofile_modified_date(self):
        """Return a Datetime object from filesystems last modified time of path."""
        try:
            return datetime.datetime.fromtimestamp(os.path.getmtime(self.videofile.path))
        except FileNotFoundError:
            return None

    def __str__(self):
        return self.videofile.name class CommentTagForm(forms.Form):
    """Form for tags, meant to be used when adding tags to Comments."""
    try:
        qs = AllowedTags.objects.get(content_type=ContentType.objects.get_for_model(Comment)).allowed_tags.all()
    except (ObjectDoesNotExist, OperationalError, ProgrammingError):
        qs = Tag.objects.all()
    tag = forms.ModelChoiceField(queryset=qs, required=False, empty_label="---", to_field_name='name',
                                 widget=forms.Select(attrs={'class': 'form-control'}), label=_lazy('Tag'))
import json
import csv
from os import path as os_path
from django.views.generic.list import ListView
from django.views.generic.detail import DetailView
from django.db.models import Q, Count
from django.db.models.fields import BooleanField
from django.http import HttpResponse, JsonResponse
from django.core.exceptions import PermissionDenied
from django.template.loader import render_to_string
from django.utils.translation import get_language
from django.db.models import Prefetch
from django.db.models import F
from django.contrib.contenttypes.models import ContentType
from django.utils.translation import ugettext as _
from django.shortcuts import get_object_or_404
from collections import defaultdict
from django.contrib import messages
from tagging.models import Tag, TaggedItem
from guardian.shortcuts import get_perms, get_objects_for_user, get_users_with_perms
from reversion.models import Version
from .forms import GlossSearchForm, TagsAddForm, GlossRelationForm, RelationForm, MorphologyForm, \
    GlossRelationSearchForm
from .models import Gloss, Dataset, Translation, GlossTranslations, GlossURL, GlossRelation, RelationToForeignSign, \
    Relation, MorphologyDefinition
from ..video.forms import GlossVideoForGlossForm
from ..video.models import GlossVideo
from ..comments import CommentTagForm
            qs = qs.filter(published=val)

        # If get has both keys hasvideo and hasnovideo, don't use them to query.
        if not ('hasvideo' in get and 'hasnovideo' in get):
            if 'hasvideo' in get and get['hasvideo'] != '':
                val = get['hasvideo'] != 'on'
                qs = qs.filter(glossvideo__isnull=val)

            if 'hasnovideo' in get and get['hasnovideo'] != '':
                val = get['hasnovideo'] == 'on'
                qs = qs.filter(glossvideo__isnull=val)

        # If gloss has multiple GlossVideos
        if 'multiplevideos' in get and get['multiplevideos'] != '' and get['multiplevideos'] == 'on':
            # Include glosses that have more than one GlossVideo
            qs = qs.annotate(videocount=Count('glossvideo')).filter(videocount__gt=1)


        # A list of phonology fieldnames
        fieldnames = ['handedness', 'strong_handshape', 'weak_handshape', 'location', 'relation_between_articulators',
                      'absolute_orientation_palm', 'absolute_orientation_fingers', 'relative_orientation_movement',
                      'relative_orientation_location', 'orientation_change', 'handshape_change', 'repeated_movement',
                      'alternating_movement', 'movement_shape', 'movement_direction', 'movement_manner',
                      'contact_type', 'phonology_other', 'mouth_gesture', 'mouthing', 'phonetic_variation',
                      'iconic_image', 'named_entity', 'semantic_field', 'number_of_occurences', ]

        """These were removed from fieldnames because they are not needed there:
        'idgloss', 'idgloss_en', 'notes',
        """


        # Language and basic property filters
        vals = get.getlist('dialect', [])
        if vals != []:
            qs = qs.filter(dialect__in=vals)

        vals = get.getlist('language', [])
        if vals != []:
            # Get languages from dataset
            qs = qs.filter(dataset__language__in=vals)

        vals = get.getlist('signlanguage', [])
        if vals != []:
            # Get sign languages from dataset
            qs = qs.filter(dataset__signlanguage__in=vals)

        if 'notes' in get and get['notes'] != '':
            qs = qs.filter(notes__icontains=get['notes'])

        # phonology and semantics field filters
        for fieldname in fieldnames:

            if fieldname in get:
                key = fieldname + '__exact'
                val = get[fieldname]

                if isinstance(Gloss._meta.get_field(fieldname), BooleanField):
                    val = {'0': '', '1': None, '2': True, '3': False}[val]

                if val != '':
                    kwargs = {key: val}
                    qs = qs.filter(**kwargs)

        if 'tags' in get and get['tags'] != '':
            vals = get.getlist('tags')

            tags = []
            for t in vals:
                tags.extend(Tag.objects.filter(pk=t))

            # search is an implicit AND so intersection
            tqs = TaggedItem.objects.get_intersection_by_model(Gloss, tags)

            # intersection
            qs = qs & tqs

            # print "J :", len(qs)

        qs = qs.distinct()

        if 'nottags' in get and get['nottags'] != '':
            vals = get.getlist('nottags')

            # print "NOT TAGS: ", vals

            tags = []
            for t in vals:
                tags.extend(Tag.objects.filter(name=t))

            # search is an implicit AND so intersection
            tqs = TaggedItem.objects.get_intersection_by_model(Gloss, tags)

            # print "NOT", tags, len(tqs)
            # exclude all of tqs from qs
            qs = [q for q in qs if q not in tqs]

            # print "K :", len(qs)

        if 'relationToForeignSign' in get and get['relationToForeignSign'] != '':
            relations = RelationToForeignSign.objects.filter(
                other_lang_gloss__icontains=get['relationToForeignSign'])
            potential_pks = [relation.gloss.pk for relation in relations]
            qs = qs.filter(pk__in=potential_pks)

        if 'hasRelationToForeignSign' in get and get['hasRelationToForeignSign'] != '0':

            pks_for_glosses_with_relations = [
                relation.gloss.pk for relation in RelationToForeignSign.objects.all()]
            print(('pks_for_glosses', pks_for_glosses_with_relations))

            # We only want glosses with a relation to a foreign sign
            if get['hasRelationToForeignSign'] == '1':
                qs = qs.filter(pk__in=pks_for_glosses_with_relations)
            # We only want glosses without a relation to a foreign sign
            elif get['hasRelationToForeignSign'] == '2':
                qs = qs.exclude(pk__in=pks_for_glosses_with_relations)

        if 'relation' in get and get['relation'] != '':
            potential_targets = Gloss.objects.filter(
                idgloss__icontains=get['relation'])

            relations = Relation.objects.filter(target__in=potential_targets)