<gh_stars>0
# coding: utf-8
#
# Copyright 2019 The Oppia Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""One-off jobs for validating prod models."""

import collections
import datetime
import itertools
import re

from constants import constants
from core import jobs
from core.domain import activity_domain
from core.domain import collection_domain
from core.domain import collection_services
from core.domain import config_domain
from core.domain import exp_domain
from core.domain import exp_services
from core.domain import fs_domain
from core.domain import recommendations_services
from core.domain import rights_manager
from core.domain import story_domain
from core.domain import story_services
from core.platform import models
import feconf

(
    activity_models, audit_models, base_models,
    collection_models, config_models, email_models,
    exp_models, feedback_models, file_models,
    recommendations_models, story_models,
    user_models,) = (
        models.Registry.import_models([
            models.NAMES.activity, models.NAMES.audit, models.NAMES.base_model,
            models.NAMES.collection, models.NAMES.config, models.NAMES.email,
            models.NAMES.exploration, models.NAMES.feedback, models.NAMES.file,
            models.NAMES.recommendations, models.NAMES.story,
            models.NAMES.user]))
datastore_services = models.Registry.import_datastore_services()

ALLOWED_AUDIO_EXTENSIONS = feconf.ACCEPTED_AUDIO_EXTENSIONS.keys()
ALLOWED_IMAGE_EXTENSIONS = list(itertools.chain.from_iterable(
    feconf.ACCEPTED_IMAGE_FORMATS_AND_EXTENSIONS.values()))
ASSETS_PATH_REGEX = '/exploration/[A-Za-z0-9]{1,12}/assets/'
IMAGE_PATH_REGEX = (
    '%simage/[A-Za-z0-9_]{1,}\\.(%s)' % (
        ASSETS_PATH_REGEX, ('|').join(ALLOWED_IMAGE_EXTENSIONS)))
AUDIO_PATH_REGEX = (
    '%saudio/[A-Za-z0-9_]{1,}\\.(%s)' % (
        ASSETS_PATH_REGEX, ('|').join(ALLOWED_AUDIO_EXTENSIONS)))
FILE_MODELS_REGEX = '(%s|%s)' % (IMAGE_PATH_REGEX, AUDIO_PATH_REGEX)


class BaseModelValidator(object):
    """Base class for validating models."""

    # The dict to store errors found during audit of model.
    errors = collections.defaultdict(list)
    # external_instance_details is keyed by field name. Each value consists
    # of a list of (model class, external_key, external_model_instance)
    # tuples.
    external_instance_details = {}

    @classmethod
    def _get_model_id_regex(cls, unused_item):
        """Returns a regex for model id.

        This method can be overridden by subclasses, if needed.

        Args:
            unused_item: ndb.Model. Entity to validate.

        Returns:
            str. A regex pattern to be followed by the model id.
        """
        return '^[A-Za-z0-9-_]{1,%s}$' % base_models.ID_LENGTH

    @classmethod
    def _validate_model_id(cls, item):
        """Checks whether the id of model matches the regex specified for
        the model.

        Args:
            item: ndb.Model. Entity to validate.
        """
        regex_string = cls._get_model_id_regex(item)
        if not re.compile(regex_string).match(str(item.id)):
            cls.errors['model id check'].append((
                'Entity id %s: Entity id does not match regex pattern') % (
                    item.id))

    @classmethod
    def _get_model_domain_object_instance(cls, unused_item):
        """Returns a domain object instance created from the model.

        This method can be overridden by subclasses, if needed.

        Args:
            unused_item: ndb.Model. Entity to validate.

        Returns:
            *: A domain object to validate.
        """
        return None

    @classmethod
    def _validate_model_domain_object_instances(cls, item):
        """Checks that model instance passes the validation of the domain
        object for model.

        Args:
            item: ndb.Model. Entity to validate.
        """
        model_domain_object_instance = cls._get_model_domain_object_instance(
            item)

        if model_domain_object_instance is None:
            # No domain object exists for this storage model class.
            return

        try:
            model_domain_object_instance.validate()
        except Exception as e:
            cls.errors['domain object check'].append((
                'Entity id %s: Entity fails domain validation with the '
                'error %s' % (item.id, e)))

    @classmethod
    def _get_external_id_relationships(cls, item):
        """Returns a mapping of external id to model class.

        This should be implemented by subclasses.

        Args:
            item: ndb.Model. Entity to validate.

        Returns:
            dict(str, (ndb.Model, list(str)). A dictionary whose keys are
            field names of the model to validate, and whose values are tuples
            that consist of the external model class and list of keys to fetch.
        """
        raise NotImplementedError

    @classmethod
    def _validate_external_id_relationships(cls, item):
        """Check whether the external id properties on the model correspond
        to valid instances.

        Args:
            item: ndb.Model. Entity to validate.
        """
        for field_name, model_class_model_id_model_tuples in (
                cls.external_instance_details.iteritems()):
            for model_class, model_id, model in (
                    model_class_model_id_model_tuples):
                if model is None or model.deleted:
                    cls.errors['%s field check' % field_name].append((
                        'Entity id %s: based on field %s having'
                        ' value %s, expect model %s with id %s but it doesn\'t'
                        ' exist' % (
                            item.id, field_name, model_id,
                            str(model_class.__name__), model_id)))

    @classmethod
    def _fetch_external_instance_details(cls, item):
        """Fetch external models based on _get_external_id_relationships.

        This should be called before we call other _validate methods.

        Args:
            item: ndb.Model. Entity to validate.
        """
        multiple_models_keys_to_fetch = {}
        for field_name_debug, (model_class, keys_to_fetch) in (
                cls._get_external_id_relationships(item).iteritems()):
            multiple_models_keys_to_fetch[field_name_debug] = (
                model_class, keys_to_fetch)
        fetched_model_instances = (
            datastore_services.fetch_multiple_entities_by_ids_and_models(
                multiple_models_keys_to_fetch.values()))
        for (
                field_name, (model_class, field_values)), (
                    external_instance_details) in zip(
                        multiple_models_keys_to_fetch.iteritems(),
                        fetched_model_instances):
            cls.external_instance_details[field_name] = (
                zip(
                    [model_class] * len(field_values),
                    field_values, external_instance_details))

    @classmethod
    def _validate_model_time_fields(cls, item):
        """Checks the following relation for the model:
        model.created_on <= model.last_updated <= current time.

        Args:
            item: ndb.Model. Entity to validate.
        """
        if item.created_on > item.last_updated:
            cls.errors['time field relation check'].append((
                'Entity id %s: The created_on field has a value %s which is '
                'greater than the value %s of last_updated field'
                ) % (item.id, item.created_on, item.last_updated))

        current_datetime = datetime.datetime.utcnow()
        if item.last_updated > current_datetime:
            cls.errors['current time check'].append((
                'Entity id %s: The last_updated field has a value %s which is '
                'greater than the time when the job was run'
                ) % (item.id, item.last_updated))

    @classmethod
    def _get_custom_validation_functions(cls):
        """Returns the list of custom validation functions to run.

        This method can be overridden by subclasses, if needed.

        Each validation function should accept only a single arg, which is the
        model instance to validate.
        """
        return []

    @classmethod
    def validate(cls, item):
        """Run _fetch_external_instance_details and all _validate functions.

        Args:
            item: ndb.Model. Entity to validate.
        """
        cls.errors.clear()
        cls.external_instance_details.clear()
        cls._fetch_external_instance_details(item)

        cls._validate_model_id(item)
        cls._validate_model_time_fields(item)
        cls._validate_model_domain_object_instances(item)
        cls._validate_external_id_relationships(item)

        for func in cls._get_custom_validation_functions():
            func(item)


class BaseSummaryModelValidator(BaseModelValidator):
    """Base class for validating summary models."""

    @classmethod
    def _get_external_model_properties(cls):
        """Returns a tuple of external models and properties.

        This should be implemented by subclasses.

        Returns:
            tuple(str, list(tuple), dict): A tuple with first element as
                external model name, second element as a tuple of
                cls.external_instance_details and the third element
                as a properties dict with key as property name in summary
                model and value as property name in external model.
        """
        raise NotImplementedError

    @classmethod
    def _validate_external_model_properties(cls, item):
        """Validate that properties of the model match the corresponding
        properties of the external model.

        Args:
            item: ndb.Model. BaseSummaryModel to validate.
        """

        for (
                external_model_name,
                external_model_class_model_id_model_tuples,
                external_model_properties_dict
            ) in cls._get_external_model_properties():

            for (_, _, external_model) in (
                    external_model_class_model_id_model_tuples):
                # The case for missing external model is ignored here
                # since errors for missing external model are already
                # checked and stored in _validate_external_id_relationships
                # function.
                if external_model is None or external_model.deleted:
                    continue
                for (property_name, external_model_property_name) in (
                        external_model_properties_dict.iteritems()):
                    value_in_summary_model = getattr(item, property_name)
                    value_in_external_model = getattr(
                        external_model, external_model_property_name)

                    if value_in_summary_model != value_in_external_model:
                        cls.errors['%s field check' % property_name].append((
                            'Entity id %s: %s field in entity: %s does not '
                            'match corresponding %s %s field: %s') % (
                                item.id, property_name,
                                value_in_summary_model,
                                external_model_name,
                                external_model_property_name,
                                value_in_external_model))

    @classmethod
    def validate(cls, item):
        """Run _fetch_external_instance_details and all _validate functions.

        Args:
            item: ndb.Model. Entity to validate.
        """
        super(BaseSummaryModelValidator, cls).validate(item)

        cls._validate_external_model_properties(item)


class BaseSnapshotContentModelValidator(BaseModelValidator):
    """Base class for validating snapshot content models."""

    # The name of the model which is to be used in the error messages.
    # This can be overridden by subclasses, if needed.
    MODEL_NAME = 'snapshot content'

    # The name of the external model in lowercase which is used to obtain
    # the name of the key for the fetch of external model and the name
    # of the external model to be used in error messages.
    # For example, if external model is CollectionRights, then
    # EXTERNAL_MODEL_NAME = collection rights, key to fetch = collection_rights
    # Name of model to be used in error message = CollectionRights
    # This should be overridden by subclasses.
    EXTERNAL_MODEL_NAME = ''

    @classmethod
    def _get_model_id_regex(cls, unused_item):
        return '^[A-Za-z0-9]{1,%s}-\\d*$' % base_models.ID_LENGTH

    @classmethod
    def _validate_base_model_version_from_item_id(cls, item):
        """Validate that external model corresponding to item.id
        has a version greater than or equal to the version in item.id.

        Args:
            item: ndb.Model. BaseSnapshotContentModel to validate.
        """

        if cls.EXTERNAL_MODEL_NAME == '':
            raise Exception('External model name should be specified')

        external_model_name = cls.EXTERNAL_MODEL_NAME
        if item.id.startswith('rights'):
            external_model_name = external_model_name + ' rights'

        name_split_by_space = external_model_name.split(' ')
        key_to_fetch = ('_').join(name_split_by_space)
        capitalized_external_model_name = ('').join([
            val.capitalize() for val in name_split_by_space])

        external_model_class_model_id_model_tuples = (
            cls.external_instance_details['%s_ids' % key_to_fetch])

        version = item.id[item.id.rfind('-') + 1:]
        for (_, _, external_model) in (
                external_model_class_model_id_model_tuples):
            # The case for missing external model is ignored here
            # since errors for missing external model are already
            # checked and stored in _validate_external_id_relationships
            # function.
            if external_model is None or external_model.deleted:
                continue
            if int(external_model.version) < int(version):
                cls.errors[
                    '%s model version check' % cls.EXTERNAL_MODEL_NAME].append((
                        'Entity id %s: %s model corresponding to '
                        'id %s has a version %s which is less than '
                        'the version %s in %s model id' % (
                            item.id, capitalized_external_model_name,
                            external_model.id, external_model.version, version,
                            cls.MODEL_NAME)))

    @classmethod
    def validate(cls, item):
        """Run _fetch_external_instance_details and all _validate functions.

        Args:
            item: ndb.Model. Entity to validate.
        """
        super(BaseSnapshotContentModelValidator, cls).validate(item)

        cls._validate_base_model_version_from_item_id(item)


class BaseSnapshotMetadataModelValidator(BaseSnapshotContentModelValidator):
    """Base class for validating snapshot metadata models."""

    MODEL_NAME = 'snapshot metadata'

    @classmethod
    def _validate_commit_type(cls, item):
        """Validates that commit type is valid.

        Args:
            item: ndb.Model. Entity to validate.
        """
        if item.commit_type not in (
                base_models.VersionedModel.COMMIT_TYPE_CHOICES):
            cls.errors['commit type check'].append((
                'Entity id %s: Commit type %s is not allowed') % (
                    item.id, item.commit_type))

    @classmethod
    def _get_change_domain_class(cls, unused_item):
        """Returns a Change domain class.

        This should be implemented by subclasses.

        Args:
            unused_item: ndb.Model. Entity to validate.

        Returns:
            change_domain.BaseChange: A domain object class for the
                changes made by commit commands of the model.
        """
        raise NotImplementedError

    @classmethod
    def _validate_commit_cmds_schema(cls, item):
        """Validates schema of commit commands in commit_cmds dict.

        Args:
            item: ndb.Model. Entity to validate.
        """
        change_domain_object = cls._get_change_domain_class(item)
        if change_domain_object is None:
            # This is for cases where id of the entity is invalid
            # and no commit command domain object is found for the entity.
            # For example, if a CollectionCommitLogEntryModel does
            # not have id starting with collection/rights, there is
            # no commit command domain object defined for this model.
            cls.errors['commit cmd check'].append(
                'Entity id %s: No commit command domain object defined '
                'for entity with commands: %s' % (item.id, item.commit_cmds))
            return
        for commit_cmd_dict in item.commit_cmds:
            if not commit_cmd_dict:
                continue
            try:
                change_domain_object(commit_cmd_dict)
            except Exception as e:
                cmd_name = commit_cmd_dict.get('cmd')
                cls.errors['commit cmd %s check' % cmd_name].append((
                    'Entity id %s: Commit command domain validation for '
                    'command: %s failed with error: %s') % (
                        item.id, commit_cmd_dict, e))

    @classmethod
    def validate(cls, item):
        """Run _fetch_external_instance_details and all _validate functions.

        Args:
            item: ndb.Model. Entity to validate.
        """
        super(BaseSnapshotMetadataModelValidator, cls).validate(item)

        cls._validate_commit_type(item)
        cls._validate_commit_cmds_schema(item)


class BaseCommitLogEntryModelValidator(BaseSnapshotMetadataModelValidator):
    """Base class for validating commit log entry models."""

    MODEL_NAME = 'commit log entry'

    @classmethod
    def _validate_post_commit_status(cls, item):
        """Validates that post_commit_status is either public or private.

        Args:
            item: ndb.Model. Entity to validate.
        """
        if item.post_commit_status not in [
                feconf.POST_COMMIT_STATUS_PUBLIC,
                feconf.POST_COMMIT_STATUS_PRIVATE]:
            cls.errors['post commit status check'].append((
                'Entity id %s: Post commit status %s is invalid') % (
                    item.id, item.post_commit_status))

    @classmethod
    def _validate_post_commit_is_private(cls, item):
        """Validates that post_commit_is_private is true iff
        post_commit_status is private.

        Args:
            item: ndb.Model. Entity to validate.
        """
        if item.post_commit_status == feconf.POST_COMMIT_STATUS_PRIVATE and (
                not item.post_commit_is_private):
            cls.errors['post commit is private check'].append((
                'Entity id %s: Post commit status is private but '
                'post_commit_is_private is False') % item.id)

        if item.post_commit_status == feconf.POST_COMMIT_STATUS_PUBLIC and (
                item.post_commit_is_private):
            cls.errors['post commit is private check'].append((
                'Entity id %s: Post commit status is public but '
                'post_commit_is_private is True') % item.id)

    @classmethod
    def validate(cls, item):
        """Run _fetch_external_instance_details and all _validate functions.

        Args:
            item: ndb.Model. Entity to validate.
        """
        super(BaseCommitLogEntryModelValidator, cls).validate(item)

        cls._validate_post_commit_status(item)
        cls._validate_post_commit_is_private(item)


class ActivityReferencesModelValidator(BaseModelValidator):
    """Class for validating ActivityReferencesModels."""

    @classmethod
    def _get_model_id_regex(cls, unused_item):
        # Valid id: featured.
        regex_string = '^(%s)$' % '|'.join(
            feconf.ALL_ACTIVITY_REFERENCE_LIST_TYPES)
        return regex_string

    @classmethod
    def _get_model_domain_object_instance(cls, item):
        activity_references_list = []

        try:
            for reference in item.activity_references:
                activity_references_list.append(
                    activity_domain.ActivityReference(
                        reference['type'], reference['id']))
        except Exception as e:
            cls.errors['fetch properties'].append((
                'Entity id %s: Entity properties cannot be fetched completely '
                'with the error %s') % (item.id, e))
            return None

        return activity_domain.ActivityReferences(activity_references_list)

    @classmethod
    def _get_external_id_relationships(cls, item):
        exploration_ids = []
        collection_ids = []

        try:
            for reference in item.activity_references:
                if reference['type'] == constants.ACTIVITY_TYPE_EXPLORATION:
                    exploration_ids.append(reference['id'])
                elif reference['type'] == constants.ACTIVITY_TYPE_COLLECTION:
                    collection_ids.append(reference['id'])
        except Exception as e:
            cls.errors['fetch properties'].append((
                'Entity id %s: Entity properties cannot be fetched completely '
                'with the error %s') % (item.id, e))
            return {}

        return {
            'exploration_ids': (exp_models.ExplorationModel, exploration_ids),
            'collection_ids': (
                collection_models.CollectionModel, collection_ids),
        }


class RoleQueryAuditModelValidator(BaseModelValidator):
    """Class for validating RoleQueryAuditModels."""

    @classmethod
    def _get_model_id_regex(cls, item):
        # Valid id: [user_id].[timestamp_in_sec].[intent].[random_number]
        regex_string = '^%s\\.\\d*\\.%s\\.\\d*$' % (item.user_id, item.intent)
        return regex_string

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {'user_ids': (user_models.UserSettingsModel, [item.user_id])}


class CollectionModelValidator(BaseModelValidator):
    """Class for validating CollectionModel."""

    @classmethod
    def _get_model_domain_object_instance(cls, item):
        return collection_services.get_collection_from_model(item)

    @classmethod
    def _get_external_id_relationships(cls, item):
        snapshot_model_ids = [
            '%s-%d' % (item.id, version)
            for version in range(1, item.version + 1)]
        return {
            'exploration_ids': (
                exp_models.ExplorationModel,
                [node['exploration_id'] for node in item.collection_contents[
                    'nodes']]),
            'collection_commit_log_entry_ids': (
                collection_models.CollectionCommitLogEntryModel,
                ['collection-%s-%s' % (item.id, version) for version in range(
                    1, item.version + 1)]),
            'collection_summary_ids': (
                collection_models.CollectionSummaryModel, [item.id]),
            'collection_rights_ids': (
                collection_models.CollectionRightsModel, [item.id]),
            'snapshot_metadata_ids': (
                collection_models.CollectionSnapshotMetadataModel,
                snapshot_model_ids),
            'snapshot_content_ids': (
                collection_models.CollectionSnapshotContentModel,
                snapshot_model_ids),
        }


class CollectionSnapshotMetadataModelValidator(
        BaseSnapshotMetadataModelValidator):
    """Class for validating CollectionSnapshotMetadataModel."""

    EXTERNAL_MODEL_NAME = 'collection'

    @classmethod
    def _get_change_domain_class(cls, unused_item):
        return collection_domain.CollectionChange

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'collection_ids': (
                collection_models.CollectionModel,
                [item.id[:item.id.find('-')]]),
            'committer_ids': (
                user_models.UserSettingsModel, [item.committer_id])
        }


class CollectionSnapshotContentModelValidator(
        BaseSnapshotContentModelValidator):
    """Class for validating CollectionSnapshotContentModel."""

    EXTERNAL_MODEL_NAME = 'collection'

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'collection_ids': (
                collection_models.CollectionModel,
                [item.id[:item.id.find('-')]]),
        }


class CollectionRightsModelValidator(BaseModelValidator):
    """Class for validating CollectionRightsModel."""

    @classmethod
    def _get_external_id_relationships(cls, item):
        snapshot_model_ids = [
            '%s-%d' % (item.id, version)
            for version in range(1, item.version + 1)]
        return {
            'collection_ids': (
                collection_models.CollectionModel, [item.id]),
            'owner_user_ids': (
                user_models.UserSettingsModel, item.owner_ids),
            'editor_user_ids': (
                user_models.UserSettingsModel, item.editor_ids),
            'viewer_user_ids': (
                user_models.UserSettingsModel, item.viewer_ids),
            'snapshot_metadata_ids': (
                collection_models.CollectionRightsSnapshotMetadataModel,
                snapshot_model_ids),
            'snapshot_content_ids': (
                collection_models.CollectionRightsSnapshotContentModel,
                snapshot_model_ids),
        }

    @classmethod
    def _validate_first_published_msec(cls, item):
        """Validate that first published time of model is less than current
        time.

        Args:
            item: ndb.Model. CollectionRightsModel to validate.
        """
        if not item.first_published_msec:
            return

        epoch = datetime.datetime.utcfromtimestamp(0)
        current_msec = (
            datetime.datetime.utcnow() - epoch).total_seconds() * 1000.0
        if item.first_published_msec > current_msec:
            cls.errors['first published msec check'].append((
                'Entity id %s: The first_published_msec field has a value %s '
                'which is greater than the time when the job was run'
                ) % (item.id, item.first_published_msec))

    @classmethod
    def _get_custom_validation_functions(cls):
        return [cls._validate_first_published_msec]


class CollectionRightsSnapshotMetadataModelValidator(
        BaseSnapshotMetadataModelValidator):
    """Class for validating CollectionRightsSnapshotMetadataModel."""

    EXTERNAL_MODEL_NAME = 'collection rights'

    @classmethod
    def _get_change_domain_class(cls, unused_item):
        return rights_manager.CollectionRightsChange

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'collection_rights_ids': (
                collection_models.CollectionRightsModel,
                [item.id[:item.id.find('-')]]),
            'committer_ids': (
                user_models.UserSettingsModel, [item.committer_id])
        }


class CollectionRightsSnapshotContentModelValidator(
        BaseSnapshotContentModelValidator):
    """Class for validating CollectionRightsSnapshotContentModel."""

    EXTERNAL_MODEL_NAME = 'collection rights'

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'collection_rights_ids': (
                collection_models.CollectionRightsModel,
                [item.id[:item.id.find('-')]]),
        }


class CollectionCommitLogEntryModelValidator(BaseCommitLogEntryModelValidator):
    """Class for validating CollectionCommitLogEntryModel."""

    EXTERNAL_MODEL_NAME = 'collection'

    @classmethod
    def _get_model_id_regex(cls, item):
        # Valid id: [collection/rights]-[collection_id]-[collection_version].
        regex_string = '^(collection|rights)-%s-\\d*$' % (
            item.collection_id)

        return regex_string

    @classmethod
    def _get_change_domain_class(cls, item):
        if item.id.startswith('rights'):
            return rights_manager.CollectionRightsChange
        elif item.id.startswith('collection'):
            return collection_domain.CollectionChange
        else:
            # The case of invalid id is being ignored here since this
            # case will already be checked by the id regex test.
            return None

    @classmethod
    def _get_external_id_relationships(cls, item):
        external_id_relationships = {
            'collection_ids': (
                collection_models.CollectionModel, [item.collection_id]),
        }
        if item.id.startswith('rights'):
            external_id_relationships['collection_rights_ids'] = (
                collection_models.CollectionRightsModel, [item.collection_id])
        return external_id_relationships


class CollectionSummaryModelValidator(BaseSummaryModelValidator):
    """Class for validating CollectionSummaryModel."""

    @classmethod
    def _get_model_domain_object_instance(cls, item):
        return collection_services.get_collection_summary_from_model(item)

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'collection_ids': (
                collection_models.CollectionModel, [item.id]),
            'collection_rights_ids': (
                collection_models.CollectionRightsModel, [item.id]),
            'owner_user_ids': (
                user_models.UserSettingsModel, item.owner_ids),
            'editor_user_ids': (
                user_models.UserSettingsModel, item.editor_ids),
            'viewer_user_ids': (
                user_models.UserSettingsModel, item.viewer_ids),
            'contributor_user_ids': (
                user_models.UserSettingsModel, item.contributor_ids)
        }

    @classmethod
    def _validate_contributors_summary(cls, item):
        """Validate that contributor ids match the contributor ids obtained
        from contributors summary.

        Args:
            item: ndb.Model. CollectionSummaryModel to validate.
        """
        contributor_ids_from_contributors_summary = (
            item.contributors_summary.keys())
        if sorted(item.contributor_ids) != sorted(
                contributor_ids_from_contributors_summary):
            cls.errors['contributors summary check'].append((
                'Entity id %s: Contributor ids: %s do not match the '
                'contributor ids obtained using contributors summary: %s') % (
                    item.id, sorted(item.contributor_ids),
                    sorted(contributor_ids_from_contributors_summary)))

    @classmethod
    def _validate_node_count(cls, item):
        """Validate that node_count of model is equal to number of nodes
        in CollectionModel.collection_contents.

        Args:
            item: ndb.Model. CollectionSummaryModel to validate.
        """
        collection_model_class_model_id_model_tuples = (
            cls.external_instance_details['collection_ids'])

        for (_, _, collection_model) in (
                collection_model_class_model_id_model_tuples):
            # The case for missing collection external model is ignored here
            # since errors for missing collection external model are already
            # checked and stored in _validate_external_id_relationships
            # function.
            if collection_model is None or collection_model.deleted:
                continue
            nodes = collection_model.collection_contents['nodes']
            if item.node_count != len(nodes):
                cls.errors['node count check'].append((
                    'Entity id %s: Node count: %s does not match the number of '
                    'nodes in collection_contents dict: %s') % (
                        item.id, item.node_count, nodes))

    @classmethod
    def _validate_ratings_is_empty(cls, item):
        """Validate that ratings for the entity is empty.

        Args:
            item: ndb.Model. CollectionSummaryModel to validate.
        """
        if item.ratings:
            cls.errors['ratings check'].append(
                'Entity id %s: Expected ratings for the entity to be '
                'empty but received %s' % (item.id, item.ratings))

    @classmethod
    def _get_external_model_properties(cls):
        collection_model_class_model_id_model_tuples = (
            cls.external_instance_details['collection_ids'])
        collection_rights_model_class_model_id_model_tuples = (
            cls.external_instance_details['collection_rights_ids'])

        collection_model_properties_dict = {
            'title': 'title',
            'category': 'category',
            'objective': 'objective',
            'language_code': 'language_code',
            'tags': 'tags',
            'collection_model_created_on': 'created_on',
            'collection_model_last_updated': 'last_updated'
        }

        collection_rights_model_properties_dict = {
            'status': 'status',
            'community_owned': 'community_owned',
            'owner_ids': 'owner_ids',
            'editor_ids': 'editor_ids',
            'viewer_ids': 'viewer_ids',
        }

        return [(
            'collection',
            collection_model_class_model_id_model_tuples,
            collection_model_properties_dict
        ), (
            'collection rights',
            collection_rights_model_class_model_id_model_tuples,
            collection_rights_model_properties_dict
        )]

    @classmethod
    def _get_custom_validation_functions(cls):
        return [
            cls._validate_node_count,
            cls._validate_ratings_is_empty,
            cls._validate_contributors_summary,
            ]


class ConfigPropertyModelValidator(BaseModelValidator):
    """Class for validating ConfigPropertyModel."""

    @classmethod
    def _get_model_id_regex(cls, unused_item):
        return '^.*$'

    @classmethod
    def _get_external_id_relationships(cls, item):
        snapshot_model_ids = [
            '%s-%d' % (item.id, version)
            for version in range(1, item.version + 1)]
        return {
            'snapshot_metadata_ids': (
                config_models.ConfigPropertySnapshotMetadataModel,
                snapshot_model_ids),
            'snapshot_content_ids': (
                config_models.ConfigPropertySnapshotContentModel,
                snapshot_model_ids),
        }


class ConfigPropertySnapshotMetadataModelValidator(
        BaseSnapshotMetadataModelValidator):
    """Class for validating ConfigPropertySnapshotMetadataModel."""

    EXTERNAL_MODEL_NAME = 'config property'

    @classmethod
    def _get_model_id_regex(cls, unused_item):
        return '^.*-\\d*$'

    @classmethod
    def _get_change_domain_class(cls, unused_item):
        return config_domain.ConfigPropertyChange

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'config_property_ids': (
                config_models.ConfigPropertyModel,
                [item.id[:item.id.find('-')]]),
            'committer_ids': (
                user_models.UserSettingsModel, [item.committer_id])
        }


class ConfigPropertySnapshotContentModelValidator(
        BaseSnapshotContentModelValidator):
    """Class for validating ConfigPropertySnapshotContentModel."""

    EXTERNAL_MODEL_NAME = 'config property'

    @classmethod
    def _get_model_id_regex(cls, unused_item):
        return '^.*-\\d*$'

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'config_property_ids': (
                config_models.ConfigPropertyModel,
                [item.id[:item.id.find('-')]]),
        }


class SentEmailModelValidator(BaseModelValidator):
    """Class for validating SentEmailModels."""

    @classmethod
    def _get_model_id_regex(cls, item):
        # Valid id: [intent].[random hash]
        regex_string = '^%s\\.\\.[A-Za-z0-9]{1,%s}$' % (
            item.intent, base_models.ID_LENGTH)
        return regex_string

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'recipient_id': (
                user_models.UserSettingsModel, [item.recipient_id]),
            'sender_id': (user_models.UserSettingsModel, [item.sender_id]),
        }

    @classmethod
    def _validate_sent_datetime(cls, item):
        """Validate that sent_datetime of model is less than current time.

        Args:
            item: ndb.Model. SentEmailModel to validate.
        """
        current_datetime = datetime.datetime.utcnow()
        if item.sent_datetime > current_datetime:
            cls.errors['sent datetime check'].append((
                'Entity id %s: The sent_datetime field has a value %s which is '
                'greater than the time when the job was run'
                ) % (item.id, item.sent_datetime))

    @classmethod
    def _validate_sender_email(cls, item):
        """Validate that sender email corresponds to email of user obtained
        by using the sender_id.

        Args:
            item: ndb.Model. SentEmailModel to validate.
        """
        sender_model_class_model_id_model_tuples = (
            cls.external_instance_details['sender_id'])

        for (_, _, sender_model) in (
                sender_model_class_model_id_model_tuples):
            # The case for missing sender external model is ignored here
            # since errors for missing sender external model are already
            # checked and stored in _validate_external_id_relationships
            # function.
            if sender_model is None or sender_model.deleted:
                continue
            if sender_model.email != item.sender_email:
                cls.errors['sender email check'].append((
                    'Entity id %s: Sender email %s in entity does not '
                    'match with email %s of user obtained through '
                    'sender id %s') % (
                        item.id, item.sender_email, sender_model.email,
                        item.sender_id))

    @classmethod
    def _validate_recipient_email(cls, item):
        """Validate that recipient email corresponds to email of user obtained
        by using the recipient_id.

        Args:
            item: ndb.Model. SentEmailModel to validate.
        """
        recipient_model_class_model_id_model_tuples = (
            cls.external_instance_details['recipient_id'])

        for (_, _, recipient_model) in (
                recipient_model_class_model_id_model_tuples):
            # The case for missing recipient external model is ignored here
            # since errors for missing recipient external model are already
            # checked and stored in _validate_external_id_relationships
            # function.
            if recipient_model is None or recipient_model.deleted:
                continue
            if recipient_model.email != item.recipient_email:
                cls.errors['recipient email check'].append((
                    'Entity id %s: Recipient email %s in entity does '
                    'not match with email %s of user obtained through '
                    'recipient id %s') % (
                        item.id, item.recipient_email,
                        recipient_model.email, item.recipient_id))

    @classmethod
    def _get_custom_validation_functions(cls):
        return [
            cls._validate_sent_datetime,
            cls._validate_sender_email,
            cls._validate_recipient_email]


class BulkEmailModelValidator(BaseModelValidator):
    """Class for validating BulkEmailModels."""

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'recipient_id': (
                user_models.UserSettingsModel, item.recipient_ids),
            'sender_id': (user_models.UserSettingsModel, [item.sender_id]),
        }

    @classmethod
    def _validate_sent_datetime(cls, item):
        """Validate that sent_datetime of model is less than current time.

        Args:
            item: ndb.Model. BulkEmailModel to validate.
        """
        current_datetime = datetime.datetime.utcnow()
        if item.sent_datetime > current_datetime:
            cls.errors['sent datetime check'].append((
                'Entity id %s: The sent_datetime field has a value %s which is '
                'greater than the time when the job was run'
                ) % (item.id, item.sent_datetime))

    @classmethod
    def _validate_sender_email(cls, item):
        """Validate that sender email corresponds to email of user obtained
        by using the sender_id.

        Args:
            item: ndb.Model. BulkEmailModel to validate.
        """
        sender_model_class_model_id_model_tuples = (
            cls.external_instance_details['sender_id'])

        for (_, _, sender_model) in (
                sender_model_class_model_id_model_tuples):
            # The case for missing sender external model is ignored here
            # since errors for missing sender external model are already
            # checked and stored in _validate_external_id_relationships
            # function.
            if sender_model is None or sender_model.deleted:
                continue
            if sender_model.email != item.sender_email:
                cls.errors['sender email check'].append((
                    'Entity id %s: Sender email %s in entity does not '
                    'match with email %s of user obtained through '
                    'sender id %s') % (
                        item.id, item.sender_email, sender_model.email,
                        item.sender_id))

    @classmethod
    def _get_custom_validation_functions(cls):
        return [
            cls._validate_sent_datetime,
            cls._validate_sender_email]


class GeneralFeedbackEmailReplyToIdModelValidator(BaseModelValidator):
    """Class for validating GeneralFeedbackEmailReplyToIdModels."""

    @classmethod
    def _get_model_id_regex(cls, unused_item):
        return (
            '^\\d*\\.(exploration|topic)\\.[A-Za-z0-9]{1,12}\\.'
            '[A-Za-z0-9=+/]{1,}')

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'item.id.user_id': (
                user_models.UserSettingsModel, [
                    item.id[:item.id.find('.')]]),
            'item.id.thread_id': (
                feedback_models.GeneralFeedbackThreadModel, [
                    item.id[item.id.find('.') + 1:]]),
        }

    @classmethod
    def _validate_reply_to_id_length(cls, item):
        """Validate that reply_to_id length is less than or equal to
        REPLY_TO_ID_LENGTH.

        Args:
            item: ndb.Model. GeneralFeedbackEmailReplyToIdModel to validate.
        """
        # The reply_to_id of model is created using utils.get_random_int
        # method by using a upper bound as email_models.REPLY_TO_ID_LENGTH.
        # So, the reply_to_id length should be less than or equal to
        # email_models.REPLY_TO_ID_LENGTH.
        if len(item.reply_to_id) > email_models.REPLY_TO_ID_LENGTH:
            cls.errors['reply_to_id length check'].append((
                'Entity id %s: reply_to_id %s should have length less than or '
                'equal to %s but instead has length %s' % (
                    item.id, item.reply_to_id, email_models.REPLY_TO_ID_LENGTH,
                    len(item.reply_to_id))))

    @classmethod
    def _get_custom_validation_functions(cls):
        return [cls._validate_reply_to_id_length]


class ExplorationModelValidator(BaseModelValidator):
    """Class for validating ExplorationModel."""

    @classmethod
    def _get_model_domain_object_instance(cls, item):
        return exp_services.get_exploration_from_model(item)

    @classmethod
    def _get_external_id_relationships(cls, item):
        snapshot_model_ids = [
            '%s-%d' % (item.id, version)
            for version in range(1, item.version + 1)]
        return {
            'exploration_commit_log_entry_ids': (
                exp_models.ExplorationCommitLogEntryModel,
                ['exploration-%s-%s' % (item.id, version) for version in range(
                    1, item.version + 1)]),
            'exp_summary_ids': (
                exp_models.ExpSummaryModel, [item.id]),
            'exploration_rights_ids': (
                exp_models.ExplorationRightsModel, [item.id]),
            'snapshot_metadata_ids': (
                exp_models.ExplorationSnapshotMetadataModel,
                snapshot_model_ids),
            'snapshot_content_ids': (
                exp_models.ExplorationSnapshotContentModel,
                snapshot_model_ids),
        }


class ExplorationSnapshotMetadataModelValidator(
        BaseSnapshotMetadataModelValidator):
    """Class for validating ExplorationSnapshotMetadataModel."""

    EXTERNAL_MODEL_NAME = 'exploration'

    @classmethod
    def _get_change_domain_class(cls, unused_item):
        return exp_domain.ExplorationChange

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'exploration_ids': (
                exp_models.ExplorationModel, [item.id[:item.id.find('-')]]),
            'committer_ids': (
                user_models.UserSettingsModel, [item.committer_id])
        }


class ExplorationSnapshotContentModelValidator(
        BaseSnapshotContentModelValidator):
    """Class for validating ExplorationSnapshotContentModel."""

    EXTERNAL_MODEL_NAME = 'exploration'

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'exploration_ids': (
                exp_models.ExplorationModel, [item.id[:item.id.find('-')]]),
        }


class ExplorationRightsModelValidator(BaseModelValidator):
    """Class for validating ExplorationRightsModel."""

    @classmethod
    def _get_external_id_relationships(cls, item):
        cloned_from_exploration_id = []
        if item.cloned_from:
            cloned_from_exploration_id.append(item.cloned_from)
        snapshot_model_ids = [
            '%s-%d' % (item.id, version)
            for version in range(1, item.version + 1)]
        return {
            'exploration_ids': (
                exp_models.ExplorationModel, [item.id]),
            'cloned_from_exploration_ids': (
                exp_models.ExplorationModel,
                cloned_from_exploration_id),
            'owner_user_ids': (
                user_models.UserSettingsModel, item.owner_ids),
            'editor_user_ids': (
                user_models.UserSettingsModel, item.editor_ids),
            'viewer_user_ids': (
                user_models.UserSettingsModel, item.viewer_ids),
            'snapshot_metadata_ids': (
                exp_models.ExplorationRightsSnapshotMetadataModel,
                snapshot_model_ids),
            'snapshot_content_ids': (
                exp_models.ExplorationRightsSnapshotContentModel,
                snapshot_model_ids),
        }

    @classmethod
    def _validate_first_published_msec(cls, item):
        """Validate that first published time of model is less than current
        time.

        Args:
            item: ndb.Model. ExplorationRightsModel to validate.
        """
        if not item.first_published_msec:
            return

        epoch = datetime.datetime.utcfromtimestamp(0)
        current_msec = (
            datetime.datetime.utcnow() - epoch).total_seconds() * 1000.0
        if item.first_published_msec > current_msec:
            cls.errors['first published msec check'].append((
                'Entity id %s: The first_published_msec field has a value %s '
                'which is greater than the time when the job was run'
                ) % (item.id, item.first_published_msec))

    @classmethod
    def _get_custom_validation_functions(cls):
        return [cls._validate_first_published_msec]


class ExplorationRightsSnapshotMetadataModelValidator(
        BaseSnapshotMetadataModelValidator):
    """Class for validating ExplorationRightsSnapshotMetadataModel."""

    EXTERNAL_MODEL_NAME = 'exploration rights'

    @classmethod
    def _get_change_domain_class(cls, unused_item):
        return rights_manager.ExplorationRightsChange

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'exploration_rights_ids': (
                exp_models.ExplorationRightsModel,
                [item.id[:item.id.find('-')]]),
            'committer_ids': (
                user_models.UserSettingsModel, [item.committer_id])
        }


class ExplorationRightsSnapshotContentModelValidator(
        BaseSnapshotContentModelValidator):
    """Class for validating ExplorationRightsSnapshotContentModel."""

    EXTERNAL_MODEL_NAME = 'exploration rights'

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'exploration_rights_ids': (
                exp_models.ExplorationRightsModel,
                [item.id[:item.id.find('-')]]),
        }


class ExplorationCommitLogEntryModelValidator(BaseCommitLogEntryModelValidator):
    """Class for validating ExplorationCommitLogEntryModel."""

    EXTERNAL_MODEL_NAME = 'exploration'

    @classmethod
    def _get_model_id_regex(cls, item):
        # Valid id: [exploration/rights]-[exploration_id]-[exploration-version].
        regex_string = '^(exploration|rights)-%s-\\d*$' % (
            item.exploration_id)

        return regex_string

    @classmethod
    def _get_change_domain_class(cls, item):
        if item.id.startswith('rights'):
            return rights_manager.ExplorationRightsChange
        elif item.id.startswith('exploration'):
            return exp_domain.ExplorationChange
        else:
            # The case of invalid id is being ignored here since this
            # case will already be checked by the id regex test.
            return None

    @classmethod
    def _get_external_id_relationships(cls, item):
        external_id_relationships = {
            'exploration_ids': (
                exp_models.ExplorationModel, [item.exploration_id]),
        }
        if item.id.startswith('rights'):
            external_id_relationships['exploration_rights_ids'] = (
                exp_models.ExplorationRightsModel, [item.exploration_id])
        return external_id_relationships


class ExpSummaryModelValidator(BaseSummaryModelValidator):
    """Class for validating ExpSummaryModel."""

    @classmethod
    def _get_model_domain_object_instance(cls, item):
        return exp_services.get_exploration_summary_from_model(item)

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'exploration_ids': (
                exp_models.ExplorationModel, [item.id]),
            'exploration_rights_ids': (
                exp_models.ExplorationRightsModel, [item.id]),
            'owner_user_ids': (
                user_models.UserSettingsModel, item.owner_ids),
            'editor_user_ids': (
                user_models.UserSettingsModel, item.editor_ids),
            'viewer_user_ids': (
                user_models.UserSettingsModel, item.viewer_ids),
            'contributor_user_ids': (
                user_models.UserSettingsModel, item.contributor_ids)
        }

    @classmethod
    def _validate_contributors_summary(cls, item):
        """Validate that contributor ids match the contributor ids obtained
        from contributors summary.

        Args:
            item: ndb.Model. ExpSummaryModel to validate.
        """
        contributor_ids_from_contributors_summary = (
            item.contributors_summary.keys())
        if sorted(item.contributor_ids) != sorted(
                contributor_ids_from_contributors_summary):
            cls.errors['contributors summary check'].append((
                'Entity id %s: Contributor ids: %s do not match the '
                'contributor ids obtained using contributors summary: %s') % (
                    item.id, sorted(item.contributor_ids),
                    sorted(contributor_ids_from_contributors_summary)))

    @classmethod
    def _validate_first_published_msec(cls, item):
        """Validate that first published time of model is less than current
        time.

        Args:
            item: ndb.Model. ExpSummaryModel to validate.
        """
        if not item.first_published_msec:
            return

        epoch = datetime.datetime.utcfromtimestamp(0)
        current_msec = (
            datetime.datetime.utcnow() - epoch).total_seconds() * 1000.0
        if item.first_published_msec > current_msec:
            cls.errors['first published msec check'].append((
                'Entity id %s: The first_published_msec field has a value %s '
                'which is greater than the time when the job was run'
                ) % (item.id, item.first_published_msec))

    @classmethod
    def _validate_exploration_model_last_updated(cls, item):
        """Validate that item.exploration_model_last_updated matches the
        time when a last commit was made by a human contributor.

        Args:
            item: ndb.Model. ExpSummaryModel to validate.
        """
        exploration_model_class_model_id_model_tuples = (
            cls.external_instance_details['exploration_ids'])
        for (_, _, exploration_model) in (
                exploration_model_class_model_id_model_tuples):
            # The case for missing exploration external model is ignored here
            # since errors for missing exploration external model are already
            # checked and stored in _validate_external_id_relationships
            # function.
            if exploration_model is None or exploration_model.deleted:
                continue
            last_human_update_ms = exp_services.get_last_updated_by_human_ms(
                exploration_model.id)
            last_human_update_time = datetime.datetime.fromtimestamp(
                last_human_update_ms / 1000.0)
            if item.exploration_model_last_updated != last_human_update_time:
                cls.errors['exploration model last updated check'].append((
                    'Entity id %s: The exploration_model_last_updated '
                    'field: %s does not match the last time a commit was '
                    'made by a human contributor: %s') % (
                        item.id, item.exploration_model_last_updated,
                        last_human_update_time))

    @classmethod
    def _get_external_model_properties(cls):
        exploration_model_class_model_id_model_tuples = (
            cls.external_instance_details['exploration_ids'])
        exploration_rights_model_class_model_id_model_tuples = (
            cls.external_instance_details['exploration_rights_ids'])

        exploration_model_properties_dict = {
            'title': 'title',
            'category': 'category',
            'objective': 'objective',
            'language_code': 'language_code',
            'tags': 'tags',
            'exploration_model_created_on': 'created_on',
        }

        exploration_rights_model_properties_dict = {
            'first_published_msec': 'first_published_msec',
            'status': 'status',
            'community_owned': 'community_owned',
            'owner_ids': 'owner_ids',
            'editor_ids': 'editor_ids',
            'viewer_ids': 'viewer_ids',
        }

        return [(
            'exploration',
            exploration_model_class_model_id_model_tuples,
            exploration_model_properties_dict
        ), (
            'exploration rights',
            exploration_rights_model_class_model_id_model_tuples,
            exploration_rights_model_properties_dict
        )]

    @classmethod
    def _get_custom_validation_functions(cls):
        return [
            cls._validate_first_published_msec,
            cls._validate_contributors_summary,
            cls._validate_exploration_model_last_updated]


class FileMetadataModelValidator(BaseModelValidator):
    """Class for validating FileMetadataModel."""

    @classmethod
    def _get_model_id_regex(cls, unused_item):
        return '^%s$' % FILE_MODELS_REGEX

    @classmethod
    def _get_external_id_relationships(cls, item):
        snapshot_model_ids = [
            '%s-%d' % (item.id, version)
            for version in range(1, item.version + 1)]

        # Item id is of the format:
        # /exploration/exp_id/assets/(image|audio)/filepath.
        exp_val = '/exploration/'
        assets_val = '/assets'
        start_index = item.id.find(exp_val) + len(exp_val)
        end_index = item.id.find(assets_val)
        exp_id = item.id[start_index:end_index]

        return {
            'snapshot_metadata_ids': (
                file_models.FileMetadataSnapshotMetadataModel,
                snapshot_model_ids),
            'snapshot_content_ids': (
                file_models.FileMetadataSnapshotContentModel,
                snapshot_model_ids),
            'exploration_ids': (exp_models.ExplorationModel, [exp_id])
        }


class FileMetadataSnapshotMetadataModelValidator(
        BaseSnapshotMetadataModelValidator):
    """Class for validating FileMetadataSnapshotMetadataModel."""

    EXTERNAL_MODEL_NAME = 'file metadata'

    @classmethod
    def _get_model_id_regex(cls, unused_item):
        return '%s-\\d*$' % FILE_MODELS_REGEX

    @classmethod
    def _get_change_domain_class(cls, unused_item):
        return fs_domain.FileMetadataChange

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'file_metadata_ids': (
                file_models.FileMetadataModel,
                [item.id[:item.id.find('-')]]),
            'committer_ids': (
                user_models.UserSettingsModel, [item.committer_id])
        }


class FileMetadataSnapshotContentModelValidator(
        BaseSnapshotContentModelValidator):
    """Class for validating FileMetadataSnapshotContentModel."""

    EXTERNAL_MODEL_NAME = 'file metadata'

    @classmethod
    def _get_model_id_regex(cls, unused_item):
        return '%s-\\d*$' % FILE_MODELS_REGEX

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'file_metadata_ids': (
                file_models.FileMetadataModel,
                [item.id[:item.id.find('-')]]),
        }


class FileModelValidator(BaseModelValidator):
    """Class for validating FileModel."""

    @classmethod
    def _get_model_id_regex(cls, unused_item):
        return '^%s$' % FILE_MODELS_REGEX

    @classmethod
    def _get_external_id_relationships(cls, item):
        snapshot_model_ids = [
            '%s-%d' % (item.id, version)
            for version in range(1, item.version + 1)]

        # Item id is of the format:
        # /exploration/exp_id/assets/(image|audio)/filepath.
        exp_val = '/exploration/'
        assets_val = '/assets'
        start_index = item.id.find(exp_val) + len(exp_val)
        end_index = item.id.find(assets_val)
        exp_id = item.id[start_index:end_index]

        return {
            'snapshot_metadata_ids': (
                file_models.FileSnapshotMetadataModel,
                snapshot_model_ids),
            'snapshot_content_ids': (
                file_models.FileSnapshotContentModel,
                snapshot_model_ids),
            'exploration_ids': (exp_models.ExplorationModel, [exp_id])
        }


class FileSnapshotMetadataModelValidator(BaseSnapshotMetadataModelValidator):
    """Class for validating FileSnapshotMetadataModel."""

    EXTERNAL_MODEL_NAME = 'file'

    @classmethod
    def _get_model_id_regex(cls, unused_item):
        return '%s-\\d*$' % FILE_MODELS_REGEX

    @classmethod
    def _get_change_domain_class(cls, unused_item):
        return fs_domain.FileChange

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'file_ids': (
                file_models.FileModel,
                [item.id[:item.id.find('-')]]),
            'committer_ids': (
                user_models.UserSettingsModel, [item.committer_id])
        }


class FileSnapshotContentModelValidator(BaseSnapshotContentModelValidator):
    """Class for validating FileSnapshotContentModel."""

    EXTERNAL_MODEL_NAME = 'file'

    @classmethod
    def _get_model_id_regex(cls, unused_item):
        return '%s-\\d*$' % FILE_MODELS_REGEX

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'file_ids': (
                file_models.FileModel,
                [item.id[:item.id.find('-')]]),
        }


class ExplorationRecommendationsModelValidator(BaseModelValidator):
    """Class for validating ExplorationRecommendationsModel."""

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'exploration_ids': (
                exp_models.ExplorationModel,
                [item.id] + item.recommended_exploration_ids),
        }

    @classmethod
    def _validate_item_id_not_in_recommended_exploration_ids(cls, item):
        """Validate that model id is not present in recommended exploration ids.

        Args:
            item: ndb.Model. ExplorationRecommendationsModel to validate.
        """
        if item.id in item.recommended_exploration_ids:
            cls.errors['item exploration id check'].append((
                'Entity id %s: The exploration id: %s for which the entity is '
                'created is also present in the recommended exploration ids '
                'for entity') % (item.id, item.id))

    @classmethod
    def _get_custom_validation_functions(cls):
        return [cls._validate_item_id_not_in_recommended_exploration_ids]


class TopicSimilaritiesModelValidator(BaseModelValidator):
    """Class for validating TopicSimilaritiesModel."""

    @classmethod
    def _get_model_id_regex(cls, unused_item):
        # Valid id: topics.
        return '^%s$' % recommendations_models.TOPIC_SIMILARITIES_ID

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {}

    @classmethod
    def _validate_topic_similarities(cls, item):
        """Validate the topic similarities to be symmetric and have real
        values between 0.0 and 1.0.

        Args:
            item: ndb.Model. TopicSimilaritiesModel to validate.
        """

        topics = item.content.keys()
        data = '%s\n' % (',').join(topics)

        for topic1 in topics:
            similarity_list = []
            for topic2 in item.content[topic1]:
                similarity_list.append(str(item.content[topic1][topic2]))
            if len(similarity_list):
                data = data + '%s\n' % (',').join(similarity_list)

        try:
            recommendations_services.validate_topic_similarities(data)
        except Exception as e:
            cls.errors['topic similarity check'].append(
                'Entity id %s: Topic similarity validation for content: %s '
                'fails with error: %s' % (item.id, item.content, e))

    @classmethod
    def _get_custom_validation_functions(cls):
        return [cls._validate_topic_similarities]


class StoryModelValidator(BaseModelValidator):
    """Class for validating StoryModel."""

    @classmethod
    def _get_model_domain_object_instance(cls, item):
        return story_services.get_story_from_model(item)

    @classmethod
    def _get_external_id_relationships(cls, item):
        snapshot_model_ids = [
            '%s-%d' % (item.id, version)
            for version in range(1, item.version + 1)]
        return {
            'story_commit_log_entry_ids': (
                story_models.StoryCommitLogEntryModel,
                ['story-%s-%s' % (item.id, version) for version in range(
                    1, item.version + 1)]),
            'story_summary_ids': (
                story_models.StorySummaryModel, [item.id]),
            'story_rights_ids': (
                story_models.StoryRightsModel, [item.id]),
            'snapshot_metadata_ids': (
                story_models.StorySnapshotMetadataModel,
                snapshot_model_ids),
            'snapshot_content_ids': (
                story_models.StorySnapshotContentModel,
                snapshot_model_ids),
            'exploration_ids': (
                exp_models.ExplorationModel,
                [node['exploration_id'] for node in (
                    item.story_contents['nodes'])])
        }


class StorySnapshotMetadataModelValidator(BaseSnapshotMetadataModelValidator):
    """Class for validating StorySnapshotMetadataModel."""

    EXTERNAL_MODEL_NAME = 'story'

    @classmethod
    def _get_change_domain_class(cls, unused_item):
        return story_domain.StoryChange

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'story_ids': (
                story_models.StoryModel, [item.id[:item.id.find('-')]]),
            'committer_ids': (
                user_models.UserSettingsModel, [item.committer_id])
        }


class StorySnapshotContentModelValidator(BaseSnapshotContentModelValidator):
    """Class for validating StorySnapshotContentModel."""

    EXTERNAL_MODEL_NAME = 'story'

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'story_ids': (
                story_models.StoryModel, [item.id[:item.id.find('-')]]),
        }


class StoryRightsModelValidator(BaseModelValidator):
    """Class for validating StoryRightsModel."""

    @classmethod
    def _get_external_id_relationships(cls, item):
        snapshot_model_ids = [
            '%s-%d' % (item.id, version)
            for version in range(1, item.version + 1)]
        return {
            'story_ids': (
                story_models.StoryModel, [item.id]),
            'manager_user_ids': (
                user_models.UserSettingsModel, item.manager_ids),
            'snapshot_metadata_ids': (
                story_models.StoryRightsSnapshotMetadataModel,
                snapshot_model_ids),
            'snapshot_content_ids': (
                story_models.StoryRightsSnapshotContentModel,
                snapshot_model_ids),
        }


class StoryRightsSnapshotMetadataModelValidator(
        BaseSnapshotMetadataModelValidator):
    """Class for validating StoryRightsSnapshotMetadataModel."""

    EXTERNAL_MODEL_NAME = 'story rights'

    @classmethod
    def _get_change_domain_class(cls, unused_item):
        return story_domain.StoryRightsChange

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'story_rights_ids': (
                story_models.StoryRightsModel,
                [item.id[:item.id.find('-')]]),
            'committer_ids': (
                user_models.UserSettingsModel, [item.committer_id])
        }


class StoryRightsSnapshotContentModelValidator(
        BaseSnapshotContentModelValidator):
    """Class for validating StoryRightsSnapshotContentModel."""

    EXTERNAL_MODEL_NAME = 'story rights'

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'story_rights_ids': (
                story_models.StoryRightsModel,
                [item.id[:item.id.find('-')]]),
        }


class StoryCommitLogEntryModelValidator(BaseCommitLogEntryModelValidator):
    """Class for validating StoryCommitLogEntryModel."""

    EXTERNAL_MODEL_NAME = 'story'

    @classmethod
    def _get_model_id_regex(cls, item):
        # Valid id: [story]-[story_id]-[story_version].
        regex_string = '^(story)-%s-\\d*$' % (
            item.story_id)

        return regex_string

    @classmethod
    def _get_change_domain_class(cls, item):
        if item.id.startswith('story'):
            return story_domain.StoryChange
        else:
            # The case of invalid id is being ignored here since this
            # case will already be checked by the id regex test.
            return None

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'story_ids': (
                story_models.StoryModel, [item.story_id]),
        }


class StorySummaryModelValidator(BaseSummaryModelValidator):
    """Class for validating StorySummaryModel."""

    @classmethod
    def _get_model_domain_object_instance(cls, item):
        return story_services.get_story_summary_from_model(item)

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'story_ids': (
                story_models.StoryModel, [item.id]),
            'story_rights_ids': (
                story_models.StoryRightsModel, [item.id]),
        }

    @classmethod
    def _validate_node_count(cls, item):
        """Validate that node_count of model is equal to number of nodes
        in StoryModel.story_contents.

        Args:
            item: ndb.Model. StorySummaryModel to validate.
        """
        story_model_class_model_id_model_tuples = cls.external_instance_details[
            'story_ids']

        for (_, _, story_model) in story_model_class_model_id_model_tuples:
            # The case for missing story external model is ignored here
            # since errors for missing story external model are already
            # checked and stored in _validate_external_id_relationships
            # function.
            if story_model is None or story_model.deleted:
                continue
            nodes = story_model.story_contents['nodes']
            if item.node_count != len(nodes):
                cls.errors['node count check'].append((
                    'Entity id %s: Node count: %s does not match the '
                    'number of nodes in story_contents dict: %s') % (
                        item.id, item.node_count, nodes))

    @classmethod
    def _get_external_model_properties(cls):
        story_model_class_model_id_model_tuples = cls.external_instance_details[
            'story_ids']

        story_model_properties_dict = {
            'title': 'title',
            'language_code': 'language_code',
            'description': 'description',
            'story_model_created_on': 'created_on',
            'story_model_last_updated': 'last_updated'
        }

        return [(
            'story',
            story_model_class_model_id_model_tuples,
            story_model_properties_dict
        )]

    @classmethod
    def _get_custom_validation_functions(cls):
        return [cls._validate_node_count]


class UserSubscriptionsModelValidator(BaseModelValidator):
    """Class for validating UserSubscriptionsModels."""

    @classmethod
    def _get_model_id_regex(cls, unused_item):
        return '^\\d*$'

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'activity_ids': (exp_models.ExplorationModel, item.activity_ids),
            'collection_ids': (
                collection_models.CollectionModel,
                item.collection_ids),
            'general_feedback_thread_ids': (
                feedback_models.GeneralFeedbackThreadModel,
                item.general_feedback_thread_ids),
            'creator_ids': (user_models.UserSettingsModel, item.creator_ids),
            'id': (user_models.UserSettingsModel, [item.id]),
        }


MODEL_TO_VALIDATOR_MAPPING = {
    activity_models.ActivityReferencesModel: ActivityReferencesModelValidator,
    audit_models.RoleQueryAuditModel: RoleQueryAuditModelValidator,
    collection_models.CollectionModel: CollectionModelValidator,
    collection_models.CollectionSnapshotMetadataModel: (
        CollectionSnapshotMetadataModelValidator),
    collection_models.CollectionSnapshotContentModel: (
        CollectionSnapshotContentModelValidator),
    collection_models.CollectionRightsModel: CollectionRightsModelValidator,
    collection_models.CollectionRightsSnapshotMetadataModel: (
        CollectionRightsSnapshotMetadataModelValidator),
    collection_models.CollectionRightsSnapshotContentModel: (
        CollectionRightsSnapshotContentModelValidator),
    collection_models.CollectionCommitLogEntryModel: (
        CollectionCommitLogEntryModelValidator),
    collection_models.CollectionSummaryModel: CollectionSummaryModelValidator,
    config_models.ConfigPropertyModel: ConfigPropertyModelValidator,
    config_models.ConfigPropertySnapshotMetadataModel: (
        ConfigPropertySnapshotMetadataModelValidator),
    config_models.ConfigPropertySnapshotContentModel: (
        ConfigPropertySnapshotContentModelValidator),
    email_models.SentEmailModel: SentEmailModelValidator,
    email_models.BulkEmailModel: BulkEmailModelValidator,
    email_models.GeneralFeedbackEmailReplyToIdModel: (
        GeneralFeedbackEmailReplyToIdModelValidator),
    exp_models.ExplorationModel: ExplorationModelValidator,
    exp_models.ExplorationSnapshotMetadataModel: (
        ExplorationSnapshotMetadataModelValidator),
    exp_models.ExplorationSnapshotContentModel: (
        ExplorationSnapshotContentModelValidator),
    exp_models.ExplorationRightsModel: ExplorationRightsModelValidator,
    exp_models.ExplorationRightsSnapshotMetadataModel: (
        ExplorationRightsSnapshotMetadataModelValidator),
    exp_models.ExplorationRightsSnapshotContentModel: (
        ExplorationRightsSnapshotContentModelValidator),
    exp_models.ExplorationCommitLogEntryModel: (
        ExplorationCommitLogEntryModelValidator),
    exp_models.ExpSummaryModel: ExpSummaryModelValidator,
    file_models.FileMetadataModel: FileMetadataModelValidator,
    file_models.FileMetadataSnapshotMetadataModel: (
        FileMetadataSnapshotMetadataModelValidator),
    file_models.FileMetadataSnapshotContentModel: (
        FileMetadataSnapshotContentModelValidator),
    file_models.FileModel: FileModelValidator,
    file_models.FileSnapshotMetadataModel: FileSnapshotMetadataModelValidator,
    file_models.FileSnapshotContentModel: FileSnapshotContentModelValidator,
    recommendations_models.ExplorationRecommendationsModel: (
        ExplorationRecommendationsModelValidator),
    recommendations_models.TopicSimilaritiesModel: (
        TopicSimilaritiesModelValidator),
    story_models.StoryModel: StoryModelValidator,
    story_models.StorySnapshotMetadataModel: (
        StorySnapshotMetadataModelValidator),
    story_models.StorySnapshotContentModel: (
        StorySnapshotContentModelValidator),
    story_models.StoryRightsModel: StoryRightsModelValidator,
    story_models.StoryRightsSnapshotMetadataModel: (
        StoryRightsSnapshotMetadataModelValidator),
    story_models.StoryRightsSnapshotContentModel: (
        StoryRightsSnapshotContentModelValidator),
    story_models.StoryCommitLogEntryModel: (
        StoryCommitLogEntryModelValidator),
    story_models.StorySummaryModel: StorySummaryModelValidator,
    user_models.UserSubscriptionsModel: UserSubscriptionsModelValidator,
}


class ProdValidationAuditOneOffJob(jobs.BaseMapReduceOneOffJobManager):
    """Job that audits and validates production models."""

    @classmethod
    def entity_classes_to_map_over(cls):
        raise NotImplementedError

    @staticmethod
    def map(model_instance):
        if not model_instance.deleted:
            model_name = model_instance.__class__.__name__
            validator_cls = MODEL_TO_VALIDATOR_MAPPING[type(model_instance)]
            validator = validator_cls()
            validator.validate(model_instance)
            if len(validator.errors) > 0:
                for error_key, error_val in (
                        validator.errors.iteritems()):
                    yield (
                        'failed validation check for %s of %s' % (
                            error_key, model_name),
                        (',').join(set(error_val)))
            else:
                yield (
                    'fully-validated %s' % model_name, 1)

    @staticmethod
    def reduce(key, values):
        if 'fully-validated' in key:
            yield (key, len(values))
        else:
            yield (key, values)


class ActivityReferencesModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates ActivityReferencesModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [activity_models.ActivityReferencesModel]


class RoleQueryAuditModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates RoleQueryAuditModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [audit_models.RoleQueryAuditModel]


class CollectionModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates CollectionModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [collection_models.CollectionModel]


class CollectionSnapshotMetadataModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates CollectionSnapshotMetadataModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [collection_models.CollectionSnapshotMetadataModel]


class CollectionSnapshotContentModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates CollectionSnapshotContentModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [collection_models.CollectionSnapshotContentModel]


class CollectionRightsModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates CollectionRightsModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [collection_models.CollectionRightsModel]


class CollectionRightsSnapshotMetadataModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates CollectionRightsSnapshotMetadataModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [collection_models.CollectionRightsSnapshotMetadataModel]


class CollectionRightsSnapshotContentModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates CollectionRightsSnapshotContentModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [collection_models.CollectionRightsSnapshotContentModel]


class CollectionCommitLogEntryModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates CollectionCommitLogEntryModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [collection_models.CollectionCommitLogEntryModel]


class CollectionSummaryModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates CollectionSummaryModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [collection_models.CollectionSummaryModel]


class ConfigPropertyModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates ConfigPropertyModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [config_models.ConfigPropertyModel]


class ConfigPropertySnapshotMetadataModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates ConfigPropertySnapshotMetadataModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [config_models.ConfigPropertySnapshotMetadataModel]


class ConfigPropertySnapshotContentModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates ConfigPropertySnapshotContentModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [config_models.ConfigPropertySnapshotContentModel]


class SentEmailModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates SentEmailModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [email_models.SentEmailModel]


class BulkEmailModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates BulkEmailModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [email_models.BulkEmailModel]


class GeneralFeedbackEmailReplyToIdModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates GeneralFeedbackEmailReplyToIdModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [email_models.GeneralFeedbackEmailReplyToIdModel]


class ExplorationModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates ExplorationModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [exp_models.ExplorationModel]


class ExplorationSnapshotMetadataModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates ExplorationSnapshotMetadataModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [exp_models.ExplorationSnapshotMetadataModel]


class ExplorationSnapshotContentModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates ExplorationSnapshotContentModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [exp_models.ExplorationSnapshotContentModel]


class ExplorationRightsModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates ExplorationRightsModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [exp_models.ExplorationRightsModel]


class ExplorationRightsSnapshotMetadataModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates ExplorationRightsSnapshotMetadataModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [exp_models.ExplorationRightsSnapshotMetadataModel]


class ExplorationRightsSnapshotContentModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates ExplorationRightsSnapshotContentModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [exp_models.ExplorationRightsSnapshotContentModel]


class ExplorationCommitLogEntryModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates ExplorationCommitLogEntryModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [exp_models.ExplorationCommitLogEntryModel]


class ExpSummaryModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates ExpSummaryModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [exp_models.ExpSummaryModel]


class FileMetadataModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates FileMetadataModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [file_models.FileMetadataModel]


class FileModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates FileModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [file_models.FileModel]


class FileSnapshotMetadataModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates FileSnapshotMetadataModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [file_models.FileSnapshotMetadataModel]


class FileSnapshotContentModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates FileSnapshotContentModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [file_models.FileSnapshotContentModel]


class FileMetadataSnapshotMetadataModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates FileMetadataSnapshotMetadataModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [file_models.FileMetadataSnapshotMetadataModel]


class FileMetadataSnapshotContentModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates FileMetadataSnapshotContentModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [file_models.FileMetadataSnapshotContentModel]


class ExplorationRecommendationsModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates ExplorationRecommendationsModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [recommendations_models.ExplorationRecommendationsModel]


class TopicSimilaritiesModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates TopicSimilaritiesModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [recommendations_models.TopicSimilaritiesModel]


class StoryModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates StoryModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [story_models.StoryModel]


class StorySnapshotMetadataModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates StorySnapshotMetadataModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [story_models.StorySnapshotMetadataModel]


class StorySnapshotContentModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates StorySnapshotContentModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [story_models.StorySnapshotContentModel]


class StoryRightsModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates StoryRightsModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [story_models.StoryRightsModel]


class StoryRightsSnapshotMetadataModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates StoryRightsSnapshotMetadataModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [story_models.StoryRightsSnapshotMetadataModel]


class StoryRightsSnapshotContentModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates StoryRightsSnapshotContentModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [story_models.StoryRightsSnapshotContentModel]


class StoryCommitLogEntryModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates StoryCommitLogEntryModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [story_models.StoryCommitLogEntryModel]


class StorySummaryModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates StorySummaryModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [story_models.StorySummaryModel]


class UserSubscriptionsModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates UserSubscriptionsModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [user_models.UserSubscriptionsModel]

# coding: utf-8
#
# Copyright 2019 The Oppia Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""One-off jobs for validating prod models."""

import collections
import datetime
import itertools
import re

from constants import constants
from core import jobs
from core.domain import activity_domain
from core.domain import collection_domain
from core.domain import collection_services
from core.domain import config_domain
from core.domain import exp_domain
from core.domain import exp_services
from core.domain import fs_domain
from core.domain import recommendations_services
from core.domain import rights_manager
from core.domain import story_domain
from core.domain import story_services
from core.platform import models
import feconf

(
    activity_models, audit_models, base_models,
    collection_models, config_models, email_models,
    exp_models, feedback_models, file_models,
    recommendations_models, story_models,
    user_models,) = (
        models.Registry.import_models([
            models.NAMES.activity, models.NAMES.audit, models.NAMES.base_model,
            models.NAMES.collection, models.NAMES.config, models.NAMES.email,
            models.NAMES.exploration, models.NAMES.feedback, models.NAMES.file,
            models.NAMES.recommendations, models.NAMES.story,
            models.NAMES.user]))
datastore_services = models.Registry.import_datastore_services()

ALLOWED_AUDIO_EXTENSIONS = feconf.ACCEPTED_AUDIO_EXTENSIONS.keys()
ALLOWED_IMAGE_EXTENSIONS = list(itertools.chain.from_iterable(
    feconf.ACCEPTED_IMAGE_FORMATS_AND_EXTENSIONS.values()))
ASSETS_PATH_REGEX = '/exploration/[A-Za-z0-9]{1,12}/assets/'
IMAGE_PATH_REGEX = (
    '%simage/[A-Za-z0-9_]{1,}\\.(%s)' % (
        ASSETS_PATH_REGEX, ('|').join(ALLOWED_IMAGE_EXTENSIONS)))
AUDIO_PATH_REGEX = (
    '%saudio/[A-Za-z0-9_]{1,}\\.(%s)' % (
        ASSETS_PATH_REGEX, ('|').join(ALLOWED_AUDIO_EXTENSIONS)))
FILE_MODELS_REGEX = '(%s|%s)' % (IMAGE_PATH_REGEX, AUDIO_PATH_REGEX)


class BaseModelValidator(object):
    """Base class for validating models."""

    # The dict to store errors found during audit of model.
    errors = collections.defaultdict(list)
    # external_instance_details is keyed by field name. Each value consists
    # of a list of (model class, external_key, external_model_instance)
    # tuples.
    external_instance_details = {}

    @classmethod
    def _get_model_id_regex(cls, unused_item):
        """Returns a regex for model id.

        This method can be overridden by subclasses, if needed.

        Args:
            unused_item: ndb.Model. Entity to validate.

        Returns:
            str. A regex pattern to be followed by the model id.
        """
        return '^[A-Za-z0-9-_]{1,%s}$' % base_models.ID_LENGTH

    @classmethod
    def _validate_model_id(cls, item):
        """Checks whether the id of model matches the regex specified for
        the model.

        Args:
            item: ndb.Model. Entity to validate.
        """
        regex_string = cls._get_model_id_regex(item)
        if not re.compile(regex_string).match(str(item.id)):
            cls.errors['model id check'].append((
                'Entity id %s: Entity id does not match regex pattern') % (
                    item.id))

    @classmethod
    def _get_model_domain_object_instance(cls, unused_item):
        """Returns a domain object instance created from the model.

        This method can be overridden by subclasses, if needed.

        Args:
            unused_item: ndb.Model. Entity to validate.

        Returns:
            *: A domain object to validate.
        """
        return None

    @classmethod
    def _validate_model_domain_object_instances(cls, item):
        """Checks that model instance passes the validation of the domain
        object for model.

        Args:
            item: ndb.Model. Entity to validate.
        """
        model_domain_object_instance = cls._get_model_domain_object_instance(
            item)

        if model_domain_object_instance is None:
            # No domain object exists for this storage model class.
            return

        try:
            model_domain_object_instance.validate()
        except Exception as e:
            cls.errors['domain object check'].append((
                'Entity id %s: Entity fails domain validation with the '
                'error %s' % (item.id, e)))

    @classmethod
    def _get_external_id_relationships(cls, item):
        """Returns a mapping of external id to model class.

        This should be implemented by subclasses.

        Args:
            item: ndb.Model. Entity to validate.

        Returns:
            dict(str, (ndb.Model, list(str)). A dictionary whose keys are
            field names of the model to validate, and whose values are tuples
            that consist of the external model class and list of keys to fetch.
        """
        raise NotImplementedError

    @classmethod
    def _validate_external_id_relationships(cls, item):
        """Check whether the external id properties on the model correspond
        to valid instances.

        Args:
            item: ndb.Model. Entity to validate.
        """
        for field_name, model_class_model_id_model_tuples in (
                cls.external_instance_details.iteritems()):
            for model_class, model_id, model in (
                    model_class_model_id_model_tuples):
                if model is None or model.deleted:
                    cls.errors['%s field check' % field_name].append((
                        'Entity id %s: based on field %s having'
                        ' value %s, expect model %s with id %s but it doesn\'t'
                        ' exist' % (
                            item.id, field_name, model_id,
                            str(model_class.__name__), model_id)))

    @classmethod
    def _fetch_external_instance_details(cls, item):
        """Fetch external models based on _get_external_id_relationships.

        This should be called before we call other _validate methods.

        Args:
            item: ndb.Model. Entity to validate.
        """
        multiple_models_keys_to_fetch = {}
        for field_name_debug, (model_class, keys_to_fetch) in (
                cls._get_external_id_relationships(item).iteritems()):
            multiple_models_keys_to_fetch[field_name_debug] = (
                model_class, keys_to_fetch)
        fetched_model_instances = (
            datastore_services.fetch_multiple_entities_by_ids_and_models(
                multiple_models_keys_to_fetch.values()))
        for (
                field_name, (model_class, field_values)), (
                    external_instance_details) in zip(
                        multiple_models_keys_to_fetch.iteritems(),
                        fetched_model_instances):
            cls.external_instance_details[field_name] = (
                zip(
                    [model_class] * len(field_values),
                    field_values, external_instance_details))

    @classmethod
    def _validate_model_time_fields(cls, item):
        """Checks the following relation for the model:
        model.created_on <= model.last_updated <= current time.

        Args:
            item: ndb.Model. Entity to validate.
        """
        if item.created_on > item.last_updated:
            cls.errors['time field relation check'].append((
                'Entity id %s: The created_on field has a value %s which is '
                'greater than the value %s of last_updated field'
                ) % (item.id, item.created_on, item.last_updated))

        current_datetime = datetime.datetime.utcnow()
        if item.last_updated > current_datetime:
            cls.errors['current time check'].append((
                'Entity id %s: The last_updated field has a value %s which is '
                'greater than the time when the job was run'
                ) % (item.id, item.last_updated))

    @classmethod
    def _get_custom_validation_functions(cls):
        """Returns the list of custom validation functions to run.

        This method can be overridden by subclasses, if needed.

        Each validation function should accept only a single arg, which is the
        model instance to validate.
        """
        return []

    @classmethod
    def validate(cls, item):
        """Run _fetch_external_instance_details and all _validate functions.

        Args:
            item: ndb.Model. Entity to validate.
        """
        cls.errors.clear()
        cls.external_instance_details.clear()
        cls._fetch_external_instance_details(item)

        cls._validate_model_id(item)
        cls._validate_model_time_fields(item)
        cls._validate_model_domain_object_instances(item)
        cls._validate_external_id_relationships(item)

        for func in cls._get_custom_validation_functions():
            func(item)


class BaseSummaryModelValidator(BaseModelValidator):
    """Base class for validating summary models."""

    @classmethod
    def _get_external_model_properties(cls):
        """Returns a tuple of external models and properties.

        This should be implemented by subclasses.

        Returns:
            tuple(str, list(tuple), dict): A tuple with first element as
                external model name, second element as a tuple of
                cls.external_instance_details and the third element
                as a properties dict with key as property name in summary
                model and value as property name in external model.
        """
        raise NotImplementedError

    @classmethod
    def _validate_external_model_properties(cls, item):
        """Validate that properties of the model match the corresponding
        properties of the external model.

        Args:
            item: ndb.Model. BaseSummaryModel to validate.
        """

        for (
                external_model_name,
                external_model_class_model_id_model_tuples,
                external_model_properties_dict
            ) in cls._get_external_model_properties():

            for (_, _, external_model) in (
                    external_model_class_model_id_model_tuples):
                # The case for missing external model is ignored here
                # since errors for missing external model are already
                # checked and stored in _validate_external_id_relationships
                # function.
                if external_model is None or external_model.deleted:
                    continue
                for (property_name, external_model_property_name) in (
                        external_model_properties_dict.iteritems()):
                    value_in_summary_model = getattr(item, property_name)
                    value_in_external_model = getattr(
                        external_model, external_model_property_name)

                    if value_in_summary_model != value_in_external_model:
                        cls.errors['%s field check' % property_name].append((
                            'Entity id %s: %s field in entity: %s does not '
                            'match corresponding %s %s field: %s') % (
                                item.id, property_name,
                                value_in_summary_model,
                                external_model_name,
                                external_model_property_name,
                                value_in_external_model))

    @classmethod
    def validate(cls, item):
        """Run _fetch_external_instance_details and all _validate functions.

        Args:
            item: ndb.Model. Entity to validate.
        """
        super(BaseSummaryModelValidator, cls).validate(item)

        cls._validate_external_model_properties(item)


class BaseSnapshotContentModelValidator(BaseModelValidator):
    """Base class for validating snapshot content models."""

    # The name of the model which is to be used in the error messages.
    # This can be overridden by subclasses, if needed.
    MODEL_NAME = 'snapshot content'

    # The name of the external model in lowercase which is used to obtain
    # the name of the key for the fetch of external model and the name
    # of the external model to be used in error messages.
    # For example, if external model is CollectionRights, then
    # EXTERNAL_MODEL_NAME = collection rights, key to fetch = collection_rights
    # Name of model to be used in error message = CollectionRights
    # This should be overridden by subclasses.
    EXTERNAL_MODEL_NAME = ''

    @classmethod
    def _get_model_id_regex(cls, unused_item):
        return '^[A-Za-z0-9]{1,%s}-\\d*$' % base_models.ID_LENGTH

    @classmethod
    def _validate_base_model_version_from_item_id(cls, item):
        """Validate that external model corresponding to item.id
        has a version greater than or equal to the version in item.id.

        Args:
            item: ndb.Model. BaseSnapshotContentModel to validate.
        """

        if cls.EXTERNAL_MODEL_NAME == '':
            raise Exception('External model name should be specified')

        external_model_name = cls.EXTERNAL_MODEL_NAME
        if item.id.startswith('rights'):
            external_model_name = external_model_name + ' rights'

        name_split_by_space = external_model_name.split(' ')
        key_to_fetch = ('_').join(name_split_by_space)
        capitalized_external_model_name = ('').join([
            val.capitalize() for val in name_split_by_space])

        external_model_class_model_id_model_tuples = (
            cls.external_instance_details['%s_ids' % key_to_fetch])

        version = item.id[item.id.rfind('-') + 1:]
        for (_, _, external_model) in (
                external_model_class_model_id_model_tuples):
            # The case for missing external model is ignored here
            # since errors for missing external model are already
            # checked and stored in _validate_external_id_relationships
            # function.
            if external_model is None or external_model.deleted:
                continue
            if int(external_model.version) < int(version):
                cls.errors[
                    '%s model version check' % cls.EXTERNAL_MODEL_NAME].append((
                        'Entity id %s: %s model corresponding to '
                        'id %s has a version %s which is less than '
                        'the version %s in %s model id' % (
                            item.id, capitalized_external_model_name,
                            external_model.id, external_model.version, version,
                            cls.MODEL_NAME)))

    @classmethod
    def validate(cls, item):
        """Run _fetch_external_instance_details and all _validate functions.

        Args:
            item: ndb.Model. Entity to validate.
        """
        super(BaseSnapshotContentModelValidator, cls).validate(item)

        cls._validate_base_model_version_from_item_id(item)


class BaseSnapshotMetadataModelValidator(BaseSnapshotContentModelValidator):
    """Base class for validating snapshot metadata models."""

    MODEL_NAME = 'snapshot metadata'

    @classmethod
    def _validate_commit_type(cls, item):
        """Validates that commit type is valid.

        Args:
            item: ndb.Model. Entity to validate.
        """
        if item.commit_type not in (
                base_models.VersionedModel.COMMIT_TYPE_CHOICES):
            cls.errors['commit type check'].append((
                'Entity id %s: Commit type %s is not allowed') % (
                    item.id, item.commit_type))

    @classmethod
    def _get_change_domain_class(cls, unused_item):
        """Returns a Change domain class.

        This should be implemented by subclasses.

        Args:
            unused_item: ndb.Model. Entity to validate.

        Returns:
            change_domain.BaseChange: A domain object class for the
                changes made by commit commands of the model.
        """
        raise NotImplementedError

    @classmethod
    def _validate_commit_cmds_schema(cls, item):
        """Validates schema of commit commands in commit_cmds dict.

        Args:
            item: ndb.Model. Entity to validate.
        """
        change_domain_object = cls._get_change_domain_class(item)
        if change_domain_object is None:
            # This is for cases where id of the entity is invalid
            # and no commit command domain object is found for the entity.
            # For example, if a CollectionCommitLogEntryModel does
            # not have id starting with collection/rights, there is
            # no commit command domain object defined for this model.
            cls.errors['commit cmd check'].append(
                'Entity id %s: No commit command domain object defined '
                'for entity with commands: %s' % (item.id, item.commit_cmds))
            return
        for commit_cmd_dict in item.commit_cmds:
            if not commit_cmd_dict:
                continue
            try:
                change_domain_object(commit_cmd_dict)
            except Exception as e:
                cmd_name = commit_cmd_dict.get('cmd')
                cls.errors['commit cmd %s check' % cmd_name].append((
                    'Entity id %s: Commit command domain validation for '
                    'command: %s failed with error: %s') % (
                        item.id, commit_cmd_dict, e))

    @classmethod
    def validate(cls, item):
        """Run _fetch_external_instance_details and all _validate functions.

        Args:
            item: ndb.Model. Entity to validate.
        """
        super(BaseSnapshotMetadataModelValidator, cls).validate(item)

        cls._validate_commit_type(item)
        cls._validate_commit_cmds_schema(item)


class BaseCommitLogEntryModelValidator(BaseSnapshotMetadataModelValidator):
    """Base class for validating commit log entry models."""

    MODEL_NAME = 'commit log entry'

    @classmethod
    def _validate_post_commit_status(cls, item):
        """Validates that post_commit_status is either public or private.

        Args:
            item: ndb.Model. Entity to validate.
        """
        if item.post_commit_status not in [
                feconf.POST_COMMIT_STATUS_PUBLIC,
                feconf.POST_COMMIT_STATUS_PRIVATE]:
            cls.errors['post commit status check'].append((
                'Entity id %s: Post commit status %s is invalid') % (
                    item.id, item.post_commit_status))

    @classmethod
    def _validate_post_commit_is_private(cls, item):
        """Validates that post_commit_is_private is true iff
        post_commit_status is private.

        Args:
            item: ndb.Model. Entity to validate.
        """
        if item.post_commit_status == feconf.POST_COMMIT_STATUS_PRIVATE and (
                not item.post_commit_is_private):
            cls.errors['post commit is private check'].append((
                'Entity id %s: Post commit status is private but '
                'post_commit_is_private is False') % item.id)

        if item.post_commit_status == feconf.POST_COMMIT_STATUS_PUBLIC and (
                item.post_commit_is_private):
            cls.errors['post commit is private check'].append((
                'Entity id %s: Post commit status is public but '
                'post_commit_is_private is True') % item.id)

    @classmethod
    def validate(cls, item):
        """Run _fetch_external_instance_details and all _validate functions.

        Args:
            item: ndb.Model. Entity to validate.
        """
        super(BaseCommitLogEntryModelValidator, cls).validate(item)

        cls._validate_post_commit_status(item)
        cls._validate_post_commit_is_private(item)


class ActivityReferencesModelValidator(BaseModelValidator):
    """Class for validating ActivityReferencesModels."""

    @classmethod
    def _get_model_id_regex(cls, unused_item):
        # Valid id: featured.
        regex_string = '^(%s)$' % '|'.join(
            feconf.ALL_ACTIVITY_REFERENCE_LIST_TYPES)
        return regex_string

    @classmethod
    def _get_model_domain_object_instance(cls, item):
        activity_references_list = []

        try:
            for reference in item.activity_references:
                activity_references_list.append(
                    activity_domain.ActivityReference(
                        reference['type'], reference['id']))
        except Exception as e:
            cls.errors['fetch properties'].append((
                'Entity id %s: Entity properties cannot be fetched completely '
                'with the error %s') % (item.id, e))
            return None

        return activity_domain.ActivityReferences(activity_references_list)

    @classmethod
    def _get_external_id_relationships(cls, item):
        exploration_ids = []
        collection_ids = []

        try:
            for reference in item.activity_references:
                if reference['type'] == constants.ACTIVITY_TYPE_EXPLORATION:
                    exploration_ids.append(reference['id'])
                elif reference['type'] == constants.ACTIVITY_TYPE_COLLECTION:
                    collection_ids.append(reference['id'])
        except Exception as e:
            cls.errors['fetch properties'].append((
                'Entity id %s: Entity properties cannot be fetched completely '
                'with the error %s') % (item.id, e))
            return {}

        return {
            'exploration_ids': (exp_models.ExplorationModel, exploration_ids),
            'collection_ids': (
                collection_models.CollectionModel, collection_ids),
        }


class RoleQueryAuditModelValidator(BaseModelValidator):
    """Class for validating RoleQueryAuditModels."""

    @classmethod
    def _get_model_id_regex(cls, item):
        # Valid id: [user_id].[timestamp_in_sec].[intent].[random_number]
        regex_string = '^%s\\.\\d*\\.%s\\.\\d*$' % (item.user_id, item.intent)
        return regex_string

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {'user_ids': (user_models.UserSettingsModel, [item.user_id])}


class CollectionModelValidator(BaseModelValidator):
    """Class for validating CollectionModel."""

    @classmethod
    def _get_model_domain_object_instance(cls, item):
        return collection_services.get_collection_from_model(item)

    @classmethod
    def _get_external_id_relationships(cls, item):
        snapshot_model_ids = [
            '%s-%d' % (item.id, version)
            for version in range(1, item.version + 1)]
        return {
            'exploration_ids': (
                exp_models.ExplorationModel,
                [node['exploration_id'] for node in item.collection_contents[
                    'nodes']]),
            'collection_commit_log_entry_ids': (
                collection_models.CollectionCommitLogEntryModel,
                ['collection-%s-%s' % (item.id, version) for version in range(
                    1, item.version + 1)]),
            'collection_summary_ids': (
                collection_models.CollectionSummaryModel, [item.id]),
            'collection_rights_ids': (
                collection_models.CollectionRightsModel, [item.id]),
            'snapshot_metadata_ids': (
                collection_models.CollectionSnapshotMetadataModel,
                snapshot_model_ids),
            'snapshot_content_ids': (
                collection_models.CollectionSnapshotContentModel,
                snapshot_model_ids),
        }


class CollectionSnapshotMetadataModelValidator(
        BaseSnapshotMetadataModelValidator):
    """Class for validating CollectionSnapshotMetadataModel."""

    EXTERNAL_MODEL_NAME = 'collection'

    @classmethod
    def _get_change_domain_class(cls, unused_item):
        return collection_domain.CollectionChange

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'collection_ids': (
                collection_models.CollectionModel,
                [item.id[:item.id.find('-')]]),
            'committer_ids': (
                user_models.UserSettingsModel, [item.committer_id])
        }


class CollectionSnapshotContentModelValidator(
        BaseSnapshotContentModelValidator):
    """Class for validating CollectionSnapshotContentModel."""

    EXTERNAL_MODEL_NAME = 'collection'

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'collection_ids': (
                collection_models.CollectionModel,
                [item.id[:item.id.find('-')]]),
        }


class CollectionRightsModelValidator(BaseModelValidator):
    """Class for validating CollectionRightsModel."""

    @classmethod
    def _get_external_id_relationships(cls, item):
        snapshot_model_ids = [
            '%s-%d' % (item.id, version)
            for version in range(1, item.version + 1)]
        return {
            'collection_ids': (
                collection_models.CollectionModel, [item.id]),
            'owner_user_ids': (
                user_models.UserSettingsModel, item.owner_ids),
            'editor_user_ids': (
                user_models.UserSettingsModel, item.editor_ids),
            'viewer_user_ids': (
                user_models.UserSettingsModel, item.viewer_ids),
            'snapshot_metadata_ids': (
                collection_models.CollectionRightsSnapshotMetadataModel,
                snapshot_model_ids),
            'snapshot_content_ids': (
                collection_models.CollectionRightsSnapshotContentModel,
                snapshot_model_ids),
        }

    @classmethod
    def _validate_first_published_msec(cls, item):
        """Validate that first published time of model is less than current
        time.

        Args:
            item: ndb.Model. CollectionRightsModel to validate.
        """
        if not item.first_published_msec:
            return

        epoch = datetime.datetime.utcfromtimestamp(0)
        current_msec = (
            datetime.datetime.utcnow() - epoch).total_seconds() * 1000.0
        if item.first_published_msec > current_msec:
            cls.errors['first published msec check'].append((
                'Entity id %s: The first_published_msec field has a value %s '
                'which is greater than the time when the job was run'
                ) % (item.id, item.first_published_msec))

    @classmethod
    def _get_custom_validation_functions(cls):
        return [cls._validate_first_published_msec]


class CollectionRightsSnapshotMetadataModelValidator(
        BaseSnapshotMetadataModelValidator):
    """Class for validating CollectionRightsSnapshotMetadataModel."""

    EXTERNAL_MODEL_NAME = 'collection rights'

    @classmethod
    def _get_change_domain_class(cls, unused_item):
        return rights_manager.CollectionRightsChange

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'collection_rights_ids': (
                collection_models.CollectionRightsModel,
                [item.id[:item.id.find('-')]]),
            'committer_ids': (
                user_models.UserSettingsModel, [item.committer_id])
        }


class CollectionRightsSnapshotContentModelValidator(
        BaseSnapshotContentModelValidator):
    """Class for validating CollectionRightsSnapshotContentModel."""

    EXTERNAL_MODEL_NAME = 'collection rights'

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'collection_rights_ids': (
                collection_models.CollectionRightsModel,
                [item.id[:item.id.find('-')]]),
        }


class CollectionCommitLogEntryModelValidator(BaseCommitLogEntryModelValidator):
    """Class for validating CollectionCommitLogEntryModel."""

    EXTERNAL_MODEL_NAME = 'collection'

    @classmethod
    def _get_model_id_regex(cls, item):
        # Valid id: [collection/rights]-[collection_id]-[collection_version].
        regex_string = '^(collection|rights)-%s-\\d*$' % (
            item.collection_id)

        return regex_string

    @classmethod
    def _get_change_domain_class(cls, item):
        if item.id.startswith('rights'):
            return rights_manager.CollectionRightsChange
        elif item.id.startswith('collection'):
            return collection_domain.CollectionChange
        else:
            # The case of invalid id is being ignored here since this
            # case will already be checked by the id regex test.
            return None

    @classmethod
    def _get_external_id_relationships(cls, item):
        external_id_relationships = {
            'collection_ids': (
                collection_models.CollectionModel, [item.collection_id]),
        }
        if item.id.startswith('rights'):
            external_id_relationships['collection_rights_ids'] = (
                collection_models.CollectionRightsModel, [item.collection_id])
        return external_id_relationships


class CollectionSummaryModelValidator(BaseSummaryModelValidator):
    """Class for validating CollectionSummaryModel."""

    @classmethod
    def _get_model_domain_object_instance(cls, item):
        return collection_services.get_collection_summary_from_model(item)

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'collection_ids': (
                collection_models.CollectionModel, [item.id]),
            'collection_rights_ids': (
                collection_models.CollectionRightsModel, [item.id]),
            'owner_user_ids': (
                user_models.UserSettingsModel, item.owner_ids),
            'editor_user_ids': (
                user_models.UserSettingsModel, item.editor_ids),
            'viewer_user_ids': (
                user_models.UserSettingsModel, item.viewer_ids),
            'contributor_user_ids': (
                user_models.UserSettingsModel, item.contributor_ids)
        }

    @classmethod
    def _validate_contributors_summary(cls, item):
        """Validate that contributor ids match the contributor ids obtained
        from contributors summary.

        Args:
            item: ndb.Model. CollectionSummaryModel to validate.
        """
        contributor_ids_from_contributors_summary = (
            item.contributors_summary.keys())
        if sorted(item.contributor_ids) != sorted(
                contributor_ids_from_contributors_summary):
            cls.errors['contributors summary check'].append((
                'Entity id %s: Contributor ids: %s do not match the '
                'contributor ids obtained using contributors summary: %s') % (
                    item.id, sorted(item.contributor_ids),
                    sorted(contributor_ids_from_contributors_summary)))

    @classmethod
    def _validate_node_count(cls, item):
        """Validate that node_count of model is equal to number of nodes
        in CollectionModel.collection_contents.

        Args:
            item: ndb.Model. CollectionSummaryModel to validate.
        """
        collection_model_class_model_id_model_tuples = (
            cls.external_instance_details['collection_ids'])

        for (_, _, collection_model) in (
                collection_model_class_model_id_model_tuples):
            # The case for missing collection external model is ignored here
            # since errors for missing collection external model are already
            # checked and stored in _validate_external_id_relationships
            # function.
            if collection_model is None or collection_model.deleted:
                continue
            nodes = collection_model.collection_contents['nodes']
            if item.node_count != len(nodes):
                cls.errors['node count check'].append((
                    'Entity id %s: Node count: %s does not match the number of '
                    'nodes in collection_contents dict: %s') % (
                        item.id, item.node_count, nodes))

    @classmethod
    def _validate_ratings_is_empty(cls, item):
        """Validate that ratings for the entity is empty.

        Args:
            item: ndb.Model. CollectionSummaryModel to validate.
        """
        if item.ratings:
            cls.errors['ratings check'].append(
                'Entity id %s: Expected ratings for the entity to be '
                'empty but received %s' % (item.id, item.ratings))

    @classmethod
    def _get_external_model_properties(cls):
        collection_model_class_model_id_model_tuples = (
            cls.external_instance_details['collection_ids'])
        collection_rights_model_class_model_id_model_tuples = (
            cls.external_instance_details['collection_rights_ids'])

        collection_model_properties_dict = {
            'title': 'title',
            'category': 'category',
            'objective': 'objective',
            'language_code': 'language_code',
            'tags': 'tags',
            'collection_model_created_on': 'created_on',
            'collection_model_last_updated': 'last_updated'
        }

        collection_rights_model_properties_dict = {
            'status': 'status',
            'community_owned': 'community_owned',
            'owner_ids': 'owner_ids',
            'editor_ids': 'editor_ids',
            'viewer_ids': 'viewer_ids',
        }

        return [(
            'collection',
            collection_model_class_model_id_model_tuples,
            collection_model_properties_dict
        ), (
            'collection rights',
            collection_rights_model_class_model_id_model_tuples,
            collection_rights_model_properties_dict
        )]

    @classmethod
    def _get_custom_validation_functions(cls):
        return [
            cls._validate_node_count,
            cls._validate_ratings_is_empty,
            cls._validate_contributors_summary,
            ]


class ConfigPropertyModelValidator(BaseModelValidator):
    """Class for validating ConfigPropertyModel."""

    @classmethod
    def _get_model_id_regex(cls, unused_item):
        return '^.*$'

    @classmethod
    def _get_external_id_relationships(cls, item):
        snapshot_model_ids = [
            '%s-%d' % (item.id, version)
            for version in range(1, item.version + 1)]
        return {
            'snapshot_metadata_ids': (
                config_models.ConfigPropertySnapshotMetadataModel,
                snapshot_model_ids),
            'snapshot_content_ids': (
                config_models.ConfigPropertySnapshotContentModel,
                snapshot_model_ids),
        }


class ConfigPropertySnapshotMetadataModelValidator(
        BaseSnapshotMetadataModelValidator):
    """Class for validating ConfigPropertySnapshotMetadataModel."""

    EXTERNAL_MODEL_NAME = 'config property'

    @classmethod
    def _get_model_id_regex(cls, unused_item):
        return '^.*-\\d*$'

    @classmethod
    def _get_change_domain_class(cls, unused_item):
        return config_domain.ConfigPropertyChange

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'config_property_ids': (
                config_models.ConfigPropertyModel,
                [item.id[:item.id.find('-')]]),
            'committer_ids': (
                user_models.UserSettingsModel, [item.committer_id])
        }


class ConfigPropertySnapshotContentModelValidator(
        BaseSnapshotContentModelValidator):
    """Class for validating ConfigPropertySnapshotContentModel."""

    EXTERNAL_MODEL_NAME = 'config property'

    @classmethod
    def _get_model_id_regex(cls, unused_item):
        return '^.*-\\d*$'

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'config_property_ids': (
                config_models.ConfigPropertyModel,
                [item.id[:item.id.find('-')]]),
        }


class SentEmailModelValidator(BaseModelValidator):
    """Class for validating SentEmailModels."""

    @classmethod
    def _get_model_id_regex(cls, item):
        # Valid id: [intent].[random hash]
        regex_string = '^%s\\.\\.[A-Za-z0-9]{1,%s}$' % (
            item.intent, base_models.ID_LENGTH)
        return regex_string

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'recipient_id': (
                user_models.UserSettingsModel, [item.recipient_id]),
            'sender_id': (user_models.UserSettingsModel, [item.sender_id]),
        }

    @classmethod
    def _validate_sent_datetime(cls, item):
        """Validate that sent_datetime of model is less than current time.

        Args:
            item: ndb.Model. SentEmailModel to validate.
        """
        current_datetime = datetime.datetime.utcnow()
        if item.sent_datetime > current_datetime:
            cls.errors['sent datetime check'].append((
                'Entity id %s: The sent_datetime field has a value %s which is '
                'greater than the time when the job was run'
                ) % (item.id, item.sent_datetime))

    @classmethod
    def _validate_sender_email(cls, item):
        """Validate that sender email corresponds to email of user obtained
        by using the sender_id.

        Args:
            item: ndb.Model. SentEmailModel to validate.
        """
        sender_model_class_model_id_model_tuples = (
            cls.external_instance_details['sender_id'])

        for (_, _, sender_model) in (
                sender_model_class_model_id_model_tuples):
            # The case for missing sender external model is ignored here
            # since errors for missing sender external model are already
            # checked and stored in _validate_external_id_relationships
            # function.
            if sender_model is None or sender_model.deleted:
                continue
            if sender_model.email != item.sender_email:
                cls.errors['sender email check'].append((
                    'Entity id %s: Sender email %s in entity does not '
                    'match with email %s of user obtained through '
                    'sender id %s') % (
                        item.id, item.sender_email, sender_model.email,
                        item.sender_id))

    @classmethod
    def _validate_recipient_email(cls, item):
        """Validate that recipient email corresponds to email of user obtained
        by using the recipient_id.

        Args:
            item: ndb.Model. SentEmailModel to validate.
        """
        recipient_model_class_model_id_model_tuples = (
            cls.external_instance_details['recipient_id'])

        for (_, _, recipient_model) in (
                recipient_model_class_model_id_model_tuples):
            # The case for missing recipient external model is ignored here
            # since errors for missing recipient external model are already
            # checked and stored in _validate_external_id_relationships
            # function.
            if recipient_model is None or recipient_model.deleted:
                continue
            if recipient_model.email != item.recipient_email:
                cls.errors['recipient email check'].append((
                    'Entity id %s: Recipient email %s in entity does '
                    'not match with email %s of user obtained through '
                    'recipient id %s') % (
                        item.id, item.recipient_email,
                        recipient_model.email, item.recipient_id))

    @classmethod
    def _get_custom_validation_functions(cls):
        return [
            cls._validate_sent_datetime,
            cls._validate_sender_email,
            cls._validate_recipient_email]


class BulkEmailModelValidator(BaseModelValidator):
    """Class for validating BulkEmailModels."""

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'recipient_id': (
                user_models.UserSettingsModel, item.recipient_ids),
            'sender_id': (user_models.UserSettingsModel, [item.sender_id]),
        }

    @classmethod
    def _validate_sent_datetime(cls, item):
        """Validate that sent_datetime of model is less than current time.

        Args:
            item: ndb.Model. BulkEmailModel to validate.
        """
        current_datetime = datetime.datetime.utcnow()
        if item.sent_datetime > current_datetime:
            cls.errors['sent datetime check'].append((
                'Entity id %s: The sent_datetime field has a value %s which is '
                'greater than the time when the job was run'
                ) % (item.id, item.sent_datetime))

    @classmethod
    def _validate_sender_email(cls, item):
        """Validate that sender email corresponds to email of user obtained
        by using the sender_id.

        Args:
            item: ndb.Model. BulkEmailModel to validate.
        """
        sender_model_class_model_id_model_tuples = (
            cls.external_instance_details['sender_id'])

        for (_, _, sender_model) in (
                sender_model_class_model_id_model_tuples):
            # The case for missing sender external model is ignored here
            # since errors for missing sender external model are already
            # checked and stored in _validate_external_id_relationships
            # function.
            if sender_model is None or sender_model.deleted:
                continue
            if sender_model.email != item.sender_email:
                cls.errors['sender email check'].append((
                    'Entity id %s: Sender email %s in entity does not '
                    'match with email %s of user obtained through '
                    'sender id %s') % (
                        item.id, item.sender_email, sender_model.email,
                        item.sender_id))

    @classmethod
    def _get_custom_validation_functions(cls):
        return [
            cls._validate_sent_datetime,
            cls._validate_sender_email]


class GeneralFeedbackEmailReplyToIdModelValidator(BaseModelValidator):
    """Class for validating GeneralFeedbackEmailReplyToIdModels."""

    @classmethod
    def _get_model_id_regex(cls, unused_item):
        return (
            '^\\d*\\.(exploration|topic)\\.[A-Za-z0-9]{1,12}\\.'
            '[A-Za-z0-9=+/]{1,}')

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'item.id.user_id': (
                user_models.UserSettingsModel, [
                    item.id[:item.id.find('.')]]),
            'item.id.thread_id': (
                feedback_models.GeneralFeedbackThreadModel, [
                    item.id[item.id.find('.') + 1:]]),
        }

    @classmethod
    def _validate_reply_to_id_length(cls, item):
        """Validate that reply_to_id length is less than or equal to
        REPLY_TO_ID_LENGTH.

        Args:
            item: ndb.Model. GeneralFeedbackEmailReplyToIdModel to validate.
        """
        # The reply_to_id of model is created using utils.get_random_int
        # method by using a upper bound as email_models.REPLY_TO_ID_LENGTH.
        # So, the reply_to_id length should be less than or equal to
        # email_models.REPLY_TO_ID_LENGTH.
        if len(item.reply_to_id) > email_models.REPLY_TO_ID_LENGTH:
            cls.errors['reply_to_id length check'].append((
                'Entity id %s: reply_to_id %s should have length less than or '
                'equal to %s but instead has length %s' % (
                    item.id, item.reply_to_id, email_models.REPLY_TO_ID_LENGTH,
                    len(item.reply_to_id))))

    @classmethod
    def _get_custom_validation_functions(cls):
        return [cls._validate_reply_to_id_length]


class ExplorationModelValidator(BaseModelValidator):
    """Class for validating ExplorationModel."""

    @classmethod
    def _get_model_domain_object_instance(cls, item):
        return exp_services.get_exploration_from_model(item)

    @classmethod
    def _get_external_id_relationships(cls, item):
        snapshot_model_ids = [
            '%s-%d' % (item.id, version)
            for version in range(1, item.version + 1)]
        return {
            'exploration_commit_log_entry_ids': (
                exp_models.ExplorationCommitLogEntryModel,
                ['exploration-%s-%s' % (item.id, version) for version in range(
                    1, item.version + 1)]),
            'exp_summary_ids': (
                exp_models.ExpSummaryModel, [item.id]),
            'exploration_rights_ids': (
                exp_models.ExplorationRightsModel, [item.id]),
            'snapshot_metadata_ids': (
                exp_models.ExplorationSnapshotMetadataModel,
                snapshot_model_ids),
            'snapshot_content_ids': (
                exp_models.ExplorationSnapshotContentModel,
                snapshot_model_ids),
        }


class ExplorationSnapshotMetadataModelValidator(
        BaseSnapshotMetadataModelValidator):
    """Class for validating ExplorationSnapshotMetadataModel."""

    EXTERNAL_MODEL_NAME = 'exploration'

    @classmethod
    def _get_change_domain_class(cls, unused_item):
        return exp_domain.ExplorationChange

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'exploration_ids': (
                exp_models.ExplorationModel, [item.id[:item.id.find('-')]]),
            'committer_ids': (
                user_models.UserSettingsModel, [item.committer_id])
        }


class ExplorationSnapshotContentModelValidator(
        BaseSnapshotContentModelValidator):
    """Class for validating ExplorationSnapshotContentModel."""

    EXTERNAL_MODEL_NAME = 'exploration'

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'exploration_ids': (
                exp_models.ExplorationModel, [item.id[:item.id.find('-')]]),
        }


class ExplorationRightsModelValidator(BaseModelValidator):
    """Class for validating ExplorationRightsModel."""

    @classmethod
    def _get_external_id_relationships(cls, item):
        cloned_from_exploration_id = []
        if item.cloned_from:
            cloned_from_exploration_id.append(item.cloned_from)
        snapshot_model_ids = [
            '%s-%d' % (item.id, version)
            for version in range(1, item.version + 1)]
        return {
            'exploration_ids': (
                exp_models.ExplorationModel, [item.id]),
            'cloned_from_exploration_ids': (
                exp_models.ExplorationModel,
                cloned_from_exploration_id),
            'owner_user_ids': (
                user_models.UserSettingsModel, item.owner_ids),
            'editor_user_ids': (
                user_models.UserSettingsModel, item.editor_ids),
            'viewer_user_ids': (
                user_models.UserSettingsModel, item.viewer_ids),
            'snapshot_metadata_ids': (
                exp_models.ExplorationRightsSnapshotMetadataModel,
                snapshot_model_ids),
            'snapshot_content_ids': (
                exp_models.ExplorationRightsSnapshotContentModel,
                snapshot_model_ids),
        }

    @classmethod
    def _validate_first_published_msec(cls, item):
        """Validate that first published time of model is less than current
        time.

        Args:
            item: ndb.Model. ExplorationRightsModel to validate.
        """
        if not item.first_published_msec:
            return

        epoch = datetime.datetime.utcfromtimestamp(0)
        current_msec = (
            datetime.datetime.utcnow() - epoch).total_seconds() * 1000.0
        if item.first_published_msec > current_msec:
            cls.errors['first published msec check'].append((
                'Entity id %s: The first_published_msec field has a value %s '
                'which is greater than the time when the job was run'
                ) % (item.id, item.first_published_msec))

    @classmethod
    def _get_custom_validation_functions(cls):
        return [cls._validate_first_published_msec]


class ExplorationRightsSnapshotMetadataModelValidator(
        BaseSnapshotMetadataModelValidator):
    """Class for validating ExplorationRightsSnapshotMetadataModel."""

    EXTERNAL_MODEL_NAME = 'exploration rights'

    @classmethod
    def _get_change_domain_class(cls, unused_item):
        return rights_manager.ExplorationRightsChange

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'exploration_rights_ids': (
                exp_models.ExplorationRightsModel,
                [item.id[:item.id.find('-')]]),
            'committer_ids': (
                user_models.UserSettingsModel, [item.committer_id])
        }


class ExplorationRightsSnapshotContentModelValidator(
        BaseSnapshotContentModelValidator):
    """Class for validating ExplorationRightsSnapshotContentModel."""

    EXTERNAL_MODEL_NAME = 'exploration rights'

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'exploration_rights_ids': (
                exp_models.ExplorationRightsModel,
                [item.id[:item.id.find('-')]]),
        }


class ExplorationCommitLogEntryModelValidator(BaseCommitLogEntryModelValidator):
    """Class for validating ExplorationCommitLogEntryModel."""

    EXTERNAL_MODEL_NAME = 'exploration'

    @classmethod
    def _get_model_id_regex(cls, item):
        # Valid id: [exploration/rights]-[exploration_id]-[exploration-version].
        regex_string = '^(exploration|rights)-%s-\\d*$' % (
            item.exploration_id)

        return regex_string

    @classmethod
    def _get_change_domain_class(cls, item):
        if item.id.startswith('rights'):
            return rights_manager.ExplorationRightsChange
        elif item.id.startswith('exploration'):
            return exp_domain.ExplorationChange
        else:
            # The case of invalid id is being ignored here since this
            # case will already be checked by the id regex test.
            return None

    @classmethod
    def _get_external_id_relationships(cls, item):
        external_id_relationships = {
            'exploration_ids': (
                exp_models.ExplorationModel, [item.exploration_id]),
        }
        if item.id.startswith('rights'):
            external_id_relationships['exploration_rights_ids'] = (
                exp_models.ExplorationRightsModel, [item.exploration_id])
        return external_id_relationships


class ExpSummaryModelValidator(BaseSummaryModelValidator):
    """Class for validating ExpSummaryModel."""

    @classmethod
    def _get_model_domain_object_instance(cls, item):
        return exp_services.get_exploration_summary_from_model(item)

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'exploration_ids': (
                exp_models.ExplorationModel, [item.id]),
            'exploration_rights_ids': (
                exp_models.ExplorationRightsModel, [item.id]),
            'owner_user_ids': (
                user_models.UserSettingsModel, item.owner_ids),
            'editor_user_ids': (
                user_models.UserSettingsModel, item.editor_ids),
            'viewer_user_ids': (
                user_models.UserSettingsModel, item.viewer_ids),
            'contributor_user_ids': (
                user_models.UserSettingsModel, item.contributor_ids)
        }

    @classmethod
    def _validate_contributors_summary(cls, item):
        """Validate that contributor ids match the contributor ids obtained
        from contributors summary.

        Args:
            item: ndb.Model. ExpSummaryModel to validate.
        """
        contributor_ids_from_contributors_summary = (
            item.contributors_summary.keys())
        if sorted(item.contributor_ids) != sorted(
                contributor_ids_from_contributors_summary):
            cls.errors['contributors summary check'].append((
                'Entity id %s: Contributor ids: %s do not match the '
                'contributor ids obtained using contributors summary: %s') % (
                    item.id, sorted(item.contributor_ids),
                    sorted(contributor_ids_from_contributors_summary)))

    @classmethod
    def _validate_first_published_msec(cls, item):
        """Validate that first published time of model is less than current
        time.

        Args:
            item: ndb.Model. ExpSummaryModel to validate.
        """
        if not item.first_published_msec:
            return

        epoch = datetime.datetime.utcfromtimestamp(0)
        current_msec = (
            datetime.datetime.utcnow() - epoch).total_seconds() * 1000.0
        if item.first_published_msec > current_msec:
            cls.errors['first published msec check'].append((
                'Entity id %s: The first_published_msec field has a value %s '
                'which is greater than the time when the job was run'
                ) % (item.id, item.first_published_msec))

    @classmethod
    def _validate_exploration_model_last_updated(cls, item):
        """Validate that item.exploration_model_last_updated matches the
        time when a last commit was made by a human contributor.

        Args:
            item: ndb.Model. ExpSummaryModel to validate.
        """
        exploration_model_class_model_id_model_tuples = (
            cls.external_instance_details['exploration_ids'])
        for (_, _, exploration_model) in (
                exploration_model_class_model_id_model_tuples):
            # The case for missing exploration external model is ignored here
            # since errors for missing exploration external model are already
            # checked and stored in _validate_external_id_relationships
            # function.
            if exploration_model is None or exploration_model.deleted:
                continue
            last_human_update_ms = exp_services.get_last_updated_by_human_ms(
                exploration_model.id)
            last_human_update_time = datetime.datetime.fromtimestamp(
                last_human_update_ms / 1000.0)
            if item.exploration_model_last_updated != last_human_update_time:
                cls.errors['exploration model last updated check'].append((
                    'Entity id %s: The exploration_model_last_updated '
                    'field: %s does not match the last time a commit was '
                    'made by a human contributor: %s') % (
                        item.id, item.exploration_model_last_updated,
                        last_human_update_time))

    @classmethod
    def _get_external_model_properties(cls):
        exploration_model_class_model_id_model_tuples = (
            cls.external_instance_details['exploration_ids'])
        exploration_rights_model_class_model_id_model_tuples = (
            cls.external_instance_details['exploration_rights_ids'])

        exploration_model_properties_dict = {
            'title': 'title',
            'category': 'category',
            'objective': 'objective',
            'language_code': 'language_code',
            'tags': 'tags',
            'exploration_model_created_on': 'created_on',
        }

        exploration_rights_model_properties_dict = {
            'first_published_msec': 'first_published_msec',
            'status': 'status',
            'community_owned': 'community_owned',
            'owner_ids': 'owner_ids',
            'editor_ids': 'editor_ids',
            'viewer_ids': 'viewer_ids',
        }

        return [(
            'exploration',
            exploration_model_class_model_id_model_tuples,
            exploration_model_properties_dict
        ), (
            'exploration rights',
            exploration_rights_model_class_model_id_model_tuples,
            exploration_rights_model_properties_dict
        )]

    @classmethod
    def _get_custom_validation_functions(cls):
        return [
            cls._validate_first_published_msec,
            cls._validate_contributors_summary,
            cls._validate_exploration_model_last_updated]


class FileMetadataModelValidator(BaseModelValidator):
    """Class for validating FileMetadataModel."""

    @classmethod
    def _get_model_id_regex(cls, unused_item):
        return '^%s$' % FILE_MODELS_REGEX

    @classmethod
    def _get_external_id_relationships(cls, item):
        snapshot_model_ids = [
            '%s-%d' % (item.id, version)
            for version in range(1, item.version + 1)]

        # Item id is of the format:
        # /exploration/exp_id/assets/(image|audio)/filepath.
        exp_val = '/exploration/'
        assets_val = '/assets'
        start_index = item.id.find(exp_val) + len(exp_val)
        end_index = item.id.find(assets_val)
        exp_id = item.id[start_index:end_index]

        return {
            'snapshot_metadata_ids': (
                file_models.FileMetadataSnapshotMetadataModel,
                snapshot_model_ids),
            'snapshot_content_ids': (
                file_models.FileMetadataSnapshotContentModel,
                snapshot_model_ids),
            'exploration_ids': (exp_models.ExplorationModel, [exp_id])
        }


class FileMetadataSnapshotMetadataModelValidator(
        BaseSnapshotMetadataModelValidator):
    """Class for validating FileMetadataSnapshotMetadataModel."""

    EXTERNAL_MODEL_NAME = 'file metadata'

    @classmethod
    def _get_model_id_regex(cls, unused_item):
        return '%s-\\d*$' % FILE_MODELS_REGEX

    @classmethod
    def _get_change_domain_class(cls, unused_item):
        return fs_domain.FileMetadataChange

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'file_metadata_ids': (
                file_models.FileMetadataModel,
                [item.id[:item.id.find('-')]]),
            'committer_ids': (
                user_models.UserSettingsModel, [item.committer_id])
        }


class FileMetadataSnapshotContentModelValidator(
        BaseSnapshotContentModelValidator):
    """Class for validating FileMetadataSnapshotContentModel."""

    EXTERNAL_MODEL_NAME = 'file metadata'

    @classmethod
    def _get_model_id_regex(cls, unused_item):
        return '%s-\\d*$' % FILE_MODELS_REGEX

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'file_metadata_ids': (
                file_models.FileMetadataModel,
                [item.id[:item.id.find('-')]]),
        }


class FileModelValidator(BaseModelValidator):
    """Class for validating FileModel."""

    @classmethod
    def _get_model_id_regex(cls, unused_item):
        return '^%s$' % FILE_MODELS_REGEX

    @classmethod
    def _get_external_id_relationships(cls, item):
        snapshot_model_ids = [
            '%s-%d' % (item.id, version)
            for version in range(1, item.version + 1)]

        # Item id is of the format:
        # /exploration/exp_id/assets/(image|audio)/filepath.
        exp_val = '/exploration/'
        assets_val = '/assets'
        start_index = item.id.find(exp_val) + len(exp_val)
        end_index = item.id.find(assets_val)
        exp_id = item.id[start_index:end_index]

        return {
            'snapshot_metadata_ids': (
                file_models.FileSnapshotMetadataModel,
                snapshot_model_ids),
            'snapshot_content_ids': (
                file_models.FileSnapshotContentModel,
                snapshot_model_ids),
            'exploration_ids': (exp_models.ExplorationModel, [exp_id])
        }


class FileSnapshotMetadataModelValidator(BaseSnapshotMetadataModelValidator):
    """Class for validating FileSnapshotMetadataModel."""

    EXTERNAL_MODEL_NAME = 'file'

    @classmethod
    def _get_model_id_regex(cls, unused_item):
        return '%s-\\d*$' % FILE_MODELS_REGEX

    @classmethod
    def _get_change_domain_class(cls, unused_item):
        return fs_domain.FileChange

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'file_ids': (
                file_models.FileModel,
                [item.id[:item.id.find('-')]]),
            'committer_ids': (
                user_models.UserSettingsModel, [item.committer_id])
        }


class FileSnapshotContentModelValidator(BaseSnapshotContentModelValidator):
    """Class for validating FileSnapshotContentModel."""

    EXTERNAL_MODEL_NAME = 'file'

    @classmethod
    def _get_model_id_regex(cls, unused_item):
        return '%s-\\d*$' % FILE_MODELS_REGEX

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'file_ids': (
                file_models.FileModel,
                [item.id[:item.id.find('-')]]),
        }


class ExplorationRecommendationsModelValidator(BaseModelValidator):
    """Class for validating ExplorationRecommendationsModel."""

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'exploration_ids': (
                exp_models.ExplorationModel,
                [item.id] + item.recommended_exploration_ids),
        }

    @classmethod
    def _validate_item_id_not_in_recommended_exploration_ids(cls, item):
        """Validate that model id is not present in recommended exploration ids.

        Args:
            item: ndb.Model. ExplorationRecommendationsModel to validate.
        """
        if item.id in item.recommended_exploration_ids:
            cls.errors['item exploration id check'].append((
                'Entity id %s: The exploration id: %s for which the entity is '
                'created is also present in the recommended exploration ids '
                'for entity') % (item.id, item.id))

    @classmethod
    def _get_custom_validation_functions(cls):
        return [cls._validate_item_id_not_in_recommended_exploration_ids]


class TopicSimilaritiesModelValidator(BaseModelValidator):
    """Class for validating TopicSimilaritiesModel."""

    @classmethod
    def _get_model_id_regex(cls, unused_item):
        # Valid id: topics.
        return '^%s$' % recommendations_models.TOPIC_SIMILARITIES_ID

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {}

    @classmethod
    def _validate_topic_similarities(cls, item):
        """Validate the topic similarities to be symmetric and have real
        values between 0.0 and 1.0.

        Args:
            item: ndb.Model. TopicSimilaritiesModel to validate.
        """

        topics = item.content.keys()
        data = '%s\n' % (',').join(topics)

        for topic1 in topics:
            similarity_list = []
            for topic2 in item.content[topic1]:
                similarity_list.append(str(item.content[topic1][topic2]))
            if len(similarity_list):
                data = data + '%s\n' % (',').join(similarity_list)

        try:
            recommendations_services.validate_topic_similarities(data)
        except Exception as e:
            cls.errors['topic similarity check'].append(
                'Entity id %s: Topic similarity validation for content: %s '
                'fails with error: %s' % (item.id, item.content, e))

    @classmethod
    def _get_custom_validation_functions(cls):
        return [cls._validate_topic_similarities]


class StoryModelValidator(BaseModelValidator):
    """Class for validating StoryModel."""

    @classmethod
    def _get_model_domain_object_instance(cls, item):
        return story_services.get_story_from_model(item)

    @classmethod
    def _get_external_id_relationships(cls, item):
        snapshot_model_ids = [
            '%s-%d' % (item.id, version)
            for version in range(1, item.version + 1)]
        return {
            'story_commit_log_entry_ids': (
                story_models.StoryCommitLogEntryModel,
                ['story-%s-%s' % (item.id, version) for version in range(
                    1, item.version + 1)]),
            'story_summary_ids': (
                story_models.StorySummaryModel, [item.id]),
            'story_rights_ids': (
                story_models.StoryRightsModel, [item.id]),
            'snapshot_metadata_ids': (
                story_models.StorySnapshotMetadataModel,
                snapshot_model_ids),
            'snapshot_content_ids': (
                story_models.StorySnapshotContentModel,
                snapshot_model_ids),
            'exploration_ids': (
                exp_models.ExplorationModel,
                [node['exploration_id'] for node in (
                    item.story_contents['nodes'])])
        }


class StorySnapshotMetadataModelValidator(BaseSnapshotMetadataModelValidator):
    """Class for validating StorySnapshotMetadataModel."""

    EXTERNAL_MODEL_NAME = 'story'

    @classmethod
    def _get_change_domain_class(cls, unused_item):
        return story_domain.StoryChange

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'story_ids': (
                story_models.StoryModel, [item.id[:item.id.find('-')]]),
            'committer_ids': (
                user_models.UserSettingsModel, [item.committer_id])
        }


class StorySnapshotContentModelValidator(BaseSnapshotContentModelValidator):
    """Class for validating StorySnapshotContentModel."""

    EXTERNAL_MODEL_NAME = 'story'

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'story_ids': (
                story_models.StoryModel, [item.id[:item.id.find('-')]]),
        }


class StoryRightsModelValidator(BaseModelValidator):
    """Class for validating StoryRightsModel."""

    @classmethod
    def _get_external_id_relationships(cls, item):
        snapshot_model_ids = [
            '%s-%d' % (item.id, version)
            for version in range(1, item.version + 1)]
        return {
            'story_ids': (
                story_models.StoryModel, [item.id]),
            'manager_user_ids': (
                user_models.UserSettingsModel, item.manager_ids),
            'snapshot_metadata_ids': (
                story_models.StoryRightsSnapshotMetadataModel,
                snapshot_model_ids),
            'snapshot_content_ids': (
                story_models.StoryRightsSnapshotContentModel,
                snapshot_model_ids),
        }


class StoryRightsSnapshotMetadataModelValidator(
        BaseSnapshotMetadataModelValidator):
    """Class for validating StoryRightsSnapshotMetadataModel."""

    EXTERNAL_MODEL_NAME = 'story rights'

    @classmethod
    def _get_change_domain_class(cls, unused_item):
        return story_domain.StoryRightsChange

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'story_rights_ids': (
                story_models.StoryRightsModel,
                [item.id[:item.id.find('-')]]),
            'committer_ids': (
                user_models.UserSettingsModel, [item.committer_id])
        }


class StoryRightsSnapshotContentModelValidator(
        BaseSnapshotContentModelValidator):
    """Class for validating StoryRightsSnapshotContentModel."""

    EXTERNAL_MODEL_NAME = 'story rights'

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'story_rights_ids': (
                story_models.StoryRightsModel,
                [item.id[:item.id.find('-')]]),
        }


class StoryCommitLogEntryModelValidator(BaseCommitLogEntryModelValidator):
    """Class for validating StoryCommitLogEntryModel."""

    EXTERNAL_MODEL_NAME = 'story'

    @classmethod
    def _get_model_id_regex(cls, item):
        # Valid id: [story]-[story_id]-[story_version].
        regex_string = '^(story)-%s-\\d*$' % (
            item.story_id)

        return regex_string

    @classmethod
    def _get_change_domain_class(cls, item):
        if item.id.startswith('story'):
            return story_domain.StoryChange
        else:
            # The case of invalid id is being ignored here since this
            # case will already be checked by the id regex test.
            return None

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'story_ids': (
                story_models.StoryModel, [item.story_id]),
        }


class StorySummaryModelValidator(BaseSummaryModelValidator):
    """Class for validating StorySummaryModel."""

    @classmethod
    def _get_model_domain_object_instance(cls, item):
        return story_services.get_story_summary_from_model(item)

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'story_ids': (
                story_models.StoryModel, [item.id]),
            'story_rights_ids': (
                story_models.StoryRightsModel, [item.id]),
        }

    @classmethod
    def _validate_node_count(cls, item):
        """Validate that node_count of model is equal to number of nodes
        in StoryModel.story_contents.

        Args:
            item: ndb.Model. StorySummaryModel to validate.
        """
        story_model_class_model_id_model_tuples = cls.external_instance_details[
            'story_ids']

        for (_, _, story_model) in story_model_class_model_id_model_tuples:
            # The case for missing story external model is ignored here
            # since errors for missing story external model are already
            # checked and stored in _validate_external_id_relationships
            # function.
            if story_model is None or story_model.deleted:
                continue
            nodes = story_model.story_contents['nodes']
            if item.node_count != len(nodes):
                cls.errors['node count check'].append((
                    'Entity id %s: Node count: %s does not match the '
                    'number of nodes in story_contents dict: %s') % (
                        item.id, item.node_count, nodes))

    @classmethod
    def _get_external_model_properties(cls):
        story_model_class_model_id_model_tuples = cls.external_instance_details[
            'story_ids']

        story_model_properties_dict = {
            'title': 'title',
            'language_code': 'language_code',
            'description': 'description',
            'story_model_created_on': 'created_on',
            'story_model_last_updated': 'last_updated'
        }

        return [(
            'story',
            story_model_class_model_id_model_tuples,
            story_model_properties_dict
        )]

    @classmethod
    def _get_custom_validation_functions(cls):
        return [cls._validate_node_count]


class UserSubscriptionsModelValidator(BaseModelValidator):
    """Class for validating UserSubscriptionsModels."""

    @classmethod
    def _get_model_id_regex(cls, unused_item):
        return '^\\d*$'

    @classmethod
    def _get_external_id_relationships(cls, item):
        return {
            'activity_ids': (exp_models.ExplorationModel, item.activity_ids),
            'collection_ids': (
                collection_models.CollectionModel,
                item.collection_ids),
            'general_feedback_thread_ids': (
                feedback_models.GeneralFeedbackThreadModel,
                item.general_feedback_thread_ids),
            'creator_ids': (user_models.UserSettingsModel, item.creator_ids),
            'id': (user_models.UserSettingsModel, [item.id]),
        }


MODEL_TO_VALIDATOR_MAPPING = {
    activity_models.ActivityReferencesModel: ActivityReferencesModelValidator,
    audit_models.RoleQueryAuditModel: RoleQueryAuditModelValidator,
    collection_models.CollectionModel: CollectionModelValidator,
    collection_models.CollectionSnapshotMetadataModel: (
        CollectionSnapshotMetadataModelValidator),
    collection_models.CollectionSnapshotContentModel: (
        CollectionSnapshotContentModelValidator),
    collection_models.CollectionRightsModel: CollectionRightsModelValidator,
    collection_models.CollectionRightsSnapshotMetadataModel: (
        CollectionRightsSnapshotMetadataModelValidator),
    collection_models.CollectionRightsSnapshotContentModel: (
        CollectionRightsSnapshotContentModelValidator),
    collection_models.CollectionCommitLogEntryModel: (
        CollectionCommitLogEntryModelValidator),
    collection_models.CollectionSummaryModel: CollectionSummaryModelValidator,
    config_models.ConfigPropertyModel: ConfigPropertyModelValidator,
    config_models.ConfigPropertySnapshotMetadataModel: (
        ConfigPropertySnapshotMetadataModelValidator),
    config_models.ConfigPropertySnapshotContentModel: (
        ConfigPropertySnapshotContentModelValidator),
    email_models.SentEmailModel: SentEmailModelValidator,
    email_models.BulkEmailModel: BulkEmailModelValidator,
    email_models.GeneralFeedbackEmailReplyToIdModel: (
        GeneralFeedbackEmailReplyToIdModelValidator),
    exp_models.ExplorationModel: ExplorationModelValidator,
    exp_models.ExplorationSnapshotMetadataModel: (
        ExplorationSnapshotMetadataModelValidator),
    exp_models.ExplorationSnapshotContentModel: (
        ExplorationSnapshotContentModelValidator),
    exp_models.ExplorationRightsModel: ExplorationRightsModelValidator,
    exp_models.ExplorationRightsSnapshotMetadataModel: (
        ExplorationRightsSnapshotMetadataModelValidator),
    exp_models.ExplorationRightsSnapshotContentModel: (
        ExplorationRightsSnapshotContentModelValidator),
    exp_models.ExplorationCommitLogEntryModel: (
        ExplorationCommitLogEntryModelValidator),
    exp_models.ExpSummaryModel: ExpSummaryModelValidator,
    file_models.FileMetadataModel: FileMetadataModelValidator,
    file_models.FileMetadataSnapshotMetadataModel: (
        FileMetadataSnapshotMetadataModelValidator),
    file_models.FileMetadataSnapshotContentModel: (
        FileMetadataSnapshotContentModelValidator),
    file_models.FileModel: FileModelValidator,
    file_models.FileSnapshotMetadataModel: FileSnapshotMetadataModelValidator,
    file_models.FileSnapshotContentModel: FileSnapshotContentModelValidator,
    recommendations_models.ExplorationRecommendationsModel: (
        ExplorationRecommendationsModelValidator),
    recommendations_models.TopicSimilaritiesModel: (
        TopicSimilaritiesModelValidator),
    story_models.StoryModel: StoryModelValidator,
    story_models.StorySnapshotMetadataModel: (
        StorySnapshotMetadataModelValidator),
    story_models.StorySnapshotContentModel: (
        StorySnapshotContentModelValidator),
    story_models.StoryRightsModel: StoryRightsModelValidator,
    story_models.StoryRightsSnapshotMetadataModel: (
        StoryRightsSnapshotMetadataModelValidator),
    story_models.StoryRightsSnapshotContentModel: (
        StoryRightsSnapshotContentModelValidator),
    story_models.StoryCommitLogEntryModel: (
        StoryCommitLogEntryModelValidator),
    story_models.StorySummaryModel: StorySummaryModelValidator,
    user_models.UserSubscriptionsModel: UserSubscriptionsModelValidator,
}


class ProdValidationAuditOneOffJob(jobs.BaseMapReduceOneOffJobManager):
    """Job that audits and validates production models."""

    @classmethod
    def entity_classes_to_map_over(cls):
        raise NotImplementedError

    @staticmethod
    def map(model_instance):
        if not model_instance.deleted:
            model_name = model_instance.__class__.__name__
            validator_cls = MODEL_TO_VALIDATOR_MAPPING[type(model_instance)]
            validator = validator_cls()
            validator.validate(model_instance)
            if len(validator.errors) > 0:
                for error_key, error_val in (
                        validator.errors.iteritems()):
                    yield (
                        'failed validation check for %s of %s' % (
                            error_key, model_name),
                        (',').join(set(error_val)))
            else:
                yield (
                    'fully-validated %s' % model_name, 1)

    @staticmethod
    def reduce(key, values):
        if 'fully-validated' in key:
            yield (key, len(values))
        else:
            yield (key, values)


class ActivityReferencesModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates ActivityReferencesModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [activity_models.ActivityReferencesModel]


class RoleQueryAuditModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates RoleQueryAuditModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [audit_models.RoleQueryAuditModel]


class CollectionModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates CollectionModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [collection_models.CollectionModel]


class CollectionSnapshotMetadataModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates CollectionSnapshotMetadataModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [collection_models.CollectionSnapshotMetadataModel]


class CollectionSnapshotContentModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates CollectionSnapshotContentModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [collection_models.CollectionSnapshotContentModel]


class CollectionRightsModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates CollectionRightsModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [collection_models.CollectionRightsModel]


class CollectionRightsSnapshotMetadataModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates CollectionRightsSnapshotMetadataModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [collection_models.CollectionRightsSnapshotMetadataModel]


class CollectionRightsSnapshotContentModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates CollectionRightsSnapshotContentModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [collection_models.CollectionRightsSnapshotContentModel]


class CollectionCommitLogEntryModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates CollectionCommitLogEntryModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [collection_models.CollectionCommitLogEntryModel]


class CollectionSummaryModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates CollectionSummaryModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [collection_models.CollectionSummaryModel]


class ConfigPropertyModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates ConfigPropertyModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [config_models.ConfigPropertyModel]


class ConfigPropertySnapshotMetadataModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates ConfigPropertySnapshotMetadataModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [config_models.ConfigPropertySnapshotMetadataModel]


class ConfigPropertySnapshotContentModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates ConfigPropertySnapshotContentModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [config_models.ConfigPropertySnapshotContentModel]


class SentEmailModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates SentEmailModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [email_models.SentEmailModel]


class BulkEmailModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates BulkEmailModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [email_models.BulkEmailModel]


class GeneralFeedbackEmailReplyToIdModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates GeneralFeedbackEmailReplyToIdModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [email_models.GeneralFeedbackEmailReplyToIdModel]


class ExplorationModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates ExplorationModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [exp_models.ExplorationModel]


class ExplorationSnapshotMetadataModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates ExplorationSnapshotMetadataModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [exp_models.ExplorationSnapshotMetadataModel]


class ExplorationSnapshotContentModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates ExplorationSnapshotContentModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [exp_models.ExplorationSnapshotContentModel]


class ExplorationRightsModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates ExplorationRightsModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [exp_models.ExplorationRightsModel]


class ExplorationRightsSnapshotMetadataModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates ExplorationRightsSnapshotMetadataModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [exp_models.ExplorationRightsSnapshotMetadataModel]


class ExplorationRightsSnapshotContentModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates ExplorationRightsSnapshotContentModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [exp_models.ExplorationRightsSnapshotContentModel]


class ExplorationCommitLogEntryModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates ExplorationCommitLogEntryModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [exp_models.ExplorationCommitLogEntryModel]


class ExpSummaryModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates ExpSummaryModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [exp_models.ExpSummaryModel]


class FileMetadataModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates FileMetadataModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [file_models.FileMetadataModel]


class FileModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates FileModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [file_models.FileModel]


class FileSnapshotMetadataModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates FileSnapshotMetadataModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [file_models.FileSnapshotMetadataModel]


class FileSnapshotContentModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates FileSnapshotContentModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [file_models.FileSnapshotContentModel]


class FileMetadataSnapshotMetadataModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates FileMetadataSnapshotMetadataModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [file_models.FileMetadataSnapshotMetadataModel]


class FileMetadataSnapshotContentModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates FileMetadataSnapshotContentModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [file_models.FileMetadataSnapshotContentModel]


class ExplorationRecommendationsModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates ExplorationRecommendationsModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [recommendations_models.ExplorationRecommendationsModel]


class TopicSimilaritiesModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates TopicSimilaritiesModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [recommendations_models.TopicSimilaritiesModel]


class StoryModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates StoryModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [story_models.StoryModel]


class StorySnapshotMetadataModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates StorySnapshotMetadataModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [story_models.StorySnapshotMetadataModel]


class StorySnapshotContentModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates StorySnapshotContentModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [story_models.StorySnapshotContentModel]


class StoryRightsModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates StoryRightsModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [story_models.StoryRightsModel]


class StoryRightsSnapshotMetadataModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates StoryRightsSnapshotMetadataModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [story_models.StoryRightsSnapshotMetadataModel]


class StoryRightsSnapshotContentModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates StoryRightsSnapshotContentModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [story_models.StoryRightsSnapshotContentModel]


class StoryCommitLogEntryModelAuditOneOffJob(
        ProdValidationAuditOneOffJob):
    """Job that audits and validates StoryCommitLogEntryModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [story_models.StoryCommitLogEntryModel]


class StorySummaryModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates StorySummaryModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [story_models.StorySummaryModel]


class UserSubscriptionsModelAuditOneOffJob(ProdValidationAuditOneOffJob):
    """Job that audits and validates UserSubscriptionsModel."""

    @classmethod
    def entity_classes_to_map_over(cls):
        return [user_models.UserSubscriptionsModel]
# coding: utf-8 # # Copyright 2019 The Oppia Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #      http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS-IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. One-off jobs for validating prod models. Base class for validating models. # The dict to store errors found during audit of model. # external_instance_details is keyed by field name. Each value consists # of a list of (model class, external_key, external_model_instance) # tuples. Returns a regex for model id.          This method can be overridden by subclasses, if needed.          Args:             unused_item: ndb.Model. Entity to validate.          Returns:             str. A regex pattern to be followed by the model id. Checks whether the id of model matches the regex specified for         the model.          Args:             item: ndb.Model. Entity to validate. Returns a domain object instance created from the model.          This method can be overridden by subclasses, if needed.          Args:             unused_item: ndb.Model. Entity to validate.          Returns:             *: A domain object to validate. Checks that model instance passes the validation of the domain         object for model.          Args:             item: ndb.Model. Entity to validate. # No domain object exists for this storage model class. Returns a mapping of external id to model class.          This should be implemented by subclasses.          Args:             item: ndb.Model. Entity to validate.          Returns:             dict(str, (ndb.Model, list(str)). A dictionary whose keys are             field names of the model to validate, and whose values are tuples             that consist of the external model class and list of keys to fetch. Check whether the external id properties on the model correspond         to valid instances.          Args:             item: ndb.Model. Entity to validate. Fetch external models based on _get_external_id_relationships.          This should be called before we call other _validate methods.          Args:             item: ndb.Model. Entity to validate. Checks the following relation for the model:         model.created_on <= model.last_updated <= current time.          Args:             item: ndb.Model. Entity to validate. Returns the list of custom validation functions to run.          This method can be overridden by subclasses, if needed.          Each validation function should accept only a single arg, which is the         model instance to validate. Run _fetch_external_instance_details and all _validate functions.          Args:             item: ndb.Model. Entity to validate. Base class for validating summary models. Returns a tuple of external models and properties.          This should be implemented by subclasses.          Returns:             tuple(str, list(tuple), dict): A tuple with first element as                 external model name, second element as a tuple of                 cls.external_instance_details and the third element                 as a properties dict with key as property name in summary                 model and value as property name in external model. Validate that properties of the model match the corresponding         properties of the external model.          Args:             item: ndb.Model. BaseSummaryModel to validate. # The case for missing external model is ignored here # since errors for missing external model are already # checked and stored in _validate_external_id_relationships # function. Run _fetch_external_instance_details and all _validate functions.          Args:             item: ndb.Model. Entity to validate. Base class for validating snapshot content models. # The name of the model which is to be used in the error messages. # This can be overridden by subclasses, if needed. # The name of the external model in lowercase which is used to obtain # the name of the key for the fetch of external model and the name # of the external model to be used in error messages. # For example, if external model is CollectionRights, then # EXTERNAL_MODEL_NAME = collection rights, key to fetch = collection_rights # Name of model to be used in error message = CollectionRights # This should be overridden by subclasses. Validate that external model corresponding to item.id         has a version greater than or equal to the version in item.id.          Args:             item: ndb.Model. BaseSnapshotContentModel to validate. # The case for missing external model is ignored here # since errors for missing external model are already # checked and stored in _validate_external_id_relationships # function. Run _fetch_external_instance_details and all _validate functions.          Args:             item: ndb.Model. Entity to validate. Base class for validating snapshot metadata models. Validates that commit type is valid.          Args:             item: ndb.Model. Entity to validate. Returns a Change domain class.          This should be implemented by subclasses.          Args:             unused_item: ndb.Model. Entity to validate.          Returns:             change_domain.BaseChange: A domain object class for the                 changes made by commit commands of the model. Validates schema of commit commands in commit_cmds dict.          Args:             item: ndb.Model. Entity to validate. # This is for cases where id of the entity is invalid # and no commit command domain object is found for the entity. # For example, if a CollectionCommitLogEntryModel does # not have id starting with collection/rights, there is # no commit command domain object defined for this model. Run _fetch_external_instance_details and all _validate functions.          Args:             item: ndb.Model. Entity to validate. Base class for validating commit log entry models. Validates that post_commit_status is either public or private.          Args:             item: ndb.Model. Entity to validate. Validates that post_commit_is_private is true iff         post_commit_status is private.          Args:             item: ndb.Model. Entity to validate. Run _fetch_external_instance_details and all _validate functions.          Args:             item: ndb.Model. Entity to validate. Class for validating ActivityReferencesModels. # Valid id: featured. Class for validating RoleQueryAuditModels. # Valid id: [user_id].[timestamp_in_sec].[intent].[random_number] Class for validating CollectionModel. Class for validating CollectionSnapshotMetadataModel. Class for validating CollectionSnapshotContentModel. Class for validating CollectionRightsModel. Validate that first published time of model is less than current         time.          Args:             item: ndb.Model. CollectionRightsModel to validate. Class for validating CollectionRightsSnapshotMetadataModel. Class for validating CollectionRightsSnapshotContentModel. Class for validating CollectionCommitLogEntryModel. # Valid id: [collection/rights]-[collection_id]-[collection_version]. # The case of invalid id is being ignored here since this # case will already be checked by the id regex test. Class for validating CollectionSummaryModel. Validate that contributor ids match the contributor ids obtained         from contributors summary.          Args:             item: ndb.Model. CollectionSummaryModel to validate. Validate that node_count of model is equal to number of nodes         in CollectionModel.collection_contents.          Args:             item: ndb.Model. CollectionSummaryModel to validate. # The case for missing collection external model is ignored here # since errors for missing collection external model are already # checked and stored in _validate_external_id_relationships # function. Validate that ratings for the entity is empty.          Args:             item: ndb.Model. CollectionSummaryModel to validate. Class for validating ConfigPropertyModel. Class for validating ConfigPropertySnapshotMetadataModel. Class for validating ConfigPropertySnapshotContentModel. Class for validating SentEmailModels. # Valid id: [intent].[random hash] Validate that sent_datetime of model is less than current time.          Args:             item: ndb.Model. SentEmailModel to validate. Validate that sender email corresponds to email of user obtained         by using the sender_id.          Args:             item: ndb.Model. SentEmailModel to validate. # The case for missing sender external model is ignored here # since errors for missing sender external model are already # checked and stored in _validate_external_id_relationships # function. Validate that recipient email corresponds to email of user obtained         by using the recipient_id.          Args:             item: ndb.Model. SentEmailModel to validate. # The case for missing recipient external model is ignored here # since errors for missing recipient external model are already # checked and stored in _validate_external_id_relationships # function. Class for validating BulkEmailModels. Validate that sent_datetime of model is less than current time.          Args:             item: ndb.Model. BulkEmailModel to validate. Validate that sender email corresponds to email of user obtained         by using the sender_id.          Args:             item: ndb.Model. BulkEmailModel to validate. # The case for missing sender external model is ignored here # since errors for missing sender external model are already # checked and stored in _validate_external_id_relationships # function. Class for validating GeneralFeedbackEmailReplyToIdModels. Validate that reply_to_id length is less than or equal to         REPLY_TO_ID_LENGTH.          Args:             item: ndb.Model. GeneralFeedbackEmailReplyToIdModel to validate. # The reply_to_id of model is created using utils.get_random_int # method by using a upper bound as email_models.REPLY_TO_ID_LENGTH. # So, the reply_to_id length should be less than or equal to # email_models.REPLY_TO_ID_LENGTH. Class for validating ExplorationModel. Class for validating ExplorationSnapshotMetadataModel. Class for validating ExplorationSnapshotContentModel. Class for validating ExplorationRightsModel. Validate that first published time of model is less than current         time.          Args:             item: ndb.Model. ExplorationRightsModel to validate. Class for validating ExplorationRightsSnapshotMetadataModel. Class for validating ExplorationRightsSnapshotContentModel. Class for validating ExplorationCommitLogEntryModel. # Valid id: [exploration/rights]-[exploration_id]-[exploration-version]. # The case of invalid id is being ignored here since this # case will already be checked by the id regex test. Class for validating ExpSummaryModel. Validate that contributor ids match the contributor ids obtained         from contributors summary.          Args:             item: ndb.Model. ExpSummaryModel to validate. Validate that first published time of model is less than current         time.          Args:             item: ndb.Model. ExpSummaryModel to validate. Validate that item.exploration_model_last_updated matches the         time when a last commit was made by a human contributor.          Args:             item: ndb.Model. ExpSummaryModel to validate. # The case for missing exploration external model is ignored here # since errors for missing exploration external model are already # checked and stored in _validate_external_id_relationships # function. Class for validating FileMetadataModel. # Item id is of the format: # /exploration/exp_id/assets/(image|audio)/filepath. Class for validating FileMetadataSnapshotMetadataModel. Class for validating FileMetadataSnapshotContentModel. Class for validating FileModel. # Item id is of the format: # /exploration/exp_id/assets/(image|audio)/filepath. Class for validating FileSnapshotMetadataModel. Class for validating FileSnapshotContentModel. Class for validating ExplorationRecommendationsModel. Validate that model id is not present in recommended exploration ids.          Args:             item: ndb.Model. ExplorationRecommendationsModel to validate. Class for validating TopicSimilaritiesModel. # Valid id: topics. Validate the topic similarities to be symmetric and have real         values between 0.0 and 1.0.          Args:             item: ndb.Model. TopicSimilaritiesModel to validate. Class for validating StoryModel. Class for validating StorySnapshotMetadataModel. Class for validating StorySnapshotContentModel. Class for validating StoryRightsModel. Class for validating StoryRightsSnapshotMetadataModel. Class for validating StoryRightsSnapshotContentModel. Class for validating StoryCommitLogEntryModel. # Valid id: [story]-[story_id]-[story_version]. # The case of invalid id is being ignored here since this # case will already be checked by the id regex test. Class for validating StorySummaryModel. Validate that node_count of model is equal to number of nodes         in StoryModel.story_contents.          Args:             item: ndb.Model. StorySummaryModel to validate. # The case for missing story external model is ignored here # since errors for missing story external model are already # checked and stored in _validate_external_id_relationships # function. Class for validating UserSubscriptionsModels. Job that audits and validates production models. Job that audits and validates ActivityReferencesModel. Job that audits and validates RoleQueryAuditModel. Job that audits and validates CollectionModel. Job that audits and validates CollectionSnapshotMetadataModel. Job that audits and validates CollectionSnapshotContentModel. Job that audits and validates CollectionRightsModel. Job that audits and validates CollectionRightsSnapshotMetadataModel. Job that audits and validates CollectionRightsSnapshotContentModel. Job that audits and validates CollectionCommitLogEntryModel. Job that audits and validates CollectionSummaryModel. Job that audits and validates ConfigPropertyModel. Job that audits and validates ConfigPropertySnapshotMetadataModel. Job that audits and validates ConfigPropertySnapshotContentModel. Job that audits and validates SentEmailModel. Job that audits and validates BulkEmailModel. Job that audits and validates GeneralFeedbackEmailReplyToIdModel. Job that audits and validates ExplorationModel. Job that audits and validates ExplorationSnapshotMetadataModel. Job that audits and validates ExplorationSnapshotContentModel. Job that audits and validates ExplorationRightsModel. Job that audits and validates ExplorationRightsSnapshotMetadataModel. Job that audits and validates ExplorationRightsSnapshotContentModel. Job that audits and validates ExplorationCommitLogEntryModel. Job that audits and validates ExpSummaryModel. Job that audits and validates FileMetadataModel. Job that audits and validates FileModel. Job that audits and validates FileSnapshotMetadataModel. Job that audits and validates FileSnapshotContentModel. Job that audits and validates FileMetadataSnapshotMetadataModel. Job that audits and validates FileMetadataSnapshotContentModel. Job that audits and validates ExplorationRecommendationsModel. Job that audits and validates TopicSimilaritiesModel. Job that audits and validates StoryModel. Job that audits and validates StorySnapshotMetadataModel. Job that audits and validates StorySnapshotContentModel. Job that audits and validates StoryRightsModel. Job that audits and validates StoryRightsSnapshotMetadataModel. Job that audits and validates StoryRightsSnapshotContentModel. Job that audits and validates StoryCommitLogEntryModel. Job that audits and validates StorySummaryModel. Job that audits and validates UserSubscriptionsModel.