class Cluster(set):
  """
  Represents a cluster(set of similar documents)
  """

  def __init__(self, docList):
    """
    Instantiates a new cluster object.
    Requires 'docList': a list of docIds comprising the cluster
    """

    set.__init__(self, docList)

    docList.sort()
    self.id = hashlib.md5("-".join(docList).encode("latin1")).hexdigest().upper()
    self.isCurrent = 'unknown'
    self.isProcessed = False

  def process(self):
    """
    Processes the cluster to include metadata of consisting documents,
    and overall metadata of cluster like category, location, feeds, etc.
    """

    self.categories = []
    self.countries = []
    self.locales = []
    self.publishers = []
    self.languages = []
    self.articles = [] # contains non-duplicate articles
    self.duplicates = []
    self.lastPubTime = 0

    docKeys = list(super(Cluster, self).__iter__())
    docs = _getDocsInParallel(docKeys)
    docsAdded = []
    for doc in docs:
      docKey = doc.key
      if not _isDuplicateArticle(doc, docsAdded):
        # note that this gets passed all the way till browser
        # don't add any internal stuff here
        self.articles.append({
          'id': docKey,
          'title': _getDocTitle(doc),
          'publisher': doc.tags.get(TAG_PUBLISHER_DETAILS, ""),
          'link': doc.tags.get(DOCTAG_URL, "#"),
          'summaryText': _getDocSummary(doc),
          'highlights': doc.tags.get(LINKTAG_HIGHLIGHTS, []),
          'images': _getImagesForDoc(doc),
          'lang': doc.tags.get(FEEDTAG_LANG, ""),
          'publishedOn': doc.tags.get(LINKTAG_PUBTIME, 0)
        })
        docsAdded.append(doc)
      else:
        self.duplicates.append(docKey)

      if doc.tags.get(FEEDTAG_CATEGORY):
        self.categories.append(doc.tags[FEEDTAG_CATEGORY])
      if doc.tags.get(FEEDTAG_COUNTRY):
        self.countries.append(doc.tags[FEEDTAG_COUNTRY])
      if doc.tags.get(FEEDTAG_LOCALE):
        self.locales.append(doc.tags[FEEDTAG_LOCALE])
      if doc.tags.get(TAG_PUBLISHER):
        self.publishers.append(doc.tags[TAG_PUBLISHER])
      if doc.tags.get(FEEDTAG_LANG):
        self.languages.append(doc.tags[FEEDTAG_LANG])
      if doc.tags.get(LINKTAG_PUBTIME, 0) > self.lastPubTime:
        self.lastPubTime =  doc.tags.get(LINKTAG_PUBTIME)

    nArticles = len(self.articles) + len(self.duplicates)
    self.articles.sort(key=itemgetter('publishedOn'), reverse=True)

    #remove duplicates
    self.categories = _removeDuplicatesAndOutliers(self.categories, nArticles)
    self.countries = _removeDuplicatesAndOutliers(self.countries, nArticles)
    self.locales = _removeDuplicatesAndOutliers(self.locales, nArticles)
    self.publishers = list(set(self.publishers))
    self.languages = list(set(self.languages))
    self.isProcessed = True

  def deserializeFromString(self, clusterString):
    clusterDict = json.loads(clusterString)
    self.__init__(clusterDict['docKeys'])
    self.articles = clusterDict['articles']
    self.categories = clusterDict['categories']
    self.countries = clusterDict['countries']
    self.locales = clusterDict['locales']
    self.publishers = clusterDict['publishers']
    self.languages = clusterDict['languages']
    self.duplicates = clusterDict['duplicates']
    self.isCurrent = clusterDict['isCurrent']
    self.lastPubTime = float(clusterDict['lastPubTime'])
    self.isProcessed = True

  def serializeToString(self):
    """
    Serialize cluster to string. Only works for processed clusters
    """

    if not self.isProcessed:
      raise ValueError('Only processed clusters can be serialized')

    return json.dumps({
      'clusterId': self.id,
      'docKeys': list(super(Cluster, self).__iter__()),
      'articles': self.articles,
      'categories': self.categories,
      'countries': self.countries,
      'locales': self.locales,
      'publishers': self.publishers,
      'languages': self.languages,
      'duplicates': self.duplicates,
      'isCurrent': self.isCurrent,
      'lastPubTime': self.lastPubTime
    }) class ClusterManager:
    """
    Manage clusters stored in cloud.
    """

    def __init__(self):
        """
        Instantiates a new instance of ClusterManager class

        """

        self.clusterTableManager = ClusterTableManager()
        self.docManager = DocManager()
        self.processedClusterStore = ProcessedClusterStore()

    def getProcessedCluster(self, cluster):
        return self.processedClusterStore.getProcessedCluster(cluster)

    def processNewCluster(self, cluster):
        cluster.isCurrent = 'true'
        cluster = self.processedClusterStore.processAndSaveCluster(cluster)
        self.clusterTableManager.addCluster(cluster)

    def __getProcessedClusterArticles(self, cluster):
        cluster = self.getProcessedCluster(cluster)
        return cluster.articles

    def __getClusterResponse(self, cluster, filters = None):
        articles = self.__getProcessedClusterArticles(
            self.__filterDocsInCluster(cluster, filters))

        title = articles[0]['title'];
        description = articles[0]['title'] + " - " + \
            articles[0]['publisher'][PUBLISHER_DETAILS_NAME] + ".";
        if len(articles) > 1:
            description += " " + articles[1]['title'] + " - " + \
              articles[1]['publisher'][PUBLISHER_DETAILS_NAME] + ".";

        return {
            "articles": articles,
            "title": title,
            "description": description,
            "locales": cluster.locales,
            "languages": cluster.languages,
            "importance": self.__computeClusterRankingScore(cluster)
        }

    def __computeClusterRankingScore(self, cluster):
        return (0.3 * (len(cluster) - len(cluster.duplicates))) + \
            (0.7 * len(cluster.publishers))

    def __sortClustersByImportance(self, clusters):
        clusterList = list(clusters)
        clusterList.sort(key = self.__computeClusterRankingScore, reverse=True)
        return clusterList;

    def __filterClusters(self, clusterList, filters):
        if not filters:
            return clusterList

        if CLUSTERS_FILTER_LANGUAGES in filters:
            clusterList = [cluster for cluster in clusterList if not \
                set(filters[CLUSTERS_FILTER_LANGUAGES]).isdisjoint(cluster.languages)]

        return clusterList;

    def __filterDocsInCluster(self, cluster, filters):
        if not filters:
            return cluster

        filteredDocs = []

        for docKey in cluster:
            isDocAllowed = True;
            doc = self.docManager.get(docKey)

            if CLUSTERS_FILTER_LANGUAGES in filters:
                if doc.tags[FEEDTAG_LANG] not in filters[CLUSTERS_FILTER_LANGUAGES]:
                    isDocAllowed = False

            if isDocAllowed:
                filteredDocs.append(docKey)

        return Cluster(filteredDocs)

    def __constructQueryResponse(self, clusters, skip, top, filters=None):
        response = []
        clusterList = list(clusters)
        clusterList = self.__filterClusters(clusterList, filters)
        clusterList = self.__sortClustersByImportance(clusterList)

        for cluster in clusterList[skip:(skip + top)]:
            try:
                response.append(self.__getClusterResponse(cluster, filters))

            except Exception as e:
                logging.exception(
                    "Could not construct query response for cluster id %s",
                    cluster.id);
                continue

        return response;

    def queryByCategoryAndCountry(self, category, country, skip, top, filters=None):
        clusters = self.clusterTableManager.queryByCategoryAndCountry(
            category,
            country)
        return self.__constructQueryResponse(clusters, skip, top, filters)

    def queryByLocale(self, locale, skip, top, filters=None):
        clusters = self.clusterTableManager.queryByLocale(locale)
        response = []

        return self.__constructQueryResponse(clusters, skip, top, filters)

    def queryByDocId(self, docId, filters=None):
        cluster = self.clusterTableManager.queryByDocId(docId)
        if not cluster:
            return None
        else:
            return self.__constructQueryResponse([cluster], 0, 1, filters)[0]

    def putCurrentClusters(self, clusters):
        jobManager = ClusterJobManager()

        existingClusters = list(self.getCurrentClusters())
        newClusters = [cluster for cluster in clusters
            if cluster not in existingClusters]
        expiredClusters = [cluster for cluster in existingClusters
            if cluster not in clusters]

        for cluster in newClusters:
            job = WorkerJob(
                JOB_PROCESSNEWCLUSTER,
                { JOBARG_PROCESSNEWCLUSTER_CLUSTER : list(cluster)})
            jobManager.enqueueJob(job)
            logging.info(
                "Put process new cluster job. Cluster id: %s.",
                cluster.id)

        logging.info("Number of clusters to delete are: %i", len(expiredClusters))
        self.clusterTableManager.deleteClusters(expiredClusters)
        self.processedClusterStore.deleteClusters(expiredClusters)

    def getCurrentClusters(self):
        return self.clusterTableManager.getCurrentClusters()

    def getCurrentDocs(self):
        currentClusters = self.getCurrentClusters()
        return (doc for cluster in currentClusters for doc in cluster)

    def archiveOldClusters(self):
        return self.clusterTableManager.archiveOldClusters();

    def reprocessCurrentClusters(self):
        currentClusters = self.getCurrentClusters()
        for cluster in currentClusters:
            cluster = self.processedClusterStore.processAndSaveCluster(cluster)

        self.clusterTableManager.addClusters(currentClusters)

    def getNotfiableClustersForLocale(self, jobId, locale):
        logging.info("Fetching clusters for locale %s. %s", locale, jobId)

        clusters = list(self.clusterTableManager.queryByLocale(locale))

        importantClusters = [cluster for cluster in clusters if \
            self.__computeClusterRankingScore(cluster) > NOTIFICATION_IMPORTANCE_THRESHOLD]
        importantClusters = self.__sortClustersByImportance(importantClusters)

        lessImportantClusters = [cluster for cluster in clusters if \
            self.__computeClusterRankingScore(cluster) > NOTIFICATION_IMPORTANCE_THRESHOLD_LOW]
        lessImportantClusters = self.__sortClustersByImportance(lessImportantClusters)

        notifiableClusters = importantClusters

        # if we have too less notifiable clusters, send notification for less important clusters.
        if len(notifiableClusters) < NOTIFICATION_MIN_NUMBER_THRESHOLD:
            notifiableClusters = lessImportantClusters[:NOTIFICATION_MIN_NUMBER_THRESHOLD]

        # if we have too many notifiable clusters, limit the number to avoid spamming.
        if len(notifiableClusters) > NOTIFICATION_MAX_NUMBER_THRESHOLD:
            notifiableClusters = notifiableClusters[:NOTIFICATION_MAX_NUMBER_THRESHOLD]

        logging.info("Number of notfiable clusters are: %i. %s", len(notifiableClusters), jobId)
        return notifiableClusters class DistanceTableManager:
    """
    Manage doc distance pairs stored on AWS dynamo db database.

    Contains functions for CRUD on table, adding and querying entries etc.
    """

    def __init__(self):
        """
        Instantiates a new instance of DistanceTableManager class
        """

        self.tableConnString = os.environ['DISTTABLE_CONNECTIONSTRING']
        self.__table = None

    def __getTable(self):
        """
        Get the distances table.
        """

        if not self.__table:
            self.__table = getDbTable(self.tableConnString)

        return self.__table

    def createFreshTable(self):
        """
        Create a fresh empty distance table.
        """

        # delete existing table if it exists
        try:
            self.__getTable().delete()
            time.sleep(10)
        except:
            pass;# do nothing. Maybe there was no existing table

        # create new table
        tableConnectionParams = parseConnectionString(
            self.tableConnString)
        return Table.create(
            tableConnectionParams['name'],
            schema = [
                HashKey('from'),
                RangeKey('to')
            ], throughput = {
                'read': 1,
                'write': 2,
            }, global_indexes = [
                GlobalAllIndex('reverseIndex', parts = [
                    HashKey('to'),
                    RangeKey('from')
                ],
                throughput = {
                    'read': 1,
                    'write': 2,
                })
            ],
            connection = getDbConnection(tableConnectionParams))

    def addEntry(self, docId1, docId2, distance):
        """
        Add a entry in the distances table.
        """

        table = self.__getTable()

        table.put_item(data = {
            'from' : min(docId1, docId2),
            'to' : max(docId1, docId2),
            'distance' : json.dumps(distance)},
            overwrite = True)

    def getDistance(self, docId1, docId2):
        """
        Get the distance between two entries.
        """

        queryFrom = min(docId1, docId2)
        queryTo = max(docId1, docId2)

        table = self.__getTable()
        entry = list(table.query_2(from__eq = queryFrom, to__eq = queryTo))

        if not entry:
          return 0
        else:
          return json.loads(entry[0]['distance'])

    def getEntries(self):
        """
        Get the distance entries.
        Return results in a generator
        """

        table = self.__getTable()
        scanResults = table.scan()
        return ((result['from'], result['to'], json.loads(result['distance']))
                for result in scanResults)

    def getDistanceMatrix(self):
        """
        Get the distance entries in matrix form.
        """

        entries = self.getEntries()
        dMatrix = {}

        for entry in entries:
            if entry[0] not in dMatrix:
                dMatrix[entry[0]] = {}
            dMatrix[entry[0]][entry[1]] = entry[2]

        return dMatrix

    def getCloseDocs(self, docId):
        """
        Get the documents closest to a particular doc
        """

        docEntries = self.__queryByDocId(docId)
        closeDocs = []

        for entry in docEntries:
            if json.loads(entry['distance']) > 0.1:
                if entry['from'] == docId:
                    closeDocs.append(entry['to'])
                else:
                    closeDocs.append(entry['from'])

        return closeDocs

    def cleanUpDoc(self, docId):
        """
        Cleanup a document from the distance table
        """

        docEntries = self.__queryByDocId(docId)

        table = self.__getTable()
        with table.batch_write() as batch:
            for entry in docEntries:
                batch.delete_item(**{
                    'from': entry['from'],
                    'to': entry['to']
                })

    def deleteEntry(self, docId1, docId2):
        """
        Delete a entry from the distances table.
        """

        table = self.__getTable()

        table.delete_item(**{
            'from': min(docId1, docId2),
            'to': max(docId1, docId2)
        })

    def __queryByDocId(self, docId):
        table = self.__getTable()

        entries = list(table.query_2(from__eq = docId)) +\
            list(table.query_2(to__eq = docId, index = 'reverseIndex'))

        return entries class Doc:
  """
  Represents a document.

  Each doc consists of an auto-generated id, original, processed content
  and some tags(which are key-value pairs to help in clustering).
  """

  def __init__(self, key, content, tags):
    """
    Instantiates a new document object.

    """

    self.key = key.upper();
    self.content = content;
    self.tags = tags; def getDocEnglishTitle(doc):
  docLang = doc.tags[FEEDTAG_LANG]

  if docLang != LANG_ENGLISH:
    return doc.tags[DOCTAG_TRANSLATED_TITLE]
  else:
    return doc.tags[LINKTAG_TITLE] def getDocEnglishSummaryText(doc):
  docLang = doc.tags[FEEDTAG_LANG]

  if docLang != LANG_ENGLISH:
    return doc.tags[DOCTAG_TRANSLATED_SUMMARYTEXT]
  else:
    return doc.tags[LINKTAG_SUMMARYTEXT] def getDocEnglishContent(doc):
  docLang = doc.tags[FEEDTAG_LANG]

  if docLang != LANG_ENGLISH:
    return doc.tags[DOCTAG_TRANSLATED_CONTENT]
  else:
    return doc.content def getDocComparisionScore(jobInfo, doc1, doc2):
    score = 0
    if (doc1.tags[FEEDTAG_LANG] == LANG_ENGLISH) and \
        (doc2.tags[FEEDTAG_LANG] == LANG_ENGLISH):
        score = computeEnglishDocsSimScore(jobInfo, doc1, doc2)
        logger.info("Compared using shingles. %s", jobInfo)
    else:
        score = computeDocSimScoreUsingEntities(jobInfo, doc1, doc2)
        # make it slightly easier for non-english docs to get clustered.
        score = score * 1.15
        logger.info("Compared using entities. %s", jobInfo)

    if FEEDTAG_LOCALE in doc1.tags and FEEDTAG_LOCALE in doc2.tags and \
        doc1.tags[FEEDTAG_LOCALE] != doc2.tags[FEEDTAG_LOCALE]:

        logger.info(
            "The two docs are from different locations. Adding penalty. %s",
            jobInfo)
        score = score - 0.4
        if score < 0:
            score = 0

    if doc1.tags[TAG_PUBLISHER] != doc2.tags[TAG_PUBLISHER]:
        # make it slightly easier for different-publisher-docs to get clustered
        score = score * 1.1

    doc1Publisher = doc1.tags[TAG_PUBLISHER]
    doc2Publisher = doc2.tags[TAG_PUBLISHER]
    if doc1Publisher == doc2Publisher:
        penaltyPublishers = ['TOI', 'TelanganaToday', 'Hindu']
        if doc1Publisher in penaltyPublishers :
            logger.info(
                "Adding penalty for same publisher %s. %s",
                doc1Publisher,
                jobInfo)
            score = score*0.8

    if score > 1:
        score = 1

    logger.info("Comparision score: %s. %s", str(score), jobInfo)
    return score class DocManager:
    """
    Manage documents stored in cloud.

    Contains functions for CRUD operations on documents
    """

    def __init__(self):
        """
        Instantiates a new instance of DocManager class

        'bucketConnString' : connection string of s3 bucket in which docs
        are stored.
        """

        self.bucketConnString = os.environ['DOCSBUCKET_CONNECTIONSTRING']
        self.cache = getCache()
        self.__cacheExpiry= 900

    def __getBucket(self):
        bucketConnParams = parseConnectionString(self.bucketConnString)
        conn = getS3Connection(self.bucketConnString)

        return conn.get_bucket(bucketConnParams['bucketName'], validate=False)

    def __isDocNew(self, key, timeLimit):
        if _getEpochSecs(key.last_modified) < timeLimit:
            return False

        doc = self.get(key.name)
        return (doc.tags[LINKTAG_PUBTIME] > timeLimit) and \
            (FEEDTAG_DO_NOT_CLUSTER not in doc.tags)

    def put(self, doc):
        k = Key(self.__getBucket())
        k.key = doc.key

        # not storing tags directly in blob's metadata as the maximum size
        # allowed there is only 2kb.
        tags = dict(doc.tags)
        tags['content'] = doc.content
        keyContents = json.dumps(tags)
        k.set_contents_from_string(keyContents)
        self.cache.set(k.key, keyContents, self.__cacheExpiry)

    def get(self, docKey):
        keyContents = self.cache.get(docKey)
        if not keyContents:
            k = Key(self.__getBucket())
            k.key = docKey
            keyContents = k.get_contents_as_string()
            self.cache.set(docKey, keyContents, self.__cacheExpiry)

        storedTags = json.loads(keyContents)
        content = storedTags.pop('content', None)
        tags = storedTags

        return Doc(docKey, content, tags)

    def delete(self, docKey):
        k = Key(self.__getBucket())
        k.key = docKey
        k.delete()
        self.cache.delete(docKey) class EntityTableManager:
    """
    Manage entity-docid pairs stored on AWS dynamo db database.

    Contains functions for CRUD on table, adding and querying entries etc.
    """

    def __init__(self):
        """
        Instantiates a new instance of EntityTableManager class
        """

        self.tableConnString = os.environ['ENTITYTABLE_CONNECTIONSTRING'];
        self.__table = None

    def __getTable(self):
        """
        Get the entity table.
        """

        if not self.__table:
            tableConnectionParams = parseConnectionString(
                self.tableConnString);

            self.__table = Table(
                tableConnectionParams['name'],
                connection = getDbConnection(tableConnectionParams));

        return self.__table;

    def __removeDuplicateEntities(self, entities):
        """
        Remove entities which might result in duplicate encodings.
        """

        seenEncodedEntities = set()
        uniqueEntitiesList = []

        for entity in entities:
            encodedEntity = EncodedEntity(entity).encoded
            if encodedEntity not in seenEncodedEntities:
                seenEncodedEntities.add(encodedEntity)
                uniqueEntitiesList.append(entity)

        return uniqueEntitiesList

    def createFreshTable(self):
        """
        Create a fresh empty entity table.
        """

        # delete existing table if it exists
        try:
            self.__getTable().delete();
            time.sleep(10)
        except:
            pass;# do nothing. Maybe there was no existing table

        # create new table
        tableConnectionParams = parseConnectionString(
            self.tableConnString);
        return Table.create(
            tableConnectionParams['name'],
            schema = [
                HashKey('entity'),
                RangeKey('docId')
            ], throughput = {
                'read': 1,
                'write': 4,
            }, global_indexes = [
                GlobalAllIndex('docIdIndex', parts = [
                    HashKey('docId'),
                    RangeKey('entity')
                ],
                throughput = {
                    'read': 1,
                    'write': 4,
                })
            ],
            connection = getDbConnection(tableConnectionParams))

    def addEntries(self, docId, entities):
        """
        Add entries in entities table for entities and docId passed.
        """

        entities = self.__removeDuplicateEntities(entities)
        table = self.__getTable()
        with table.batch_write() as batch:
            for entity in entities:
                encodedEntity = EncodedEntity(entity)
                if encodedEntity.encoded:
                    batch.put_item(data={
                        'docId': docId,
                        'entity': encodedEntity.encoded,
                        'plain': encodedEntity.plain},
                        overwrite = True)

    def getEntityWeight(self, entity):
        encodedEntity = EncodedEntity(entity)
        if not encodedEntity.encoded:
            return 0.0

        docCount = len(list(
            self.queryByEntity(encodedEntity.encoded)))
        if docCount > 50:
           return 0.0;
        else:
           return (50.0 - docCount)/50

    def queryByEntity(self, entity):
        """
        Retrieve list of docId's for the passed entity.
        """

        table = self.__getTable()
        return (row['docId'] for row in table.query_2(
            entity__eq = entity))

    def queryByDocId(self, docId):
        """
        Retrieve list of entities for the passed docId.
        """

        table = self.__getTable()
        return (row['entity'] for row in table.query_2(
            docId__eq = docId,
            index = 'docIdIndex'))

    def cleanUpDocEntities(self, docId):
        """
        Cleanup all entities of doc.
        """

        entities = self.queryByDocId(docId)
        table = self.__getTable()

        with table.batch_write() as batch:
            for entity in entities:
                batch.delete_item(docId=docId, entity=entity) class MinerJobManager(JobManager):
    """
    Helper class to enqueue and dequeue jobs to the miner job queue.
    """

    def __init__(self):
        JobManager.__init__(self, 'MINER_JOBSQUEUE_CONNECTIONSTRING') class ShingleTableManager:
    """
    Manage shingle-docid pairs stored on AWS dynamo db database.

    Contains functions for CRUD on table, adding and querying entries etc.
    """

    def __init__(self):
        """
        Instantiates a new instance of ShingleTableManager class
        """

        self.tableConnString = os.environ['SHINGLETABLE_CONNECTIONSTRING']
        self.__table = None

    def __getTable(self):
        """
        Get the shingle table.
        """

        if not self.__table:
            shingleTableConnectionParams = parseConnectionString(
                self.tableConnString);

            self.__table = Table(
                shingleTableConnectionParams['name'],
                connection = getDbConnection(shingleTableConnectionParams));

        return self.__table;

    def addEntries(self, docId, shingles):
        """
        Add a entries in shingles table for shingles and docId passed.
        """

        shingles = list(set(shingles)) # remove duplicate shingles
        shingles.sort()
        shingles = shingles[:MAX_SHINGLES_PER_DOC]

        shingleTable = self.__getTable()
        with shingleTable.batch_write() as batch:
            for shingle in shingles:
                batch.put_item(data={
                    'docId': docId,
                    'shingle': shingle})

    def queryByShingle(self, shingle):
        """
        Retrieve list of docId's for the passed shingle.
        """

        shingleTable = self.__getTable()
        return (row['docId'] for row in shingleTable.query_2(
            shingle__eq = shingle))

    def queryByDocId(self, docId):
        """
        Retrieve list of shingles for the passed docId.
        """

        shingleTable = self.__getTable()
        return (row['shingle'] for row in shingleTable.query_2(
            docId__eq = docId,
            index = 'docIdIndex'))

    def cleanUpDocShingles(self, docId):
        """
        Cleanup all shingles of doc.
        """

        shingles = self.queryByDocId(docId)
        shingleTable = self.__getTable()

        with shingleTable.batch_write() as batch:
            for shingle in shingles:
                batch.delete_item(docId=docId, shingle=shingle)

    def scan(self):
        """
        List all the shingle entries currently present.
        """

        shingleTable = self.__getTable()
        scanResults = ((row['docId'], row['shingle']) for row in shingleTable.scan())

        return scanResults class WorkerJob:
    """
    Represents a job which can be processed by one of the worker roles.
    """

    def __init__(self, jobName, jobParams, jobId = None):
        """
        Instantiates a new worker job object.
        Requires 'jobName': a string representing name of the job
        Requires 'jobParams': a dictionary where keys represnt job parameter
            names, and corresponding values the job parameter values.
        Optional 'jobId': a identier for this job. If not provided a
            randomly generated alphanumeric string is used.
        """

        if (jobId is None):
            self.jobId = _generateRandomJobId()
        else :
            self.jobId = jobId

        self.jobName = jobName
        self.jobParams = jobParams

    def deserializeFromString(self, serializedJob):
        """
        Sets this worker job object to the specified serialized string
        representation.
        """

        tempDict = json.loads(serializedJob)
        jobName = tempDict.pop('jobName', None)
        jobId = tempDict.pop('jobId', None)
        self.__init__(jobName, tempDict, jobId)

    def serializeToString(self):
        """
        Serialize to a human-readable string representation of this object
        """

        tempDict = dict(self.jobParams)
        tempDict['jobName'] = self.jobName
        tempDict['jobId'] = self.jobId
        return json.dumps(tempDict)
import itertools
import logging
import json
from retrying import retry
from .constants import *
from .cluster import Cluster
from .clusterManager import ClusterManager
from .distanceTableManager import DistanceTableManager
from .doc import Doc
from .docHelper import getDocEnglishTitle, getDocEnglishSummaryText, \
  getDocEnglishContent, getDocComparisionScore
from .docManager import DocManager
from .entityTableManager import EntityTableManager
from .minerJobManager import MinerJobManager
from .loggingHelper import *
from .shingleTableManager import ShingleTableManager
from .workerJob import WorkerJob
from . import textHelperNltk as th



logger = logging.getLogger('clusteringJobs')

SIMSCORE_MIN_THRESHOLD = 0.07

def compareDocsAfterFetch(jobId, doc1, doc2, distanceTableManager):
    jobInfo = "Doc1 id: " + doc1.key + " Doc2 id: " + doc2.key \
              + ". Job id: " + jobId


    score = getDocComparisionScore(jobInfo, doc1, doc2)