def getFamilyFromId(famId, familyDB, relationsDB):
    """
    Build a full family record (husb,wife,marriage,children)
    from a familyId
    """
    famRecord = familyDB.find_one({'_id': famId})
    if famRecord is None: return None
    famRecord['husb'] = None
    famRecord['wife'] = None
    famRecord['children'] = []
    for member in relationsDB.find({'famId': famRecord['_id']}):
        if member['relTyp'] == 'child': famRecord['children'].append(member['persId'])
        else: famRecord[member['relTyp']] = member['persId']
        #if 'husb' in member: famRecord['husb'] = member['husb']
        #elif 'wife' in member: famRecord['wife'] = member['wife']
        #elif 'child' in member: famRecord['children'].append(member['child'])
    return famRecord def matchFam(tFamId, rFamId, conf):
    #KOLLA EVT accept match as optional param and not do everything?
    famMatchData = {}
    famMatchSummary = {}
    famMatchData['workid'] = tFamId
    famMatchData['matchid'] = rFamId
    stat = set()
    ##AAFIX
    #tFam = conf['families'].find_one({'_id': tFamId})
    tFam = getFamilyFromId( tFamId, conf['families'], conf['relations'])
    #rFam =conf['match_families'].find_one({'_id': rFamId})
    rFam = getFamilyFromId( rFamId, conf['match_families'], conf['match_relations'])
    try: famMatchData['workRefId'] = tFam['refId']
    except: famMatchData['workRefId'] = tFamId
    try: famMatchData['matchRefId'] = rFam['refId']
    except: famMatchData['matchRefId'] = rFamId
    famMatchData['marriage'] = {}
    if 'marriage' in tFam: famMatchData['marriage']['work'] = tFam['marriage']
    if 'marriage' in rFam: famMatchData['marriage']['match'] = rFam['marriage']
    antPartner = 0
    for partner in ('husb','wife'):
        famMatchData[partner] = {'status': 'notMatched'}
        famMatchSummary[partner] = {'status': 'notMatched'}
        if (tFam[partner]) and (rFam[partner]):
            antPartner += 1
            st = getMatchPers(tFam[partner],rFam[partner], conf)
            if not st:
                #logging.debug('do matchPers %s %s %s', partner, tFam, rFam)
                st = matchPers( conf['persons'].find_one({'_id': tFam[partner]}),
                            conf['match_persons'].find_one({'_id': rFam[partner]}), conf )
                #logging.debug('Insert new unmatched parent %s %s %s',
                #              st['pwork']['refId'], st['pmatch']['refId'],st['status'])
                conf['matches'].insert(st)
            famMatchData[partner] = st
            famMatchSummary[partner] = st['status']
            if (st['status'] in common.statOK): stat.add('Match')
            elif (st['status'] in common.statManuell): stat.add('Manuell')
            elif (st['status'] in common.statEjOK): stat.add('EjMatch')
        elif tFam[partner]:
            famMatchData[partner]['pwork'] = conf['persons'].find_one({'_id': tFam[partner]})
        elif rFam[partner]:
            famMatchData[partner]['pmatch'] = conf['match_persons'].find_one({'_id': rFam[partner]})
    #Children ...
    tmpchilds = tFam['children']  #getChildrenUid('TmpMatch', str(tmpfuid))
    rgdchilds = rFam['children']  #getChildrenUid('RGD', str(rgdfuid))
    rgddone=[]
    tmpdone=[]
    if not tmpchilds: tmpchilds=[]
    if not rgdchilds: rgdchilds=[]
#checkOK??
    famMatchData['children'] = []
    famMatchSummary['children'] = set()
    #All matched persons
    for cond in ( [common.statOK,'Match'], [common.statManuell,'Manuell'],
                  [common.statEjOK, 'EjMatch'] ):
        for chtmp in tmpchilds:
            iuid = chtmp    #['I_uid']
            if iuid in tmpdone: continue
            for chrgd in rgdchilds:
                rgdiuid = chrgd   #['I_uid']
                if rgdiuid in rgddone: continue
                #FIXif chtmp['sex'] != chrgd['sex']: continue  #FIX
                st = getMatchPers(iuid,rgdiuid, conf)
                if (st and (st['status'] in cond[0])):
                    stat.add(cond[1])
                    rgddone.append(rgdiuid)
                    tmpdone.append(iuid)
                    try:
                        st['sort'] = st['pwork']['birth']['date']
                    except:
                        try:
                            st['sort'] = st['pmatch']['birth']['date']
                        except:
                            st['sort'] = '0'
                    famMatchData['children'].append(st)
                    famMatchSummary['children'].add(st['status'])
                    break
    #Try to find match
    for chtmp in tmpchilds:
        iuid = chtmp #['I_uid']
        if iuid in tmpdone: continue
        for chrgd in rgdchilds:
            rgdiuid = chrgd #['I_uid']
            if rgdiuid in rgddone: continue
            st = getMatchPers(iuid,rgdiuid, conf)
            if (st and (st['status'] == 'split')): continue
            #FIXif chtmp['sex'] != chrgd['sex']: continue  #FIX
            mt = matchPers( conf['persons'].find_one({'_id': iuid}),
                            conf['match_persons'].find_one({'_id': rgdiuid}), conf )
            if mt['status'] in common.statOK.union(common.statManuell):
            #    logging.debug('Inserting new unmatched child %s %s %s',
            #                  mt['status'], mt['pwork']['refId'], mt['pmatch']['refId'])
##WHY THIS??
                conf['matches'].insert(mt) #FIX Problem when doing dublMatch
                if mt['status'] in common.statOK: stat.add('Match')
                else: stat.add('Manuell')
                rgddone.append(rgdiuid)
                tmpdone.append(iuid)
                try:
                    mt['sort'] = mt['pwork']['birth']['date']
                except:
                    try:
                        mt['sort'] = mt['pmatch']['birth']['date']
                    except:
                        mt['sort'] = '0'
                famMatchData['children'].append(mt)
                famMatchSummary['children'].add(mt['status'])
                break
            t = conf['persons'].find_one({'_id': iuid})
            r = conf['match_persons'].find_one({'_id': rgdiuid})
            if matchNamnFDate( t, r ):
                stat.add('Manuell')
                rgddone.append(rgdiuid)
                tmpdone.append(iuid)
                try:
                    sort = t['birth']['date']
                except:
                    try:
                        sort = r['birth']['date']
                    except:
                        sort = '0'
                famMatchData['children'].append({'status': 'rManuell', 'workid': iuid, 'pwork': t,
                                                 'matchid': rgdiuid, 'pmatch': r, 'sort': sort})
                famMatchSummary['children'].add('rManuell')
                #Insert into matches also? Can't do setOKperson if pair not in person matches
                mt['status'] = 'rManuell'
                #logging.debug('Inserting new name/date matched child %s %s %s',
                #              mt['status'], mt['pwork']['refId'], mt['pmatch']['refId'])
                conf['matches'].insert(mt) #FIX Problem when doing dublMatch
                break
    #All the rest
    for chtmp in tmpchilds:
        if chtmp in tmpdone: continue
        p = conf['persons'].find_one({'_id': chtmp})
        try:
            sort = p['birth']['date']
        except:
            sort = '0'
        famMatchData['children'].append({'status': '', 'workid': chtmp, 'pwork': p, 'sort': sort})
    for chtmp in rgdchilds:
        if chtmp in rgddone: continue
        p = conf['match_persons'].find_one({'_id': chtmp})
        try:
            sort = p['birth']['date']
        except:
            sort = '0'
        famMatchData['children'].append({'status': '', 'matchid': chtmp, 'pmatch': p, 'sort': sort})

    #calc status
    if (('EjMatch' in stat) and ('Match' in stat)):
        status = 'Manuell'
    elif 'Manuell' in stat:
        status = 'Manuell'
    elif (('EjMatch' in stat) and ('Manuell' not in stat) and ('Match' not in stat)):
        status = 'EjMatch'
    elif (('EjMatch' not in stat) and ('Manuell' not in stat) and ('Match' in stat)):
        status = 'Match'
    else: status = 'Manuell'
    famMatchData['status'] = status
    famMatchSummary['status'] = status
    famMatchSummary['children'] = list(famMatchSummary['children'])
    famMatchData['summary'] = famMatchSummary
    #sort children in birth-order
    famMatchData['children'] = sorted(famMatchData['children'], key=lambda c: c['sort'])
    return famMatchData def setFamOK(wid, mid, conf, famlist = None, button = False):
    fam_matches = conf['fam_matches']
    matches = conf['matches']
    families = conf['families']
    if button:
        kod ='OK'
        negKod = 'EjOK'
    else:
        kod = 'rOK'
        negKod = 'rEjOK'
    #logging.debug('work=%s match=%s button=%s kod=%s', wid, mid, button, kod)
    if not famlist:
        #make famList from wid, mid (familyIDs)
        #Sanity check
        if fam_matches.find({'workid': wid, 'matchid': mid}).count() != 1:
            logging.error('SetFamOK - not exactly 1 fam_match for %s %s', wid, mid) #ERR?
        match = fam_matches.find_one({'workid': wid, 'matchid': mid})
        famlist = [['F',match['workid'],match['matchid'],match['workRefId'],match['matchRefId']]]
        for partner in ('husb', 'wife'):
            try:  #FIX? if only one of pwork and pmatch exists??
                famlist.append(['P', match[partner]['pwork']['_id'],
                                match[partner]['pmatch']['_id'],
                               match[partner]['pwork']['refId'],
                               match[partner]['pmatch']['refId']])
            except:
                pass
                #logging.debug('SetFamOK - No match for %s %s %s', wid, mid, partner) #byt till pass
        for ch in match.get('children',[]):
            try:  #FIX?
                famlist.append(['P', ch['workid'], ch['matchid']])
            except:
                if 'pwork' in ch:
                    pass
                    #print 'SetFamOK - No match for child work', ch['pwork']['refId']  #byt till pass
                elif 'pmatch' in ch:
                    pass
                    #print 'SetFamOK - No match for child match', ch['pmatch']['refId'] #byt till pass
                else:
                    logging.error('SetFamOK - No match for child %s', ch)   #byt till pass  ERR?
    indlist = []
    tmpfuid = '0'
    #logging.debug('famlist=%s', famlist)
    for pair in famlist:
        if pair[0]=='P':  #individ
            setOKperson(pair[1], pair[2], button = button) #Also updates family status
    stat = ''
    for pair in famlist:
        if pair[0]=='F': #familj
            #Force all familymatches to negKod except workid, matchid
            workid = pair[1]
            matchid = pair[2]
            #Set all family pairs famA,* and *,famB to negKod
            for famMatch in fam_matches.find({'$or': [{'workid': workid},{'matchid': matchid}]},
                                             {'workid': 1, 'matchid': 1}):
                if famMatch['workid']==workid and famMatch['matchid']==matchid: continue
                else: setEjOKfamily(str(famMatch['workid']), str(famMatch['matchid']), code=negKod)
    return '' def setEjOKfamily(wid, mid, code='EjOK'):
#Villkor: inget matchat barn eller båda föräldrarna matchade.
#Ska testas + evt felmeddelande
    #logging.debug('setEjOKfamily %s %s %s', wid, mid, code)    #res =
    match = common.config['fam_matches'].find_one({'workid': wid, 'matchid': mid})
    if not match: return ''
    if (('husb' in match) and (match['husb']['status'] in common.statOK) and
        ('wife' in match) and (match['wife']['status'] in common.statOK)):
        return 'Not allowed: Both husb and wife matched'
    for ch in match['children']:
        if ch['status'] in common.statOK:
            return 'Not allowed: Matched child'
    res = ''
    for ch in match['children']:
        if ch['status'] in common.statManuell:
            #setEjOKperson calls updateFam
            res += setEjOKperson(str(ch['workid']), str(ch['matchid']), code=code)
    common.config['fam_matches'].update_one({'workid': wid, 'matchid': mid}, {'$set': {'status': 'FamEjOK'}})
    return res def setOKperson(wid, mid, button = True):
    indlist = []
    if button:
        kod ='OK'
        negKod = 'EjOK'
    else:
        kod = 'rOK'
        negKod = 'rEjOK'
    for mt in common.config['matches'].find({'$or': [{'workid': wid},{'matchid': mid}]},
                  {'workid': True, 'matchid': True, 'status': True}):
        #logging.debug('In setOKperson found match %s', mt['_id'])
        if mt['workid']==wid and  mt['matchid']==mid:
            if common.checkStatusUpdate(mt['status'], kod):
                #logging.debug('wid=%s mid=%s status old=%s new=%s',
                #              wid, mid, mt['status'], kod)
                common.config['matches'].update_one({'_id': mt['_id']}, {'$set': {'status': kod}})
                if mt['status'] in common.statOK: continue  #No other updates needed
                indlist.append(mt['workid'])
                ##TEST
                #check if wid,mid med i något familjepar som inte är med i fam_matches
                #   insert minimum fam_match med status 'candidate' - svårt!!
                #   it gets updated later by updateFamMatch
                for role in ('husb', 'wife', 'children'):
                    #tFam = common.config['families'].find({role: wid}, {'_id': 1} )
                    tFam = common.config['relations'].find({'persId': wid, 'relTyp': role})
                    #rFam = common.config['match_families'].find({role: mid}, {'_id': 1} )
                    rFam = common.config['match_relations'].find({'persId': mid, 'relTyp': role})
                    for f in tFam:
                        for ff in rFam:
                            if not common.config['fam_matches'].find_one({'workid': f['famId'], 'matchid': ff['famId']}):
                                common.config['fam_matches'].insert_one(matchFam(f['famId'], ff['famId'], common.config))
                           #Check multifam-resolution From match.py
                ##TEST
        elif common.checkStatusUpdate(mt['status'], negKod):
            #logging.debug('wid=%s mid=%s status old=%s new=%s',
            #              mt['workid'], mt['matchid'], mt['status'], negKod)
            common.config['matches'].update_one({'_id': mt['_id']}, {'$set': {'status': negKod}})
            if mt['status'] in common.statEjOK: continue  #No other updates needed
            indlist.append(mt['workid'])
    if not indlist:
        #logging.debug('No status changing updates')
        return 'No status changing updates'
    flist = set()
    for id in indlist:  #get list of involved individuals
        #for fam in common.config['families'].find({'$or': [{'husb': id}, {'wife': id}, {'children': id}]}, {'_id': True}):
        #    flist.add(fam['_id'])
        for fam in common.config['relations'].find({'persId': id}, {'famId': True}):
            flist.add(fam['famId'])
    return updateFamMatch(flist, common.config) class matchtext:
    """
    calculates the text representation used for initial selection of match-kandidates
    """
    def __init__(self):
        #caching does not improve speed of lucene indexing
        pass

    def luceneFix(self, text):
        for c in '+-&|!(){}[]^"~*?:,.':
            text = text.replace(c,'')
        return text.strip()

    def placeText(self, event):
        txt = ''
        if 'normPlaceUid' in event:
        #event['normPlaceUid'] can be either INT or STR
            try:
                txt = 'NPlace' + str(event['normPlaceUid'])
            except Exception, e:
                txt = 'NPlace' + event['normPlaceUid']
        elif 'place' in event:
            txt = 'Place' + event['place']
        return txt.replace(' ','')

    def eventText(self, event, prefix):
        txt = ''
        if 'date' in event:
            txt += prefix+'DateY' + event['date'][0:4]
            if len(event['date'])==8:
                txt += ' '+prefix+'Date' + event['date']
            if 'normPlaceUid' in event:
                txt += ' '+prefix+'DateY' + event['date'][0:4] + prefix + self.placeText(event)
        if self.placeText(event):
            txt += ' '+prefix + self.placeText(event)
        return txt.strip()

    ###########################
    def personText(self, pers):
        if not pers: return ''  #tmp FIX for inconsistensies in DB
        mtxt = ''
        namelist=[]
        if 'grpNameGiven' in pers:
            for nn in pers['grpNameGiven'].split(' '):
                mtxt += ' G' + nn
                namelist.append(nn)
        if 'grpNameLast' in pers:
            for nn in pers['grpNameLast'].split(' '):
                mtxt += ' L' + nn
                namelist.append(nn)
        if namelist: mtxt += ' ' + 'N'.join(sorted(namelist))

        if 'name' in pers:
            #mtxt += ' ' + pers['name'].replace('*','').replace('/',' ').replace('(',' ').replace('[','').replace(']','').replace(')',' ')
            mtxt += ' ' + pers['name']
        #    mtxt += ' sex' + p.sex()
        for ev in ('birth', 'death'):
            if ev in pers:
                if ev == 'birth': prefix = 'B'
                elif ev == 'death': prefix = 'D'
                mtxt += ' ' + self.eventText(pers[ev], prefix)

        return mtxt.strip()

    ###########################
    def matchtextPerson(self, p, personDB, familyDB, relationDB):
            matchtext = self.personText(p)
            #Add father and mother
            try:
                fam = getFamilyFromChild(p['_id'], familyDB, relationDB)
                if fam:
                    mtxt = set()
                    if fam['husb']:
                        for item in self.personText(personDB.find_one({'_id': fam['husb']})).split():
                            mtxt.add('Father'+item)
                    if fam['wife']:
                        for item in self.personText(personDB.find_one({'_id': fam['wife']})).split():
                            mtxt.add('Mother'+item)
                    matchtext += ' ' + ' '.join(mtxt)
            except: pass
            return self.luceneFix(matchtext)

    ###########################
    #Not used?

    def familyText(self, fam):
        if 'marriage' in fam:
            mtxt = self.eventText(fam['marriage'], 'M')
        else:
            mtxt = ''
        return mtxt
    ###########################
    def matchtextFamily(self, family, familyDB, pers_list, relations):
        fam = getFamilyFromId(family['_id'], familyDB, relations)
        matchtext = self.familyText(fam)
        mtxt = set()
        #Add HUSB o WIFE
        if fam['husb']:
            for item in self.personText(pers_list.find_one({'_id': fam['husb']})).split():
                mtxt.add('HUSB'+item)
        if fam['wife']:
            for item in self.personText(pers_list.find_one({'_id': fam['wife']})).split():
                mtxt.add('WIFE'+item)
        #Add children
        for ch in fam['children']:
            for item in self.personText(pers_list.find_one({'_id': ch})).split():
                mtxt.add('CHILD' + item)
        matchtext += ' ' + ' '.join(mtxt)
        return self.luceneFix(matchtext) class luceneDB:
    """
    Handles all details for the free-text database Lucene
    """
    def __init__(self, dbName, dropDB=False):
        #self.initObject = lucene.initVM() #default 2048? #vmargs=['-Djava.awt.headless=true']
        """
        attachCurrentThread(name, asDaemon)
        Before a thread created in Python or elsewhere but not in the Java VM
        can be used with the Java VM, this method needs to be invoked.
        The two arguments it takes are optional and self-explanatory.
        """
        #self.initObject.attachCurrentThread('LuceneDB', True)
        luceneVM.attachCurrentThread('LuceneDB')
        self.analyzer = StandardAnalyzer() #split on whitespace, no trunkation or stemming
        self.indexDir = None
        self.searcher = None
        (user,db) = dbName.split('_', 1)
        directory = "./files/"+user+'/'+db+'/LuceneIndex'
        if dropDB: shutil.rmtree(directory)
        self.indexDir = SimpleFSDirectory(Paths.get(directory)) #creates directory if not exists

    def personText(self, person):
        txt = []
        txt.append(person['name'])
        for field, value in person.iteritems():
            if field in ('birth', 'death'):
                for key, val in value.iteritems():
                    if key == 'place': txt.append(val)
                    elif key == 'date':
                        txt.append(val)
                        if len(val) > 4: txt.append(val[0:4])
            elif field in ('_id', 'refId'): txt.append(value)
        return ' '.join(txt).lower()

    def dummyIndex(self):
        """
        Create a dummy index - to avoid problems updating it
        """
        config = IndexWriterConfig(self.analyzer)
        config.setOpenMode(IndexWriterConfig.OpenMode.CREATE)
        writer = IndexWriter(self.indexDir, config)
        doc = Document()
        doc.add(Field('uid', 'dummy', StringField.TYPE_STORED))
        writer.addDocument(doc)
        writer.commit()
        writer.close()
        return

    def index(self, personDB, familyDB, relationDB):
        """
        indexes a database
        Field match includes information about parents and is used to find matches
        Field text has Ids, names, places, and dates and is used to find a person/family
        """
        config = IndexWriterConfig(self.analyzer)
        config.setOpenMode(IndexWriterConfig.OpenMode.CREATE)
        writer = IndexWriter(self.indexDir, config)
        #indexWriter.setRAMBufferSizeMB(256)  #?

        mt = matchtext()

        for p in personDB.find({}, no_cursor_timeout=True):
            matchtxt = mt.matchtextPerson(p, personDB, familyDB, relationDB)
            doc = Document()
            doc.add(Field('uid',str(p['_id']), StringField.TYPE_STORED))
            doc.add(Field('sex',str(p['sex']), StringField.TYPE_STORED))
            doc.add(Field("match", matchtxt, TextField.TYPE_NOT_STORED))
            doc.add(Field("text", mt.luceneFix(self.personText(p)), TextField.TYPE_NOT_STORED))
            writer.addDocument(doc)

        #Family matchtext
        for f in familyDB.find():
            #matchtxt = mt.matchtextFamily(f, familyDB, personDB, relationDB)
            doc = Document()
            doc.add(Field('uid',str(f['_id']), StringField.TYPE_STORED))
            #doc.add(Field('sex','FAM', StringField.TYPE_STORED))
            #doc.add(Field("match", matchtxt, TextField.TYPE_NOT_STORED))
            txt = f['_id']
            if 'refId' in f: txt += ' ' + f['refId']
            doc.add(Field("text", txt, TextField.TYPE_NOT_STORED))
            writer.addDocument(doc)

        writer.commit()
        writer.close()
        return

    def updateDeleteRec(self, pid1, pid2, personDB, familyDB, relationDB):
        config = IndexWriterConfig(self.analyzer)
        config.setOpenMode(IndexWriterConfig.OpenMode.APPEND)
        writer = IndexWriter(self.indexDir, config)
        mt = matchtext()
        writer.deleteDocuments(Term('uid', pid1))
        writer.deleteDocuments(Term('uid', pid2))
        p = personDB.find_one({'_id': pid1})
        matchtxt = mt.matchtextPerson(p, personDB, familyDB, relationDB)
        doc = Document()
        doc.add(Field('uid',str(pid1), StringField.TYPE_STORED))
        doc.add(Field('sex',str(p['sex']), StringField.TYPE_STORED))
        doc.add(Field("match", matchtxt, TextField.TYPE_NOT_STORED))
        doc.add(Field("text", mt.luceneFix(self.personText(p)), TextField.TYPE_NOT_STORED))
        writer.addDocument(doc)
        writer.commit()
        writer.close()
        self.searcher = IndexSearcher(DirectoryReader.open(self.indexDir))
        return

    def deleteRec(self, pid):
        config = IndexWriterConfig(self.analyzer)
        config.setOpenMode(IndexWriterConfig.OpenMode.APPEND)
        writer = IndexWriter(self.indexDir, config)
        writer.deleteDocuments(Term('uid', pid))
        writer.commit()
        writer.close()
        self.searcher = IndexSearcher(DirectoryReader.open(self.indexDir))
        return

    def query(self, txt, ant=10):
        """Searches for a person or family by id, name, place, or date"""
        q = QueryParser("text", self.analyzer).parse(txt.replace('/', '\/').lower())
        if not self.searcher:
            self.searcher = IndexSearcher(DirectoryReader.open(self.indexDir))
        scoreDocs = self.searcher.search(q, ant).scoreDocs
        hits = []
        for scoreDoc in scoreDocs:
            doc = self.searcher.doc(scoreDoc.doc)
            hits.append([doc.get("uid"), scoreDoc.score])
        return hits

    def search(self, q, sex, ant=5, config = None):
        """Searches for a match"""
        query = QueryParser("match", self.analyzer).parse(q.replace('/', '\/'))
        #Hur lägga till sex?
        if not self.searcher:
            self.searcher = IndexSearcher(DirectoryReader.open(self.indexDir))
        scoreDocs = self.searcher.search(query, ant).scoreDocs
        hits = []
        for scoreDoc in scoreDocs:
            doc = self.searcher.doc(scoreDoc.doc)
            if sex == doc.get("sex"):
                hits.append([doc.get("uid"), scoreDoc.score])
        return hits def nameDiff(work, match):
#    return ( (work['grpNameGiven'] != match['grpNameGiven']) or
#             (work['grpNameLast']  != match['grpNameLast']) )
#Redo so that only real diffs count, 'Lars Erik' and 'Lars' is OK
    for grp in ('grpNameGiven', 'grpNameLast'):
        s1 = set(work[grp].split())
        s2 = set(match[grp].split())
        if s1.issubset(s2) or s2.issubset(s1): pass
        else: return True
    return False def eventDiff(work, match, events, items = ('date', 'normPlaceUid')):
    for typ in events:
        if (typ in work) and (typ in match):
#            for item in ('date', 'normPlaceUid'):
            for item in items:
                try:
                    if work[typ][item] != match[typ][item]:
                        if (item=='date') and (len(work[typ][item])==4 or
                                               len(match[typ][item])==4 ):
                            if work[typ][item][0:4] != match[typ][item][0:4]:
                                return True
                        else: return True
                except:
                    pass
    return False def updateFamMatch(flist, conf):
    for fl in flist:  #Updatera familjestatus
#Om familjen inte matchad?
        for match in conf['fam_matches'].find({'workid': fl}, {'workid': True, 'matchid': True, 'workRefId': True, 'matchRefId': True, 'status': True}):
##EVT testa på status FamEjOK??
            if match['status'] == 'FamEjOK': continue
##EVT
            famMatchData = matchFam(match['workid'], match['matchid'], conf)
            #logging.debug('Uppdating Fam-match %s %s Old=%s New=%s', match['workRefId'],
            #              match['matchRefId'], match['status'], famMatchData['status'])
            conf['fam_matches'].delete_one({'_id': match['_id']})
            conf['fam_matches'].insert_one(famMatchData)
    return ''
import common
import argparse, time, sys, os, logging
from collections import defaultdict
from matchUtils import *
from dbUtils import getFamilyFromId
from utils import matchFam, setFamOK, setEjOKfamily, setOKperson
from matchtext import matchtext
from luceneDB import luceneDB
from uiUtils import nameDiff, eventDiff
from utils import updateFamMatch
#  deleteRec does not update merged events FIXED with updateDeleteRec
#searchDB = luceneDB(mDBname, dropDB=True)
#searchDB.index(config['match_persons'], config['match_families'], config['match_relations'])
#Should be optimized??
person_list = config['persons']
fam_list = config['families']

matches = config['matches']
matches.drop()
fam_matches = config['fam_matches']
fam_matches.drop()
config['flags'].drop()

match_person = config['match_persons']
match_family = config['match_families']

dubltmp = defaultdict(list)
dbltmpNs = defaultdict(float)
ant=0
for p in person_list.find({}, no_cursor_timeout=True):
    matchtxt = mt_tmp.matchtextPerson(p, person_list, fam_list, config['relations'])
    if not matchtxt:
        logging.error('No matchtextdata for %s, %s',p['_id'],p['refId'])
        continue       ##########FIX!!!!!!!!!!
    candidates = searchDB.search(matchtxt, p['sex'], 2) #Lucene search
    #Om inga kandidat-matcher? Inget problem dom är inte Match
    sc = 0
    for (kid,score) in candidates:
        #do not break if little data - WORSE
        #if score < 35.0 and (len(matchtxt.split())>10): break  #breakoff score point for considering  match
        if score < 30.0: break  #breakoff score point for considering  match
        if (score> sc): sc = score
        candidate = match_person.find_one({'_id': kid})
        ##PROBLEM FIX Lucene indexing
        if not candidate: continue
        ##PROBLEM
        matchdata = matchPers(p, candidate, config, score)
        #FIX EVT: lägg in mönster (autoOK, autoCheck -> EjOK) (multimatch Resolve) här
        #logging.debug('Insert main matching for %s, %s',p['refId'], candidate['refId'])
        matches.insert(matchdata)
#        if p['_id']=='P_980100': print matchdata
        ant += 1
#se mail 'Stickprov' Juni 5 2015
#        if matchdata['status'] in common.statEjOK: break
        if  matchdata['status'] in common.statOK.union(common.statManuell):
            dubltmp[p['_id']].append(candidate['_id'])
            dbltmpNs[p['_id'], candidate['_id']] = matchdata['nodesim']
        #break if Match
        if matchdata['status'] == 'Match': break
        #break if score is less than 1/3 of max score
        if (sc/score > 3.0): break
logging.info('%d person matchings inserted', ant)
logging.info('Time %s',time.time() - t0)
#Families match-status calculated from person match-status => No SVM
ant = 0
fams = set()
for match in  matches.find({'status': {'$in': list(common.statOK.union(common.statManuell))}}):
    for role in ('husb', 'wife', 'child'):
        tFam = []
        rFam = []
        for tRel in config['relations'].find({'relTyp': role, 'persId': match['workid']}):
            tFam.append(fam_list.find_one({'_id': tRel['famId']}, {'_id': 1, 'marriage.date': 1}))
        for tRel in config['match_relations'].find({'relTyp': role, 'persId': match['matchid']}):
            rFam.append(match_family.find_one({'_id': tRel['famId']},
                                              {'_id': 1, 'marriage.date': 1}))
        tfams = []
        tDone = []
#take all combinations of families
#if several posibilities for 1 work-family keep only the the pair where marriage-date matches
        for f in tFam:
            for ff in rFam:
                try:
                    if len(f['marriage']['date'])>4 and f['marriage']['date'] == ff['marriage']['date']:
                        fams.add((f['_id'], ff['_id']))
                        tDone.append(f['_id'])
                    else:
                        tfams.append([f['_id'], f['marriage']['date'], ff['_id'], ff['marriage']['date']])
                except:
                    fams.add((f['_id'], ff['_id']))
        for l in tfams:
            if l[0] not in tDone:
                fams.add((l[0], l[2]))
famMatchSummary = {}
for (tFamId,rFamId) in fams:    #  for all involved families
    famMatchData = matchFam(tFamId, rFamId, config)
    fam_matches.insert(famMatchData)
    ant += 1
logging.info('%d family matchings inserted', ant)
logging.info('Time %s',time.time() - t0)
############################
#EVT SVM for fam-matches?
############################
if noFamSVM: #default
    logging.info('Matching All done')
    sys.exit()
logging.info('Doing SVM family match, incl split-ifying')

if 'famfeatureSet' in config:
    famSVMfeatures = getattr(importlib.import_module('featureSet'), config['famfeatureSet'])
    svmFamModel = svm_load_model('conf/' + config['famfeatureSet'] + '.model')
else:
    famSVMfeatures = getattr(importlib.import_module('featureSet'), 'famBaseline')
    svmFamModel = svm_load_model('conf/famBaseline.model')
antChanged = 0
antSVM = 0
###Test all family matches gives better results
for fmatch in config['fam_matches'].find().batch_size(50):
    work = getFamilyFromId(fmatch['workid'] , config['families'], config['relations'])
    match = getFamilyFromId(fmatch['matchid'], config['match_families'], config['match_relations'])
    #Run through children and change 'EjMatch' for unreasonable matches to 'split'
    changes = False
    for mt in config['matches'].find({'$and': [
              {'status': {'$in': list(common.statEjOK)}},
              {'workid': {'$in': work['children']}},
              {'matchid': {'$in': match['children']}}
              ]}):
        #AA GET EVENTS
        workchild = config['persons'].find_one({'_id': mt['workid']})
        matchchild = config['match_persons'].find_one({'_id': mt['matchid']})
        if (nameDiff(workchild, matchchild) and 

            eventDiff(workchild, matchchild, ('birth','death'), ('date',))):