def listPersons(filters, multi, page=1, limit=10):
    matches = common.config['matches']
    #line_count = matches.find({}).count()
    #total = int(line_count / limit)+1
    start = limit * page - limit
    filter = []
    for (f,v) in filters.iteritems():
        if v=='checked': filter.append(f)
    rows = [['#','Status','Ant','Id','Namn',u'Född',u'Död','-']]
    i=start
    args = {'where': 'visa', 'what': '/view/persons'}
#is refId uniq within a contribution??
    aggrPipe = [{'$match': {'status': {'$in': filter}}}]
    if multi['DB2'] == 'checked':
        aggrPipe.append({'$project': {'refId': '$pmatch.refId', '_id': '$pmatch.refId',
                                  'wid': '$workid', 'mid': '$matchid',
                                  'namn': '$pmatch.name', 'status': 1,
                                  'birth': '$pmatch.birth', 'death': '$pmatch.death',
                                  'count': {'$concat': ['1']}}})
    else:
        aggrPipe.append({'$project': {'refId': '$pwork.refId', '_id': '$pwork.refId',
                                  'wid': '$workid', 'mid': '$matchid',
                                  'namn': '$pwork.name', 'status': 1,
                                  'birth': '$pwork.birth', 'death': '$pwork.death',
                                  'count': {'$concat': ['1']}}})
    #SORT BY refId
    if multi['DB1'] == 'checked':
        aggrPipe.append({'$group': {'_id': '$refId', 'count': {'$sum': 1},
                                    'namn': {'$first': '$namn'}, 'wid': {'$first': '$wid'},
                                    'status': {'$first': '$status'}}})
        aggrPipe.append({'$match': {'count': {'$gt': 1}}})
    if multi['DB2'] == 'checked':
        aggrPipe.append({'$group': {'_id': '$refId', 'count': {'$sum': 1},
                                    'namn': {'$first': '$namn'}, 'mid': {'$first': '$mid'},
                                    'status': {'$first': '$status'}}})
        aggrPipe.append({'$match': {'count': {'$gt': 1}}})
    try:
        tot = len(list(matches.aggregate(aggrPipe)))
    except:
        tot = '??'
    #SORT BY count, refId
    aggrPipe.append({'$sort': {'count': -1, '_id': 1}})
    aggrPipe.append({'$skip': start})
    aggrPipe.append({'$limit': limit})
    for match in matches.aggregate(aggrPipe):
        #print match
#FIX id, refId, namn, sum, född. död, visa
        if match['count'] == '1':
            args['wid'] = str(match['wid'])
            args['mid'] = str(match['mid'])
        elif multi['DB1'] == 'checked':
            args['wid'] = str(match['wid'])
            args['mid'] = ''
            match['status'] = ''
        elif multi['DB2'] == 'checked':
            args['wid'] = ''
            args['mid'] = str(match['mid'])
            match['status'] = ''
        else:
            args['wid'] = ''
            args['mid'] = ''
        refId = match['_id']
        tnm = match['namn']
        try: tfd = eventDisp(match['birth'])
        except: tfd = '-'
        try: tdd = eventDisp(match['death'])
        except: tdd = '-'
        visa = '<button onclick="doAction('+str(args)+')">Visa</button>'
        i += 1
        rows.append([i, match['status'], match['count'], refId, tnm, tfd, tdd, visa])
    return (rows,tot) def listFamilies(filters, multi, page=1, limit=10):
    fam_matches = common.config['fam_matches']
    start = limit * page - limit
    filter = []
    for (f,v) in filters.iteritems():
        if v=='checked': filter.append(f)
    rows = [['#','Status','Ant','Id','Namn',u'Gift','-']]
    i=start
    args = {'where': 'visa', 'what': '/view/families'}
#is refId uniq within a contribution??
    aggrPipe = [{'$match': {'status': {'$in': filter}}}]
    if multi['DB2'] == 'checked':
        aggrPipe.append({'$project': {'refId': '$matchRefId', '_id': '$matchRefId',
                                  'wid': '$workid', 'mid': '$matchid',
                                  'namn': {'$concat': [ { '$ifNull': ['$husb.pmatch.name','-']},
                                                       '<br/>',
                                                       { '$ifNull': ['$wife.pmatch.name','-']} ]},
                                  'status': 1, 'marriage': '$marriage',
                                  'count': {'$concat': ['1']}}})
    else:
        aggrPipe.append({'$project': {'refId': '$workRefId', '_id': '$workRefId',
                                  'wid': '$workid', 'mid': '$matchid',
                                  'namn': {'$concat': [ { '$ifNull': ['$husb.pwork.name','-']},
                                                       '<br/>',
                                                       { '$ifNull': ['$wife.pwork.name','-']} ]},
                                  'status': 1, 'marriage': '$marriage',
                                  'count': {'$concat': ['1']}}})
    if (multi['DB1'] == 'checked'):
        aggrPipe.append({'$group': {'_id': '$refId', 'count': {'$sum': 1},
                                    'namn': {'$first': '$namn'}, 'wid': {'$first': '$wid'},
                                    'marriage': {'$first': '$marriage'},
                                    'status': {'$first': '$status'}}})
        aggrPipe.append({'$match': {'count': {'$gt': 1}}})
    if (multi['DB2'] == 'checked'):
        aggrPipe.append({'$group': {'_id': '$refId', 'count': {'$sum': 1},
                                    'namn': {'$first': '$namn'}, 'mid': {'$first': '$mid'},
                                    'marriage': {'$first': '$marriage'},
                                    'status': {'$first': '$status'}}})
        aggrPipe.append({'$match': {'count': {'$gt': 1}}})
    #fix to avoid to big bson objects
    #calc tot
    try:
        tot = len(list(fam_matches.aggregate(aggrPipe)))
    except:
        tot = '??'
    #SORT BY count, refId
    aggrPipe.append({'$sort': {'count': -1, '_id': 1}})
    aggrPipe.append({'$skip': start})
    aggrPipe.append({'$limit': limit})
    for match in fam_matches.aggregate(aggrPipe):
#        tdd = '-'
        tfd = '-'
        if match['count'] == '1':
            args['wid'] = str(match['wid'])
            args['mid'] = str(match['mid'])
            try: tfd = eventDisp(match['marriage']['work'])
            except: tfd = '-'
        elif multi['DB1'] == 'checked':
            args['wid'] = str(match['wid'])
            args['mid'] = ''
            try: tfd = eventDisp(match['marriage']['work'])
            except: tfd = '-'
            match['status'] = ''  #dont show status when multilist
        elif multi['DB2'] == 'checked':
            args['wid'] = ''
            args['mid'] = str(match['mid'])
            try: tfd = eventDisp(match['marriage']['match'])
            except: tfd = '-'
            match['status'] = ''  #dont show status when multilist
        else:
            args['wid'] = ''
            args['mid'] = ''
        refId = match['_id']
        tnm = match['namn']
        visa = '<button onclick="doAction('+str(args)+')">Visa</button>'
        i += 1
#        rows.append([i, match['status'], match['count'], refId, tnm, tfd, tdd, visa])
        rows.append([i, match['status'], match['count'], refId, tnm, tfd, visa])
    return (rows, tot) def personView(wid, mid):
    #show personMatch
    res = []
##BUG FIX handle as families: if wid & mid elif mid elif wid ...
#    if mid:
#        matches = common.config['matches'].find({'workid': ObjectId(wid), 'matchid': ObjectId(mid)})
#    else:
#        matches = common.config['matches'].find({'workid': ObjectId(wid)})
##
    if wid and mid:
        matches = common.config['matches'].find({'workid': wid, 'matchid': mid})
    elif wid:
        """
        AA0 debug
        # multilista => only statOK and statManuell
        matches = common.config['matches'].find({'$and': [{'workid': wid},
                {'status': {'$in': list(common.statOK.union(common.statManuell))}}]})
        """
        matches = common.config['matches'].find({'workid': wid})

    elif mid:
        # multilista => only statOK and statManuell
        matches = common.config['matches'].find({'$and': [{'matchid': mid},
                {'status': {'$in': list(common.statOK.union(common.statManuell))}}]})
    else: matches = []
##
    for pmatch in matches:
        #print 'Doing', pmatch['pwork']['refId'], pmatch['pmatch']['refId']
        #FIX Filter om match-status
        prow = persMatchDisp('Person', pmatch)
        #show familyMatch
        mid = pmatch['matchid']
#        if not wid: wid = str(pmatch['workid'])
        wid = str(pmatch['workid'])
        ftab = []
        #Match exists children?
        #print 'children'
        fmatch = common.config['fam_matches'].find_one({'children.pwork._id': wid, 'children.pmatch._id': mid})
        if fmatch: ftab = famDisp(None, None, fmatch)
        else:  #HUSB
            #print 'husb'
            fmatch = common.config['fam_matches'].find_one({'husb.pwork._id': wid, 'husb.pmatch._id': mid})
            if fmatch: ftab = famDisp(None, None, fmatch)
            else:  #WIFE
                #print 'wife',wid,mid
                fmatch = common.config['fam_matches'].find_one({'wife.pwork._id': wid, 'wife.pmatch._id': mid})
                if fmatch: ftab = famDisp(None, None, fmatch)
                else:
                    #print 'matchchild'
                    #wfamid = common.config['families'].find_one({'children': ObjectId(wid)}, {'_id': True})
                    wfamid = common.config['relations'].find_one({'relTyp': 'child', 'persId': wid})
                    #mfamid = common.config['match_families'].find_one({'children': mid}, {'_id': True})
                    mfamid = common.config['match_relations'].find_one({'relTyp': 'child', 'persId': wid})
                    if wfamid and mfamid:
                        try:
                            fmatch = matchFam(wfamid['famId'], mfamid['famId'], common.config)
                            ftab = famDisp(None, None, fmatch)
                        except: pass
                    else:
                        #print 'matchhusb'
                        #wfamid = common.config['families'].find_one({'husb': ObjectId(wid)}, {'_id': True})
                        #mfamid = common.config['match_families'].find_one({'husb': mid}, {'_id': True})
                        wfamid = common.config['relations'].find_one({'relTyp': 'husb', 'persId': wid})
                        mfamid = common.config['match_relations'].find_one({'relTyp': 'husb', 'persId': wid})
                        if wfamid and mfamid:
                            try:
                                fmatch = matchFam(wfamid['famId'], mfamid['famId'], common.config)
                                ftab = famDisp(None, None, fmatch)
                            except: pass
                        else:
                            #print 'matchwife'
                            #wfamid = common.config['families'].find_one({'wife': ObjectId(wid)}, {'_id': True})
                            #mfamid = common.config['match_families'].find_one({'wife': mid}, {'_id': True})
                            wfamid = common.config['relations'].find_one({'relTyp': 'wife', 'persId': wid})
                            mfamid = common.config['match_relations'].find_one({'relTyp': 'wife', 'persId': wid})
                            if wfamid and mfamid:
                                try:
                                    fmatch = matchFam(wfamid['famId'], mfamid['famId'], common.config)
                                    ftab = famDisp(None, None, fmatch)
                                except: pass

        if fmatch:
            #print 'fmatch', fmatch['workRefId'], fmatch['matchRefId']
#            res.append((prow, str(pmatch['workid']), str(pmatch['matchid']),
#                        ftab, fmatch['workRefId'], fmatch['matchRefId']))
            res.append((prow, str(pmatch['workid']), str(pmatch['matchid']),
                        ftab, fmatch['workid'], fmatch['matchid']))
        else: res.append((prow, str(pmatch['workid']), str(pmatch['matchid']), ftab,None,None))
    return res def familyView(wid, mid):
    res = []
    if wid and mid:
        matches = common.config['fam_matches'].find({'workid': wid, 'matchid': mid})
    elif wid:
        # multilista => only statOK and statManuell
#        matches = common.config['fam_matches'].find({'workid': ObjectId(wid)})
        matches = common.config['fam_matches'].find({'$and': [{'workid': wid},
                {'status': {'$in': list(common.statOK.union(common.statManuell))}}]})
    elif mid:
        # multilista => only statOK and statManuell
#        matches = common.config['fam_matches'].find({'matchid': ObjectId(mid)})
        matches = common.config['fam_matches'].find({'$and': [{'matchid': mid},
                {'status': {'$in': list(common.statOK.union(common.statManuell))}}]})
    else: matches = []
    for fmatch in matches:
        ftab = famDisp(None, None, fmatch)
        res.append(([], None, None, ftab,fmatch['workid'],fmatch['matchid']))
    return res def famDisp( tmpfid, rgdfid, match = None ):
    if not match:
        #match = common.config['fam_matches'].find_one({'workid': ObjectId(tmpfid), 'matchid': ObjectId(rgdfid)})
        match = common.config['fam_matches'].find_one({'workid': tmpfid, 'matchid': rgdfid})
    tab = []
    tab.append(['', '<b>'+common.config['workDB'].split('_', 1)[1]+'</b>','','',
                '', '<b>'+common.config['matchDB'].split('_', 1)[1]+'</b>','','',''])
    tab.append(['', u'Namn/refId', u'Född', u'Död', '', u'Namn/refId', u'Född', u'Död', ''])
    for role in ('husb', 'wife'):
        try: tab.append(persMatchDisp(role, match[role], match['workid'], match['matchid']))
#        except: pass
        except Exception,e:
#            print 'Except in famDisp', e
            txt = [role,'?']
            try: txt.extend(persDisp(match[role]['pwork']))
            except: txt.extend(['-','_','-'])
            txt.extend([''])
            try: txt.extend(persDisp(match[role]['pmatch']))
            except: txt.extend(['-','_','-'])
            tab.append(txt)
    marr = ['', match['workRefId']]
    try: marr.append(eventDisp(match['marriage']['work']))
    except: marr.append('-')
    flag = common.config['flags'].find_one({"typ" : "IgnoreFamilyMatch",
                                            "workFam" : match['workid'],
                                            "matchFam" : match['matchid']})
    ignargs = {'where': 'res', 'what': '/actions/ignoreRelation', 'role': 'family',
               'workFam': str(match['workid']), 'matchFam': str(match['matchid'])}
    ignButt = '<br><button onclick="doAction('+str(ignargs)+')">Ignorera match</button>'
    ign = 'Marriage/ '+match['status']
    if flag: ign += '<br><b>Ignorerad</b>'
    elif match['status'] in ('Match', 'OK', 'rOK'): ign += ignButt
    marr.extend(['', ign, match['matchRefId']])
    try: marr.append(eventDisp(match['marriage']['match']))
    except: marr.append('-')
    marr.append('')
    marr.append('')
    tab.append(marr)
    tab.append(['','','','','','','','',''])
    for ch in match['children']:
        tab.append(persMatchDisp('child', ch, match['workid'], match['matchid']))
    return tab def persMatchDisp(role, pmI, workfamId = None, matchfamId = None):
    #HACK because status of persons in database fam_match are not always updated
    try:
        pm = common.config['matches'].find_one({'workid': pmI['workid'], 'matchid': pmI['matchid']})
    except:
        pm = pmI
    #FIX color dep status
    ignargs = {'where': 'res', 'what': '/actions/ignoreRelation', 'role': role,
               'workFam': str(workfamId), 'matchFam': str(matchfamId)}
    ign1 = ''
    ign2 = ''
    if role in ('husb', 'wife', 'child'):
        try:
            ignargs['pid'] = str(pm['pwork']['_id'])
            ignargs['fid'] = str(workfamId)
            flag = common.config['flags'].find_one({"typ" : "IgnoreRelation",
                                                    'persId': ignargs['pid'], 'relTyp': role,
                                                    'famId': ignargs['fid']})
            if flag: ign1 = '<b>Ignorerad</b>'
            else:
                ign1 = '<button onclick="doAction('+str(ignargs)+')">Ignorera relation</button>'
        except: pass
        try:
            ignargs['pid'] = str(pm['pmatch']['_id'])
            ignargs['fid'] = str(matchfamId)
            flag = common.config['flags'].find_one({"typ" : "IgnoreRelation",
                                                    'persId': ignargs['pid'], 'relTyp': role,
                                                    'famId': ignargs['fid']})
            if flag: ign2 = '<b>Ignorerad</b>'
            else:
                ign2 = '<button onclick="doAction('+str(ignargs)+')">Ignorera relation</button>'
        except: pass
    #cell1 = '<br/>'.join([role.capitalize(), ign1, ign2])
    #txt = [cell1, pm['status']]
    txt = [ign1]
    if 'pwork' in pm:
        txt.extend(persDisp(pm['pwork']))
    else:
        txt.extend(['-','-','-'])
    #persmatch buttons - if statements can be optimized
    args =  {'where': 'visa', 'what': '/view/persons', 'wid': '', 'mid': ''}
    if 'pwork' in pm: args['wid'] = str(pm['pwork']['_id'])
    if 'pmatch' in pm: args['mid'] = str(pm['pmatch']['_id'])
    persButton = '<button onclick="doAction('+str(args)+')">Visa</button>'
    if (role == 'child') and (pm['status'] in common.statEjOK):
        args['what'] = '/actions/split'
        persButton += '<br><button onclick="doAction('+str(args)+')">Split</button>'
    edit = ''
    try:
        graphArgs = {'where': 'graph', 'what': '/graph', 'role': role, 
                     'wid': str(pm['pwork']['_id']), 'mid': str(pm['pmatch']['_id'])}
        edit = '<button onclick="doAction('+str(graphArgs)+')">Graf</button>'
    except:
        pass
    #txt.extend([persButton + '<br/>' + edit])
    txt.extend(['<br/>'.join([role.capitalize()+'/ '+pm['status'], persButton, edit])])
    try: txt.extend(persDisp(pm['pmatch']))
    except: txt.extend(['-','_','-'])
    txt.extend([ign2])
    return txt def listPersonSkillnad(page=1, limit=10, diffType=None):
    start = limit * page - limit
    matches = common.config['matches']
    rows = [['#',u'Namn/refId', u'Född', u'Död','|', u'Namn/refId', u'Född', u'Död','Visa']]
    i = 0
    args = {'where': 'visa', 'what': '/view/persons', 'buttons': 'No'}
    for persMatch in matches.find({'status':
          {'$in': list(common.statOK.union(common.statManuell))}}):
#        if (nameDiff(persMatch['pwork'], persMatch['pmatch']) or
        if (nameSpellDiff(persMatch['pwork'], persMatch['pmatch']) or
            eventDiff(persMatch['pwork'], persMatch['pmatch'], ('birth','death') )) :
#Limit list to certain type of diff
            #date
            if ((diffType == 'Datum') and 
                not eventDiff(persMatch['pwork'], persMatch['pmatch'],
                              ('birth','death'), ('date',))): continue
            #place
            if ((diffType == 'Plats') and 
                not eventDiff(persMatch['pwork'], persMatch['pmatch'],
                              ('birth','death'), ('normPlaceUid',))): continue
            #name
            if ((diffType == 'Namn') and
                not nameDiff(persMatch['pwork'], persMatch['pmatch'])): continue
            #name-groups
            if ((diffType == 'NamnGrupp') and
                not nameGroupDiff(persMatch['pwork'], persMatch['pmatch'])): continue
            #name-groups
            if ((diffType == 'NamnStavning') and
                not nameSpellDiff(persMatch['pwork'], persMatch['pmatch'])): continue
#end limit
            i += 1
            if i<=start: continue
            if i>start+limit:
                continue
#                break
            row = [str(i)]
            row1 = persDisp(persMatch['pwork'])
            row2 = persDisp(persMatch['pmatch'])
#Code below depends on order of what persMatch returns
#0=name, 1=birth, 2=death
            if persMatch['pwork']['name'].replace('*','').replace('/',' ').replace('(',' ').replace(')',' ') == persMatch['pmatch']['name'].replace('*','').replace('/',' ').replace('(',' ').replace(')',' '):
                row.append(row1[0])
            else:
                row.append('<b>' + row1[0] + '</b>')
            if eventDiff(persMatch['pwork'], persMatch['pmatch'], ('birth',) ):
                row.append(row1[1])
            else:
                row.append('')
            if eventDiff(persMatch['pwork'], persMatch['pmatch'], ('death',) ):
                row.append(row1[2])
            else:
                row.append('')
            row.append('|')

            if persMatch['pwork']['name'].replace('*','').replace('/',' ').replace('(',' ').replace(')',' ') == persMatch['pmatch']['name'].replace('*','').replace('/',' ').replace('(',' ').replace(')',' '):
#                row.append('')
                #row.append(row2[0].split('gedcom_',1)[1]) #Need gedcomId
                row.append(row2[0]) #Need gedcomId
            else:
                row.append(row2[0])
            if eventDiff(persMatch['pwork'], persMatch['pmatch'], ('birth',) ):
                row.append(row2[1])
            else:
                row.append('')
            if eventDiff(persMatch['pwork'], persMatch['pmatch'], ('death',) ):
                row.append(row2[2])
            else:
                row.append('')
            args['wid'] = str(persMatch['workid'])
            args['mid'] = str(persMatch['matchid'])
            row.append('<button onclick="doAction('+str(args)+')">Visa</button>')
            rows.append(row)
    return (rows,i) def listFamiljeSkillnad(page):
    return ([['<h4>NOT implemented yet</h4>','']], 0) def ignoreRelation(persId, famId, role, workFam=None, matchFam=None):
    if role == 'family':
        common.config['flags'].insert_one({'typ': 'IgnoreFamilyMatch', 'workFam': workFam,
                                   'matchFam': matchFam})
        return 'Flagga: IgnoreFamilyMatch %s -> %s<br>' % (workFam, matchFam)
    else:
        common.config['flags'].insert_one({'typ': 'IgnoreRelation', 'persId': persId,
                                           'relTyp': role, 'famId': famId})
        return 'Flagga: IgnoreRelation person=%s roll=%s familj=%s<br>' % (persId, role, famId)
    return '' def getFlags(personId, famId, role):
    #flags = [['oid','refId','famRefId','text'],[personId, '', '', 'Showing flags for']]
    flags = [['oid','refId','famId','text'],[personId, '', '', 'Showing flags for']]
    for flag in common.config['flags'].find({'workid': ObjectId(personId)}):
        #flags.append(flag)
        flags.append([flag['personid'],'',flag['famid'],flag['text']])
    return flags def setOKfamily(wid, mid):
    return setFamOK(wid, mid, common.config, famlist = None, button = True) def setEjOKfamily(wid, mid, code='EjOK'):
#Villkor: inget matchat barn eller båda föräldrarna matchade.
#Ska testas + evt felmeddelande
    #logging.debug('setEjOKfamily %s %s %s', wid, mid, code)    #res =
    match = common.config['fam_matches'].find_one({'workid': wid, 'matchid': mid})
    if not match: return ''
    if (('husb' in match) and (match['husb']['status'] in common.statOK) and
        ('wife' in match) and (match['wife']['status'] in common.statOK)):
        return 'Not allowed: Both husb and wife matched'
    for ch in match['children']:
        if ch['status'] in common.statOK:
            return 'Not allowed: Matched child'
    res = ''
    for ch in match['children']:
        if ch['status'] in common.statManuell:
            #setEjOKperson calls updateFam
            res += setEjOKperson(str(ch['workid']), str(ch['matchid']), code=code)
    common.config['fam_matches'].update_one({'workid': wid, 'matchid': mid}, {'$set': {'status': 'FamEjOK'}})
    return res def setOKperson(wid, mid, button = True):
    indlist = []
    if button:
        kod ='OK'
        negKod = 'EjOK'
    else:
        kod = 'rOK'
        negKod = 'rEjOK'
    for mt in common.config['matches'].find({'$or': [{'workid': wid},{'matchid': mid}]},
                  {'workid': True, 'matchid': True, 'status': True}):
        #logging.debug('In setOKperson found match %s', mt['_id'])
        if mt['workid']==wid and  mt['matchid']==mid:
            if common.checkStatusUpdate(mt['status'], kod):
                #logging.debug('wid=%s mid=%s status old=%s new=%s',
                #              wid, mid, mt['status'], kod)
                common.config['matches'].update_one({'_id': mt['_id']}, {'$set': {'status': kod}})
                if mt['status'] in common.statOK: continue  #No other updates needed
                indlist.append(mt['workid'])
                ##TEST
                #check if wid,mid med i något familjepar som inte är med i fam_matches
                #   insert minimum fam_match med status 'candidate' - svårt!!
                #   it gets updated later by updateFamMatch
                for role in ('husb', 'wife', 'children'):
                    #tFam = common.config['families'].find({role: wid}, {'_id': 1} )
                    tFam = common.config['relations'].find({'persId': wid, 'relTyp': role})
                    #rFam = common.config['match_families'].find({role: mid}, {'_id': 1} )
                    rFam = common.config['match_relations'].find({'persId': mid, 'relTyp': role})
                    for f in tFam:
                        for ff in rFam:
                            if not common.config['fam_matches'].find_one({'workid': f['famId'], 'matchid': ff['famId']}):
                                common.config['fam_matches'].insert_one(matchFam(f['famId'], ff['famId'], common.config))
                           #Check multifam-resolution From match.py
                ##TEST
        elif common.checkStatusUpdate(mt['status'], negKod):
            #logging.debug('wid=%s mid=%s status old=%s new=%s',
            #              mt['workid'], mt['matchid'], mt['status'], negKod)
            common.config['matches'].update_one({'_id': mt['_id']}, {'$set': {'status': negKod}})
            if mt['status'] in common.statEjOK: continue  #No other updates needed
            indlist.append(mt['workid'])
    if not indlist:
        #logging.debug('No status changing updates')
        return 'No status changing updates'
    flist = set()
    for id in indlist:  #get list of involved individuals
        #for fam in common.config['families'].find({'$or': [{'husb': id}, {'wife': id}, {'children': id}]}, {'_id': True}):
        #    flist.add(fam['_id'])
        for fam in common.config['relations'].find({'persId': id}, {'famId': True}):
            flist.add(fam['famId'])
    return updateFamMatch(flist, common.config) def setEjOKperson(wid, mid, code='EjOK'):
    mt = common.config['matches'].find_one({'workid': wid, 'matchid': mid})
    if not common.checkStatusUpdate(mt['status'], code): return 'setEjOKperson Update not done'
    #logging.debug('setEjOKperson work=%s match=%s code=%s', wid, mid, code)
    common.config['matches'].update_one({'_id': mt['_id']}, {'$set': {'status': code}})
    indlist = [wid]
    flist = set()
    for id in indlist:  #get list of involved individuals
        #for fam in common.config['families'].find({'$or': [{'husb': id}, {'wife': id}, {'children': id}]}, {'_id': True}):
        #    flist.add(fam['_id'])
        for fam in common.config['relations'].find({'persId': id}, {'famId': True}):
            flist.add(fam['famId'])
    return updateFamMatch(flist, common.config) def split(wid, mid):
    """ Precondition personmatch in status statEjOK
    Parameters: wid workid, mid matchid for persons
    Algorithm:
      delete personmatch wid,mid
      generate famList from wid,mid
      update family matches for famList
    """
##    common.config['matches'].remove({'workid': wid, 'matchid': mid})
#TEST
    common.config['matches'].update_one({'workid': wid, 'matchid': mid},
                                    {'$set': {'status': 'split'}})
#TEST
    famList = set()
    for doc in common.config['fam_matches'].find({'$or':
                [{'husb.workid': wid, 'husb.matchid': mid},
                 {'wife.workid': wid, 'wife.matchid': mid}]}, {'workid': 1}):
        famList.add(doc['workid'])
    for doc in common.config['fam_matches'].find({'children.workid': wid}):
        for ch in doc['children']:
            if 'matchid' in ch and ch['matchid'] == mid:
                famList.add(doc['workid'])
                break
    #logging.debug('split %s %s %s', wid,mid, famList)
    return updateFamMatch(famList, common.config) def dbfind(db, q):
    #return 'Q:' + q.field
    mapQ = {'ID': '_id', 'name': 'name', 'refId': 'refId',
            'Birth date': 'birth.date'}
    import pprint
    query = {}
    #if q.field == 'refId': query['refId'] = q.val
    #else: return '???'
#    res = common.admClient[db][q.coll].find_one(query)
    query = {mapQ[q.field]: q.val}
    resTxt = ''
    for res in common.admClient[db][q.coll].find(query):
        resTxt += pprint.pformat(res) + "\n\n"
    if not resTxt:
        resTxt = 'No hits'
    return '<pre>' + resTxt + '</pre>' def familyViewAll(skipAnt=0):
    res = []
    buttons = True
    wid = None
    mid = None
    from collections import OrderedDict
    import re
    filter = OrderedDict()
    for stat in (common.statOK, common.statManuell, common.statEjOK):
        for s in list(stat): filter[s] = ''
    for s in list(common.statOK): filter[s] = 'checked'
    for s in list(common.statManuell): filter[s] = 'checked'

    #hitta multimatch OK/Manuell
    (tmp, tmp1) = listFamilies(filter, {'DB1': 'checked', 'DB2': ''}, page=1, limit=1)
    if len(tmp)>1:
        #'wid': '54302ae7d6dd3d665c3cd4cd',
        visa = tmp[1][6]
        m = re.search(r"\'wid\': \'([^']+)\',", visa)
        if m:
            wid = m.group(1)
        else: print 'No wid in DB1'
    else:
        (tmp, tmp1) = listFamilies(filter, {'DB1': '', 'DB2': 'checked'}, page=1, limit=1)
        if len(tmp)>1:
            print 'DB2', tmp
            visa = tmp[1][6]
            m = re.search(r"\'mid\': \'([^']+)\',", visa)
            if m: mid = ObjectId(m.group(1))
            else: print 'No mid in DB2'
    if not wid and not mid:
        print 'No DB1, DB2'
        #Alla manuella
        mt = common.config['fam_matches'].find_one({'status': {'$in': list(common.statManuell)}},
                                                   {'matchid': 1, 'workid': 1})
        if mt:
            wid = mt['workid']
            mid = mt['matchid']
        else: return res
    matches = common.config['fam_matches'].find({'$and': [
                    {'$or': [{'workid': wid}, {'matchid': mid}]},
                    {'status': {'$in': list(common.statOK.union(common.statManuell))}}
                    ]})
    for fmatch in matches:
        ftab = famDisp(None, None, fmatch)
        res.append(([], None, None, ftab,fmatch['workid'],fmatch['matchid']))
    return res def workFlowUI(user, workingDir):
    files = []
    if not workingDir:
        rootdir = "./files/" 
        if user == 'guest':
            #get temporary directory (do NOT use '_' in directorynames)
            workingDir = ''
            tempdir = rootdir + 'GUEST'
            for i in xrange(1000):
                workingDir = '%s%04d' % (tempdir, i)
                if not os.path.isdir(workingDir):
                    try:
                        os.mkdir(workingDir)
                        user = '%s%04d' % (user, i)
                        break
                    except OSError:
                        continue
        else:
            workingDir = rootdir + user
            if not os.path.isdir(workingDir):
                os.mkdir(workingDir)
    for file in glob.glob(workingDir + "/*.GED_*"):
        files.append(os.path.basename(file))
    for file in glob.glob(workingDir + "/*.ged_*"):
        files.append(os.path.basename(file))
    dbs = []
    #8 = len rootdir FIX!
    activeUser = workingDir[8:]
    for db in client.database_names():
        if db.startswith(activeUser+'_') or (user == 'admin'):
            dbs.append(db)
    return (files, dbs, workingDir, activeUser) def doUpload(dir, fileitem):
    # Test if the file was uploaded
    if fileitem.filename:
       message = ''
       # strip leading path from file name to avoid directory traversal attacks
       fn = os.path.basename(fileitem.filename)
       rootdir = dir + '/'
       fdir = rootdir + fn.split('.')[0]
       if os.path.isdir(fdir): shutil.rmtree(fdir)
       try:
           os.mkdir(fdir)
       except Exception, e:
           message += str(e)
       open(fdir+'/'+fn, 'wb').write(fileitem.file.read())
       message += 'The file "' + fn + '" was uploaded successfully'
       return (message, fdir, fn) def cleanUp(activeUser, dir):
    #only called for the user 'guest_****'
    if activeUser.startswith('guest') or activeUser.startswith('GUEST'):
        #remove directory and delete databases
        for db in client.database_names():
            if db.startswith(activeUser):
                #mongodump --db 'db' > dir/'db'  #FIX
                print 'cleanUp dropping', db
                client.drop_database(db)
        os.rename(dir, dir + '_' + str(time.time())) def getDBselect(what, db1, activeUser, workingDir):
    #if db.startswith(activeUser) or (user == 'admin'):  #Sanity check/security FIX
    if what == 'manualMatch':
        submitValue = 'do Manual Matching'
    elif what == 'sanity':
        submitValue = 'do Sanity Check'
    elif what == 'listSkillnad':
        submitValue = 'do Visa Skillnad'
    elif what == 'famMatches':
        submitValue = 'ladda ner matchade familjer'
    elif what == 'listDubl':
        submitValue = 'do Visa Duplicates'
    elif what == 'merge':
        submitValue = 'do Merge'
    else:
        submitValue = 'do ????'
    db = client[db1]
    selOptions = ''
    for coll in db.collection_names():
        #Manual Match => check that collection xxx_match finns
        if coll.startswith('matches_'):
            #listDubl uses match against itself - not valid for others
            if what != 'listDubl' and coll.endswith(db1): continue
            elif what == 'listDubl' and coll[8:] != db1: continue
            selOptions += '<option>'+coll[8:]+'</option>'  #Remove 'matches_' to get databasename
    if selOptions:
        noDefault = u'<option>Välj databas II</option>'
        return '<select name="matchDB">' + noDefault + selOptions + '</select><p><input type="submit" value="'+submitValue+'" />'
    else: return 'No valid choices' def listOldLogs( user, database ):
    if not database: return "Ingen databas vald<br>"
    import zipfile
    mess = 'Logfiler ' + database + "<br>\n"
    (tmp, db) = database.split('_', 1)
    utf8 = db + '.ged_UTF8'
    utf8U = db + '.GED_UTF8'
    zipf = zipfile.ZipFile('./files/'+user+'/'+db+'/Logs.zip', 'w')
    for fil in ('Log', 'Info.txt', 'Note.txt', 'Check_lista.txt', 'RGDN.txt', 'RGDO.txt',
                'RGDD.txt', 'RGDK.CSV', 'RGDXL.txt', utf8, utf8U):
        filepath = './files/'+user+'/'+db+'/'+fil
        if os.path.isfile(filepath):
            mess += '<a href="/getFile?fil='+filepath+'">'+fil+'</a><br>'
            zipf.write(filepath)
    for (dirpath, dirnames, filenames) in os.walk('./files/'+user+'/'+db):
        for fil in filenames:
            if fil.endswith('.log') or fil.endswith('.err'):
                mess += '<a href="/getFile?fil='+os.path.join(dirpath, fil)+'">'+fil+'</a><br>'
                zipf.write(os.path.join(dirpath, fil))
        break
    zipf.close()
    mess += '<a href="/getFile?fil='+'./files/'+user+'/'+db+'/Logs.zip'+'">All logs as zip-archive</a><br>'
    mess += '<a href="/getFile?fil='+'./files/'+user+'/'+db+'/DgDub.txt'+'">Dubblettfil till DISGEN</a><br>'
    return mess def genGraph(centerPersonId, families, persons, relations, directory, title):
    global mapPersId
    centerPersonId = centerPersonId
    filnamn = directory+'/graph.gv'
    gvFil = open(filnamn, 'wb')
    gvFil = codecs.getwriter('UTF-8')(gvFil)

    gvFil.write('digraph G {charset=utf8; overlap=false; rankdir = LR; ratio = compress; ranksep = 0.25; nodesep = 0.03;fontname=Helvetica; fontsize=16; fontcolor=black; label="'+title+'"; labelloc=t;')
    gvFil.write("\n")
    mapFamc = {}
    mapPersId = {}
    persList = set()
    famList = []
    partnerList = []
    #for f in families.find({'$or': [{'husb': centerPersonId},
    #                                {'wife':  centerPersonId} ]}):
    #NoParents = True
    #for f in families.find({'children': centerPersonId}):
    #    NoParents = False
    #    famList.append(f)
    NoParents = True
    for f in relations.find({'persId': centerPersonId}):
        famList.append(getFamilyFromId(f['famId'], families, relations))
        if f['relTyp'] == 'child': NoParents = False
    if NoParents: 
        dummyFam['children'] = [centerPersonId]
        famList.append(dummyFam)
    for fam in famList:
        for partner in ('husb', 'wife'):
            if partner in fam:
                #persList.add(fam[partner])
                #partnerList.append(fam[partner])
                if fam[partner] == centerPersonId:
                    printNode(fam[partner], 'shape="tab", style=filled, fillcolor="aquamarine"', persons, gvFil)
                else:
                    printNode(fam[partner], 'shape="folder", style=filled, fillcolor="lightyellow"', persons, gvFil)
        prev = None
        for ch in fam['children']:
            mapFamc[ch] = fam['_id']
            if ch == centerPersonId:
                printNode(ch, 'shape="tab", style=filled, fillcolor="aquamarine"', persons, gvFil)
            else:
                printNode(ch, 'shape="box", style=filled, fillcolor="whitesmoke"', persons, gvFil)
            if prev:
                gvFil.write(mapPersId[prev]+' -> '+mapPersId[ch]+' [style=invis, label="", len=0.02];'+"\n")
            prev = ch
        gvFil.write('{rank=same; ')
        for ch in fam['children']:
            gvFil.write(mapPersId[ch] + '; ')
        gvFil.write("}\n")

    for fam in famList:
        txt = '<FONT POINT-SIZE="8.0">' + fam['refId'] + '<br/>'
        for ev in ('marriage',):
            if ev in fam:
                txt += ev+':'+eventDisp(fam[ev]).replace('<br>',', ')
        txt += '</FONT>'
#No URL for family-graphs
        gvFil.write(fam['_id'] + '[label=<'+txt+'>, style=filled, shape="note"];')
        gvFil.write("\n")
        for ch in fam['children']:
            gvFil.write(fam['_id'] + '->' + mapPersId[ch])
            gvFil.write("\n")
        if 'wife' in fam and fam['wife']:
            gvFil.write(mapPersId[fam['wife']] + '->' + fam['_id'])
            gvFil.write("\n")
        if 'husb' in fam and fam['husb']:
            gvFil.write(mapPersId[fam['husb']] + '->' + fam['_id'])
            gvFil.write("\n")
    gvFil.write( "}\n" )
    gvFil.close()
    os.system('dot -Tsvg -O '+filnamn)
    fil = open(filnamn+'.svg' , 'rb')
    graph = fil.read()
    fil.close()
    return graph def editList(config, typ):
    global dubblList
    tit = 'Verktyg databas: ' + config['workDB']
    doTyp = []
    if typ in ('child', 'family', 'relation', 'dubblett', 'dubblettFind'):
        doTyp.append(typ)
    else:
        dubblList = []
        doTyp.append('child')
        doTyp.append('family')
    childErrs = [['Typ', 'Person', 'Familjer', 'action']]
    famErrs = [['Typ', 'Personer', 'Familj', 'action']]
    relErrs = [['Typ', 'Person', '', 'action']]
    dubbls = []
    (cErr, fErr, rErr) = sanity(config['persons'], config['families'],
                                    config['relations'], do=doTyp)
    cErr = repairChild(cErr, config['persons'], config['families'],
                       config['relations'], config['originalData'])
    for (pers, chFams) in cErr:
        child = "%s %s" % (pers['_id'], pers['name'])
        args = {'where': 'visa', 'what': '/view/relErr', 'typ': 'child',
                'person': str(pers['_id']), 'family': str(':'.join(chFams))}
        visa = '<button onclick="doAction('+str(args)+')">Visa</button>'
        childErrs.append(['Child', child, '; '.join(chFams), visa])
    #1 husb/wife per family
    fErr = repairFam(fErr, config['persons'], config['families'],
                       config['relations'], config['originalData'])
    for (famId, persList) in fErr:
        person = []
        pids = []
        for pers in persList:
            person.append("%s %s" % (pers['_id'], pers['name']))
            pids.append(pers['_id'])
        args = {'where': 'visa', 'what': '/view/relErr', 'typ': 'partner',
                'person': str(':'.join(pids)), 'family': str(famId)}
        visa = '<button onclick="doAction('+str(args)+')">Visa</button>'
        famErrs.append(['Partner', '<br>'.join(person), famId, visa])
    rErr = repairRel(rErr, config['persons'], config['families'],
                       config['relations'], config['originalData'])
    for pid in rErr:
        pers = config['persons'].find_one({'_id': pid})
        person = "%s %s" % (pers['_id'], pers['name'])
        args = {'where': 'visa', 'what': '/view/relErr', 'typ': 'noRel',
                'person': str(pers['_id']), 'family': ''}
        visa = '<button onclick="doAction('+str(args)+')">Visa</button>'
        relErrs.append(['Inga relationer', person, '', visa])
    if typ in ('dubblett', 'dubblettFind'):
        tit = 'Dubblett editor databas: ' + config['workDB']
        if len(dubblList) == 0 or typ=='dubblettFind':
            searchDB = luceneDB(config['workDB'])
            from matchtext import matchtext
            mt_tmp = matchtext()
            tab = []
            done = []
            tab.append(['Score',u'Namn/Id', 'Kandidat Id'])
            for person in config['persons'].find():
                matchtxt = mt_tmp.matchtextPerson(person, config['persons'],
                                                  config['families'], config['relations'])
                txt = []
                for term in matchtxt.split():
                    if term.startswith('Father') or term.startswith('Mother'): continue
                    txt.append(term)
                #candidates = search(' '.join(txt), person['sex'], 4) #Lucene search
                candidates = searchDB.search(' '.join(txt), person['sex'], 4) #Lucene search
                pstr = "%s %s" % (person['_id'], person['name'])
                for (kid,score) in candidates:
                    if kid == person['_id']: continue
                    if ';'.join([person['_id'],kid]) in done: continue
                    cand = config['persons'].find_one({'_id': kid})
                    if not cand: continue
                    try:
                        if abs(int(person['birth']['date'][0:4]) - int(cand['birth']['date'][0:4])) > 10:
                            continue
                    except:pass
                    args = {'where': 'visa', 'what': '/view/relErr', 'typ': 'dubblett',
                            'person': str(person['_id']), 'family': str(kid)}
                    visa = '<button onclick="doAction('+str(args)+')">Visa</button>'
                    tab.append(["%3.0f" % (score), pstr, kid, visa])
                    done.append(';'.join([kid,person['_id']]))
            dubblList = sorted(tab, key=itemgetter(0), reverse=True)[0:25]
            findAndMergeDuplFams(config['persons'], config['families'],
                                 config['relations'], config['originalData'])
            #OBS does not update luceneDB - OK?
        dubbls = dubblList
    return (tit, childErrs, famErrs, relErrs, dubbls) def viewChildErr(personIds, familyIds, config):
    tab = []
    drelargs = {'where': 'visa', 'what': '/actions/delRelation',
             'id1': '', 'id2': ''}
    tab.append(['',u'Namn/refId', u'Född', u'Död', '', u'Namn/refId', u'Född', u'Död', ''])
    fam1 = getFamilyFromId(familyIds[0], config['families'], config['relations'])
    fam2 = getFamilyFromId(familyIds[1], config['families'], config['relations'])
    for role in ('husb', 'wife'):
        drelargs['id1'] = str(fam1[role])
        drelargs['id2'] = str(familyIds[0])
        button = '<br><button onclick="doAction('+str(drelargs)+')">Ta bort relation</button>'
        t = [role + button]
        t.extend(persTab(fam1[role], config['persons']))
        if fam1[role]==fam2[role] and fam1[role]: t.append('Match')
        else: t.append('')
        t.extend(persTab(fam2[role], config['persons']))
        drelargs['id1'] = str(fam2[role])
        drelargs['id2'] = str(familyIds[1])
        button = '<button onclick="doAction('+str(drelargs)+')">Ta bort relation</button>'
        t.append(button)
        tab.append(t)
    try: marr1 = eventDisp(fam1['marriage'])
    except: marr1 = '-'
    try: marr2 = eventDisp(fam2['marriage'])
    except: marr2 = '-'
    fid1 = fam1['_id']
    fid2 = fam2['_id']
    if 'refId' in fam1: fid1 = '; '.join([fam1['refId'], fam1['_id']])
    if 'refId' in fam2: fid2 = '; '.join([fam2['refId'], fam2['_id']])
    args =  {'where': 'visa', 'what': '/actions/mergeFam',
             'id1': str(fam1['_id']), 'id2': str(fam2['_id'])}
    button = '<button onclick="doAction('+str(args)+')">Samma familj</button>'
    tab.append(['', fid1, marr1, '', button, fid2, marr2])
    tab.append(['', '', '', '', '', '', ''])
    done = []
    chDates = defaultdict(list)
    for chId in fam1['children']:
        k = 0
        person = config['persons'].find_one({'_id': chId}, {'birth.date': 1})
        if person and 'birth' in person and 'date' in person['birth']:
            if len(person['birth']['date']) == 8: k = int(person['birth']['date'])
            elif len(person['birth']['date']) == 4: k = int(person['birth']['date'])*10000
        chDates[k].append((1,chId))
        done.append(chId)
    for chId in fam2['children']:
        if chId in done: continue
        k = 0
        person = config['persons'].find_one({'_id': chId}, {'birth.date': 1, '_id': 0})
        if person and 'date' in person['birth']:
            if len(person['birth']['date']) == 8: k = int(person['birth']['date'])
            elif len(person['birth']['date']) == 4: k = int(person['birth']['date'])*10000
        chDates[k].append((2,chId))
    for k in sorted(chDates):
        for (pos,chId) in chDates[k]:
            drelargs['id1'] = str(chId)
            drelargs['id2'] = str(fam1['_id'])
            button = '<button onclick="doAction('+str(drelargs)+')">Ta bort relation</button>'
            t = [button]
            if pos == 1 and chId in fam2['children']:
                t.extend(persTab(chId, config['persons']))
                t.append('Match')
                t.extend(persTab(chId, config['persons']))
            elif pos==1:
                t.extend(persTab(chId, config['persons']))
                t.append('')
                t.extend(['', '', ''])
            elif pos==2:
                t.extend(['', '', ''])
                t.append('')
                t.extend(persTab(chId, config['persons']))
            drelargs['id2'] = str(fam2['_id'])
            button = '<button onclick="doAction('+str(drelargs)+')">Ta bort relation</button>'
            t.append(button)
            tab.append(t)
    graph = genGraphFam(personIds, familyIds[0], familyIds[1], config['persons'],
                        config['families'], config['relations'])
    return (tab, graph) def viewPartnerErr(personIds, familyIds, config):
    tab = []
    tab.append(['',u'Namn/refId', u'Född', u'Död', 'Familj', u'Namn/refId', u'Född', u'Död'])
    t = ['']
    t.extend(persTab(personIds[0], config['persons']))
    fam = config['families'].find_one({'_id': familyIds}, {'refId': 1})
    fid = fam['_id']
    if 'refId' in fam: fid = '; '.join([fam['refId'], fam['_id']])
    args =  {'where': 'visa', 'what': '/actions/mergePers',
             'id1': str(personIds[0]), 'id2': str(personIds[1])}
    button = '<br><button onclick="doAction('+str(args)+')">Samma person</button>'
    t.append(fid + button)
    t.extend(persTab(personIds[1], config['persons']))
    tab.append(t)
    graph = genGraphPers(personIds[0], personIds[1], familyIds, config['persons'],
                        config['families'], config['relations'])
    return (tab,graph) def viewNoRelErr(personIds, familyIds, config):
    #from luceneUtils import setupDir, search
    #setupDir(config['workDB'])
    searchDB = luceneDB(config['workDB'])
    from matchtext import matchtext
    mt_tmp = matchtext()
    person = config['persons'].find_one({'_id': personIds})
    matchtxt = mt_tmp.matchtextPerson(person, config['persons'],
                                      config['families'], config['relations'])
    candidates = searchDB.search(matchtxt, person['sex'], 5) #Lucene search
    tab = []
    tab.append(['Score',u'Namn/refId', u'Född', u'Död', '', u'Namn/refId', u'Född', u'Död'])
    for (kid,score) in candidates:
        if kid == personIds: continue
        cand = config['persons'].find_one({'_id': kid})
        if not cand: continue
        try:
            if abs(int(person['birth']['date'][0:4]) - int(cand['birth']['date'][0:4])) > 10:
                continue
        except:pass
        t = []
        t.append("%3.0f" % (score))
        t.extend(persTab(personIds, config['persons']))
        args =  {'where': 'visa', 'what': '/actions/mergePers',
                 'id1': str(personIds), 'id2': str(kid)}
        button = '<br><button onclick="doAction('+str(args)+')">Samma person</button>'
        t.append(button)
        t.extend(persTab(kid, config['persons']))
        tab.append(t)
    return (tab,'') def viewDubbl(personId, kandidateId, config, find=False):
    tab = []
    tab.append(['',u'Namn/refId', u'Född', u'Död', '', u'Namn/refId', u'Född', u'Död'])
    t = ['']
    t.extend(persTab(personId, config['persons']))
    args =  {'where': 'visa', 'what': '/actions/mergePers',
             'id1': str(personId), 'id2': str(kandidateId)}
    button = '<br><button onclick="doAction('+str(args)+')">Samma person</button>'
    t.append(button)
    t.extend(persTab(kandidateId, config['persons']))
    tab.append(t)
    graph = genGraphPers(personId, kandidateId, None, config['persons'],
                        config['families'], config['relations'])
    return (tab,graph) def viewQueryHits(Id, config):
    tab = []
    tab.append(['',u'Namn/refId', u'Född', u'Död', ''])
    t = ['']
    if Id.startswith('P_'):
        t.extend(persTab(Id, config['persons']))
        graph = genGraphPers(Id, None, None, config['persons'],
                        config['families'], config['relations'])
    elif Id.startswith('F_'):
        t.extend([Id, '', ''])
        graph = ''
    t.append('')
    tab.append(t)
    return (tab,graph) def mergePers(pId1, pId2, personDB, familyDB, relationDB, origDB):
    #print '  Merging persons', pId2, 'into', pId1
    origDB.update_one({'recordId': pId1, 'type': 'person'},
                      {'$push': {'map': pId2}})
    for r in relationDB.find({'persId': pId2}):
        relationDB.delete_one(r)
        del(r['_id'])
        r['persId'] = pId1
        relationDB.replace_one(r, r, upsert=True)
    #merge birth/death
    for ev in ('birth', 'death'):
        Events = []
        for pid in (pId1, pId2):
            event = personDB.find_one({'_id': pid}, {'_id': False, ev: True})
            if event: Events.append(event[ev])
        if Events:
            personDB.update_one({'_id': pId1}, {'$set':
                                         {ev: mergeEvent(Events)}})
    personDB.delete_one({'_id': pId2}) #delete person pId2
    # FIX Need name of DB:
    searchDB = luceneDB(personDB.full_name.split('.')[0])
    #searchDB.deleteRec(pId2)
    searchDB.updateDeleteRec(pId1, pId2, personDB, familyDB, relationDB)
    #Evt check if pId1 barn i två familjer och inga problem => delete den familjen
    return def mergeFam(fId1, fId2, personDB, familyDB, relationDB, origDB, updateLucene=False):
    #print '  Merging families', fId2, 'into', fId1
    origDB.update_one({'recordId': fId1, 'type': 'family'},
                      {'$push': {'map': fId2}})
    #Test fId1:husb/wife == fId2:husb/wife -- evt merge persons?
    partners = {}
    partners['husb']=set()
    partners['wife']=set()
    for fid in (fId1, fId2):
        for r in relationDB.find({'famId': fid,
                                 '$or': [{'relTyp': 'husb'}, {'relTyp': 'wife'}]}):
            partners[r['relTyp']].add(r['persId'])
    for partner in ('husb', 'wife'):
        if len(partners[partner]) == 2:
            p = list(partners[partner])
            mergePers(p[0], p[1], personDB, familyDB, relationDB, origDB)
    for r in relationDB.find({'famId': fId2}):
        relationDB.delete_one(r)
        del(r['_id'])
        r['famId'] = fId1
        relationDB.replace_one(r, r, upsert=True)
    #merge marriage events
    marrEvents = []
    for fid in (fId1, fId2):
        marr = familyDB.find_one({'_id': fid}, {'_id': False, 'marriage': True})
        if marr: marrEvents.append(marr['marriage'])
    if marrEvents:
        #print 'marrEvents', marrEvents
        familyDB.update_one({'_id': fId1}, {'$set':
                                    {'marriage': mergeEvent(marrEvents)}})
    familyDB.delete_one({'_id': fId2}) #delete family fId2
    if updateLucene:
        searchDB = luceneDB(personDB.full_name.split('.')[0])
        searchDB.deleteRec(fId2)
    return def doQuery(q, config):
    searchDB = luceneDB(config['workDB'])
    res = [['Namn, Id', 'Född', 'Död', 'score', 'action']]
    for uid, score in searchDB.query(q):
        #print uid, score
        pers = config['persons'].find_one({'_id': uid})
        t = persDisp(pers)
        t[0] += '<br/>' + uid
        t.append("%3.0f" % (score))
        args = {'where': 'visa', 'what': '/view/relErr', 'typ': 'queryhits',
                'person': str(uid), 'family': ''}
        visa = '<button onclick="doAction('+str(args)+')">Visa</button>'
        t.append(visa)
        res.append(t)
    tit = u'Resultat sökning i %s efter "%s"' % (config['workDB'], q)
    #tit = config['workDB']
    return (tit, res)
from collections import OrderedDict
from uiUtils import listPersons, listFamilies, personView, familyView, famDisp, persMatchDisp
from uiUtils import listPersonSkillnad, listFamiljeSkillnad, ignoreRelation, getFlags#, addFlag
from utils import setOKfamily, setEjOKfamily, setOKperson, setEjOKperson, split#, kopplaLoss
from uiUtils import dbfind,familyViewAll
from workFlow import workFlowUI, doUpload, cleanUp, getDBselect, listOldLogs
from graphUtils import genGraph
from relationEdit import editList, viewChildErr, viewPartnerErr, viewNoRelErr, viewDubbl, viewQueryHits
from mergeUtils import mergePers, mergeFam
from queryUtils import doQuery
from bottle import response
from beaker.middleware import SessionMiddleware
from cork import Cork
    from datetime import datetime
    from time import sleep
        from time import sleep
    from uiUtils import persDisp
    from uiUtils import persDisp
        from openpyxl import Workbook
        from openpyxl.writer.write_only import WriteOnlyCell
        from openpyxl.styles import colors
        from openpyxl.styles import Border, Side, Font, Color, PatternFill, Alignment
        from openpyxl.utils import get_column_letter
import codecs, time, logging, os.path
import conf.config, common
import bottle
    import subprocess
    import tempfile
    import subprocess
    import subprocess
           import shutil
        import tempfile
    import StringIO
        import csv
    import debugUtils
    import socket
                           famErrs=[], relErrs=[], dubbletter=[], hits=res) #TMP

#Likheter
@bottle.route('/downloadFamMatches')
@authorize()
def downloadFamMatches():
    output = StringIO.StringIO()
    done = set()
    fileFormat = bottle.request.query.fileFormat
    titleRow = [bottle.request.query.workDB,'','','',bottle.request.query.matchDB,'','']
    if fileFormat == 'xlsx':
        wb = Workbook(write_only=True)
        ws = wb.create_sheet()
        ws.title = "RGD matches"
        greyFill = PatternFill(start_color='DDDDDD',
                   end_color='DDDDDD',
                   fill_type='solid')
        greenFill = PatternFill(start_color='50FF50',
                   end_color='50FF50',
                   fill_type='solid')
        yellowFill = PatternFill(start_color='FFFF50',
                   end_color='FFFF50',
                   fill_type='solid')
        redFill = PatternFill(start_color='FF5050',
                   end_color='FF5050',
                   fill_type='solid')
        thick = Side(border_style="thick", color="000000")
        thin = Side(border_style="thin", color="000000")
        rowVals = []
        for val in titleRow:
            cell = WriteOnlyCell(ws, value=val.encode("utf-8"))
            cell.font = Font(bold=True)
            rowVals.append(cell)
        ws.append(rowVals)
    else:
        CSV =  csv.writer(output, dialect='excel')
        CSV.writerow([s.encode("utf-8") for s in titleRow])
    for fmatch in common.config['fam_matches'].find({'status':
                          {'$in': list(common.statOK.union(common.statManuell))}}):
        rows = famDisp(None, None, fmatch)
        line = False
        lines=0
        for r in rows:
            lines += 1
            if lines == 1: continue
            try:
                k1 = r[1].split('<br/>')[1]
            except:
                k1 = ''
            try:
                k2 = r[5].split('<br/>')[1]
            except:
                k2 = ''
            key = k1+';'+k2
            done.add(key)
            if 'Ignorerad' in r[0]: ignRelI = True
            else: ignRelI = False
            if 'Ignorerad' in r[8]: ignRelII = True
            else: ignRelII = False
            ignFam = False
            #remove html-code for buttons
            try:
                if 'Ignorerad' in r[4]: ignFam = True
                r[4] = r[4].split('<')[0]
                if ign: r[4] += ' Ignorerad'
            except:
                pass
            if r == ['', '', '', '', '', '', '', '', '']:
                line = True
                if fileFormat == 'xlsx': continue
            if fileFormat == 'xlsx':
                rowVals = []
                i=0
                green = False
                yellow = False
                red = False
                #if r[4].endswith(('EjMatch', 'EjOK', 'rEjOK')): red = True
                #elif r[4].endswith(('Manuell', 'rManuell')): yellow = True
                #elif r[4].endswith(('Match', 'OK', 'rOK')): green = True
                if ('EjMatch' in r[4]) or ('EjOK' in r[4]): red = True
                elif ('Match' in r[4]) or ('OK' in r[4]): green = True
                elif 'Manuell' in r[4]: yellow = True
                for val in r[1:8]:
                    i+=1
                    if i == 4: #separator between workDB and matchDB
                        cell = WriteOnlyCell(ws, value=val)
                        cell.border = Border(top=thin, left=thick, right=thick, bottom=thin)
                    else:
                        cell = WriteOnlyCell(ws,
                                value=val.replace('<br/>', "\n").rstrip().encode("utf-8"))
                        cell.alignment = Alignment(wrapText=True)
                        cell.border = Border(top=thin, left=thin, right=thin, bottom=thin)
                    if lines <=2: #head
                        cell.font = Font(bold=True)
                        cell.border = Border(top=thick, left=thin, right=thin, bottom=thick)
                        cell.fill = greyFill
                    elif line:
                        if i == 4: #separator between workDB and matchDB
                            cell.border = Border(top=thick, left=thick, right=thick, bottom=thin)
                        else:
                            cell.border = Border(top=thick, left=thin, right=thin, bottom=thin)
                    if (i == 4) and ignFam: cell.fill = redFill
                    elif (i==1) and ignRelI: cell.fill = redFill
                    elif (i==8) and ignRelII: cell.fill = redFill
                    elif green: cell.fill = greenFill
                    elif yellow:  cell.fill = yellowFill
                    elif red:  cell.fill = redFill
                    rowVals.append(cell)
                #line = False
                ws.append(rowVals)
            else:
                CSV.writerow([s.replace('<br/>', "\n").rstrip().encode("utf-8") for s in r])
        if fileFormat != 'xlsx':
            CSV.writerow(['#####', '#####', '#####', '#####', '#####', '|', '#####', '#####', '#####'])
    #List matched persons without families
    for persmatch in common.config['matches'].find({'status':
                          {'$in': list(common.statOK.union(common.statManuell))}}):
        if persmatch['pwork']['refId']+';'+persmatch['pmatch']['refId'] in done: continue
        rows = []
        rows.append(['', u'Namn/refId', u'Född', u'Död', '', u'Namn/refId', u'Född', u'Död', ''])

        rows.append(persMatchDisp('Person', persmatch))