<p>Few points:</p>

<ul>
<li><p>I don't see resizing, aligning and whitening of the input face image that is fed into the network.</p></li>
<li><p>You cannot add a fixed margin of 50 to a variable-sized face. There has to be a scaling such that the face region fills almost the same region in every input image.</p></li>
<li><p>I am not sure about the model you are using, but if you are using <a href="https://arxiv.org/abs/1503.03832" rel="nofollow noreferrer">FaceNet</a>, your accepted matching threshold, 0.1, seems to be very low. It will not accept any matches unless it is the same exact image (with a distance of 0.0), or has a very minimal variation from the gallery image.</p></li>
</ul>
Incorrect facenet recognition
<p>I've been working on a face recognition attendance management system. I've built the pipeline from scratch but in the end,the script recognizes the wrong face among a group of 10 classes.
I've implemented the following pipeline using Tensorflow and Python.</p>

<ol>
<li>Capture images, resize, align them using dlib's shape predictor and store them in named folders for later comparison while performing recognition.</li>
<li><p>Pickle the images into a <code>data.pickle</code> file for later deserialization.</p></li>
<li><p>Using OpenCV to implement MTCNN algorithm to detect faces in a frame captured by webcam</p></li>
<li>passing these frames into a facenet network to create 128-D embeddings and compared accordingly with the embeddings present in pickle database.</li>
</ol>

<p>Given Below is the main file which runs step 3 and 4:</p>

<pre><code>from keras import backend as K
import time
from multiprocessing.dummy import Pool
K.set_image_data_format('channels_first')
import cv2
import os
import glob
import numpy as np
from numpy import genfromtxt
import tensorflow as tf
from keras.models import load_model
from fr_utils import *
from inception_blocks_v2 import *
from mtcnn.mtcnn import MTCNN
import dlib
from imutils import face_utils
import imutils
import pickle
from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import train_test_split


FRmodel = load_model('face-rec_Google.h5')
# detector = dlib.get_frontal_face_detector()
detector = MTCNN()
# FRmodel = faceRecoModel(input_shape=(3, 96, 96))
#
# # detector = dlib.get_frontal_face_detector()
# # predictor = dlib.shape_predictor("shape_predictor_68_face_landmarks.dat")
# def triplet_loss(y_true, y_pred, alpha = 0.3):
#     """
#     Implementation of the triplet loss as defined by formula (3)
#
#     Arguments:
#     y_pred -- python list containing three objects:
#             anchor -- the encodings for the anchor images, of shape (None, 128)
#             positive -- the encodings for the positive images, of shape (None, 128)
#             negative -- the encodings for the negative images, of shape (None, 128)
#
#     Returns:
#     loss -- real number, value of the loss
#     """
#
#     anchor, positive, negative = y_pred[0], y_pred[1], y_pred[2]
#
#     pos_dist = tf.reduce_sum(tf.square(tf.subtract(anchor, positive)), axis=-1)
#     neg_dist = tf.reduce_sum(tf.square(tf.subtract(anchor, negative)), axis=-1)
#     basic_loss = tf.add(tf.subtract(pos_dist, neg_dist), alpha)
#     loss = tf.reduce_sum(tf.maximum(basic_loss, 0.0))
#
#     return loss
#
# FRmodel.compile(optimizer = 'adam', loss = triplet_loss, metrics = ['accuracy'])
# load_weights_from_FaceNet(FRmodel)
def ret_model():
    return FRmodel

def prepare_database():
    pickle_in = open("data.pickle","rb")
    database =  pickle.load(pickle_in)
    return database

def unpickle_something(pickle_file):
    pickle_in = open(pickle_file,"rb")
    unpickled_file =  pickle.load(pickle_in)
    return unpickled_file


def webcam_face_recognizer(database):

    cv2.namedWindow("preview")
    vc = cv2.VideoCapture(0)

    while vc.isOpened():
        ret, frame = vc.read()
        img_rgb = cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
        img = frame
        # We do not want to detect a new identity while the program is in the process of identifying another person
        img = process_frame(img,img)

        cv2.imshow("Preview", img)
        cv2.waitKey(1)

    vc.release()

def process_frame(img, frame):
    """
    Determine whether the current frame contains the faces of people from our database
    """
    # rects = detector(img)
    rects = detector.detect_faces(img)
    # Loop through all the faces detected and determine whether or not they are in the database
    identities = []
    for (i,rect) in enumerate(rects):
        (x,y,w,h) = rect['box'][0],rect['box'][1],rect['box'][2],rect['box'][3]
        img = cv2.rectangle(frame,(x, y),(x+w, y+h),(255,0,0),2)

        identity = find_identity(frame, x-50, y-50, x+w+50, y+h+50)
        cv2.putText(img, identity,(10,500), cv2.FONT_HERSHEY_SIMPLEX , 4,(255,255,255),2,cv2.LINE_AA)

        if identity is not None:
            identities.append(identity)

    if identities != []:
        cv2.imwrite('example.png',img)

    return img

def find_identity(frame, x,y,w,h):
    """
    Determine whether the face contained within the bounding box exists in our database

    x1,y1_____________
    |                 |
    |                 |
    |_________________x2,y2

    """
    height, width, channels = frame.shape
    # The padding is necessary since the OpenCV face detector creates the bounding box around the face and not the head
    part_image = frame[y:y+h, x:x+w]

    return who_is_it(part_image, database, FRmodel)

def who_is_it(image, database, model):

    encoding = img_to_encoding(image, model)


    min_dist = 100
    # Loop over the database dictionary's names and encodings.
    for (name, db_enc) in database.items():

        # Compute L2 distance between the target "encoding" and the current "emb" from the database.
        dist = np.linalg.norm(db_enc.flatten() - encoding.flatten())

        print('distance for %s is %s' %(name, dist))

        # If this distance is less than the min_dist, then set min_dist to dist, and identity to name
        if dist &lt; min_dist:
            min_dist = dist
            identity = name

        if min_dist &gt;0.1:
            print('Unknown person')
        else:
            print(identity)
    return identity


if __name__ == "__main__":
    database = prepare_database()
    webcam_face_recognizer(database)
</code></pre>

<p>What am I doing wrong here?
<strong>Here the FRmodel is the facenet trained model</strong></p>