title: Online Face Recognition using OpenCV
body: <p>I am trying to implement online face recognition using the webcam. I am using this two websites as references</p>
  
  <p><a href="http://www.shervinemami.co.cc/faceRecognition.html#recognizeFace" rel="nofollow">shervinemami.co.cc</a><br>
  <a href="http://www.cognotics.com/opencv/servo_2007_series/part_5/page_3.html" rel="nofollow">cognotics.com</a></p>
  
  <p>I have few questions:</p>
  
  <p>In face recognition, there are 6 steps: </p>
  
  <ol>
  <li>Grab a frame from the camera</li>
  <li>Detect a face within the image</li>
  <li>Crop the frame to show just the face</li>
  <li>Convert the frame to greyscale</li>
  <li>Preprocess the image</li>
  <li>Recognize the person in the image.</li>
  </ol>
  
  <p>I am able to do the first five steps. Last step i am not able to do. I am not sure how to  link step 5 to step 6.</p>
  
  <p>I have already created the train.txt file and test.txt file which contains the information of the training and testing images. I have already added the functions such as learn(), doPCA() to the code...</p>
  
  <p>But the point is how to use these functions in the main to recognize the image that is already preprocessed.</p>
  
  <p>Need some help on it...</p>
  
  <p>Attached the code below:</p>
  
  <pre><code>// Real-time.cpp : Defines the entry point for the console application.
  
  #include "stdafx.h"
  #include &lt;cv.h&gt;
  #include &lt;cxcore.h&gt;
  #include &lt;highgui.h&gt;
  #include &lt;cvaux.h&gt;
  
  IplImage ** faceImgArr        = 0; // array of face images
  CvMat    *  personNumTruthMat = 0; // array of person numbers
  int nTrainFaces               = 0; // the number of training images
  int nEigens                   = 0; // the number of eigenvalues
  IplImage * pAvgTrainImg       = 0; // the average image
  IplImage ** eigenVectArr      = 0; // eigenvectors
  CvMat * eigenValMat           = 0; // eigenvalues
  CvMat * projectedTrainFaceMat = 0; // projected training faces
  
  
  IplImage* getCameraFrame(CvCapture* &amp;camera);
  IplImage* detectFaces( IplImage *img ,CvHaarClassifierCascade* facecascade,CvMemStorage* storage );
  CvRect detectFaceInImage(IplImage *inputImg, CvHaarClassifierCascade* cascade);
  IplImage* preprocess( IplImage* inputImg);
  IplImage* resizeImage(const IplImage *origImg, int newWidth,
      int newHeight, bool keepAspectRatio);
  void learn();
  void recognize();
  void doPCA();
  void storeTrainingData();
  int  loadTrainingData(CvMat ** pTrainPersonNumMat);
  int  findNearestNeighbor(float * projectedTestFace);
  int  loadFaceImgArray(char * filename);
  
  int _tmain(int argc, _TCHAR* argv[])
  {
      CvCapture* camera = 0;  // The camera device.
      CvMemStorage            *storage;
      cvNamedWindow( "Realtime:", CV_WINDOW_AUTOSIZE);
      char *faceCascadeFilename = "C:/OpenCV2.1/data/haarcascades/haarcascade_frontalface_alt.xml";
      CvHaarClassifierCascade* faceCascade;
      faceCascade = (CvHaarClassifierCascade*)cvLoad(faceCascadeFilename, 0, 0, 0);
      storage = cvCreateMemStorage( 0 );
  
      learn();
  
      while ( cvWaitKey(10) != 27 )   // Quit on "Escape" key
          {   
          IplImage *frame = getCameraFrame(camera);
          //IplImage* resized=cvCreateImage(cvSize(420,240),frame-&gt;depth,3);
          //cvResizeWindow( "Image:", 640, 480);
          //cvResize(frame,resized);
          //cvShowImage( "Realtime:", resized );
          IplImage *imgA = resizeImage(frame, 420,240, true);
          IplImage *frame1 = detectFaces(imgA,faceCascade,storage);
          frame1 = preprocess(frame1);
          }   
      // Free the camera.
      cvReleaseCapture( &amp;camera );
      cvReleaseMemStorage( &amp;storage );
      return 0;
  }
  
  IplImage* getCameraFrame(CvCapture* &amp;camera)
  {
      IplImage *frame;
      int w, h;
  
      // If the camera hasn't been initialized, then open it.
      if (!camera) {
          printf("Acessing the camera ...\n");
          camera = cvCreateCameraCapture( 0 );
          if (!camera) {
              printf("Couldn't access the camera.\n");
              exit(1);
          }
          // Try to set the camera resolution to 320 x 240.
          cvSetCaptureProperty(camera, CV_CAP_PROP_FRAME_WIDTH, 320);
          cvSetCaptureProperty(camera, CV_CAP_PROP_FRAME_HEIGHT, 240);
          // Get the first frame, to make sure the camera is initialized.
          frame = cvQueryFrame( camera );
          if (frame) {
              w = frame-&gt;width;
              h = frame-&gt;height;
              printf("Got the camera at %dx%d resolution.\n", w, h);
          }
          // Wait a little, so that the camera can auto-adjust its brightness.
          Sleep(1000);    // (in milliseconds)
      }
  
      // Wait until the next camera frame is ready, then grab it.
      frame = cvQueryFrame( camera );
      if (!frame) {
          printf("Couldn't grab a camera frame.\n");
          exit(1);
      }
      return frame;
  }
  
  CvRect detectFaceInImage(IplImage *inputImg, CvHaarClassifierCascade* cascade)
  {
      // Smallest face size.
      CvSize minFeatureSize = cvSize(20, 20);
      // Only search for 1 face.
      int flags = CV_HAAR_FIND_BIGGEST_OBJECT | CV_HAAR_DO_ROUGH_SEARCH;
      // How detailed should the search be.
      float search_scale_factor = 1.1f;
      IplImage *detectImg;
      IplImage *greyImg = 0;
      CvMemStorage* storage;
      CvRect rc;
      double t;
      CvSeq* rects;
      CvSize size;
      int i, ms, nFaces;
  
      storage = cvCreateMemStorage(0);
      cvClearMemStorage( storage );
  
  
      // If the image is color, use a greyscale copy of the image.
      detectImg = (IplImage*)inputImg;
      if (inputImg-&gt;nChannels &gt; 1) {
          size = cvSize(inputImg-&gt;width, inputImg-&gt;height);
          greyImg = cvCreateImage(size, IPL_DEPTH_8U, 1 );
          cvCvtColor( inputImg, greyImg, CV_BGR2GRAY );
          detectImg = greyImg;    // Use the greyscale image.
      }
  
      // Detect all the faces in the greyscale image.
      t = (double)cvGetTickCount();
      rects = cvHaarDetectObjects( detectImg, cascade, storage,
              search_scale_factor, 3, flags, minFeatureSize);
      t = (double)cvGetTickCount() - t;
      ms = cvRound( t / ((double)cvGetTickFrequency() * 1000.0) );
      nFaces = rects-&gt;total;
      printf("Face Detection took %d ms and found %d objects\n", ms, nFaces);
  
      // Get the first detected face (the biggest).
      if (nFaces &gt; 0)
          rc = *(CvRect*)cvGetSeqElem( rects, 0 );
      else
          rc = cvRect(-1,-1,-1,-1);   // Couldn't find the face.
  
      if (greyImg)
          cvReleaseImage( &amp;greyImg );
      cvReleaseMemStorage( &amp;storage );
      //cvReleaseHaarClassifierCascade( &amp;cascade );
  
      return rc;  // Return the biggest face found, or (-1,-1,-1,-1).
  }
  
  IplImage* detectFaces( IplImage *img ,CvHaarClassifierCascade* facecascade,CvMemStorage* storage )
  {
      int i;
      CvRect *r;
      CvSeq *faces = cvHaarDetectObjects(
              img,
              facecascade,
              storage,
              1.1,
              3,
              0 /*CV_HAAR_DO_CANNY_PRUNNING*/,
              cvSize( 40, 40 ) );
  
      int padding_width = 30; // pixels
      int padding_height = 30; // pixels
  
      for( i = 0 ; i &lt; ( faces ? faces-&gt;total : 0 ) ; i++ ) {
          r = ( CvRect* )cvGetSeqElem( faces, i );
          cvRectangle( img,
                       cvPoint( r-&gt;x, r-&gt;y ),
                       cvPoint( r-&gt;x + r-&gt;width, r-&gt;y + r-&gt;height ),
                       CV_RGB( 255, 0, 0 ), 1, 8, 0 );
      }
  
      cvShowImage( "Realtime:", img );
  
      //cropping the face
      cvSetImageROI(img, cvRect(r-&gt;x,r-&gt;y,r-&gt;width,r-&gt;height));
      IplImage *img2 = cvCreateImage(cvGetSize(img), 
                              img-&gt;depth, 
                                img-&gt;nChannels);
      cvCopy(img, img2, NULL);
      cvResetImageROI(img);
  
      return img;
  }
  
  IplImage* preprocess( IplImage* inputImg){
      IplImage *detectImg, *greyImg = 0;
      IplImage *imageProcessed;
      CvSize size;
      detectImg = (IplImage*)inputImg;
      if (inputImg-&gt;nChannels &gt; 1) {
          size = cvSize(inputImg-&gt;width, inputImg-&gt;height);
          greyImg = cvCreateImage(size, IPL_DEPTH_8U, 1 );
          cvCvtColor( inputImg, greyImg, CV_BGR2GRAY );
          detectImg = greyImg;    // Use the greyscale image.
      }
  
      imageProcessed = cvCreateImage(cvSize(inputImg-&gt;width, inputImg-&gt;height), IPL_DEPTH_8U, 1);
      cvResize(detectImg, imageProcessed, CV_INTER_LINEAR);
      cvEqualizeHist(imageProcessed, imageProcessed);
      return imageProcessed;
  }
  
  IplImage* resizeImage(const IplImage *origImg, int newWidth,
      int newHeight, bool keepAspectRatio)
  {
      IplImage *outImg = 0;
      int origWidth;
      int origHeight;
      if (origImg) {
          origWidth = origImg-&gt;width;
          origHeight = origImg-&gt;height;
      }
      if (newWidth &lt;= 0 || newHeight &lt;= 0 || origImg == 0
          || origWidth &lt;= 0 || origHeight &lt;= 0) {
          //cerr &lt;&lt; "ERROR: Bad desired image size of " &lt;&lt; newWidth
          //  &lt;&lt; "x" &lt;&lt; newHeight &lt;&lt; " in resizeImage().\n";
          exit(1);
      }
  
      if (keepAspectRatio) {
          // Resize the image without changing its aspect ratio,
          // by cropping off the edges and enlarging the middle section.
          CvRect r;
          // input aspect ratio
          float origAspect = (origWidth / (float)origHeight);
          // output aspect ratio
          float newAspect = (newWidth / (float)newHeight);
          // crop width to be origHeight * newAspect
          if (origAspect &gt; newAspect) {
              int tw = (origHeight * newWidth) / newHeight;
              r = cvRect((origWidth - tw)/2, 0, tw, origHeight);
          }
          else {  // crop height to be origWidth / newAspect
              int th = (origWidth * newHeight) / newWidth;
              r = cvRect(0, (origHeight - th)/2, origWidth, th);
          }
          IplImage *croppedImg = cropImage(origImg, r);
  
          // Call this function again, with the new aspect ratio image.
          // Will do a scaled image resize with the correct aspect ratio.
          outImg = resizeImage(croppedImg, newWidth, newHeight, false);
          cvReleaseImage( &amp;croppedImg );
  
      }
      else {
  
          // Scale the image to the new dimensions,
          // even if the aspect ratio will be changed.
          outImg = cvCreateImage(cvSize(newWidth, newHeight),
              origImg-&gt;depth, origImg-&gt;nChannels);
          if (newWidth &gt; origImg-&gt;width &amp;&amp; newHeight &gt; origImg-&gt;height) {
              // Make the image larger
              cvResetImageROI((IplImage*)origImg);
              // CV_INTER_LINEAR: good at enlarging.
              // CV_INTER_CUBIC: good at enlarging.           
              cvResize(origImg, outImg, CV_INTER_LINEAR);
          }
          else {
              // Make the image smaller
              cvResetImageROI((IplImage*)origImg);
              // CV_INTER_AREA: good at shrinking (decimation) only.
              cvResize(origImg, outImg, CV_INTER_AREA);
          }
  
      }
      return outImg;
  }
  
  void learn()
  {
      int i, offset;
  
      // load training data
      nTrainFaces = loadFaceImgArray("C:/Users/HP/Desktop/OpenCV/50_images_of_15_people.txt");
      if( nTrainFaces &lt; 2 )
      {
          fprintf(stderr,
                  "Need 2 or more training faces\n"
                  "Input file contains only %d\n", nTrainFaces);
          return;
      }
  
      // do PCA on the training faces
      doPCA();
  
      // project the training images onto the PCA subspace
      projectedTrainFaceMat = cvCreateMat( nTrainFaces, nEigens, CV_32FC1 );
      offset = projectedTrainFaceMat-&gt;step / sizeof(float);
      for(i=0; i&lt;nTrainFaces; i++)
      {
          //int offset = i * nEigens;
          cvEigenDecomposite(
              faceImgArr[i],
              nEigens,
              eigenVectArr,
              0, 0,
              pAvgTrainImg,
              //projectedTrainFaceMat-&gt;data.fl + i*nEigens);
              projectedTrainFaceMat-&gt;data.fl + i*offset);
      }
  
      // store the recognition data as an xml file
      storeTrainingData();
  }
  
  void recognize()
  {
      int i, nTestFaces  = 0;         // the number of test images
      CvMat * trainPersonNumMat = 0;  // the person numbers during training
      float * projectedTestFace = 0;
  
      // load test images and ground truth for person number
      nTestFaces = loadFaceImgArray("C:/Users/HP/Desktop/OpenCV/test.txt");
      printf("%d test faces loaded\n", nTestFaces);
  
      // load the saved training data
      if( !loadTrainingData( &amp;trainPersonNumMat ) ) return;
  
      // project the test images onto the PCA subspace
      projectedTestFace = (float *)cvAlloc( nEigens*sizeof(float) );
      for(i=0; i&lt;nTestFaces; i++)
      {
          int iNearest, nearest, truth;
  
          // project the test image onto the PCA subspace
          cvEigenDecomposite(
              faceImgArr[i],
              nEigens,
              eigenVectArr,
              0, 0,
              pAvgTrainImg,
              projectedTestFace);
  
          iNearest = findNearestNeighbor(projectedTestFace);
          truth    = personNumTruthMat-&gt;data.i[i];
          nearest  = trainPersonNumMat-&gt;data.i[iNearest];
  
          printf("nearest = %d, Truth = %d\n", nearest, truth);
      }
  }
  
  int loadTrainingData(CvMat ** pTrainPersonNumMat)
  {
      CvFileStorage * fileStorage;
      int i;
  
      // create a file-storage interface
      fileStorage = cvOpenFileStorage( "facedata.xml", 0, CV_STORAGE_READ );
      if( !fileStorage )
      {
          fprintf(stderr, "Can't open facedata.xml\n");
          return 0;
      }
  
      nEigens = cvReadIntByName(fileStorage, 0, "nEigens", 0);
      nTrainFaces = cvReadIntByName(fileStorage, 0, "nTrainFaces", 0);
      *pTrainPersonNumMat = (CvMat *)cvReadByName(fileStorage, 0, "trainPersonNumMat", 0);
      eigenValMat  = (CvMat *)cvReadByName(fileStorage, 0, "eigenValMat", 0);
      projectedTrainFaceMat = (CvMat *)cvReadByName(fileStorage, 0, "projectedTrainFaceMat", 0);
      pAvgTrainImg = (IplImage *)cvReadByName(fileStorage, 0, "avgTrainImg", 0);
      eigenVectArr = (IplImage **)cvAlloc(nTrainFaces*sizeof(IplImage *));
      for(i=0; i&lt;nEigens; i++)
      {
          char varname[200];
          sprintf( varname, "eigenVect_%d", i );
          eigenVectArr[i] = (IplImage *)cvReadByName(fileStorage, 0, varname, 0);
      }
  
      // release the file-storage interface
      cvReleaseFileStorage( &amp;fileStorage );
  
      return 1;
  }
  
  void storeTrainingData()
  {
      CvFileStorage * fileStorage;
      int i;
  
      // create a file-storage interface
      fileStorage = cvOpenFileStorage( "facedata.xml", 0, CV_STORAGE_WRITE );
  
      // store all the data
      cvWriteInt( fileStorage, "nEigens", nEigens );
      cvWriteInt( fileStorage, "nTrainFaces", nTrainFaces );
      cvWrite(fileStorage, "trainPersonNumMat", personNumTruthMat, cvAttrList(0,0));
      cvWrite(fileStorage, "eigenValMat", eigenValMat, cvAttrList(0,0));
      cvWrite(fileStorage, "projectedTrainFaceMat", projectedTrainFaceMat, cvAttrList(0,0));
      cvWrite(fileStorage, "avgTrainImg", pAvgTrainImg, cvAttrList(0,0));
      for(i=0; i&lt;nEigens; i++)
      {
          char varname[200];
          sprintf( varname, "eigenVect_%d", i );
          cvWrite(fileStorage, varname, eigenVectArr[i], cvAttrList(0,0));
      }
  
      // release the file-storage interface
      cvReleaseFileStorage( &amp;fileStorage );
  }
  
  int findNearestNeighbor(float * projectedTestFace)
  {
      //double leastDistSq = 1e12;
      double leastDistSq = DBL_MAX;
      int i, iTrain, iNearest = 0;
  
      for(iTrain=0; iTrain&lt;nTrainFaces; iTrain++)
      {
          double distSq=0;
  
          for(i=0; i&lt;nEigens; i++)
          {
              float d_i =
                  projectedTestFace[i] -
                  projectedTrainFaceMat-&gt;data.fl[iTrain*nEigens + i];
              //distSq += d_i*d_i / eigenValMat-&gt;data.fl[i];  // Mahalanobis
              distSq += d_i*d_i; // Euclidean
          }
  
          if(distSq &lt; leastDistSq)
          {
              leastDistSq = distSq;
              iNearest = iTrain;
          }
      }
  
      return iNearest;
  }
  
  void doPCA()
  {
      int i;
      CvTermCriteria calcLimit;
      CvSize faceImgSize;
  
      // set the number of eigenvalues to use
      nEigens = nTrainFaces-1;
  
      // allocate the eigenvector images
      faceImgSize.width  = faceImgArr[0]-&gt;width;
      faceImgSize.height = faceImgArr[0]-&gt;height;
      eigenVectArr = (IplImage**)cvAlloc(sizeof(IplImage*) * nEigens);
      for(i=0; i&lt;nEigens; i++)
          eigenVectArr[i] = cvCreateImage(faceImgSize, IPL_DEPTH_32F, 1);
  
      // allocate the eigenvalue array
      eigenValMat = cvCreateMat( 1, nEigens, CV_32FC1 );
  
      // allocate the averaged image
      pAvgTrainImg = cvCreateImage(faceImgSize, IPL_DEPTH_32F, 1);
  
      // set the PCA termination criterion
      calcLimit = cvTermCriteria( CV_TERMCRIT_ITER, nEigens, 1);
  
      // compute average image, eigenvalues, and eigenvectors
      cvCalcEigenObjects(
          nTrainFaces,
          (void*)faceImgArr,
          (void*)eigenVectArr,
          CV_EIGOBJ_NO_CALLBACK,
          0,
          0,
          &amp;calcLimit,
          pAvgTrainImg,
          eigenValMat-&gt;data.fl);
  
      cvNormalize(eigenValMat, eigenValMat, 1, 0, CV_L1, 0);
  }
  
  int loadFaceImgArray(char * filename)
  {
      FILE * imgListFile = 0;
      char imgFilename[512];
      int iFace, nFaces=0;
  
  
      // open the input file
      if( !(imgListFile = fopen(filename, "r")) )
      {
          fprintf(stderr, "Can\'t open file %s\n", filename);
          return 0;
      }
  
      // count the number of faces
      while( fgets(imgFilename, 512, imgListFile) ) ++nFaces;
      rewind(imgListFile);
  
      // allocate the face-image array and person number matrix
      faceImgArr        = (IplImage **)cvAlloc( nFaces*sizeof(IplImage *) );
      personNumTruthMat = cvCreateMat( 1, nFaces, CV_32SC1 );
  
      // store the face images in an array
      for(iFace=0; iFace&lt;nFaces; iFace++)
      {
          // read person number and name of image file
          fscanf(imgListFile,
              "%d %s", personNumTruthMat-&gt;data.i+iFace, imgFilename);
  
          // load the face image
          faceImgArr[iFace] = cvLoadImage(imgFilename, CV_LOAD_IMAGE_GRAYSCALE);
  
          if( !faceImgArr[iFace] )
          {
              fprintf(stderr, "Can\'t load image from %s\n", imgFilename);
              return 0;
          }
      }
  
      fclose(imgListFile);
  
      return nFaces;
  }
  </code></pre>