opencv delphi能用吗 opencv dlib

转载

技术极先锋 2024-08-16 18:03:39

文章标签 opencv delphi能用吗 #include #define android 文章分类 计算机视觉人工智能

opencv 与dlib 结合实现人脸融合

融合流程

检测面部标记
旋转、缩放和转换第二张图像，使之与第一张图像相适应
调整第二张图像的色彩平衡，使之与第一个相匹配
把第二张图像的特性混合在第一张图像中

实现流程

找到人脸矩阵

使用dlib提取面部标记

用Dlib实现了论文One Millisecond Face Alignment with an Ensemble of Regression Trees中的算法

（ http://www.csc.kth.se/~vahidk/papers/KazemiCVPR14.pdf ，作者为Vahid Kazemi 和Josephine Sullivan）。
算法本身非常复杂，但dlib接口使用起来非常简单

人脸矩阵是由68个像素点组成，根据对应的像素的对应相应的五官

1-16代码脸型

17-21 22-26 分别代表眉毛

27-35代表鼻子

36-41 42-47 代码眼睛

48-68 带表嘴巴

get_landmarks()函数将一个图像转化成numpy数组，并返回一个68 x2元素矩阵，输入图像的每个特征点对应每行的一个x，y坐标。

特征提取器（predictor）要一个粗糙的边界框作为算法输入，由传统的能返回一个矩形列表的人脸检测器（detector）提供，其每个矩形列表在图像中对应一个脸。
为了构建特征提取器，预训练模型必不可少，相关模型可从dlib sourceforge库下载 ( http://sourceforge.net/projects/dclib/files/dlib/v18.10/shape_predictor_68_face_landmarks.dat.bz2 )。

调整脸部图像对齐

原理

现在我们已经有了两个标记矩阵，每行有一组坐标对应一个特定的面部特征（如第30行给出的鼻子的坐标）。我们现在要搞清楚如何旋转、翻译和规模化第一个向量，使它们尽可能适合第二个向量的点。想法是，可以用相同的变换在第一个图像上覆盖第二个图像。

实现步骤

代码分别实现了下面几步：

1．将输入矩阵转换为浮点数。这是之后步骤的必要条件。

2．每一个点集减去它的矩心。一旦为这两个新的点集找到了一个最佳的缩放和旋转方法，这两个矩心c1和c2就可以用来找到完整的解决方案。

3．同样，每一个点集除以它的标准偏差。这消除了问题的组件缩放偏差。

4．使用Singular Value Decomposition计算旋转部分。可以在维基百科上看到关于解决正交普氏问题的细节（https://en.wikipedia.org/wiki/Orthogonal_Procrustes_problem ）。

5.利用透视变换矩阵（ https://en.wikipedia.org/wiki/Transformation_matrix#Affine_transformations ）返回完整的转化。

源代码

#include <string.h>
 #include <jni.h>
 #include <android/log.h>
 #include <string>
 #include <stdio.h>
 #include <vector>
 #include <iostream>
 #include <opencv2/opencv.hpp>
 #include <opencv2/core/core.hpp>
 #include <opencv2/imgproc/imgproc.hpp>
 #include <opencv2/highgui/highgui.hpp>


 #include <dlib/image_processing/frontal_face_detector.h>
 #include <dlib/image_processing/render_face_detections.h>
 #include <dlib/image_loader/load_image.h>
 #include <dlib/image_processing.h>
 #include <dlib/opencv/cv_image.h>
 #include <dlib/gui_widgets.h>
 #include <dlib/image_io.h>
 #include <dlib/opencv.h>
 #include <iostream>


 using namespace dlib;
 using namespace std;
 using namespace cv;


 #define LOG_TAG "People_Det-JNI"
 #define LOGV(...) __android_log_print(ANDROID_LOG_VERBOSE, LOG_TAG, __VA_ARGS__)
 #define LOGD(...) __android_log_print(ANDROID_LOG_DEBUG, LOG_TAG, __VA_ARGS__)
 #define LOGI(...) __android_log_print(ANDROID_LOG_INFO, LOG_TAG, __VA_ARGS__)
 #define LOGW(...) __android_log_print(ANDROID_LOG_WARN, LOG_TAG, __VA_ARGS__)
 #define LOGE(...) __android_log_print(ANDROID_LOG_ERROR, LOG_TAG, __VA_ARGS__)


 /*
 #ifdef __cplusplus
 extern "C" {
 #endif
 */


 static dlib::shape_predictor msp;
 static frontal_face_detector detector;
 static dlib::full_object_detection FACE_MARK1,FACE_MARK2;
 static Mat wrap_dst, src_img2, mask;


 dlib::full_object_detection detectface(dlib::cv_image<dlib::bgr_pixel> mat)
 {
 dlib::full_object_detection mark;
 std::vector<dlib::rectangle> mRets;
 mRets = detector(mat);
 if(mRets.size() > 0)
 mark = msp(mat, mRets[0]);
 return mark;
 }


 jboolean facesRotate_UseData(JNIEnv* env, jobject thiz,
 jintArray face1,jint w1,jint h1,
 jintArray face2,jint w2,jint h2)
 {
 jint *cbuf1;
 jint *cbuf2;
 cbuf1 = env->GetIntArrayElements(face1, JNI_FALSE);
 cbuf2 = env->GetIntArrayElements(face2, JNI_FALSE);
 if(cbuf1 == NULL || cbuf1 == NULL){
 return JNI_FALSE;
 }


 Mat src_img(h1, w1, CV_8UC4, (unsigned char*)cbuf1);
 Mat src_img1(h2, w2, CV_8UC4, (unsigned char*)cbuf2);
 cvtColor(src_img,src_img,CV_RGBA2RGB);
 cvtColor(src_img1,src_img1,CV_RGBA2RGB);


 if(!src_img.data || !src_img1.data)
 LOGD("data error");


 dlib::cv_image<dlib::bgr_pixel> img(src_img);
 dlib::cv_image<dlib::bgr_pixel> img1(src_img1);
 FACE_MARK1 = detectface(img);
 FACE_MARK2 = detectface(img1);


 std::vector<cv::Point2f> srcTri;
 std::vector<cv::Point2f> dstTri;


 for(int i = 0;i < 68;i++) {
 srcTri.push_back(cv::Point2f(FACE_MARK1.part(i).x(),FACE_MARK1.part(i).y()));
 dstTri.push_back(cv::Point2f(FACE_MARK2.part(i).x(),FACE_MARK2.part(i).y()));
 }
 cv::Mat warp_mat;
 warp_mat = cv::findHomography(srcTri,dstTri,0);
 //the faceLine's size is 68,including all face data,such as:
 //the index 0 - 16:jaw data
 //the index 17 - 68:face data
 //the index 17 - 21:right brow data
 //the index 22 - 26:left brow data
 //the index 27 - 34:nose data
 //the index 36 - 41:right eye data
 //the index 42 - 47:left eye data
 //the index 48 - 60:mouth data


 src_img2 = src_img1;
 cv::warpPerspective(src_img,wrap_dst,warp_mat,src_img1.size());
 dlib::cv_image<dlib::bgr_pixel> img3(wrap_dst);
 FACE_MARK1 = detectface(img3);
 Mat new_img = Mat::zeros(src_img1.size(),CV_8UC3);
 std::vector<image_window::overlay_line> faceLine;
 faceLine = render_face_detections(FACE_MARK1);
 for(int i = 17 ;i < 61; i++){
 cv::Point p1(faceLine[i].p1.x(),faceLine[i].p1.y());
 cv::Point p2(faceLine[i].p2.x(),faceLine[i].p2.y());
 cv::line(new_img,p1,p2,cv::Scalar(255,255,255),1,cv::LINE_8);
 }
 imwrite("/data/data/com.example.faceswapdemo/files/newimg.png",new_img);
 int blur_amount = faceLine[41].p1.x() - faceLine[47].p1.x();
 int blur_mask = blur_amount;
     if (blur_mask % 2 == 0) {
     blur_mask += 1;


     }
     LOGD("blur_mask%d-------",blur_mask);
 GaussianBlur(new_img,mask,Size(blur_mask,blur_mask),0,0);


 GaussianBlur(mask,mask,Size(blur_mask,blur_mask),0,0);
 // FileStorage fs("/data/data/com.example.faceswapdemo/files/face.xml",FileStorage::WRITE);
 // fs << "mask" << mask;
 blur_amount = blur_amount*3/5;
     if (blur_amount % 2 == 0) {
         blur_amount += 1;
     }
     LOGD("blur_amount%d-------",blur_amount);
     Mat blur0 = Mat(wrap_dst.size(), wrap_dst.type());
     GaussianBlur(wrap_dst, blur0, Size( blur_amount, blur_amount ), 0, 0 );
     Mat blur1= Mat(src_img1.size(), src_img1.type());
     GaussianBlur(src_img1, blur1, Size( blur_amount, blur_amount ), 0, 0 );
     Mat out = Mat::zeros( blur1.size(), blur1.type());
 for (int y = 0; y < blur1.rows; y++) {
 for (int x = 0; x < blur1.cols; x++) {
 for (int c = 0; c < 3; c++) {
 if ((blur0.at<Vec3b>(y, x)[c]) > 0) {


 out.at<Vec3b>(y, x)[c] = saturate_cast<uchar>(
 (wrap_dst.at<Vec3b>(y, x)[c])
 * (blur1.at<Vec3b>(y, x)[c])
 / (blur0.at<Vec3b>(y, x)[c]));
 } else {
 out.at<Vec3b>(y, x)[c] = saturate_cast<uchar>(
 (wrap_dst.at<Vec3b>(y, x)[c])
 * (blur1.at<Vec3b>(y, x)[c]));
 }
 }
 }
 }




     for(int y = 0; y < src_img1.rows; y++ )
         {
             for(int x = 0; x < src_img1.cols; x++ )
             {
             if((mask.at<Vec3b>(y,x)[0]) > 0) {
                     for(int c = 0; c < 3; c++ )
                     {
                     src_img1.at<Vec3b>(y,x)[c]= (out.at<Vec3b>(y,x)[c]);
                     }
             }


             }
         }
 //    src_img1 += 128 * (src_img1 <= 1.0);
 // for(int i = 0; i < faceLine1.size(); i++) {
 // //mark the all face data
 // cv::Point p1(faceLine1[i].p1.x(),faceLine[i].p1.y());
 // cv::Point p2(faceLine1[i].p2.x(),faceLine[i].p2.y());
 // cv::line(src_img,p1,p2,cv::Scalar(0,255,0),1,cv::LINE_8);
 // }
 //
 // for(int i = 0; i < faceLine2.size(); i++) {
 // //mark the all face data
 // cv::Point p1(faceLine2[i].p1.x(),faceLine[i].p1.y());
 // cv::Point p2(faceLine2[i].p2.x(),faceLine[i].p2.y());
 // cv::line(src_img1,p1,p2,cv::Scalar(0,255,0),1,cv::LINE_8);
 // }
 //Size dsize = Size(src_img1.cols,src_img1.rows);
 //Mat image2 = Mat(dsize,wrap_dst.type());
 //LOGD("img1 w:%d h:%d",src_img1.cols,src_img1.rows);
 //resize(wrap_dst, image2,dsize);
 //LOGD("dst w:%d h:%d",image2.cols,image2.rows);
 imwrite("/data/data/com.example.faceswapdemo/files/data.jpg",src_img1);
 env->ReleaseIntArrayElements(face1, cbuf1, 0);
 env->ReleaseIntArrayElements(face2, cbuf2, 0);
 return JNI_TRUE;
 }






 jboolean studytrain()
 {
 dlib::deserialize("/data/data/com.example.faceswapdemo/files/shape_predictor_68_face_landmarks.dat") >> msp;
 detector = get_frontal_face_detector();
 return JNI_TRUE;
 }


 jboolean facecheck(JNIEnv* env, jobject thiz,
 jintArray face1,jint w1,jint h1,
 jintArray face2,jint w2,jint h2)
 {


 return facesRotate_UseData(env,thiz,face1,w1,h1,face2,w2,h2);
 }


 jboolean removefaceswap()
 {
 return JNI_TRUE;
 }


 void startfaceswap()
 {


 }


 void endfaceswap()
 {


 }


 void facemapping()
 {


 }


 void facefusion()
 {


 }


 void faceswap()
 {


 }


 void setfaceswapcallback()
 {


 }






 /*jintArray facesRotate_UsePath(JNIEnv* env, jobject thiz,jstring path1,jstring path2)
 {


 LOGD("in det");
 frontal_face_detector detector = get_frontal_face_detector();
 char * str1;
 str1=(char*)env->GetStringUTFChars(path1,NULL);
 char * str2;
 str2=(char*)env->GetStringUTFChars(path2,NULL);
 LOGD("load picture");
 Mat src_img = imread(str1,1);
 Mat src_img1 = imread(str2,1);
 LOGD("%d-----img1-----%d",src_img.cols,src_img.rows);
 LOGD("%d-----img1-----%d",src_img1.cols,src_img1.rows);
 LOGD("opencv img to dlib img");
 if(!src_img.data || !src_img1.data)
 LOGD("data error");
 dlib::cv_image<dlib::bgr_pixel> img(src_img);
 dlib::cv_image<dlib::bgr_pixel> img1(src_img1);


 LOGD("start detector");
     std::vector<dlib::rectangle> mRets;
     std::vector<dlib::rectangle> mRets1;
     double t = (double)getTickCount();
 mRets = detector(img);
 mRets1 = detector(img1);
 t = ((double)getTickCount()-t)/getTickFrequency();
 dlib::shape_predictor msp;
 LOGD("detect use %lf s",t);


 t = (double)getTickCount();
 dlib::deserialize("/data/data/com.example.faceswapdemo/files/shape_predictor_68_face_landmarks.dat") >> msp;
 t = ((double)getTickCount()-t)/getTickFrequency();
 LOGD("load local mark use %lf s",t);


 LOGD("get face mark from the picture");
 t = (double)getTickCount();
 dlib::full_object_detection shape1,shape2;
 if (mRets.size() != 0) {
 for (unsigned long j = 0; j < mRets.size(); ++j)
 shape1 = msp(img, mRets[j]);
 }
 if (mRets1.size() != 0) {
 for (unsigned long j = 0; j < mRets1.size(); ++j)
 shape2 = msp(img1, mRets1[j]);
 }
 t = ((double)getTickCount()-t)/getTickFrequency();
 LOGD("--get face mark use %lf s---",t);


 LOGD("--use face mark to get mapping matrix---");
 t = (double)getTickCount();
 std::vector<cv::Point2f> srcTri;
 std::vector<cv::Point2f> dstTri;
 for(int i = 0;i < 16;i++) {
 srcTri.push_back(cv::Point2f(shape1.part(i).x(),shape1.part(i).y()));
 dstTri.push_back(cv::Point2f(shape2.part(i).x(),shape2.part(i).y()));
 }
 LOGD("--got special points---");
 cv::Mat warp_mat;
 warp_mat = cv::findHomography(srcTri,dstTri,0);
 LOGD("%d---get change matrix-----%d",warp_mat.cols,warp_mat.rows);
 uchar* p1;
 for(int i = 0;i < 3;i ++) {
    p1 = warp_mat.ptr<uchar>(i);
    for(int j = 0;j < 3;j ++)
     LOGD("matrix-----%d",(int)p1[j]);
     }
 //the faceLine's size is 68,including all face data,such as:
 //the index 0 - 16:jaw data
 //the index 17 - 68:face data
 //the index 17 - 21:right brow data
 //the index 22 - 26:left brow data
 //the index 27 - 34:nose data
 //the index 36 - 41:right eye data
 //the index 42 - 47:left eye data
 //the index 48 - 60:mouth data


 LOGD("use the mapping matrix to change the picture");
 Mat wrap_dst;
 cv::warpPerspective(src_img,wrap_dst,warp_mat,src_img.size());
 t = ((double)getTickCount()-t)/getTickFrequency();
 LOGD("change picture use %lf s",t);
 int size = wrap_dst.cols * wrap_dst.rows;
 jintArray result = env->NewIntArray(size);
 uchar* p;
 p = wrap_dst.ptr(0);
 cv::imwrite("/data/data/com.example.faceswapdemo/files/face_wrap1.png", wrap_dst);
 env->SetIntArrayRegion(result, 0, size, (int*)p);
 env->ReleaseStringUTFChars(path1, str1);
 env->ReleaseStringUTFChars(path2, str2);
 return result;
 }*/


 void testmat(jlong addr1,jlong addr2)
 {
 Mat& src = *(Mat*)addr1;
 Mat& src1 = *(Mat*)addr2;
 // Mat src1 = Mat(*((Mat*)addr2));
 cvtColor(src1,src1,CV_RGBA2RGB);
 cvtColor(src,src,CV_BGR2GRAY);
 }




 static JNINativeMethod gMethods[] = {
     {"facecheck", "([III[III)Z", (void*)facecheck},
     {"studytrain","()Z",(void*) studytrain},
 //    {"faceRotateUsePath", "(Ljava/lang/String;Ljava/lang/String;)[I", (void*)facesRotate_UsePath},
     //    {"testMat","(JJ)V",(void*) testmat},
     // {"faceRotate", "()I", (void*)facesRotate},
 };


 /*
 * 为某一个类注册本地方法
 */
 static int registerNativeMethods(JNIEnv* env
         , const char* className
         , JNINativeMethod* gMethods, int numMethods) {
     jclass clazz;
     clazz = env->FindClass(className);
     if (clazz == NULL) {
         return JNI_FALSE;
     }
     if (env->RegisterNatives(clazz, gMethods, numMethods) < 0) {
         return JNI_FALSE;
     }


     return JNI_TRUE;
 }


 /*
 * 为所有类注册本地方法
 */
 static int registerNatives(JNIEnv* env) {
     const char* kClassName = "com/example/faceswapdemo/FaceUtils";//指定要注册的类
     return registerNativeMethods(env, kClassName, gMethods,
     sizeof(gMethods)/sizeof(gMethods[0]));
 }


 jint JNIEXPORT JNICALL JNI_OnLoad(JavaVM* vm, void* reserved)
 {
     LOGE("JNI On Load");
     JNIEnv* env = NULL;
     jint result = -1;


     if (vm->GetEnv((void**)&env, JNI_VERSION_1_4) != JNI_OK) {
         LOGE("GetEnv failed!");
         return result;
     }
     LOGE("start register!");
     if (!registerNatives(env)) {//注册
     LOGE("register failed!");
         return result;
     }
     return JNI_VERSION_1_4;
 }


 /*
 #ifdef __cplusplus
 }
 #endif
 */