Merge pull request #72 from HyperInspire/dev/eye_status

Dev/eye status

CMakeLists.txt

@@ -9,7 +9,7 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3")
 # Current version
 set(INSPIRE_FACE_VERSION_MAJOR 1)
 set(INSPIRE_FACE_VERSION_MINOR 1)
-set(INSPIRE_FACE_VERSION_PATCH 2)
+set(INSPIRE_FACE_VERSION_PATCH 3)
 
 # Converts the version number to a string
 string(CONCAT INSPIRE_FACE_VERSION_MAJOR_STR ${INSPIRE_FACE_VERSION_MAJOR})

cpp/inspireface/CMakeLists.txt

@@ -55,6 +55,7 @@ set(SOURCE_FILES ${SOURCE_FILES} ${CMAKE_CURRENT_SOURCE_DIR}/middleware/model_ar
 link_directories(${MNN_LIBS})
 
 if(ISF_BUILD_SHARED_LIBS)
+    add_definitions("-DISF_BUILD_SHARED_LIBS")
     add_library(InspireFace SHARED ${SOURCE_FILES})
 else()
     add_library(InspireFace STATIC ${SOURCE_FILES})

cpp/inspireface/c_api/inspireface.cc

@@ -281,6 +281,33 @@ HResult HFGetFaceBasicTokenSize(HPInt32 bufferSize) {
     return HSUCCEED;
 }
 
+HResult HFGetNumOfFaceDenseLandmark(HPInt32 num) {
+    *num = 106;
+    return HSUCCEED;
+}
+
+HResult HFGetFaceDenseLandmarkFromFaceToken(HFFaceBasicToken singleFace, HPoint2f* landmarks, HInt32 num) {
+    if (num != 106) {
+        return HERR_SESS_LANDMARK_NUM_NOT_MATCH;
+    }
+    inspire::FaceBasicData data;
+    data.dataSize = singleFace.size;
+    data.data = singleFace.data;
+    HyperFaceData face = {0};
+    HInt32 ret;
+    ret = DeserializeHyperFaceData((char* )data.data, data.dataSize, face);
+    if (ret != HSUCCEED) {
+        return ret;
+    }
+    for (size_t i = 0; i < num; i++)
+    {
+        landmarks[i].x = face.densityLandmark[i].x;
+        landmarks[i].y = face.densityLandmark[i].y;
+    }
+
+    return HSUCCEED;
+}
+
 HResult HFFeatureHubFaceSearchThresholdSetting(float threshold) {
     FEATURE_HUB->SetRecognitionThreshold(threshold);
     return HSUCCEED;
@@ -549,7 +576,7 @@ HResult HFMultipleFacePipelineProcessOptional(HFSession session, HFImageStream s
     }
     if (customOption & HF_ENABLE_INTERACTION) {
         param.enable_interaction_liveness = true;
-    }
+    } 
 
 
     HResult ret;
@@ -633,6 +660,21 @@ HResult HFFaceQualityDetect(HFSession session, HFFaceBasicToken singleFace, HFlo
 
 }
 
+HResult HFGetFaceIntereactionResult(HFSession session, PHFFaceIntereactionResult result) {
+     if (session == nullptr) {
+        return HERR_INVALID_CONTEXT_HANDLE;
+    }
+    HF_FaceAlgorithmSession *ctx = (HF_FaceAlgorithmSession* ) session;
+    if (ctx == nullptr) {
+        return HERR_INVALID_CONTEXT_HANDLE;
+    }
+    result->num = ctx->impl.GetFaceInteractionLeftEyeStatusCache().size();
+    result->leftEyeStatusConfidence = (HFloat* )ctx->impl.GetFaceInteractionLeftEyeStatusCache().data();
+    result->rightEyeStatusConfidence = (HFloat* )ctx->impl.GetFaceInteractionRightEyeStatusCache().data();
+
+    return HSUCCEED;
+}
+
 HResult HFFeatureHubGetFaceCount(HInt32* count) {
     *count = FEATURE_HUB->GetFaceFeatureCount();
     return HSUCCEED;

cpp/inspireface/c_api/inspireface.h

@@ -10,7 +10,7 @@
 #include "herror.h"
 
 #if defined(_WIN32)
-#ifdef HYPER_BUILD_SHARED_LIB
+#ifdef ISF_BUILD_SHARED_LIBS
 #define HYPER_CAPI_EXPORT __declspec(dllexport)
 #else
 #define HYPER_CAPI_EXPORT
@@ -298,6 +298,23 @@ HYPER_CAPI_EXPORT extern HResult HFCopyFaceBasicToken(HFFaceBasicToken token, HP
  */
 HYPER_CAPI_EXPORT extern HResult HFGetFaceBasicTokenSize(HPInt32 bufferSize);
 
+/**
+ * @brief Retrieve the number of dense facial landmarks.
+ * @param num Number of dense facial landmarks
+ * @return HResult indicating the success or failure of the operation.
+ */
+HYPER_CAPI_EXPORT extern HResult HFGetNumOfFaceDenseLandmark(HPInt32 num);
+
+/**
+ * @brief When you pass in a valid facial token, you can retrieve a set of dense facial landmarks. 
+ *          The memory for the dense landmarks must be allocated by you.
+ * @param singleFace Basic token representing a single face.
+ * @param landmarks Pre-allocated memory address of the array for 2D floating-point coordinates.
+ * @param num Number of landmark points
+ * @return HResult indicating the success or failure of the operation.
+ */
+HYPER_CAPI_EXPORT extern HResult HFGetFaceDenseLandmarkFromFaceToken(HFFaceBasicToken singleFace, HPoint2f* landmarks, HInt32 num);
+
 /************************************************************************
 * Face Recognition
 ************************************************************************/
@@ -618,6 +635,18 @@ HYPER_CAPI_EXPORT extern HResult HFGetFaceQualityConfidence(HFSession session, P
  */
 HYPER_CAPI_EXPORT extern HResult HFFaceQualityDetect(HFSession session, HFFaceBasicToken singleFace, HFloat *confidence);
 
+
+/**
+ * @brief Some facial states in the face interaction module.
+ */
+typedef struct HFFaceIntereactionResult {
+    HInt32 num;                             ///< Number of faces detected.
+    HPFloat leftEyeStatusConfidence;        ///< Left eye state: confidence close to 1 means open, close to 0 means closed.
+    HPFloat rightEyeStatusConfidence;       ///< Right eye state: confidence close to 1 means open, close to 0 means closed.
+} HFFaceIntereactionResult, *PHFFaceIntereactionResult;
+
+HYPER_CAPI_EXPORT extern HResult HFGetFaceIntereactionResult(HFSession session, PHFFaceIntereactionResult result);
+
 /************************************************************************
 * System Function
 ************************************************************************/

cpp/inspireface/c_api/intypedef.h

@@ -31,5 +31,9 @@ typedef struct HFaceRect {
     HInt32 height;        ///< Height of the rectangle.
 } HFaceRect;         ///< Rectangle representing a face region.
 
+typedef struct HPoint2f{
+    HFloat x;          ///< X-coordinate
+    HFloat y;          ///< Y-coordinate
+} HPoint2f;
 
 #endif //HYPERFACEREPO_INTYPEDEF_H

cpp/inspireface/common/face_data/data_tools.h

@@ -96,6 +96,15 @@ inline HyperFaceData INSPIRE_API FaceObjectToHyperFaceData(const FaceObject& obj
     data.face3DAngle.pitch = obj.high_result.pitch;
     data.face3DAngle.roll = obj.high_result.roll;
     data.face3DAngle.yaw = obj.high_result.yaw;
+
+
+    const auto &lmk = obj.landmark_smooth_aux_.back();
+    for (size_t i = 0; i < lmk.size(); i++)
+    {
+        data.densityLandmark[i].x = lmk[i].x;
+        data.densityLandmark[i].y = lmk[i].y;
+    }
+
 
     return data;
 }

cpp/inspireface/common/face_data/face_data_type.h

@@ -57,15 +57,16 @@ typedef struct TransMatrix {
 * Struct to represent hyper face data.
 */
 typedef struct HyperFaceData {
-    int trackState;         ///< Track state
-    int inGroupIndex;       ///< Index within a group
-    int trackId;            ///< Track ID
-    int trackCount;         ///< Track count
-    FaceRect rect;          ///< Face rectangle
-    TransMatrix trans;      ///< Transformation matrix
-    Point2F keyPoints[5];   ///< Key points (e.g., landmarks)
-    Face3DAngle face3DAngle; ///< 3D face angles
-    float quality[5];       ///< Quality values for key points
+    int trackState;                 ///< Track state
+    int inGroupIndex;               ///< Index within a group
+    int trackId;                    ///< Track ID
+    int trackCount;                 ///< Track count
+    FaceRect rect;                  ///< Face rectangle
+    TransMatrix trans;              ///< Transformation matrix
+    Point2F keyPoints[5];           ///< Key points (e.g., landmarks)
+    Face3DAngle face3DAngle;        ///< 3D face angles
+    float quality[5];               ///< Quality values for key points
+    Point2F densityLandmark[106];   ///< Face density landmark
 } HyperFaceData;
 
 } // namespace inspire

cpp/inspireface/common/face_info/face_object.h

@@ -312,6 +312,10 @@ class INSPIRE_API FaceObject {
         face_id_ = id;
     }
 
+    std::vector<float> left_eye_status_;
+
+    std::vector<float> right_eye_status_;
+
 private:
     TRACK_STATE tracking_state_;
 //  std::shared_ptr<FaceAction> face_action_;

cpp/inspireface/face_context.cpp

@@ -62,6 +62,8 @@ int32_t FaceContext::FaceDetectAndTrack(CameraStream &image) {
     m_yaw_results_cache_.clear();
     m_pitch_results_cache_.clear();
     m_quality_score_results_cache_.clear();
+    m_react_left_eye_results_cache_.clear();
+    m_react_right_eye_results_cache_.clear();
     if (m_face_track_ == nullptr) {
         return HERR_SESS_TRACKER_FAILURE;
     }
@@ -129,6 +131,8 @@ int32_t FaceContext::FacesProcess(CameraStream &image, const std::vector<HyperFa
     std::lock_guard<std::mutex> lock(m_mtx_);
     m_mask_results_cache_.resize(faces.size(), -1.0f);
     m_rgb_liveness_results_cache_.resize(faces.size(), -1.0f);
+    m_react_left_eye_results_cache_.resize(faces.size(), -1.0f);
+    m_react_right_eye_results_cache_.resize(faces.size(), -1.0f);
     for (int i = 0; i < faces.size(); ++i) {
         const auto &face = faces[i];
         // RGB Liveness Detect
@@ -161,6 +165,38 @@ int32_t FaceContext::FacesProcess(CameraStream &image, const std::vector<HyperFa
                 return ret;
             }
         }
+        // Face interaction
+        if (param.enable_interaction_liveness) {
+            auto ret = m_face_pipeline_->Process(image, face, PROCESS_INTERACTION);
+            if (ret != HSUCCEED) {
+                return ret;
+            }
+            // Get eyes status
+            m_react_left_eye_results_cache_[i] = m_face_pipeline_->eyesStatusCache[0];
+            m_react_right_eye_results_cache_[i] = m_face_pipeline_->eyesStatusCache[1];
+            // Special handling:  ff it is a tracking state, it needs to be filtered
+            if (face.trackState > 0)
+            {   
+                auto idx = face.inGroupIndex;
+                if (idx < m_face_track_->trackingFace.size()) {
+                    auto& target = m_face_track_->trackingFace[idx];
+                    if (target.GetTrackingId() == face.trackId) {
+                        auto new_eye_left = EmaFilter(m_face_pipeline_->eyesStatusCache[0], target.left_eye_status_, 8, 0.2f);
+                        auto new_eye_right = EmaFilter(m_face_pipeline_->eyesStatusCache[1], target.right_eye_status_, 8, 0.2f);
+                        if (face.trackState > 1) {
+                            // The filtered value can be obtained only in the tracking state
+                            m_react_left_eye_results_cache_[i] = new_eye_left;
+                            m_react_right_eye_results_cache_[i] = new_eye_right;
+                        }
+
+                    } else {
+                        INSPIRE_LOGD("Serialized objects cannot connect to trace objects in memory, and there may be some problems");
+                    }
+                } else {
+                    INSPIRE_LOGW("The index of the trace object does not match the trace list in memory, and there may be some problems");
+                }
+            }
+        }
 
     }
 
@@ -212,6 +248,13 @@ const std::vector<float>& FaceContext::GetFaceQualityScoresResultsCache() const
     return m_quality_score_results_cache_;
 }
 
+const std::vector<float>& FaceContext::GetFaceInteractionLeftEyeStatusCache() const {
+    return m_react_left_eye_results_cache_;
+}
+
+const std::vector<float>& FaceContext::GetFaceInteractionRightEyeStatusCache() const {
+    return m_react_right_eye_results_cache_;
+}
 
 const Embedded& FaceContext::GetFaceFeatureCache() const {
     return m_face_feature_cache_;

cpp/inspireface/face_context.h

@@ -232,6 +232,18 @@ class INSPIRE_API FaceContext {
      */
     const std::vector<float>& GetFaceQualityScoresResultsCache() const;
 
+    /**
+     * @brief Gets the cache of left eye status predict results.
+     * @return A const reference to a vector containing eye status predict results.
+     */
+    const std::vector<float>& GetFaceInteractionLeftEyeStatusCache() const;
+
+    /**
+     * @brief Gets the cache of right eye status predict results.
+     * @return A const reference to a vector containing eye status predict results.
+     */
+    const std::vector<float>& GetFaceInteractionRightEyeStatusCache() const;
+
     /**
      * @brief Gets the cache of the current face features.
      * @return A const reference to the Embedded object containing current face feature data.
@@ -263,6 +275,8 @@ class INSPIRE_API FaceContext {
     std::vector<float> m_mask_results_cache_;                       ///< Cache for mask detection results
     std::vector<float> m_rgb_liveness_results_cache_;               ///< Cache for RGB liveness detection results
     std::vector<float> m_quality_score_results_cache_;               ///< Cache for RGB face quality score results
+    std::vector<float> m_react_left_eye_results_cache_;               ///< Cache for Left eye state in face interaction
+    std::vector<float> m_react_right_eye_results_cache_;               ///< Cache for Right eye state in face interaction
     Embedded m_face_feature_cache_;                                ///< Cache for current face feature data
 
     std::mutex m_mtx_;                                             ///< Mutex for thread safety.

cpp/inspireface/herror.h

@@ -27,6 +27,7 @@
 #define HERR_SESS_TRACKER_FAILURE             (HERR_SESS_BASE+3)               // Tracker module not initialized
 #define HERR_SESS_INVALID_RESOURCE            (HERR_SESS_BASE+10)              // Invalid static resource
 #define HERR_SESS_NUM_OF_MODELS_NOT_MATCH     (HERR_SESS_BASE+11)              // Number of models does not match
+#define HERR_SESS_LANDMARK_NUM_NOT_MATCH      (HERR_SESS_BASE+20)              // The number of input landmark points does not match
 
 #define HERR_SESS_PIPELINE_FAILURE            (HERR_SESS_BASE+8)               // Pipeline module not initialized
 

cpp/inspireface/information.h

@@ -7,6 +7,6 @@
 
 #define INSPIRE_FACE_VERSION_MAJOR_STR "1"
 #define INSPIRE_FACE_VERSION_MINOR_STR "1"
-#define INSPIRE_FACE_VERSION_PATCH_STR "2"
+#define INSPIRE_FACE_VERSION_PATCH_STR "3"
 
 #endif //HYPERFACEREPO_INFORMATION_H

cpp/inspireface/middleware/inference_helper/inference_helper_mnn.cpp

@@ -249,6 +249,15 @@ int32_t InferenceHelperMnn::PreProcess(const std::vector<InputTensorInfo>& input
             /* Convert color type */
 //            LOGD("input_tensor_info.image_info.channel: %d", input_tensor_info.image_info.channel);
 //            LOGD("input_tensor_info.GetChannel(): %d", input_tensor_info.GetChannel());
+
+            // !!!!!! BUG !!!!!!!!!
+            // When initializing, setting the image channel to 3 and the tensor channel to 1, 
+            // and configuring the processing to convert the color image to grayscale may cause some bugs. 
+            // For example, the image channel might automatically change to 1. 
+            // This issue has not been fully investigated, 
+            // so it's necessary to manually convert the image to grayscale before input.
+            // !!!!!! BUG !!!!!!!!!
+
             if ((input_tensor_info.image_info.channel == 3) && (input_tensor_info.GetChannel() == 3)) {
                 image_processconfig.sourceFormat = (input_tensor_info.image_info.is_bgr) ? MNN::CV::BGR : MNN::CV::RGB;
                 if (input_tensor_info.image_info.swap_color) {

cpp/inspireface/middleware/utils.h

@@ -481,11 +481,9 @@ inline cv::Mat ScaleAffineMatrix(const cv::Mat &affine, float scale,
     return m;
 }
 
-template<typename T>
-inline int ArgMax(const std::vector<T> data, int start, int end) {
-    int diff = std::max_element(data.begin() + start, data.begin() + end) -
-               (data.begin() + start);
-    return diff;
+template<class ForwardIterator>
+inline size_t argmax(ForwardIterator first, ForwardIterator last) {
+    return std::distance(first, std::max_element(first, last));
 }
 
 inline void RotPoints(std::vector<cv::Point2f> &pts, float angle) {
@@ -650,6 +648,52 @@ inline bool isShortestSideGreaterThan(const cv::Rect_<T>& rect, T value, float s
     return shortestSide > value;
 }
 
+/**
+ * @brief Computes the affine transformation matrix for face cropping.
+ * @param rect Rectangle representing the face in the image.
+ * @return cv::Mat The computed affine transformation matrix.
+ */
+inline cv::Mat ComputeCropMatrix(const cv::Rect2f &rect, int width, int height) {
+    float x = rect.x;
+    float y = rect.y;
+    float w = rect.width;
+    float h = rect.height;
+    float cx = x + w / 2;
+    float cy = y + h / 2;
+    float length = std::max(w, h) * 1.5 / 2;
+    float x1 = cx - length;
+    float y1 = cy - length;
+    float x2 = cx + length;
+    float y2 = cy + length;
+    cv::Rect2f padding_rect(x1, y1, x2 - x1, y2 - y1);
+    std::vector<cv::Point2f> rect_pts = Rect2Points(padding_rect);
+    rect_pts.erase(rect_pts.end() - 1);
+    std::vector<cv::Point2f> dst_pts = {{0, 0}, {(float )width, 0}, {(float )width, (float )height}};
+    cv::Mat m = cv::getAffineTransform(rect_pts, dst_pts);
+
+    return m;
+}
+
+
+// Exponential Moving Average (EMA) filter function
+inline float EmaFilter(float currentProb, std::vector<float>& history, int max, float alpha = 0.2f) {
+    // Add current probability to history
+    history.push_back(currentProb);
+
+    // Trim history if it exceeds max size
+    if (history.size() > max) {
+        history.erase(history.begin(), history.begin() + (history.size() - max));
+    }
+
+    // Compute EMA
+    float ema = history[0];  // Initial value
+    for (size_t i = 1; i < history.size(); ++i) {
+        ema = alpha * history[i] + (1 - alpha) * ema;
+    }
+
+    return ema;
+}
+
 }   // namespace inspire
 
 #endif