xrt: implements xrt/ipc plumbing for XR_ANDROID_face_tracking

+103

src/xrt/include/xrt/xrt_defines.h

··· 1181 1181 _(XRT_INPUT_HTC_LIP_FACE_TRACKING , XRT_INPUT_NAME(0x0602, FACE_TRACKING)) \ 1182 1182 _(XRT_INPUT_FB_FACE_TRACKING2_AUDIO , XRT_INPUT_NAME(0x0603, FACE_TRACKING)) \ 1183 1183 _(XRT_INPUT_FB_FACE_TRACKING2_VISUAL , XRT_INPUT_NAME(0x0604, FACE_TRACKING)) \ 1184 + _(XRT_INPUT_ANDROID_FACE_TRACKING , XRT_INPUT_NAME(0x0605, FACE_TRACKING)) \ 1184 1185 \ 1185 1186 _(XRT_INPUT_GENERIC_BODY_TRACKING , XRT_INPUT_NAME(0x0700, BODY_TRACKING)) \ 1186 1187 _(XRT_INPUT_FB_BODY_TRACKING , XRT_INPUT_NAME(0x0701, BODY_TRACKING)) \ ··· 1764 1765 #define XRT_FACIAL_EXPRESSION_EYE_COUNT_HTC 14 1765 1766 #define XRT_FACIAL_EXPRESSION_LIP_COUNT_HTC 37 1766 1767 1768 + enum xrt_face_confidence_regions_android 1769 + { 1770 + XRT_FACE_CONFIDENCE_REGIONS_LOWER_ANDROID = 0, 1771 + XRT_FACE_CONFIDENCE_REGIONS_LEFT_UPPER_ANDROID = 1, 1772 + XRT_FACE_CONFIDENCE_REGIONS_RIGHT_UPPER_ANDROID = 2, 1773 + }; 1774 + 1775 + enum xrt_face_parameter_indices_android 1776 + { 1777 + XRT_FACE_PARAMETER_INDICES_BROW_LOWERER_L_ANDROID = 0, 1778 + XRT_FACE_PARAMETER_INDICES_BROW_LOWERER_R_ANDROID = 1, 1779 + XRT_FACE_PARAMETER_INDICES_CHEEK_PUFF_L_ANDROID = 2, 1780 + XRT_FACE_PARAMETER_INDICES_CHEEK_PUFF_R_ANDROID = 3, 1781 + XRT_FACE_PARAMETER_INDICES_CHEEK_RAISER_L_ANDROID = 4, 1782 + XRT_FACE_PARAMETER_INDICES_CHEEK_RAISER_R_ANDROID = 5, 1783 + XRT_FACE_PARAMETER_INDICES_CHEEK_SUCK_L_ANDROID = 6, 1784 + XRT_FACE_PARAMETER_INDICES_CHEEK_SUCK_R_ANDROID = 7, 1785 + XRT_FACE_PARAMETER_INDICES_CHIN_RAISER_B_ANDROID = 8, 1786 + XRT_FACE_PARAMETER_INDICES_CHIN_RAISER_T_ANDROID = 9, 1787 + XRT_FACE_PARAMETER_INDICES_DIMPLER_L_ANDROID = 10, 1788 + XRT_FACE_PARAMETER_INDICES_DIMPLER_R_ANDROID = 11, 1789 + XRT_FACE_PARAMETER_INDICES_EYES_CLOSED_L_ANDROID = 12, 1790 + XRT_FACE_PARAMETER_INDICES_EYES_CLOSED_R_ANDROID = 13, 1791 + XRT_FACE_PARAMETER_INDICES_EYES_LOOK_DOWN_L_ANDROID = 14, 1792 + XRT_FACE_PARAMETER_INDICES_EYES_LOOK_DOWN_R_ANDROID = 15, 1793 + XRT_FACE_PARAMETER_INDICES_EYES_LOOK_LEFT_L_ANDROID = 16, 1794 + XRT_FACE_PARAMETER_INDICES_EYES_LOOK_LEFT_R_ANDROID = 17, 1795 + XRT_FACE_PARAMETER_INDICES_EYES_LOOK_RIGHT_L_ANDROID = 18, 1796 + XRT_FACE_PARAMETER_INDICES_EYES_LOOK_RIGHT_R_ANDROID = 19, 1797 + XRT_FACE_PARAMETER_INDICES_EYES_LOOK_UP_L_ANDROID = 20, 1798 + XRT_FACE_PARAMETER_INDICES_EYES_LOOK_UP_R_ANDROID = 21, 1799 + XRT_FACE_PARAMETER_INDICES_INNER_BROW_RAISER_L_ANDROID = 22, 1800 + XRT_FACE_PARAMETER_INDICES_INNER_BROW_RAISER_R_ANDROID = 23, 1801 + XRT_FACE_PARAMETER_INDICES_JAW_DROP_ANDROID = 24, 1802 + XRT_FACE_PARAMETER_INDICES_JAW_SIDEWAYS_LEFT_ANDROID = 25, 1803 + XRT_FACE_PARAMETER_INDICES_JAW_SIDEWAYS_RIGHT_ANDROID = 26, 1804 + XRT_FACE_PARAMETER_INDICES_JAW_THRUST_ANDROID = 27, 1805 + XRT_FACE_PARAMETER_INDICES_LID_TIGHTENER_L_ANDROID = 28, 1806 + XRT_FACE_PARAMETER_INDICES_LID_TIGHTENER_R_ANDROID = 29, 1807 + XRT_FACE_PARAMETER_INDICES_LIP_CORNER_DEPRESSOR_L_ANDROID = 30, 1808 + XRT_FACE_PARAMETER_INDICES_LIP_CORNER_DEPRESSOR_R_ANDROID = 31, 1809 + XRT_FACE_PARAMETER_INDICES_LIP_CORNER_PULLER_L_ANDROID = 32, 1810 + XRT_FACE_PARAMETER_INDICES_LIP_CORNER_PULLER_R_ANDROID = 33, 1811 + XRT_FACE_PARAMETER_INDICES_LIP_FUNNELER_LB_ANDROID = 34, 1812 + XRT_FACE_PARAMETER_INDICES_LIP_FUNNELER_LT_ANDROID = 35, 1813 + XRT_FACE_PARAMETER_INDICES_LIP_FUNNELER_RB_ANDROID = 36, 1814 + XRT_FACE_PARAMETER_INDICES_LIP_FUNNELER_RT_ANDROID = 37, 1815 + XRT_FACE_PARAMETER_INDICES_LIP_PRESSOR_L_ANDROID = 38, 1816 + XRT_FACE_PARAMETER_INDICES_LIP_PRESSOR_R_ANDROID = 39, 1817 + XRT_FACE_PARAMETER_INDICES_LIP_PUCKER_L_ANDROID = 40, 1818 + XRT_FACE_PARAMETER_INDICES_LIP_PUCKER_R_ANDROID = 41, 1819 + XRT_FACE_PARAMETER_INDICES_LIP_STRETCHER_L_ANDROID = 42, 1820 + XRT_FACE_PARAMETER_INDICES_LIP_STRETCHER_R_ANDROID = 43, 1821 + XRT_FACE_PARAMETER_INDICES_LIP_SUCK_LB_ANDROID = 44, 1822 + XRT_FACE_PARAMETER_INDICES_LIP_SUCK_LT_ANDROID = 45, 1823 + XRT_FACE_PARAMETER_INDICES_LIP_SUCK_RB_ANDROID = 46, 1824 + XRT_FACE_PARAMETER_INDICES_LIP_SUCK_RT_ANDROID = 47, 1825 + XRT_FACE_PARAMETER_INDICES_LIP_TIGHTENER_L_ANDROID = 48, 1826 + XRT_FACE_PARAMETER_INDICES_LIP_TIGHTENER_R_ANDROID = 49, 1827 + XRT_FACE_PARAMETER_INDICES_LIPS_TOWARD_ANDROID = 50, 1828 + XRT_FACE_PARAMETER_INDICES_LOWER_LIP_DEPRESSOR_L_ANDROID = 51, 1829 + XRT_FACE_PARAMETER_INDICES_LOWER_LIP_DEPRESSOR_R_ANDROID = 52, 1830 + XRT_FACE_PARAMETER_INDICES_MOUTH_LEFT_ANDROID = 53, 1831 + XRT_FACE_PARAMETER_INDICES_MOUTH_RIGHT_ANDROID = 54, 1832 + XRT_FACE_PARAMETER_INDICES_NOSE_WRINKLER_L_ANDROID = 55, 1833 + XRT_FACE_PARAMETER_INDICES_NOSE_WRINKLER_R_ANDROID = 56, 1834 + XRT_FACE_PARAMETER_INDICES_OUTER_BROW_RAISER_L_ANDROID = 57, 1835 + XRT_FACE_PARAMETER_INDICES_OUTER_BROW_RAISER_R_ANDROID = 58, 1836 + XRT_FACE_PARAMETER_INDICES_UPPER_LID_RAISER_L_ANDROID = 59, 1837 + XRT_FACE_PARAMETER_INDICES_UPPER_LID_RAISER_R_ANDROID = 60, 1838 + XRT_FACE_PARAMETER_INDICES_UPPER_LIP_RAISER_L_ANDROID = 61, 1839 + XRT_FACE_PARAMETER_INDICES_UPPER_LIP_RAISER_R_ANDROID = 62, 1840 + XRT_FACE_PARAMETER_INDICES_TONGUE_OUT_ANDROID = 63, 1841 + XRT_FACE_PARAMETER_INDICES_TONGUE_LEFT_ANDROID = 64, 1842 + XRT_FACE_PARAMETER_INDICES_TONGUE_RIGHT_ANDROID = 65, 1843 + XRT_FACE_PARAMETER_INDICES_TONGUE_UP_ANDROID = 66, 1844 + XRT_FACE_PARAMETER_INDICES_TONGUE_DOWN_ANDROID = 67, 1845 + }; 1846 + 1847 + enum xrt_face_tracking_state_android 1848 + { 1849 + XRT_FACE_TRACKING_STATE_PAUSED_ANDROID = 0, 1850 + XRT_FACE_TRACKING_STATE_STOPPED_ANDROID = 1, 1851 + XRT_FACE_TRACKING_STATE_TRACKING_ANDROID = 2, 1852 + }; 1853 + 1854 + #define XRT_FACE_PARAMETER_COUNT_ANDROID 68 1855 + 1856 + #define XRT_FACE_REGION_CONFIDENCE_COUNT_ANDROID 3 1857 + 1767 1858 struct xrt_facial_base_expression_set_htc 1768 1859 { 1769 1860 int64_t sample_time_ns; ··· 1796 1887 bool is_eye_following_blendshapes_valid; 1797 1888 }; 1798 1889 1890 + struct xrt_facial_expression_set_android 1891 + { 1892 + // ordered by xrt_face_parameter_indices_android 1893 + float parameters[XRT_FACE_PARAMETER_COUNT_ANDROID]; 1894 + float region_confidences[XRT_FACE_REGION_CONFIDENCE_COUNT_ANDROID]; 1895 + 1896 + uint64_t sample_time_ns; 1897 + 1898 + XRT_ALIGNAS(8) bool is_valid; 1899 + }; 1900 + 1799 1901 struct xrt_facial_expression_set 1800 1902 { 1801 1903 union { ··· 1803 1905 struct xrt_facial_eye_expression_set_htc eye_expression_set_htc; 1804 1906 struct xrt_facial_lip_expression_set_htc lip_expression_set_htc; 1805 1907 struct xrt_facial_expression_set2_fb face_expression_set2_fb; 1908 + struct xrt_facial_expression_set_android face_expression_set_android; 1806 1909 }; 1807 1910 }; 1808 1911

+23

src/xrt/include/xrt/xrt_device.h

··· 406 406 struct xrt_facial_expression_set *out_value); 407 407 408 408 /*! 409 + * @brief Gets the face tracking calibration state 410 + * 411 + * @param[in] xdev The device. 412 + * @param[in] out_value Is face tracking calibrated? 413 + * 414 + * @see xrt_input_name 415 + */ 416 + xrt_result_t (*get_face_calibration_state_android)(struct xrt_device *xdev, bool *out_face_is_calibrated); 417 + 418 + /*! 409 419 * @brief Get the body skeleton in T-pose, used to query the skeleton hierarchy, scale, proportions etc 410 420 * 411 421 * @param[in] xdev The device. ··· 753 763 struct xrt_facial_expression_set *out_value) 754 764 { 755 765 return xdev->get_face_tracking(xdev, facial_expression_type, at_timestamp_ns, out_value); 766 + } 767 + 768 + /*! 769 + * Helper function for @ref xrt_device::get_face_calibration_state_android. 770 + * 771 + * @copydoc xrt_device::get_face_calibration_state_android 772 + * 773 + * @public @memberof xrt_device 774 + */ 775 + static inline xrt_result_t 776 + xrt_device_get_face_calibration_state_android(struct xrt_device *xdev, bool *out_face_is_calibrated) 777 + { 778 + return xdev->get_face_calibration_state_android(xdev, out_face_is_calibrated); 756 779 } 757 780 758 781 /*!

+11

src/xrt/ipc/server/ipc_server_handler.c

··· 2572 2572 } 2573 2573 2574 2574 xrt_result_t 2575 + ipc_handle_device_device_get_face_calibration_state_android(volatile struct ipc_client_state *ics, 2576 + uint32_t id, 2577 + bool *out_face_is_calibrated) 2578 + { 2579 + const uint32_t device_id = id; 2580 + struct xrt_device *xdev = NULL; 2581 + GET_XDEV_OR_RETURN(ics, device_id, xdev); 2582 + return xrt_device_get_face_calibration_state_android(xdev, out_face_is_calibrated); 2583 + } 2584 + 2585 + xrt_result_t 2575 2586 ipc_handle_device_get_body_skeleton(volatile struct ipc_client_state *ics, 2576 2587 uint32_t id, 2577 2588 enum xrt_input_name body_tracking_type,

+9

src/xrt/ipc/shared/proto/50-device.json

··· 181 181 ] 182 182 }, 183 183 184 + "device_device_get_face_calibration_state_android": { 185 + "in": [ 186 + {"name": "id", "type": "uint32_t"} 187 + ], 188 + "out": [ 189 + {"name": "face_is_calibrated", "type": "bool"} 190 + ] 191 + }, 192 + 184 193 "device_get_body_skeleton": { 185 194 "in": [ 186 195 {"name": "id", "type": "uint32_t"},

+84

src/xrt/state_trackers/oxr/oxr_face_tracking_android.c

··· 14 14 #include "oxr_objects.h" 15 15 #include "oxr_logger.h" 16 16 #include "oxr_handle.h" 17 + #include "oxr_xret.h" 18 + #include "oxr_two_call.h" 17 19 18 20 static XrResult 19 21 oxr_face_tracker_android_destroy_cb(struct oxr_logger *log, struct oxr_handle_base *hb) ··· 29 31 const XrFaceTrackerCreateInfoANDROID *createInfo, 30 32 XrFaceTrackerANDROID *faceTracker) 31 33 { 34 + bool supported = false; 35 + oxr_system_get_face_tracking_android_support(log, sess->sys->inst, &supported); 36 + if (!supported) { 37 + return oxr_error(log, XR_ERROR_FEATURE_UNSUPPORTED, "System does not support Android face tracking"); 38 + } 39 + 40 + struct xrt_device *xdev = GET_XDEV_BY_ROLE(sess->sys, face); 41 + if (xdev == NULL) { 42 + return oxr_error(log, XR_ERROR_FEATURE_UNSUPPORTED, "No device found for face tracking role"); 43 + } 44 + 45 + if (!xdev->supported.face_tracking) { 46 + return oxr_error(log, XR_ERROR_FEATURE_UNSUPPORTED, "Device does not support HTC facial tracking"); 47 + } 48 + 49 + struct oxr_face_tracker_android *face_tracker_android = NULL; 50 + OXR_ALLOCATE_HANDLE_OR_RETURN(log, face_tracker_android, OXR_XR_DEBUG_FTRACKER, 51 + oxr_face_tracker_android_destroy_cb, &sess->handle); 52 + 53 + face_tracker_android->sess = sess; 54 + face_tracker_android->xdev = xdev; 55 + 56 + *faceTracker = oxr_face_tracker_android_to_openxr(face_tracker_android); 32 57 return oxr_session_success_result(sess); 33 58 } 34 59 ··· 38 63 const XrFaceStateGetInfoANDROID *getInfo, 39 64 XrFaceStateANDROID *faceStateOutput) 40 65 { 66 + /*! 67 + * OXR_TWO_CALL_CHECK_* macro usage here is not technically necessary because validation 68 + * is handled in the API layer before this function is called, but we still re-use them here 69 + * for declarative purposes of handling two-call patterns of setting capacities on the first call. 70 + */ 71 + 72 + /*! 73 + * Use the goto macro variant because a two-call check needs to happen with region confidences 74 + * as well; we can't early exit for only one of them. 75 + */ 76 + XrResult xres = XR_SUCCESS; 77 + OXR_TWO_CALL_CHECK_GOTO(log, faceStateOutput->parametersCapacityInput, 78 + (&faceStateOutput->parametersCountOutput), XRT_FACE_PARAMETER_COUNT_ANDROID, xres, 79 + region_confidences_check); 80 + region_confidences_check: 81 + if (xres != XR_SUCCESS) { 82 + return xres; 83 + } 84 + 85 + OXR_TWO_CALL_CHECK_ONLY(log, faceStateOutput->regionConfidencesCapacityInput, 86 + (&faceStateOutput->regionConfidencesCountOutput), 87 + XRT_FACE_REGION_CONFIDENCE_COUNT_ANDROID, xres); 88 + 89 + const struct oxr_instance *inst = facial_tracker_android->sess->sys->inst; 90 + const int64_t at_timestamp_ns = time_state_ts_to_monotonic_ns(inst->timekeeping, getInfo->time); 91 + 92 + struct xrt_facial_expression_set facial_expression_set_result = {0}; 93 + const xrt_result_t xret = 94 + xrt_device_get_face_tracking(facial_tracker_android->xdev, XRT_INPUT_ANDROID_FACE_TRACKING, at_timestamp_ns, 95 + &facial_expression_set_result); 96 + OXR_CHECK_XRET(log, facial_tracker_android->sess, xret, "oxr_get_face_state_android"); 97 + 98 + const struct xrt_facial_expression_set_android *face_expression_set_android = 99 + &facial_expression_set_result.face_expression_set_android; 100 + 101 + faceStateOutput->isValid = face_expression_set_android->is_valid; 102 + if (faceStateOutput->isValid == XR_FALSE) { 103 + return XR_SUCCESS; 104 + } 105 + 106 + faceStateOutput->sampleTime = 107 + time_state_monotonic_to_ts_ns(inst->timekeeping, face_expression_set_android->sample_time_ns); 108 + 109 + if (faceStateOutput->parametersCapacityInput) { 110 + memcpy(faceStateOutput->parameters, face_expression_set_android->parameters, 111 + sizeof(float) * faceStateOutput->parametersCapacityInput); 112 + } 113 + 114 + if (faceStateOutput->regionConfidencesCapacityInput) { 115 + memcpy(faceStateOutput->regionConfidences, face_expression_set_android->region_confidences, 116 + sizeof(float) * faceStateOutput->regionConfidencesCapacityInput); 117 + } 41 118 42 119 return oxr_session_success_result(facial_tracker_android->sess); 43 120 } ··· 47 124 struct oxr_face_tracker_android *facial_tracker_android, 48 125 XrBool32 *faceIsCalibratedOutput) 49 126 { 127 + bool face_is_calibrated = false; 128 + const xrt_result_t xret = 129 + xrt_device_get_face_calibration_state_android(facial_tracker_android->xdev, &face_is_calibrated); 130 + OXR_CHECK_XRET(log, facial_tracker_android->sess, xret, "oxr_get_face_calibration_state_android"); 131 + 132 + *faceIsCalibratedOutput = face_is_calibrated; 133 + 50 134 return oxr_session_success_result(facial_tracker_android->sess); 51 135 }

+3

src/xrt/state_trackers/oxr/oxr_objects.h

··· 1124 1124 oxr_system_get_force_feedback_support(struct oxr_logger *log, struct oxr_instance *inst); 1125 1125 1126 1126 void 1127 + oxr_system_get_face_tracking_android_support(struct oxr_logger *log, struct oxr_instance *inst, bool *supported); 1128 + 1129 + void 1127 1130 oxr_system_get_face_tracking_htc_support(struct oxr_logger *log, 1128 1131 struct oxr_instance *inst, 1129 1132 bool *supports_eye,

+36

src/xrt/state_trackers/oxr/oxr_system.c

··· 406 406 } 407 407 408 408 void 409 + oxr_system_get_face_tracking_android_support(struct oxr_logger *log, struct oxr_instance *inst, bool *supported) 410 + { 411 + assert(supported); 412 + 413 + *supported = false; 414 + struct oxr_system *sys = &inst->system; 415 + const struct xrt_device *face_xdev = GET_XDEV_BY_ROLE(sys, face); 416 + 417 + if (face_xdev == NULL || !face_xdev->supported.face_tracking || face_xdev->inputs == NULL) { 418 + return; 419 + } 420 + 421 + for (size_t input_idx = 0; input_idx < face_xdev->input_count; ++input_idx) { 422 + const struct xrt_input *input = &face_xdev->inputs[input_idx]; 423 + if (input->name == XRT_INPUT_ANDROID_FACE_TRACKING) { 424 + *supported = true; 425 + return; 426 + } 427 + } 428 + } 429 + 430 + void 409 431 oxr_system_get_face_tracking_htc_support(struct oxr_logger *log, 410 432 struct oxr_instance *inst, 411 433 bool *supports_eye, ··· 567 589 } 568 590 } 569 591 #endif 592 + 593 + #ifdef OXR_HAVE_ANDROID_face_tracking 594 + XrSystemFaceTrackingPropertiesANDROID *android_face_tracking_props = NULL; 595 + if (sys->inst->extensions.ANDROID_face_tracking) { 596 + android_face_tracking_props = OXR_GET_OUTPUT_FROM_CHAIN( 597 + properties, XR_TYPE_SYSTEM_FACE_TRACKING_PROPERTIES_ANDROID, XrSystemFaceTrackingPropertiesANDROID); 598 + } 599 + 600 + if (android_face_tracking_props) { 601 + bool supported = false; 602 + oxr_system_get_face_tracking_android_support(log, sys->inst, &supported); 603 + android_face_tracking_props->supportsFaceTracking = supported; 604 + } 605 + #endif // OXR_HAVE_HTC_facial_tracking 570 606 571 607 #ifdef OXR_HAVE_HTC_facial_tracking 572 608 XrSystemFacialTrackingPropertiesHTC *htc_facial_tracking_props = NULL;

Configure Feed

Configure Feed