xrt: Remove old_rgb hand tracking

src/xrt/drivers/CMakeLists.txt

···
 			aux_util
 			aux_math
 			t_ht_mercury
-			t_ht_old_rgb
 			hand_async
 		)
 

src/xrt/drivers/ht/ht_driver.c

···
 
 // Save me, Obi-Wan!
 
-#include "../../tracking/hand/old_rgb/rgb_interface.h"
 #include "../../tracking/hand/mercury/hg_interface.h"
 
 #ifdef XRT_BUILD_DRIVER_DEPTHAI
···
 int
 ht_device_create(struct xrt_frame_context *xfctx,
                  struct t_stereo_camera_calibration *calib,
-                 enum t_hand_tracking_algorithm algorithm_choice,
                  struct t_camera_extra_info extra_camera_info,
                  struct xrt_slam_sinks **out_sinks,
                  struct xrt_device **out_device)
···
 
 	struct t_hand_tracking_sync *sync = NULL;
 
-	switch (algorithm_choice) {
-	case HT_ALGORITHM_MERCURY: {
-		sync = t_hand_tracking_sync_mercury_create(calib, extra_camera_info);
-	} break;
-	case HT_ALGORITHM_OLD_RGB: {
-		//!@todo Either have this deal with the output space correctly, or have everything use LEFT_CAMERA
-		sync = t_hand_tracking_sync_old_rgb_create(calib);
-	}
-	}
+	sync = t_hand_tracking_sync_mercury_create(calib, extra_camera_info);
+
 	struct ht_device *htd = ht_device_create_common(calib, false, xfctx, sync);
 
 	HT_DEBUG(htd, "Hand Tracker initialized!");

src/xrt/drivers/ht/ht_interface.h

···
  *
  * @param xfctx Frame context to attach the tracker to
  * @param calib Calibration struct for stereo camera
- * @param algorithm_choice Which algorithm to use for hand tracking
  * @param out_sinks Sinks to stream camera data to
  * @param out_device Newly created hand tracker "device"
  * @return int 0 on success
···
 int
 ht_device_create(struct xrt_frame_context *xfctx,
                  struct t_stereo_camera_calibration *calib,
-                 enum t_hand_tracking_algorithm algorithm_choice,
                  struct t_camera_extra_info extra_camera_info,
                  struct xrt_slam_sinks **out_sinks,
                  struct xrt_device **out_device);

src/xrt/drivers/rift_s/rift_s_tracker.c

···
 	extra_camera_info.views[0].camera_orientation = CAMERA_ORIENTATION_90;
 	extra_camera_info.views[1].camera_orientation = CAMERA_ORIENTATION_90;
 
-	int create_status = ht_device_create(xfctx,                //
-	                                     t->stereo_calib,      //
-	                                     HT_ALGORITHM_MERCURY, //
+	int create_status = ht_device_create(xfctx,           //
+	                                     t->stereo_calib, //
 	                                     extra_camera_info,
 	                                     &sinks, //
 	                                     &device);

src/xrt/drivers/wmr/wmr_hmd.c

···
 	extra_camera_info.views[0].boundary_type = HT_IMAGE_BOUNDARY_NONE;
 	extra_camera_info.views[1].boundary_type = HT_IMAGE_BOUNDARY_NONE;
 
-	int create_status = ht_device_create(&wh->tracking.xfctx,  //
-	                                     stereo_calib,         //
-	                                     HT_ALGORITHM_MERCURY, //
-	                                     extra_camera_info,    //
-	                                     &sinks,               //
+	int create_status = ht_device_create(&wh->tracking.xfctx, //
+	                                     stereo_calib,        //
+	                                     extra_camera_info,   //
+	                                     &sinks,              //
 	                                     &device);
 	if (create_status != 0) {
 		return create_status;

src/xrt/include/tracking/t_hand_tracking.h

···
 };
 
 /*!
- * @brief Which hand-tracking algorithm should we use?
- *
- * Never use HT_ALGORITHM_OLD_RGB. The tracking quality is extremely poor.
- * @ingroup xrt_iface
- */
-enum t_hand_tracking_algorithm
-{
-	HT_ALGORITHM_MERCURY,
-	HT_ALGORITHM_OLD_RGB
-};
-
-/*!
  * Synchronously processes frames and returns two hands.
  */
 struct t_hand_tracking_sync

src/xrt/state_trackers/gui/gui_scene_hand_tracking_demo.c

···
 	int create_status = ht_device_create( //
 	    &usysd->xfctx,                    //
 	    calib,                            //
-	    HT_ALGORITHM_MERCURY,             //
 	    extra_camera_info,                //
 	    &hand_sinks,                      //
 	    &ht_dev);                         //

src/xrt/targets/common/target_builder_lighthouse.c

···
 DEBUG_GET_ONCE_BOOL_OPTION(vive_over_survive, "VIVE_OVER_SURVIVE", false)
 DEBUG_GET_ONCE_BOOL_OPTION(vive_slam, "VIVE_SLAM", false)
 DEBUG_GET_ONCE_TRISTATE_OPTION(lh_handtracking, "LH_HANDTRACKING")
-DEBUG_GET_ONCE_BOOL_OPTION(ht_use_old_rgb, "HT_USE_OLD_RGB", false)
 
 #define LH_TRACE(...) U_LOG_IFL_T(debug_get_log_option_lh_log(), __VA_ARGS__)
 #define LH_DEBUG(...) U_LOG_IFL_D(debug_get_log_option_lh_log(), __VA_ARGS__)
···
 	info.views[0].boundary.circle.normalized_radius = 0.55;
 	info.views[1].boundary.circle.normalized_radius = 0.55;
 
-	bool old_rgb = debug_get_bool_option_ht_use_old_rgb();
-	enum t_hand_tracking_algorithm ht_algorithm = old_rgb ? HT_ALGORITHM_OLD_RGB : HT_ALGORITHM_MERCURY;
-
 	struct xrt_device *ht_device = NULL;
 	int create_status = ht_device_create(&lhs->devices->xfctx, //
 	                                     stereo_calib,         //
-	                                     ht_algorithm,         //
 	                                     info,                 //
 	                                     &sinks,               //
 	                                     &ht_device);
···
 	struct xrt_frame_sink *entry_left_sink = NULL;
 	struct xrt_frame_sink *entry_right_sink = NULL;
 	struct xrt_frame_sink *entry_sbs_sink = NULL;
-	bool old_rgb_ht = debug_get_bool_option_ht_use_old_rgb();
 
-	if (slam_enabled && hand_enabled && !old_rgb_ht) {
+	if (slam_enabled && hand_enabled) {
 		u_sink_split_create(&lhs->devices->xfctx, slam_sinks->left, hand_sinks->left, &entry_left_sink);
 		u_sink_split_create(&lhs->devices->xfctx, slam_sinks->right, hand_sinks->right, &entry_right_sink);
 		u_sink_stereo_sbs_to_slam_sbs_create(&lhs->devices->xfctx, entry_left_sink, entry_right_sink,
 		                                     &entry_sbs_sink);
 		u_sink_create_format_converter(&lhs->devices->xfctx, XRT_FORMAT_L8, entry_sbs_sink, &entry_sbs_sink);
-	} else if (slam_enabled && hand_enabled && old_rgb_ht) {
-		struct xrt_frame_sink *hand_sbs = NULL;
-		struct xrt_frame_sink *slam_sbs = NULL;
-		u_sink_stereo_sbs_to_slam_sbs_create(&lhs->devices->xfctx, hand_sinks->left, hand_sinks->right,
-		                                     &hand_sbs);
-		u_sink_stereo_sbs_to_slam_sbs_create(&lhs->devices->xfctx, slam_sinks->left, slam_sinks->right,
-		                                     &slam_sbs);
-		u_sink_create_format_converter(&lhs->devices->xfctx, XRT_FORMAT_L8, slam_sbs, &slam_sbs);
-		u_sink_split_create(&lhs->devices->xfctx, slam_sbs, hand_sbs, &entry_sbs_sink);
 	} else if (slam_enabled) {
 		entry_left_sink = slam_sinks->left;
 		entry_right_sink = slam_sinks->right;
···
 		                                     &entry_sbs_sink);
 		u_sink_create_format_converter(&lhs->devices->xfctx, XRT_FORMAT_L8, entry_sbs_sink, &entry_sbs_sink);
 	} else if (hand_enabled) {
-		enum xrt_format fmt = old_rgb_ht ? XRT_FORMAT_R8G8B8 : XRT_FORMAT_L8;
 		entry_left_sink = hand_sinks->left;
 		entry_right_sink = hand_sinks->right;
 		u_sink_stereo_sbs_to_slam_sbs_create(&lhs->devices->xfctx, entry_left_sink, entry_right_sink,
 		                                     &entry_sbs_sink);
-		u_sink_create_format_converter(&lhs->devices->xfctx, fmt, entry_sbs_sink, &entry_sbs_sink);
+		u_sink_create_format_converter(&lhs->devices->xfctx, XRT_FORMAT_L8, entry_sbs_sink, &entry_sbs_sink);
 	} else {
 		LH_WARN("No visual trackers were set");
 		return false;

src/xrt/targets/common/target_builder_north_star.c

···
 	extra_camera_info.views[0].boundary_type = HT_IMAGE_BOUNDARY_NONE;
 	extra_camera_info.views[1].boundary_type = HT_IMAGE_BOUNDARY_NONE;
 
-	int create_status = ht_device_create(&usysd->xfctx,        //
-	                                     calib,                //
-	                                     HT_ALGORITHM_MERCURY, //
-	                                     extra_camera_info,    //
-	                                     &hand_sinks,          //
+	int create_status = ht_device_create(&usysd->xfctx,     //
+	                                     calib,             //
+	                                     extra_camera_info, //
+	                                     &hand_sinks,       //
 	                                     out_hand_device);
 	t_stereo_camera_calibration_reference(&calib, NULL);
 	if (create_status != 0) {

src/xrt/tracking/hand/CMakeLists.txt

···
 # Copyright 2022, Collabora, Ltd.
 # SPDX-License-Identifier: BSL-1.0
 
-add_subdirectory(old_rgb)
 add_subdirectory(mercury)
 
 ###

src/xrt/tracking/hand/old_rgb/CMakeLists.txt

···
-# Copyright 2019-2022, Collabora, Ltd.
-# SPDX-License-Identifier: BSL-1.0
-
-# Old RGB hand tracking library.
-add_library(
-	t_ht_old_rgb STATIC
-	rgb_hand_math.hpp
-	rgb_image_math.hpp
-	rgb_interface.h
-	rgb_model.hpp
-	rgb_nms.hpp
-	rgb_sync.cpp
-	rgb_sync.hpp
-	)
-target_link_libraries(
-	t_ht_old_rgb
-	PUBLIC aux-includes xrt-external-cjson
-	PRIVATE
-		aux_math
-		aux_tracking
-		aux_os
-		aux_util
-		ONNXRuntime::ONNXRuntime
-		${OpenCV_LIBRARIES}
-	)
-if(XRT_HAVE_OPENCV)
-	target_include_directories(
-		t_ht_old_rgb SYSTEM PRIVATE ${OpenCV_INCLUDE_DIRS} ${EIGEN3_INCLUDE_DIR}
-		)
-	target_link_libraries(t_ht_old_rgb PUBLIC ${OpenCV_LIBRARIES})
-endif()

src/xrt/tracking/hand/old_rgb/readme.md

···
-<!--
-Copyright 2021, Collabora, Ltd.
-Authors:
-Moses Turner <[e-mail]>
-SPDX-License-Identifier: BSL-1.0
--->
-
-# What is this?
-This is a driver to do optical hand tracking. The actual code mostly written by Moses Turner, with tons of help from Marcus Edel, Jakob Bornecrantz, Ryan Pavlik, and Christoph Haag. Jakob Bornecrantz and Marcus Edel are the main people who gathered training data for the initial Collabora models.
-
-In `main` it only works with Valve Index, although we've used a lot of Luxonis cameras in development. With additional work, it should work fine with devices like the T265, or PS4/PS5 cam, should there be enough interest for any of those.
-
-Under good lighting, I would say it's around as good as Oculus Quest 2's hand tracking. Not that I'm trying to make any claims; that's just what I honestly would tell somebody if they are wondering if it's worth testing out.
-
-
-# How to get started
-## Get dependencies
-### Get OpenCV
-Each distro has its own way to get OpenCV, and it can change at any time; there's no specific reason to trust this documentation over anything else.
-
-Having said that, on Ubuntu, it would look something like
-
-```
-sudo apt install libopencv-dev libopencv-contrib-dev
-```
-
-Or you could build it from source, or get it from one of the other 1000s of package managers. Whatever floats your boat.
-
-### Get ONNXRuntime
-I followed the instructions here: https://onnxruntime.ai/docs/how-to/build/inferencing.html#linux
-
-then had to do
-```
-cd build/Linux/RelWithDebInfo/
-sudo make install
-```
-
-### Get the ML models
-Make sure you have git-lfs installed, then run ./scripts/get-ht-models.sh. Should work fine.
-
-## Building the driver
-Once onnxruntime is installed, you should be able to build like normal with CMake or Meson.
-
-If it properly found everything, - CMake should say
-
-```
--- Found ONNXRUNTIME: /usr/local/include/onnxruntime
-
-[...]
-
--- #    DRIVER_HANDTRACKING: ON
-```
-
-and Meson should say
-
-```
-Run-time dependency libonnxruntime found: YES 1.8.2
-
-[...]
-
-Message: Configuration done!
-Message:     drivers:  [...] handtracking, [...]
-```
-
-## Running the driver
-Currently, it's only set up to work on Valve Index.
-
-So, the two things you can do are
-* Use the `survive` driver with both controllers off - It should automagically start hand tracking upon not finding any controllers.
-* Use the `vive` driver with `VIVE_USE_HANDTRACKING=ON` and it should work the same as the survive driver.
-
-You can see if the driver is working with `openxr-simple-playground`, StereoKit, or any other app you know of. Poke me (Moses) if you find any other cool hand-tracking apps; I'm always looking for more!
-
-# Tips and tricks
-
-This tracking likes to be in a bright, evenly-lit room with multiple light sources. Turn all the lights on, see if you can find any lamps. If the ML models can see well, the tracking quality can get surprisingly nice.
-
-Sometimes, the tracking fails when it can see more than one hand. As the tracking gets better (we train better ML models and squash more bugs) this should happen less often or not at all. If it does, put one of your hands down, and it should resume tracking the remaining hand just fine.
-
-# Future improvements
-
-* Get more training data; train better ML models.
-* Improve the tracking math
-  * Be smarter about keeping tracking lock on a hand
-  * Try predicting the next bounding box based on the estimated keypoints of the last few frames instead of uncritically trusting the detection model, and not run the detection model *every single* frame.
-  * Instead of directly doing disparity on the observed keypoints, use a kinematic model of the hand and fit that to the 2D observations - this should get rid of a *lot* of jitter and make it look better to the end user if the ML models fail
-  * Make something that also works with non-stereo (mono, trinocular, or N cameras) camera setups
-* Optionally run the ML models on GPU - currently, everything's CPU bound which could be sub-optimal under some circumstances
-* Write a lot of generic code so that you can run this on any stereo camera
-* More advanced prediction/interpolation code that doesn't care at all about the input frame cadence. One-euro filters are pretty good about this, but we can get better!

src/xrt/tracking/hand/old_rgb/rgb_hand_math.hpp

···
-#pragma once
-
-// Copyright 2021, Collabora, Ltd.
-// SPDX-License-Identifier: BSL-1.0
-/*!
- * @file
- * @brief Helper math to do things with 3D hands for the camera-based hand tracker
- * @author Moses Turner <[e-mail]>
- * @author Nick Klingensmith <[e-mail]>
- * @ingroup drv_ht
- */
-
-#include "math/m_api.h"
-#include "math/m_vec3.h"
-
-#include "rgb_sync.hpp"
-#include "util/u_time.h"
-#include "xrt/xrt_defines.h"
-
-static constexpr int num_real_joints = 21;
-
-float
-sumOfHandJointDistances(const Hand3D &one, const Hand3D &two)
-{
-	float dist = 0.0f;
-	for (int i = 0; i < num_real_joints; i++) {
-		dist += m_vec3_len(one.kps[i] - two.kps[i]);
-	}
-	return dist;
-}
-
-float
-errHandHistory(const HandHistory3D &history_hand, const Hand3D &present_hand)
-{
-	// Remember we never have to deal with an empty hand. Can always read the last element.
-	return sumOfHandJointDistances(history_hand.last_hands_unfiltered.back(), present_hand);
-}
-
-float
-errHandDisparity(const Hand2D &left_rays, const Hand2D &right_rays)
-{
-	float error_y_diff = 0.0f;
-	for (int i = 0; i < 21; i++) {
-		float diff_y = fabsf(left_rays.kps[i].y - right_rays.kps[i].y);
-		// Big question about what's the best loss function. Gut feeling was "I should be using sum of squared
-		// errors" but I don't really know. Using just sum of errors for now. Ideally it'd also be not very
-		// sensitive to one or two really bad outliers.
-		error_y_diff += diff_y;
-	}
-	// U_LOG_E("stereo camera err is %f, y_disparity is %f", err_stereo_camera, error_y_diff);
-	return error_y_diff;
-}
-
-void
-applyThumbIndexDrag(Hand3D *hand)
-{
-	// TERRIBLE HACK.
-	// Puts the thumb and pointer a bit closer together to be better at triggering XR clients' pinch detection.
-	static const float max_radius = 0.05;
-	static const float min_radius = 0.00;
-
-	// no min drag, min drag always 0.
-	static const float max_drag = 0.85f;
-
-	xrt_vec3 thumb = hand->kps[THMB_TIP];
-	xrt_vec3 index = hand->kps[INDX_TIP];
-	xrt_vec3 ttp = index - thumb;
-	float length = m_vec3_len(ttp);
-	if ((length > max_radius)) {
-		return;
-	}
-
-
-	float amount = math_map_ranges(length, min_radius, max_radius, max_drag, 0.0f);
-
-	hand->kps[THMB_TIP] = m_vec3_lerp(thumb, index, amount * 0.5f);
-	hand->kps[INDX_TIP] = m_vec3_lerp(index, thumb, amount * 0.5f);
-}
-
-static inline xrt_vec3
-get_joint_position(struct xrt_hand_joint_set *set, xrt_hand_joint jt)
-{
-	return set->values.hand_joint_set_default[jt].relation.pose.position;
-}
-
-template <size_t N>
-static inline void
-set_finger(struct xrt_hand_joint_set *set,
-           const xrt_vec3 &pinky_to_index_prox,
-           const std::array<xrt_hand_joint, N> &finger)
-{
-	for (size_t i = 0; i < N - 1; i++) {
-		// Don't do fingertips. (Fingertip would be index 4.)
-		struct xrt_vec3 forwards =
-		    m_vec3_normalize(get_joint_position(set, finger[i + 1]) - get_joint_position(set, finger[i]));
-		struct xrt_vec3 backwards = m_vec3_mul_scalar(forwards, -1.0f);
-
-		struct xrt_vec3 left = m_vec3_orthonormalize(forwards, pinky_to_index_prox);
-		// float dot = m_vec3_dot(backwards, left);
-		// assert((m_vec3_dot(backwards,left) == 0.0f));
-		math_quat_from_plus_x_z(&left, &backwards,
-		                        &set->values.hand_joint_set_default[finger[i]].relation.pose.orientation);
-	}
-	// Do fingertip! Per XR_EXT_hand_tracking, just copy the distal joint's orientation. Doing anything else
-	// is wrong.
-	set->values.hand_joint_set_default[finger[N - 1]].relation.pose.orientation =
-	    set->values.hand_joint_set_default[finger[N - 2]].relation.pose.orientation;
-}
-
-void
-applyJointOrientations(struct xrt_hand_joint_set *set, bool is_right)
-{
-	// The real rule to follow is that each joint's "X" axis is along the axis along which it can bend.
-	// The nature of our estimation makes this a bit difficult, but these should work okay-ish under perfect
-	// conditions
-	if (set->is_active == false) {
-		return;
-	}
-
-	auto gl = [&](xrt_hand_joint jt) { return get_joint_position(set, jt); };
-
-	xrt_vec3 pinky_prox = gl(XRT_HAND_JOINT_LITTLE_PROXIMAL);
-
-	xrt_vec3 index_prox = gl(XRT_HAND_JOINT_INDEX_PROXIMAL);
-
-
-	xrt_vec3 pinky_to_index_prox = m_vec3_normalize(index_prox - pinky_prox);
-	if (is_right) {
-		pinky_to_index_prox = m_vec3_mul_scalar(pinky_to_index_prox, -1.0f);
-	}
-
-	using Finger = std::array<xrt_hand_joint, 5>;
-	static const std::array<Finger, 4> fingers_with_joints_in_them = {{
-
-	    {XRT_HAND_JOINT_INDEX_METACARPAL, XRT_HAND_JOINT_INDEX_PROXIMAL, XRT_HAND_JOINT_INDEX_INTERMEDIATE,
-	     XRT_HAND_JOINT_INDEX_DISTAL, XRT_HAND_JOINT_INDEX_TIP},
-
-	    {XRT_HAND_JOINT_MIDDLE_METACARPAL, XRT_HAND_JOINT_MIDDLE_PROXIMAL, XRT_HAND_JOINT_MIDDLE_INTERMEDIATE,
-	     XRT_HAND_JOINT_MIDDLE_DISTAL, XRT_HAND_JOINT_MIDDLE_TIP},
-
-	    {XRT_HAND_JOINT_RING_METACARPAL, XRT_HAND_JOINT_RING_PROXIMAL, XRT_HAND_JOINT_RING_INTERMEDIATE,
-	     XRT_HAND_JOINT_RING_DISTAL, XRT_HAND_JOINT_RING_TIP},
-
-	    {XRT_HAND_JOINT_LITTLE_METACARPAL, XRT_HAND_JOINT_LITTLE_PROXIMAL, XRT_HAND_JOINT_LITTLE_INTERMEDIATE,
-	     XRT_HAND_JOINT_LITTLE_DISTAL, XRT_HAND_JOINT_LITTLE_TIP},
-
-	}};
-	for (Finger const &finger : fingers_with_joints_in_them) {
-		set_finger(set, pinky_to_index_prox, finger);
-	}
-
-	// wrist!
-	// Not the best but acceptable. Eventually, probably, do triangle of wrist pinky prox and index prox.
-	set->values.hand_joint_set_default[XRT_HAND_JOINT_WRIST].relation.pose.orientation =
-	    set->values.hand_joint_set_default[XRT_HAND_JOINT_MIDDLE_METACARPAL].relation.pose.orientation;
-
-
-	// palm!
-	set->values.hand_joint_set_default[XRT_HAND_JOINT_PALM].relation.pose.orientation =
-	    set->values.hand_joint_set_default[XRT_HAND_JOINT_MIDDLE_METACARPAL].relation.pose.orientation;
-
-	// thumb!
-	// When I look at Ultraleap tracking, there's like, a "plane" made by the tip, distal and proximal (and kinda
-	// MCP, but least squares fitting a plane is too hard for my baby brain) Normal to this plane is the +X, and
-	// obviously forwards to the next joint is the -Z.
-	xrt_vec3 thumb_prox_to_dist = gl(XRT_HAND_JOINT_THUMB_DISTAL) - gl(XRT_HAND_JOINT_THUMB_PROXIMAL);
-	xrt_vec3 thumb_dist_to_tip = gl(XRT_HAND_JOINT_THUMB_TIP) - gl(XRT_HAND_JOINT_THUMB_DISTAL);
-	xrt_vec3 plane_normal{};
-	if (!is_right) {
-		math_vec3_cross(&thumb_prox_to_dist, &thumb_dist_to_tip, &plane_normal);
-	} else {
-		math_vec3_cross(&thumb_dist_to_tip, &thumb_prox_to_dist, &plane_normal);
-	}
-	constexpr std::array<enum xrt_hand_joint, 4> thumbs = {XRT_HAND_JOINT_THUMB_METACARPAL,
-	                                                       XRT_HAND_JOINT_THUMB_PROXIMAL,
-	                                                       XRT_HAND_JOINT_THUMB_DISTAL, XRT_HAND_JOINT_THUMB_TIP};
-	//! @todo this code isn't quite the same as set_finger, can we make it the same so we can use that?
-	for (int i = 0; i < 3; i++) {
-		struct xrt_vec3 backwards =
-		    m_vec3_mul_scalar(m_vec3_normalize(gl(thumbs[i + 1]) - gl(thumbs[i])), -1.0f);
-
-		struct xrt_vec3 left = m_vec3_orthonormalize(backwards, plane_normal);
-		math_quat_from_plus_x_z(&left, &backwards,
-		                        &set->values.hand_joint_set_default[thumbs[i]].relation.pose.orientation);
-	}
-	struct xrt_quat *tip = &set->values.hand_joint_set_default[XRT_HAND_JOINT_THUMB_TIP].relation.pose.orientation;
-	struct xrt_quat *distal =
-	    &set->values.hand_joint_set_default[XRT_HAND_JOINT_THUMB_DISTAL].relation.pose.orientation;
-	memcpy(tip, distal, sizeof(struct xrt_quat));
-}
-
-float
-handednessJointSet(Hand3D *set)
-{
-	// Guess if hand is left or right.
-	// Left is negative, right is positive.
-
-
-	// xrt_vec3 middle_mcp = gl(XRT_HAND_JOINT_MIDDLE_METACARPAL);
-
-	xrt_vec3 pinky_prox = set->kps[LITL_PXM]; // gl(XRT_HAND_JOINT_LITTLE_PROXIMAL);
-
-	xrt_vec3 index_prox = set->kps[INDX_PXM]; // gl(XRT_HAND_JOINT_INDEX_PROXIMAL);
-
-	xrt_vec3 pinky_to_index_prox = m_vec3_normalize(index_prox - pinky_prox);
-
-	float handedness = 0.0f;
-
-	for (int i : {INDX_PXM, MIDL_PXM, RING_PXM, LITL_PXM}) {
-		xrt_vec3 prox = set->kps[i];
-		xrt_vec3 intr = set->kps[i + 1];
-		xrt_vec3 dist = set->kps[i + 2];
-		xrt_vec3 tip = set->kps[i + 3];
-
-		xrt_vec3 prox_to_int = m_vec3_normalize(intr - prox);
-		xrt_vec3 int_to_dist = m_vec3_normalize(dist - intr);
-		xrt_vec3 dist_to_tip = m_vec3_normalize(tip - dist);
-
-		xrt_vec3 checks[2];
-
-		math_vec3_cross(&prox_to_int, &int_to_dist, &checks[0]);
-		math_vec3_cross(&int_to_dist, &dist_to_tip, &checks[1]);
-
-		handedness += m_vec3_dot(m_vec3_normalize(pinky_to_index_prox), (checks[0]));
-		handedness += m_vec3_dot(m_vec3_normalize(pinky_to_index_prox), (checks[1]));
-	}
-	set->handedness = handedness / (4 * 2);
-	return set->handedness;
-}
-
-void
-handednessHandHistory3D(HandHistory3D *history)
-{
-
-	float inter = handednessJointSet(&history->last_hands_unfiltered.back());
-
-	if ((fabsf(inter) > 0.3f) || (fabsf(history->handedness) < 0.3f)) {
-		history->handedness += inter;
-	}
-	static const int max_handedness = 2.0f;
-	if (history->handedness > max_handedness) {
-		history->handedness = max_handedness;
-	} else if (history->handedness < -max_handedness) {
-		history->handedness = -max_handedness;
-	}
-}
-
-void
-handEuroFiltersInit(HandHistory3D *history, double fc_min, double fc_min_d, double beta)
-{
-	for (int i = 0; i < 21; i++) {
-		m_filter_euro_vec3_init(&history->filters[i], fc_min, fc_min_d, beta);
-	}
-}
-
-static double
-calc_smoothing_alpha(double Fc, double dt)
-{
-	/* Calculate alpha = (1 / (1 + tau/dt)) where tau = 1.0 / (2 * pi * Fc),
-	 * this is a straight rearrangement with fewer divisions */
-	double r = 2.0 * M_PI * Fc * dt;
-	return r / (r + 1.0);
-}
-
-static double
-exp_smooth(double alpha, double y, double prev_y)
-{
-	return alpha * y + (1.0 - alpha) * prev_y;
-}
-
-void
-handEuroFiltersRun(struct HandTracking *htd, HandHistory3D *f, Hand3D *out_hand)
-{
-	// Assume present hand is in element 0!
-#if 0
-	// float vals[4] = {0.5, 0.33, 0.1, 0.07};
-	float vals[4] = {0.9, 0.09, 0.009, 0.001};
-	auto m = f->last_hands_unfiltered.size() - 1;
-	double ts_out = (vals[0] * (double)f->last_hands_unfiltered.get_at_age(std::min(m, 0))->timestamp) +
-	                (vals[1] * (double)f->last_hands_unfiltered.get_at_age(std::min(m, 1))->timestamp) +
-	                (vals[2] * (double)f->last_hands_unfiltered.get_at_age(std::min(m, 2))->timestamp) +
-	                (vals[3] * (double)f->last_hands_unfiltered.get_at_age(std::min(m, 3))->timestamp);
-	out_hand->timestamp = (uint64_t)ts_out;
-
-	for (int kp_idx = 0; kp_idx < 21; kp_idx++) {
-		for (int hist_idx = 0; hist_idx < 4; hist_idx++) {
-			float *in_y_arr =
-			    (float *)&f->last_hands_unfiltered.get_at_age(std::min(m, hist_idx))->kps[kp_idx];
-			float *out_y_arr = (float *)&out_hand->kps[kp_idx];
-			for (int i = 0; i < 3; i++) {
-				out_y_arr[i] += in_y_arr[i] * vals[hist_idx];
-			}
-		}
-	}
-#elif 0
-	for (int i = 0; i < 21; i++) {
-		m_filter_euro_vec3_run(&f->filters[i], f->last_hands_unfiltered.back().timestamp,
-		                       &f->last_hands_unfiltered.back().kps[i], &out_hand->kps[i]);
-	}
-	// conspicuously wrong!
-	out_hand->timestamp = f->last_hands_unfiltered.back().timestamp;
-#else
-
-	if (!f->have_prev_hand) {
-		f->last_hands_filtered.push_back(f->last_hands_unfiltered.back());
-		uint64_t ts = f->last_hands_unfiltered.back().timestamp;
-		f->prev_ts_for_alpha = ts;
-		f->first_ts = ts;
-		f->prev_filtered_ts = ts;
-		f->prev_dy = 0;
-		f->have_prev_hand = true;
-		*out_hand = f->last_hands_unfiltered.back();
-	}
-	uint64_t ts = f->last_hands_unfiltered.back().timestamp;
-	double dt, alpha_d;
-	dt = (double)(ts - f->prev_ts_for_alpha) / U_TIME_1S_IN_NS;
-
-	double abs_dy =
-	    (sumOfHandJointDistances(f->last_hands_unfiltered.back(), f->last_hands_filtered.back()) / 21.0f) * 0.7f;
-	alpha_d = calc_smoothing_alpha(htd->dynamic_config.hand_fc_min_d.val, dt);
-
-	double alpha, fc_cutoff;
-	f->prev_dy = exp_smooth(alpha_d, abs_dy, f->prev_dy);
-
-	fc_cutoff = htd->dynamic_config.hand_fc_min.val + htd->dynamic_config.hand_beta.val * f->prev_dy;
-	alpha = calc_smoothing_alpha(fc_cutoff, dt);
-	HT_DEBUG(htd, "dt is %f, abs_dy is %f, alpha is %f", dt, abs_dy, alpha);
-
-	for (int i = 0; i < 21; i++) {
-		out_hand->kps[i].x =
-		    exp_smooth(alpha, f->last_hands_unfiltered.back().kps[i].x, f->last_hands_filtered.back().kps[i].x);
-		out_hand->kps[i].y =
-		    exp_smooth(alpha, f->last_hands_unfiltered.back().kps[i].y, f->last_hands_filtered.back().kps[i].y);
-		out_hand->kps[i].z =
-		    exp_smooth(alpha, f->last_hands_unfiltered.back().kps[i].z, f->last_hands_filtered.back().kps[i].z);
-	}
-	double prev_ts_offset = (double)(f->prev_filtered_ts - f->first_ts);
-	double current_ts_offset = (double)(ts - f->first_ts);
-	double new_filtered_ts_offset = exp_smooth(alpha, current_ts_offset, prev_ts_offset);
-	uint64_t new_filtered_ts = (uint64_t)(new_filtered_ts_offset) + f->first_ts;
-	out_hand->timestamp = new_filtered_ts;
-	f->prev_filtered_ts = out_hand->timestamp;
-	f->prev_ts_for_alpha = ts; // NOT the filtered timestamp. NO.
-#endif
-}
-
-bool
-rejectTooFar(struct HandTracking *htd, Hand3D *hand)
-{
-	static const float max_dist = 1.0f; // this sucks too - make it bigger if you can.
-	const float max_dist_from_camera_sqrd = max_dist * max_dist;
-	for (int i = 0; i < 21; i++) {
-		xrt_vec3 pos = hand->kps[i];
-		float len = m_vec3_len_sqrd(pos); // Faster.
-		if (len > max_dist_from_camera_sqrd) {
-			goto reject;
-		}
-	}
-	return true;
-
-reject:
-	HT_TRACE(htd, "Rejected too far!");
-	return false;
-}
-
-bool
-rejectTooClose(struct HandTracking *htd, Hand3D *hand)
-{
-	const float min_dist = 0.12f; // Be a bit aggressive here - it's nice to not let people see our tracking fail
-	                              // when the hands are way too close
-	const float min_dist_from_camera_sqrd = min_dist * min_dist;
-
-	for (int i = 0; i < 21; i++) {
-		xrt_vec3 pos = hand->kps[i];
-		float len = m_vec3_len_sqrd(pos); // Faster.
-		if (len < min_dist_from_camera_sqrd) {
-			goto reject;
-		}
-		if (pos.z > min_dist) { // remember negative-Z is forward!
-			goto reject;
-		}
-	}
-	return true;
-
-reject:
-	HT_TRACE(htd, "Rejected too close!");
-	return false;
-}
-
-bool
-rejectTinyPalm(struct HandTracking *htd, Hand3D *hand)
-{
-	// This one sucks, because some people really have tiny hands. If at some point you can stop using it, stop
-	// using it.
-	// Weird scoping so that we can still do gotos
-
-	{
-		float len = m_vec3_len(hand->kps[WRIST] - hand->kps[INDX_PXM]);
-		if ((len < 0.03f || len > 0.25f)) {
-			goto reject;
-		}
-	}
-
-	{
-		float len = m_vec3_len(hand->kps[WRIST] - hand->kps[MIDL_PXM]);
-		if (len < 0.03f || len > 0.25f) {
-			goto reject;
-		}
-	}
-
-	return true;
-
-reject:
-	HT_TRACE(htd, "Rejected because too big or too small!");
-	return false;
-}

src/xrt/tracking/hand/old_rgb/rgb_image_math.hpp

···
-// Copyright 2021, Collabora, Ltd.
-// SPDX-License-Identifier: BSL-1.0
-/*!
- * @file
- * @brief Helper math to do things with images for the camera-based hand tracker
- * @author Moses Turner <[e-mail]>
- * @ingroup drv_ht
- */
-#pragma once
-
-#include "math/m_vec2.h"
-#include "math/m_vec3.h"
-
-#include <opencv2/imgproc.hpp>
-#include <opencv2/core/mat.hpp>
-#include <opencv2/core/types.hpp>
-
-/*!
- * This is a template so that we can use xrt_vec3 or xrt_vec2.
- * Please don't use this for anything other than xrt_vec3 or xrt_vec2!
- */
-template <typename T>
-T
-transformVecBy2x3(T in, cv::Matx23f warp_back)
-{
-	T rr;
-	rr.x = (in.x * warp_back(0, 0)) + (in.y * warp_back(0, 1)) + warp_back(0, 2);
-	rr.y = (in.x * warp_back(1, 0)) + (in.y * warp_back(1, 1)) + warp_back(1, 2);
-	return rr;
-}
-
-cv::Scalar
-hsv2rgb(float fH, float fS, float fV)
-{
-	const float fC = fV * fS; // Chroma
-	const float fHPrime = fmod(fH / 60.0, 6);
-	const float fX = fC * (1 - fabs(fmod(fHPrime, 2) - 1));
-	const float fM = fV - fC;
-
-	float fR, fG, fB;
-
-	if (0 <= fHPrime && fHPrime < 1) {
-		fR = fC;
-		fG = fX;
-		fB = 0;
-	} else if (1 <= fHPrime && fHPrime < 2) {
-		fR = fX;
-		fG = fC;
-		fB = 0;
-	} else if (2 <= fHPrime && fHPrime < 3) {
-		fR = 0;
-		fG = fC;
-		fB = fX;
-	} else if (3 <= fHPrime && fHPrime < 4) {
-		fR = 0;
-		fG = fX;
-		fB = fC;
-	} else if (4 <= fHPrime && fHPrime < 5) {
-		fR = fX;
-		fG = 0;
-		fB = fC;
-	} else if (5 <= fHPrime && fHPrime < 6) {
-		fR = fC;
-		fG = 0;
-		fB = fX;
-	} else {
-		fR = 0;
-		fG = 0;
-		fB = 0;
-	}
-
-	fR += fM;
-	fG += fM;
-	fB += fM;
-	return {fR * 255.0f, fG * 255.0f, fB * 255.0f};
-}
-
-struct xrt_vec3
-raycoord(struct ht_view *htv, struct xrt_vec3 model_out)
-{
-	cv::Mat in_px_coords(1, 1, CV_32FC2);
-	float *write_in;
-	write_in = in_px_coords.ptr<float>(0);
-	write_in[0] = model_out.x;
-	write_in[1] = model_out.y;
-	cv::Mat out_ray(1, 1, CV_32FC2);
-
-	cv::fisheye::undistortPoints(in_px_coords, out_ray, htv->cameraMatrix, htv->distortion);
-
-
-	float n_x = out_ray.at<float>(0, 0);
-	float n_y = out_ray.at<float>(0, 1);
-
-
-	struct xrt_vec3 n = {n_x, n_y, 1.0f};
-
-	cv::Matx33f R = htv->rotate_camera_to_stereo_camera;
-
-	struct xrt_vec3 o = {
-	    (n.x * R(0, 0)) + (n.y * R(0, 1)) + (n.z * R(0, 2)),
-	    (n.x * R(1, 0)) + (n.y * R(1, 1)) + (n.z * R(1, 2)),
-	    (n.x * R(2, 0)) + (n.y * R(2, 1)) + (n.z * R(2, 2)),
-	};
-
-	math_vec3_scalar_mul(1.0f / o.z, &o);
-	return o;
-}
-
-cv::Matx23f
-blackbar(const cv::Mat &in, cv::Mat &out, xrt_size out_size)
-{
-#if 1
-	// Easy to think about, always right, but pretty slow:
-	// Get a matrix from the original to the scaled down / blackbar'd image, then get one that goes back.
-	// Then just warpAffine() it.
-	// Easy in programmer time - never have to worry about off by one, special cases. We can come back and optimize
-	// later.
-
-	// Do the black bars need to be on top and bottom, or on left and right?
-	float scale_down_w = (float)out_size.w / (float)in.cols; // 128/1280 = 0.1
-	float scale_down_h = (float)out_size.h / (float)in.rows; // 128/800 =  0.16
-
-	float scale_down = fmin(scale_down_w, scale_down_h); // 0.1
-
-	float width_inside = (float)in.cols * scale_down;
-	float height_inside = (float)in.rows * scale_down;
-
-	float translate_x = (out_size.w - width_inside) / 2;  // should be 0 for 1280x800
-	float translate_y = (out_size.h - height_inside) / 2; // should be (1280-800)/2 = 240
-
-	cv::Matx23f go;
-	// clang-format off
-	go(0,0) = scale_down;  go(0,1) = 0.0f;                  go(0,2) = translate_x;
-	go(1,0) = 0.0f;                  go(1,1) = scale_down;  go(1,2) = translate_y;
-	// clang-format on
-
-	cv::warpAffine(in, out, go, cv::Size(out_size.w, out_size.h));
-
-	cv::Matx23f ret;
-
-	// clang-format off
-	ret(0,0) = 1.0f/scale_down;  ret(0,1) = 0.0f;             ret(0,2) = -translate_x/scale_down;
-	ret(1,0) = 0.0f;             ret(1,1) = 1.0f/scale_down;  ret(1,2) = -translate_y/scale_down;
-	// clang-format on
-
-	return ret;
-#else
-	// Fast, always wrong if the input isn't square. You'd end up using something like this, plus some
-	// copyMakeBorder if you want to optimize.
-	if (aspect_ratio_input == aspect_ratio_output) {
-		cv::resize(in, out, {out_size.w, out_size.h});
-		cv::Matx23f ret;
-		float scale_from_out_to_in = (float)in.cols / (float)out_size.w;
-		// clang-format off
-		ret(0,0) = scale_from_out_to_in;  ret(0,1) = 0.0f;                  ret(0,2) = 0.0f;
-		ret(1,0) = 0.0f;                  ret(1,1) = scale_from_out_to_in;  ret(1,2) = 0.0f;
-		// clang-format on
-		cv::imshow("hi", out);
-		cv::waitKey(1);
-		return ret;
-	}
-	assert(!"Uh oh! Unimplemented!");
-	return {};
-#endif
-}
-
-void
-handDot(cv::Mat &mat, xrt_vec2 place, float radius, float hue, float intensity, int type)
-{
-	cv::circle(mat, {(int)place.x, (int)place.y}, radius, hsv2rgb(hue * 360.0f, intensity, intensity), type);
-}
-
-void
-centerAndRotationFromJoints(struct ht_view *htv,
-                            const xrt_vec2 *wrist,
-                            const xrt_vec2 *index,
-                            const xrt_vec2 *middle,
-                            const xrt_vec2 *little,
-                            xrt_vec2 *out_center,
-                            xrt_vec2 *out_wrist_to_middle)
-{
-	// Close to what Mediapipe does, but slightly different - just uses the middle proximal instead of "estimating"
-	// it from the pinky and index.
-	// at the end of the day I should probably do that basis vector filtering thing to get a nicer middle metacarpal
-	// from 6 keypoints (not thumb proximal) OR SHOULD I. because distortion. hmm
-
-	// Feel free to look at the way MP does it, you can see it's different.
-	// https://github.com/google/mediapipe/blob/master/mediapipe/modules/holistic_landmark/calculators/hand_detections_from_pose_to_rects_calculator.cc
-
-	// struct xrt_vec2 hand_center = m_vec2_mul_scalar(middle, 0.5) + m_vec2_mul_scalar(index, 0.5*(2.0f/3.0f)) +
-	// m_vec2_mul_scalar(little, 0.5f*((1.0f/3.0f))); // Middle proximal, straight-up.
-	// U_LOG_E("%f %f  %f %f  %f %f  %f %f  ", wrist.x, wrist.y, index.x, index.y, middle.x, middle.y, little.x,
-	// little.y);
-	*out_center = m_vec2_lerp(*middle, m_vec2_lerp(*index, *little, 1.0f / 3.0f), 0.25f);
-
-	*out_wrist_to_middle = *out_center - *wrist;
-}
-
-struct DetectionModelOutput
-rotatedRectFromJoints(struct ht_view *htv, xrt_vec2 center, xrt_vec2 wrist_to_middle, DetectionModelOutput *out)
-{
-	float box_size = m_vec2_len(wrist_to_middle) * 2.0f * 1.73f;
-
-	double rot = atan2(wrist_to_middle.x, wrist_to_middle.y) * (-180.0f / M_PI);
-
-	out->rotation = rot;
-	out->size = box_size;
-	out->center = center;
-
-	cv::RotatedRect rrect =
-	    cv::RotatedRect(cv::Point2f(out->center.x, out->center.y), cv::Size2f(out->size, out->size), out->rotation);
-
-
-	cv::Point2f vertices[4];
-	rrect.points(vertices);
-	if (htv->htd->debug_scribble && htv->htd->dynamic_config.scribble_bounding_box) {
-		for (int i = 0; i < 4; i++) {
-			cv::Scalar b = cv::Scalar(10, 30, 30);
-			if (i == 3) {
-				b = cv::Scalar(255, 255, 0);
-			}
-			cv::line(htv->debug_out_to_this, vertices[i], vertices[(i + 1) % 4], b, 2);
-		}
-	}
-	// topright is 0. bottomright is 1. bottomleft is 2. topleft is 3.
-
-	cv::Point2f src_tri[3] = {vertices[3], vertices[2], vertices[1]}; // top-left, bottom-left, bottom-right
-
-	cv::Point2f dest_tri[3] = {cv::Point2f(0, 0), cv::Point2f(0, 224), cv::Point2f(224, 224)};
-
-	out->warp_there = getAffineTransform(src_tri, dest_tri);
-	out->warp_back = getAffineTransform(dest_tri, src_tri);
-
-	// out->wrist = wrist;
-
-	return *out;
-}
-
-void
-planarize(const cv::Mat &input, uint8_t *output)
-{
-	// output better be the right size, because we are not doing any bounds checking!
-	assert(input.isContinuous());
-	int lix = input.cols;
-	int liy = input.rows;
-	cv::Mat planes[3];
-	cv::split(input, planes);
-	cv::Mat red = planes[0];
-	cv::Mat green = planes[1];
-	cv::Mat blue = planes[2];
-	memcpy(output, red.data, lix * liy);
-	memcpy(output + (lix * liy), green.data, lix * liy);
-	memcpy(output + (lix * liy * 2), blue.data, lix * liy);
-}

src/xrt/tracking/hand/old_rgb/rgb_interface.h

···
-// Copyright 2022, Collabora, Ltd.
-// SPDX-License-Identifier: BSL-1.0
-/*!
- * @file
- * @brief  Public interface of old rgb hand tracking.
- * @author Jakob Bornecrantz <[e-mail]>
- * @ingroup aux_tracking
- */
-
-#include "tracking/t_tracking.h"
-#include "tracking/t_hand_tracking.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-
-/*!
- * Create a old style RGB hand tracking pipeline.
- *
- * @ingroup aux_tracking
- */
-struct t_hand_tracking_sync *
-t_hand_tracking_sync_old_rgb_create(struct t_stereo_camera_calibration *calib);
-
-
-#ifdef __cplusplus
-} // extern "C"
-#endif

src/xrt/tracking/hand/old_rgb/rgb_model.hpp

···
-// Copyright 2021, Collabora, Ltd.
-// SPDX-License-Identifier: BSL-1.0
-/*!
- * @file
- * @brief Code to run machine learning models for camera-based hand tracker.
- * @author Moses Turner <[e-mail]>
- * @author Marcus Edel <[e-mail]>
- * @author Simon Zeni <[e-mail]>
- * @ingroup drv_ht
- */
-
-// Many C api things were stolen from here (MIT license):
-// https://github.com/microsoft/onnxruntime-inference-examples/blob/main/c_cxx/fns_candy_style_transfer/fns_candy_style_transfer.c
-#pragma once
-
-#include "rgb_sync.hpp"
-#include "rgb_image_math.hpp"
-#include "rgb_nms.hpp"
-
-#include <onnxruntime_c_api.h>
-
-#include <filesystem>
-#include <array>
-
-#undef HEAVY_SCRIBBLE
-
-// forward-declare
-struct OrtApi;
-struct OrtEnv;
-struct OrtMemoryInfo;
-struct OrtSession;
-struct OrtSessionOptions;
-struct OrtValue;
-
-namespace xrt::tracking::hand::old_rgb {
-
-
-// struct ht_device;
-
-class ht_model
-{
-	HandTracking *device = nullptr;
-
-	const OrtApi *api = nullptr;
-	OrtEnv *env = nullptr;
-
-	OrtMemoryInfo *palm_detection_meminfo = nullptr;
-	OrtSession *palm_detection_session = nullptr;
-	OrtValue *palm_detection_tensor = nullptr;
-	std::array<float, 3 * 128 * 128> palm_detection_data;
-
-	std::mutex hand_landmark_lock;
-	OrtMemoryInfo *hand_landmark_meminfo = nullptr;
-	OrtSession *hand_landmark_session = nullptr;
-	OrtValue *hand_landmark_tensor = nullptr;
-	std::array<float, 3 * 224 * 224> hand_landmark_data;
-
-	void
-	init_palm_detection(OrtSessionOptions *opts);
-	void
-	init_hand_landmark(OrtSessionOptions *opts);
-
-public:
-	ht_model(struct HandTracking *htd);
-	~ht_model();
-
-	std::vector<Palm7KP>
-	palm_detection(ht_view *htv, const cv::Mat &input);
-	Hand2D
-	hand_landmark(const cv::Mat input);
-};
-
-
-/*
- * Anchors data taken from mediapipe's palm detection, used for single-shot detector model.
- *
- * See:
- * https://google.github.io/mediapipe/solutions/hands.html#palm-detection-model
- * https://github.com/google/mediapipe/blob/v0.8.8/mediapipe/calculators/tflite/ssd_anchors_calculator.cc#L101
- * https://github.com/google/mediapipe/blob/v0.8.8/mediapipe/modules/palm_detection/palm_detection_cpu.pbtxt#L60
- */
-struct anchor
-{
-	float x, y;
-};
-
-static const struct anchor anchors[896]{
-    {0.031250, 0.031250}, {0.031250, 0.031250}, {0.093750, 0.031250}, {0.093750, 0.031250}, //
-    {0.156250, 0.031250}, {0.156250, 0.031250}, {0.218750, 0.031250}, {0.218750, 0.031250}, //
-    {0.281250, 0.031250}, {0.281250, 0.031250}, {0.343750, 0.031250}, {0.343750, 0.031250}, //
-    {0.406250, 0.031250}, {0.406250, 0.031250}, {0.468750, 0.031250}, {0.468750, 0.031250}, //
-    {0.531250, 0.031250}, {0.531250, 0.031250}, {0.593750, 0.031250}, {0.593750, 0.031250}, //
-    {0.656250, 0.031250}, {0.656250, 0.031250}, {0.718750, 0.031250}, {0.718750, 0.031250}, //
-    {0.781250, 0.031250}, {0.781250, 0.031250}, {0.843750, 0.031250}, {0.843750, 0.031250}, //
-    {0.906250, 0.031250}, {0.906250, 0.031250}, {0.968750, 0.031250}, {0.968750, 0.031250}, //
-    {0.031250, 0.093750}, {0.031250, 0.093750}, {0.093750, 0.093750}, {0.093750, 0.093750}, //
-    {0.156250, 0.093750}, {0.156250, 0.093750}, {0.218750, 0.093750}, {0.218750, 0.093750}, //
-    {0.281250, 0.093750}, {0.281250, 0.093750}, {0.343750, 0.093750}, {0.343750, 0.093750}, //
-    {0.406250, 0.093750}, {0.406250, 0.093750}, {0.468750, 0.093750}, {0.468750, 0.093750}, //
-    {0.531250, 0.093750}, {0.531250, 0.093750}, {0.593750, 0.093750}, {0.593750, 0.093750}, //
-    {0.656250, 0.093750}, {0.656250, 0.093750}, {0.718750, 0.093750}, {0.718750, 0.093750}, //
-    {0.781250, 0.093750}, {0.781250, 0.093750}, {0.843750, 0.093750}, {0.843750, 0.093750}, //
-    {0.906250, 0.093750}, {0.906250, 0.093750}, {0.968750, 0.093750}, {0.968750, 0.093750}, //
-    {0.031250, 0.156250}, {0.031250, 0.156250}, {0.093750, 0.156250}, {0.093750, 0.156250}, //
-    {0.156250, 0.156250}, {0.156250, 0.156250}, {0.218750, 0.156250}, {0.218750, 0.156250}, //
-    {0.281250, 0.156250}, {0.281250, 0.156250}, {0.343750, 0.156250}, {0.343750, 0.156250}, //
-    {0.406250, 0.156250}, {0.406250, 0.156250}, {0.468750, 0.156250}, {0.468750, 0.156250}, //
-    {0.531250, 0.156250}, {0.531250, 0.156250}, {0.593750, 0.156250}, {0.593750, 0.156250}, //
-    {0.656250, 0.156250}, {0.656250, 0.156250}, {0.718750, 0.156250}, {0.718750, 0.156250}, //
-    {0.781250, 0.156250}, {0.781250, 0.156250}, {0.843750, 0.156250}, {0.843750, 0.156250}, //
-    {0.906250, 0.156250}, {0.906250, 0.156250}, {0.968750, 0.156250}, {0.968750, 0.156250}, //
-    {0.031250, 0.218750}, {0.031250, 0.218750}, {0.093750, 0.218750}, {0.093750, 0.218750}, //
-    {0.156250, 0.218750}, {0.156250, 0.218750}, {0.218750, 0.218750}, {0.218750, 0.218750}, //
-    {0.281250, 0.218750}, {0.281250, 0.218750}, {0.343750, 0.218750}, {0.343750, 0.218750}, //
-    {0.406250, 0.218750}, {0.406250, 0.218750}, {0.468750, 0.218750}, {0.468750, 0.218750}, //
-    {0.531250, 0.218750}, {0.531250, 0.218750}, {0.593750, 0.218750}, {0.593750, 0.218750}, //
-    {0.656250, 0.218750}, {0.656250, 0.218750}, {0.718750, 0.218750}, {0.718750, 0.218750}, //
-    {0.781250, 0.218750}, {0.781250, 0.218750}, {0.843750, 0.218750}, {0.843750, 0.218750}, //
-    {0.906250, 0.218750}, {0.906250, 0.218750}, {0.968750, 0.218750}, {0.968750, 0.218750}, //
-    {0.031250, 0.281250}, {0.031250, 0.281250}, {0.093750, 0.281250}, {0.093750, 0.281250}, //
-    {0.156250, 0.281250}, {0.156250, 0.281250}, {0.218750, 0.281250}, {0.218750, 0.281250}, //
-    {0.281250, 0.281250}, {0.281250, 0.281250}, {0.343750, 0.281250}, {0.343750, 0.281250}, //
-    {0.406250, 0.281250}, {0.406250, 0.281250}, {0.468750, 0.281250}, {0.468750, 0.281250}, //
-    {0.531250, 0.281250}, {0.531250, 0.281250}, {0.593750, 0.281250}, {0.593750, 0.281250}, //
-    {0.656250, 0.281250}, {0.656250, 0.281250}, {0.718750, 0.281250}, {0.718750, 0.281250}, //
-    {0.781250, 0.281250}, {0.781250, 0.281250}, {0.843750, 0.281250}, {0.843750, 0.281250}, //
-    {0.906250, 0.281250}, {0.906250, 0.281250}, {0.968750, 0.281250}, {0.968750, 0.281250}, //
-    {0.031250, 0.343750}, {0.031250, 0.343750}, {0.093750, 0.343750}, {0.093750, 0.343750}, //
-    {0.156250, 0.343750}, {0.156250, 0.343750}, {0.218750, 0.343750}, {0.218750, 0.343750}, //
-    {0.281250, 0.343750}, {0.281250, 0.343750}, {0.343750, 0.343750}, {0.343750, 0.343750}, //
-    {0.406250, 0.343750}, {0.406250, 0.343750}, {0.468750, 0.343750}, {0.468750, 0.343750}, //
-    {0.531250, 0.343750}, {0.531250, 0.343750}, {0.593750, 0.343750}, {0.593750, 0.343750}, //
-    {0.656250, 0.343750}, {0.656250, 0.343750}, {0.718750, 0.343750}, {0.718750, 0.343750}, //
-    {0.781250, 0.343750}, {0.781250, 0.343750}, {0.843750, 0.343750}, {0.843750, 0.343750}, //
-    {0.906250, 0.343750}, {0.906250, 0.343750}, {0.968750, 0.343750}, {0.968750, 0.343750}, //
-    {0.031250, 0.406250}, {0.031250, 0.406250}, {0.093750, 0.406250}, {0.093750, 0.406250}, //
-    {0.156250, 0.406250}, {0.156250, 0.406250}, {0.218750, 0.406250}, {0.218750, 0.406250}, //
-    {0.281250, 0.406250}, {0.281250, 0.406250}, {0.343750, 0.406250}, {0.343750, 0.406250}, //
-    {0.406250, 0.406250}, {0.406250, 0.406250}, {0.468750, 0.406250}, {0.468750, 0.406250}, //
-    {0.531250, 0.406250}, {0.531250, 0.406250}, {0.593750, 0.406250}, {0.593750, 0.406250}, //
-    {0.656250, 0.406250}, {0.656250, 0.406250}, {0.718750, 0.406250}, {0.718750, 0.406250}, //
-    {0.781250, 0.406250}, {0.781250, 0.406250}, {0.843750, 0.406250}, {0.843750, 0.406250}, //
-    {0.906250, 0.406250}, {0.906250, 0.406250}, {0.968750, 0.406250}, {0.968750, 0.406250}, //
-    {0.031250, 0.468750}, {0.031250, 0.468750}, {0.093750, 0.468750}, {0.093750, 0.468750}, //
-    {0.156250, 0.468750}, {0.156250, 0.468750}, {0.218750, 0.468750}, {0.218750, 0.468750}, //
-    {0.281250, 0.468750}, {0.281250, 0.468750}, {0.343750, 0.468750}, {0.343750, 0.468750}, //
-    {0.406250, 0.468750}, {0.406250, 0.468750}, {0.468750, 0.468750}, {0.468750, 0.468750}, //
-    {0.531250, 0.468750}, {0.531250, 0.468750}, {0.593750, 0.468750}, {0.593750, 0.468750}, //
-    {0.656250, 0.468750}, {0.656250, 0.468750}, {0.718750, 0.468750}, {0.718750, 0.468750}, //
-    {0.781250, 0.468750}, {0.781250, 0.468750}, {0.843750, 0.468750}, {0.843750, 0.468750}, //
-    {0.906250, 0.468750}, {0.906250, 0.468750}, {0.968750, 0.468750}, {0.968750, 0.468750}, //
-    {0.031250, 0.531250}, {0.031250, 0.531250}, {0.093750, 0.531250}, {0.093750, 0.531250}, //
-    {0.156250, 0.531250}, {0.156250, 0.531250}, {0.218750, 0.531250}, {0.218750, 0.531250}, //
-    {0.281250, 0.531250}, {0.281250, 0.531250}, {0.343750, 0.531250}, {0.343750, 0.531250}, //
-    {0.406250, 0.531250}, {0.406250, 0.531250}, {0.468750, 0.531250}, {0.468750, 0.531250}, //
-    {0.531250, 0.531250}, {0.531250, 0.531250}, {0.593750, 0.531250}, {0.593750, 0.531250}, //
-    {0.656250, 0.531250}, {0.656250, 0.531250}, {0.718750, 0.531250}, {0.718750, 0.531250}, //
-    {0.781250, 0.531250}, {0.781250, 0.531250}, {0.843750, 0.531250}, {0.843750, 0.531250}, //
-    {0.906250, 0.531250}, {0.906250, 0.531250}, {0.968750, 0.531250}, {0.968750, 0.531250}, //
-    {0.031250, 0.593750}, {0.031250, 0.593750}, {0.093750, 0.593750}, {0.093750, 0.593750}, //
-    {0.156250, 0.593750}, {0.156250, 0.593750}, {0.218750, 0.593750}, {0.218750, 0.593750}, //
-    {0.281250, 0.593750}, {0.281250, 0.593750}, {0.343750, 0.593750}, {0.343750, 0.593750}, //
-    {0.406250, 0.593750}, {0.406250, 0.593750}, {0.468750, 0.593750}, {0.468750, 0.593750}, //
-    {0.531250, 0.593750}, {0.531250, 0.593750}, {0.593750, 0.593750}, {0.593750, 0.593750}, //
-    {0.656250, 0.593750}, {0.656250, 0.593750}, {0.718750, 0.593750}, {0.718750, 0.593750}, //
-    {0.781250, 0.593750}, {0.781250, 0.593750}, {0.843750, 0.593750}, {0.843750, 0.593750}, //
-    {0.906250, 0.593750}, {0.906250, 0.593750}, {0.968750, 0.593750}, {0.968750, 0.593750}, //
-    {0.031250, 0.656250}, {0.031250, 0.656250}, {0.093750, 0.656250}, {0.093750, 0.656250}, //
-    {0.156250, 0.656250}, {0.156250, 0.656250}, {0.218750, 0.656250}, {0.218750, 0.656250}, //
-    {0.281250, 0.656250}, {0.281250, 0.656250}, {0.343750, 0.656250}, {0.343750, 0.656250}, //
-    {0.406250, 0.656250}, {0.406250, 0.656250}, {0.468750, 0.656250}, {0.468750, 0.656250}, //
-    {0.531250, 0.656250}, {0.531250, 0.656250}, {0.593750, 0.656250}, {0.593750, 0.656250}, //
-    {0.656250, 0.656250}, {0.656250, 0.656250}, {0.718750, 0.656250}, {0.718750, 0.656250}, //
-    {0.781250, 0.656250}, {0.781250, 0.656250}, {0.843750, 0.656250}, {0.843750, 0.656250}, //
-    {0.906250, 0.656250}, {0.906250, 0.656250}, {0.968750, 0.656250}, {0.968750, 0.656250}, //
-    {0.031250, 0.718750}, {0.031250, 0.718750}, {0.093750, 0.718750}, {0.093750, 0.718750}, //
-    {0.156250, 0.718750}, {0.156250, 0.718750}, {0.218750, 0.718750}, {0.218750, 0.718750}, //
-    {0.281250, 0.718750}, {0.281250, 0.718750}, {0.343750, 0.718750}, {0.343750, 0.718750}, //
-    {0.406250, 0.718750}, {0.406250, 0.718750}, {0.468750, 0.718750}, {0.468750, 0.718750}, //
-    {0.531250, 0.718750}, {0.531250, 0.718750}, {0.593750, 0.718750}, {0.593750, 0.718750}, //
-    {0.656250, 0.718750}, {0.656250, 0.718750}, {0.718750, 0.718750}, {0.718750, 0.718750}, //
-    {0.781250, 0.718750}, {0.781250, 0.718750}, {0.843750, 0.718750}, {0.843750, 0.718750}, //
-    {0.906250, 0.718750}, {0.906250, 0.718750}, {0.968750, 0.718750}, {0.968750, 0.718750}, //
-    {0.031250, 0.781250}, {0.031250, 0.781250}, {0.093750, 0.781250}, {0.093750, 0.781250}, //
-    {0.156250, 0.781250}, {0.156250, 0.781250}, {0.218750, 0.781250}, {0.218750, 0.781250}, //
-    {0.281250, 0.781250}, {0.281250, 0.781250}, {0.343750, 0.781250}, {0.343750, 0.781250}, //
-    {0.406250, 0.781250}, {0.406250, 0.781250}, {0.468750, 0.781250}, {0.468750, 0.781250}, //
-    {0.531250, 0.781250}, {0.531250, 0.781250}, {0.593750, 0.781250}, {0.593750, 0.781250}, //
-    {0.656250, 0.781250}, {0.656250, 0.781250}, {0.718750, 0.781250}, {0.718750, 0.781250}, //
-    {0.781250, 0.781250}, {0.781250, 0.781250}, {0.843750, 0.781250}, {0.843750, 0.781250}, //
-    {0.906250, 0.781250}, {0.906250, 0.781250}, {0.968750, 0.781250}, {0.968750, 0.781250}, //
-    {0.031250, 0.843750}, {0.031250, 0.843750}, {0.093750, 0.843750}, {0.093750, 0.843750}, //
-    {0.156250, 0.843750}, {0.156250, 0.843750}, {0.218750, 0.843750}, {0.218750, 0.843750}, //
-    {0.281250, 0.843750}, {0.281250, 0.843750}, {0.343750, 0.843750}, {0.343750, 0.843750}, //
-    {0.406250, 0.843750}, {0.406250, 0.843750}, {0.468750, 0.843750}, {0.468750, 0.843750}, //
-    {0.531250, 0.843750}, {0.531250, 0.843750}, {0.593750, 0.843750}, {0.593750, 0.843750}, //
-    {0.656250, 0.843750}, {0.656250, 0.843750}, {0.718750, 0.843750}, {0.718750, 0.843750}, //
-    {0.781250, 0.843750}, {0.781250, 0.843750}, {0.843750, 0.843750}, {0.843750, 0.843750}, //
-    {0.906250, 0.843750}, {0.906250, 0.843750}, {0.968750, 0.843750}, {0.968750, 0.843750}, //
-    {0.031250, 0.906250}, {0.031250, 0.906250}, {0.093750, 0.906250}, {0.093750, 0.906250}, //
-    {0.156250, 0.906250}, {0.156250, 0.906250}, {0.218750, 0.906250}, {0.218750, 0.906250}, //
-    {0.281250, 0.906250}, {0.281250, 0.906250}, {0.343750, 0.906250}, {0.343750, 0.906250}, //
-    {0.406250, 0.906250}, {0.406250, 0.906250}, {0.468750, 0.906250}, {0.468750, 0.906250}, //
-    {0.531250, 0.906250}, {0.531250, 0.906250}, {0.593750, 0.906250}, {0.593750, 0.906250}, //
-    {0.656250, 0.906250}, {0.656250, 0.906250}, {0.718750, 0.906250}, {0.718750, 0.906250}, //
-    {0.781250, 0.906250}, {0.781250, 0.906250}, {0.843750, 0.906250}, {0.843750, 0.906250}, //
-    {0.906250, 0.906250}, {0.906250, 0.906250}, {0.968750, 0.906250}, {0.968750, 0.906250}, //
-    {0.031250, 0.968750}, {0.031250, 0.968750}, {0.093750, 0.968750}, {0.093750, 0.968750}, //
-    {0.156250, 0.968750}, {0.156250, 0.968750}, {0.218750, 0.968750}, {0.218750, 0.968750}, //
-    {0.281250, 0.968750}, {0.281250, 0.968750}, {0.343750, 0.968750}, {0.343750, 0.968750}, //
-    {0.406250, 0.968750}, {0.406250, 0.968750}, {0.468750, 0.968750}, {0.468750, 0.968750}, //
-    {0.531250, 0.968750}, {0.531250, 0.968750}, {0.593750, 0.968750}, {0.593750, 0.968750}, //
-    {0.656250, 0.968750}, {0.656250, 0.968750}, {0.718750, 0.968750}, {0.718750, 0.968750}, //
-    {0.781250, 0.968750}, {0.781250, 0.968750}, {0.843750, 0.968750}, {0.843750, 0.968750}, //
-    {0.906250, 0.968750}, {0.906250, 0.968750}, {0.968750, 0.968750}, {0.968750, 0.968750}, //
-    {0.062500, 0.062500}, {0.062500, 0.062500}, {0.062500, 0.062500}, {0.062500, 0.062500}, //
-    {0.062500, 0.062500}, {0.062500, 0.062500}, {0.187500, 0.062500}, {0.187500, 0.062500}, //
-    {0.187500, 0.062500}, {0.187500, 0.062500}, {0.187500, 0.062500}, {0.187500, 0.062500}, //
-    {0.312500, 0.062500}, {0.312500, 0.062500}, {0.312500, 0.062500}, {0.312500, 0.062500}, //
-    {0.312500, 0.062500}, {0.312500, 0.062500}, {0.437500, 0.062500}, {0.437500, 0.062500}, //
-    {0.437500, 0.062500}, {0.437500, 0.062500}, {0.437500, 0.062500}, {0.437500, 0.062500}, //
-    {0.562500, 0.062500}, {0.562500, 0.062500}, {0.562500, 0.062500}, {0.562500, 0.062500}, //
-    {0.562500, 0.062500}, {0.562500, 0.062500}, {0.687500, 0.062500}, {0.687500, 0.062500}, //
-    {0.687500, 0.062500}, {0.687500, 0.062500}, {0.687500, 0.062500}, {0.687500, 0.062500}, //
-    {0.812500, 0.062500}, {0.812500, 0.062500}, {0.812500, 0.062500}, {0.812500, 0.062500}, //
-    {0.812500, 0.062500}, {0.812500, 0.062500}, {0.937500, 0.062500}, {0.937500, 0.062500}, //
-    {0.937500, 0.062500}, {0.937500, 0.062500}, {0.937500, 0.062500}, {0.937500, 0.062500}, //
-    {0.062500, 0.187500}, {0.062500, 0.187500}, {0.062500, 0.187500}, {0.062500, 0.187500}, //
-    {0.062500, 0.187500}, {0.062500, 0.187500}, {0.187500, 0.187500}, {0.187500, 0.187500}, //
-    {0.187500, 0.187500}, {0.187500, 0.187500}, {0.187500, 0.187500}, {0.187500, 0.187500}, //
-    {0.312500, 0.187500}, {0.312500, 0.187500}, {0.312500, 0.187500}, {0.312500, 0.187500}, //
-    {0.312500, 0.187500}, {0.312500, 0.187500}, {0.437500, 0.187500}, {0.437500, 0.187500}, //
-    {0.437500, 0.187500}, {0.437500, 0.187500}, {0.437500, 0.187500}, {0.437500, 0.187500}, //
-    {0.562500, 0.187500}, {0.562500, 0.187500}, {0.562500, 0.187500}, {0.562500, 0.187500}, //
-    {0.562500, 0.187500}, {0.562500, 0.187500}, {0.687500, 0.187500}, {0.687500, 0.187500}, //
-    {0.687500, 0.187500}, {0.687500, 0.187500}, {0.687500, 0.187500}, {0.687500, 0.187500}, //
-    {0.812500, 0.187500}, {0.812500, 0.187500}, {0.812500, 0.187500}, {0.812500, 0.187500}, //
-    {0.812500, 0.187500}, {0.812500, 0.187500}, {0.937500, 0.187500}, {0.937500, 0.187500}, //
-    {0.937500, 0.187500}, {0.937500, 0.187500}, {0.937500, 0.187500}, {0.937500, 0.187500}, //
-    {0.062500, 0.312500}, {0.062500, 0.312500}, {0.062500, 0.312500}, {0.062500, 0.312500}, //
-    {0.062500, 0.312500}, {0.062500, 0.312500}, {0.187500, 0.312500}, {0.187500, 0.312500}, //
-    {0.187500, 0.312500}, {0.187500, 0.312500}, {0.187500, 0.312500}, {0.187500, 0.312500}, //
-    {0.312500, 0.312500}, {0.312500, 0.312500}, {0.312500, 0.312500}, {0.312500, 0.312500}, //
-    {0.312500, 0.312500}, {0.312500, 0.312500}, {0.437500, 0.312500}, {0.437500, 0.312500}, //
-    {0.437500, 0.312500}, {0.437500, 0.312500}, {0.437500, 0.312500}, {0.437500, 0.312500}, //
-    {0.562500, 0.312500}, {0.562500, 0.312500}, {0.562500, 0.312500}, {0.562500, 0.312500}, //
-    {0.562500, 0.312500}, {0.562500, 0.312500}, {0.687500, 0.312500}, {0.687500, 0.312500}, //
-    {0.687500, 0.312500}, {0.687500, 0.312500}, {0.687500, 0.312500}, {0.687500, 0.312500}, //
-    {0.812500, 0.312500}, {0.812500, 0.312500}, {0.812500, 0.312500}, {0.812500, 0.312500}, //
-    {0.812500, 0.312500}, {0.812500, 0.312500}, {0.937500, 0.312500}, {0.937500, 0.312500}, //
-    {0.937500, 0.312500}, {0.937500, 0.312500}, {0.937500, 0.312500}, {0.937500, 0.312500}, //
-    {0.062500, 0.437500}, {0.062500, 0.437500}, {0.062500, 0.437500}, {0.062500, 0.437500}, //
-    {0.062500, 0.437500}, {0.062500, 0.437500}, {0.187500, 0.437500}, {0.187500, 0.437500}, //
-    {0.187500, 0.437500}, {0.187500, 0.437500}, {0.187500, 0.437500}, {0.187500, 0.437500}, //
-    {0.312500, 0.437500}, {0.312500, 0.437500}, {0.312500, 0.437500}, {0.312500, 0.437500}, //
-    {0.312500, 0.437500}, {0.312500, 0.437500}, {0.437500, 0.437500}, {0.437500, 0.437500}, //
-    {0.437500, 0.437500}, {0.437500, 0.437500}, {0.437500, 0.437500}, {0.437500, 0.437500}, //
-    {0.562500, 0.437500}, {0.562500, 0.437500}, {0.562500, 0.437500}, {0.562500, 0.437500}, //
-    {0.562500, 0.437500}, {0.562500, 0.437500}, {0.687500, 0.437500}, {0.687500, 0.437500}, //
-    {0.687500, 0.437500}, {0.687500, 0.437500}, {0.687500, 0.437500}, {0.687500, 0.437500}, //
-    {0.812500, 0.437500}, {0.812500, 0.437500}, {0.812500, 0.437500}, {0.812500, 0.437500}, //
-    {0.812500, 0.437500}, {0.812500, 0.437500}, {0.937500, 0.437500}, {0.937500, 0.437500}, //
-    {0.937500, 0.437500}, {0.937500, 0.437500}, {0.937500, 0.437500}, {0.937500, 0.437500}, //
-    {0.062500, 0.562500}, {0.062500, 0.562500}, {0.062500, 0.562500}, {0.062500, 0.562500}, //
-    {0.062500, 0.562500}, {0.062500, 0.562500}, {0.187500, 0.562500}, {0.187500, 0.562500}, //
-    {0.187500, 0.562500}, {0.187500, 0.562500}, {0.187500, 0.562500}, {0.187500, 0.562500}, //
-    {0.312500, 0.562500}, {0.312500, 0.562500}, {0.312500, 0.562500}, {0.312500, 0.562500}, //
-    {0.312500, 0.562500}, {0.312500, 0.562500}, {0.437500, 0.562500}, {0.437500, 0.562500}, //
-    {0.437500, 0.562500}, {0.437500, 0.562500}, {0.437500, 0.562500}, {0.437500, 0.562500}, //
-    {0.562500, 0.562500}, {0.562500, 0.562500}, {0.562500, 0.562500}, {0.562500, 0.562500}, //
-    {0.562500, 0.562500}, {0.562500, 0.562500}, {0.687500, 0.562500}, {0.687500, 0.562500}, //
-    {0.687500, 0.562500}, {0.687500, 0.562500}, {0.687500, 0.562500}, {0.687500, 0.562500}, //
-    {0.812500, 0.562500}, {0.812500, 0.562500}, {0.812500, 0.562500}, {0.812500, 0.562500}, //
-    {0.812500, 0.562500}, {0.812500, 0.562500}, {0.937500, 0.562500}, {0.937500, 0.562500}, //
-    {0.937500, 0.562500}, {0.937500, 0.562500}, {0.937500, 0.562500}, {0.937500, 0.562500}, //
-    {0.062500, 0.687500}, {0.062500, 0.687500}, {0.062500, 0.687500}, {0.062500, 0.687500}, //
-    {0.062500, 0.687500}, {0.062500, 0.687500}, {0.187500, 0.687500}, {0.187500, 0.687500}, //
-    {0.187500, 0.687500}, {0.187500, 0.687500}, {0.187500, 0.687500}, {0.187500, 0.687500}, //
-    {0.312500, 0.687500}, {0.312500, 0.687500}, {0.312500, 0.687500}, {0.312500, 0.687500}, //
-    {0.312500, 0.687500}, {0.312500, 0.687500}, {0.437500, 0.687500}, {0.437500, 0.687500}, //
-    {0.437500, 0.687500}, {0.437500, 0.687500}, {0.437500, 0.687500}, {0.437500, 0.687500}, //
-    {0.562500, 0.687500}, {0.562500, 0.687500}, {0.562500, 0.687500}, {0.562500, 0.687500}, //
-    {0.562500, 0.687500}, {0.562500, 0.687500}, {0.687500, 0.687500}, {0.687500, 0.687500}, //
-    {0.687500, 0.687500}, {0.687500, 0.687500}, {0.687500, 0.687500}, {0.687500, 0.687500}, //
-    {0.812500, 0.687500}, {0.812500, 0.687500}, {0.812500, 0.687500}, {0.812500, 0.687500}, //
-    {0.812500, 0.687500}, {0.812500, 0.687500}, {0.937500, 0.687500}, {0.937500, 0.687500}, //
-    {0.937500, 0.687500}, {0.937500, 0.687500}, {0.937500, 0.687500}, {0.937500, 0.687500}, //
-    {0.062500, 0.812500}, {0.062500, 0.812500}, {0.062500, 0.812500}, {0.062500, 0.812500}, //
-    {0.062500, 0.812500}, {0.062500, 0.812500}, {0.187500, 0.812500}, {0.187500, 0.812500}, //
-    {0.187500, 0.812500}, {0.187500, 0.812500}, {0.187500, 0.812500}, {0.187500, 0.812500}, //
-    {0.312500, 0.812500}, {0.312500, 0.812500}, {0.312500, 0.812500}, {0.312500, 0.812500}, //
-    {0.312500, 0.812500}, {0.312500, 0.812500}, {0.437500, 0.812500}, {0.437500, 0.812500}, //
-    {0.437500, 0.812500}, {0.437500, 0.812500}, {0.437500, 0.812500}, {0.437500, 0.812500}, //
-    {0.562500, 0.812500}, {0.562500, 0.812500}, {0.562500, 0.812500}, {0.562500, 0.812500}, //
-    {0.562500, 0.812500}, {0.562500, 0.812500}, {0.687500, 0.812500}, {0.687500, 0.812500}, //
-    {0.687500, 0.812500}, {0.687500, 0.812500}, {0.687500, 0.812500}, {0.687500, 0.812500}, //
-    {0.812500, 0.812500}, {0.812500, 0.812500}, {0.812500, 0.812500}, {0.812500, 0.812500}, //
-    {0.812500, 0.812500}, {0.812500, 0.812500}, {0.937500, 0.812500}, {0.937500, 0.812500}, //
-    {0.937500, 0.812500}, {0.937500, 0.812500}, {0.937500, 0.812500}, {0.937500, 0.812500}, //
-    {0.062500, 0.937500}, {0.062500, 0.937500}, {0.062500, 0.937500}, {0.062500, 0.937500}, //
-    {0.062500, 0.937500}, {0.062500, 0.937500}, {0.187500, 0.937500}, {0.187500, 0.937500}, //
-    {0.187500, 0.937500}, {0.187500, 0.937500}, {0.187500, 0.937500}, {0.187500, 0.937500}, //
-    {0.312500, 0.937500}, {0.312500, 0.937500}, {0.312500, 0.937500}, {0.312500, 0.937500}, //
-    {0.312500, 0.937500}, {0.312500, 0.937500}, {0.437500, 0.937500}, {0.437500, 0.937500}, //
-    {0.437500, 0.937500}, {0.437500, 0.937500}, {0.437500, 0.937500}, {0.437500, 0.937500}, //
-    {0.562500, 0.937500}, {0.562500, 0.937500}, {0.562500, 0.937500}, {0.562500, 0.937500}, //
-    {0.562500, 0.937500}, {0.562500, 0.937500}, {0.687500, 0.937500}, {0.687500, 0.937500}, //
-    {0.687500, 0.937500}, {0.687500, 0.937500}, {0.687500, 0.937500}, {0.687500, 0.937500}, //
-    {0.812500, 0.937500}, {0.812500, 0.937500}, {0.812500, 0.937500}, {0.812500, 0.937500}, //
-    {0.812500, 0.937500}, {0.812500, 0.937500}, {0.937500, 0.937500}, {0.937500, 0.937500}, //
-    {0.937500, 0.937500}, {0.937500, 0.937500}, {0.937500, 0.937500}, {0.937500, 0.937500}, //
-};
-
-#define ORT(expr)                                                                                                      \
-	do {                                                                                                           \
-		OrtStatus *status = this->api->expr;                                                                   \
-		if (status != nullptr) {                                                                               \
-			const char *msg = this->api->GetErrorMessage(status);                                          \
-			HT_ERROR(this->device, "[%s:%d]: %s\n", __FILE__, __LINE__, msg);                              \
-			this->api->ReleaseStatus(status);                                                              \
-			assert(false);                                                                                 \
-		}                                                                                                      \
-	} while (0)
-
-void
-ht_model::init_palm_detection(OrtSessionOptions *opts)
-{
-	// Both models have slightly different shapes, preventing us to constexpr the input shape
-	std::array<int64_t, 4> input_shape;
-	std::array<std::string, 1> input_names;
-
-	std::filesystem::path path = this->device->startup_config.model_slug;
-	if (this->device->startup_config.keypoint_estimation_use_mediapipe) {
-		path /= "palm_detection_MEDIAPIPE.onnx";
-
-		input_shape = {1, 3, 128, 128};
-		input_names = {"input"};
-	} else {
-		path /= "palm_detection_COLLABORA.onnx";
-
-		input_shape = {1, 128, 128, 3};
-		input_names = {"input:0"};
-	}
-
-	HT_DEBUG(this->device, "Loading palm detection model from file '%s'", path.c_str());
-	ORT(CreateSession(this->env, path.c_str(), opts, &this->palm_detection_session));
-	assert(this->palm_detection_session);
-
-	constexpr size_t input_size = 3 * 128 * 128;
-
-	ORT(CreateTensorWithDataAsOrtValue(this->palm_detection_meminfo, this->palm_detection_data.data(),
-	                                   input_size * sizeof(float), input_shape.data(), input_shape.size(),
-	                                   ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, &this->palm_detection_tensor));
-
-	assert(this->palm_detection_tensor);
-	int is_tensor;
-	ORT(IsTensor(this->palm_detection_tensor, &is_tensor));
-	assert(is_tensor);
-}
-
-void
-ht_model::init_hand_landmark(OrtSessionOptions *opts)
-{
-	std::filesystem::path path = this->device->startup_config.model_slug;
-	if (this->device->startup_config.keypoint_estimation_use_mediapipe) {
-		path /= "hand_landmark_MEDIAPIPE.onnx";
-	} else {
-		path /= "hand_landmark_COLLABORA.onnx";
-	}
-
-	HT_DEBUG(this->device, "Loading hand landmark model from file '%s'", path.c_str());
-	ORT(CreateSession(this->env, path.c_str(), opts, &this->hand_landmark_session));
-	assert(this->hand_landmark_session);
-
-	constexpr size_t input_size = 3 * 224 * 224;
-
-	constexpr std::array<int64_t, 4> input_shape = {1, 3, 224, 224};
-	ORT(CreateTensorWithDataAsOrtValue(this->hand_landmark_meminfo, this->hand_landmark_data.data(),
-	                                   input_size * sizeof(float), input_shape.data(), input_shape.size(),
-	                                   ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, &this->hand_landmark_tensor));
-
-	assert(this->hand_landmark_tensor != nullptr);
-	int is_tensor;
-	ORT(IsTensor(hand_landmark_tensor, &is_tensor));
-	assert(is_tensor);
-}
-
-ht_model::ht_model(struct HandTracking *htd) : device(htd), api(OrtGetApiBase()->GetApi(ORT_API_VERSION))
-{
-	ORT(CreateEnv(ORT_LOGGING_LEVEL_WARNING, "monado_ht", &this->env));
-
-	ORT(CreateCpuMemoryInfo(OrtArenaAllocator, OrtMemTypeDefault, &this->palm_detection_meminfo));
-	ORT(CreateCpuMemoryInfo(OrtArenaAllocator, OrtMemTypeDefault, &this->hand_landmark_meminfo));
-
-	OrtSessionOptions *opts = nullptr;
-	ORT(CreateSessionOptions(&opts));
-
-	// TODO review options, config for threads?
-	ORT(SetSessionGraphOptimizationLevel(opts, ORT_ENABLE_ALL));
-	ORT(SetIntraOpNumThreads(opts, 1));
-
-	this->init_palm_detection(opts);
-	this->init_hand_landmark(opts);
-
-	this->api->ReleaseSessionOptions(opts);
-}
-
-
-ht_model::~ht_model()
-{
-	this->api->ReleaseMemoryInfo(this->palm_detection_meminfo);
-	this->api->ReleaseSession(this->palm_detection_session);
-	this->api->ReleaseValue(this->palm_detection_tensor);
-
-	this->api->ReleaseMemoryInfo(this->hand_landmark_meminfo);
-	this->api->ReleaseSession(this->hand_landmark_session);
-	this->api->ReleaseValue(this->hand_landmark_tensor);
-
-	this->api->ReleaseEnv(this->env);
-}
-
-std::vector<Palm7KP>
-ht_model::palm_detection(ht_view *htv, const cv::Mat &input)
-{
-	// TODO use opencv to handle input preprocessing
-	constexpr int hd_size = 128;
-	constexpr size_t nb_planes = 3;
-	constexpr size_t size = hd_size * hd_size * nb_planes;
-
-	cv::Mat img;
-	cv::Matx23f back_from_blackbar = blackbar(input, img, {hd_size, hd_size});
-
-	float scale_factor = back_from_blackbar(0, 0); // 960/128
-	assert(img.isContinuous());
-	constexpr float mean = 128.0f;
-	constexpr float std = 128.0f;
-
-	if (htv->htd->startup_config.palm_detection_use_mediapipe) {
-		std::vector<uint8_t> combined_planes(size);
-		planarize(img, combined_planes.data());
-		for (size_t i = 0; i < size; i++) {
-			float val = (float)combined_planes[i];
-			this->palm_detection_data[i] = (val - mean) / std;
-		}
-	} else {
-
-		assert(img.isContinuous());
-
-		for (size_t i = 0; i < size; i++) {
-			int val = img.data[i];
-
-			this->palm_detection_data[i] = (val - mean) / std;
-		}
-	}
-
-	const char *input_names[1];
-	if (this->device->startup_config.keypoint_estimation_use_mediapipe) {
-		input_names[0] = "input";
-	} else {
-		input_names[0] = "input:0";
-	}
-
-	static const char *const output_names[] = {"classificators", "regressors"};
-
-	OrtValue *output_tensor[] = {nullptr, nullptr};
-	ORT(Run(this->palm_detection_session, nullptr, input_names, &this->palm_detection_tensor, 1, output_names, 2,
-	        output_tensor));
-
-	// TODO define types to handle data
-	float *classificators = nullptr;
-	float *regressors = nullptr;
-
-	// Output is 896 floats
-	ORT(GetTensorMutableData(output_tensor[0], (void **)&classificators));
-
-	// Output is 896 * 18 floats
-	ORT(GetTensorMutableData(output_tensor[1], (void **)&regressors));
-
-	std::vector<NMSPalm> detections;
-	for (size_t i = 0; i < 896; ++i) {
-		const float score = 1.0 / (1.0 + exp(-classificators[i]));
-
-		// Let a lot of detections in - they'll be slowly rejected later
-		if (score <= this->device->dynamic_config.nms_threshold.val) {
-			continue;
-		}
-
-		const struct anchor *anchor = &anchors[i];
-
-		// Boundary box.
-		NMSPalm det;
-
-		float anchx = anchor->x * 128;
-		float anchy = anchor->y * 128;
-
-		float shiftx = regressors[i * 18];
-		float shifty = regressors[i * 18 + 1];
-
-		float w = regressors[i * 18 + 2];
-		float h = regressors[i * 18 + 3];
-
-		float cx = shiftx + anchx;
-		float cy = shifty + anchy;
-
-		struct xrt_vec2 *kps = det.keypoints;
-
-		kps[0] = {regressors[i * 18 + 4], regressors[i * 18 + 5]};
-		kps[1] = {regressors[i * 18 + 6], regressors[i * 18 + 7]};
-		kps[2] = {regressors[i * 18 + 8], regressors[i * 18 + 9]};
-		kps[3] = {regressors[i * 18 + 10], regressors[i * 18 + 11]};
-		kps[4] = {regressors[i * 18 + 12], regressors[i * 18 + 13]};
-		kps[5] = {regressors[i * 18 + 14], regressors[i * 18 + 15]};
-		kps[6] = {regressors[i * 18 + 16], regressors[i * 18 + 17]};
-
-
-		for (int i = 0; i < 7; i++) {
-			struct xrt_vec2 *b = &kps[i];
-			b->x += anchx;
-			b->y += anchy;
-		}
-
-		det.bbox.w = w;
-		det.bbox.h = h;
-		det.bbox.cx = cx;
-		det.bbox.cy = cy;
-		det.confidence = score;
-		detections.push_back(det);
-
-		if (htv->htd->debug_scribble && (htv->htd->dynamic_config.scribble_raw_detections)) {
-			xrt_vec2 center = transformVecBy2x3(xrt_vec2{cx, cy}, back_from_blackbar);
-
-			float sz = det.bbox.w * scale_factor;
-
-			cv::rectangle(htv->debug_out_to_this,
-			              {(int)(center.x - (sz / 2)), (int)(center.y - (sz / 2)), (int)sz, (int)sz},
-			              hsv2rgb(0.0f, math_map_ranges(det.confidence, 0.0f, 1.0f, 1.5f, -0.1f),
-			                      math_map_ranges(det.confidence, 0.0f, 1.0f, 0.2f, 1.4f)),
-			              1);
-
-			for (int i = 0; i < 7; i++) {
-				handDot(htv->debug_out_to_this, transformVecBy2x3(kps[i], back_from_blackbar),
-				        det.confidence * 7, ((float)i) * (360.0f / 7.0f), det.confidence, 1);
-			}
-		}
-	}
-
-	this->api->ReleaseValue(output_tensor[0]);
-	this->api->ReleaseValue(output_tensor[1]);
-
-	std::vector<Palm7KP> output;
-	if (detections.empty()) {
-		return output;
-	}
-
-	std::vector<NMSPalm> nms_palms = filterBoxesWeightedAvg(detections, htv->htd->dynamic_config.nms_iou.val);
-
-	for (const NMSPalm &cooler : nms_palms) {
-
-		// Display box
-
-		struct xrt_vec2 tl = {cooler.bbox.cx - cooler.bbox.w / 2, cooler.bbox.cy - cooler.bbox.h / 2};
-		struct xrt_vec2 bob = transformVecBy2x3(tl, back_from_blackbar);
-		float sz = cooler.bbox.w * scale_factor;
-
-		if (htv->htd->debug_scribble && htv->htd->dynamic_config.scribble_nms_detections) {
-			cv::rectangle(htv->debug_out_to_this, {(int)bob.x, (int)bob.y, (int)sz, (int)sz},
-			              hsv2rgb(180.0f, math_map_ranges(cooler.confidence, 0.0f, 1.0f, 0.8f, -0.1f),
-			                      math_map_ranges(cooler.confidence, 0.0f, 1.0f, 0.2f, 1.4f)),
-			              2);
-			for (int i = 0; i < 7; i++) {
-				handDot(htv->debug_out_to_this,
-				        transformVecBy2x3(cooler.keypoints[i], back_from_blackbar),
-				        cooler.confidence * 14, ((float)i) * (360.0f / 7.0f), cooler.confidence, 3);
-			}
-		}
-
-
-		Palm7KP this_element;
-
-		for (int i = 0; i < 7; i++) {
-			struct xrt_vec2 b = cooler.keypoints[i];
-			this_element.kps[i] = transformVecBy2x3(b, back_from_blackbar);
-		}
-		this_element.confidence = cooler.confidence;
-
-		output.push_back(this_element);
-	}
-
-
-	return output;
-}
-
-Hand2D
-ht_model::hand_landmark(const cv::Mat input)
-{
-	std::scoped_lock lock(this->hand_landmark_lock);
-
-	// TODO use opencv to handle input preprocessing
-	constexpr size_t lix = 224;
-	constexpr size_t liy = 224;
-	constexpr size_t nb_planes = 3;
-	cv::Mat planes[nb_planes];
-
-	constexpr size_t size = lix * liy * nb_planes;
-
-	std::vector<uint8_t> combined_planes(size);
-	planarize(input, combined_planes.data());
-
-	// Normalize - supposedly, the keypoint estimator wants keypoints in [0,1]
-	for (size_t i = 0; i < size; i++) {
-		this->hand_landmark_data[i] = (float)combined_planes[i] / 255.0;
-	}
-
-	static const char *const input_names[] = {"input_1"};
-	static const char *const output_names[] = {"Identity", "Identity_1", "Identity_2"};
-
-	OrtValue *output_tensor[] = {nullptr, nullptr, nullptr};
-	ORT(Run(this->hand_landmark_session, nullptr, input_names, &this->hand_landmark_tensor, 1, output_names, 3,
-	        output_tensor));
-
-	Hand2D hand{};
-
-	float *landmarks = nullptr;
-
-	// Should give a pointer to data that is freed on g_ort->ReleaseValue(output_tensor[0]);.
-	ORT(GetTensorMutableData(output_tensor[0], (void **)&landmarks));
-
-	constexpr int stride = 3;
-	for (size_t i = 0; i < 21; i++) {
-		int rt = i * stride;
-		float x = landmarks[rt];
-		float y = landmarks[rt + 1];
-		float z = landmarks[rt + 2];
-		hand.kps[i].x = x;
-		hand.kps[i].y = y;
-		hand.kps[i].z = z;
-	}
-
-	this->api->ReleaseValue(output_tensor[0]);
-	this->api->ReleaseValue(output_tensor[1]);
-	this->api->ReleaseValue(output_tensor[2]);
-
-	return hand;
-}
-
-} // namespace xrt::tracking::hand::old_rgb

src/xrt/tracking/hand/old_rgb/rgb_nms.hpp

···
-// Copyright 2021-2022, Collabora, Ltd.
-// SPDX-License-Identifier: BSL-1.0
-/*!
- * @file
- * @brief  Code to deal with bounding boxes for camera-based hand-tracking.
- * @author Moses Turner <[e-mail]>
- * @author Marcus Edel <[e-mail]>
- * @ingroup drv_ht
- */
-
-#include "rgb_sync.hpp"
-#include <math.h>
-
-#include "util/u_box_iou.hpp"
-
-using namespace xrt::auxiliary::util::box_iou;
-struct NMSPalm
-{
-	Box bbox;
-	struct xrt_vec2 keypoints[7];
-	float confidence;
-};
-
-
-
-static NMSPalm
-weightedAvgBoxes(const std::vector<NMSPalm> &detections)
-{
-	float weight = 0.0f; // or, sum_confidences.
-	float cx = 0.0f;
-	float cy = 0.0f;
-	float size = 0.0f;
-	NMSPalm out = {};
-
-	for (const NMSPalm &detection : detections) {
-		weight += detection.confidence;
-		cx += detection.bbox.cx * detection.confidence;
-		cy += detection.bbox.cy * detection.confidence;
-		size += detection.bbox.w * .5 * detection.confidence;
-		size += detection.bbox.h * .5 * detection.confidence;
-
-		for (int i = 0; i < 7; i++) {
-			out.keypoints[i].x += detection.keypoints[i].x * detection.confidence;
-			out.keypoints[i].y += detection.keypoints[i].y * detection.confidence;
-		}
-	}
-	cx /= weight;
-	cy /= weight;
-	size /= weight;
-	for (int i = 0; i < 7; i++) {
-		out.keypoints[i].x /= weight;
-		out.keypoints[i].y /= weight;
-	}
-
-
-	float bare_confidence = weight / detections.size();
-
-	// desmos \frac{1}{1+e^{-.5x}}-.5
-
-	float steep = 0.2;
-	float cent = 0.5;
-
-	float exp = detections.size();
-
-	float sigmoid_addendum = (1.0f / (1.0f + pow(M_E, (-steep * exp)))) - cent;
-
-	float diff_bare_to_one = 1.0f - bare_confidence;
-
-	out.confidence = bare_confidence + (sigmoid_addendum * diff_bare_to_one);
-
-	// U_LOG_E("Bare %f num %f sig %f diff %f out %f", bare_confidence, exp, sigmoid_addendum, diff_bare_to_one,
-	// out.confidence);
-
-	out.bbox.cx = cx;
-	out.bbox.cy = cy;
-	out.bbox.w = size;
-	out.bbox.h = size;
-	return out;
-}
-
-std::vector<NMSPalm>
-filterBoxesWeightedAvg(const std::vector<NMSPalm> &detections, float min_iou)
-{
-	std::vector<std::vector<NMSPalm>> overlaps;
-	std::vector<NMSPalm> outs;
-
-	// U_LOG_D("\n\nStarting filtering boxes. There are %zu boxes to look at.\n", detections.size());
-	for (const NMSPalm &detection : detections) {
-		// U_LOG_D("Starting looking at one detection\n");
-		bool foundAHome = false;
-		for (size_t i = 0; i < outs.size(); i++) {
-			float iou = boxIOU(outs[i].bbox, detection.bbox);
-			// U_LOG_D("IOU is %f\n", iou);
-			// U_LOG_D("Outs box is %f %f %f %f", outs[i].bbox.cx, outs[i].bbox.cy, outs[i].bbox.w,
-			// outs[i].bbox.h)
-			if (iou > min_iou) {
-				// This one intersects with the whole thing
-				overlaps[i].push_back(detection);
-				outs[i] = weightedAvgBoxes(overlaps[i]);
-				foundAHome = true;
-				break;
-			}
-		}
-		if (!foundAHome) {
-			// U_LOG_D("No home\n");
-			overlaps.push_back({detection});
-			outs.push_back({detection});
-		} else {
-			// U_LOG_D("Found a home!\n");
-		}
-	}
-	// U_LOG_D("Sizeeeeeeeeeeeeeeeeeeeee is %zu\n", outs.size());
-	return outs;
-}

src/xrt/tracking/hand/old_rgb/rgb_sync.cpp

···
-// Copyright 2022, Collabora, Ltd.
-// SPDX-License-Identifier: BSL-1.0
-/*!
- * @file
- * @brief  Old RGB hand tracking main file.
- * @author Jakob Bornecrantz <[e-mail]>
- * @ingroup aux_tracking
- */
-
-#include "rgb_interface.h"
-#include "rgb_sync.hpp"
-#include "xrt/xrt_frame.h"
-
-
-using namespace xrt::tracking::hand::old_rgb;
-
-
-
-#include "xrt/xrt_defines.h"
-
-#include "math/m_vec2.h"
-#include "util/u_frame.h"
-#include "util/u_trace_marker.h"
-
-
-#include "templates/NaivePermutationSort.hpp"
-
-#include <future>
-
-
-// Copyright 2021, Collabora, Ltd.
-// SPDX-License-Identifier: BSL-1.0
-/*!
- * @file
- * @brief  Camera based hand tracking driver code.
- * @author Moses Turner <[e-mail]>
- * @author Jakob Bornecrantz <[e-mail]>
- * @ingroup drv_ht
- */
-
-#if defined(EXPERIMENTAL_DATASET_RECORDING)
-#include "gstreamer/gst_pipeline.h"
-#include "gstreamer/gst_sink.h"
-#endif
-
-#include "xrt/xrt_defines.h"
-#include "xrt/xrt_frame.h"
-#include "xrt/xrt_frameserver.h"
-
-#include "os/os_time.h"
-#include "os/os_threading.h"
-
-#include "math/m_api.h"
-#include "math/m_eigen_interop.hpp"
-
-#include "util/u_device.h"
-#include "util/u_frame.h"
-#include "util/u_hand_tracking.h"
-#include "util/u_sink.h"
-#include "util/u_format.h"
-#include "util/u_logging.h"
-#include "util/u_time.h"
-#include "util/u_trace_marker.h"
-#include "util/u_time.h"
-#include "util/u_json.h"
-#include "util/u_config_json.h"
-
-#include "tracking/t_frame_cv_mat_wrapper.hpp"
-#include "tracking/t_calibration_opencv.hpp"
-
-#include "rgb_hand_math.hpp"
-#include "rgb_image_math.hpp"
-#include "rgb_model.hpp"
-
-#include <cjson/cJSON.h>
-#include <opencv2/core/mat.hpp>
-#include <opencv2/calib3d.hpp>
-
-#include <math.h>
-#include <float.h>
-#include <stdio.h>
-#include <unistd.h>
-#include <string.h>
-
-#include <cmath>
-
-#include <limits>
-#include <thread>
-#include <future>
-#include <fstream>
-#include <numeric>
-#include <sstream>
-#include <iostream>
-#include <exception>
-#include <algorithm>
-
-
-
-// Flags to tell state tracker that these are indeed valid joints
-static const enum xrt_space_relation_flags valid_flags_ht = (enum xrt_space_relation_flags)(
-    XRT_SPACE_RELATION_ORIENTATION_VALID_BIT | XRT_SPACE_RELATION_ORIENTATION_TRACKED_BIT |
-    XRT_SPACE_RELATION_POSITION_VALID_BIT | XRT_SPACE_RELATION_POSITION_TRACKED_BIT);
-
-
-static void
-htProcessJoint(struct HandTracking *htd,
-               struct xrt_vec3 model_out,
-               struct xrt_hand_joint_set *hand,
-               enum xrt_hand_joint idx)
-{
-	hand->values.hand_joint_set_default[idx].relation.relation_flags = valid_flags_ht;
-	hand->values.hand_joint_set_default[idx].relation.pose.position.x = model_out.x;
-	hand->values.hand_joint_set_default[idx].relation.pose.position.y = model_out.y;
-	hand->values.hand_joint_set_default[idx].relation.pose.position.z = model_out.z;
-}
-
-static float
-errHistory2D(const HandHistory2DBBox &past, const Palm7KP &present)
-{
-	if (!past.htAlgorithm_approves) {
-		// U_LOG_E("Returning big number because htAlgorithm told me to!");
-		return 100000000000000000000000000000.0f;
-	}
-	float sum_of_lengths = m_vec2_len(past.wrist_unfiltered.back() - past.middle_unfiltered.back()) +
-	                       m_vec2_len(present.kps[WRIST_7KP] - present.kps[MIDDLE_7KP]);
-
-	float sum_of_distances = (m_vec2_len(past.wrist_unfiltered.back() - present.kps[WRIST_7KP]) +
-	                          m_vec2_len(past.middle_unfiltered.back() - present.kps[MIDDLE_7KP]));
-
-
-	float final = sum_of_distances / sum_of_lengths;
-
-	return final;
-}
-
-static std::vector<Hand2D>
-htImageToKeypoints(struct ht_view *htv)
-{
-	struct HandTracking *htd = htv->htd;
-	ht_model *htm = htv->htm;
-
-	cv::Mat raw_input = htv->run_model_on_this;
-
-	// Get a list of palms - drop confidences and ssd bounding boxes, just keypoints.
-
-
-	std::vector<Palm7KP> hand_detections = htm->palm_detection(htv, raw_input);
-
-	std::vector<bool> used_histories;
-	std::vector<bool> used_detections;
-
-	std::vector<size_t> history_indices;
-	std::vector<size_t> detection_indices;
-	std::vector<float> dontuse;
-
-
-	// Strategy here is: We have a big list of palms. Match 'em up to previous palms.
-	naive_sort_permutation_by_error<HandHistory2DBBox, Palm7KP>(htv->bbox_histories, hand_detections,
-
-	                                                            // bools
-	                                                            used_histories, used_detections,
-
-	                                                            history_indices, detection_indices, dontuse,
-	                                                            errHistory2D, 1.0f);
-
-	// Here's the trick - we use the associated bbox_filter to get an output but *never commit* the noisy 128x128
-	// detection; instead later on we commit the (hopefully) nicer palm and wrist from the 224x224 keypoint
-	// estimation.
-
-	// Add extra detections!
-	for (size_t i = 0; i < used_detections.size(); i++) {
-		if ((used_detections[i] == false) && hand_detections[i].confidence > 0.65) {
-			// Confidence to get in the door is 0.65, confidence to stay in is 0.3
-			HandHistory2DBBox hist_new = {};
-			m_filter_euro_vec2_init(&hist_new.m_filter_center, FCMIN_BBOX_POSITION, FCMIN_D_BB0X_POSITION,
-			                        BETA_BB0X_POSITION);
-			m_filter_euro_vec2_init(&hist_new.m_filter_direction, FCMIN_BBOX_ORIENTATION,
-			                        FCMIN_D_BB0X_ORIENTATION, BETA_BB0X_ORIENTATION);
-
-			htv->bbox_histories.push_back(hist_new);
-			history_indices.push_back(htv->bbox_histories.size() - 1);
-			detection_indices.push_back(i);
-		}
-	}
-
-	// Do the things for each active bbox history!
-	for (size_t i = 0; i < history_indices.size(); i++) {
-		HandHistory2DBBox *hist_of_interest = &htv->bbox_histories[history_indices[i]];
-		hist_of_interest->wrist_unfiltered.push_back(hand_detections[detection_indices[i]].kps[WRIST_7KP]);
-		hist_of_interest->index_unfiltered.push_back(hand_detections[detection_indices[i]].kps[INDEX_7KP]);
-		hist_of_interest->middle_unfiltered.push_back(hand_detections[detection_indices[i]].kps[MIDDLE_7KP]);
-		hist_of_interest->pinky_unfiltered.push_back(hand_detections[detection_indices[i]].kps[LITTLE_7KP]);
-		// Eh do the rest later
-	}
-
-	// Prune stale detections! (After we don't need {history,detection}_indices to be correct)
-	int bob = 0;
-	for (size_t i = 0; i < used_histories.size(); i++) {
-		if (used_histories[i] == false) {
-			// history never got assigned a present hand to it. treat it as stale delete it.
-
-			HT_TRACE(htv->htd, "Removing bbox from history!\n");
-			htv->bbox_histories.erase(htv->bbox_histories.begin() + i + bob);
-			bob--;
-		}
-	}
-	if (htv->bbox_histories.size() == 0) {
-		return {}; // bail early
-	}
-
-	std::vector<std::future<Hand2D>> await_list_of_hand_in_bbox; //(htv->bbox_histories.size());
-
-	std::vector<DetectionModelOutput> blah(htv->bbox_histories.size());
-
-	std::vector<Hand2D> output;
-
-	if (htv->bbox_histories.size() > 2) {
-		HT_DEBUG(htd, "More than two hands (%zu) in 2D view %i", htv->bbox_histories.size(), htv->view);
-	}
-
-	for (size_t i = 0; i < htv->bbox_histories.size(); i++) { //(BBoxHistory * entry : htv->bbox_histories) {
-		HandHistory2DBBox *entry = &htv->bbox_histories[i];
-		cv::Mat hand_rect = cv::Mat(224, 224, CV_8UC3);
-		xrt_vec2 unfiltered_middle;
-		xrt_vec2 unfiltered_direction;
-
-		centerAndRotationFromJoints(htv, &entry->wrist_unfiltered.back(), &entry->index_unfiltered.back(),
-		                            &entry->middle_unfiltered.back(), &entry->pinky_unfiltered.back(),
-		                            &unfiltered_middle, &unfiltered_direction);
-
-		xrt_vec2 filtered_middle;
-		xrt_vec2 filtered_direction;
-
-		m_filter_euro_vec2_run_no_commit(&entry->m_filter_center, htv->htd->current_frame_timestamp,
-		                                 &unfiltered_middle, &filtered_middle);
-		m_filter_euro_vec2_run_no_commit(&entry->m_filter_direction, htv->htd->current_frame_timestamp,
-		                                 &unfiltered_direction, &filtered_direction);
-
-		rotatedRectFromJoints(htv, filtered_middle, filtered_direction, &blah[i]);
-
-		warpAffine(raw_input, hand_rect, blah[i].warp_there, hand_rect.size());
-
-		await_list_of_hand_in_bbox.push_back(
-		    std::async(std::launch::async, std::bind(&ht_model::hand_landmark, htm, hand_rect)));
-	}
-
-	for (size_t i = 0; i < htv->bbox_histories.size(); i++) {
-		Hand2D in_bbox = await_list_of_hand_in_bbox[i].get();
-
-		cv::Matx23f warp_back = blah[i].warp_back;
-
-		Hand2D in_image_ray_coords;
-		Hand2D in_image_px_coords;
-
-		for (int i = 0; i < 21; i++) {
-			struct xrt_vec3 vec = in_bbox.kps[i];
-
-#if 1
-			xrt_vec3 rr = transformVecBy2x3(vec, warp_back);
-			rr.z = vec.z;
-#else
-			xrt_vec3 rr;
-			rr.x = (vec.x * warp_back(0, 0)) + (vec.y * warp_back(0, 1)) + warp_back(0, 2);
-			rr.y = (vec.x * warp_back(1, 0)) + (vec.y * warp_back(1, 1)) + warp_back(1, 2);
-			rr.z = vec.z;
-#endif
-			in_image_px_coords.kps[i] = rr;
-
-			in_image_ray_coords.kps[i] = raycoord(htv, rr);
-			if (htd->debug_scribble && htd->dynamic_config.scribble_2d_keypoints) {
-				handDot(htv->debug_out_to_this, {rr.x, rr.y}, fmax((-vec.z + 100 - 20) * .08, 2),
-				        ((float)i) / 21.0f, 0.95f, cv::FILLED);
-			}
-		}
-		xrt_vec2 wrist_in_px_coords = {in_image_px_coords.kps[WRIST].x, in_image_px_coords.kps[WRIST].y};
-		xrt_vec2 index_in_px_coords = {in_image_px_coords.kps[INDX_PXM].x, in_image_px_coords.kps[INDX_PXM].y};
-		xrt_vec2 middle_in_px_coords = {in_image_px_coords.kps[MIDL_PXM].x, in_image_px_coords.kps[MIDL_PXM].y};
-		xrt_vec2 little_in_px_coords = {in_image_px_coords.kps[LITL_PXM].x, in_image_px_coords.kps[LITL_PXM].y};
-		xrt_vec2 dontuse;
-
-		xrt_vec2 unfiltered_middle, unfiltered_direction;
-		centerAndRotationFromJoints(htv, &wrist_in_px_coords, &index_in_px_coords, &middle_in_px_coords,
-		                            &little_in_px_coords, &unfiltered_middle, &unfiltered_direction);
-
-		m_filter_euro_vec2_run(&htv->bbox_histories[i].m_filter_center, htv->htd->current_frame_timestamp,
-		                       &unfiltered_middle, &dontuse);
-
-		m_filter_euro_vec2_run(&htv->bbox_histories[i].m_filter_direction, htv->htd->current_frame_timestamp,
-		                       &unfiltered_direction, &dontuse);
-
-		output.push_back(in_image_ray_coords);
-	}
-	return output;
-}
-
-#if defined(EXPERIMENTAL_DATASET_RECORDING)
-
-static void
-jsonAddJoint(cJSON *into_this, xrt_pose loc, const char *name)
-{
-
-	cJSON *container = cJSON_CreateObject();
-	cJSON *joint_loc = cJSON_CreateArray();
-	cJSON_AddItemToArray(joint_loc, cJSON_CreateNumber(loc.position.x));
-	cJSON_AddItemToArray(joint_loc, cJSON_CreateNumber(loc.position.y));
-	cJSON_AddItemToArray(joint_loc, cJSON_CreateNumber(loc.position.z));
-
-	cJSON_AddItemToObject(container, "position", joint_loc);
-
-	cJSON *joint_rot = cJSON_CreateArray();
-
-
-	cJSON_AddItemToArray(joint_rot, cJSON_CreateNumber(loc.orientation.x));
-	cJSON_AddItemToArray(joint_rot, cJSON_CreateNumber(loc.orientation.y));
-	cJSON_AddItemToArray(joint_rot, cJSON_CreateNumber(loc.orientation.z));
-	cJSON_AddItemToArray(joint_rot, cJSON_CreateNumber(loc.orientation.w));
-
-	cJSON_AddItemToObject(container, "rotation_quat_xyzw", joint_rot);
-
-	cJSON_AddItemToObject(into_this, name, container);
-}
-
-void
-jsonMaybeAddSomeHands(struct HandTracking *htd, bool err)
-{
-	if (!htd->tracking_should_record_dataset) {
-		return;
-	}
-	cJSON *j_this_frame = cJSON_CreateObject();
-	cJSON_AddItemToObject(j_this_frame, "seq_since_start", cJSON_CreateNumber(htd->gst.current_index));
-	cJSON_AddItemToObject(j_this_frame, "seq_src", cJSON_CreateNumber(htd->frame_for_process->source_sequence));
-	cJSON_AddItemToObject(j_this_frame, "ts", cJSON_CreateNumber(htd->gst.last_frame_ns));
-
-	cJSON *j_hands_in_frame = cJSON_AddArrayToObject(j_this_frame, "detected_hands");
-	if (!err) {
-		for (size_t idx_hand = 0; idx_hand < htd->histories_3d.size(); idx_hand++) {
-			cJSON *j_hand_in_frame = cJSON_CreateObject();
-
-			cJSON *j_uuid = cJSON_CreateNumber(htd->histories_3d[idx_hand].uuid);
-			cJSON_AddItemToObject(j_hand_in_frame, "uuid", j_uuid);
-
-			cJSON *j_handedness = cJSON_CreateNumber(htd->histories_3d[idx_hand].handedness);
-			cJSON_AddItemToObject(j_hand_in_frame, "handedness", j_handedness);
-
-			static const char *keys[21] = {
-			    "WRIST",
-
-			    "THMB_MCP", "THMB_PXM", "THMB_DST", "THMB_TIP",
-
-			    "INDX_PXM", "INDX_INT", "INDX_DST", "INDX_TIP",
-
-			    "MIDL_PXM", "MIDL_INT", "MIDL_DST", "MIDL_TIP",
-
-			    "RING_PXM", "RING_INT", "RING_DST", "RING_TIP",
-
-			    "LITL_PXM", "LITL_INT", "LITL_DST", "LITL_TIP",
-			};
-
-			for (int idx_joint = 0; idx_joint < 21; idx_joint++) {
-				// const char* key = keys[idx_joint];
-				cJSON *j_vec3 = cJSON_AddArrayToObject(j_hand_in_frame, keys[idx_joint]);
-				cJSON_AddItemToArray(
-				    j_vec3,
-				    cJSON_CreateNumber(
-				        htd->histories_3d[idx_hand].last_hands_unfiltered.back().kps[idx_joint].x));
-				cJSON_AddItemToArray(
-				    j_vec3,
-				    cJSON_CreateNumber(
-				        htd->histories_3d[idx_hand].last_hands_unfiltered.back().kps[idx_joint].y));
-				cJSON_AddItemToArray(
-				    j_vec3,
-				    cJSON_CreateNumber(
-				        htd->histories_3d[idx_hand].last_hands_unfiltered.back().kps[idx_joint].z));
-			}
-
-
-			cJSON_AddItemToArray(j_hands_in_frame, j_hand_in_frame);
-		}
-	}
-	cJSON_AddItemToArray(htd->gst.output_array, j_this_frame);
-}
-
-#endif
-
-
-
-static void
-htJointDisparityMath(struct HandTracking *htd, Hand2D *hand_in_left, Hand2D *hand_in_right, Hand3D *out_hand)
-{
-	for (int i = 0; i < 21; i++) {
-		// Believe it or not, this is where the 3D stuff happens!
-		float t = htd->baseline / (hand_in_left->kps[i].x - hand_in_right->kps[i].x);
-
-		out_hand->kps[i].z = -t;
-
-		out_hand->kps[i].x = (hand_in_left->kps[i].x * t);
-		out_hand->kps[i].y = -hand_in_left->kps[i].y * t;
-
-		out_hand->kps[i].x += htd->baseline + (hand_in_right->kps[i].x * t);
-		out_hand->kps[i].y += -hand_in_right->kps[i].y * t;
-
-		out_hand->kps[i].x *= .5;
-		out_hand->kps[i].y *= .5;
-	}
-}
-int64_t last_frame, this_frame;
-
-DEBUG_GET_ONCE_LOG_OPTION(ht_log, "HT_LOG", U_LOGGING_WARN)
-
-/*!
- * Setup helper functions.
- */
-
-static bool
-getCalibration(struct HandTracking *htd, t_stereo_camera_calibration *calibration)
-{
-	xrt::auxiliary::tracking::StereoCameraCalibrationWrapper wrap(calibration);
-	xrt_vec3 trans = {(float)wrap.camera_translation_mat(0, 0), (float)wrap.camera_translation_mat(1, 0),
-	                  (float)wrap.camera_translation_mat(2, 0)};
-	htd->baseline = m_vec3_len(trans);
-
-#if 0
-	std::cout << "\n\nTRANSLATION VECTOR IS\n" << wrap.camera_translation_mat;
-	std::cout << "\n\nROTATION FROM LEFT TO RIGHT IS\n" << wrap.camera_rotation_mat << "\n";
-#endif
-
-	cv::Matx34d P1;
-	cv::Matx34d P2;
-
-	cv::Matx44d Q;
-
-	// The only reason we're calling stereoRectify is because we want R1 and R2 for the
-	cv::stereoRectify(wrap.view[0].intrinsics_mat,                  // cameraMatrix1
-	                  wrap.view[0].distortion_mat,                  // distCoeffs1
-	                  wrap.view[1].intrinsics_mat,                  // cameraMatrix2
-	                  wrap.view[1].distortion_mat,                  // distCoeffs2
-	                  wrap.view[0].image_size_pixels_cv,            // imageSize*
-	                  wrap.camera_rotation_mat,                     // R
-	                  wrap.camera_translation_mat,                  // T
-	                  htd->views[0].rotate_camera_to_stereo_camera, // R1
-	                  htd->views[1].rotate_camera_to_stereo_camera, // R2
-	                  P1,                                           // P1
-	                  P2,                                           // P2
-	                  Q,                                            // Q
-	                  0,                                            // flags
-	                  -1.0f,                                        // alpha
-	                  cv::Size(),                                   // newImageSize
-	                  NULL,                                         // validPixROI1
-	                  NULL);                                        // validPixROI2
-
-	//* Good enough guess that view 0 and view 1 are the same size.
-
-	for (int i = 0; i < 2; i++) {
-		htd->views[i].cameraMatrix = wrap.view[i].intrinsics_mat;
-
-		htd->views[i].distortion = wrap.view[i].distortion_fisheye_mat;
-	}
-
-	htd->one_view_size_px.w = wrap.view[0].image_size_pixels.w;
-	htd->one_view_size_px.h = wrap.view[0].image_size_pixels.h;
-
-	U_LOG_E("%d %d %p %p", htd->one_view_size_px.w, htd->one_view_size_px.h, (void *)&htd->one_view_size_px.w,
-	        (void *)&htd->one_view_size_px.h);
-
-
-
-	cv::Matx33d rotate_stereo_camera_to_left_camera = htd->views[0].rotate_camera_to_stereo_camera.inv();
-
-	xrt_matrix_3x3 s;
-	s.v[0] = rotate_stereo_camera_to_left_camera(0, 0);
-	s.v[1] = rotate_stereo_camera_to_left_camera(0, 1);
-	s.v[2] = rotate_stereo_camera_to_left_camera(0, 2);
-
-	s.v[3] = rotate_stereo_camera_to_left_camera(1, 0);
-	s.v[4] = rotate_stereo_camera_to_left_camera(1, 1);
-	s.v[5] = rotate_stereo_camera_to_left_camera(1, 2);
-
-	s.v[6] = rotate_stereo_camera_to_left_camera(2, 0);
-	s.v[7] = rotate_stereo_camera_to_left_camera(2, 1);
-	s.v[8] = rotate_stereo_camera_to_left_camera(2, 2);
-
-	xrt_quat tmp;
-
-	math_quat_from_matrix_3x3(&s, &tmp);
-
-	// Weird that I have to invert this quat, right? I think at some point - like probably just before this - I must
-	// have swapped row-major and col-major - remember, if you transpose a rotation matrix, you get its inverse.
-	// Doesn't matter that I don't understand - non-inverted looks definitely wrong, inverted looks definitely
-	// right.
-	math_quat_invert(&tmp, &htd->stereo_camera_to_left_camera);
-
-#if 0
-	U_LOG_E("%f %f %f %f", htd->stereo_camera_to_left_camera.w, htd->stereo_camera_to_left_camera.x,
-	        htd->stereo_camera_to_left_camera.y, htd->stereo_camera_to_left_camera.z);
-#endif
-
-	return true;
-}
-
-#if 0
-static void
-getStartupConfig(struct HandTracking *htd, const cJSON *startup_config)
-{
-	const cJSON *palm_detection_type = u_json_get(startup_config, "palm_detection_model");
-	const cJSON *keypoint_estimation_type = u_json_get(startup_config, "keypoint_estimation_model");
-	const cJSON *uvc_wire_format = u_json_get(startup_config, "uvc_wire_format");
-
-	// IsString does its own null-checking
-	if (cJSON_IsString(palm_detection_type)) {
-		bool is_collabora = (strcmp(cJSON_GetStringValue(palm_detection_type), "collabora") == 0);
-		bool is_mediapipe = (strcmp(cJSON_GetStringValue(palm_detection_type), "mediapipe") == 0);
-		if (!is_collabora && !is_mediapipe) {
-			HT_WARN(htd, "Unknown palm detection type %s - should be \"collabora\" or \"mediapipe\"",
-			        cJSON_GetStringValue(palm_detection_type));
-		}
-		htd->startup_config.palm_detection_use_mediapipe = is_mediapipe;
-	}
-
-	if (cJSON_IsString(keypoint_estimation_type)) {
-		bool is_collabora = (strcmp(cJSON_GetStringValue(keypoint_estimation_type), "collabora") == 0);
-		bool is_mediapipe = (strcmp(cJSON_GetStringValue(keypoint_estimation_type), "mediapipe") == 0);
-		if (!is_collabora && !is_mediapipe) {
-			HT_WARN(htd, "Unknown keypoint estimation type %s - should be \"collabora\" or \"mediapipe\"",
-			        cJSON_GetStringValue(keypoint_estimation_type));
-		}
-		htd->startup_config.keypoint_estimation_use_mediapipe = is_mediapipe;
-	}
-
-	if (cJSON_IsString(uvc_wire_format)) {
-		bool is_yuv = (strcmp(cJSON_GetStringValue(uvc_wire_format), "yuv") == 0);
-		bool is_mjpeg = (strcmp(cJSON_GetStringValue(uvc_wire_format), "mjpeg") == 0);
-		if (!is_yuv && !is_mjpeg) {
-			HT_WARN(htd, "Unknown wire format type %s - should be \"yuv\" or \"mjpeg\"",
-			        cJSON_GetStringValue(uvc_wire_format));
-		}
-		if (is_yuv) {
-			HT_DEBUG(htd, "Using YUYV422!");
-			htd->startup_config.desired_format = XRT_FORMAT_YUYV422;
-		} else {
-			HT_DEBUG(htd, "Using MJPEG!");
-			htd->startup_config.desired_format = XRT_FORMAT_MJPEG;
-		}
-	}
-}
-
-static void
-getUserConfig(struct HandTracking *htd)
-{
-	// The game here is to avoid bugs + be paranoid, not to be fast. If you see something that seems "slow" - don't
-	// fix it. Any of the tracking code is way stickier than this could ever be.
-
-	struct u_config_json config_json = {};
-
-	u_config_json_open_or_create_main_file(&config_json);
-	if (!config_json.file_loaded) {
-		return;
-	}
-
-	cJSON *ht_config_json = cJSON_GetObjectItemCaseSensitive(config_json.root, "config_ht");
-	if (ht_config_json == NULL) {
-		return;
-	}
-
-	// Don't get it twisted: initializing these to NULL is not cargo-culting.
-	// Uninitialized values on the stack aren't guaranteed to be 0, so these could end up pointing to what we
-	// *think* is a valid address but what is *not* one.
-	char *startup_config_string = NULL;
-	char *dynamic_config_string = NULL;
-
-	{
-		const cJSON *startup_config_string_json = u_json_get(ht_config_json, "startup_config_index");
-		if (cJSON_IsString(startup_config_string_json)) {
-			startup_config_string = cJSON_GetStringValue(startup_config_string_json);
-		}
-
-		const cJSON *dynamic_config_string_json = u_json_get(ht_config_json, "dynamic_config_index");
-		if (cJSON_IsString(dynamic_config_string_json)) {
-			dynamic_config_string = cJSON_GetStringValue(dynamic_config_string_json);
-		}
-	}
-
-	if (startup_config_string != NULL) {
-		const cJSON *startup_config_obj =
-		    u_json_get(u_json_get(ht_config_json, "startup_configs"), startup_config_string);
-		getStartupConfig(htd, startup_config_obj);
-	}
-
-	if (dynamic_config_string != NULL) {
-		const cJSON *dynamic_config_obj =
-		    u_json_get(u_json_get(ht_config_json, "dynamic_configs"), dynamic_config_string);
-		{
-			ht_dynamic_config *hdc = &htd->dynamic_config;
-			// Do the thing
-			u_json_get_string_into_array(u_json_get(dynamic_config_obj, "name"), hdc->name, 64);
-
-			u_json_get_float(u_json_get(dynamic_config_obj, "hand_fc_min"), &hdc->hand_fc_min.val);
-			u_json_get_float(u_json_get(dynamic_config_obj, "hand_fc_min_d"), &hdc->hand_fc_min_d.val);
-			u_json_get_float(u_json_get(dynamic_config_obj, "hand_beta"), &hdc->hand_beta.val);
-
-			u_json_get_float(u_json_get(dynamic_config_obj, "nms_iou"), &hdc->nms_iou.val);
-			u_json_get_float(u_json_get(dynamic_config_obj, "nms_threshold"), &hdc->nms_threshold.val);
-
-			u_json_get_bool(u_json_get(dynamic_config_obj, "scribble_nms_detections"),
-			                &hdc->scribble_nms_detections);
-			u_json_get_bool(u_json_get(dynamic_config_obj, "scribble_raw_detections"),
-			                &hdc->scribble_raw_detections);
-			u_json_get_bool(u_json_get(dynamic_config_obj, "scribble_2d_keypoints"),
-			                &hdc->scribble_2d_keypoints);
-			u_json_get_bool(u_json_get(dynamic_config_obj, "scribble_bounding_box"),
-			                &hdc->scribble_bounding_box);
-
-			char *dco_str = cJSON_Print(dynamic_config_obj);
-			U_LOG_D("Config %s %s", dynamic_config_string, dco_str);
-			free(dco_str);
-		}
-	}
-
-
-
-	cJSON_Delete(config_json.root);
-	return;
-}
-#endif
-
-static void
-userConfigSetDefaults(struct HandTracking *htd)
-{
-	// Admit defeat: for now, Mediapipe's are still better than ours.
-	htd->startup_config.palm_detection_use_mediapipe = true;
-	htd->startup_config.keypoint_estimation_use_mediapipe = true;
-
-	// Make sure you build DebugOptimized!
-	htd->startup_config.desired_format = XRT_FORMAT_YUYV422;
-
-
-	ht_dynamic_config *hdc = &htd->dynamic_config;
-
-	hdc->scribble_nms_detections = true;
-	hdc->scribble_raw_detections = false;
-	hdc->scribble_2d_keypoints = true;
-	hdc->scribble_bounding_box = false;
-
-	hdc->hand_fc_min.min = 0.0f;
-	hdc->hand_fc_min.max = 50.0f;
-	hdc->hand_fc_min.step = 0.05f;
-	hdc->hand_fc_min.val = FCMIN_HAND;
-
-	hdc->hand_fc_min_d.min = 0.0f;
-	hdc->hand_fc_min_d.max = 50.0f;
-	hdc->hand_fc_min_d.step = 0.05f;
-	hdc->hand_fc_min_d.val = FCMIN_D_HAND;
-
-
-	hdc->hand_beta.min = 0.0f;
-	hdc->hand_beta.max = 50.0f;
-	hdc->hand_beta.step = 0.05f;
-	hdc->hand_beta.val = BETA_HAND;
-
-	hdc->max_vel.min = 0.0f;
-	hdc->max_vel.max = 50.0f;
-	hdc->max_vel.step = 0.05f;
-	hdc->max_vel.val = 30.0f; // 30 m/s; about 108 kph. If your hand is going this fast, our tracking failing is the
-	                          // least of your problems.
-
-	hdc->max_acc.min = 0.0f;
-	hdc->max_acc.max = 100.0f;
-	hdc->max_acc.step = 0.1f;
-	hdc->max_acc.val = 100.0f; // 100 m/s^2; about 10 Gs. Ditto.
-
-	hdc->nms_iou.min = 0.0f;
-	hdc->nms_iou.max = 1.0f;
-	hdc->nms_iou.step = 0.01f;
-
-
-	hdc->nms_threshold.min = 0.0f;
-	hdc->nms_threshold.max = 1.0f;
-	hdc->nms_threshold.step = 0.01f;
-
-	hdc->new_detection_threshold.min = 0.0f;
-	hdc->new_detection_threshold.max = 1.0f;
-	hdc->new_detection_threshold.step = 0.01f;
-
-
-	hdc->nms_iou.val = 0.05f;
-	hdc->nms_threshold.val = 0.3f;
-	hdc->new_detection_threshold.val = 0.6f;
-}
-
-
-static void
-getModelsFolder(struct HandTracking *htd)
-{
-// Please bikeshed me on this! I don't know where is the best place to put this stuff!
-#if 0
-	char exec_location[1024] = {};
-	readlink("/proc/self/exe", exec_location, 1024);
-
-	HT_DEBUG(htd, "Exec at %s\n", exec_location);
-
-	int end = 0;
-	while (exec_location[end] != '\0') {
-		HT_DEBUG(htd, "%d", end);
-		end++;
-	}
-
-	while (exec_location[end] != '/' && end != 0) {
-		HT_DEBUG(htd, "%d %c", end, exec_location[end]);
-		exec_location[end] = '\0';
-		end--;
-	}
-
-	strcat(exec_location, "../share/monado/hand-tracking-models/");
-	strcpy(htd->startup_config.model_slug, exec_location);
-#else
-	const char *xdg_home = getenv("XDG_CONFIG_HOME");
-	const char *home = getenv("HOME");
-	if (xdg_home != NULL) {
-		strcpy(htd->startup_config.model_slug, xdg_home);
-	} else if (home != NULL) {
-		strcpy(htd->startup_config.model_slug, home);
-	} else {
-		assert(false);
-	}
-	strcat(htd->startup_config.model_slug, "/.local/share/monado/hand-tracking-models/");
-#endif
-}
-
-
-
-static void
-htExitFrame(struct HandTracking *htd,
-            bool err,
-            struct xrt_hand_joint_set final_hands_ordered_by_handedness[2],
-            uint64_t timestamp,
-            struct xrt_hand_joint_set *out_left,
-            struct xrt_hand_joint_set *out_right,
-            uint64_t *out_timestamp_ns)
-{
-
-	os_mutex_lock(&htd->openxr_hand_data_mediator);
-	*out_timestamp_ns = timestamp;
-
-	if (err) {
-		out_left->is_active = false;
-		out_right->is_active = false;
-	} else {
-		*out_left = final_hands_ordered_by_handedness[0];
-		*out_right = final_hands_ordered_by_handedness[1];
-
-
-		HT_DEBUG(htd, "Adding ts %zu", htd->hands_for_openxr_timestamp);
-	}
-	os_mutex_unlock(&htd->openxr_hand_data_mediator);
-#ifdef EXPERIMENTAL_DATASET_RECORDING
-	if (htd->tracking_should_record_dataset) {
-		// Add nothing-entry to json file.
-		jsonMaybeAddSomeHands(htd, err);
-		htd->gst.current_index++;
-	}
-#endif
-}
-
-/*
- *
- * Member functions.
- *
- */
-
-HandTracking::HandTracking()
-{
-	this->base.process = &HandTracking::cCallbackProcess;
-	this->base.destroy = &HandTracking::cCallbackDestroy;
-}
-
-HandTracking::~HandTracking()
-{
-	//
-}
-
-//!@todo vVERY BAD
-static void
-combine_frames_r8g8b8_hack(struct xrt_frame *l, struct xrt_frame *r, struct xrt_frame *f)
-{
-	// SINK_TRACE_MARKER();
-
-	uint32_t height = l->height;
-
-	for (uint32_t y = 0; y < height; y++) {
-		uint8_t *dst = f->data + f->stride * y;
-		uint8_t *src = l->data + l->stride * y;
-
-		for (uint32_t x = 0; x < l->width * 3; x++) {
-			*dst++ = *src++;
-		}
-
-		dst = f->data + f->stride * y + l->width * 3;
-		src = r->data + r->stride * y;
-		for (uint32_t x = 0; x < r->width * 3; x++) {
-			*dst++ = *src++;
-		}
-	}
-}
-
-void
-HandTracking::cCallbackProcess(struct t_hand_tracking_sync *ht_sync,
-                               struct xrt_frame *left_frame,
-                               struct xrt_frame *right_frame,
-                               struct xrt_hand_joint_set *out_left_hand,
-                               struct xrt_hand_joint_set *out_right_hand,
-                               uint64_t *out_timestamp_ns)
-{
-	XRT_TRACE_MARKER();
-
-	HandTracking *htd = (struct HandTracking *)ht_sync;
-
-	// U_LOG_E("htd is at %p", htd);
-
-	htd->current_frame_timestamp = left_frame->timestamp;
-
-	int64_t start, end;
-	start = os_monotonic_get_ns();
-
-
-	/*
-	 * Setup views.
-	 */
-
-	assert(left_frame->width == right_frame->width);
-	assert(left_frame->height == right_frame->height);
-
-	const int full_height = left_frame->height;
-	const int full_width = left_frame->width * 2;
-
-	const int view_width = htd->one_view_size_px.w;
-	const int view_height = htd->one_view_size_px.h;
-
-	assert(full_height == view_height);
-
-	const cv::Size full_size = cv::Size(full_width, full_height);
-	const cv::Size view_size = cv::Size(view_width, view_height);
-	const cv::Point view_offsets[2] = {cv::Point(0, 0), cv::Point(view_width, 0)};
-
-	// cv::Mat full_frame(full_size, CV_8UC3, htd->frame_for_process->data, htd->frame_for_process->stride);
-	htd->views[0].run_model_on_this = cv::Mat(view_size, CV_8UC3, left_frame->data, left_frame->stride);
-	htd->views[1].run_model_on_this = cv::Mat(view_size, CV_8UC3, right_frame->data, right_frame->stride);
-
-
-	// Convenience
-	uint64_t timestamp = left_frame->timestamp;
-
-	htd->debug_scribble = u_sink_debug_is_active(&htd->debug_sink);
-
-	cv::Mat debug_output = {};
-	xrt_frame *debug_frame = nullptr;
-
-
-	if (htd->debug_scribble) {
-		u_frame_create_one_off(XRT_FORMAT_R8G8B8, full_width, full_height, &debug_frame);
-		combine_frames_r8g8b8_hack(left_frame, right_frame, debug_frame);
-
-		debug_output = cv::Mat(full_size, CV_8UC3, debug_frame->data, debug_frame->stride);
-		htd->views[0].debug_out_to_this = debug_output(cv::Rect(view_offsets[0], view_size));
-		htd->views[1].debug_out_to_this = debug_output(cv::Rect(view_offsets[1], view_size));
-	}
-
-
-	/*
-	 * Do the hand tracking!
-	 */
-
-	std::future<std::vector<Hand2D>> future_left =
-	    std::async(std::launch::async, htImageToKeypoints, &htd->views[0]);
-	std::future<std::vector<Hand2D>> future_right =
-	    std::async(std::launch::async, htImageToKeypoints, &htd->views[1]);
-	std::vector<Hand2D> hands_in_left_view = future_left.get();
-	std::vector<Hand2D> hands_in_right_view = future_right.get();
-
-	end = os_monotonic_get_ns();
-
-
-	this_frame = os_monotonic_get_ns();
-
-	double time_ms = (double)(end - start) / (double)U_TIME_1MS_IN_NS;
-	double _1_time = 1 / (time_ms * 0.001);
-
-	char t[64];
-	char t2[64];
-	sprintf(t, "% 8.2f ms", time_ms);
-	sprintf(t2, "% 8.2f fps", _1_time);
-	last_frame = this_frame;
-
-
-	if (htd->debug_scribble) {
-		cv::putText(debug_output, t, cv::Point(30, 60), cv::FONT_HERSHEY_SIMPLEX, 1.0f, cv::Scalar(0, 255, 0),
-		            4);
-		cv::putText(debug_output, t2, cv::Point(30, 100), cv::FONT_HERSHEY_SIMPLEX, 1.0f, cv::Scalar(0, 255, 0),
-		            4);
-	} else {
-		HT_DEBUG(htd, "%s", t);
-		HT_DEBUG(htd, "%s", t2);
-	}
-
-
-
-	if (htd->debug_scribble) {
-		u_sink_debug_push_frame(&htd->debug_sink, debug_frame);
-		xrt_frame_reference(&debug_frame, NULL);
-	}
-
-	// Bail early this frame if no hands were detected.
-	// In the long run, this'll be a silly thing - we shouldn't always take the detection model's word for it
-	// especially when part of the pipeline is an arbitrary confidence threshold.
-	if (hands_in_left_view.size() == 0 || hands_in_right_view.size() == 0) {
-		htExitFrame(htd, true, NULL, timestamp, out_left_hand, out_right_hand, out_timestamp_ns);
-		return;
-	}
-
-	std::vector<Hand3D> possible_3d_hands;
-
-	// for every possible combination of hands in left view and hands in right view,
-	for (size_t idx_l = 0; idx_l < hands_in_left_view.size(); idx_l++) {
-		for (size_t idx_r = 0; idx_r < hands_in_right_view.size(); idx_r++) {
-			Hand3D cur_hand = {};
-
-			Hand2D &left_2d = hands_in_left_view[idx_l];
-			Hand2D &right_2d = hands_in_right_view[idx_r];
-
-			// Calculate a 3D hand for this combination
-			htJointDisparityMath(htd, &hands_in_left_view[idx_l], &hands_in_right_view[idx_r], &cur_hand);
-			cur_hand.timestamp = timestamp;
-			cur_hand.rejected_by_smush = false;
-
-			cur_hand.idx_l = idx_l;
-			cur_hand.idx_r = idx_r;
-
-			// Calculate a y-disparity for this combination
-			cur_hand.y_disparity_error = errHandDisparity(left_2d, right_2d);
-
-			possible_3d_hands.push_back(cur_hand);
-		}
-	}
-
-	HT_DEBUG(htd, "Starting with %zu hands!", possible_3d_hands.size());
-
-	// For each pair of 3D hands we just made
-	for (size_t idx_one = 0; idx_one < possible_3d_hands.size(); idx_one++) {
-		for (size_t idx_two = 0; idx_two < possible_3d_hands.size(); idx_two++) {
-			if ((idx_one <= idx_two)) {
-				continue;
-			}
-
-			// See if this pair is suspiciously close together.
-			// If it is, then this pairing is wrong - this is what was causing the "hands smushing together"
-			// issue - we weren't catching these reliably.
-			float errr = sumOfHandJointDistances(possible_3d_hands[idx_one], possible_3d_hands[idx_two]);
-			HT_TRACE(htd, "%zu %zu is smush %f", idx_one, idx_two, errr);
-			if (errr < 0.03f * 21.0f) {
-				possible_3d_hands[idx_one].rejected_by_smush = true;
-				possible_3d_hands[idx_two].rejected_by_smush = true;
-			}
-		}
-	}
-
-	std::vector<Hand3D> hands_unfiltered;
-
-	for (Hand3D hand : possible_3d_hands) {
-		// If none of these are false, then all our heuristics indicate this is a real hand, so we add it to our
-		// list of real hands.
-		bool selected = !hand.rejected_by_smush &&       //
-		                hand.y_disparity_error < 1.0f && //
-		                rejectTooClose(htd, &hand) &&    //
-		                rejectTooFar(htd, &hand) &&      //
-		                rejectTinyPalm(htd, &hand);
-		if (selected) {
-			HT_TRACE(htd, "Pushing back with y-error %f", hand.y_disparity_error);
-			hands_unfiltered.push_back(hand);
-		}
-	}
-
-
-	std::vector<bool> past_hands_taken;
-	std::vector<bool> present_hands_taken;
-
-	std::vector<size_t> past_indices;
-	std::vector<size_t> present_indices;
-	std::vector<float> flow_errors;
-
-
-	float max_dist_between_frames = 1.0f;
-
-	naive_sort_permutation_by_error<HandHistory3D, Hand3D>(htd->histories_3d, // past
-	                                                       hands_unfiltered,  // present
-
-
-	                                                       // outputs
-	                                                       past_hands_taken, present_hands_taken, past_indices,
-	                                                       present_indices, flow_errors, errHandHistory,
-	                                                       (max_dist_between_frames * 21.0f)
-
-	);
-
-
-	for (size_t i = 0; i < past_indices.size(); i++) {
-		htd->histories_3d[past_indices[i]].last_hands_unfiltered.push_back(
-		    hands_unfiltered[present_indices[i]]);
-	}
-	// The preceding may not do anything, because we'll start out with no hand histories! All the numbers of
-	// elements should be zero.
-
-
-	for (size_t i = 0; i < present_hands_taken.size(); i++) {
-		if (present_hands_taken[i] == false) {
-			// if this hand never got assigned to a history
-			HandHistory3D history_new;
-			history_new.uuid = rand(); // Not a great uuid, huh? Good enough for us, this only has to be
-			                           // unique across say an hour period max.
-			handEuroFiltersInit(&history_new, FCMIN_HAND, FCMIN_D_HAND, BETA_HAND);
-			history_new.last_hands_unfiltered.push_back(hands_unfiltered[i]);
-			// history_new.
-			htd->histories_3d.push_back(
-			    history_new); // Add something to the end - don't initialize any of it.
-		}
-	}
-
-	int bob = 0;
-	for (size_t i = 0; i < past_hands_taken.size(); i++) {
-		if (past_hands_taken[i] == false) {
-			htd->histories_3d.erase(htd->histories_3d.begin() + i + bob);
-			bob--;
-		}
-	}
-
-	if (htd->histories_3d.size() == 0) {
-		HT_DEBUG(htd, "Bailing");
-		htExitFrame(htd, true, NULL, timestamp, out_left_hand, out_right_hand, out_timestamp_ns);
-		return;
-	}
-
-	size_t num_hands = htd->histories_3d.size();
-	// if (num_hands > 2) {
-	HT_DEBUG(htd, "Ending with %zu hands!",
-	         num_hands); // this is quite bad, but rarely happens.
-	                     // }
-
-	// Here, we go back to our bbox_histories and remove the histories for any bounding boxes that never turned into
-	// good hands.
-
-	// Iterate over all hands we're keeping track of, compute their current handedness.
-	std::vector<size_t> valid_2d_idxs[2];
-
-
-	for (size_t i = 0; i < htd->histories_3d.size(); i++) {
-		// U_LOG_E("Valid hand %zu l_idx %i r_idx %i", i, htd->histories_3d[i].last_hands[0]->idx_l,
-		//         htd->histories_3d[i].last_hands[0]->idx_r);
-		valid_2d_idxs[0].push_back(htd->histories_3d[i].last_hands_unfiltered.back().idx_l);
-		valid_2d_idxs[1].push_back(htd->histories_3d[i].last_hands_unfiltered.back().idx_r);
-		handednessHandHistory3D(&htd->histories_3d[i]);
-	}
-
-	// Almost certainly not the cleanest way of doing this but leave me alone
-	// Per camera view
-	for (int view = 0; view < 2; view++) {
-		// Per entry in bbox_histories
-		for (size_t hist_idx = 0; hist_idx < htd->views[view].bbox_histories.size(); hist_idx++) {
-			// See if this entry in bbox_histories ever turned into a 3D hand. If not, we notify (in a very
-			// silly way) htImageToKeypoints that it should go away because it was an erroneous detection.
-			for (size_t valid_idx : valid_2d_idxs[view]) {
-				if (valid_idx == hist_idx) {
-					htd->views[view].bbox_histories[hist_idx].htAlgorithm_approves = true;
-					break;
-				} else {
-					htd->views[view].bbox_histories[hist_idx].htAlgorithm_approves = false;
-				}
-			}
-		}
-	}
-
-	// Whoo! Okay, now we have some unfiltered hands in htd->histories_3d[i].last_hands[0]! Euro filter them!
-
-	std::vector<Hand3D> filtered_hands(num_hands);
-
-	for (size_t hand_index = 0; hand_index < num_hands; hand_index++) {
-		handEuroFiltersRun(htd, &htd->histories_3d[hand_index], &filtered_hands[hand_index]);
-		htd->histories_3d[hand_index].last_hands_filtered.push_back(filtered_hands[hand_index]);
-		applyThumbIndexDrag(&filtered_hands[hand_index]);
-		filtered_hands[hand_index].handedness = htd->histories_3d[hand_index].handedness;
-	}
-
-	std::vector<size_t> xr_indices;
-	std::vector<Hand3D *> hands_to_use;
-
-	if (filtered_hands.size() == 1) {
-		if (filtered_hands[0].handedness < 0) {
-			// Left
-			xr_indices = {0};
-			hands_to_use = {&filtered_hands[0]};
-		} else {
-			xr_indices = {1};
-			hands_to_use = {&filtered_hands[0]};
-		}
-	} else {
-		// filtered_hands better be two for now.
-		if (filtered_hands[0].handedness < filtered_hands[1].handedness) {
-			xr_indices = {0, 1};
-			hands_to_use = {&filtered_hands[0], &filtered_hands[1]};
-		} else {
-			xr_indices = {1, 0};
-			hands_to_use = {&filtered_hands[0], &filtered_hands[1]};
-		}
-	}
-
-	struct xrt_hand_joint_set final_hands_ordered_by_handedness[2];
-	memset(&final_hands_ordered_by_handedness[0], 0, sizeof(xrt_hand_joint_set));
-	memset(&final_hands_ordered_by_handedness[1], 0, sizeof(xrt_hand_joint_set));
-	final_hands_ordered_by_handedness[0].is_active = false;
-	final_hands_ordered_by_handedness[1].is_active = false;
-
-	for (size_t i = 0; (i < xr_indices.size()); i++) {
-		Hand3D *hand = hands_to_use[i];
-
-		struct xrt_hand_joint_set *put_in_set = &final_hands_ordered_by_handedness[xr_indices[i]];
-
-		xrt_vec3 wrist = hand->kps[0];
-
-		xrt_vec3 index_prox = hand->kps[5];
-		xrt_vec3 middle_prox = hand->kps[9];
-		xrt_vec3 ring_prox = hand->kps[13];
-		xrt_vec3 pinky_prox = hand->kps[17];
-
-		xrt_vec3 middle_to_index = m_vec3_sub(index_prox, middle_prox);
-		xrt_vec3 middle_to_ring = m_vec3_sub(ring_prox, middle_prox);
-		xrt_vec3 middle_to_pinky = m_vec3_sub(pinky_prox, middle_prox);
-
-		xrt_vec3 three_fourths_down_middle_mcp =
-		    m_vec3_add(m_vec3_mul_scalar(wrist, 3.0f / 4.0f), m_vec3_mul_scalar(middle_prox, 1.0f / 4.0f));
-
-		xrt_vec3 middle_metacarpal = three_fourths_down_middle_mcp;
-
-		float s = 0.6f;
-
-		xrt_vec3 index_metacarpal = middle_metacarpal + m_vec3_mul_scalar(middle_to_index, s);
-		xrt_vec3 ring_metacarpal = middle_metacarpal + m_vec3_mul_scalar(middle_to_ring, s);
-		xrt_vec3 pinky_metacarpal = middle_metacarpal + m_vec3_mul_scalar(middle_to_pinky, s);
-
-		float palm_ness = 0.33;
-		xrt_vec3 palm =
-		    m_vec3_add(m_vec3_mul_scalar(wrist, palm_ness), m_vec3_mul_scalar(middle_prox, (1.0f - palm_ness)));
-
-
-
-		htProcessJoint(htd, palm, put_in_set, XRT_HAND_JOINT_PALM);
-
-		htProcessJoint(htd, hand->kps[0], put_in_set, XRT_HAND_JOINT_WRIST);
-		htProcessJoint(htd, hand->kps[1], put_in_set, XRT_HAND_JOINT_THUMB_METACARPAL);
-		htProcessJoint(htd, hand->kps[2], put_in_set, XRT_HAND_JOINT_THUMB_PROXIMAL);
-		htProcessJoint(htd, hand->kps[3], put_in_set, XRT_HAND_JOINT_THUMB_DISTAL);
-		htProcessJoint(htd, hand->kps[4], put_in_set, XRT_HAND_JOINT_THUMB_TIP);
-
-		htProcessJoint(htd, index_metacarpal, put_in_set, XRT_HAND_JOINT_INDEX_METACARPAL);
-		htProcessJoint(htd, hand->kps[5], put_in_set, XRT_HAND_JOINT_INDEX_PROXIMAL);
-		htProcessJoint(htd, hand->kps[6], put_in_set, XRT_HAND_JOINT_INDEX_INTERMEDIATE);
-		htProcessJoint(htd, hand->kps[7], put_in_set, XRT_HAND_JOINT_INDEX_DISTAL);
-		htProcessJoint(htd, hand->kps[8], put_in_set, XRT_HAND_JOINT_INDEX_TIP);
-
-		htProcessJoint(htd, middle_metacarpal, put_in_set, XRT_HAND_JOINT_MIDDLE_METACARPAL);
-		htProcessJoint(htd, hand->kps[9], put_in_set, XRT_HAND_JOINT_MIDDLE_PROXIMAL);
-		htProcessJoint(htd, hand->kps[10], put_in_set, XRT_HAND_JOINT_MIDDLE_INTERMEDIATE);
-		htProcessJoint(htd, hand->kps[11], put_in_set, XRT_HAND_JOINT_MIDDLE_DISTAL);
-		htProcessJoint(htd, hand->kps[12], put_in_set, XRT_HAND_JOINT_MIDDLE_TIP);
-
-		htProcessJoint(htd, ring_metacarpal, put_in_set, XRT_HAND_JOINT_RING_METACARPAL);
-		htProcessJoint(htd, hand->kps[13], put_in_set, XRT_HAND_JOINT_RING_PROXIMAL);
-		htProcessJoint(htd, hand->kps[14], put_in_set, XRT_HAND_JOINT_RING_INTERMEDIATE);
-		htProcessJoint(htd, hand->kps[15], put_in_set, XRT_HAND_JOINT_RING_DISTAL);
-		htProcessJoint(htd, hand->kps[16], put_in_set, XRT_HAND_JOINT_RING_TIP);
-
-		htProcessJoint(htd, pinky_metacarpal, put_in_set, XRT_HAND_JOINT_LITTLE_METACARPAL);
-		htProcessJoint(htd, hand->kps[17], put_in_set, XRT_HAND_JOINT_LITTLE_PROXIMAL);
-		htProcessJoint(htd, hand->kps[18], put_in_set, XRT_HAND_JOINT_LITTLE_INTERMEDIATE);
-		htProcessJoint(htd, hand->kps[19], put_in_set, XRT_HAND_JOINT_LITTLE_DISTAL);
-		htProcessJoint(htd, hand->kps[20], put_in_set, XRT_HAND_JOINT_LITTLE_TIP);
-
-		put_in_set->is_active = true;
-		math_pose_identity(&put_in_set->hand_pose.pose);
-
-
-		put_in_set->hand_pose.pose.orientation = htd->stereo_camera_to_left_camera;
-
-		put_in_set->hand_pose.relation_flags = valid_flags_ht;
-
-		u_hand_joints_apply_joint_width(put_in_set);
-		applyJointOrientations(put_in_set, xr_indices[i]);
-	}
-	htExitFrame(htd, false, final_hands_ordered_by_handedness, filtered_hands[0].timestamp, out_left_hand,
-	            out_right_hand, out_timestamp_ns);
-}
-
-void
-HandTracking::cCallbackDestroy(t_hand_tracking_sync *ht_sync)
-{
-	auto ht_ptr = &HandTracking::fromC(ht_sync);
-
-	u_sink_debug_destroy(&ht_ptr->debug_sink);
-
-	delete ht_ptr->views[0].htm;
-	delete ht_ptr->views[1].htm;
-	delete ht_ptr;
-}
-
-
-/*
- *
- * 'Exported' functions.
- *
- */
-
-extern "C" t_hand_tracking_sync *
-t_hand_tracking_sync_old_rgb_create(struct t_stereo_camera_calibration *calib)
-{
-	XRT_TRACE_MARKER();
-
-	auto htd = new HandTracking();
-
-	U_LOG_E("htd is at %p", (void *)htd);
-
-	// Setup logging first. We like logging.
-	htd->log_level = debug_get_log_option_ht_log();
-
-	/*
-	 * Get configuration
-	 */
-
-	u_sink_debug_init(&htd->debug_sink);
-	assert(calib != NULL);
-	getCalibration(htd, calib);
-	// Set defaults - most people won't have a config json and it won't get past here.
-	userConfigSetDefaults(htd);
-	getModelsFolder(htd);
-
-
-	htd->views[0].htd = htd;
-	htd->views[1].htd = htd; // :)
-
-	htd->views[0].htm = new ht_model(htd);
-	htd->views[1].htm = new ht_model(htd);
-
-	htd->views[0].view = 0;
-	htd->views[1].view = 1;
-
-	u_var_add_root(htd, "Camera-based Hand Tracker", true);
-
-	u_var_add_draggable_f32(htd, &htd->dynamic_config.hand_fc_min, "hand_fc_min");
-	u_var_add_draggable_f32(htd, &htd->dynamic_config.hand_fc_min_d, "hand_fc_min_d");
-	u_var_add_draggable_f32(htd, &htd->dynamic_config.hand_beta, "hand_beta");
-	u_var_add_draggable_f32(htd, &htd->dynamic_config.nms_iou, "nms_iou");
-	u_var_add_draggable_f32(htd, &htd->dynamic_config.nms_threshold, "nms_threshold");
-	u_var_add_draggable_f32(htd, &htd->dynamic_config.new_detection_threshold, "new_detection_threshold");
-
-	u_var_add_bool(htd, &htd->dynamic_config.scribble_raw_detections, "Scribble raw detections");
-	u_var_add_bool(htd, &htd->dynamic_config.scribble_nms_detections, "Scribble NMS detections");
-	u_var_add_bool(htd, &htd->dynamic_config.scribble_2d_keypoints, "Scribble 2D keypoints");
-	u_var_add_bool(htd, &htd->dynamic_config.scribble_bounding_box, "Scribble bounding box");
-
-	u_var_add_sink_debug(htd, &htd->debug_sink, "i");
-
-
-	HT_DEBUG(htd, "Hand Tracker initialized!");
-
-
-	return &htd->base;
-}

src/xrt/tracking/hand/old_rgb/rgb_sync.hpp

···
-// Copyright 2022, Collabora, Ltd.
-// SPDX-License-Identifier: BSL-1.0
-/*!
- * @file
- * @brief  Old RGB hand tracking header.
- * @author Jakob Bornecrantz <[e-mail]>
- * @author Moses Turner <[e-mail]>
- * @ingroup tracking
- */
-
-#pragma once
-
-#include "tracking/t_hand_tracking.h"
-
-#include "os/os_threading.h"
-
-#include "xrt/xrt_device.h"
-#include "xrt/xrt_prober.h"
-#include "xrt/xrt_frame.h"
-#include "xrt/xrt_frameserver.h"
-
-#include "math/m_api.h"
-#include "math/m_vec3.h"
-#include "math/m_filter_one_euro.h"
-
-#include "util/u_var.h"
-#include "util/u_json.h"
-#include "util/u_sink.h"
-#include "util/u_debug.h"
-#include "util/u_device.h"
-
-#include "util/u_template_historybuf.hpp"
-
-#include <opencv2/opencv.hpp>
-
-#include <vector>
-namespace xrt::tracking::hand::old_rgb {
-
-using namespace xrt::auxiliary::util;
-
-#define HT_TRACE(htd, ...) U_LOG_IFL_T(htd->log_level, __VA_ARGS__)
-#define HT_DEBUG(htd, ...) U_LOG_IFL_D(htd->log_level, __VA_ARGS__)
-#define HT_INFO(htd, ...) U_LOG_IFL_I(htd->log_level, __VA_ARGS__)
-#define HT_WARN(htd, ...) U_LOG_IFL_W(htd->log_level, __VA_ARGS__)
-#define HT_ERROR(htd, ...) U_LOG_IFL_E(htd->log_level, __VA_ARGS__)
-
-#undef EXPERIMENTAL_DATASET_RECORDING
-
-#define FCMIN_BBOX_ORIENTATION 3.0f
-#define FCMIN_D_BB0X_ORIENTATION 10.0f
-#define BETA_BB0X_ORIENTATION 0.0f
-
-#define FCMIN_BBOX_POSITION 30.0f
-#define FCMIN_D_BB0X_POSITION 25.0f
-#define BETA_BB0X_POSITION 0.01f
-
-
-
-#define FCMIN_HAND 4.0f
-#define FCMIN_D_HAND 12.0f
-#define BETA_HAND 0.0083f
-
-class ht_model;
-
-enum HandJoint7Keypoint
-{
-	WRIST_7KP = 0,
-	INDEX_7KP = 1,
-	MIDDLE_7KP = 2,
-	RING_7KP = 3,
-	LITTLE_7KP = 4,
-	THUMB_METACARPAL_7KP = 5,
-	THMB_PROXIMAL_7KP = 6
-};
-
-enum HandJoint21Keypoint
-{
-	WRIST = 0,
-
-	THMB_MCP = 1,
-	THMB_PXM = 2,
-	THMB_DST = 3,
-	THMB_TIP = 4,
-
-	INDX_PXM = 5,
-	INDX_INT = 6,
-	INDX_DST = 7,
-	INDX_TIP = 8,
-
-	MIDL_PXM = 9,
-	MIDL_INT = 10,
-	MIDL_DST = 11,
-	MIDL_TIP = 12,
-
-	RING_PXM = 13,
-	RING_INT = 14,
-	RING_DST = 15,
-	RING_TIP = 16,
-
-	LITL_PXM = 17,
-	LITL_INT = 18,
-	LITL_DST = 19,
-	LITL_TIP = 20
-};
-
-struct Palm7KP
-{
-	struct xrt_vec2 kps[7];
-	float confidence; // between 0 and 1
-};
-
-struct DetectionModelOutput
-{
-	float rotation;
-	float size;
-	xrt_vec2 center;
-	Palm7KP palm;
-
-	cv::Matx23f warp_there;
-	cv::Matx23f warp_back;
-};
-
-// To keep you on your toes. *Don't* think the 2D hand is the same as the 3D!
-struct Hand2D
-{
-	struct xrt_vec3 kps[21];
-	// Third value is depth from ML model. Do not believe the depth.
-};
-
-struct Hand3D
-{
-	struct xrt_vec3 kps[21];
-	float y_disparity_error;
-	float flow_error;
-	int idx_l;
-	int idx_r;
-	bool rejected_by_smush; // init to false.
-
-	float handedness;
-	uint64_t timestamp;
-};
-
-
-struct HandHistory3D
-{
-	// Index 0 is current frame, index 1 is last frame, index 2 is second to last frame.
-	// No particular reason to keep the last 5 frames. we only really only use the current and last one.
-	float handedness;
-	bool have_prev_hand = false;
-	double prev_dy;
-	uint64_t prev_ts_for_alpha; // also in last_hands_unfiltered.back() but go away.
-
-	uint64_t first_ts;
-	uint64_t prev_filtered_ts;
-
-	HistoryBuffer<Hand3D, 10> last_hands_unfiltered;
-	HistoryBuffer<Hand3D, 10> last_hands_filtered;
-
-	// Euro filter for 21kps.
-	m_filter_euro_vec3 filters[21];
-	int uuid;
-};
-
-struct HandHistory2DBBox
-{
-	m_filter_euro_vec2 m_filter_center;
-	m_filter_euro_vec2 m_filter_direction;
-
-	HistoryBuffer<xrt_vec2, 50> wrist_unfiltered;
-	HistoryBuffer<xrt_vec2, 50> index_unfiltered;
-	HistoryBuffer<xrt_vec2, 50> middle_unfiltered;
-	HistoryBuffer<xrt_vec2, 50> pinky_unfiltered;
-	bool htAlgorithm_approves = false;
-};
-
-// Forward declaration for ht_view
-struct HandTracking;
-
-struct ht_view
-{
-	HandTracking *htd;
-	ht_model *htm;
-	int view;
-
-	cv::Matx<double, 4, 1> distortion;
-	cv::Matx<double, 3, 3> cameraMatrix;
-	cv::Matx33d rotate_camera_to_stereo_camera; // R1 or R2
-
-	cv::Mat run_model_on_this;
-	cv::Mat debug_out_to_this;
-
-	std::vector<HandHistory2DBBox> bbox_histories;
-};
-
-struct ht_dynamic_config
-{
-	char name[64];
-	struct u_var_draggable_f32 hand_fc_min;
-	struct u_var_draggable_f32 hand_fc_min_d;
-	struct u_var_draggable_f32 hand_beta;
-	struct u_var_draggable_f32 max_vel;
-	struct u_var_draggable_f32 max_acc;
-	struct u_var_draggable_f32 nms_iou;
-	struct u_var_draggable_f32 nms_threshold;
-	struct u_var_draggable_f32 new_detection_threshold;
-	bool scribble_raw_detections;
-	bool scribble_nms_detections;
-	bool scribble_2d_keypoints;
-	bool scribble_bounding_box;
-};
-
-struct ht_startup_config
-{
-	bool palm_detection_use_mediapipe = false;
-	bool keypoint_estimation_use_mediapipe = false;
-	enum xrt_format desired_format;
-	char model_slug[1024];
-};
-
-/*!
- * Main class of old style RGB hand tracking.
- *
- * @ingroup aux_tracking
- */
-struct HandTracking
-{
-public:
-	// Base thing, has to be first.
-	t_hand_tracking_sync base = {};
-
-	struct u_sink_debug debug_sink = {};
-
-	struct xrt_size one_view_size_px = {};
-
-#if defined(EXPERIMENTAL_DATASET_RECORDING)
-	struct
-	{
-		struct u_var_button start_json_record = {};
-	} gui = {};
-
-	struct
-	{
-		struct gstreamer_pipeline *gp = nullptr;
-		struct gstreamer_sink *gs = nullptr;
-		struct xrt_frame_sink *sink = nullptr;
-		struct xrt_frame_context xfctx = {};
-		uint64_t offset_ns = {};
-		uint64_t last_frame_ns = {};
-		uint64_t current_index = {};
-
-		cJSON *output_root = nullptr;
-		cJSON *output_array = nullptr;
-	} gst = {};
-#endif
-
-	struct ht_view views[2] = {};
-
-	float baseline = {};
-	struct xrt_quat stereo_camera_to_left_camera = {};
-
-	uint64_t current_frame_timestamp = {};
-
-	std::vector<HandHistory3D> histories_3d = {};
-
-	struct os_mutex openxr_hand_data_mediator = {};
-	struct xrt_hand_joint_set hands_for_openxr[2] = {};
-	uint64_t hands_for_openxr_timestamp = {};
-
-	// Only change these when you have unlocked_between_frames, ie. when the hand tracker is between frames.
-	bool tracking_should_die = {};
-	bool tracking_should_record_dataset = {};
-	struct os_mutex unlocked_between_frames = {};
-
-	// Change this whenever you want
-	volatile bool debug_scribble = true;
-
-	struct ht_startup_config startup_config = {};
-	struct ht_dynamic_config dynamic_config = {};
-
-	enum u_logging_level log_level = U_LOGGING_INFO;
-
-public:
-	explicit HandTracking();
-	~HandTracking();
-
-	static inline HandTracking &
-	fromC(t_hand_tracking_sync *ht_sync)
-	{
-		return *reinterpret_cast<HandTracking *>(ht_sync);
-	}
-
-	static void
-	cCallbackProcess(struct t_hand_tracking_sync *ht_sync,
-	                 struct xrt_frame *left_frame,
-	                 struct xrt_frame *right_frame,
-	                 struct xrt_hand_joint_set *out_left_hand,
-	                 struct xrt_hand_joint_set *out_right_hand,
-	                 uint64_t *out_timestamp_ns);
-
-	static void
-	cCallbackDestroy(t_hand_tracking_sync *ht_sync);
-};
-
-
-} // namespace xrt::tracking::hand::old_rgb

src/xrt/tracking/hand/old_rgb/templates/NaivePermutationSort.hpp

···
-// Copyright 2021, Collabora, Ltd.
-// SPDX-License-Identifier: BSL-1.0
-/*!
- * @file
- * @brief  Camera based hand tracking sorting implementation.
- * @author Moses Turner <[e-mail]>
- * @ingroup drv_ht
- */
-
-#pragma once
-
-#include <math.h>
-#include <vector>
-#include <algorithm>
-#include <iostream>
-// Other thing: sort by speed? like, if our thing must have suddenly changed directions, add to error?
-// Easy enough to do using more complicated structs.
-// Like a past thing with position, velocity and timestamp - present thing with position and timestamp.
-
-// typedef bool booool;
-
-struct psort_atom_t
-{
-	size_t idx_1;
-	size_t idx_2;
-	float err;
-};
-
-
-bool
-comp_err(psort_atom_t one, psort_atom_t two)
-{
-	return (one.err < two.err);
-}
-
-
-template <typename Tp_1, typename Tp_2>
-void
-naive_sort_permutation_by_error(
-    // Inputs - shall be initialized with real data before calling. This function shall not modify them in any way.
-    std::vector<Tp_1> &in_1,
-    std::vector<Tp_2> &in_2,
-
-    // Outputs - shall be uninitialized. This function shall initialize them to the right size and fill them with the
-    // proper values.
-    std::vector<bool> &used_1,
-    std::vector<bool> &used_2,
-    std::vector<size_t> &out_indices_1,
-    std::vector<size_t> &out_indices_2,
-    std::vector<float> &out_errs,
-
-    float (*calc_error)(const Tp_1 &one, const Tp_2 &two),
-    float max_err = std::numeric_limits<float>::max())
-{
-	used_1 = std::vector<bool>(in_1.size()); // silly? Unsure.
-	used_2 = std::vector<bool>(in_2.size());
-
-	size_t out_size = std::min(in_1.size(), in_2.size());
-
-	out_indices_1.reserve(out_size);
-	out_indices_2.reserve(out_size);
-
-	std::vector<psort_atom_t> associations;
-
-	for (size_t idx_1 = 0; idx_1 < in_1.size(); idx_1++) {
-		for (size_t idx_2 = 0; idx_2 < in_2.size(); idx_2++) {
-			float err = calc_error(in_1[idx_1], in_2[idx_2]);
-			if (err > 0.0f) {
-				// Negative error means the error calculator thought there was something so bad with
-				// these that they shouldn't be considered at all.
-				associations.push_back({idx_1, idx_2, err});
-			}
-		}
-	}
-
-	std::sort(associations.begin(), associations.end(), comp_err);
-
-	for (size_t i = 0; i < associations.size(); i++) {
-		psort_atom_t chonk = associations[i];
-		if (used_1[chonk.idx_1] || used_2[chonk.idx_2] || (chonk.err > max_err)) {
-			continue;
-		}
-		used_1[chonk.idx_1] = true;
-		used_2[chonk.idx_2] = true;
-
-		out_indices_1.push_back(chonk.idx_1);
-		out_indices_2.push_back(chonk.idx_2);
-
-		out_errs.push_back(chonk.err);
-	}
-}