Merge pull request #88411 from dsnopek/webxr-hand-tracking

WebXR: Add support for hand tracking
This commit is contained in:
Rémi Verschelde 2024-02-23 22:57:46 +01:00
commit cf9de66389
No known key found for this signature in database
GPG key ID: C3336907360768E1
8 changed files with 182 additions and 26 deletions

View file

@ -153,6 +153,10 @@
</method>
</methods>
<members>
<member name="enabled_features" type="String" setter="" getter="get_enabled_features">
A comma-separated list of features that were successfully enabled by [method XRInterface.initialize] when setting up the WebXR session.
This may include features requested by setting [member required_features] and [member optional_features].
</member>
<member name="optional_features" type="String" setter="set_optional_features" getter="get_optional_features">
A comma-seperated list of optional features used by [method XRInterface.initialize] when setting up the WebXR session.
If a user's browser or device doesn't support one of the given features, initialization will continue, but you won't be able to use the requested feature.

View file

@ -45,7 +45,7 @@ enum WebXRInputEvent {
};
typedef void (*GodotWebXRSupportedCallback)(char *p_session_mode, int p_supported);
typedef void (*GodotWebXRStartedCallback)(char *p_reference_space_type);
typedef void (*GodotWebXRStartedCallback)(char *p_reference_space_type, char *p_enabled_features);
typedef void (*GodotWebXREndedCallback)();
typedef void (*GodotWebXRFailedCallback)(char *p_message);
typedef void (*GodotWebXRInputEventCallback)(int p_event_type, int p_input_source_id);
@ -85,7 +85,10 @@ extern bool godot_webxr_update_input_source(
int *r_button_count,
float *r_buttons,
int *r_axes_count,
float *r_axes);
float *r_axes,
int *r_has_hand_data,
float *r_hand_joints,
float *r_hand_radii);
extern char *godot_webxr_get_visibility_state();
extern int godot_webxr_get_bounds_geometry(float **r_points);

View file

@ -318,9 +318,11 @@ const GodotWebXR = {
// callback don't bubble up here and cause Godot to try the
// next reference space.
window.setTimeout(function () {
const c_str = GodotRuntime.allocString(reference_space_type);
onstarted(c_str);
GodotRuntime.free(c_str);
const reference_space_c_str = GodotRuntime.allocString(reference_space_type);
const enabled_features_c_str = GodotRuntime.allocString(Array.from(session.enabledFeatures).join(","));
onstarted(reference_space_c_str, enabled_features_c_str);
GodotRuntime.free(reference_space_c_str);
GodotRuntime.free(enabled_features_c_str);
}, 0);
}
@ -479,8 +481,8 @@ const GodotWebXR = {
},
godot_webxr_update_input_source__proxy: 'sync',
godot_webxr_update_input_source__sig: 'iiiiiiiiiiii',
godot_webxr_update_input_source: function (p_input_source_id, r_target_pose, r_target_ray_mode, r_touch_index, r_has_grip_pose, r_grip_pose, r_has_standard_mapping, r_button_count, r_buttons, r_axes_count, r_axes) {
godot_webxr_update_input_source__sig: 'iiiiiiiiiiiiiii',
godot_webxr_update_input_source: function (p_input_source_id, r_target_pose, r_target_ray_mode, r_touch_index, r_has_grip_pose, r_grip_pose, r_has_standard_mapping, r_button_count, r_buttons, r_axes_count, r_axes, r_has_hand_data, r_hand_joints, r_hand_radii) {
if (!GodotWebXR.session || !GodotWebXR.frame) {
return 0;
}
@ -563,6 +565,19 @@ const GodotWebXR = {
GodotRuntime.setHeapValue(r_button_count, button_count, 'i32');
GodotRuntime.setHeapValue(r_axes_count, axes_count, 'i32');
// Hand tracking data.
let has_hand_data = false;
if (input_source.hand && r_hand_joints != 0 && r_hand_radii != 0) {
const hand_joint_array = new Float32Array(25 * 16);
const hand_radii_array = new Float32Array(25);
if (frame.fillPoses(input_source.hand.values(), space, hand_joint_array) && frame.fillJointRadii(input_source.hand.values(), hand_radii_array)) {
GodotRuntime.heapCopy(HEAPF32, hand_joint_array, r_hand_joints);
GodotRuntime.heapCopy(HEAPF32, hand_radii_array, r_hand_radii);
has_hand_data = true;
}
}
GodotRuntime.setHeapValue(r_has_hand_data, has_hand_data ? 1 : 0, 'i32');
return true;
},

View file

@ -229,13 +229,27 @@ XRFrame.prototype.session;
XRFrame.prototype.getViewerPose = function (referenceSpace) {};
/**
*
* @param {XRSpace} space
* @param {XRSpace} baseSpace
* @return {XRPose}
*/
XRFrame.prototype.getPose = function (space, baseSpace) {};
/**
* @param {Array<XRSpace>} spaces
* @param {XRSpace} baseSpace
* @param {Float32Array} transforms
* @return {boolean}
*/
XRFrame.prototype.fillPoses = function (spaces, baseSpace, transforms) {};
/**
* @param {Array<XRJointSpace>} jointSpaces
* @param {Float32Array} radii
* @return {boolean}
*/
XRFrame.prototype.fillJointRadii = function (jointSpaces, radii) {};
/**
* @constructor
*/
@ -498,11 +512,51 @@ XRInputSource.prototype.targetRayMode;
*/
XRInputSource.prototype.targetRaySpace;
/**
* @type {?XRHand}
*/
XRInputSource.prototype.hand;
/**
* @constructor
*/
function XRHand() {};
/**
* Note: In fact, XRHand acts like a Map<string, XRJointSpace>, but I don't know
* how to represent that here. So, we're just giving the one method we call.
*
* @return {Array<XRJointSpace>}
*/
XRHand.prototype.values = function () {};
/**
* @type {number}
*/
XRHand.prototype.size;
/**
* @param {string} key
* @return {XRJointSpace}
*/
XRHand.prototype.get = function (key) {};
/**
* @constructor
*/
function XRSpace() {};
/**
* @constructor
* @extends {XRSpace}
*/
function XRJointSpace() {};
/**
* @type {string}
*/
XRJointSpace.prototype.jointName;
/**
* @constructor
*/

View file

@ -41,6 +41,7 @@ void WebXRInterface::_bind_methods() {
ClassDB::bind_method(D_METHOD("set_optional_features", "optional_features"), &WebXRInterface::set_optional_features);
ClassDB::bind_method(D_METHOD("get_optional_features"), &WebXRInterface::get_optional_features);
ClassDB::bind_method(D_METHOD("get_reference_space_type"), &WebXRInterface::get_reference_space_type);
ClassDB::bind_method(D_METHOD("get_enabled_features"), &WebXRInterface::get_enabled_features);
ClassDB::bind_method(D_METHOD("set_requested_reference_space_types", "requested_reference_space_types"), &WebXRInterface::set_requested_reference_space_types);
ClassDB::bind_method(D_METHOD("get_requested_reference_space_types"), &WebXRInterface::get_requested_reference_space_types);
ClassDB::bind_method(D_METHOD("is_input_source_active", "input_source_id"), &WebXRInterface::is_input_source_active);
@ -56,6 +57,7 @@ void WebXRInterface::_bind_methods() {
ADD_PROPERTY(PropertyInfo(Variant::STRING, "optional_features", PROPERTY_HINT_NONE), "set_optional_features", "get_optional_features");
ADD_PROPERTY(PropertyInfo(Variant::STRING, "requested_reference_space_types", PROPERTY_HINT_NONE), "set_requested_reference_space_types", "get_requested_reference_space_types");
ADD_PROPERTY(PropertyInfo(Variant::STRING, "reference_space_type", PROPERTY_HINT_NONE), "", "get_reference_space_type");
ADD_PROPERTY(PropertyInfo(Variant::STRING, "enabled_features", PROPERTY_HINT_NONE), "", "get_enabled_features");
ADD_PROPERTY(PropertyInfo(Variant::STRING, "visibility_state", PROPERTY_HINT_NONE), "", "get_visibility_state");
ADD_SIGNAL(MethodInfo("session_supported", PropertyInfo(Variant::STRING, "session_mode"), PropertyInfo(Variant::BOOL, "supported")));

View file

@ -62,6 +62,7 @@ public:
virtual void set_requested_reference_space_types(String p_requested_reference_space_types) = 0;
virtual String get_requested_reference_space_types() const = 0;
virtual String get_reference_space_type() const = 0;
virtual String get_enabled_features() const = 0;
virtual bool is_input_source_active(int p_input_source_id) const = 0;
virtual Ref<XRPositionalTracker> get_input_source_tracker(int p_input_source_id) const = 0;
virtual TargetRayMode get_input_source_target_ray_mode(int p_input_source_id) const = 0;

View file

@ -41,6 +41,7 @@
#include "scene/main/window.h"
#include "servers/rendering/renderer_compositor.h"
#include "servers/rendering/rendering_server_globals.h"
#include "servers/xr/xr_hand_tracker.h"
#include <emscripten.h>
#include <stdlib.h>
@ -49,22 +50,23 @@ void _emwebxr_on_session_supported(char *p_session_mode, int p_supported) {
XRServer *xr_server = XRServer::get_singleton();
ERR_FAIL_NULL(xr_server);
Ref<XRInterface> interface = xr_server->find_interface("WebXR");
Ref<WebXRInterfaceJS> interface = xr_server->find_interface("WebXR");
ERR_FAIL_COND(interface.is_null());
String session_mode = String(p_session_mode);
interface->emit_signal(SNAME("session_supported"), session_mode, p_supported ? true : false);
}
void _emwebxr_on_session_started(char *p_reference_space_type) {
void _emwebxr_on_session_started(char *p_reference_space_type, char *p_enabled_features) {
XRServer *xr_server = XRServer::get_singleton();
ERR_FAIL_NULL(xr_server);
Ref<XRInterface> interface = xr_server->find_interface("WebXR");
Ref<WebXRInterfaceJS> interface = xr_server->find_interface("WebXR");
ERR_FAIL_COND(interface.is_null());
String reference_space_type = String(p_reference_space_type);
static_cast<WebXRInterfaceJS *>(interface.ptr())->_set_reference_space_type(reference_space_type);
interface->_set_reference_space_type(reference_space_type);
interface->_set_enabled_features(p_enabled_features);
interface->emit_signal(SNAME("session_started"));
}
@ -72,7 +74,7 @@ void _emwebxr_on_session_ended() {
XRServer *xr_server = XRServer::get_singleton();
ERR_FAIL_NULL(xr_server);
Ref<XRInterface> interface = xr_server->find_interface("WebXR");
Ref<WebXRInterfaceJS> interface = xr_server->find_interface("WebXR");
ERR_FAIL_COND(interface.is_null());
interface->uninitialize();
@ -83,7 +85,7 @@ void _emwebxr_on_session_failed(char *p_message) {
XRServer *xr_server = XRServer::get_singleton();
ERR_FAIL_NULL(xr_server);
Ref<XRInterface> interface = xr_server->find_interface("WebXR");
Ref<WebXRInterfaceJS> interface = xr_server->find_interface("WebXR");
ERR_FAIL_COND(interface.is_null());
interface->uninitialize();
@ -96,17 +98,17 @@ extern "C" EMSCRIPTEN_KEEPALIVE void _emwebxr_on_input_event(int p_event_type, i
XRServer *xr_server = XRServer::get_singleton();
ERR_FAIL_NULL(xr_server);
Ref<XRInterface> interface = xr_server->find_interface("WebXR");
Ref<WebXRInterfaceJS> interface = xr_server->find_interface("WebXR");
ERR_FAIL_COND(interface.is_null());
((WebXRInterfaceJS *)interface.ptr())->_on_input_event(p_event_type, p_input_source_id);
interface->_on_input_event(p_event_type, p_input_source_id);
}
extern "C" EMSCRIPTEN_KEEPALIVE void _emwebxr_on_simple_event(char *p_signal_name) {
XRServer *xr_server = XRServer::get_singleton();
ERR_FAIL_NULL(xr_server);
Ref<XRInterface> interface = xr_server->find_interface("WebXR");
Ref<WebXRInterfaceJS> interface = xr_server->find_interface("WebXR");
ERR_FAIL_COND(interface.is_null());
StringName signal_name = StringName(p_signal_name);
@ -149,14 +151,14 @@ String WebXRInterfaceJS::get_requested_reference_space_types() const {
return requested_reference_space_types;
}
void WebXRInterfaceJS::_set_reference_space_type(String p_reference_space_type) {
reference_space_type = p_reference_space_type;
}
String WebXRInterfaceJS::get_reference_space_type() const {
return reference_space_type;
}
String WebXRInterfaceJS::get_enabled_features() const {
return enabled_features;
}
bool WebXRInterfaceJS::is_input_source_active(int p_input_source_id) const {
ERR_FAIL_INDEX_V(p_input_source_id, input_source_count, false);
return input_sources[p_input_source_id].active;
@ -256,7 +258,9 @@ bool WebXRInterfaceJS::initialize() {
return false;
}
// we must create a tracker for our head
enabled_features.clear();
// We must create a tracker for our head.
head_transform.basis = Basis();
head_transform.origin = Vector3();
head_tracker.instantiate();
@ -265,7 +269,7 @@ bool WebXRInterfaceJS::initialize() {
head_tracker->set_tracker_desc("Players head");
xr_server->add_tracker(head_tracker);
// make this our primary interface
// Make this our primary interface.
xr_server->set_primary_interface(this);
// Clear render_targetsize to make sure it gets reset to the new size.
@ -301,6 +305,14 @@ void WebXRInterfaceJS::uninitialize() {
head_tracker.unref();
}
for (int i = 0; i < HAND_MAX; i++) {
if (hand_trackers[i].is_valid()) {
xr_server->remove_hand_tracker(i == 0 ? "/user/left" : "/user/right");
hand_trackers[i].unref();
}
}
if (xr_server->get_primary_interface() == this) {
// no longer our primary interface
xr_server->set_primary_interface(nullptr);
@ -321,7 +333,8 @@ void WebXRInterfaceJS::uninitialize() {
}
texture_cache.clear();
reference_space_type = "";
reference_space_type.clear();
enabled_features.clear();
initialized = false;
};
};
@ -572,6 +585,9 @@ void WebXRInterfaceJS::_update_input_source(int p_input_source_id) {
float buttons[10];
int axes_count;
float axes[10];
int has_hand_data;
float hand_joints[WEBXR_HAND_JOINT_MAX * 16];
float hand_radii[WEBXR_HAND_JOINT_MAX];
input_source.active = godot_webxr_update_input_source(
p_input_source_id,
@ -584,7 +600,10 @@ void WebXRInterfaceJS::_update_input_source(int p_input_source_id) {
&button_count,
buttons,
&axes_count,
axes);
axes,
&has_hand_data,
hand_joints,
hand_radii);
if (!input_source.active) {
if (input_source.tracker.is_valid()) {
@ -683,6 +702,56 @@ void WebXRInterfaceJS::_update_input_source(int p_input_source_id) {
touches[touch_index].position = position;
}
}
if (p_input_source_id < 2) {
Ref<XRHandTracker> hand_tracker = hand_trackers[p_input_source_id];
if (has_hand_data) {
// Transform orientations to match Godot Humanoid skeleton.
const Basis bone_adjustment(
Vector3(-1.0, 0.0, 0.0),
Vector3(0.0, 0.0, -1.0),
Vector3(0.0, -1.0, 0.0));
if (unlikely(hand_tracker.is_null())) {
hand_tracker.instantiate();
hand_tracker->set_hand(p_input_source_id == 0 ? XRHandTracker::HAND_LEFT : XRHandTracker::HAND_RIGHT);
// These flags always apply, since WebXR doesn't give us enough insight to be more fine grained.
BitField<XRHandTracker::HandJointFlags> joint_flags(XRHandTracker::HAND_JOINT_FLAG_POSITION_VALID | XRHandTracker::HAND_JOINT_FLAG_ORIENTATION_VALID | XRHandTracker::HAND_JOINT_FLAG_POSITION_TRACKED | XRHandTracker::HAND_JOINT_FLAG_ORIENTATION_TRACKED);
for (int godot_joint = 0; godot_joint < XRHandTracker::HAND_JOINT_MAX; godot_joint++) {
hand_tracker->set_hand_joint_flags((XRHandTracker::HandJoint)godot_joint, joint_flags);
}
hand_trackers[p_input_source_id] = hand_tracker;
xr_server->add_hand_tracker(p_input_source_id == 0 ? "/user/left" : "/user/right", hand_tracker);
}
hand_tracker->set_has_tracking_data(true);
for (int webxr_joint = 0; webxr_joint < WEBXR_HAND_JOINT_MAX; webxr_joint++) {
XRHandTracker::HandJoint godot_joint = (XRHandTracker::HandJoint)(webxr_joint + 1);
Transform3D joint_transform = _js_matrix_to_transform(hand_joints + (16 * webxr_joint));
joint_transform.basis *= bone_adjustment;
hand_tracker->set_hand_joint_transform(godot_joint, joint_transform);
hand_tracker->set_hand_joint_radius(godot_joint, hand_radii[webxr_joint]);
}
// WebXR doesn't have a palm joint, so we calculate it by finding the middle of the middle finger metacarpal bone.
{
// 10 is the WebXR middle finger metacarpal joint, and 12 is the offset to the transform origin.
const float *start_pos = hand_joints + (10 * 16) + 12;
// 11 is the WebXR middle finger phalanx proximal joint, and 12 is the offset to the transform origin.
const float *end_pos = hand_joints + (11 * 16) + 12;
Transform3D palm_transform;
palm_transform.origin = (Vector3(start_pos[0], start_pos[1], start_pos[2]) + Vector3(end_pos[0], end_pos[1], end_pos[2])) / 2.0;
hand_tracker->set_hand_joint_transform(XRHandTracker::HAND_JOINT_PALM, palm_transform);
}
} else if (hand_tracker.is_valid()) {
hand_tracker->set_has_tracking_data(false);
}
}
}
void WebXRInterfaceJS::_on_input_event(int p_event_type, int p_input_source_id) {

View file

@ -56,6 +56,7 @@ private:
String optional_features;
String requested_reference_space_types;
String reference_space_type;
String enabled_features;
Size2 render_targetsize;
RBMap<unsigned int, RID> texture_cache;
@ -73,6 +74,10 @@ private:
int touch_index = -1;
} input_sources[input_source_count];
static const int WEBXR_HAND_JOINT_MAX = 25;
static const int HAND_MAX = 2;
Ref<XRHandTracker> hand_trackers[HAND_MAX];
RID color_texture;
RID depth_texture;
@ -94,8 +99,8 @@ public:
virtual String get_optional_features() const override;
virtual void set_requested_reference_space_types(String p_requested_reference_space_types) override;
virtual String get_requested_reference_space_types() const override;
void _set_reference_space_type(String p_reference_space_type);
virtual String get_reference_space_type() const override;
virtual String get_enabled_features() const override;
virtual bool is_input_source_active(int p_input_source_id) const override;
virtual Ref<XRPositionalTracker> get_input_source_tracker(int p_input_source_id) const override;
virtual TargetRayMode get_input_source_target_ray_mode(int p_input_source_id) const override;
@ -129,6 +134,9 @@ public:
void _on_input_event(int p_event_type, int p_input_source_id);
inline void _set_reference_space_type(String p_reference_space_type) { reference_space_type = p_reference_space_type; }
inline void _set_enabled_features(String p_enabled_features) { enabled_features = p_enabled_features; }
WebXRInterfaceJS();
~WebXRInterfaceJS();