Add support for the OpenXR Eye gaze interaction extension

Co-authored-by: Bastiaan Olij <mux213@gmail.com>

doc/classes/ProjectSettings.xml

@@ -2718,6 +2718,9 @@
 		<member name="xr/openxr/environment_blend_mode" type="int" setter="" getter="" default="&quot;0&quot;">
 			Specify how OpenXR should blend in the environment. This is specific to certain AR and passthrough devices where camera images are blended in by the XR compositor.
 		</member>
+		<member name="xr/openxr/extensions/eye_gaze_interaction" type="bool" setter="" getter="" default="false">
+			Specify whether to enable eye tracking for this project. Depending on the platform, additional export configuration may be needed.
+		</member>
 		<member name="xr/openxr/form_factor" type="int" setter="" getter="" default="&quot;0&quot;">
 			Specify whether OpenXR should be configured for an HMD or a hand held device.
 		</member>

main/main.cpp

@@ -2113,6 +2113,9 @@ Error Main::setup(const char *execpath, int argc, char *argv[], bool p_second_ph
 	GLOBAL_DEF_BASIC("xr/openxr/submit_depth_buffer", false);
 	GLOBAL_DEF_BASIC("xr/openxr/startup_alert", true);
 
+	// XR project extensions settings.
+	GLOBAL_DEF_BASIC("xr/openxr/extensions/eye_gaze_interaction", false);
+
 #ifdef TOOLS_ENABLED
 	// Disabled for now, using XR inside of the editor we'll be working on during the coming months.
 

modules/openxr/SCsub

@@ -106,6 +106,7 @@ if env["opengl3"] and env["platform"] != "macos":
 
 env_openxr.add_source_files(module_obj, "extensions/openxr_palm_pose_extension.cpp")
 env_openxr.add_source_files(module_obj, "extensions/openxr_composition_layer_depth_extension.cpp")
+env_openxr.add_source_files(module_obj, "extensions/openxr_eye_gaze_interaction.cpp")
 env_openxr.add_source_files(module_obj, "extensions/openxr_htc_controller_extension.cpp")
 env_openxr.add_source_files(module_obj, "extensions/openxr_htc_vive_tracker_extension.cpp")
 env_openxr.add_source_files(module_obj, "extensions/openxr_huawei_controller_extension.cpp")

modules/openxr/action_map/openxr_action_map.cpp

@@ -206,7 +206,8 @@ void OpenXRActionMap::create_default_action_sets() {
 			"/user/vive_tracker_htcx/role/waist,"
 			"/user/vive_tracker_htcx/role/chest,"
 			"/user/vive_tracker_htcx/role/camera,"
-			"/user/vive_tracker_htcx/role/keyboard");
+			"/user/vive_tracker_htcx/role/keyboard,"
+			"/user/eyes_ext");
 	Ref<OpenXRAction> aim_pose = action_set->add_new_action("aim_pose", "Aim pose", OpenXRAction::OPENXR_ACTION_POSE, "/user/hand/left,/user/hand/right");
 	Ref<OpenXRAction> grip_pose = action_set->add_new_action("grip_pose", "Grip pose", OpenXRAction::OPENXR_ACTION_POSE, "/user/hand/left,/user/hand/right");
 	Ref<OpenXRAction> palm_pose = action_set->add_new_action("palm_pose", "Palm pose", OpenXRAction::OPENXR_ACTION_POSE, "/user/hand/left,/user/hand/right");
@@ -503,6 +504,11 @@ void OpenXRActionMap::create_default_action_sets() {
 			"/user/vive_tracker_htcx/role/camera/output/haptic,"
 			"/user/vive_tracker_htcx/role/keyboard/output/haptic");
 	add_interaction_profile(profile);
+
+	// Create our eye gaze interaction profile
+	profile = OpenXRInteractionProfile::new_profile("/interaction_profiles/ext/eye_gaze_interaction");
+	profile->add_new_binding(default_pose, "/user/eyes_ext/input/gaze_ext/pose");
+	add_interaction_profile(profile);
 }
 
 void OpenXRActionMap::create_editor_action_sets() {

modules/openxr/doc_classes/OpenXRInterface.xml

@@ -77,6 +77,13 @@
 				Returns [code]true[/code] if the given action set is active.
 			</description>
 		</method>
+		<method name="is_eye_gaze_interaction_supported">
+			<return type="bool" />
+			<description>
+				Returns the capabilities of the eye gaze interaction extension.
+				[b]Note:[/b] This only returns a valid value after OpenXR has been initialized.
+			</description>
+		</method>
 		<method name="is_foveation_supported" qualifiers="const">
 			<return type="bool" />
 			<description>

modules/openxr/extensions/openxr_eye_gaze_interaction.cpp

@@ -0,0 +1,98 @@
+/**************************************************************************/
+/*  openxr_eye_gaze_interaction.cpp                                       */
+/**************************************************************************/
+/*                         This file is part of:                          */
+/*                             GODOT ENGINE                               */
+/*                        https://godotengine.org                         */
+/**************************************************************************/
+/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
+/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur.                  */
+/*                                                                        */
+/* Permission is hereby granted, free of charge, to any person obtaining  */
+/* a copy of this software and associated documentation files (the        */
+/* "Software"), to deal in the Software without restriction, including    */
+/* without limitation the rights to use, copy, modify, merge, publish,    */
+/* distribute, sublicense, and/or sell copies of the Software, and to     */
+/* permit persons to whom the Software is furnished to do so, subject to  */
+/* the following conditions:                                              */
+/*                                                                        */
+/* The above copyright notice and this permission notice shall be         */
+/* included in all copies or substantial portions of the Software.        */
+/*                                                                        */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,        */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF     */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY   */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,   */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE      */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                 */
+/**************************************************************************/
+
+#include "openxr_eye_gaze_interaction.h"
+
+#include "core/os/os.h"
+
+#include "../action_map/openxr_interaction_profile_metadata.h"
+
+OpenXREyeGazeInteractionExtension *OpenXREyeGazeInteractionExtension::singleton = nullptr;
+
+OpenXREyeGazeInteractionExtension *OpenXREyeGazeInteractionExtension::get_singleton() {
+	ERR_FAIL_NULL_V(singleton, nullptr);
+	return singleton;
+}
+
+OpenXREyeGazeInteractionExtension::OpenXREyeGazeInteractionExtension() {
+	singleton = this;
+}
+
+OpenXREyeGazeInteractionExtension::~OpenXREyeGazeInteractionExtension() {
+	singleton = nullptr;
+}
+
+HashMap<String, bool *> OpenXREyeGazeInteractionExtension::get_requested_extensions() {
+	HashMap<String, bool *> request_extensions;
+
+	request_extensions[XR_EXT_EYE_GAZE_INTERACTION_EXTENSION_NAME] = &available;
+
+	return request_extensions;
+}
+
+void *OpenXREyeGazeInteractionExtension::set_system_properties_and_get_next_pointer(void *p_next_pointer) {
+	if (!available) {
+		return p_next_pointer;
+	}
+
+	properties.type = XR_TYPE_SYSTEM_EYE_GAZE_INTERACTION_PROPERTIES_EXT;
+	properties.next = p_next_pointer;
+	properties.supportsEyeGazeInteraction = false;
+
+	return &properties;
+}
+
+bool OpenXREyeGazeInteractionExtension::is_available() {
+	return available;
+}
+
+bool OpenXREyeGazeInteractionExtension::supports_eye_gaze_interaction() {
+	// The extension being available only means that the OpenXR Runtime supports the extension.
+	// The `supportsEyeGazeInteraction` is set to true if the device also supports this.
+	// Thus both need to be true.
+	// In addition, on mobile runtimes, the proper permission needs to be granted.
+	if (available && properties.supportsEyeGazeInteraction) {
+		return !OS::get_singleton()->has_feature("mobile") || OS::get_singleton()->has_feature("PERMISSION_XR_EXT_eye_gaze_interaction");
+	}
+
+	return false;
+}
+
+void OpenXREyeGazeInteractionExtension::on_register_metadata() {
+	OpenXRInteractionProfileMetadata *metadata = OpenXRInteractionProfileMetadata::get_singleton();
+	ERR_FAIL_NULL(metadata);
+
+	// Eyes top path
+	metadata->register_top_level_path("Eye gaze tracker", "/user/eyes_ext", XR_EXT_EYE_GAZE_INTERACTION_EXTENSION_NAME);
+
+	// Eye gaze interaction
+	metadata->register_interaction_profile("Eye gaze", "/interaction_profiles/ext/eye_gaze_interaction", XR_EXT_EYE_GAZE_INTERACTION_EXTENSION_NAME);
+	metadata->register_io_path("/interaction_profiles/ext/eye_gaze_interaction", "Gaze pose", "/user/eyes_ext", "/user/eyes_ext/input/gaze_ext/pose", "", OpenXRAction::OPENXR_ACTION_POSE);
+}

modules/openxr/extensions/openxr_eye_gaze_interaction.h

@@ -0,0 +1,58 @@
+/**************************************************************************/
+/*  openxr_eye_gaze_interaction.h                                         */
+/**************************************************************************/
+/*                         This file is part of:                          */
+/*                             GODOT ENGINE                               */
+/*                        https://godotengine.org                         */
+/**************************************************************************/
+/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
+/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur.                  */
+/*                                                                        */
+/* Permission is hereby granted, free of charge, to any person obtaining  */
+/* a copy of this software and associated documentation files (the        */
+/* "Software"), to deal in the Software without restriction, including    */
+/* without limitation the rights to use, copy, modify, merge, publish,    */
+/* distribute, sublicense, and/or sell copies of the Software, and to     */
+/* permit persons to whom the Software is furnished to do so, subject to  */
+/* the following conditions:                                              */
+/*                                                                        */
+/* The above copyright notice and this permission notice shall be         */
+/* included in all copies or substantial portions of the Software.        */
+/*                                                                        */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,        */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF     */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY   */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,   */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE      */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                 */
+/**************************************************************************/
+
+#ifndef OPENXR_EYE_GAZE_INTERACTION_H
+#define OPENXR_EYE_GAZE_INTERACTION_H
+
+#include "openxr_extension_wrapper.h"
+
+class OpenXREyeGazeInteractionExtension : public OpenXRExtensionWrapper {
+public:
+	static OpenXREyeGazeInteractionExtension *get_singleton();
+
+	OpenXREyeGazeInteractionExtension();
+	~OpenXREyeGazeInteractionExtension();
+
+	virtual HashMap<String, bool *> get_requested_extensions() override;
+	virtual void *set_system_properties_and_get_next_pointer(void *p_next_pointer) override;
+
+	bool is_available();
+	bool supports_eye_gaze_interaction();
+
+	virtual void on_register_metadata() override;
+
+private:
+	static OpenXREyeGazeInteractionExtension *singleton;
+
+	bool available = false;
+	XrSystemEyeGazeInteractionPropertiesEXT properties;
+};
+
+#endif // OPENXR_EYE_GAZE_INTERACTION_H

modules/openxr/openxr_interface.cpp

@@ -36,6 +36,8 @@
 
 #include "extensions/openxr_hand_tracking_extension.h"
 
+#include "extensions/openxr_eye_gaze_interaction.h"
+
 void OpenXRInterface::_bind_methods() {
 	// lifecycle signals
 	ADD_SIGNAL(MethodInfo("session_begun"));
@@ -119,6 +121,8 @@ void OpenXRInterface::_bind_methods() {
 	BIND_ENUM_CONSTANT(HAND_JOINT_LITTLE_DISTAL);
 	BIND_ENUM_CONSTANT(HAND_JOINT_LITTLE_TIP);
 	BIND_ENUM_CONSTANT(HAND_JOINT_MAX);
+
+	ClassDB::bind_method(D_METHOD("is_eye_gaze_interaction_supported"), &OpenXRInterface::is_eye_gaze_interaction_supported);
 }
 
 StringName OpenXRInterface::get_name() const {
@@ -152,7 +156,9 @@ PackedStringArray OpenXRInterface::get_suggested_tracker_names() const {
 		"/user/vive_tracker_htcx/role/waist",
 		"/user/vive_tracker_htcx/role/chest",
 		"/user/vive_tracker_htcx/role/camera",
-		"/user/vive_tracker_htcx/role/keyboard"
+		"/user/vive_tracker_htcx/role/keyboard",
+
+		"/user/eyes_ext",
 	};
 
 	return arr;
@@ -705,6 +711,21 @@ Array OpenXRInterface::get_available_display_refresh_rates() const {
 	}
 }
 
+bool OpenXRInterface::is_eye_gaze_interaction_supported() {
+	if (openxr_api == nullptr) {
+		return false;
+	} else if (!openxr_api->is_initialized()) {
+		return false;
+	} else {
+		OpenXREyeGazeInteractionExtension *eye_gaze_ext = OpenXREyeGazeInteractionExtension::get_singleton();
+		if (eye_gaze_ext == nullptr) {
+			return false;
+		} else {
+			return eye_gaze_ext->supports_eye_gaze_interaction();
+		}
+	}
+}
+
 bool OpenXRInterface::is_action_set_active(const String &p_action_set) const {
 	for (ActionSet *action_set : action_sets) {
 		if (action_set->action_set_name == p_action_set) {

modules/openxr/openxr_interface.h

@@ -107,6 +107,8 @@ public:
 	virtual PackedStringArray get_suggested_tracker_names() const override;
 	virtual TrackingStatus get_tracking_status() const override;
 
+	bool is_eye_gaze_interaction_supported();
+
 	bool initialize_on_startup() const;
 	virtual bool is_initialized() const override;
 	virtual bool initialize() override;

modules/openxr/register_types.cpp

@@ -42,6 +42,7 @@
 #include "scene/openxr_hand.h"
 
 #include "extensions/openxr_composition_layer_depth_extension.h"
+#include "extensions/openxr_eye_gaze_interaction.h"
 #include "extensions/openxr_fb_display_refresh_rate_extension.h"
 #include "extensions/openxr_fb_passthrough_extension_wrapper.h"
 #include "extensions/openxr_hand_tracking_extension.h"
@@ -104,6 +105,9 @@ void initialize_openxr_module(ModuleInitializationLevel p_level) {
 #endif
 
 			// register our other extensions
+			if (GLOBAL_GET("xr/openxr/extensions/eye_gaze_interaction") && (!OS::get_singleton()->has_feature("mobile") || OS::get_singleton()->has_feature(XR_EXT_EYE_GAZE_INTERACTION_EXTENSION_NAME))) {
+				OpenXRAPI::register_extension_wrapper(memnew(OpenXREyeGazeInteractionExtension));
+			}
 			OpenXRAPI::register_extension_wrapper(memnew(OpenXRPalmPoseExtension));
 			OpenXRAPI::register_extension_wrapper(memnew(OpenXRPicoControllerExtension));
 			OpenXRAPI::register_extension_wrapper(memnew(OpenXRCompositionLayerDepthExtension));

platform/android/java/editor/src/main/java/org/godotengine/editor/GodotEditor.kt

@@ -91,10 +91,6 @@ open class GodotEditor : GodotActivity() {
 	private val commandLineParams = ArrayList<String>()
 
 	override fun onCreate(savedInstanceState: Bundle?) {
-		// We exclude certain permissions from the set we request at startup, as they'll be
-		// requested on demand based on use-cases.
-		PermissionsUtil.requestManifestPermissions(this, setOf(Manifest.permission.RECORD_AUDIO))
-
 		val params = intent.getStringArrayExtra(EXTRA_COMMAND_LINE_PARAMS)
 		Log.d(TAG, "Received parameters ${params.contentToString()}")
 		updateCommandLineParams(params)

platform/android/java/lib/src/org/godotengine/godot/Godot.kt

@@ -927,6 +927,19 @@ class Godot(private val context: Context) : SensorEventListener {
 		return PermissionsUtil.getGrantedPermissions(getActivity())
 	}
 
+	/**
+	 * Return true if the given feature is supported.
+	 */
+	@Keep
+	private fun hasFeature(feature: String): Boolean {
+		for (plugin in pluginRegistry.allPlugins) {
+			if (plugin.supportsFeature(feature)) {
+				return true
+			}
+		}
+		return false
+	}
+
 	/**
 	 * Get the list of gdextension modules to register.
 	 */

platform/android/java/lib/src/org/godotengine/godot/GodotActivity.kt

@@ -30,12 +30,15 @@
 
 package org.godotengine.godot
 
+import android.Manifest
 import android.app.Activity
 import android.content.Intent
+import android.content.pm.PackageManager
 import android.os.Bundle
 import android.util.Log
 import androidx.annotation.CallSuper
 import androidx.fragment.app.FragmentActivity
+import org.godotengine.godot.utils.PermissionsUtil
 import org.godotengine.godot.utils.ProcessPhoenix
 
 /**
@@ -62,6 +65,10 @@ abstract class GodotActivity : FragmentActivity(), GodotHost {
 		private set
 
 	override fun onCreate(savedInstanceState: Bundle?) {
+		// We exclude certain permissions from the set we request at startup, as they'll be
+		// requested on demand based on use-cases.
+		PermissionsUtil.requestManifestPermissions(this, setOf(Manifest.permission.RECORD_AUDIO))
+
 		super.onCreate(savedInstanceState)
 		setContentView(R.layout.godot_app_layout)
 
@@ -148,6 +155,14 @@ abstract class GodotActivity : FragmentActivity(), GodotHost {
 	override fun onRequestPermissionsResult(requestCode: Int, permissions: Array<String>, grantResults: IntArray) {
 		super.onRequestPermissionsResult(requestCode, permissions, grantResults)
 		godotFragment?.onRequestPermissionsResult(requestCode, permissions, grantResults)
+
+		if (requestCode == PermissionsUtil.REQUEST_ALL_PERMISSION_REQ_CODE) {
+			Log.d(TAG, "Received permissions request result..")
+			for (i in permissions.indices) {
+				val permissionGranted = grantResults[i] == PackageManager.PERMISSION_GRANTED
+				Log.d(TAG, "Permission ${permissions[i]} ${if (permissionGranted) { "granted"} else { "denied" }}")
+			}
+		}
 	}
 
 	override fun onBackPressed() {

platform/android/java/lib/src/org/godotengine/godot/plugin/GodotPlugin.java

@@ -315,6 +315,15 @@ public abstract class GodotPlugin {
 		return true;
 	}
 
+	/**
+	 * Returns whether the plugin supports the given feature tag.
+	 *
+	 * @see <a href="https://docs.godotengine.org/en/stable/tutorials/export/feature_tags.html">Feature tags</a>
+	 */
+	public boolean supportsFeature(String featureTag) {
+		return false;
+	}
+
 	/**
 	 * Runs the specified action on the UI thread. If the current thread is the UI
 	 * thread, then the action is executed immediately. If the current thread is

platform/android/java/lib/src/org/godotengine/godot/utils/PermissionsUtil.java

@@ -160,6 +160,7 @@ public final class PermissionsUtil {
 			try {
 				if (manifestPermission.equals(Manifest.permission.MANAGE_EXTERNAL_STORAGE)) {
 					if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R && !Environment.isExternalStorageManager()) {
+						Log.d(TAG, "Requesting permission " + manifestPermission);
 						try {
 							Intent intent = new Intent(Settings.ACTION_MANAGE_APP_ALL_FILES_ACCESS_PERMISSION);
 							intent.setData(Uri.parse(String.format("package:%s", activity.getPackageName())));
@@ -173,6 +174,7 @@ public final class PermissionsUtil {
 					PermissionInfo permissionInfo = getPermissionInfo(activity, manifestPermission);
 					int protectionLevel = Build.VERSION.SDK_INT >= Build.VERSION_CODES.P ? permissionInfo.getProtection() : permissionInfo.protectionLevel;
 					if (protectionLevel == PermissionInfo.PROTECTION_DANGEROUS && ContextCompat.checkSelfPermission(activity, manifestPermission) != PackageManager.PERMISSION_GRANTED) {
+						Log.d(TAG, "Requesting permission " + manifestPermission);
 						requestedPermissions.add(manifestPermission);
 					}
 				}

platform/android/java_godot_wrapper.cpp

@@ -82,6 +82,7 @@ GodotJavaWrapper::GodotJavaWrapper(JNIEnv *p_env, jobject p_activity, jobject p_
 	_end_benchmark_measure = p_env->GetMethodID(godot_class, "nativeEndBenchmarkMeasure", "(Ljava/lang/String;)V");
 	_dump_benchmark = p_env->GetMethodID(godot_class, "nativeDumpBenchmark", "(Ljava/lang/String;)V");
 	_get_gdextension_list_config_file = p_env->GetMethodID(godot_class, "getGDExtensionConfigFiles", "()[Ljava/lang/String;");
+	_has_feature = p_env->GetMethodID(godot_class, "hasFeature", "(Ljava/lang/String;)Z");
 }
 
 GodotJavaWrapper::~GodotJavaWrapper() {
@@ -373,3 +374,15 @@ void GodotJavaWrapper::dump_benchmark(const String &benchmark_file) {
 		env->CallVoidMethod(godot_instance, _dump_benchmark, j_benchmark_file);
 	}
 }
+
+bool GodotJavaWrapper::has_feature(const String &p_feature) const {
+	if (_has_feature) {
+		JNIEnv *env = get_jni_env();
+		ERR_FAIL_NULL_V(env, false);
+
+		jstring j_feature = env->NewStringUTF(p_feature.utf8().get_data());
+		return env->CallBooleanMethod(godot_instance, _has_feature, j_feature);
+	} else {
+		return false;
+	}
+}

platform/android/java_godot_wrapper.h

@@ -73,6 +73,7 @@ private:
 	jmethodID _begin_benchmark_measure = nullptr;
 	jmethodID _end_benchmark_measure = nullptr;
 	jmethodID _dump_benchmark = nullptr;
+	jmethodID _has_feature = nullptr;
 
 public:
 	GodotJavaWrapper(JNIEnv *p_env, jobject p_activity, jobject p_godot_instance);
@@ -110,6 +111,9 @@ public:
 
 	// Return the list of gdextensions config file.
 	Vector<String> get_gdextension_list_config_file() const;
+
+	// Return true if the given feature is supported.
+	bool has_feature(const String &p_feature) const;
 };
 
 #endif // JAVA_GODOT_WRAPPER_H

platform/android/os_android.cpp

@@ -749,6 +749,11 @@ bool OS_Android::_check_internal_feature_support(const String &p_feature) {
 		return true;
 	}
 #endif
+
+	if (godot_java->has_feature(p_feature)) {
+		return true;
+	}
+
 	return false;
 }