Merge pull request #99768 from dsnopek/openxr-vulkan-foveated-rendering

OpenXR: Use the `XR_FB_foveation_vulkan` extension to get the density map for VRS

modules/openxr/extensions/openxr_extension_wrapper.h

@@ -192,4 +192,5 @@ public:
 	virtual void cleanup_swapchain_graphics_data(void **p_swapchain_graphics_data) = 0; // `cleanup_swapchain_graphics_data` cleans up the data held in our implementation dependent data structure and should free up its memory.
 	virtual bool create_projection_fov(const XrFovf p_fov, double p_z_near, double p_z_far, Projection &r_camera_matrix) = 0; // `create_projection_fov` creates a proper projection matrix based on asymmetric FOV data provided by OpenXR.
 	virtual RID get_texture(void *p_swapchain_graphics_data, int p_image_index) = 0; // `get_texture` returns a Godot texture RID for the current active texture in our swapchain.
+	virtual RID get_density_map(void *p_swapchain_graphics_data, int p_image_index) = 0; // `get_density_map` returns a Godot texture RID for the current active density map in our swapchain (if any).
 };

modules/openxr/extensions/openxr_fb_foveation_extension.cpp

@@ -31,6 +31,8 @@
 #include "openxr_fb_foveation_extension.h"
 #include "core/config/project_settings.h"
 
+#include "../openxr_platform_inc.h"
+
 OpenXRFBFoveationExtension *OpenXRFBFoveationExtension::singleton = nullptr;
 
 OpenXRFBFoveationExtension *OpenXRFBFoveationExtension::get_singleton() {
@@ -51,6 +53,12 @@ OpenXRFBFoveationExtension::OpenXRFBFoveationExtension(const String &p_rendering
 	swapchain_create_info_foveation_fb.type = XR_TYPE_SWAPCHAIN_CREATE_INFO_FOVEATION_FB;
 	swapchain_create_info_foveation_fb.next = nullptr;
 	swapchain_create_info_foveation_fb.flags = 0;
+
+	if (rendering_driver == "opengl3") {
+		swapchain_create_info_foveation_fb.flags = XR_SWAPCHAIN_CREATE_FOVEATION_SCALED_BIN_BIT_FB;
+	} else if (rendering_driver == "vulkan") {
+		swapchain_create_info_foveation_fb.flags = XR_SWAPCHAIN_CREATE_FOVEATION_FRAGMENT_DENSITY_MAP_BIT_FB;
+	}
 }
 
 OpenXRFBFoveationExtension::~OpenXRFBFoveationExtension() {
@@ -61,12 +69,11 @@ OpenXRFBFoveationExtension::~OpenXRFBFoveationExtension() {
 HashMap<String, bool *> OpenXRFBFoveationExtension::get_requested_extensions() {
 	HashMap<String, bool *> request_extensions;
 
-	if (rendering_driver == "vulkan") {
-		// This is currently only supported on OpenGL, but we may add Vulkan support in the future...
+	request_extensions[XR_FB_FOVEATION_EXTENSION_NAME] = &fb_foveation_ext;
+	request_extensions[XR_FB_FOVEATION_CONFIGURATION_EXTENSION_NAME] = &fb_foveation_configuration_ext;
 
-	} else if (rendering_driver == "opengl3") {
-		request_extensions[XR_FB_FOVEATION_EXTENSION_NAME] = &fb_foveation_ext;
-		request_extensions[XR_FB_FOVEATION_CONFIGURATION_EXTENSION_NAME] = &fb_foveation_configuration_ext;
+	if (rendering_driver == "vulkan") {
+		request_extensions[XR_FB_FOVEATION_VULKAN_EXTENSION_NAME] = &fb_foveation_vulkan_ext;
 	}
 
 	return request_extensions;
@@ -89,7 +96,11 @@ void OpenXRFBFoveationExtension::on_instance_destroyed() {
 }
 
 bool OpenXRFBFoveationExtension::is_enabled() const {
-	return swapchain_update_state_ext != nullptr && swapchain_update_state_ext->is_enabled() && fb_foveation_ext && fb_foveation_configuration_ext;
+	bool enabled = swapchain_update_state_ext != nullptr && swapchain_update_state_ext->is_enabled() && fb_foveation_ext && fb_foveation_configuration_ext;
+	if (rendering_driver == "vulkan") {
+		enabled = enabled && fb_foveation_vulkan_ext;
+	}
+	return enabled;
 }
 
 void *OpenXRFBFoveationExtension::set_swapchain_create_info_and_get_next_pointer(void *p_next_pointer) {

modules/openxr/extensions/openxr_fb_foveation_extension.h

@@ -35,11 +35,6 @@
 // Other Android based devices are implementing this as well, see:
 // https://github.khronos.org/OpenXR-Inventory/extension_support.html#XR_FB_foveation
 
-// Note: Currently we only support this for OpenGL.
-// This extension works on enabling foveated rendering on the swapchain.
-// Vulkan does not render 3D content directly to the swapchain image
-// hence this extension can't be used.
-
 #include "../openxr_api.h"
 #include "../util.h"
 #include "openxr_extension_wrapper.h"
@@ -81,6 +76,7 @@ private:
 	String rendering_driver;
 	bool fb_foveation_ext = false;
 	bool fb_foveation_configuration_ext = false;
+	bool fb_foveation_vulkan_ext = false;
 
 	// Configuration
 	XrFoveationLevelFB foveation_level = XR_FOVEATION_LEVEL_NONE_FB;

modules/openxr/extensions/platform/openxr_metal_extension.h

@@ -53,6 +53,7 @@ public:
 	virtual void cleanup_swapchain_graphics_data(void **p_swapchain_graphics_data) override;
 	virtual bool create_projection_fov(const XrFovf p_fov, double p_z_near, double p_z_far, Projection &r_camera_matrix) override;
 	virtual RID get_texture(void *p_swapchain_graphics_data, int p_image_index) override;
+	virtual RID get_density_map(void *p_swapchain_graphics_data, int p_image_index) override { return RID(); }
 
 private:
 	static XrGraphicsBindingMetalKHR graphics_binding_metal;

modules/openxr/extensions/platform/openxr_opengl_extension.h

@@ -55,6 +55,7 @@ public:
 	virtual void cleanup_swapchain_graphics_data(void **p_swapchain_graphics_data) override;
 	virtual bool create_projection_fov(const XrFovf p_fov, double p_z_near, double p_z_far, Projection &r_camera_matrix) override;
 	virtual RID get_texture(void *p_swapchain_graphics_data, int p_image_index) override;
+	virtual RID get_density_map(void *p_swapchain_graphics_data, int p_image_index) override { return RID(); }
 
 private:
 	static OpenXROpenGLExtension *singleton;

modules/openxr/extensions/platform/openxr_vulkan_extension.cpp

@@ -31,6 +31,7 @@
 #include "openxr_vulkan_extension.h"
 
 #include "../../openxr_util.h"
+#include "../openxr_fb_foveation_extension.h"
 
 #include "core/string/print_string.h"
 #include "servers/rendering/renderer_rd/effects/copy_effects.h"
@@ -240,6 +241,7 @@ void OpenXRVulkanExtension::get_usable_depth_formats(Vector<int64_t> &p_usable_s
 
 bool OpenXRVulkanExtension::get_swapchain_image_data(XrSwapchain p_swapchain, int64_t p_swapchain_format, uint32_t p_width, uint32_t p_height, uint32_t p_sample_count, uint32_t p_array_size, void **r_swapchain_graphics_data) {
 	LocalVector<XrSwapchainImageVulkanKHR> images;
+	LocalVector<XrSwapchainImageFoveationVulkanFB> density_images;
 
 	RenderingServer *rendering_server = RenderingServer::get_singleton();
 	ERR_FAIL_NULL_V(rendering_server, false);
@@ -261,6 +263,20 @@ bool OpenXRVulkanExtension::get_swapchain_image_data(XrSwapchain p_swapchain, in
 		image.image = VK_NULL_HANDLE;
 	}
 
+	if (OpenXRFBFoveationExtension::get_singleton()->is_enabled()) {
+		density_images.resize(swapchain_length);
+
+		for (uint64_t i = 0; i < swapchain_length; i++) {
+			density_images[i].type = XR_TYPE_SWAPCHAIN_IMAGE_FOVEATION_VULKAN_FB;
+			density_images[i].next = nullptr;
+			density_images[i].image = VK_NULL_HANDLE;
+			density_images[i].width = 0;
+			density_images[i].height = 0;
+
+			images[i].next = &density_images[i];
+		}
+	}
+
 	result = xrEnumerateSwapchainImages(p_swapchain, swapchain_length, &swapchain_length, (XrSwapchainImageBaseHeader *)images.ptr());
 	if (XR_FAILED(result)) {
 		print_line("OpenXR: Failed to get swapchaim images [", OpenXRAPI::get_singleton()->get_error_string(result), "]");
@@ -351,9 +367,12 @@ bool OpenXRVulkanExtension::get_swapchain_image_data(XrSwapchain p_swapchain, in
 	}
 
 	Vector<RID> texture_rids;
+	Vector<RID> density_map_rids;
 
 	// create Godot texture objects for each entry in our swapchain
-	for (const XrSwapchainImageVulkanKHR &swapchain_image : images) {
+	for (uint32_t i = 0; i < swapchain_length; i++) {
+		const XrSwapchainImageVulkanKHR &swapchain_image = images[i];
+
 		RID image_rid = rendering_device->texture_create_from_extension(
 				p_array_size == 1 ? RenderingDevice::TEXTURE_TYPE_2D : RenderingDevice::TEXTURE_TYPE_2D_ARRAY,
 				format,
@@ -366,9 +385,27 @@ bool OpenXRVulkanExtension::get_swapchain_image_data(XrSwapchain p_swapchain, in
 				p_array_size);
 
 		texture_rids.push_back(image_rid);
+
+		if (OpenXRFBFoveationExtension::get_singleton()->is_enabled() && density_images[i].image != VK_NULL_HANDLE) {
+			RID density_map_rid = rendering_device->texture_create_from_extension(
+					p_array_size == 1 ? RenderingDevice::TEXTURE_TYPE_2D : RenderingDevice::TEXTURE_TYPE_2D_ARRAY,
+					RD::DATA_FORMAT_R8G8_UNORM,
+					RenderingDevice::TEXTURE_SAMPLES_1,
+					RD::TEXTURE_USAGE_COLOR_ATTACHMENT_BIT | RD::TEXTURE_USAGE_SAMPLING_BIT | RD::TEXTURE_USAGE_STORAGE_BIT | RD::TEXTURE_USAGE_VRS_ATTACHMENT_BIT,
+					(uint64_t)density_images[i].image,
+					density_images[i].width,
+					density_images[i].height,
+					1,
+					p_array_size);
+
+			density_map_rids.push_back(density_map_rid);
+		} else {
+			density_map_rids.push_back(RID());
+		}
 	}
 
 	data->texture_rids = texture_rids;
+	data->density_map_rids = density_map_rids;
 
 	return true;
 }
@@ -395,6 +432,14 @@ RID OpenXRVulkanExtension::get_texture(void *p_swapchain_graphics_data, int p_im
 	return data->texture_rids[p_image_index];
 }
 
+RID OpenXRVulkanExtension::get_density_map(void *p_swapchain_graphics_data, int p_image_index) {
+	SwapchainGraphicsData *data = (SwapchainGraphicsData *)p_swapchain_graphics_data;
+	ERR_FAIL_NULL_V(data, RID());
+
+	ERR_FAIL_INDEX_V(p_image_index, data->density_map_rids.size(), RID());
+	return data->density_map_rids[p_image_index];
+}
+
 void OpenXRVulkanExtension::cleanup_swapchain_graphics_data(void **p_swapchain_graphics_data) {
 	if (*p_swapchain_graphics_data == nullptr) {
 		return;
@@ -413,6 +458,13 @@ void OpenXRVulkanExtension::cleanup_swapchain_graphics_data(void **p_swapchain_g
 	}
 	data->texture_rids.clear();
 
+	for (int i = 0; i < data->density_map_rids.size(); i++) {
+		if (data->density_map_rids[i].is_valid()) {
+			rendering_device->free(data->density_map_rids[i]);
+		}
+	}
+	data->density_map_rids.clear();
+
 	memdelete(data);
 	*p_swapchain_graphics_data = nullptr;
 }

modules/openxr/extensions/platform/openxr_vulkan_extension.h

@@ -62,6 +62,7 @@ public:
 	virtual void cleanup_swapchain_graphics_data(void **p_swapchain_graphics_data) override;
 	virtual bool create_projection_fov(const XrFovf p_fov, double p_z_near, double p_z_far, Projection &r_camera_matrix) override;
 	virtual RID get_texture(void *p_swapchain_graphics_data, int p_image_index) override;
+	virtual RID get_density_map(void *p_swapchain_graphics_data, int p_image_index) override;
 
 private:
 	static OpenXRVulkanExtension *singleton;
@@ -70,6 +71,7 @@ private:
 	struct SwapchainGraphicsData {
 		bool is_multiview;
 		Vector<RID> texture_rids;
+		Vector<RID> density_map_rids;
 	};
 
 	bool check_graphics_api_support(XrVersion p_desired_version);

modules/openxr/openxr_api.cpp

@@ -267,6 +267,16 @@ RID OpenXRAPI::OpenXRSwapChainInfo::get_image() {
 	}
 }
 
+RID OpenXRAPI::OpenXRSwapChainInfo::get_density_map() {
+	OpenXRAPI *openxr_api = OpenXRAPI::get_singleton();
+
+	if (image_acquired && openxr_api && openxr_api->get_graphics_extension()) {
+		return openxr_api->get_graphics_extension()->get_density_map(swapchain_graphics_data, image_index);
+	} else {
+		return RID();
+	}
+}
+
 ////////////////////////////////////
 // OpenXRAPI
 
@@ -2369,6 +2379,17 @@ RID OpenXRAPI::get_depth_texture() {
 	}
 }
 
+RID OpenXRAPI::get_density_map_texture() {
+	ERR_NOT_ON_RENDER_THREAD_V(RID());
+
+	OpenXRFBFoveationExtension *fov_ext = OpenXRFBFoveationExtension::get_singleton();
+	if (fov_ext && fov_ext->is_enabled()) {
+		return render_state.main_swapchains[OPENXR_SWAPCHAIN_COLOR].get_density_map();
+	}
+
+	return RID();
+}
+
 void OpenXRAPI::set_velocity_texture(RID p_render_target) {
 	velocity_texture = p_render_target;
 }

modules/openxr/openxr_api.h

@@ -73,6 +73,7 @@ public:
 		bool acquire(bool &p_should_render);
 		bool release();
 		RID get_image();
+		RID get_density_map();
 	};
 
 private:
@@ -501,6 +502,7 @@ public:
 	XrSwapchain get_color_swapchain();
 	RID get_color_texture();
 	RID get_depth_texture();
+	RID get_density_map_texture();
 	void set_velocity_texture(RID p_render_target);
 	RID get_velocity_texture();
 	void set_velocity_depth_texture(RID p_render_target);

modules/openxr/openxr_interface.cpp

@@ -1546,6 +1546,11 @@ RID OpenXRInterface::get_vrs_texture() {
 		return RID();
 	}
 
+	RID density_map = openxr_api->get_density_map_texture();
+	if (density_map.is_valid()) {
+		return density_map;
+	}
+
 	PackedVector2Array eye_foci;
 
 	Size2 target_size = get_render_target_size();
@@ -1561,6 +1566,19 @@ RID OpenXRInterface::get_vrs_texture() {
 	return xr_vrs.make_vrs_texture(target_size, eye_foci);
 }
 
+XRInterface::VRSTextureFormat OpenXRInterface::get_vrs_texture_format() {
+	if (!openxr_api) {
+		return XR_VRS_TEXTURE_FORMAT_UNIFIED;
+	}
+
+	RID density_map = openxr_api->get_density_map_texture();
+	if (density_map.is_valid()) {
+		return XR_VRS_TEXTURE_FORMAT_FRAGMENT_DENSITY_MAP;
+	}
+
+	return XR_VRS_TEXTURE_FORMAT_UNIFIED;
+}
+
 void OpenXRInterface::set_cpu_level(PerfSettingsLevel p_level) {
 	OpenXRPerformanceSettingsExtension *performance_settings_ext = OpenXRPerformanceSettingsExtension::get_singleton();
 	if (performance_settings_ext && performance_settings_ext->is_available()) {

modules/openxr/openxr_interface.h

@@ -290,6 +290,7 @@ public:
 	Vector3 get_hand_joint_angular_velocity(Hand p_hand, HandJoints p_joint) const;
 
 	virtual RID get_vrs_texture() override;
+	virtual VRSTextureFormat get_vrs_texture_format() override;
 
 	// Performance settings.
 	enum PerfSettingsLevel {