From 15c1a7636164567fd0d324003fe8848f8247f0a6 Mon Sep 17 00:00:00 2001 From: Bastiaan Olij Date: Fri, 7 May 2021 23:19:04 +1000 Subject: Add stereoscopic rendering through multiview --- modules/gdnative/gdnative_api.json | 6 +- modules/gdnative/include/xr/godot_xr.h | 27 ++++++--- modules/gdnative/xr/xr_interface_gdnative.cpp | 53 ++++++++++++---- modules/gdnative/xr/xr_interface_gdnative.h | 14 +++-- modules/glslang/register_types.cpp | 4 ++ modules/mobile_vr/mobile_vr_interface.cpp | 87 +++++++++++++++++++++------ modules/mobile_vr/mobile_vr_interface.h | 14 +++-- modules/webxr/webxr_interface_js.cpp | 27 +++++++-- modules/webxr/webxr_interface_js.h | 5 +- 9 files changed, 178 insertions(+), 59 deletions(-) (limited to 'modules') diff --git a/modules/gdnative/gdnative_api.json b/modules/gdnative/gdnative_api.json index 175a6a3c91..8c65447e5d 100644 --- a/modules/gdnative/gdnative_api.json +++ b/modules/gdnative/gdnative_api.json @@ -5068,7 +5068,7 @@ }, { "name": "godot_xr_get_worldscale", - "return_type": "godot_float", + "return_type": "godot_real_t", "arguments": [] }, { @@ -5189,7 +5189,7 @@ "p_exis" ], [ - "godot_float", + "godot_real_t", "p_value" ], [ @@ -5200,7 +5200,7 @@ }, { "name": "godot_xr_get_controller_rumble", - "return_type": "godot_float", + "return_type": "godot_real_t", "arguments": [ [ "godot_int", diff --git a/modules/gdnative/include/xr/godot_xr.h b/modules/gdnative/include/xr/godot_xr.h index e09244f3f9..53cb830cbb 100644 --- a/modules/gdnative/include/xr/godot_xr.h +++ b/modules/gdnative/include/xr/godot_xr.h @@ -41,8 +41,8 @@ extern "C" { // version info to detect whether a call is available // Use these to populate version in your plugin -#define GODOTVR_API_MAJOR 1 -#define GODOTVR_API_MINOR 1 +#define GODOTVR_API_MAJOR 4 +#define GODOTVR_API_MINOR 0 typedef struct { godot_gdnative_api_version version; /* version of our API */ @@ -52,24 +52,31 @@ typedef struct { godot_int (*get_capabilities)(const void *); godot_bool (*get_anchor_detection_is_enabled)(const void *); void (*set_anchor_detection_is_enabled)(void *, godot_bool); - godot_bool (*is_stereo)(const void *); + godot_int (*get_view_count)(const void *); godot_bool (*is_initialized)(const void *); godot_bool (*initialize)(void *); void (*uninitialize)(void *); godot_vector2 (*get_render_targetsize)(const void *); - godot_transform3d (*get_transform_for_eye)(void *, godot_int, godot_transform3d *); - void (*fill_projection_for_eye)(void *, godot_float *, godot_int, godot_float, godot_float, godot_float); - void (*commit_for_eye)(void *, godot_int, godot_rid *, godot_rect2 *); + + godot_transform3d (*get_camera_transform)(void *); + godot_transform3d (*get_transform_for_view)(void *, godot_int, godot_transform3d *); + void (*fill_projection_for_view)(void *, godot_real_t *, godot_int, godot_real_t, godot_real_t, godot_real_t); + void (*commit_views)(void *, godot_rid *, godot_rect2 *); + void (*process)(void *); - godot_int (*get_external_texture_for_eye)(void *, godot_int); void (*notification)(void *, godot_int); godot_int (*get_camera_feed_id)(void *); + + // possibly deprecate but adding/keeping as a reminder these are in Godot 3 + void (*commit_for_eye)(void *, godot_int, godot_rid *, godot_rect2 *); + godot_int (*get_external_texture_for_eye)(void *, godot_int); + godot_int (*get_external_depth_for_eye)(void *, godot_int); } godot_xr_interface_gdnative; void GDAPI godot_xr_register_interface(const godot_xr_interface_gdnative *p_interface); // helper functions to access XRServer data -godot_float GDAPI godot_xr_get_worldscale(); +godot_real_t GDAPI godot_xr_get_worldscale(); godot_transform3d GDAPI godot_xr_get_reference_frame(); // helper functions for rendering @@ -81,8 +88,8 @@ godot_int GDAPI godot_xr_add_controller(char *p_device_name, godot_int p_hand, g void GDAPI godot_xr_remove_controller(godot_int p_controller_id); void GDAPI godot_xr_set_controller_transform(godot_int p_controller_id, godot_transform3d *p_transform, godot_bool p_tracks_orientation, godot_bool p_tracks_position); void GDAPI godot_xr_set_controller_button(godot_int p_controller_id, godot_int p_button, godot_bool p_is_pressed); -void GDAPI godot_xr_set_controller_axis(godot_int p_controller_id, godot_int p_axis, godot_float p_value, godot_bool p_can_be_negative); -godot_float GDAPI godot_xr_get_controller_rumble(godot_int p_controller_id); +void GDAPI godot_xr_set_controller_axis(godot_int p_controller_id, godot_int p_axis, godot_real_t p_value, godot_bool p_can_be_negative); +godot_real_t GDAPI godot_xr_get_controller_rumble(godot_int p_controller_id); #ifdef __cplusplus } diff --git a/modules/gdnative/xr/xr_interface_gdnative.cpp b/modules/gdnative/xr/xr_interface_gdnative.cpp index 258fb75000..ff959affa3 100644 --- a/modules/gdnative/xr/xr_interface_gdnative.cpp +++ b/modules/gdnative/xr/xr_interface_gdnative.cpp @@ -70,8 +70,13 @@ void XRInterfaceGDNative::set_interface(const godot_xr_interface_gdnative *p_int // this should only be called once, just being paranoid.. if (interface) { cleanup(); + interface = NULL; } + // validate + ERR_FAIL_NULL(p_interface); + ERR_FAIL_COND_MSG(p_interface->version.major < 4, "This is an incompatible GDNative XR plugin."); + // bind to our interface interface = p_interface; @@ -119,14 +124,14 @@ int XRInterfaceGDNative::get_camera_feed_id() { return (unsigned int)interface->get_camera_feed_id(data); } -bool XRInterfaceGDNative::is_stereo() { - bool stereo; +uint32_t XRInterfaceGDNative::get_view_count() { + uint32_t view_count; - ERR_FAIL_COND_V(interface == nullptr, false); + ERR_FAIL_COND_V(interface == nullptr, 1); - stereo = interface->is_stereo(data); + view_count = interface->get_view_count(data); - return stereo; + return view_count; } bool XRInterfaceGDNative::is_initialized() const { @@ -173,28 +178,52 @@ Size2 XRInterfaceGDNative::get_render_targetsize() { return *vec; } -Transform3D XRInterfaceGDNative::get_transform_for_eye(XRInterface::Eyes p_eye, const Transform3D &p_cam_transform) { +Transform3D XRInterfaceGDNative::get_camera_transform() { Transform3D *ret; ERR_FAIL_COND_V(interface == nullptr, Transform3D()); - godot_transform3d t = interface->get_transform_for_eye(data, (int)p_eye, (godot_transform3d *)&p_cam_transform); + godot_transform3d t = interface->get_camera_transform(data); ret = (Transform3D *)&t; return *ret; } -CameraMatrix XRInterfaceGDNative::get_projection_for_eye(XRInterface::Eyes p_eye, real_t p_aspect, real_t p_z_near, real_t p_z_far) { +Transform3D XRInterfaceGDNative::get_transform_for_view(uint32_t p_view, const Transform3D &p_cam_transform) { + Transform3D *ret; + + ERR_FAIL_COND_V(interface == nullptr, Transform3D()); + + godot_transform3d t = interface->get_transform_for_view(data, (int)p_view, (godot_transform3d *)&p_cam_transform); + + ret = (Transform3D *)&t; + + return *ret; +} + +CameraMatrix XRInterfaceGDNative::get_projection_for_view(uint32_t p_view, real_t p_aspect, real_t p_z_near, real_t p_z_far) { CameraMatrix cm; ERR_FAIL_COND_V(interface == nullptr, CameraMatrix()); - interface->fill_projection_for_eye(data, (godot_float *)cm.matrix, (godot_int)p_eye, p_aspect, p_z_near, p_z_far); + interface->fill_projection_for_view(data, (godot_real_t *)cm.matrix, (godot_int)p_view, p_aspect, p_z_near, p_z_far); return cm; } +Vector XRInterfaceGDNative::commit_views(RID p_render_target, const Rect2 &p_screen_rect) { + // possibly move this as a member variable and add a callback to populate? + Vector blit_to_screen; + + ERR_FAIL_COND_V(interface == nullptr, blit_to_screen); + + // must implement + interface->commit_views(data, (godot_rid *)&p_render_target, (godot_rect2 *)&p_screen_rect); + + return blit_to_screen; +} + unsigned int XRInterfaceGDNative::get_external_texture_for_eye(XRInterface::Eyes p_eye) { ERR_FAIL_COND_V(interface == nullptr, 0); @@ -234,7 +263,7 @@ void GDAPI godot_xr_register_interface(const godot_xr_interface_gdnative *p_inte XRServer::get_singleton()->add_interface(new_interface); } -godot_float GDAPI godot_xr_get_worldscale() { +godot_real_t GDAPI godot_xr_get_worldscale() { XRServer *xr_server = XRServer::get_singleton(); ERR_FAIL_NULL_V(xr_server, 1.0); @@ -388,7 +417,7 @@ void GDAPI godot_xr_set_controller_button(godot_int p_controller_id, godot_int p } } -void GDAPI godot_xr_set_controller_axis(godot_int p_controller_id, godot_int p_axis, godot_float p_value, godot_bool p_can_be_negative) { +void GDAPI godot_xr_set_controller_axis(godot_int p_controller_id, godot_int p_axis, godot_real_t p_value, godot_bool p_can_be_negative) { XRServer *xr_server = XRServer::get_singleton(); ERR_FAIL_NULL(xr_server); @@ -407,7 +436,7 @@ void GDAPI godot_xr_set_controller_axis(godot_int p_controller_id, godot_int p_a } } -godot_float GDAPI godot_xr_get_controller_rumble(godot_int p_controller_id) { +godot_real_t GDAPI godot_xr_get_controller_rumble(godot_int p_controller_id) { XRServer *xr_server = XRServer::get_singleton(); ERR_FAIL_NULL_V(xr_server, 0.0); diff --git a/modules/gdnative/xr/xr_interface_gdnative.h b/modules/gdnative/xr/xr_interface_gdnative.h index 42098ebeff..42e9206c1f 100644 --- a/modules/gdnative/xr/xr_interface_gdnative.h +++ b/modules/gdnative/xr/xr_interface_gdnative.h @@ -72,20 +72,24 @@ public: /** rendering and internal **/ virtual Size2 get_render_targetsize() override; - virtual bool is_stereo() override; - virtual Transform3D get_transform_for_eye(XRInterface::Eyes p_eye, const Transform3D &p_cam_transform) override; + virtual uint32_t get_view_count() override; + virtual Transform3D get_camera_transform() override; + virtual Transform3D get_transform_for_view(uint32_t p_view, const Transform3D &p_cam_transform) override; // we expose a Vector version of this function to GDNative Vector _get_projection_for_eye(XRInterface::Eyes p_eye, real_t p_aspect, real_t p_z_near, real_t p_z_far); // and a CameraMatrix version to XRServer - virtual CameraMatrix get_projection_for_eye(XRInterface::Eyes p_eye, real_t p_aspect, real_t p_z_near, real_t p_z_far) override; + virtual CameraMatrix get_projection_for_view(uint32_t p_view, real_t p_aspect, real_t p_z_near, real_t p_z_far) override; - virtual unsigned int get_external_texture_for_eye(XRInterface::Eyes p_eye) override; - virtual void commit_for_eye(XRInterface::Eyes p_eye, RID p_render_target, const Rect2 &p_screen_rect) override; + virtual Vector commit_views(RID p_render_target, const Rect2 &p_screen_rect) override; virtual void process() override; virtual void notification(int p_what) override; + + // deprecated + virtual void commit_for_eye(XRInterface::Eyes p_eye, RID p_render_target, const Rect2 &p_screen_rect) override; + virtual unsigned int get_external_texture_for_eye(XRInterface::Eyes p_eye) override; }; #endif // XR_INTERFACE_GDNATIVE_H diff --git a/modules/glslang/register_types.cpp b/modules/glslang/register_types.cpp index 8979eabfc3..730c6b89f7 100644 --- a/modules/glslang/register_types.cpp +++ b/modules/glslang/register_types.cpp @@ -116,6 +116,10 @@ static Vector _compile_shader_glsl(RenderingDevice::ShaderStage p_stage } } + if (p_capabilities->supports_multiview) { + preamble += "#define has_VK_KHR_multiview 1\n"; + } + if (preamble != "") { shader.setPreamble(preamble.c_str()); } diff --git a/modules/mobile_vr/mobile_vr_interface.cpp b/modules/mobile_vr/mobile_vr_interface.cpp index 40b1745c35..590b95ab79 100644 --- a/modules/mobile_vr/mobile_vr_interface.cpp +++ b/modules/mobile_vr/mobile_vr_interface.cpp @@ -300,9 +300,9 @@ real_t MobileVRInterface::get_k2() const { return k2; }; -bool MobileVRInterface::is_stereo() { +uint32_t MobileVRInterface::get_view_count() { // needs stereo... - return true; + return 2; }; bool MobileVRInterface::is_initialized() const { @@ -361,7 +361,29 @@ Size2 MobileVRInterface::get_render_targetsize() { return target_size; }; -Transform3D MobileVRInterface::get_transform_for_eye(XRInterface::Eyes p_eye, const Transform3D &p_cam_transform) { +Transform3D MobileVRInterface::get_camera_transform() { + _THREAD_SAFE_METHOD_ + + Transform3D transform_for_eye; + + XRServer *xr_server = XRServer::get_singleton(); + ERR_FAIL_NULL_V(xr_server, transform_for_eye); + + if (initialized) { + float world_scale = xr_server->get_world_scale(); + + // just scale our origin point of our transform + Transform3D hmd_transform; + hmd_transform.basis = orientation; + hmd_transform.origin = Vector3(0.0, eye_height * world_scale, 0.0); + + transform_for_eye = (xr_server->get_reference_frame()) * hmd_transform; + } + + return transform_for_eye; +}; + +Transform3D MobileVRInterface::get_transform_for_view(uint32_t p_view, const Transform3D &p_cam_transform) { _THREAD_SAFE_METHOD_ Transform3D transform_for_eye; @@ -374,12 +396,12 @@ Transform3D MobileVRInterface::get_transform_for_eye(XRInterface::Eyes p_eye, co // we don't need to check for the existence of our HMD, doesn't affect our values... // note * 0.01 to convert cm to m and * 0.5 as we're moving half in each direction... - if (p_eye == XRInterface::EYE_LEFT) { + if (p_view == 0) { transform_for_eye.origin.x = -(intraocular_dist * 0.01 * 0.5 * world_scale); - } else if (p_eye == XRInterface::EYE_RIGHT) { + } else if (p_view == 1) { transform_for_eye.origin.x = intraocular_dist * 0.01 * 0.5 * world_scale; } else { - // for mono we don't reposition, we want our center position. + // should not have any other values.. }; // just scale our origin point of our transform @@ -396,21 +418,13 @@ Transform3D MobileVRInterface::get_transform_for_eye(XRInterface::Eyes p_eye, co return transform_for_eye; }; -CameraMatrix MobileVRInterface::get_projection_for_eye(XRInterface::Eyes p_eye, real_t p_aspect, real_t p_z_near, real_t p_z_far) { +CameraMatrix MobileVRInterface::get_projection_for_view(uint32_t p_view, real_t p_aspect, real_t p_z_near, real_t p_z_far) { _THREAD_SAFE_METHOD_ CameraMatrix eye; - if (p_eye == XRInterface::EYE_MONO) { - ///@TODO for now hardcode some of this, what is really needed here is that this needs to be in sync with the real camera's properties - // which probably means implementing a specific class for iOS and Android. For now this is purely here as an example. - // Note also that if you use a normal viewport with AR/VR turned off you can still use the tracker output of this interface - // to position a stock standard Godot camera and have control over this. - // This will make more sense when we implement ARkit on iOS (probably a separate interface). - eye.set_perspective(60.0, p_aspect, p_z_near, p_z_far, false); - } else { - eye.set_for_hmd(p_eye == XRInterface::EYE_LEFT ? 1 : 2, p_aspect, intraocular_dist, display_width, display_to_lens, oversample, p_z_near, p_z_far); - }; + aspect = p_aspect; + eye.set_for_hmd(p_view + 1, p_aspect, intraocular_dist, display_width, display_to_lens, oversample, p_z_near, p_z_far); return eye; }; @@ -440,6 +454,45 @@ void MobileVRInterface::commit_for_eye(XRInterface::Eyes p_eye, RID p_render_tar eye_center.y = 0.0; } +Vector MobileVRInterface::commit_views(RID p_render_target, const Rect2 &p_screen_rect) { + _THREAD_SAFE_METHOD_ + + Vector blit_to_screen; + + // We must have a valid render target + ERR_FAIL_COND_V(!p_render_target.is_valid(), blit_to_screen); + + // Because we are rendering to our device we must use our main viewport! + ERR_FAIL_COND_V(p_screen_rect == Rect2(), blit_to_screen); + + // and add our blits + BlitToScreen blit; + blit.render_target = p_render_target; + blit.multi_view.use_layer = true; + blit.lens_distortion.apply = true; + blit.lens_distortion.k1 = k1; + blit.lens_distortion.k2 = k2; + blit.lens_distortion.upscale = oversample; + blit.lens_distortion.aspect_ratio = aspect; + + // left eye + blit.rect = p_screen_rect; + blit.rect.size.width *= 0.5; + blit.multi_view.layer = 0; + blit.lens_distortion.eye_center.x = ((-intraocular_dist / 2.0) + (display_width / 4.0)) / (display_width / 2.0); + blit_to_screen.push_back(blit); + + // right eye + blit.rect = p_screen_rect; + blit.rect.size.width *= 0.5; + blit.rect.position.x = blit.rect.size.width; + blit.multi_view.layer = 1; + blit.lens_distortion.eye_center.x = ((intraocular_dist / 2.0) - (display_width / 4.0)) / (display_width / 2.0); + blit_to_screen.push_back(blit); + + return blit_to_screen; +} + void MobileVRInterface::process() { _THREAD_SAFE_METHOD_ diff --git a/modules/mobile_vr/mobile_vr_interface.h b/modules/mobile_vr/mobile_vr_interface.h index aad40ebb5b..29ce0f92c8 100644 --- a/modules/mobile_vr/mobile_vr_interface.h +++ b/modules/mobile_vr/mobile_vr_interface.h @@ -63,9 +63,9 @@ private: real_t display_to_lens = 4.0; real_t oversample = 1.5; - //@TODO not yet used, these are needed in our distortion shader... real_t k1 = 0.215; real_t k2 = 0.215; + real_t aspect = 1.0; /* logic for processing our sensor data, this was originally in our positional tracker logic but I think @@ -138,16 +138,20 @@ public: virtual void uninitialize() override; virtual Size2 get_render_targetsize() override; - virtual bool is_stereo() override; - virtual Transform3D get_transform_for_eye(XRInterface::Eyes p_eye, const Transform3D &p_cam_transform) override; - virtual CameraMatrix get_projection_for_eye(XRInterface::Eyes p_eye, real_t p_aspect, real_t p_z_near, real_t p_z_far) override; - virtual void commit_for_eye(XRInterface::Eyes p_eye, RID p_render_target, const Rect2 &p_screen_rect) override; + virtual uint32_t get_view_count() override; + virtual Transform3D get_camera_transform() override; + virtual Transform3D get_transform_for_view(uint32_t p_view, const Transform3D &p_cam_transform) override; + virtual CameraMatrix get_projection_for_view(uint32_t p_view, real_t p_aspect, real_t p_z_near, real_t p_z_far) override; + virtual Vector commit_views(RID p_render_target, const Rect2 &p_screen_rect) override; virtual void process() override; virtual void notification(int p_what) override {} MobileVRInterface(); ~MobileVRInterface(); + + // deprecated + virtual void commit_for_eye(XRInterface::Eyes p_eye, RID p_render_target, const Rect2 &p_screen_rect) override; }; #endif // !MOBILE_VR_INTERFACE_H diff --git a/modules/webxr/webxr_interface_js.cpp b/modules/webxr/webxr_interface_js.cpp index a628cb0549..13981e73e1 100644 --- a/modules/webxr/webxr_interface_js.cpp +++ b/modules/webxr/webxr_interface_js.cpp @@ -265,7 +265,7 @@ void WebXRInterfaceJS::uninitialize() { }; }; -Transform WebXRInterfaceJS::_js_matrix_to_transform(float *p_js_matrix) { +Transform3D WebXRInterfaceJS::_js_matrix_to_transform(float *p_js_matrix) { Transform3D transform; transform.basis.elements[0].x = p_js_matrix[0]; @@ -305,13 +305,30 @@ Size2 WebXRInterfaceJS::get_render_targetsize() { return render_targetsize; }; -Transform WebXRInterfaceJS::get_transform_for_eye(XRInterface::Eyes p_eye, const Transform3D &p_cam_transform) { +Transform3D WebXRInterfaceJS::get_camera_transform() { Transform3D transform_for_eye; XRServer *xr_server = XRServer::get_singleton(); ERR_FAIL_NULL_V(xr_server, transform_for_eye); - float *js_matrix = godot_webxr_get_transform_for_eye(p_eye); + float *js_matrix = godot_webxr_get_transform_for_eye(0); + if (!initialized || js_matrix == nullptr) { + return transform_for_eye; + } + + transform_for_eye = _js_matrix_to_transform(js_matrix); + free(js_matrix); + + return xr_server->get_reference_frame() * transform_for_eye; +}; + +Transform3D WebXRInterfaceJS::get_transform_for_view(uint32_t p_view, const Transform3D &p_cam_transform) { + Transform3D transform_for_eye; + + XRServer *xr_server = XRServer::get_singleton(); + ERR_FAIL_NULL_V(xr_server, transform_for_eye); + + float *js_matrix = godot_webxr_get_transform_for_eye(p_view + 1); if (!initialized || js_matrix == nullptr) { transform_for_eye = p_cam_transform; return transform_for_eye; @@ -323,10 +340,10 @@ Transform WebXRInterfaceJS::get_transform_for_eye(XRInterface::Eyes p_eye, const return p_cam_transform * xr_server->get_reference_frame() * transform_for_eye; }; -CameraMatrix WebXRInterfaceJS::get_projection_for_eye(XRInterface::Eyes p_eye, real_t p_aspect, real_t p_z_near, real_t p_z_far) { +CameraMatrix WebXRInterfaceJS::get_projection_for_view(uint32_t p_view, real_t p_aspect, real_t p_z_near, real_t p_z_far) { CameraMatrix eye; - float *js_matrix = godot_webxr_get_projection_for_eye(p_eye); + float *js_matrix = godot_webxr_get_projection_for_eye(p_view + 1); if (!initialized || js_matrix == nullptr) { return eye; } diff --git a/modules/webxr/webxr_interface_js.h b/modules/webxr/webxr_interface_js.h index e9274a06e4..723ab952cd 100644 --- a/modules/webxr/webxr_interface_js.h +++ b/modules/webxr/webxr_interface_js.h @@ -84,8 +84,9 @@ public: virtual Size2 get_render_targetsize() override; virtual bool is_stereo() override; - virtual Transform get_transform_for_eye(XRInterface::Eyes p_eye, const Transform3D &p_cam_transform) override; - virtual CameraMatrix get_projection_for_eye(XRInterface::Eyes p_eye, real_t p_aspect, real_t p_z_near, real_t p_z_far) override; + virtual Transform3D get_camera_transform() override; + virtual Transform3D get_transform_for_view(uint32_t p_view, const Transform3D &p_cam_transform) override; + virtual CameraMatrix get_projection_for_view(uint32_t p_view, real_t p_aspect, real_t p_z_near, real_t p_z_far) override; virtual unsigned int get_external_texture_for_eye(XRInterface::Eyes p_eye) override; virtual void commit_for_eye(XRInterface::Eyes p_eye, RID p_render_target, const Rect2 &p_screen_rect) override; -- cgit v1.2.3