From 98e5cd24dbe1151e129883731b26ec8f521ee57f Mon Sep 17 00:00:00 2001 From: Bastiaan Olij Date: Wed, 26 Jan 2022 12:25:20 +1100 Subject: Improve XRInterface hooks into rendering --- doc/classes/XRInterfaceExtension.xml | 50 ++++++++++++++++++++++++-- doc/classes/XRServer.xml | 18 ---------- modules/mobile_vr/mobile_vr_interface.cpp | 2 +- modules/mobile_vr/mobile_vr_interface.h | 2 +- modules/webxr/webxr_interface_js.cpp | 2 +- modules/webxr/webxr_interface_js.h | 2 +- servers/rendering/renderer_viewport.cpp | 47 ++++++++++++++---------- servers/rendering/rendering_server_default.cpp | 6 ++++ servers/xr/xr_interface.cpp | 3 -- servers/xr/xr_interface.h | 9 +++-- servers/xr/xr_interface_extension.cpp | 36 +++++++++++++++---- servers/xr/xr_interface_extension.h | 11 ++++-- servers/xr_server.cpp | 50 +++++++++++++------------- servers/xr_server.h | 18 +++++----- 14 files changed, 163 insertions(+), 93 deletions(-) diff --git a/doc/classes/XRInterfaceExtension.xml b/doc/classes/XRInterfaceExtension.xml index d2bb6aa59a..cd09f83335 100644 --- a/doc/classes/XRInterfaceExtension.xml +++ b/doc/classes/XRInterfaceExtension.xml @@ -9,46 +9,52 @@ - + - - + Called if interface is active and queues have been submitted. + Return [code]true[/code] if anchor detection is enabled for this interface. + Returns the camera feed id for the [CameraFeed] registered with the [CameraServer] that should be presented as the background on an AR capable device (if applicable). + Returns the [Transform3D] that positions the [XRCamera3D] in the world. + Returns the capabilities of this interface. + Returns the name of this interface. + Returns an [PackedVector3Array] that denotes the play areas boundaries (if applicable). + Returns the [enum XRInterface.PlayAreaMode] that sets up our play area. @@ -58,27 +64,32 @@ + Returns the projection matrix for the given view as a [PackedFloat64Array]. + Returns the size of our render target for this interface, this overrides the size of the [Viewport] marked as the xr viewport. + Returns a [PackedStringArray] with pose names configured by this interface. Note that user configuration can override this list. + Returns a [PackedStringArray] with tracker names configured by this interface. Note that user configuration can override this list. + Returns a [enum XRInterface.TrackingStatus] specifying the current status of our tracking. @@ -86,50 +97,80 @@ + Returns a [Transform3D] for a given view. + Returns the number of views this interface requires, 1 for mono, 2 for stereoscopic. + Initializes the interface, returns [code]true[/code] on success. + Returns [code]true[/code] if this interface has been initialised. + Informs the interface of an applicable system notification. + + + + + + + + Called after the XR [Viewport] draw logic has completed. + + + + + + + Called if this is our primary [XRInterfaceExtension] before we start processing a [Viewport] for every active XR [Viewport], returns [code]true[/code] if that viewport should be rendered. An XR interface may return [code]false[/code] if the user has taken off their headset and we can pause rendering. + + + + + + Called if this [XRInterfaceExtension] is active before rendering starts, most XR interfaces will sync tracking at this point in time. + Called if this [XRInterfaceExtension] is active before our physics and game process is called. most XR interfaces will update its [XRPositionalTracker]s at this point in time. + Enables anchor detection on this interface if supported. + Set the play area mode for this interface. + Returns [code]true[/code] if this interface supports this play area mode. @@ -141,11 +182,13 @@ + Triggers a haptic pulse to be emitted on the specified tracker. + Uninitialize the interface. @@ -169,6 +212,7 @@ + Returns a valid [RID] for a texture to which we should render the current frame if supported by the interface. diff --git a/doc/classes/XRServer.xml b/doc/classes/XRServer.xml index 3a7fdea8d0..335ca4c35f 100644 --- a/doc/classes/XRServer.xml +++ b/doc/classes/XRServer.xml @@ -69,24 +69,6 @@ Returns a list of available interfaces the ID and name of each interface. - - - - Returns the absolute timestamp (in μs) of the last [XRServer] commit of the AR/VR eyes to [RenderingServer]. The value comes from an internal call to [method Time.get_ticks_usec]. - - - - - - Returns the duration (in μs) of the last frame. This is computed as the difference between [method get_last_commit_usec] and [method get_last_process_usec] when committing. - - - - - - Returns the absolute timestamp (in μs) of the last [XRServer] process callback. The value comes from an internal call to [method Time.get_ticks_usec]. - - diff --git a/modules/mobile_vr/mobile_vr_interface.cpp b/modules/mobile_vr/mobile_vr_interface.cpp index 49044c4afe..8cd23ffb24 100644 --- a/modules/mobile_vr/mobile_vr_interface.cpp +++ b/modules/mobile_vr/mobile_vr_interface.cpp @@ -463,7 +463,7 @@ CameraMatrix MobileVRInterface::get_projection_for_view(uint32_t p_view, double return eye; }; -Vector MobileVRInterface::commit_views(RID p_render_target, const Rect2 &p_screen_rect) { +Vector MobileVRInterface::post_draw_viewport(RID p_render_target, const Rect2 &p_screen_rect) { _THREAD_SAFE_METHOD_ Vector blit_to_screen; diff --git a/modules/mobile_vr/mobile_vr_interface.h b/modules/mobile_vr/mobile_vr_interface.h index 47dc33c0c7..8ecca3a2ae 100644 --- a/modules/mobile_vr/mobile_vr_interface.h +++ b/modules/mobile_vr/mobile_vr_interface.h @@ -151,7 +151,7 @@ public: virtual Transform3D get_camera_transform() override; virtual Transform3D get_transform_for_view(uint32_t p_view, const Transform3D &p_cam_transform) override; virtual CameraMatrix get_projection_for_view(uint32_t p_view, double p_aspect, double p_z_near, double p_z_far) override; - virtual Vector commit_views(RID p_render_target, const Rect2 &p_screen_rect) override; + virtual Vector post_draw_viewport(RID p_render_target, const Rect2 &p_screen_rect) override; virtual void process() override; diff --git a/modules/webxr/webxr_interface_js.cpp b/modules/webxr/webxr_interface_js.cpp index 8eb0d8ff90..86b857f72c 100644 --- a/modules/webxr/webxr_interface_js.cpp +++ b/modules/webxr/webxr_interface_js.cpp @@ -385,7 +385,7 @@ CameraMatrix WebXRInterfaceJS::get_projection_for_view(uint32_t p_view, double p return eye; } -Vector WebXRInterfaceJS::commit_views(RID p_render_target, const Rect2 &p_screen_rect) { +Vector WebXRInterfaceJS::post_draw_viewport(RID p_render_target, const Rect2 &p_screen_rect) { Vector blit_to_screen; if (!initialized) { diff --git a/modules/webxr/webxr_interface_js.h b/modules/webxr/webxr_interface_js.h index 8eddfbe484..31858194f6 100644 --- a/modules/webxr/webxr_interface_js.h +++ b/modules/webxr/webxr_interface_js.h @@ -88,7 +88,7 @@ public: virtual Transform3D get_camera_transform() override; virtual Transform3D get_transform_for_view(uint32_t p_view, const Transform3D &p_cam_transform) override; virtual CameraMatrix get_projection_for_view(uint32_t p_view, double p_aspect, double p_z_near, double p_z_far) override; - virtual Vector commit_views(RID p_render_target, const Rect2 &p_screen_rect) override; + virtual Vector post_draw_viewport(RID p_render_target, const Rect2 &p_screen_rect) override; virtual void process() override; diff --git a/servers/rendering/renderer_viewport.cpp b/servers/rendering/renderer_viewport.cpp index 17a665922f..be96793ec4 100644 --- a/servers/rendering/renderer_viewport.cpp +++ b/servers/rendering/renderer_viewport.cpp @@ -549,8 +549,13 @@ void RendererViewport::draw_viewports() { // get our xr interface in case we need it Ref xr_interface; - if (XRServer::get_singleton() != nullptr) { - xr_interface = XRServer::get_singleton()->get_primary_interface(); + XRServer *xr_server = XRServer::get_singleton(); + if (xr_server != nullptr) { + // let our XR server know we're about to render our frames so we can get our frame timing + xr_server->pre_render(); + + // retrieve the interface responsible for rendering + xr_interface = xr_server->get_primary_interface(); } if (Engine::get_singleton()->is_editor_hint()) { @@ -582,19 +587,26 @@ void RendererViewport::draw_viewports() { bool visible = vp->viewport_to_screen_rect != Rect2(); - if (vp->use_xr && xr_interface.is_valid()) { - visible = true; // XR viewport is always visible regardless of update mode, output is sent to HMD. - - // Override our size, make sure it matches our required size and is created as a stereo target - Size2 xr_size = xr_interface->get_render_target_size(); - - // Would have been nice if we could call viewport_set_size here, - // but alas that takes our RID and we now have our pointer, - // also we only check if view_count changes in render_target_set_size so we need to call that for this to reliably change - vp->occlusion_buffer_dirty = vp->occlusion_buffer_dirty || (vp->size != xr_size); - vp->size = xr_size; - uint32_t view_count = xr_interface->get_view_count(); - RSG::storage->render_target_set_size(vp->render_target, vp->size.x, vp->size.y, view_count); + if (vp->use_xr) { + if (xr_interface.is_valid()) { + // Override our size, make sure it matches our required size and is created as a stereo target + Size2 xr_size = xr_interface->get_render_target_size(); + + // Would have been nice if we could call viewport_set_size here, + // but alas that takes our RID and we now have our pointer, + // also we only check if view_count changes in render_target_set_size so we need to call that for this to reliably change + vp->occlusion_buffer_dirty = vp->occlusion_buffer_dirty || (vp->size != xr_size); + vp->size = xr_size; + uint32_t view_count = xr_interface->get_view_count(); + RSG::storage->render_target_set_size(vp->render_target, vp->size.x, vp->size.y, view_count); + + // Inform xr interface we're about to render its viewport, if this returns false we don't render + visible = xr_interface->pre_draw_viewport(vp->render_target); + } else { + // don't render anything + visible = false; + vp->size = Size2(); + } } if (vp->update_mode == RS::VIEWPORT_UPDATE_ALWAYS || vp->update_mode == RS::VIEWPORT_UPDATE_ONCE) { @@ -647,7 +659,7 @@ void RendererViewport::draw_viewports() { // measure // commit our eyes - Vector blits = xr_interface->commit_views(vp->render_target, vp->viewport_to_screen_rect); + Vector blits = xr_interface->post_draw_viewport(vp->render_target, vp->viewport_to_screen_rect); if (vp->viewport_to_screen != DisplayServer::INVALID_WINDOW_ID && blits.size() > 0) { if (!blit_to_screen_list.has(vp->viewport_to_screen)) { blit_to_screen_list[vp->viewport_to_screen] = Vector(); @@ -657,9 +669,6 @@ void RendererViewport::draw_viewports() { blit_to_screen_list[vp->viewport_to_screen].push_back(blits[b]); } } - - // and for our frame timing, mark when we've finished committing our eyes - XRServer::get_singleton()->_mark_commit(); } else { RSG::storage->render_target_set_external_texture(vp->render_target, 0); diff --git a/servers/rendering/rendering_server_default.cpp b/servers/rendering/rendering_server_default.cpp index d7e9d210db..d93aad5d7b 100644 --- a/servers/rendering/rendering_server_default.cpp +++ b/servers/rendering/rendering_server_default.cpp @@ -93,6 +93,12 @@ void RenderingServerDefault::_draw(bool p_swap_buffers, double frame_step) { RSG::rasterizer->end_frame(p_swap_buffers); + XRServer *xr_server = XRServer::get_singleton(); + if (xr_server != nullptr) { + // let our XR server know we're done so we can get our frame timing + xr_server->end_frame(); + } + RSG::canvas->update_visibility_notifiers(); RSG::scene->update_visibility_notifiers(); diff --git a/servers/xr/xr_interface.cpp b/servers/xr/xr_interface.cpp index 758f2416ec..7ae111b5e7 100644 --- a/servers/xr/xr_interface.cpp +++ b/servers/xr/xr_interface.cpp @@ -168,8 +168,5 @@ XRInterface::TrackingStatus XRInterface::get_tracking_status() const { return XR_UNKNOWN_TRACKING; } -void XRInterface::notification(int p_what) { -} - void XRInterface::trigger_haptic_pulse(const String &p_action_name, const StringName &p_tracker_name, double p_frequency, double p_amplitude, double p_duration_sec, double p_delay_sec) { } diff --git a/servers/xr/xr_interface.h b/servers/xr/xr_interface.h index aee98f8fee..6e105ffc26 100644 --- a/servers/xr/xr_interface.h +++ b/servers/xr/xr_interface.h @@ -123,10 +123,13 @@ public: // note, external color/depth/vrs texture support will be added here soon. - virtual Vector commit_views(RID p_render_target, const Rect2 &p_screen_rect) = 0; /* commit rendered views to the XR interface */ - virtual void process() = 0; - virtual void notification(int p_what); + virtual void pre_render(){}; + virtual bool pre_draw_viewport(RID p_render_target) { return true; }; /* inform XR interface we are about to start our viewport draw process */ + virtual Vector post_draw_viewport(RID p_render_target, const Rect2 &p_screen_rect) = 0; /* inform XR interface we finished our viewport draw process */ + virtual void end_frame(){}; + + virtual void notification(int p_what){}; XRInterface(); ~XRInterface(); diff --git a/servers/xr/xr_interface_extension.cpp b/servers/xr/xr_interface_extension.cpp index 9dae3b162d..18131c1e89 100644 --- a/servers/xr/xr_interface_extension.cpp +++ b/servers/xr/xr_interface_extension.cpp @@ -52,9 +52,12 @@ void XRInterfaceExtension::_bind_methods() { GDVIRTUAL_BIND(_get_transform_for_view, "view", "cam_transform"); GDVIRTUAL_BIND(_get_projection_for_view, "view", "aspect", "z_near", "z_far"); - GDVIRTUAL_BIND(_commit_views, "render_target", "screen_rect"); - GDVIRTUAL_BIND(_process); + GDVIRTUAL_BIND(_pre_render); + GDVIRTUAL_BIND(_pre_draw_viewport, "render_target"); + GDVIRTUAL_BIND(_post_draw_viewport, "render_target", "screen_rect"); + GDVIRTUAL_BIND(_end_frame); + GDVIRTUAL_BIND(_notification, "what"); /** input and output **/ @@ -274,7 +277,7 @@ CameraMatrix XRInterfaceExtension::get_projection_for_view(uint32_t p_view, doub void XRInterfaceExtension::add_blit(RID p_render_target, Rect2 p_src_rect, Rect2i p_dst_rect, bool p_use_layer, uint32_t p_layer, bool p_apply_lens_distortion, Vector2 p_eye_center, double p_k1, double p_k2, double p_upscale, double p_aspect_ratio) { BlitToScreen blit; - ERR_FAIL_COND_MSG(!can_add_blits, "add_blit can only be called from an XR plugin from within _commit_views!"); + ERR_FAIL_COND_MSG(!can_add_blits, "add_blit can only be called from an XR plugin from within _post_draw_viewport!"); blit.render_target = p_render_target; blit.src_rect = p_src_rect; @@ -293,12 +296,31 @@ void XRInterfaceExtension::add_blit(RID p_render_target, Rect2 p_src_rect, Rect2 blits.push_back(blit); } -Vector XRInterfaceExtension::commit_views(RID p_render_target, const Rect2 &p_screen_rect) { +void XRInterfaceExtension::process() { + GDVIRTUAL_CALL(_process); +} + +void XRInterfaceExtension::pre_render() { + GDVIRTUAL_CALL(_pre_render); +} + +bool XRInterfaceExtension::pre_draw_viewport(RID p_render_target) { + bool do_render = true; + + if (GDVIRTUAL_CALL(_pre_draw_viewport, p_render_target, do_render)) { + return do_render; + } else { + // if not implemented we're returning true + return true; + } +} + +Vector XRInterfaceExtension::post_draw_viewport(RID p_render_target, const Rect2 &p_screen_rect) { // This is just so our XR plugin can add blits... blits.clear(); can_add_blits = true; - if (GDVIRTUAL_CALL(_commit_views, p_render_target, p_screen_rect)) { + if (GDVIRTUAL_CALL(_post_draw_viewport, p_render_target, p_screen_rect)) { return blits; } @@ -306,8 +328,8 @@ Vector XRInterfaceExtension::commit_views(RID p_render_target, con return blits; } -void XRInterfaceExtension::process() { - GDVIRTUAL_CALL(_process); +void XRInterfaceExtension::end_frame() { + GDVIRTUAL_CALL(_end_frame); } void XRInterfaceExtension::notification(int p_what) { diff --git a/servers/xr/xr_interface_extension.h b/servers/xr/xr_interface_extension.h index e22ec2b872..5a436b9fd0 100644 --- a/servers/xr/xr_interface_extension.h +++ b/servers/xr/xr_interface_extension.h @@ -109,13 +109,20 @@ public: GDVIRTUAL4R(PackedFloat64Array, _get_projection_for_view, uint32_t, double, double, double); void add_blit(RID p_render_target, Rect2 p_src_rect, Rect2i p_dst_rect, bool p_use_layer = false, uint32_t p_layer = 0, bool p_apply_lens_distortion = false, Vector2 p_eye_center = Vector2(), double p_k1 = 0.0, double p_k2 = 0.0, double p_upscale = 1.0, double p_aspect_ratio = 1.0); - virtual Vector commit_views(RID p_render_target, const Rect2 &p_screen_rect) override; - GDVIRTUAL2(_commit_views, RID, const Rect2 &); virtual void process() override; + virtual void pre_render() override; + virtual bool pre_draw_viewport(RID p_render_target) override; + virtual Vector post_draw_viewport(RID p_render_target, const Rect2 &p_screen_rect) override; + virtual void end_frame() override; virtual void notification(int p_what) override; GDVIRTUAL0(_process); + GDVIRTUAL0(_pre_render); + GDVIRTUAL1R(bool, _pre_draw_viewport, RID); + GDVIRTUAL2(_post_draw_viewport, RID, const Rect2 &); + GDVIRTUAL0(_end_frame); + GDVIRTUAL1(_notification, int); /* access to some internals we need */ diff --git a/servers/xr_server.cpp b/servers/xr_server.cpp index 69f4c69b53..dbfe76a127 100644 --- a/servers/xr_server.cpp +++ b/servers/xr_server.cpp @@ -65,10 +65,6 @@ void XRServer::_bind_methods() { ADD_PROPERTY(PropertyInfo(Variant::OBJECT, "primary_interface"), "set_primary_interface", "get_primary_interface"); - ClassDB::bind_method(D_METHOD("get_last_process_usec"), &XRServer::get_last_process_usec); - ClassDB::bind_method(D_METHOD("get_last_commit_usec"), &XRServer::get_last_commit_usec); - ClassDB::bind_method(D_METHOD("get_last_frame_usec"), &XRServer::get_last_frame_usec); - BIND_ENUM_CONSTANT(TRACKER_HEAD); BIND_ENUM_CONSTANT(TRACKER_CONTROLLER); BIND_ENUM_CONSTANT(TRACKER_BASESTATION); @@ -351,24 +347,9 @@ PackedStringArray XRServer::get_suggested_pose_names(const StringName &p_tracker return arr; } -uint64_t XRServer::get_last_process_usec() { - return last_process_usec; -}; - -uint64_t XRServer::get_last_commit_usec() { - return last_commit_usec; -}; - -uint64_t XRServer::get_last_frame_usec() { - return last_frame_usec; -}; - void XRServer::_process() { /* called from renderer_viewport.draw_viewports right before we start drawing our viewports */ - /* mark for our frame timing */ - last_process_usec = OS::get_singleton()->get_ticks_usec(); - /* process all active interfaces */ for (int i = 0; i < interfaces.size(); i++) { if (!interfaces[i].is_valid()) { @@ -379,13 +360,32 @@ void XRServer::_process() { }; }; -void XRServer::_mark_commit() { - /* time this */ - last_commit_usec = OS::get_singleton()->get_ticks_usec(); +void XRServer::pre_render() { + // called from RendererViewport.draw_viewports right before we start drawing our viewports + // note that we can have multiple interfaces active if we have interfaces that purely handle tracking - /* now store our difference as we may overwrite last_process_usec before this is accessed */ - last_frame_usec = last_commit_usec - last_process_usec; -}; + // process all active interfaces + for (int i = 0; i < interfaces.size(); i++) { + if (!interfaces[i].is_valid()) { + // ignore, not a valid reference + } else if (interfaces[i]->is_initialized()) { + interfaces.write[i]->pre_render(); + }; + }; +} + +void XRServer::end_frame() { + // called from RenderingServerDefault after Vulkan queues have been submitted + + // process all active interfaces + for (int i = 0; i < interfaces.size(); i++) { + if (!interfaces[i].is_valid()) { + // ignore, not a valid reference + } else if (interfaces[i]->is_initialized()) { + interfaces.write[i]->end_frame(); + }; + }; +} XRServer::XRServer() { singleton = this; diff --git a/servers/xr_server.h b/servers/xr_server.h index a820634bd9..d9188d2de1 100644 --- a/servers/xr_server.h +++ b/servers/xr_server.h @@ -84,10 +84,6 @@ private: Transform3D world_origin; /* our world origin point, maps a location in our virtual world to the origin point in our real world tracking volume */ Transform3D reference_frame; /* our reference frame */ - uint64_t last_process_usec; /* for frame timing, usec when we did our processing */ - uint64_t last_commit_usec; /* for frame timing, usec when we finished committing both eyes */ - uint64_t last_frame_usec; /* time it took between process and committing, we should probably average this over the last x frames */ - protected: static XRServer *singleton; @@ -175,12 +171,16 @@ public: PackedStringArray get_suggested_pose_names(const StringName &p_tracker_name) const; // Q: Should we add get_suggested_input_names and get_suggested_haptic_names even though we don't use them for the IDE? - uint64_t get_last_process_usec(); - uint64_t get_last_commit_usec(); - uint64_t get_last_frame_usec(); - + // Process is called before we handle our physics process and game process. This is where our interfaces will update controller data and such. void _process(); - void _mark_commit(); + + // Pre-render is called right before we're rendering our viewports. + // This is where interfaces such as OpenVR and OpenXR will update positioning data. + // Many of these interfaces will also do a predictive sync which ensures we run at a steady framerate. + void pre_render(); + + // End-frame is called right after Godot has finished its rendering bits. + void end_frame(); XRServer(); ~XRServer(); -- cgit v1.2.3