Fixes to editor subgizmos

* Fixed subgizmo editing on scaled nodes.
* Added more clarifications on the coordinate space of subgizmos.
* Given input priority to the transform gizmo over subgizmo selection.

2 files changed, 45 insertions, 27 deletions

diff --git a/doc/classes/EditorNode3DGizmoPlugin.xml b/doc/classes/EditorNode3DGizmoPlugin.xml
index 10d6bd8bc8..6152487eaf 100644
--- a/doc/classes/EditorNode3DGizmoPlugin.xml
+++ b/doc/classes/EditorNode3DGizmoPlugin.xml
@@ -46,7 +46,7 @@
 			</argument>
 			<description>
 				Override this method to commit a group of subgizmos being edited (see [method _subgizmos_intersect_ray] and [method _subgizmos_intersect_frustum]). This usually means creating an [UndoRedo] action for the change, using the current transforms as "do" and the [code]restore[/code] transforms as "undo".
-				If the [code]cancel[/code] argument is [code]true[/code], the [code]restore[/code] transforms should be directly set, without any [UndoRedo] action. Called for this plugin's active gizmos.
+				If the [code]cancel[/code] argument is [code]true[/code], the [code]restore[/code] transforms should be directly set, without any [UndoRedo] action. As with all subgizmo methods, transforms are given in local space respect to the gizmo's Node3D. Called for this plugin's active gizmos.
 			</description>
 		</method>
 		<method name="_create_gizmo" qualifiers="virtual">
@@ -103,7 +103,7 @@
 			<argument index="1" name="id" type="int">
 			</argument>
 			<description>
-				Override this method to return the current transform of a subgizmo. This transform will be requested at the start of an edit and used in the [code]restore[/code] argument in [method _commit_subgizmos]. Called for this plugin's active gizmos.
+				Override this method to return the current transform of a subgizmo. As with all subgizmo methods, the transform should be in local space respect to the gizmo's Node3D. This transform will be requested at the start of an edit and used in the [code]restore[/code] argument in [method _commit_subgizmos]. Called for this plugin's active gizmos.
 			</description>
 		</method>
 		<method name="_has_gizmo" qualifiers="virtual">
diff --git a/editor/plugins/node_3d_editor_plugin.cpp b/editor/plugins/node_3d_editor_plugin.cpp
index b7651c350c..7c03c983ac 100644
--- a/editor/plugins/node_3d_editor_plugin.cpp
+++ b/editor/plugins/node_3d_editor_plugin.cpp
@@ -1423,17 +1423,15 @@ void Node3DEditorViewport::_sinput(const Ref<InputEvent> &p_event) {
 					_edit.snap = spatial_editor->is_snap_enabled();
 					_edit.mode = TRANSFORM_NONE;
 
-					//gizmo has priority over everything
-
-					bool can_select_gizmos = true;
+					bool can_select_gizmos = spatial_editor->get_single_selected_node();
 
 					{
 						int idx = view_menu->get_popup()->get_item_index(VIEW_GIZMOS);
-						can_select_gizmos = view_menu->get_popup()->is_item_checked(idx);
+						can_select_gizmos = can_select_gizmos && view_menu->get_popup()->is_item_checked(idx);
 					}
 
-					if (can_select_gizmos && spatial_editor->get_single_selected_node()) {
-						Node3DEditorSelectedItem *se = editor_selection->get_node_editor_data<Node3DEditorSelectedItem>(spatial_editor->get_single_selected_node());
+					// Gizmo handles
+					if (can_select_gizmos) {
 						Vector<Ref<Node3DGizmo>> gizmos = spatial_editor->get_single_selected_node()->get_gizmos();
 
 						bool intersected_handle = false;
@@ -1444,6 +1442,40 @@ void Node3DEditorViewport::_sinput(const Ref<InputEvent> &p_event) {
 								continue;
 							}
 
+							int gizmo_handle = -1;
+							seg->handles_intersect_ray(camera, _edit.mouse_pos, b->is_shift_pressed(), gizmo_handle);
+							if (gizmo_handle != -1) {
+								_edit.gizmo = seg;
+								_edit.gizmo_handle = gizmo_handle;
+								_edit.gizmo_initial_value = seg->get_handle_value(gizmo_handle);
+								intersected_handle = true;
+								break;
+							}
+						}
+
+						if (intersected_handle) {
+							break;
+						}
+					}
+
+					// Transform gizmo
+					if (_transform_gizmo_select(_edit.mouse_pos)) {
+						break;
+					}
+
+					// Subgizmos
+					if (can_select_gizmos) {
+						Node3DEditorSelectedItem *se = editor_selection->get_node_editor_data<Node3DEditorSelectedItem>(spatial_editor->get_single_selected_node());
+						Vector<Ref<Node3DGizmo>> gizmos = spatial_editor->get_single_selected_node()->get_gizmos();
+
+						bool intersected_subgizmo = false;
+						for (int i = 0; i < gizmos.size(); i++) {
+							Ref<EditorNode3DGizmo> seg = gizmos[i];
+
+							if ((!seg.is_valid())) {
+								continue;
+							}
+
 							int subgizmo_id = seg->subgizmos_intersect_ray(camera, _edit.mouse_pos);
 							if (subgizmo_id != -1) {
 								ERR_CONTINUE(!se);
@@ -1466,30 +1498,16 @@ void Node3DEditorViewport::_sinput(const Ref<InputEvent> &p_event) {
 
 								seg->redraw();
 								spatial_editor->update_transform_gizmo();
-								intersected_handle = true;
-								break;
-							}
-
-							int gizmo_handle = -1;
-							seg->handles_intersect_ray(camera, _edit.mouse_pos, b->is_shift_pressed(), gizmo_handle);
-							if (gizmo_handle != -1) {
-								_edit.gizmo = seg;
-								_edit.gizmo_handle = gizmo_handle;
-								_edit.gizmo_initial_value = seg->get_handle_value(gizmo_handle);
-								intersected_handle = true;
+								intersected_subgizmo = true;
 								break;
 							}
 						}
 
-						if (intersected_handle) {
+						if (intersected_subgizmo) {
 							break;
 						}
 					}
 
-					if (_transform_gizmo_select(_edit.mouse_pos)) {
-						break;
-					}
-
 					clicked = ObjectID();
 
 					if ((spatial_editor->get_tool_mode() == Node3DEditor::TOOL_MODE_SELECT && b->is_command_pressed()) || spatial_editor->get_tool_mode() == Node3DEditor::TOOL_MODE_ROTATE) {
@@ -1791,7 +1809,7 @@ void Node3DEditorViewport::_sinput(const Ref<InputEvent> &p_event) {
 									Transform3D xform = GE->get();
 									Transform3D new_xform = _compute_transform(TRANSFORM_SCALE, se->original * xform, xform, motion, snap, local_coords);
 									if (!local_coords) {
-										new_xform = se->original.inverse() * new_xform;
+										new_xform = se->original.affine_inverse() * new_xform;
 									}
 									se->gizmo->set_subgizmo_transform(GE->key(), new_xform);
 								}
@@ -1889,7 +1907,7 @@ void Node3DEditorViewport::_sinput(const Ref<InputEvent> &p_event) {
 								for (Map<int, Transform3D>::Element *GE = se->subgizmos.front(); GE; GE = GE->next()) {
 									Transform3D xform = GE->get();
 									Transform3D new_xform = _compute_transform(TRANSFORM_TRANSLATE, se->original * xform, xform, motion, snap, local_coords);
-									new_xform = se->original.inverse() * new_xform;
+									new_xform = se->original.affine_inverse() * new_xform;
 									se->gizmo->set_subgizmo_transform(GE->key(), new_xform);
 								}
 							} else {
@@ -1977,7 +1995,7 @@ void Node3DEditorViewport::_sinput(const Ref<InputEvent> &p_event) {
 
 									Transform3D new_xform = _compute_transform(TRANSFORM_ROTATE, se->original * xform, xform, compute_axis, angle, local_coords);
 									if (!local_coords) {
-										new_xform = se->original.inverse() * new_xform;
+										new_xform = se->original.affine_inverse() * new_xform;
 									}
 									se->gizmo->set_subgizmo_transform(GE->key(), new_xform);
 								}