Fix the order in which additional transformations are applied in Matrix3 and Transform.

This is a part of the breaking changes proposed in PR #6865, solving the issue regarding the order of affine transformations described in #2565. This PR also fixes the affected code within Godot codebase. Includes improvements to documentation too.

Another change is, Matrix3::get_scale() will now return negative scaling when the determinant of the matrix is negative. The rationale behind this is simple: when performing a polar decomposition on a basis matrix M = R.S, we have to ensure that the determinant of R is +1, such that it is a proper rotation matrix (with no reflections) which can be represented by Euler angles or a quaternion.

Also replaced the few instances of float with real_t in Matrix3 and Transform.

Furthermore, this PR fixes an issue introduced due to the API breakage in #6865. Namely Matrix3::get_euler() now only works with proper rotation matrices. As a result, the code that wants to get the rotation portion of a transform needs to use Matrix3::get_rotation() introduced in this commit, which complements Matrix3::get_scaled(), providing both parts of the polar decomposition.

Finally, it is now possible to construct a rotation matrix from Euler angles using the new constructor Matrix3::Matrix3(const Vector3 &p_euler).

2 files changed, 10 insertions, 20 deletions

diff --git a/tools/editor/plugins/multimesh_editor_plugin.cpp b/tools/editor/plugins/multimesh_editor_plugin.cpp
index 9b19542268..4e9c4ebf9d 100644
--- a/tools/editor/plugins/multimesh_editor_plugin.cpp
+++ b/tools/editor/plugins/multimesh_editor_plugin.cpp
@@ -238,9 +238,10 @@ void MultiMeshEditor::_populate() {
 
 		Matrix3 post_xform;
 
-		post_xform.rotate(xform.basis.get_axis(0),-Math::random(-_tilt_random,_tilt_random)*Math_PI);
-		post_xform.rotate(xform.basis.get_axis(2),-Math::random(-_tilt_random,_tilt_random)*Math_PI);
 		post_xform.rotate(xform.basis.get_axis(1),-Math::random(-_rotate_random,_rotate_random)*Math_PI);
+		post_xform.rotate(xform.basis.get_axis(2),-Math::random(-_tilt_random,_tilt_random)*Math_PI);
+		post_xform.rotate(xform.basis.get_axis(0),-Math::random(-_tilt_random,_tilt_random)*Math_PI);
+
 		xform.basis = post_xform * xform.basis;
 		//xform.basis.orthonormalize();
 
diff --git a/tools/editor/plugins/spatial_editor_plugin.cpp b/tools/editor/plugins/spatial_editor_plugin.cpp
index a577e97843..b6627d72bf 100644
--- a/tools/editor/plugins/spatial_editor_plugin.cpp
+++ b/tools/editor/plugins/spatial_editor_plugin.cpp
@@ -61,8 +61,8 @@ void SpatialEditorViewport::_update_camera() {
 
 	Transform camera_transform;
 	camera_transform.translate(cursor.pos);
-	camera_transform.basis.rotate(Vector3(0, 1, 0), -cursor.y_rot);
 	camera_transform.basis.rotate(Vector3(1, 0, 0), -cursor.x_rot);
+	camera_transform.basis.rotate(Vector3(0, 1, 0), -cursor.y_rot);
 
 	if (orthogonal)
 		camera_transform.translate(0, 0, 4096);
@@ -474,8 +474,8 @@ Vector3 SpatialEditorViewport::_get_screen_to_space(const Vector3& p_pos) {
 
 	Transform camera_transform;
 	camera_transform.translate( cursor.pos );
-	camera_transform.basis.rotate(Vector3(0,1,0),-cursor.y_rot);
 	camera_transform.basis.rotate(Vector3(1,0,0),-cursor.x_rot);
+	camera_transform.basis.rotate(Vector3(0,1,0),-cursor.y_rot);
 	camera_transform.translate(0,0,cursor.distance);
 
 	return camera_transform.xform(Vector3( ((p_pos.x/get_size().width)*2.0-1.0)*screen_w, ((1.0-(p_pos.y/get_size().height))*2.0-1.0)*screen_h,-get_znear()));
@@ -1502,7 +1502,7 @@ void SpatialEditorViewport::_sinput(const InputEvent &p_event) {
 								Transform original=se->original;
 
 								Transform base=Transform( Matrix3(), _edit.center);
-								Transform t=base * (r * (base.inverse() * original));
+								Transform t=base * r * base.inverse() * original;
 
 								sp->set_global_transform(t);
 							}
@@ -1591,8 +1591,8 @@ void SpatialEditorViewport::_sinput(const InputEvent &p_event) {
 					Transform camera_transform;
 
 					camera_transform.translate(cursor.pos);
-					camera_transform.basis.rotate(Vector3(0,1,0),-cursor.y_rot);
 					camera_transform.basis.rotate(Vector3(1,0,0),-cursor.x_rot);
+					camera_transform.basis.rotate(Vector3(0,1,0),-cursor.y_rot);
 					Vector3 translation(-m.relative_x*pan_speed,m.relative_y*pan_speed,0);
 					translation*=cursor.distance/DISTANCE_DEFAULT;
 					camera_transform.translate(translation);
@@ -2803,21 +2803,10 @@ void SpatialEditor::_xform_dialog_action() {
 		rotate[i]=Math::deg2rad(xform_rotate[i]->get_text().to_double());
 		scale[i]=xform_scale[i]->get_text().to_double();
 	}
-
+	
+	t.basis.scale(scale);
+	t.basis.rotate(rotate);
 	t.origin=translate;
-	for(int i=0;i<3;i++) {
-		if (!rotate[i])
-			continue;
-		Vector3 axis;
-		axis[i]=1.0;
-		t.basis.rotate(axis,rotate[i]); // BUG(?): Angle not flipped; please check during the review of PR #6865.
-	}
-
-	for(int i=0;i<3;i++) {
-		if (scale[i]==1)
-			continue;
-		t.basis.set_axis(i,t.basis.get_axis(i)*scale[i]);
-	}
 
 
 	undo_redo->create_action(TTR("XForm Dialog"));