Use YXZ convention for Euler angles.

As discussed in issues #1479 and #9782, choosing the up axis (which is Y in Godot) as the axis of the last (or first) rotation is helpful in practical use cases.
This also aligns Godot's convention with Unity, helping with a smoother transition for people who are used to working with Unity (issue #9905).

Internally, both XYZ and YXZ functions are kept, for potential future applications.

4 files changed, 143 insertions, 18 deletions

diff --git a/core/math/matrix3.cpp b/core/math/matrix3.cpp
index b64f34d977..f2f6ff93cf 100644
--- a/core/math/matrix3.cpp
+++ b/core/math/matrix3.cpp
@@ -338,7 +338,7 @@ void Basis::set_rotation_axis_angle(const Vector3 &p_axis, real_t p_angle) {
 	rotate(p_axis, p_angle);
 }
 
-// get_euler returns a vector containing the Euler angles in the format
+// get_euler_xyz returns a vector containing the Euler angles in the format
 // (a1,a2,a3), where a3 is the angle of the first rotation, and a1 is the last
 // (following the convention they are commonly defined in the literature).
 //
@@ -348,7 +348,7 @@ void Basis::set_rotation_axis_angle(const Vector3 &p_axis, real_t p_angle) {
 // And thus, assuming the matrix is a rotation matrix, this function returns
 // the angles in the decomposition R = X(a1).Y(a2).Z(a3) where Z(a) rotates
 // around the z-axis by a and so on.
-Vector3 Basis::get_euler() const {
+Vector3 Basis::get_euler_xyz() const {
 
 	// Euler angles in XYZ convention.
 	// See https://en.wikipedia.org/wiki/Euler_angles#Rotation_matrix
@@ -366,6 +366,9 @@ Vector3 Basis::get_euler() const {
 		if (euler.y > -Math_PI * 0.5) {
 			//if rotation is Y-only, return a proper -pi,pi range like in x or z for the same case.
 			if (elements[1][0] == 0.0 && elements[0][1] == 0.0 && elements[0][0] < 0.0) {
+				euler.x = 0;
+				euler.z = 0;
+
 				if (euler.y > 0.0)
 					euler.y = Math_PI - euler.y;
 				else
@@ -389,10 +392,11 @@ Vector3 Basis::get_euler() const {
 	return euler;
 }
 
-// set_euler expects a vector containing the Euler angles in the format
-// (c,b,a), where a is the angle of the first rotation, and c is the last.
+// set_euler_xyz expects a vector containing the Euler angles in the format
+// (ax,ay,az), where ax is the angle of rotation around x axis,
+// and similar for other axes.
 // The current implementation uses XYZ convention (Z is the first rotation).
-void Basis::set_euler(const Vector3 &p_euler) {
+void Basis::set_euler_xyz(const Vector3 &p_euler) {
 
 	real_t c, s;
 
@@ -412,6 +416,78 @@ void Basis::set_euler(const Vector3 &p_euler) {
 	*this = xmat * (ymat * zmat);
 }
 
+// get_euler_yxz returns a vector containing the Euler angles in the YXZ convention,
+// as in first-Z, then-X, last-Y. The angles for X, Y, and Z rotations are returned
+// as the x, y, and z components of a Vector3 respectively.
+Vector3 Basis::get_euler_yxz() const {
+
+	// Euler angles in YXZ convention.
+	// See https://en.wikipedia.org/wiki/Euler_angles#Rotation_matrix
+	//
+	// rot =  cy*cz+sy*sx*sz    cz*sy*sx-cy*sz        cx*sy
+	//        cx*sz             cx*cz                 -sx
+	//        cy*sx*sz-cz*sy    cy*cz*sx+sy*sz        cy*cx
+
+	Vector3 euler;
+#ifdef MATH_CHECKS
+	ERR_FAIL_COND_V(is_rotation() == false, euler);
+#endif
+	real_t m12 = elements[1][2];
+
+	if (m12 < 1) {
+		if (m12 > -1) {
+			if (elements[1][0] == 0 && elements[0][1] == 0 && elements[2][2] < 0) { // use pure x rotation
+				real_t x = asin(-m12);
+				euler.y = 0;
+				euler.z = 0;
+
+				if (x > 0.0)
+					euler.x = Math_PI - x;
+				else
+					euler.x = -(Math_PI + x);
+			} else {
+				euler.x = asin(-m12);
+				euler.y = atan2(elements[0][2], elements[2][2]);
+				euler.z = atan2(elements[1][0], elements[1][1]);
+			}
+		} else { // m12 == -1
+			euler.x = Math_PI * 0.5;
+			euler.y = -atan2(-elements[0][1], elements[0][0]);
+			euler.z = 0;
+		}
+	} else { // m12 == 1
+		euler.x = -Math_PI * 0.5;
+		euler.y = -atan2(-elements[0][1], elements[0][0]);
+		euler.z = 0;
+	}
+
+	return euler;
+}
+
+// set_euler_yxz expects a vector containing the Euler angles in the format
+// (ax,ay,az), where ax is the angle of rotation around x axis,
+// and similar for other axes.
+// The current implementation uses YXZ convention (Z is the first rotation).
+void Basis::set_euler_yxz(const Vector3 &p_euler) {
+
+	real_t c, s;
+
+	c = Math::cos(p_euler.x);
+	s = Math::sin(p_euler.x);
+	Basis xmat(1.0, 0.0, 0.0, 0.0, c, -s, 0.0, s, c);
+
+	c = Math::cos(p_euler.y);
+	s = Math::sin(p_euler.y);
+	Basis ymat(c, 0.0, s, 0.0, 1.0, 0.0, -s, 0.0, c);
+
+	c = Math::cos(p_euler.z);
+	s = Math::sin(p_euler.z);
+	Basis zmat(c, -s, 0.0, s, c, 0.0, 0.0, 0.0, 1.0);
+
+	//optimizer will optimize away all this anyway
+	*this = ymat * xmat * zmat;
+}
+
 bool Basis::is_equal_approx(const Basis &a, const Basis &b) const {
 
 	for (int i = 0; i < 3; i++) {
diff --git a/core/math/matrix3.h b/core/math/matrix3.h
index 8897c692f7..74e6564578 100644
--- a/core/math/matrix3.h
+++ b/core/math/matrix3.h
@@ -84,8 +84,13 @@ public:
 	void set_rotation_euler(const Vector3 &p_euler);
 	void set_rotation_axis_angle(const Vector3 &p_axis, real_t p_angle);
 
-	Vector3 get_euler() const;
-	void set_euler(const Vector3 &p_euler);
+	Vector3 get_euler_xyz() const;
+	void set_euler_xyz(const Vector3 &p_euler);
+	Vector3 get_euler_yxz() const;
+	void set_euler_yxz(const Vector3 &p_euler);
+
+	Vector3 get_euler() const { return get_euler_yxz(); };
+	void set_euler(const Vector3 &p_euler) { set_euler_yxz(p_euler); };
 
 	void get_axis_angle(Vector3 &r_axis, real_t &r_angle) const;
 	void set_axis_angle(const Vector3 &p_axis, real_t p_phi);
diff --git a/core/math/quat.cpp b/core/math/quat.cpp
index 0bea97c2e8..5984cdf657 100644
--- a/core/math/quat.cpp
+++ b/core/math/quat.cpp
@@ -31,10 +31,11 @@
 #include "matrix3.h"
 #include "print_string.h"
 
-// set_euler expects a vector containing the Euler angles in the format
-// (c,b,a), where a is the angle of the first rotation, and c is the last.
-// The current implementation uses XYZ convention (Z is the first rotation).
-void Quat::set_euler(const Vector3 &p_euler) {
+// set_euler_xyz expects a vector containing the Euler angles in the format
+// (ax,ay,az), where ax is the angle of rotation around x axis,
+// and similar for other axes.
+// This implementation uses XYZ convention (Z is the first rotation).
+void Quat::set_euler_xyz(const Vector3 &p_euler) {
 	real_t half_a1 = p_euler.x * 0.5;
 	real_t half_a2 = p_euler.y * 0.5;
 	real_t half_a3 = p_euler.z * 0.5;
@@ -56,12 +57,48 @@ void Quat::set_euler(const Vector3 &p_euler) {
 			-sin_a1 * sin_a2 * sin_a3 + cos_a1 * cos_a2 * cos_a3);
 }
 
-// get_euler returns a vector containing the Euler angles in the format
-// (a1,a2,a3), where a3 is the angle of the first rotation, and a1 is the last.
-// The current implementation uses XYZ convention (Z is the first rotation).
-Vector3 Quat::get_euler() const {
+// get_euler_xyz returns a vector containing the Euler angles in the format
+// (ax,ay,az), where ax is the angle of rotation around x axis,
+// and similar for other axes.
+// This implementation uses XYZ convention (Z is the first rotation).
+Vector3 Quat::get_euler_xyz() const {
 	Basis m(*this);
-	return m.get_euler();
+	return m.get_euler_xyz();
+}
+
+// set_euler_yxz expects a vector containing the Euler angles in the format
+// (ax,ay,az), where ax is the angle of rotation around x axis,
+// and similar for other axes.
+// This implementation uses YXZ convention (Z is the first rotation).
+void Quat::set_euler_yxz(const Vector3 &p_euler) {
+	real_t half_a1 = p_euler.y * 0.5;
+	real_t half_a2 = p_euler.x * 0.5;
+	real_t half_a3 = p_euler.z * 0.5;
+
+	// R = Y(a1).X(a2).Z(a3) convention for Euler angles.
+	// Conversion to quaternion as listed in https://ntrs.nasa.gov/archive/nasa/casi.ntrs.nasa.gov/19770024290.pdf (page A-6)
+	// a3 is the angle of the first rotation, following the notation in this reference.
+
+	real_t cos_a1 = Math::cos(half_a1);
+	real_t sin_a1 = Math::sin(half_a1);
+	real_t cos_a2 = Math::cos(half_a2);
+	real_t sin_a2 = Math::sin(half_a2);
+	real_t cos_a3 = Math::cos(half_a3);
+	real_t sin_a3 = Math::sin(half_a3);
+
+	set(sin_a1 * cos_a2 * sin_a3 + cos_a1 * sin_a2 * cos_a3,
+			sin_a1 * cos_a2 * cos_a3 - cos_a1 * sin_a2 * sin_a3,
+			-sin_a1 * sin_a2 * cos_a3 + cos_a1 * sin_a2 * sin_a3,
+			sin_a1 * sin_a2 * sin_a3 + cos_a1 * cos_a2 * cos_a3);
+}
+
+// get_euler_yxz returns a vector containing the Euler angles in the format
+// (ax,ay,az), where ax is the angle of rotation around x axis,
+// and similar for other axes.
+// This implementation uses YXZ convention (Z is the first rotation).
+Vector3 Quat::get_euler_yxz() const {
+	Basis m(*this);
+	return m.get_euler_yxz();
 }
 
 void Quat::operator*=(const Quat &q) {
diff --git a/core/math/quat.h b/core/math/quat.h
index f22275b457..0e378eb4e4 100644
--- a/core/math/quat.h
+++ b/core/math/quat.h
@@ -51,8 +51,15 @@ public:
 	bool is_normalized() const;
 	Quat inverse() const;
 	_FORCE_INLINE_ real_t dot(const Quat &q) const;
-	void set_euler(const Vector3 &p_euler);
-	Vector3 get_euler() const;
+
+	void set_euler_xyz(const Vector3 &p_euler);
+	Vector3 get_euler_xyz() const;
+	void set_euler_yxz(const Vector3 &p_euler);
+	Vector3 get_euler_yxz() const;
+
+	void set_euler(const Vector3 &p_euler) { set_euler_yxz(p_euler); };
+	Vector3 get_euler() const { return get_euler_yxz(); };
+
 	Quat slerp(const Quat &q, const real_t &t) const;
 	Quat slerpni(const Quat &q, const real_t &t) const;
 	Quat cubic_slerp(const Quat &q, const Quat &prep, const Quat &postq, const real_t &t) const;