Assimp FBX Import support

Issues fixed:
- Updated assimp to latest and backported fixes into godot.
- Fixed file scale being ignored from FBX file.
- Fixed bone removal
- Implemented proper armature binding
- Fixed recursion not always going through the entire path
- Implemented assimp global scaling system
- Fixed assimp global scale process to support unit conversion
- Implemented proper fbx scaling
- Fixed asserts caused by missing faces in some models which could crash
- Fixed valid bone removal
- Fixed root node being overwriten by assimp which caused data loss
- Fixed armature construction so that it works with multiple roots
- Implemented basic support for FBX standard materials
- Refactoring to improve code quality and improve function reuse.
- Simplified node creation from assimp scene into subsections: create_light, create_mesh, create_bone.
- Creating meshes is now done after hierarchy is created so that the skeleton is always available.
- Added support to assimp to support file scale in all formats which call SetFileScale.
- Many other fixes provided into assimp.

Known issues:
- FBX pivots from Maya do not currently work. (workaround: for now use blender import and export to remove pivot tracks)
- Hierarchy creates an extra node for each mesh - this was done intentionally but we intended to do a pass to remove these as they're a required node.
- When an animated mesh has not executed any animation the rest pose is wrong.

Co-authored-by: K. S. Ernest (iFire) Lee <ernest.lee@chibifire.com>

6 files changed, 47 insertions, 124 deletions

diff --git a/thirdparty/assimp/code/FBX/FBXConverter.cpp b/thirdparty/assimp/code/FBX/FBXConverter.cpp
index 9f940d3226..9bd970098e 100644
--- a/thirdparty/assimp/code/FBX/FBXConverter.cpp
+++ b/thirdparty/assimp/code/FBX/FBXConverter.cpp
@@ -66,6 +66,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <vector>
 #include <sstream>
 #include <iomanip>
+#include <cstdint>
 
 
 namespace Assimp {
@@ -90,7 +91,6 @@ namespace Assimp {
         , anim_fps()
         , out(out)
         , doc(doc)
-        , mRemoveEmptyBones( removeEmptyBones )
         , mCurrentUnit(FbxUnit::cm) {
             // animations need to be converted first since this will
             // populate the node_anim_chain_bits map, which is needed
@@ -119,7 +119,6 @@ namespace Assimp {
 
             ConvertGlobalSettings();
             TransferDataToScene();
-            ConvertToUnitScale(unit);
 
             // if we didn't read any meshes set the AI_SCENE_FLAGS_INCOMPLETE
             // to make sure the scene passes assimp's validation. FBX files
@@ -685,30 +684,37 @@ namespace Assimp {
             bool ok;
 
             aiMatrix4x4 chain[TransformationComp_MAXIMUM];
+
+            ai_assert(TransformationComp_MAXIMUM < 32);
+            std::uint32_t chainBits = 0;
+            // A node won't need a node chain if it only has these.
+            const std::uint32_t chainMaskSimple = (1 << TransformationComp_Translation) + (1 << TransformationComp_Scaling) + (1 << TransformationComp_Rotation);
+            // A node will need a node chain if it has any of these.
+            const std::uint32_t chainMaskComplex = ((1 << (TransformationComp_MAXIMUM)) - 1) - chainMaskSimple;
+
             std::fill_n(chain, static_cast<unsigned int>(TransformationComp_MAXIMUM), aiMatrix4x4());
 
             // generate transformation matrices for all the different transformation components
             const float zero_epsilon = 1e-6f;
             const aiVector3D all_ones(1.0f, 1.0f, 1.0f);
-            bool is_complex = false;
 
             const aiVector3D& PreRotation = PropertyGet<aiVector3D>(props, "PreRotation", ok);
             if (ok && PreRotation.SquareLength() > zero_epsilon) {
-                is_complex = true;
+                chainBits = chainBits | (1 << TransformationComp_PreRotation);
 
                 GetRotationMatrix(Model::RotOrder::RotOrder_EulerXYZ, PreRotation, chain[TransformationComp_PreRotation]);
             }
 
             const aiVector3D& PostRotation = PropertyGet<aiVector3D>(props, "PostRotation", ok);
             if (ok && PostRotation.SquareLength() > zero_epsilon) {
-                is_complex = true;
+                chainBits = chainBits | (1 << TransformationComp_PostRotation);
 
                 GetRotationMatrix(Model::RotOrder::RotOrder_EulerXYZ, PostRotation, chain[TransformationComp_PostRotation]);
             }
 
             const aiVector3D& RotationPivot = PropertyGet<aiVector3D>(props, "RotationPivot", ok);
             if (ok && RotationPivot.SquareLength() > zero_epsilon) {
-                is_complex = true;
+                chainBits = chainBits | (1 << TransformationComp_RotationPivot) | (1 << TransformationComp_RotationPivotInverse);
 
                 aiMatrix4x4::Translation(RotationPivot, chain[TransformationComp_RotationPivot]);
                 aiMatrix4x4::Translation(-RotationPivot, chain[TransformationComp_RotationPivotInverse]);
@@ -716,21 +722,21 @@ namespace Assimp {
 
             const aiVector3D& RotationOffset = PropertyGet<aiVector3D>(props, "RotationOffset", ok);
             if (ok && RotationOffset.SquareLength() > zero_epsilon) {
-                is_complex = true;
+                chainBits = chainBits | (1 << TransformationComp_RotationOffset);
 
                 aiMatrix4x4::Translation(RotationOffset, chain[TransformationComp_RotationOffset]);
             }
 
             const aiVector3D& ScalingOffset = PropertyGet<aiVector3D>(props, "ScalingOffset", ok);
             if (ok && ScalingOffset.SquareLength() > zero_epsilon) {
-                is_complex = true;
+                chainBits = chainBits | (1 << TransformationComp_ScalingOffset);
 
                 aiMatrix4x4::Translation(ScalingOffset, chain[TransformationComp_ScalingOffset]);
             }
 
             const aiVector3D& ScalingPivot = PropertyGet<aiVector3D>(props, "ScalingPivot", ok);
             if (ok && ScalingPivot.SquareLength() > zero_epsilon) {
-                is_complex = true;
+                chainBits = chainBits | (1 << TransformationComp_ScalingPivot) | (1 << TransformationComp_ScalingPivotInverse);
 
                 aiMatrix4x4::Translation(ScalingPivot, chain[TransformationComp_ScalingPivot]);
                 aiMatrix4x4::Translation(-ScalingPivot, chain[TransformationComp_ScalingPivotInverse]);
@@ -738,22 +744,28 @@ namespace Assimp {
 
             const aiVector3D& Translation = PropertyGet<aiVector3D>(props, "Lcl Translation", ok);
             if (ok && Translation.SquareLength() > zero_epsilon) {
+                chainBits = chainBits | (1 << TransformationComp_Translation);
+
                 aiMatrix4x4::Translation(Translation, chain[TransformationComp_Translation]);
             }
 
             const aiVector3D& Scaling = PropertyGet<aiVector3D>(props, "Lcl Scaling", ok);
             if (ok && (Scaling - all_ones).SquareLength() > zero_epsilon) {
+                chainBits = chainBits | (1 << TransformationComp_Scaling);
+
                 aiMatrix4x4::Scaling(Scaling, chain[TransformationComp_Scaling]);
             }
 
             const aiVector3D& Rotation = PropertyGet<aiVector3D>(props, "Lcl Rotation", ok);
             if (ok && Rotation.SquareLength() > zero_epsilon) {
+                chainBits = chainBits | (1 << TransformationComp_Rotation);
+
                 GetRotationMatrix(rot, Rotation, chain[TransformationComp_Rotation]);
             }
 
             const aiVector3D& GeometricScaling = PropertyGet<aiVector3D>(props, "GeometricScaling", ok);
             if (ok && (GeometricScaling - all_ones).SquareLength() > zero_epsilon) {
-                is_complex = true;
+                chainBits = chainBits | (1 << TransformationComp_GeometricScaling);
                 aiMatrix4x4::Scaling(GeometricScaling, chain[TransformationComp_GeometricScaling]);
                 aiVector3D GeometricScalingInverse = GeometricScaling;
                 bool canscale = true;
@@ -768,13 +780,14 @@ namespace Assimp {
                     }
                 }
                 if (canscale) {
+                    chainBits = chainBits | (1 << TransformationComp_GeometricScalingInverse);
                     aiMatrix4x4::Scaling(GeometricScalingInverse, chain[TransformationComp_GeometricScalingInverse]);
                 }
             }
 
             const aiVector3D& GeometricRotation = PropertyGet<aiVector3D>(props, "GeometricRotation", ok);
             if (ok && GeometricRotation.SquareLength() > zero_epsilon) {
-                is_complex = true;
+                chainBits = chainBits | (1 << TransformationComp_GeometricRotation) | (1 << TransformationComp_GeometricRotationInverse);
                 GetRotationMatrix(rot, GeometricRotation, chain[TransformationComp_GeometricRotation]);
                 GetRotationMatrix(rot, GeometricRotation, chain[TransformationComp_GeometricRotationInverse]);
                 chain[TransformationComp_GeometricRotationInverse].Inverse();
@@ -782,7 +795,7 @@ namespace Assimp {
 
             const aiVector3D& GeometricTranslation = PropertyGet<aiVector3D>(props, "GeometricTranslation", ok);
             if (ok && GeometricTranslation.SquareLength() > zero_epsilon) {
-                is_complex = true;
+                chainBits = chainBits | (1 << TransformationComp_GeometricTranslation) | (1 << TransformationComp_GeometricTranslationInverse);
                 aiMatrix4x4::Translation(GeometricTranslation, chain[TransformationComp_GeometricTranslation]);
                 aiMatrix4x4::Translation(-GeometricTranslation, chain[TransformationComp_GeometricTranslationInverse]);
             }
@@ -790,12 +803,12 @@ namespace Assimp {
             // is_complex needs to be consistent with NeedsComplexTransformationChain()
             // or the interplay between this code and the animation converter would
             // not be guaranteed.
-            ai_assert(NeedsComplexTransformationChain(model) == is_complex);
+            ai_assert(NeedsComplexTransformationChain(model) == ((chainBits & chainMaskComplex) != 0));
 
             // now, if we have more than just Translation, Scaling and Rotation,
             // we need to generate a full node chain to accommodate for assimp's
             // lack to express pivots and offsets.
-            if (is_complex && doc.Settings().preservePivots) {
+            if ((chainBits & chainMaskComplex) && doc.Settings().preservePivots) {
                 FBXImporter::LogInfo("generating full transformation chain for node: " + name);
 
                 // query the anim_chain_bits dictionary to find out which chain elements
@@ -808,7 +821,7 @@ namespace Assimp {
                 for (size_t i = 0; i < TransformationComp_MAXIMUM; ++i, bit <<= 1) {
                     const TransformationComp comp = static_cast<TransformationComp>(i);
 
-                    if (chain[i].IsIdentity() && (anim_chain_bitmask & bit) == 0) {
+                    if ((chainBits & bit) == 0 && (anim_chain_bitmask & bit) == 0) {
                         continue;
                     }
 
@@ -1462,14 +1475,8 @@ namespace Assimp {
 
                     const WeightIndexArray& indices = cluster->GetIndices();
 
-                    if (indices.empty() && mRemoveEmptyBones ) {
-                        continue;
-                    }
-
                     const MatIndexArray& mats = geo.GetMaterialIndices();
 
-                    bool ok = false;
-
                     const size_t no_index_sentinel = std::numeric_limits<size_t>::max();
 
                     count_out_indices.clear();
@@ -1509,8 +1516,7 @@ namespace Assimp {
                                     out_indices.push_back(std::distance(outputVertStartIndices->begin(), it));
                                 }
 
-                                ++count_out_indices.back();
-                                ok = true;
+                                ++count_out_indices.back();                               
                             }
                         }
                     }
@@ -1518,10 +1524,8 @@ namespace Assimp {
                     // if we found at least one, generate the output bones
                     // XXX this could be heavily simplified by collecting the bone
                     // data in a single step.
-                    if (ok && mRemoveEmptyBones) {
-                        ConvertCluster(bones, model, *cluster, out_indices, index_out_indices,
+                    ConvertCluster(bones, model, *cluster, out_indices, index_out_indices,
                             count_out_indices, node_global_transform);
-                    }
                 }
             }
             catch (std::exception&) {
@@ -3532,46 +3536,6 @@ void FBXConverter::SetShadingPropertiesRaw(aiMaterial* out_mat, const PropertyTa
             out->mMetaData->Set(14, "CustomFrameRate", doc.GlobalSettings().CustomFrameRate());
         }
 
-        void FBXConverter::ConvertToUnitScale( FbxUnit unit ) {
-            if (mCurrentUnit == unit) {
-                return;
-            }
-
-            ai_real scale = 1.0;
-            if (mCurrentUnit == FbxUnit::cm) {
-                if (unit == FbxUnit::m) {
-                    scale = (ai_real)0.01;
-                } else if (unit == FbxUnit::km) {
-                    scale = (ai_real)0.00001;
-                }
-            } else if (mCurrentUnit == FbxUnit::m) {
-                if (unit == FbxUnit::cm) {
-                    scale = (ai_real)100.0;
-                } else if (unit == FbxUnit::km) {
-                    scale = (ai_real)0.001;
-                }
-            } else if (mCurrentUnit == FbxUnit::km) {
-                if (unit == FbxUnit::cm) {
-                    scale = (ai_real)100000.0;
-                } else if (unit == FbxUnit::m) {
-                    scale = (ai_real)1000.0;
-                }
-            }
-            
-            for (auto mesh : meshes) {
-                if (nullptr == mesh) {
-                    continue;
-                }
-
-                if (mesh->HasPositions()) {
-                    for (unsigned int i = 0; i < mesh->mNumVertices; ++i) {
-                        aiVector3D &pos = mesh->mVertices[i];
-                        pos *= scale;
-                    }
-                }
-            }
-        }
-
         void FBXConverter::TransferDataToScene()
         {
             ai_assert(!out->mMeshes);
diff --git a/thirdparty/assimp/code/FBX/FBXConverter.h b/thirdparty/assimp/code/FBX/FBXConverter.h
index 17a7bc56b7..b458627392 100644
--- a/thirdparty/assimp/code/FBX/FBXConverter.h
+++ b/thirdparty/assimp/code/FBX/FBXConverter.h
@@ -431,10 +431,6 @@ private:
     void ConvertGlobalSettings();
 
     // ------------------------------------------------------------------------------------------------
-    //  Will perform the conversion from a given unit to the requested unit.
-    void ConvertToUnitScale(FbxUnit unit);
-
-    // ------------------------------------------------------------------------------------------------
     // copy generated meshes, animations, lights, cameras and textures to the output scene
     void TransferDataToScene();
 
@@ -470,9 +466,6 @@ private:
 
     aiScene* const out;
     const FBX::Document& doc;
-
-    bool mRemoveEmptyBones;
-
     FbxUnit mCurrentUnit;
 };
 
diff --git a/thirdparty/assimp/code/FBX/FBXDocument.cpp b/thirdparty/assimp/code/FBX/FBXDocument.cpp
index 1af08fe6d8..506fd978dd 100644
--- a/thirdparty/assimp/code/FBX/FBXDocument.cpp
+++ b/thirdparty/assimp/code/FBX/FBXDocument.cpp
@@ -90,14 +90,6 @@ const Object* LazyObject::Get(bool dieOnError)
         return object.get();
     }
 
-    // if this is the root object, we return a dummy since there
-    // is no root object int he fbx file - it is just referenced
-    // with id 0.
-    if(id == 0L) {
-        object.reset(new Object(id, element, "Model::RootNode"));
-        return object.get();
-    }
-
     const Token& key = element.KeyToken();
     const TokenList& tokens = element.Tokens();
 
diff --git a/thirdparty/assimp/code/FBX/FBXExporter.cpp b/thirdparty/assimp/code/FBX/FBXExporter.cpp
index 153e676506..8ebc8555a2 100644
--- a/thirdparty/assimp/code/FBX/FBXExporter.cpp
+++ b/thirdparty/assimp/code/FBX/FBXExporter.cpp
@@ -1706,8 +1706,7 @@ void FBXExporter::WriteObjects ()
                     }
                     if (end) { break; }
                 }
-                limbnodes.insert(parent);
-                skeleton.insert(parent);
+                
                 // if it was the skeleton root we can finish here
                 if (end) { break; }
             }
@@ -1848,44 +1847,10 @@ void FBXExporter::WriteObjects ()
             inverse_bone_xform.Inverse();
             aiMatrix4x4 tr = inverse_bone_xform * mesh_xform;
 
-            // this should be the same as the bone's mOffsetMatrix.
-            // if it's not the same, the skeleton isn't in the bind pose.
-            float epsilon = 1e-4f; // some error is to be expected
-            float epsilon_custom = mProperties->GetPropertyFloat("BINDPOSE_EPSILON", -1);
-            if(epsilon_custom > 0)
-                epsilon = epsilon_custom;
-            bool bone_xform_okay = true;
-            if (b && ! tr.Equal(b->mOffsetMatrix, epsilon)) {
-                not_in_bind_pose.insert(b);
-                bone_xform_okay = false;
-            }
+            sdnode.AddChild("Transform", tr);
 
-            // if we have a bone we should use the mOffsetMatrix,
-            // otherwise try to just use the calculated transform.
-            if (b) {
-                sdnode.AddChild("Transform", b->mOffsetMatrix);
-            } else {
-                sdnode.AddChild("Transform", tr);
-            }
-            // note: it doesn't matter if we mix these,
-            // because if they disagree we'll throw an exception later.
-            // it could be that the skeleton is not in the bone pose
-            // but all bones are still defined,
-            // in which case this would use the mOffsetMatrix for everything
-            // and a correct skeleton would still be output.
-
-            // transformlink should be the position of the bone in world space.
-            // if the bone is in the bind pose (or nonexistent),
-            // we can just use the matrix we already calculated
-            if (bone_xform_okay) {
-                sdnode.AddChild("TransformLink", bone_xform);
-            // otherwise we can only work it out using the mesh position.
-            } else {
-                aiMatrix4x4 trl = b->mOffsetMatrix;
-                trl.Inverse();
-                trl *= mesh_xform;
-                sdnode.AddChild("TransformLink", trl);
-            }
+
+            sdnode.AddChild("TransformLink", bone_xform);
             // note: this means we ALWAYS rely on the mesh node transform
             // being unchanged from the time the skeleton was bound.
             // there's not really any way around this at the moment.
diff --git a/thirdparty/assimp/code/FBX/FBXImporter.cpp b/thirdparty/assimp/code/FBX/FBXImporter.cpp
index ec8bbd2b47..bd359dbf29 100644
--- a/thirdparty/assimp/code/FBX/FBXImporter.cpp
+++ b/thirdparty/assimp/code/FBX/FBXImporter.cpp
@@ -189,8 +189,16 @@ void FBXImporter::InternReadFile( const std::string& pFile, aiScene* pScene, IOS
         if (settings.convertToMeters) {
             unit = FbxUnit::m;
         }
+
         // convert the FBX DOM to aiScene
-        ConvertToAssimpScene(pScene,doc, settings.removeEmptyBones, unit);
+        ConvertToAssimpScene(pScene, doc, settings.removeEmptyBones, unit);
+
+        // size relative to cm
+        float size_relative_to_cm = doc.GlobalSettings().UnitScaleFactor();
+
+        // Set FBX file scale is relative to CM must be converted to M for
+        // assimp universal format (M)
+        SetFileScale( size_relative_to_cm * 0.01f);
 
         std::for_each(tokens.begin(),tokens.end(),Util::delete_fun<Token>());
     }
diff --git a/thirdparty/assimp/code/FBX/FBXMeshGeometry.cpp b/thirdparty/assimp/code/FBX/FBXMeshGeometry.cpp
index 44a0264ca0..5c9a0e309d 100644
--- a/thirdparty/assimp/code/FBX/FBXMeshGeometry.cpp
+++ b/thirdparty/assimp/code/FBX/FBXMeshGeometry.cpp
@@ -115,7 +115,6 @@ MeshGeometry::MeshGeometry(uint64_t id, const Element& element, const std::strin
 
     if(tempVerts.empty()) {
         FBXImporter::LogWarn("encountered mesh with no vertices");
-        return;
     }
 
     std::vector<int> tempFaces;
@@ -123,7 +122,6 @@ MeshGeometry::MeshGeometry(uint64_t id, const Element& element, const std::strin
 
     if(tempFaces.empty()) {
         FBXImporter::LogWarn("encountered mesh with no faces");
-        return;
     }
 
     m_vertices.reserve(tempFaces.size());
@@ -612,7 +610,10 @@ void MeshGeometry::ReadVertexDataMaterials(std::vector<int>& materials_out, cons
     const std::string& ReferenceInformationType)
 {
     const size_t face_count = m_faces.size();
-    ai_assert(face_count);
+    if(face_count <= 0)
+    {
+        return;
+    }
 
     // materials are handled separately. First of all, they are assigned per-face
     // and not per polyvert. Secondly, ReferenceInformationType=IndexToDirect