diff options
Diffstat (limited to 'thirdparty')
-rw-r--r-- | thirdparty/README.md | 32 | ||||
-rw-r--r-- | thirdparty/assimp/assimp/config.h | 16 | ||||
-rw-r--r-- | thirdparty/assimp/code/CApi/AssimpCExport.cpp | 156 | ||||
-rw-r--r-- | thirdparty/assimp/code/CApi/CInterfaceIOWrapper.cpp (renamed from thirdparty/assimp/code/CInterfaceIOWrapper.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/CApi/CInterfaceIOWrapper.h (renamed from thirdparty/assimp/code/CInterfaceIOWrapper.h) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/Assimp.cpp | 695 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/BaseImporter.cpp (renamed from thirdparty/assimp/code/BaseImporter.cpp) | 6 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/BaseProcess.cpp (renamed from thirdparty/assimp/code/BaseProcess.cpp) | 2 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/BaseProcess.h (renamed from thirdparty/assimp/code/BaseProcess.h) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/Bitmap.cpp (renamed from thirdparty/assimp/code/Bitmap.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/CreateAnimMesh.cpp (renamed from thirdparty/assimp/code/CreateAnimMesh.cpp) | 4 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/DefaultIOStream.cpp (renamed from thirdparty/assimp/code/DefaultIOStream.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/DefaultIOSystem.cpp (renamed from thirdparty/assimp/code/DefaultIOSystem.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/DefaultLogger.cpp (renamed from thirdparty/assimp/code/DefaultLogger.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/DefaultProgressHandler.h (renamed from thirdparty/assimp/code/DefaultProgressHandler.h) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/Exporter.cpp (renamed from thirdparty/assimp/code/Exporter.cpp) | 32 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/FileLogStream.h (renamed from thirdparty/assimp/code/FileLogStream.h) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/FileSystemFilter.h (renamed from thirdparty/assimp/code/FileSystemFilter.h) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/IFF.h | 102 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/Importer.cpp (renamed from thirdparty/assimp/code/Importer.cpp) | 19 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/Importer.h (renamed from thirdparty/assimp/code/Importer.h) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/ImporterRegistry.cpp (renamed from thirdparty/assimp/code/ImporterRegistry.cpp) | 96 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/PolyTools.h (renamed from thirdparty/assimp/code/PolyTools.h) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/PostStepRegistry.cpp (renamed from thirdparty/assimp/code/PostStepRegistry.cpp) | 63 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/RemoveComments.cpp (renamed from thirdparty/assimp/code/RemoveComments.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/SGSpatialSort.cpp (renamed from thirdparty/assimp/code/SGSpatialSort.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/SceneCombiner.cpp (renamed from thirdparty/assimp/code/SceneCombiner.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/ScenePreprocessor.cpp (renamed from thirdparty/assimp/code/ScenePreprocessor.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/ScenePreprocessor.h (renamed from thirdparty/assimp/code/ScenePreprocessor.h) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/ScenePrivate.h (renamed from thirdparty/assimp/code/ScenePrivate.h) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/SkeletonMeshBuilder.cpp (renamed from thirdparty/assimp/code/SkeletonMeshBuilder.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/SpatialSort.cpp (renamed from thirdparty/assimp/code/SpatialSort.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/SplitByBoneCountProcess.cpp (renamed from thirdparty/assimp/code/SplitByBoneCountProcess.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/SplitByBoneCountProcess.h (renamed from thirdparty/assimp/code/SplitByBoneCountProcess.h) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/StandardShapes.cpp (renamed from thirdparty/assimp/code/StandardShapes.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/StdOStreamLogStream.h (renamed from thirdparty/assimp/code/StdOStreamLogStream.h) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/Subdivision.cpp (renamed from thirdparty/assimp/code/Subdivision.cpp) | 7 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/TargetAnimation.cpp (renamed from thirdparty/assimp/code/TargetAnimation.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/TargetAnimation.h (renamed from thirdparty/assimp/code/TargetAnimation.h) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/Version.cpp (renamed from thirdparty/assimp/code/Version.cpp) | 2 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/VertexTriangleAdjacency.cpp (renamed from thirdparty/assimp/code/VertexTriangleAdjacency.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/VertexTriangleAdjacency.h (renamed from thirdparty/assimp/code/VertexTriangleAdjacency.h) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/Win32DebugLogStream.h (renamed from thirdparty/assimp/code/Win32DebugLogStream.h) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/assbin_chunks.h | 196 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/scene.cpp (renamed from thirdparty/assimp/code/scene.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/simd.cpp (renamed from thirdparty/assimp/code/simd.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/Common/simd.h (renamed from thirdparty/assimp/code/simd.h) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/FBX/FBXAnimation.cpp (renamed from thirdparty/assimp/code/FBXAnimation.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/FBX/FBXBinaryTokenizer.cpp (renamed from thirdparty/assimp/code/FBXBinaryTokenizer.cpp) | 12 | ||||
-rw-r--r-- | thirdparty/assimp/code/FBX/FBXCommon.h (renamed from thirdparty/assimp/code/FBXCommon.h) | 4 | ||||
-rw-r--r-- | thirdparty/assimp/code/FBX/FBXCompileConfig.h (renamed from thirdparty/assimp/code/FBXCompileConfig.h) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/FBX/FBXConverter.cpp (renamed from thirdparty/assimp/code/FBXConverter.cpp) | 268 | ||||
-rw-r--r-- | thirdparty/assimp/code/FBX/FBXConverter.h (renamed from thirdparty/assimp/code/FBXConverter.h) | 37 | ||||
-rw-r--r-- | thirdparty/assimp/code/FBX/FBXDeformer.cpp (renamed from thirdparty/assimp/code/FBXDeformer.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/FBX/FBXDocument.cpp (renamed from thirdparty/assimp/code/FBXDocument.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/FBX/FBXDocument.h (renamed from thirdparty/assimp/code/FBXDocument.h) | 4 | ||||
-rw-r--r-- | thirdparty/assimp/code/FBX/FBXDocumentUtil.cpp (renamed from thirdparty/assimp/code/FBXDocumentUtil.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/FBX/FBXDocumentUtil.h (renamed from thirdparty/assimp/code/FBXDocumentUtil.h) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/FBX/FBXExportNode.cpp (renamed from thirdparty/assimp/code/FBXExportNode.cpp) | 11 | ||||
-rw-r--r-- | thirdparty/assimp/code/FBX/FBXExportNode.h (renamed from thirdparty/assimp/code/FBXExportNode.h) | 12 | ||||
-rw-r--r-- | thirdparty/assimp/code/FBX/FBXExportProperty.cpp (renamed from thirdparty/assimp/code/FBXExportProperty.cpp) | 257 | ||||
-rw-r--r-- | thirdparty/assimp/code/FBX/FBXExportProperty.h (renamed from thirdparty/assimp/code/FBXExportProperty.h) | 48 | ||||
-rw-r--r-- | thirdparty/assimp/code/FBX/FBXExporter.cpp (renamed from thirdparty/assimp/code/FBXExporter.cpp) | 115 | ||||
-rw-r--r-- | thirdparty/assimp/code/FBX/FBXExporter.h (renamed from thirdparty/assimp/code/FBXExporter.h) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/FBX/FBXImportSettings.h (renamed from thirdparty/assimp/code/FBXImportSettings.h) | 39 | ||||
-rw-r--r-- | thirdparty/assimp/code/FBX/FBXImporter.cpp (renamed from thirdparty/assimp/code/FBXImporter.cpp) | 10 | ||||
-rw-r--r-- | thirdparty/assimp/code/FBX/FBXImporter.h (renamed from thirdparty/assimp/code/FBXImporter.h) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/FBX/FBXMaterial.cpp (renamed from thirdparty/assimp/code/FBXMaterial.cpp) | 44 | ||||
-rw-r--r-- | thirdparty/assimp/code/FBX/FBXMeshGeometry.cpp (renamed from thirdparty/assimp/code/FBXMeshGeometry.cpp) | 11 | ||||
-rw-r--r-- | thirdparty/assimp/code/FBX/FBXMeshGeometry.h (renamed from thirdparty/assimp/code/FBXMeshGeometry.h) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/FBX/FBXModel.cpp (renamed from thirdparty/assimp/code/FBXModel.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/FBX/FBXNodeAttribute.cpp (renamed from thirdparty/assimp/code/FBXNodeAttribute.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/FBX/FBXParser.cpp (renamed from thirdparty/assimp/code/FBXParser.cpp) | 12 | ||||
-rw-r--r-- | thirdparty/assimp/code/FBX/FBXParser.h (renamed from thirdparty/assimp/code/FBXParser.h) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/FBX/FBXProperties.cpp (renamed from thirdparty/assimp/code/FBXProperties.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/FBX/FBXProperties.h (renamed from thirdparty/assimp/code/FBXProperties.h) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/FBX/FBXTokenizer.cpp (renamed from thirdparty/assimp/code/FBXTokenizer.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/FBX/FBXTokenizer.h (renamed from thirdparty/assimp/code/FBXTokenizer.h) | 12 | ||||
-rw-r--r-- | thirdparty/assimp/code/FBX/FBXUtil.cpp (renamed from thirdparty/assimp/code/FBXUtil.cpp) | 135 | ||||
-rw-r--r-- | thirdparty/assimp/code/FBX/FBXUtil.h (renamed from thirdparty/assimp/code/FBXUtil.h) | 23 | ||||
-rw-r--r-- | thirdparty/assimp/code/FIReader.cpp | 1834 | ||||
-rw-r--r-- | thirdparty/assimp/code/MMD/MMDCpp14.h (renamed from thirdparty/assimp/code/MMDCpp14.h) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/MMD/MMDImporter.cpp (renamed from thirdparty/assimp/code/MMDImporter.cpp) | 12 | ||||
-rw-r--r-- | thirdparty/assimp/code/MMD/MMDImporter.h (renamed from thirdparty/assimp/code/MMDImporter.h) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/MMD/MMDPmdParser.h (renamed from thirdparty/assimp/code/MMDPmdParser.h) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/MMD/MMDPmxParser.cpp (renamed from thirdparty/assimp/code/MMDPmxParser.cpp) | 6 | ||||
-rw-r--r-- | thirdparty/assimp/code/MMD/MMDPmxParser.h (renamed from thirdparty/assimp/code/MMDPmxParser.h) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/MMD/MMDVmdParser.h (renamed from thirdparty/assimp/code/MMDVmdParser.h) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/Material/MaterialSystem.cpp (renamed from thirdparty/assimp/code/MaterialSystem.cpp) | 12 | ||||
-rw-r--r-- | thirdparty/assimp/code/Material/MaterialSystem.h (renamed from thirdparty/assimp/code/MaterialSystem.h) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/CalcTangentsProcess.cpp (renamed from thirdparty/assimp/code/CalcTangentsProcess.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/CalcTangentsProcess.h (renamed from thirdparty/assimp/code/CalcTangentsProcess.h) | 4 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/ComputeUVMappingProcess.cpp (renamed from thirdparty/assimp/code/ComputeUVMappingProcess.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/ComputeUVMappingProcess.h (renamed from thirdparty/assimp/code/ComputeUVMappingProcess.h) | 3 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/ConvertToLHProcess.cpp (renamed from thirdparty/assimp/code/ConvertToLHProcess.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/ConvertToLHProcess.h (renamed from thirdparty/assimp/code/ConvertToLHProcess.h) | 3 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/DeboneProcess.cpp (renamed from thirdparty/assimp/code/DeboneProcess.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/DeboneProcess.h (renamed from thirdparty/assimp/code/DeboneProcess.h) | 17 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/DropFaceNormalsProcess.cpp (renamed from thirdparty/assimp/code/DropFaceNormalsProcess.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/DropFaceNormalsProcess.h (renamed from thirdparty/assimp/code/DropFaceNormalsProcess.h) | 11 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/EmbedTexturesProcess.cpp (renamed from thirdparty/assimp/code/EmbedTexturesProcess.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/EmbedTexturesProcess.h (renamed from thirdparty/assimp/code/EmbedTexturesProcess.h) | 2 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/FindDegenerates.cpp (renamed from thirdparty/assimp/code/FindDegenerates.cpp) | 1 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/FindDegenerates.h (renamed from thirdparty/assimp/code/FindDegenerates.h) | 3 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/FindInstancesProcess.cpp (renamed from thirdparty/assimp/code/FindInstancesProcess.cpp) | 12 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/FindInstancesProcess.h (renamed from thirdparty/assimp/code/FindInstancesProcess.h) | 4 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/FindInvalidDataProcess.cpp (renamed from thirdparty/assimp/code/FindInvalidDataProcess.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/FindInvalidDataProcess.h (renamed from thirdparty/assimp/code/FindInvalidDataProcess.h) | 3 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/FixNormalsStep.cpp (renamed from thirdparty/assimp/code/FixNormalsStep.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/FixNormalsStep.h (renamed from thirdparty/assimp/code/FixNormalsStep.h) | 2 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/GenBoundingBoxesProcess.cpp | 115 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/GenBoundingBoxesProcess.h | 76 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/GenFaceNormalsProcess.cpp (renamed from thirdparty/assimp/code/GenFaceNormalsProcess.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/GenFaceNormalsProcess.h (renamed from thirdparty/assimp/code/GenFaceNormalsProcess.h) | 2 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/GenVertexNormalsProcess.cpp (renamed from thirdparty/assimp/code/GenVertexNormalsProcess.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/GenVertexNormalsProcess.h (renamed from thirdparty/assimp/code/GenVertexNormalsProcess.h) | 18 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/ImproveCacheLocality.cpp (renamed from thirdparty/assimp/code/ImproveCacheLocality.cpp) | 65 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/ImproveCacheLocality.h (renamed from thirdparty/assimp/code/ImproveCacheLocality.h) | 7 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/JoinVerticesProcess.cpp (renamed from thirdparty/assimp/code/JoinVerticesProcess.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/JoinVerticesProcess.h (renamed from thirdparty/assimp/code/JoinVerticesProcess.h) | 10 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/LimitBoneWeightsProcess.cpp (renamed from thirdparty/assimp/code/LimitBoneWeightsProcess.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/LimitBoneWeightsProcess.h (renamed from thirdparty/assimp/code/LimitBoneWeightsProcess.h) | 34 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/MakeVerboseFormat.cpp (renamed from thirdparty/assimp/code/MakeVerboseFormat.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/MakeVerboseFormat.h (renamed from thirdparty/assimp/code/MakeVerboseFormat.h) | 3 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/OptimizeGraph.cpp (renamed from thirdparty/assimp/code/OptimizeGraph.cpp) | 2 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/OptimizeGraph.h (renamed from thirdparty/assimp/code/OptimizeGraph.h) | 27 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/OptimizeMeshes.cpp (renamed from thirdparty/assimp/code/OptimizeMeshes.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/OptimizeMeshes.h (renamed from thirdparty/assimp/code/OptimizeMeshes.h) | 10 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/PretransformVertices.cpp (renamed from thirdparty/assimp/code/PretransformVertices.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/PretransformVertices.h (renamed from thirdparty/assimp/code/PretransformVertices.h) | 15 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/ProcessHelper.cpp (renamed from thirdparty/assimp/code/ProcessHelper.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/ProcessHelper.h (renamed from thirdparty/assimp/code/ProcessHelper.h) | 2 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/RemoveRedundantMaterials.cpp (renamed from thirdparty/assimp/code/RemoveRedundantMaterials.cpp) | 10 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/RemoveRedundantMaterials.h (renamed from thirdparty/assimp/code/RemoveRedundantMaterials.h) | 18 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/RemoveVCProcess.cpp (renamed from thirdparty/assimp/code/RemoveVCProcess.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/RemoveVCProcess.h (renamed from thirdparty/assimp/code/RemoveVCProcess.h) | 3 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/ScaleProcess.cpp (renamed from thirdparty/assimp/code/ScaleProcess.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/ScaleProcess.h (renamed from thirdparty/assimp/code/ScaleProcess.h) | 2 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/SortByPTypeProcess.cpp (renamed from thirdparty/assimp/code/SortByPTypeProcess.cpp) | 18 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/SortByPTypeProcess.h (renamed from thirdparty/assimp/code/SortByPTypeProcess.h) | 11 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/SplitLargeMeshes.cpp (renamed from thirdparty/assimp/code/SplitLargeMeshes.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/SplitLargeMeshes.h (renamed from thirdparty/assimp/code/SplitLargeMeshes.h) | 9 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/TextureTransform.cpp (renamed from thirdparty/assimp/code/TextureTransform.cpp) | 0 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/TextureTransform.h (renamed from thirdparty/assimp/code/TextureTransform.h) | 2 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/TriangulateProcess.cpp (renamed from thirdparty/assimp/code/TriangulateProcess.cpp) | 10 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/TriangulateProcess.h (renamed from thirdparty/assimp/code/TriangulateProcess.h) | 8 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/ValidateDataStructure.cpp (renamed from thirdparty/assimp/code/ValidateDataStructure.cpp) | 33 | ||||
-rw-r--r-- | thirdparty/assimp/code/PostProcessing/ValidateDataStructure.h (renamed from thirdparty/assimp/code/ValidateDataStructure.h) | 3 | ||||
-rw-r--r-- | thirdparty/assimp/code/RawLoader.cpp | 331 | ||||
-rw-r--r-- | thirdparty/assimp/include/assimp/Exporter.hpp | 2 | ||||
-rw-r--r-- | thirdparty/assimp/include/assimp/ParsingUtils.h | 3 | ||||
-rw-r--r-- | thirdparty/assimp/include/assimp/aabb.h | 76 | ||||
-rw-r--r-- | thirdparty/assimp/include/assimp/camera.h | 5 | ||||
-rw-r--r-- | thirdparty/assimp/include/assimp/color4.inl | 4 | ||||
-rw-r--r-- | thirdparty/assimp/include/assimp/config.h.in | 35 | ||||
-rw-r--r-- | thirdparty/assimp/include/assimp/defs.h | 21 | ||||
-rw-r--r-- | thirdparty/assimp/include/assimp/mesh.h | 11 | ||||
-rw-r--r-- | thirdparty/assimp/include/assimp/postprocess.h | 13 | ||||
-rw-r--r-- | thirdparty/assimp/include/assimp/scene.h | 9 | ||||
-rw-r--r-- | thirdparty/jpeg-compressor/jpgd.cpp | 55 | ||||
-rw-r--r-- | thirdparty/xatlas/xatlas.cpp | 698 | ||||
-rw-r--r-- | thirdparty/zstd/common/compiler.h | 7 | ||||
-rw-r--r-- | thirdparty/zstd/common/zstd_internal.h | 64 | ||||
-rw-r--r-- | thirdparty/zstd/compress/zstd_compress.c | 292 | ||||
-rw-r--r-- | thirdparty/zstd/compress/zstd_compress_internal.h | 104 | ||||
-rw-r--r-- | thirdparty/zstd/compress/zstd_double_fast.c | 92 | ||||
-rw-r--r-- | thirdparty/zstd/compress/zstd_fast.c | 49 | ||||
-rw-r--r-- | thirdparty/zstd/compress/zstd_lazy.c | 15 | ||||
-rw-r--r-- | thirdparty/zstd/compress/zstd_ldm.c | 2 | ||||
-rw-r--r-- | thirdparty/zstd/compress/zstd_opt.c | 77 | ||||
-rw-r--r-- | thirdparty/zstd/compress/zstdmt_compress.c | 25 | ||||
-rw-r--r-- | thirdparty/zstd/compress/zstdmt_compress.h | 1 | ||||
-rw-r--r-- | thirdparty/zstd/decompress/zstd_decompress.c | 21 | ||||
-rw-r--r-- | thirdparty/zstd/decompress/zstd_decompress_block.c | 29 | ||||
-rw-r--r-- | thirdparty/zstd/zstd.h | 74 |
175 files changed, 3575 insertions, 3464 deletions
diff --git a/thirdparty/README.md b/thirdparty/README.md index cb29eadeca..99f917dbc1 100644 --- a/thirdparty/README.md +++ b/thirdparty/README.md @@ -4,7 +4,7 @@ ## assimp - Upstream: http://github.com/assimp/assimp -- Version: git (d3d98a7ec0c8d38e1952b46dfe53f7e9233dc92d) +- Version: git (1d565b0aab5a2ee00462f18c5b8a81f6a5454a48) - License: BSD-3-Clause @@ -200,6 +200,7 @@ Important: Some files have Godot-made changes. They are marked with `// -- GODOT start --` and `// -- GODOT end --` comments. + ## libtheora - Upstream: https://www.theora.org @@ -262,18 +263,6 @@ changes to ensure they build for Javascript/HTML5. Those changes are marked with `// -- GODOT --` comments. -## wslay - -- Upstream: https://github.com/tatsuhiro-t/wslay -- Version: 1.1.0 -- License: MIT - -File extracted from upstream release tarball: - -- All `*.c` and `*.h` in `lib/` and `lib/includes/` -- `wslay.h` has a small Godot addition to fix MSVC build. - See `thirdparty/wslay/msvcfix.diff` - ## mbedtls - Upstream: https://tls.mbed.org/ @@ -508,10 +497,23 @@ They can be reapplied using the patches included in the `vhacd` folder. +## wslay + +- Upstream: https://github.com/tatsuhiro-t/wslay +- Version: 1.1.0 +- License: MIT + +File extracted from upstream release tarball: + +- All `*.c` and `*.h` in `lib/` and `lib/includes/` +- `wslay.h` has a small Godot addition to fix MSVC build. + See `thirdparty/wslay/msvcfix.diff` + + ## xatlas - Upstream: https://github.com/jpcy/xatlas -- Version: git (b7d7bb, 2019) +- Version: git (f65a664, 2019) - License: MIT Files extracted from upstream source: @@ -536,7 +538,7 @@ Files extracted from upstream source: ## zstd - Upstream: https://github.com/facebook/zstd -- Version: 1.4.0 +- Version: 1.4.1 - License: BSD-3-Clause Files extracted from upstream source: diff --git a/thirdparty/assimp/assimp/config.h b/thirdparty/assimp/assimp/config.h index 8b0634d28b..382a698268 100644 --- a/thirdparty/assimp/assimp/config.h +++ b/thirdparty/assimp/assimp/config.h @@ -647,6 +647,21 @@ enum aiComponent { "AI_CONFIG_IMPORT_FBX_EMBEDDED_TEXTURES_LEGACY_NAMING" // --------------------------------------------------------------------------- +/** @brief Set wether the FBX importer shall not remove empty bones. + * + * + * Empty bone are often used to define connections for other models. + */ +#define AI_CONFIG_IMPORT_REMOVE_EMPTY_BONES \ + "AI_CONFIG_IMPORT_REMOVE_EMPTY_BONES" + +// --------------------------------------------------------------------------- +/** @brief Set wether the FBX importer shall convert the unit from cm to m. + */ +#define AI_CONFIG_FBX_CONVERT_TO_M \ + "AI_CONFIG_FBX_CONVERT_TO_M" + +// --------------------------------------------------------------------------- /** @brief Set the vertex animation keyframe to be imported * * ASSIMP does not support vertex keyframes (only bone animation is supported). @@ -978,3 +993,4 @@ enum aiComponent { /* #cmakedefine ASSIMP_DOUBLE_PRECISION 1 */ #endif // !! AI_CONFIG_H_INC + diff --git a/thirdparty/assimp/code/CApi/AssimpCExport.cpp b/thirdparty/assimp/code/CApi/AssimpCExport.cpp new file mode 100644 index 0000000000..7557edcfc6 --- /dev/null +++ b/thirdparty/assimp/code/CApi/AssimpCExport.cpp @@ -0,0 +1,156 @@ +/* +--------------------------------------------------------------------------- +Open Asset Import Library (assimp) +--------------------------------------------------------------------------- + +Copyright (c) 2006-2019, assimp team + + + +All rights reserved. + +Redistribution and use of this software in source and binary forms, +with or without modification, are permitted provided that the following +conditions are met: + +* Redistributions of source code must retain the above + copyright notice, this list of conditions and the + following disclaimer. + +* Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the + following disclaimer in the documentation and/or other + materials provided with the distribution. + +* Neither the name of the assimp team, nor the names of its + contributors may be used to endorse or promote products + derived from this software without specific prior + written permission of the assimp team. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +--------------------------------------------------------------------------- +*/ + +/** @file AssimpCExport.cpp +Assimp C export interface. See Exporter.cpp for some notes. +*/ + +#ifndef ASSIMP_BUILD_NO_EXPORT + +#include "CInterfaceIOWrapper.h" +#include <assimp/SceneCombiner.h> +#include "Common/ScenePrivate.h" +#include <assimp/Exporter.hpp> + +using namespace Assimp; + +// ------------------------------------------------------------------------------------------------ +ASSIMP_API size_t aiGetExportFormatCount(void) +{ + return Exporter().GetExportFormatCount(); +} + +// ------------------------------------------------------------------------------------------------ +ASSIMP_API const aiExportFormatDesc* aiGetExportFormatDescription( size_t index) +{ + // Note: this is valid as the index always pertains to a built-in exporter, + // for which the returned structure is guaranteed to be of static storage duration. + Exporter exporter; + const aiExportFormatDesc* orig( exporter.GetExportFormatDescription( index ) ); + if (NULL == orig) { + return NULL; + } + + aiExportFormatDesc *desc = new aiExportFormatDesc; + desc->description = new char[ strlen( orig->description ) + 1 ](); + ::strncpy( (char*) desc->description, orig->description, strlen( orig->description ) ); + desc->fileExtension = new char[ strlen( orig->fileExtension ) + 1 ](); + ::strncpy( ( char* ) desc->fileExtension, orig->fileExtension, strlen( orig->fileExtension ) ); + desc->id = new char[ strlen( orig->id ) + 1 ](); + ::strncpy( ( char* ) desc->id, orig->id, strlen( orig->id ) ); + + return desc; +} + +// ------------------------------------------------------------------------------------------------ +ASSIMP_API void aiReleaseExportFormatDescription( const aiExportFormatDesc *desc ) { + if (NULL == desc) { + return; + } + + delete [] desc->description; + delete [] desc->fileExtension; + delete [] desc->id; + delete desc; +} + +// ------------------------------------------------------------------------------------------------ +ASSIMP_API void aiCopyScene(const aiScene* pIn, aiScene** pOut) +{ + if (!pOut || !pIn) { + return; + } + + SceneCombiner::CopyScene(pOut,pIn,true); + ScenePriv(*pOut)->mIsCopy = true; +} + + +// ------------------------------------------------------------------------------------------------ +ASSIMP_API void aiFreeScene(const C_STRUCT aiScene* pIn) +{ + // note: aiReleaseImport() is also able to delete scene copies, but in addition + // it also handles scenes with import metadata. + delete pIn; +} + + +// ------------------------------------------------------------------------------------------------ +ASSIMP_API aiReturn aiExportScene( const aiScene* pScene, const char* pFormatId, const char* pFileName, unsigned int pPreprocessing ) +{ + return ::aiExportSceneEx(pScene,pFormatId,pFileName,NULL,pPreprocessing); +} + + +// ------------------------------------------------------------------------------------------------ +ASSIMP_API aiReturn aiExportSceneEx( const aiScene* pScene, const char* pFormatId, const char* pFileName, aiFileIO* pIO, unsigned int pPreprocessing ) +{ + Exporter exp; + + if (pIO) { + exp.SetIOHandler(new CIOSystemWrapper(pIO)); + } + return exp.Export(pScene,pFormatId,pFileName,pPreprocessing); +} + + +// ------------------------------------------------------------------------------------------------ +ASSIMP_API const C_STRUCT aiExportDataBlob* aiExportSceneToBlob( const aiScene* pScene, const char* pFormatId, unsigned int pPreprocessing ) +{ + Exporter exp; + if (!exp.ExportToBlob(pScene,pFormatId,pPreprocessing)) { + return NULL; + } + const aiExportDataBlob* blob = exp.GetOrphanedBlob(); + ai_assert(blob); + + return blob; +} + +// ------------------------------------------------------------------------------------------------ +ASSIMP_API C_STRUCT void aiReleaseExportBlob( const aiExportDataBlob* pData ) +{ + delete pData; +} + +#endif // !ASSIMP_BUILD_NO_EXPORT diff --git a/thirdparty/assimp/code/CInterfaceIOWrapper.cpp b/thirdparty/assimp/code/CApi/CInterfaceIOWrapper.cpp index 5a3a49565a..5a3a49565a 100644 --- a/thirdparty/assimp/code/CInterfaceIOWrapper.cpp +++ b/thirdparty/assimp/code/CApi/CInterfaceIOWrapper.cpp diff --git a/thirdparty/assimp/code/CInterfaceIOWrapper.h b/thirdparty/assimp/code/CApi/CInterfaceIOWrapper.h index 2162320302..2162320302 100644 --- a/thirdparty/assimp/code/CInterfaceIOWrapper.h +++ b/thirdparty/assimp/code/CApi/CInterfaceIOWrapper.h diff --git a/thirdparty/assimp/code/Common/Assimp.cpp b/thirdparty/assimp/code/Common/Assimp.cpp new file mode 100644 index 0000000000..178b2c01d0 --- /dev/null +++ b/thirdparty/assimp/code/Common/Assimp.cpp @@ -0,0 +1,695 @@ +/* +--------------------------------------------------------------------------- +Open Asset Import Library (assimp) +--------------------------------------------------------------------------- + +Copyright (c) 2006-2019, assimp team + + + +All rights reserved. + +Redistribution and use of this software in source and binary forms, +with or without modification, are permitted provided that the following +conditions are met: + +* Redistributions of source code must retain the above + copyright notice, this list of conditions and the + following disclaimer. + +* Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the + following disclaimer in the documentation and/or other + materials provided with the distribution. + +* Neither the name of the assimp team, nor the names of its + contributors may be used to endorse or promote products + derived from this software without specific prior + written permission of the assimp team. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +--------------------------------------------------------------------------- +*/ +/** @file Assimp.cpp + * @brief Implementation of the Plain-C API + */ + +#include <assimp/cimport.h> +#include <assimp/LogStream.hpp> +#include <assimp/DefaultLogger.hpp> +#include <assimp/Importer.hpp> +#include <assimp/importerdesc.h> +#include <assimp/scene.h> +#include <assimp/GenericProperty.h> +#include <assimp/Exceptional.h> +#include <assimp/BaseImporter.h> + +#include "CApi/CInterfaceIOWrapper.h" +#include "Importer.h" +#include "ScenePrivate.h" + +#include <list> + +// ------------------------------------------------------------------------------------------------ +#ifndef ASSIMP_BUILD_SINGLETHREADED +# include <thread> +# include <mutex> +#endif +// ------------------------------------------------------------------------------------------------ +using namespace Assimp; + +namespace Assimp { + // underlying structure for aiPropertyStore + typedef BatchLoader::PropertyMap PropertyMap; + + /** Stores the LogStream objects for all active C log streams */ + struct mpred { + bool operator () (const aiLogStream& s0, const aiLogStream& s1) const { + return s0.callback<s1.callback&&s0.user<s1.user; + } + }; + typedef std::map<aiLogStream, Assimp::LogStream*, mpred> LogStreamMap; + + /** Stores the LogStream objects allocated by #aiGetPredefinedLogStream */ + typedef std::list<Assimp::LogStream*> PredefLogStreamMap; + + /** Local storage of all active log streams */ + static LogStreamMap gActiveLogStreams; + + /** Local storage of LogStreams allocated by #aiGetPredefinedLogStream */ + static PredefLogStreamMap gPredefinedStreams; + + /** Error message of the last failed import process */ + static std::string gLastErrorString; + + /** Verbose logging active or not? */ + static aiBool gVerboseLogging = false; + + /** will return all registered importers. */ + void GetImporterInstanceList(std::vector< BaseImporter* >& out); + + /** will delete all registered importers. */ + void DeleteImporterInstanceList(std::vector< BaseImporter* >& out); +} // namespace assimp + + +#ifndef ASSIMP_BUILD_SINGLETHREADED +/** Global mutex to manage the access to the log-stream map */ +static std::mutex gLogStreamMutex; +#endif + +// ------------------------------------------------------------------------------------------------ +// Custom LogStream implementation for the C-API +class LogToCallbackRedirector : public LogStream { +public: + explicit LogToCallbackRedirector(const aiLogStream& s) + : stream (s) { + ai_assert(NULL != s.callback); + } + + ~LogToCallbackRedirector() { +#ifndef ASSIMP_BUILD_SINGLETHREADED + std::lock_guard<std::mutex> lock(gLogStreamMutex); +#endif + // (HACK) Check whether the 'stream.user' pointer points to a + // custom LogStream allocated by #aiGetPredefinedLogStream. + // In this case, we need to delete it, too. Of course, this + // might cause strange problems, but the chance is quite low. + + PredefLogStreamMap::iterator it = std::find(gPredefinedStreams.begin(), + gPredefinedStreams.end(), (Assimp::LogStream*)stream.user); + + if (it != gPredefinedStreams.end()) { + delete *it; + gPredefinedStreams.erase(it); + } + } + + /** @copydoc LogStream::write */ + void write(const char* message) { + stream.callback(message,stream.user); + } + +private: + aiLogStream stream; +}; + +// ------------------------------------------------------------------------------------------------ +void ReportSceneNotFoundError() { + ASSIMP_LOG_ERROR("Unable to find the Assimp::Importer for this aiScene. " + "The C-API does not accept scenes produced by the C++ API and vice versa"); + + ai_assert(false); +} + +// ------------------------------------------------------------------------------------------------ +// Reads the given file and returns its content. +const aiScene* aiImportFile( const char* pFile, unsigned int pFlags) { + return aiImportFileEx(pFile,pFlags,NULL); +} + +// ------------------------------------------------------------------------------------------------ +const aiScene* aiImportFileEx( const char* pFile, unsigned int pFlags, aiFileIO* pFS) { + return aiImportFileExWithProperties(pFile, pFlags, pFS, NULL); +} + +// ------------------------------------------------------------------------------------------------ +const aiScene* aiImportFileExWithProperties( const char* pFile, unsigned int pFlags, + aiFileIO* pFS, const aiPropertyStore* props) { + ai_assert(NULL != pFile); + + const aiScene* scene = NULL; + ASSIMP_BEGIN_EXCEPTION_REGION(); + + // create an Importer for this file + Assimp::Importer* imp = new Assimp::Importer(); + + // copy properties + if(props) { + const PropertyMap* pp = reinterpret_cast<const PropertyMap*>(props); + ImporterPimpl* pimpl = imp->Pimpl(); + pimpl->mIntProperties = pp->ints; + pimpl->mFloatProperties = pp->floats; + pimpl->mStringProperties = pp->strings; + pimpl->mMatrixProperties = pp->matrices; + } + // setup a custom IO system if necessary + if (pFS) { + imp->SetIOHandler( new CIOSystemWrapper (pFS) ); + } + + // and have it read the file + scene = imp->ReadFile( pFile, pFlags); + + // if succeeded, store the importer in the scene and keep it alive + if( scene) { + ScenePrivateData* priv = const_cast<ScenePrivateData*>( ScenePriv(scene) ); + priv->mOrigImporter = imp; + } else { + // if failed, extract error code and destroy the import + gLastErrorString = imp->GetErrorString(); + delete imp; + } + + // return imported data. If the import failed the pointer is NULL anyways + ASSIMP_END_EXCEPTION_REGION(const aiScene*); + + return scene; +} + +// ------------------------------------------------------------------------------------------------ +const aiScene* aiImportFileFromMemory( + const char* pBuffer, + unsigned int pLength, + unsigned int pFlags, + const char* pHint) +{ + return aiImportFileFromMemoryWithProperties(pBuffer, pLength, pFlags, pHint, NULL); +} + +// ------------------------------------------------------------------------------------------------ +const aiScene* aiImportFileFromMemoryWithProperties( + const char* pBuffer, + unsigned int pLength, + unsigned int pFlags, + const char* pHint, + const aiPropertyStore* props) +{ + ai_assert( NULL != pBuffer ); + ai_assert( 0 != pLength ); + + const aiScene* scene = NULL; + ASSIMP_BEGIN_EXCEPTION_REGION(); + + // create an Importer for this file + Assimp::Importer* imp = new Assimp::Importer(); + + // copy properties + if(props) { + const PropertyMap* pp = reinterpret_cast<const PropertyMap*>(props); + ImporterPimpl* pimpl = imp->Pimpl(); + pimpl->mIntProperties = pp->ints; + pimpl->mFloatProperties = pp->floats; + pimpl->mStringProperties = pp->strings; + pimpl->mMatrixProperties = pp->matrices; + } + + // and have it read the file from the memory buffer + scene = imp->ReadFileFromMemory( pBuffer, pLength, pFlags,pHint); + + // if succeeded, store the importer in the scene and keep it alive + if( scene) { + ScenePrivateData* priv = const_cast<ScenePrivateData*>( ScenePriv(scene) ); + priv->mOrigImporter = imp; + } + else { + // if failed, extract error code and destroy the import + gLastErrorString = imp->GetErrorString(); + delete imp; + } + // return imported data. If the import failed the pointer is NULL anyways + ASSIMP_END_EXCEPTION_REGION(const aiScene*); + return scene; +} + +// ------------------------------------------------------------------------------------------------ +// Releases all resources associated with the given import process. +void aiReleaseImport( const aiScene* pScene) +{ + if (!pScene) { + return; + } + + ASSIMP_BEGIN_EXCEPTION_REGION(); + + // find the importer associated with this data + const ScenePrivateData* priv = ScenePriv(pScene); + if( !priv || !priv->mOrigImporter) { + delete pScene; + } + else { + // deleting the Importer also deletes the scene + // Note: the reason that this is not written as 'delete priv->mOrigImporter' + // is a suspected bug in gcc 4.4+ (http://gcc.gnu.org/bugzilla/show_bug.cgi?id=52339) + Importer* importer = priv->mOrigImporter; + delete importer; + } + + ASSIMP_END_EXCEPTION_REGION(void); +} + +// ------------------------------------------------------------------------------------------------ +ASSIMP_API const aiScene* aiApplyPostProcessing(const aiScene* pScene, + unsigned int pFlags) +{ + const aiScene* sc = NULL; + + + ASSIMP_BEGIN_EXCEPTION_REGION(); + + // find the importer associated with this data + const ScenePrivateData* priv = ScenePriv(pScene); + if( !priv || !priv->mOrigImporter) { + ReportSceneNotFoundError(); + return NULL; + } + + sc = priv->mOrigImporter->ApplyPostProcessing(pFlags); + + if (!sc) { + aiReleaseImport(pScene); + return NULL; + } + + ASSIMP_END_EXCEPTION_REGION(const aiScene*); + return sc; +} + +// ------------------------------------------------------------------------------------------------ +ASSIMP_API const aiScene *aiApplyCustomizedPostProcessing( const aiScene *scene, + BaseProcess* process, + bool requestValidation ) { + const aiScene* sc( NULL ); + + ASSIMP_BEGIN_EXCEPTION_REGION(); + + // find the importer associated with this data + const ScenePrivateData* priv = ScenePriv( scene ); + if ( NULL == priv || NULL == priv->mOrigImporter ) { + ReportSceneNotFoundError(); + return NULL; + } + + sc = priv->mOrigImporter->ApplyCustomizedPostProcessing( process, requestValidation ); + + if ( !sc ) { + aiReleaseImport( scene ); + return NULL; + } + + ASSIMP_END_EXCEPTION_REGION( const aiScene* ); + + return sc; +} + +// ------------------------------------------------------------------------------------------------ +void CallbackToLogRedirector (const char* msg, char* dt) +{ + ai_assert( NULL != msg ); + ai_assert( NULL != dt ); + LogStream* s = (LogStream*)dt; + + s->write(msg); +} + +// ------------------------------------------------------------------------------------------------ +ASSIMP_API aiLogStream aiGetPredefinedLogStream(aiDefaultLogStream pStream,const char* file) +{ + aiLogStream sout; + + ASSIMP_BEGIN_EXCEPTION_REGION(); + LogStream* stream = LogStream::createDefaultStream(pStream,file); + if (!stream) { + sout.callback = NULL; + sout.user = NULL; + } + else { + sout.callback = &CallbackToLogRedirector; + sout.user = (char*)stream; + } + gPredefinedStreams.push_back(stream); + ASSIMP_END_EXCEPTION_REGION(aiLogStream); + return sout; +} + +// ------------------------------------------------------------------------------------------------ +ASSIMP_API void aiAttachLogStream( const aiLogStream* stream ) +{ + ASSIMP_BEGIN_EXCEPTION_REGION(); + +#ifndef ASSIMP_BUILD_SINGLETHREADED + std::lock_guard<std::mutex> lock(gLogStreamMutex); +#endif + + LogStream* lg = new LogToCallbackRedirector(*stream); + gActiveLogStreams[*stream] = lg; + + if (DefaultLogger::isNullLogger()) { + DefaultLogger::create(NULL,(gVerboseLogging == AI_TRUE ? Logger::VERBOSE : Logger::NORMAL)); + } + DefaultLogger::get()->attachStream(lg); + ASSIMP_END_EXCEPTION_REGION(void); +} + +// ------------------------------------------------------------------------------------------------ +ASSIMP_API aiReturn aiDetachLogStream( const aiLogStream* stream) +{ + ASSIMP_BEGIN_EXCEPTION_REGION(); + +#ifndef ASSIMP_BUILD_SINGLETHREADED + std::lock_guard<std::mutex> lock(gLogStreamMutex); +#endif + // find the log-stream associated with this data + LogStreamMap::iterator it = gActiveLogStreams.find( *stream); + // it should be there... else the user is playing fools with us + if( it == gActiveLogStreams.end()) { + return AI_FAILURE; + } + DefaultLogger::get()->detatchStream( it->second ); + delete it->second; + + gActiveLogStreams.erase( it); + + if (gActiveLogStreams.empty()) { + DefaultLogger::kill(); + } + ASSIMP_END_EXCEPTION_REGION(aiReturn); + return AI_SUCCESS; +} + +// ------------------------------------------------------------------------------------------------ +ASSIMP_API void aiDetachAllLogStreams(void) +{ + ASSIMP_BEGIN_EXCEPTION_REGION(); +#ifndef ASSIMP_BUILD_SINGLETHREADED + std::lock_guard<std::mutex> lock(gLogStreamMutex); +#endif + Logger *logger( DefaultLogger::get() ); + if ( NULL == logger ) { + return; + } + + for (LogStreamMap::iterator it = gActiveLogStreams.begin(); it != gActiveLogStreams.end(); ++it) { + logger->detatchStream( it->second ); + delete it->second; + } + gActiveLogStreams.clear(); + DefaultLogger::kill(); + + ASSIMP_END_EXCEPTION_REGION(void); +} + +// ------------------------------------------------------------------------------------------------ +ASSIMP_API void aiEnableVerboseLogging(aiBool d) +{ + if (!DefaultLogger::isNullLogger()) { + DefaultLogger::get()->setLogSeverity((d == AI_TRUE ? Logger::VERBOSE : Logger::NORMAL)); + } + gVerboseLogging = d; +} + +// ------------------------------------------------------------------------------------------------ +// Returns the error text of the last failed import process. +const char* aiGetErrorString() +{ + return gLastErrorString.c_str(); +} + +// ----------------------------------------------------------------------------------------------- +// Return the description of a importer given its index +const aiImporterDesc* aiGetImportFormatDescription( size_t pIndex) +{ + return Importer().GetImporterInfo(pIndex); +} + +// ----------------------------------------------------------------------------------------------- +// Return the number of importers +size_t aiGetImportFormatCount(void) +{ + return Importer().GetImporterCount(); +} + +// ------------------------------------------------------------------------------------------------ +// Returns the error text of the last failed import process. +aiBool aiIsExtensionSupported(const char* szExtension) +{ + ai_assert(NULL != szExtension); + aiBool candoit=AI_FALSE; + ASSIMP_BEGIN_EXCEPTION_REGION(); + + // FIXME: no need to create a temporary Importer instance just for that .. + Assimp::Importer tmp; + candoit = tmp.IsExtensionSupported(std::string(szExtension)) ? AI_TRUE : AI_FALSE; + + ASSIMP_END_EXCEPTION_REGION(aiBool); + return candoit; +} + +// ------------------------------------------------------------------------------------------------ +// Get a list of all file extensions supported by ASSIMP +void aiGetExtensionList(aiString* szOut) +{ + ai_assert(NULL != szOut); + ASSIMP_BEGIN_EXCEPTION_REGION(); + + // FIXME: no need to create a temporary Importer instance just for that .. + Assimp::Importer tmp; + tmp.GetExtensionList(*szOut); + + ASSIMP_END_EXCEPTION_REGION(void); +} + +// ------------------------------------------------------------------------------------------------ +// Get the memory requirements for a particular import. +void aiGetMemoryRequirements(const C_STRUCT aiScene* pIn, + C_STRUCT aiMemoryInfo* in) +{ + ASSIMP_BEGIN_EXCEPTION_REGION(); + + // find the importer associated with this data + const ScenePrivateData* priv = ScenePriv(pIn); + if( !priv || !priv->mOrigImporter) { + ReportSceneNotFoundError(); + return; + } + + return priv->mOrigImporter->GetMemoryRequirements(*in); + ASSIMP_END_EXCEPTION_REGION(void); +} + +// ------------------------------------------------------------------------------------------------ +ASSIMP_API aiPropertyStore* aiCreatePropertyStore(void) +{ + return reinterpret_cast<aiPropertyStore*>( new PropertyMap() ); +} + +// ------------------------------------------------------------------------------------------------ +ASSIMP_API void aiReleasePropertyStore(aiPropertyStore* p) +{ + delete reinterpret_cast<PropertyMap*>(p); +} + +// ------------------------------------------------------------------------------------------------ +// Importer::SetPropertyInteger +ASSIMP_API void aiSetImportPropertyInteger(aiPropertyStore* p, const char* szName, int value) +{ + ASSIMP_BEGIN_EXCEPTION_REGION(); + PropertyMap* pp = reinterpret_cast<PropertyMap*>(p); + SetGenericProperty<int>(pp->ints,szName,value); + ASSIMP_END_EXCEPTION_REGION(void); +} + +// ------------------------------------------------------------------------------------------------ +// Importer::SetPropertyFloat +ASSIMP_API void aiSetImportPropertyFloat(aiPropertyStore* p, const char* szName, ai_real value) +{ + ASSIMP_BEGIN_EXCEPTION_REGION(); + PropertyMap* pp = reinterpret_cast<PropertyMap*>(p); + SetGenericProperty<ai_real>(pp->floats,szName,value); + ASSIMP_END_EXCEPTION_REGION(void); +} + +// ------------------------------------------------------------------------------------------------ +// Importer::SetPropertyString +ASSIMP_API void aiSetImportPropertyString(aiPropertyStore* p, const char* szName, + const C_STRUCT aiString* st) +{ + if (!st) { + return; + } + ASSIMP_BEGIN_EXCEPTION_REGION(); + PropertyMap* pp = reinterpret_cast<PropertyMap*>(p); + SetGenericProperty<std::string>(pp->strings,szName,std::string(st->C_Str())); + ASSIMP_END_EXCEPTION_REGION(void); +} + +// ------------------------------------------------------------------------------------------------ +// Importer::SetPropertyMatrix +ASSIMP_API void aiSetImportPropertyMatrix(aiPropertyStore* p, const char* szName, + const C_STRUCT aiMatrix4x4* mat) +{ + if (!mat) { + return; + } + ASSIMP_BEGIN_EXCEPTION_REGION(); + PropertyMap* pp = reinterpret_cast<PropertyMap*>(p); + SetGenericProperty<aiMatrix4x4>(pp->matrices,szName,*mat); + ASSIMP_END_EXCEPTION_REGION(void); +} + +// ------------------------------------------------------------------------------------------------ +// Rotation matrix to quaternion +ASSIMP_API void aiCreateQuaternionFromMatrix(aiQuaternion* quat,const aiMatrix3x3* mat) +{ + ai_assert( NULL != quat ); + ai_assert( NULL != mat ); + *quat = aiQuaternion(*mat); +} + +// ------------------------------------------------------------------------------------------------ +// Matrix decomposition +ASSIMP_API void aiDecomposeMatrix(const aiMatrix4x4* mat,aiVector3D* scaling, + aiQuaternion* rotation, + aiVector3D* position) +{ + ai_assert( NULL != rotation ); + ai_assert( NULL != position ); + ai_assert( NULL != scaling ); + ai_assert( NULL != mat ); + mat->Decompose(*scaling,*rotation,*position); +} + +// ------------------------------------------------------------------------------------------------ +// Matrix transpose +ASSIMP_API void aiTransposeMatrix3(aiMatrix3x3* mat) +{ + ai_assert(NULL != mat); + mat->Transpose(); +} + +// ------------------------------------------------------------------------------------------------ +ASSIMP_API void aiTransposeMatrix4(aiMatrix4x4* mat) +{ + ai_assert(NULL != mat); + mat->Transpose(); +} + +// ------------------------------------------------------------------------------------------------ +// Vector transformation +ASSIMP_API void aiTransformVecByMatrix3(aiVector3D* vec, + const aiMatrix3x3* mat) +{ + ai_assert( NULL != mat ); + ai_assert( NULL != vec); + *vec *= (*mat); +} + +// ------------------------------------------------------------------------------------------------ +ASSIMP_API void aiTransformVecByMatrix4(aiVector3D* vec, + const aiMatrix4x4* mat) +{ + ai_assert( NULL != mat ); + ai_assert( NULL != vec ); + + *vec *= (*mat); +} + +// ------------------------------------------------------------------------------------------------ +// Matrix multiplication +ASSIMP_API void aiMultiplyMatrix4( + aiMatrix4x4* dst, + const aiMatrix4x4* src) +{ + ai_assert( NULL != dst ); + ai_assert( NULL != src ); + *dst = (*dst) * (*src); +} + +// ------------------------------------------------------------------------------------------------ +ASSIMP_API void aiMultiplyMatrix3( + aiMatrix3x3* dst, + const aiMatrix3x3* src) +{ + ai_assert( NULL != dst ); + ai_assert( NULL != src ); + *dst = (*dst) * (*src); +} + +// ------------------------------------------------------------------------------------------------ +// Matrix identity +ASSIMP_API void aiIdentityMatrix3( + aiMatrix3x3* mat) +{ + ai_assert(NULL != mat); + *mat = aiMatrix3x3(); +} + +// ------------------------------------------------------------------------------------------------ +ASSIMP_API void aiIdentityMatrix4( + aiMatrix4x4* mat) +{ + ai_assert(NULL != mat); + *mat = aiMatrix4x4(); +} + +// ------------------------------------------------------------------------------------------------ +ASSIMP_API C_STRUCT const aiImporterDesc* aiGetImporterDesc( const char *extension ) { + if( NULL == extension ) { + return NULL; + } + const aiImporterDesc *desc( NULL ); + std::vector< BaseImporter* > out; + GetImporterInstanceList( out ); + for( size_t i = 0; i < out.size(); ++i ) { + if( 0 == strncmp( out[ i ]->GetInfo()->mFileExtensions, extension, strlen( extension ) ) ) { + desc = out[ i ]->GetInfo(); + break; + } + } + + DeleteImporterInstanceList(out); + + return desc; +} + +// ------------------------------------------------------------------------------------------------ diff --git a/thirdparty/assimp/code/BaseImporter.cpp b/thirdparty/assimp/code/Common/BaseImporter.cpp index 4803c6d6f2..0a5694aa0e 100644 --- a/thirdparty/assimp/code/BaseImporter.cpp +++ b/thirdparty/assimp/code/Common/BaseImporter.cpp @@ -320,7 +320,11 @@ std::string BaseImporter::GetExtension( const std::string& file ) { return false; } -#include "../contrib/utf8cpp/source/utf8.h" +#ifdef ASSIMP_USE_HUNTER +# include <utf8/utf8.h> +#else +# include "../contrib/utf8cpp/source/utf8.h" +#endif // ------------------------------------------------------------------------------------------------ // Convert to UTF8 data diff --git a/thirdparty/assimp/code/BaseProcess.cpp b/thirdparty/assimp/code/Common/BaseProcess.cpp index 18872c3693..e247be418d 100644 --- a/thirdparty/assimp/code/BaseProcess.cpp +++ b/thirdparty/assimp/code/Common/BaseProcess.cpp @@ -89,7 +89,7 @@ void BaseProcess::ExecuteOnScene( Importer* pImp) // and kill the partially imported data delete pImp->Pimpl()->mScene; - pImp->Pimpl()->mScene = NULL; + pImp->Pimpl()->mScene = nullptr; } } diff --git a/thirdparty/assimp/code/BaseProcess.h b/thirdparty/assimp/code/Common/BaseProcess.h index 4d5c7a76be..4d5c7a76be 100644 --- a/thirdparty/assimp/code/BaseProcess.h +++ b/thirdparty/assimp/code/Common/BaseProcess.h diff --git a/thirdparty/assimp/code/Bitmap.cpp b/thirdparty/assimp/code/Common/Bitmap.cpp index b22b71ea9e..b22b71ea9e 100644 --- a/thirdparty/assimp/code/Bitmap.cpp +++ b/thirdparty/assimp/code/Common/Bitmap.cpp diff --git a/thirdparty/assimp/code/CreateAnimMesh.cpp b/thirdparty/assimp/code/Common/CreateAnimMesh.cpp index 1a052849bb..98b60e5319 100644 --- a/thirdparty/assimp/code/CreateAnimMesh.cpp +++ b/thirdparty/assimp/code/Common/CreateAnimMesh.cpp @@ -47,10 +47,6 @@ namespace Assimp { aiAnimMesh *aiCreateAnimMesh(const aiMesh *mesh) { aiAnimMesh *animesh = new aiAnimMesh; - animesh->mVertices = NULL; - animesh->mNormals = NULL; - animesh->mTangents = NULL; - animesh->mBitangents = NULL; animesh->mNumVertices = mesh->mNumVertices; if (mesh->mVertices) { animesh->mVertices = new aiVector3D[animesh->mNumVertices]; diff --git a/thirdparty/assimp/code/DefaultIOStream.cpp b/thirdparty/assimp/code/Common/DefaultIOStream.cpp index 1c100b6189..1c100b6189 100644 --- a/thirdparty/assimp/code/DefaultIOStream.cpp +++ b/thirdparty/assimp/code/Common/DefaultIOStream.cpp diff --git a/thirdparty/assimp/code/DefaultIOSystem.cpp b/thirdparty/assimp/code/Common/DefaultIOSystem.cpp index d40b67de32..d40b67de32 100644 --- a/thirdparty/assimp/code/DefaultIOSystem.cpp +++ b/thirdparty/assimp/code/Common/DefaultIOSystem.cpp diff --git a/thirdparty/assimp/code/DefaultLogger.cpp b/thirdparty/assimp/code/Common/DefaultLogger.cpp index de3528d2b4..de3528d2b4 100644 --- a/thirdparty/assimp/code/DefaultLogger.cpp +++ b/thirdparty/assimp/code/Common/DefaultLogger.cpp diff --git a/thirdparty/assimp/code/DefaultProgressHandler.h b/thirdparty/assimp/code/Common/DefaultProgressHandler.h index bd2cce00be..bd2cce00be 100644 --- a/thirdparty/assimp/code/DefaultProgressHandler.h +++ b/thirdparty/assimp/code/Common/DefaultProgressHandler.h diff --git a/thirdparty/assimp/code/Exporter.cpp b/thirdparty/assimp/code/Common/Exporter.cpp index 8848e87f5b..090b561ae0 100644 --- a/thirdparty/assimp/code/Exporter.cpp +++ b/thirdparty/assimp/code/Common/Exporter.cpp @@ -61,15 +61,16 @@ Here we implement only the C++ interface (Assimp::Exporter). #include <assimp/mesh.h> #include <assimp/postprocess.h> #include <assimp/scene.h> - -#include "DefaultProgressHandler.h" -#include "BaseProcess.h" -#include "JoinVerticesProcess.h" -#include "MakeVerboseFormat.h" -#include "ConvertToLHProcess.h" -#include "PretransformVertices.h" #include <assimp/Exceptional.h> -#include "ScenePrivate.h" + +#include "Common/DefaultProgressHandler.h" +#include "Common/BaseProcess.h" +#include "Common/ScenePrivate.h" +#include "PostProcessing/CalcTangentsProcess.h" +#include "PostProcessing/MakeVerboseFormat.h" +#include "PostProcessing/JoinVerticesProcess.h" +#include "PostProcessing/ConvertToLHProcess.h" +#include "PostProcessing/PretransformVertices.h" #include <memory> @@ -101,6 +102,7 @@ void ExportSceneX3D(const char*, IOSystem*, const aiScene*, const ExportProperti void ExportSceneFBX(const char*, IOSystem*, const aiScene*, const ExportProperties*); void ExportSceneFBXA(const char*, IOSystem*, const aiScene*, const ExportProperties*); void ExportScene3MF( const char*, IOSystem*, const aiScene*, const ExportProperties* ); +void ExportAssimp2Json(const char* , IOSystem*, const aiScene* , const Assimp::ExportProperties*); // ------------------------------------------------------------------------------------------------ // global array of all export formats which Assimp supports in its current build @@ -161,11 +163,11 @@ Exporter::ExportFormatEntry gExporters[] = #endif #ifndef ASSIMP_BUILD_NO_ASSBIN_EXPORTER - Exporter::ExportFormatEntry( "assbin", "Assimp Binary", "assbin" , &ExportSceneAssbin, 0 ), + Exporter::ExportFormatEntry( "assbin", "Assimp Binary File", "assbin" , &ExportSceneAssbin, 0 ), #endif #ifndef ASSIMP_BUILD_NO_ASSXML_EXPORTER - Exporter::ExportFormatEntry( "assxml", "Assxml Document", "assxml" , &ExportSceneAssxml, 0 ), + Exporter::ExportFormatEntry( "assxml", "Assimp XML Document", "assxml" , &ExportSceneAssxml, 0 ), #endif #ifndef ASSIMP_BUILD_NO_X3D_EXPORTER @@ -178,7 +180,11 @@ Exporter::ExportFormatEntry gExporters[] = #endif #ifndef ASSIMP_BUILD_NO_3MF_EXPORTER - Exporter::ExportFormatEntry( "3mf", "The 3MF-File-Format", "3mf", &ExportScene3MF, 0 ) + Exporter::ExportFormatEntry( "3mf", "The 3MF-File-Format", "3mf", &ExportScene3MF, 0 ), +#endif + +#ifndef ASSIMP_BUILD_NO_ASSJSON_EXPORTER + Exporter::ExportFormatEntry( "assjson", "Assimp JSON Document", "json", &ExportAssimp2Json, 0) #endif }; @@ -288,7 +294,7 @@ void Exporter::SetProgressHandler(ProgressHandler* pHandler) { // ------------------------------------------------------------------------------------------------ const aiExportDataBlob* Exporter::ExportToBlob( const aiScene* pScene, const char* pFormatId, - unsigned int, const ExportProperties* /*pProperties*/ ) { + unsigned int pPreprocessing, const ExportProperties* pProperties) { if (pimpl->blob) { delete pimpl->blob; pimpl->blob = nullptr; @@ -298,7 +304,7 @@ const aiExportDataBlob* Exporter::ExportToBlob( const aiScene* pScene, const cha BlobIOSystem* blobio = new BlobIOSystem(); pimpl->mIOSystem = std::shared_ptr<IOSystem>( blobio ); - if (AI_SUCCESS != Export(pScene,pFormatId,blobio->GetMagicFileName())) { + if (AI_SUCCESS != Export(pScene,pFormatId,blobio->GetMagicFileName(), pPreprocessing, pProperties)) { pimpl->mIOSystem = old; return nullptr; } diff --git a/thirdparty/assimp/code/FileLogStream.h b/thirdparty/assimp/code/Common/FileLogStream.h index 740c503192..740c503192 100644 --- a/thirdparty/assimp/code/FileLogStream.h +++ b/thirdparty/assimp/code/Common/FileLogStream.h diff --git a/thirdparty/assimp/code/FileSystemFilter.h b/thirdparty/assimp/code/Common/FileSystemFilter.h index 9923cdbdd3..9923cdbdd3 100644 --- a/thirdparty/assimp/code/FileSystemFilter.h +++ b/thirdparty/assimp/code/Common/FileSystemFilter.h diff --git a/thirdparty/assimp/code/Common/IFF.h b/thirdparty/assimp/code/Common/IFF.h new file mode 100644 index 0000000000..91d7d48289 --- /dev/null +++ b/thirdparty/assimp/code/Common/IFF.h @@ -0,0 +1,102 @@ +// Definitions for the Interchange File Format (IFF) +// Alexander Gessler, 2006 +// Adapted to Assimp August 2008 + +#ifndef AI_IFF_H_INCLUDED +#define AI_IFF_H_INCLUDED + +#include <assimp/ByteSwapper.h> + +namespace Assimp { +namespace IFF { + +///////////////////////////////////////////////////////////////////////////////// +//! Describes an IFF chunk header +///////////////////////////////////////////////////////////////////////////////// +struct ChunkHeader +{ + //! Type of the chunk header - FourCC + uint32_t type; + + //! Length of the chunk data, in bytes + uint32_t length; +}; + + +///////////////////////////////////////////////////////////////////////////////// +//! Describes an IFF sub chunk header +///////////////////////////////////////////////////////////////////////////////// +struct SubChunkHeader +{ + //! Type of the chunk header - FourCC + uint32_t type; + + //! Length of the chunk data, in bytes + uint16_t length; +}; + + +#define AI_IFF_FOURCC(a,b,c,d) ((uint32_t) (((uint8_t)a << 24u) | \ + ((uint8_t)b << 16u) | ((uint8_t)c << 8u) | ((uint8_t)d))) + + +#define AI_IFF_FOURCC_FORM AI_IFF_FOURCC('F','O','R','M') + + +///////////////////////////////////////////////////////////////////////////////// +//! Load a chunk header +//! @param outFile Pointer to the file data - points to the chunk data afterwards +//! @return Copy of the chunk header +///////////////////////////////////////////////////////////////////////////////// +inline ChunkHeader LoadChunk(uint8_t*& outFile) +{ + ChunkHeader head; + ::memcpy(&head.type, outFile, 4); + outFile += 4; + ::memcpy(&head.length, outFile, 4); + outFile += 4; + AI_LSWAP4(head.length); + AI_LSWAP4(head.type); + return head; +} + +///////////////////////////////////////////////////////////////////////////////// +//! Load a sub chunk header +//! @param outFile Pointer to the file data - points to the chunk data afterwards +//! @return Copy of the sub chunk header +///////////////////////////////////////////////////////////////////////////////// +inline SubChunkHeader LoadSubChunk(uint8_t*& outFile) +{ + SubChunkHeader head; + ::memcpy(&head.type, outFile, 4); + outFile += 4; + ::memcpy(&head.length, outFile, 2); + outFile += 2; + AI_LSWAP2(head.length); + AI_LSWAP4(head.type); + return head; +} + +///////////////////////////////////////////////////////////////////////////////// +//! Read the file header and return the type of the file and its size +//! @param outFile Pointer to the file data. The buffer must at +//! least be 12 bytes large. +//! @param fileType Receives the type of the file +//! @return 0 if everything was OK, otherwise an error message +///////////////////////////////////////////////////////////////////////////////// +inline const char* ReadHeader(uint8_t* outFile, uint32_t& fileType) +{ + ChunkHeader head = LoadChunk(outFile); + if(AI_IFF_FOURCC_FORM != head.type) + { + return "The file is not an IFF file: FORM chunk is missing"; + } + ::memcpy(&fileType, outFile, 4); + AI_LSWAP4(fileType); + return 0; +} + + +}} + +#endif // !! AI_IFF_H_INCLUDED diff --git a/thirdparty/assimp/code/Importer.cpp b/thirdparty/assimp/code/Common/Importer.cpp index 65b16471cc..91b50859a0 100644 --- a/thirdparty/assimp/code/Importer.cpp +++ b/thirdparty/assimp/code/Common/Importer.cpp @@ -64,15 +64,15 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // ------------------------------------------------------------------------------------------------ // Internal headers // ------------------------------------------------------------------------------------------------ -#include "Importer.h" -#include <assimp/BaseImporter.h> -#include "BaseProcess.h" +#include "Common/Importer.h" +#include "Common/BaseProcess.h" +#include "Common/DefaultProgressHandler.h" +#include "PostProcessing/ProcessHelper.h" +#include "Common/ScenePreprocessor.h" +#include "Common/ScenePrivate.h" -#include "DefaultProgressHandler.h" +#include <assimp/BaseImporter.h> #include <assimp/GenericProperty.h> -#include "ProcessHelper.h" -#include "ScenePreprocessor.h" -#include "ScenePrivate.h" #include <assimp/MemoryIOWrapper.h> #include <assimp/Profiler.h> #include <assimp/TinyFormatter.h> @@ -86,7 +86,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <assimp/DefaultIOSystem.h> #ifndef ASSIMP_BUILD_NO_VALIDATEDS_PROCESS -# include "ValidateDataStructure.h" +# include "PostProcessing/ValidateDataStructure.h" #endif using namespace Assimp::Profiling; @@ -590,10 +590,12 @@ const aiScene* Importer::ReadFile( const char* _pFile, unsigned int pFlags) // Find an worker class which can handle the file BaseImporter* imp = NULL; + SetPropertyInteger("importerIndex", -1); for( unsigned int a = 0; a < pimpl->mImporter.size(); a++) { if( pimpl->mImporter[a]->CanRead( pFile, pimpl->mIOHandler, false)) { imp = pimpl->mImporter[a]; + SetPropertyInteger("importerIndex", a); break; } } @@ -606,6 +608,7 @@ const aiScene* Importer::ReadFile( const char* _pFile, unsigned int pFlags) for( unsigned int a = 0; a < pimpl->mImporter.size(); a++) { if( pimpl->mImporter[a]->CanRead( pFile, pimpl->mIOHandler, true)) { imp = pimpl->mImporter[a]; + SetPropertyInteger("importerIndex", a); break; } } diff --git a/thirdparty/assimp/code/Importer.h b/thirdparty/assimp/code/Common/Importer.h index a439d99c2f..a439d99c2f 100644 --- a/thirdparty/assimp/code/Importer.h +++ b/thirdparty/assimp/code/Common/Importer.h diff --git a/thirdparty/assimp/code/ImporterRegistry.cpp b/thirdparty/assimp/code/Common/ImporterRegistry.cpp index 747815fa6f..32ac3b4168 100644 --- a/thirdparty/assimp/code/ImporterRegistry.cpp +++ b/thirdparty/assimp/code/Common/ImporterRegistry.cpp @@ -56,146 +56,146 @@ corresponding preprocessor flag to selectively disable formats. // (include_new_importers_here) // ------------------------------------------------------------------------------------------------ #ifndef ASSIMP_BUILD_NO_X_IMPORTER -# include "XFileImporter.h" +# include "X/XFileImporter.h" #endif #ifndef ASSIMP_BUILD_NO_AMF_IMPORTER -# include "AMFImporter.hpp" +# include "AMF/AMFImporter.hpp" #endif #ifndef ASSIMP_BUILD_NO_3DS_IMPORTER -# include "3DSLoader.h" +# include "3DS/3DSLoader.h" #endif #ifndef ASSIMP_BUILD_NO_MD3_IMPORTER -# include "MD3Loader.h" +# include "MD3/MD3Loader.h" #endif #ifndef ASSIMP_BUILD_NO_MDL_IMPORTER -# include "MDLLoader.h" +# include "MDL/MDLLoader.h" #endif #ifndef ASSIMP_BUILD_NO_MD2_IMPORTER -# include "MD2Loader.h" +# include "MD2/MD2Loader.h" #endif #ifndef ASSIMP_BUILD_NO_PLY_IMPORTER -# include "PlyLoader.h" +# include "Ply/PlyLoader.h" #endif #ifndef ASSIMP_BUILD_NO_ASE_IMPORTER -# include "ASELoader.h" +# include "ASE/ASELoader.h" #endif #ifndef ASSIMP_BUILD_NO_OBJ_IMPORTER -# include "ObjFileImporter.h" +# include "Obj/ObjFileImporter.h" #endif #ifndef ASSIMP_BUILD_NO_HMP_IMPORTER -# include "HMPLoader.h" +# include "HMP/HMPLoader.h" #endif #ifndef ASSIMP_BUILD_NO_SMD_IMPORTER -# include "SMDLoader.h" +# include "SMD/SMDLoader.h" #endif #ifndef ASSIMP_BUILD_NO_MDC_IMPORTER -# include "MDCLoader.h" +# include "MDC/MDCLoader.h" #endif #ifndef ASSIMP_BUILD_NO_MD5_IMPORTER -# include "MD5Loader.h" +# include "MD5/MD5Loader.h" #endif #ifndef ASSIMP_BUILD_NO_STL_IMPORTER -# include "STLLoader.h" +# include "STL/STLLoader.h" #endif #ifndef ASSIMP_BUILD_NO_LWO_IMPORTER -# include "LWOLoader.h" +# include "LWO/LWOLoader.h" #endif #ifndef ASSIMP_BUILD_NO_DXF_IMPORTER -# include "DXFLoader.h" +# include "DXF/DXFLoader.h" #endif #ifndef ASSIMP_BUILD_NO_NFF_IMPORTER -# include "NFFLoader.h" +# include "NFF/NFFLoader.h" #endif #ifndef ASSIMP_BUILD_NO_RAW_IMPORTER -# include "RawLoader.h" +# include "Raw/RawLoader.h" #endif #ifndef ASSIMP_BUILD_NO_SIB_IMPORTER -# include "SIBImporter.h" +# include "SIB/SIBImporter.h" #endif #ifndef ASSIMP_BUILD_NO_OFF_IMPORTER -# include "OFFLoader.h" +# include "OFF/OFFLoader.h" #endif #ifndef ASSIMP_BUILD_NO_AC_IMPORTER -# include "ACLoader.h" +# include "AC/ACLoader.h" #endif #ifndef ASSIMP_BUILD_NO_BVH_IMPORTER -# include "BVHLoader.h" +# include "BVH/BVHLoader.h" #endif #ifndef ASSIMP_BUILD_NO_IRRMESH_IMPORTER -# include "IRRMeshLoader.h" +# include "Irr/IRRMeshLoader.h" #endif #ifndef ASSIMP_BUILD_NO_IRR_IMPORTER -# include "IRRLoader.h" +# include "Irr/IRRLoader.h" #endif #ifndef ASSIMP_BUILD_NO_Q3D_IMPORTER -# include "Q3DLoader.h" +# include "Q3D/Q3DLoader.h" #endif #ifndef ASSIMP_BUILD_NO_B3D_IMPORTER -# include "B3DImporter.h" +# include "B3D/B3DImporter.h" #endif #ifndef ASSIMP_BUILD_NO_COLLADA_IMPORTER -# include "ColladaLoader.h" +# include "Collada/ColladaLoader.h" #endif #ifndef ASSIMP_BUILD_NO_TERRAGEN_IMPORTER -# include "TerragenLoader.h" +# include "Terragen/TerragenLoader.h" #endif #ifndef ASSIMP_BUILD_NO_CSM_IMPORTER -# include "CSMLoader.h" +# include "CSM/CSMLoader.h" #endif #ifndef ASSIMP_BUILD_NO_3D_IMPORTER -# include "UnrealLoader.h" +# include "Unreal/UnrealLoader.h" #endif #ifndef ASSIMP_BUILD_NO_LWS_IMPORTER -# include "LWSLoader.h" +# include "LWS/LWSLoader.h" #endif #ifndef ASSIMP_BUILD_NO_OGRE_IMPORTER -# include "OgreImporter.h" +# include "Ogre/OgreImporter.h" #endif #ifndef ASSIMP_BUILD_NO_OPENGEX_IMPORTER -# include "OpenGEXImporter.h" +# include "OpenGEX/OpenGEXImporter.h" #endif #ifndef ASSIMP_BUILD_NO_MS3D_IMPORTER -# include "MS3DLoader.h" +# include "MS3D/MS3DLoader.h" #endif #ifndef ASSIMP_BUILD_NO_COB_IMPORTER -# include "COBLoader.h" +# include "COB/COBLoader.h" #endif #ifndef ASSIMP_BUILD_NO_BLEND_IMPORTER -# include "BlenderLoader.h" +# include "Blender/BlenderLoader.h" #endif #ifndef ASSIMP_BUILD_NO_Q3BSP_IMPORTER -# include "Q3BSPFileImporter.h" +# include "Q3BSP/Q3BSPFileImporter.h" #endif #ifndef ASSIMP_BUILD_NO_NDO_IMPORTER -# include "NDOLoader.h" +# include "NDO/NDOLoader.h" #endif #ifndef ASSIMP_BUILD_NO_IFC_IMPORTER # include "Importer/IFC/IFCLoader.h" #endif #ifndef ASSIMP_BUILD_NO_XGL_IMPORTER -# include "XGLLoader.h" +# include "XGL/XGLLoader.h" #endif #ifndef ASSIMP_BUILD_NO_FBX_IMPORTER -# include "FBXImporter.h" +# include "FBX/FBXImporter.h" #endif #ifndef ASSIMP_BUILD_NO_ASSBIN_IMPORTER -# include "AssbinLoader.h" +# include "Assbin/AssbinLoader.h" #endif #ifndef ASSIMP_BUILD_NO_GLTF_IMPORTER -# include "glTFImporter.h" -# include "glTF2Importer.h" +# include "glTF/glTFImporter.h" +# include "glTF2/glTF2Importer.h" #endif #ifndef ASSIMP_BUILD_NO_C4D_IMPORTER -# include "C4DImporter.h" +# include "C4D/C4DImporter.h" #endif #ifndef ASSIMP_BUILD_NO_3MF_IMPORTER -# include "D3MFImporter.h" +# include "3MF/D3MFImporter.h" #endif #ifndef ASSIMP_BUILD_NO_X3D_IMPORTER -# include "X3DImporter.hpp" +# include "X3D/X3DImporter.hpp" #endif #ifndef ASSIMP_BUILD_NO_MMD_IMPORTER -# include "MMDImporter.h" +# include "MMD/MMDImporter.h" #endif #ifndef ASSIMP_BUILD_NO_STEP_IMPORTER # include "Importer/StepFile/StepFileImporter.h" @@ -364,7 +364,7 @@ void GetImporterInstanceList(std::vector< BaseImporter* >& out) void DeleteImporterInstanceList(std::vector< BaseImporter* >& deleteList){ for(size_t i= 0; i<deleteList.size();++i){ delete deleteList[i]; - deleteList[i]=NULL; + deleteList[i]=nullptr; }//for } diff --git a/thirdparty/assimp/code/PolyTools.h b/thirdparty/assimp/code/Common/PolyTools.h index fbbda0e7d1..fbbda0e7d1 100644 --- a/thirdparty/assimp/code/PolyTools.h +++ b/thirdparty/assimp/code/Common/PolyTools.h diff --git a/thirdparty/assimp/code/PostStepRegistry.cpp b/thirdparty/assimp/code/Common/PostStepRegistry.cpp index 15b4a28843..ef58f8ddfd 100644 --- a/thirdparty/assimp/code/PostStepRegistry.cpp +++ b/thirdparty/assimp/code/Common/PostStepRegistry.cpp @@ -48,89 +48,93 @@ directly (unless you are adding new steps), instead use the corresponding preprocessor flag to selectively disable steps. */ -#include "ProcessHelper.h" +#include "PostProcessing/ProcessHelper.h" #ifndef ASSIMP_BUILD_NO_CALCTANGENTS_PROCESS -# include "CalcTangentsProcess.h" +# include "PostProcessing/CalcTangentsProcess.h" #endif #ifndef ASSIMP_BUILD_NO_JOINVERTICES_PROCESS -# include "JoinVerticesProcess.h" +# include "PostProcessing/JoinVerticesProcess.h" #endif #if !(defined ASSIMP_BUILD_NO_MAKELEFTHANDED_PROCESS && defined ASSIMP_BUILD_NO_FLIPUVS_PROCESS && defined ASSIMP_BUILD_NO_FLIPWINDINGORDER_PROCESS) -# include "ConvertToLHProcess.h" +# include "PostProcessing/ConvertToLHProcess.h" #endif #ifndef ASSIMP_BUILD_NO_TRIANGULATE_PROCESS -# include "TriangulateProcess.h" +# include "PostProcessing/TriangulateProcess.h" #endif #ifndef ASSIMP_BUILD_NO_DROPFACENORMALS_PROCESS -# include "DropFaceNormalsProcess.h" +# include "PostProcessing/DropFaceNormalsProcess.h" #endif #ifndef ASSIMP_BUILD_NO_GENFACENORMALS_PROCESS -# include "GenFaceNormalsProcess.h" +# include "PostProcessing/GenFaceNormalsProcess.h" #endif #ifndef ASSIMP_BUILD_NO_GENVERTEXNORMALS_PROCESS -# include "GenVertexNormalsProcess.h" +# include "PostProcessing/GenVertexNormalsProcess.h" #endif #ifndef ASSIMP_BUILD_NO_REMOVEVC_PROCESS -# include "RemoveVCProcess.h" +# include "PostProcessing/RemoveVCProcess.h" #endif #ifndef ASSIMP_BUILD_NO_SPLITLARGEMESHES_PROCESS -# include "SplitLargeMeshes.h" +# include "PostProcessing/SplitLargeMeshes.h" #endif #ifndef ASSIMP_BUILD_NO_PRETRANSFORMVERTICES_PROCESS -# include "PretransformVertices.h" +# include "PostProcessing/PretransformVertices.h" #endif #ifndef ASSIMP_BUILD_NO_LIMITBONEWEIGHTS_PROCESS -# include "LimitBoneWeightsProcess.h" +# include "PostProcessing/LimitBoneWeightsProcess.h" #endif #ifndef ASSIMP_BUILD_NO_VALIDATEDS_PROCESS -# include "ValidateDataStructure.h" +# include "PostProcessing/ValidateDataStructure.h" #endif #ifndef ASSIMP_BUILD_NO_IMPROVECACHELOCALITY_PROCESS -# include "ImproveCacheLocality.h" +# include "PostProcessing/ImproveCacheLocality.h" #endif #ifndef ASSIMP_BUILD_NO_FIXINFACINGNORMALS_PROCESS -# include "FixNormalsStep.h" +# include "PostProcessing/FixNormalsStep.h" #endif #ifndef ASSIMP_BUILD_NO_REMOVE_REDUNDANTMATERIALS_PROCESS -# include "RemoveRedundantMaterials.h" +# include "PostProcessing/RemoveRedundantMaterials.h" #endif #if (!defined ASSIMP_BUILD_NO_EMBEDTEXTURES_PROCESS) -# include "EmbedTexturesProcess.h" +# include "PostProcessing/EmbedTexturesProcess.h" #endif #ifndef ASSIMP_BUILD_NO_FINDINVALIDDATA_PROCESS -# include "FindInvalidDataProcess.h" +# include "PostProcessing/FindInvalidDataProcess.h" #endif #ifndef ASSIMP_BUILD_NO_FINDDEGENERATES_PROCESS -# include "FindDegenerates.h" +# include "PostProcessing/FindDegenerates.h" #endif #ifndef ASSIMP_BUILD_NO_SORTBYPTYPE_PROCESS -# include "SortByPTypeProcess.h" +# include "PostProcessing/SortByPTypeProcess.h" #endif #ifndef ASSIMP_BUILD_NO_GENUVCOORDS_PROCESS -# include "ComputeUVMappingProcess.h" +# include "PostProcessing/ComputeUVMappingProcess.h" #endif #ifndef ASSIMP_BUILD_NO_TRANSFORMTEXCOORDS_PROCESS -# include "TextureTransform.h" +# include "PostProcessing/TextureTransform.h" #endif #ifndef ASSIMP_BUILD_NO_FINDINSTANCES_PROCESS -# include "FindInstancesProcess.h" +# include "PostProcessing/FindInstancesProcess.h" #endif #ifndef ASSIMP_BUILD_NO_OPTIMIZEMESHES_PROCESS -# include "OptimizeMeshes.h" +# include "PostProcessing/OptimizeMeshes.h" #endif #ifndef ASSIMP_BUILD_NO_OPTIMIZEGRAPH_PROCESS -# include "OptimizeGraph.h" +# include "PostProcessing/OptimizeGraph.h" #endif #ifndef ASSIMP_BUILD_NO_SPLITBYBONECOUNT_PROCESS -# include "SplitByBoneCountProcess.h" +# include "Common/SplitByBoneCountProcess.h" #endif #ifndef ASSIMP_BUILD_NO_DEBONE_PROCESS -# include "DeboneProcess.h" +# include "PostProcessing/DeboneProcess.h" #endif #if (!defined ASSIMP_BUILD_NO_GLOBALSCALE_PROCESS) -# include "ScaleProcess.h" +# include "PostProcessing/ScaleProcess.h" #endif +#if (!defined ASSIMP_BUILD_NO_GENBOUNDINGBOXES_PROCESS) +# include "PostProcessing/GenBoundingBoxesProcess.h" +#endif + namespace Assimp { @@ -246,6 +250,9 @@ void GetPostProcessingStepInstanceList(std::vector< BaseProcess* >& out) #if (!defined ASSIMP_BUILD_NO_IMPROVECACHELOCALITY_PROCESS) out.push_back( new ImproveCacheLocalityProcess()); #endif +#if (!defined ASSIMP_BUILD_NO_GENBOUNDINGBOXES_PROCESS) + out.push_back(new GenBoundingBoxesProcess); +#endif } } diff --git a/thirdparty/assimp/code/RemoveComments.cpp b/thirdparty/assimp/code/Common/RemoveComments.cpp index 91700a7699..91700a7699 100644 --- a/thirdparty/assimp/code/RemoveComments.cpp +++ b/thirdparty/assimp/code/Common/RemoveComments.cpp diff --git a/thirdparty/assimp/code/SGSpatialSort.cpp b/thirdparty/assimp/code/Common/SGSpatialSort.cpp index 120070b0aa..120070b0aa 100644 --- a/thirdparty/assimp/code/SGSpatialSort.cpp +++ b/thirdparty/assimp/code/Common/SGSpatialSort.cpp diff --git a/thirdparty/assimp/code/SceneCombiner.cpp b/thirdparty/assimp/code/Common/SceneCombiner.cpp index e445bd7434..e445bd7434 100644 --- a/thirdparty/assimp/code/SceneCombiner.cpp +++ b/thirdparty/assimp/code/Common/SceneCombiner.cpp diff --git a/thirdparty/assimp/code/ScenePreprocessor.cpp b/thirdparty/assimp/code/Common/ScenePreprocessor.cpp index 432a3d7666..432a3d7666 100644 --- a/thirdparty/assimp/code/ScenePreprocessor.cpp +++ b/thirdparty/assimp/code/Common/ScenePreprocessor.cpp diff --git a/thirdparty/assimp/code/ScenePreprocessor.h b/thirdparty/assimp/code/Common/ScenePreprocessor.h index 3f4c8d7c3f..3f4c8d7c3f 100644 --- a/thirdparty/assimp/code/ScenePreprocessor.h +++ b/thirdparty/assimp/code/Common/ScenePreprocessor.h diff --git a/thirdparty/assimp/code/ScenePrivate.h b/thirdparty/assimp/code/Common/ScenePrivate.h index f336aafc9a..f336aafc9a 100644 --- a/thirdparty/assimp/code/ScenePrivate.h +++ b/thirdparty/assimp/code/Common/ScenePrivate.h diff --git a/thirdparty/assimp/code/SkeletonMeshBuilder.cpp b/thirdparty/assimp/code/Common/SkeletonMeshBuilder.cpp index 06cfe034e9..06cfe034e9 100644 --- a/thirdparty/assimp/code/SkeletonMeshBuilder.cpp +++ b/thirdparty/assimp/code/Common/SkeletonMeshBuilder.cpp diff --git a/thirdparty/assimp/code/SpatialSort.cpp b/thirdparty/assimp/code/Common/SpatialSort.cpp index a4f3a4e4b8..a4f3a4e4b8 100644 --- a/thirdparty/assimp/code/SpatialSort.cpp +++ b/thirdparty/assimp/code/Common/SpatialSort.cpp diff --git a/thirdparty/assimp/code/SplitByBoneCountProcess.cpp b/thirdparty/assimp/code/Common/SplitByBoneCountProcess.cpp index 2ef66a9afc..2ef66a9afc 100644 --- a/thirdparty/assimp/code/SplitByBoneCountProcess.cpp +++ b/thirdparty/assimp/code/Common/SplitByBoneCountProcess.cpp diff --git a/thirdparty/assimp/code/SplitByBoneCountProcess.h b/thirdparty/assimp/code/Common/SplitByBoneCountProcess.h index 6c904a9df4..6c904a9df4 100644 --- a/thirdparty/assimp/code/SplitByBoneCountProcess.h +++ b/thirdparty/assimp/code/Common/SplitByBoneCountProcess.h diff --git a/thirdparty/assimp/code/StandardShapes.cpp b/thirdparty/assimp/code/Common/StandardShapes.cpp index 2e5100130f..2e5100130f 100644 --- a/thirdparty/assimp/code/StandardShapes.cpp +++ b/thirdparty/assimp/code/Common/StandardShapes.cpp diff --git a/thirdparty/assimp/code/StdOStreamLogStream.h b/thirdparty/assimp/code/Common/StdOStreamLogStream.h index 893e261a2b..893e261a2b 100644 --- a/thirdparty/assimp/code/StdOStreamLogStream.h +++ b/thirdparty/assimp/code/Common/StdOStreamLogStream.h diff --git a/thirdparty/assimp/code/Subdivision.cpp b/thirdparty/assimp/code/Common/Subdivision.cpp index 19db223a55..60c54939f5 100644 --- a/thirdparty/assimp/code/Subdivision.cpp +++ b/thirdparty/assimp/code/Common/Subdivision.cpp @@ -43,9 +43,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <assimp/Subdivision.h> #include <assimp/SceneCombiner.h> #include <assimp/SpatialSort.h> -#include "ProcessHelper.h" #include <assimp/Vertex.h> #include <assimp/ai_assert.h> + +#include "PostProcessing/ProcessHelper.h" + #include <stdio.h> using namespace Assimp; @@ -56,8 +58,7 @@ void mydummy() {} * implementation is basing on recursive refinement. Directly evaluating the result is also * possible and much quicker, but it depends on lengthy matrix lookup tables. */ // ------------------------------------------------------------------------------------------------ -class CatmullClarkSubdivider : public Subdivider -{ +class CatmullClarkSubdivider : public Subdivider { public: void Subdivide (aiMesh* mesh, aiMesh*& out, unsigned int num, bool discard_input); void Subdivide (aiMesh** smesh, size_t nmesh, diff --git a/thirdparty/assimp/code/TargetAnimation.cpp b/thirdparty/assimp/code/Common/TargetAnimation.cpp index b8062499ff..b8062499ff 100644 --- a/thirdparty/assimp/code/TargetAnimation.cpp +++ b/thirdparty/assimp/code/Common/TargetAnimation.cpp diff --git a/thirdparty/assimp/code/TargetAnimation.h b/thirdparty/assimp/code/Common/TargetAnimation.h index 91634ab5aa..91634ab5aa 100644 --- a/thirdparty/assimp/code/TargetAnimation.h +++ b/thirdparty/assimp/code/Common/TargetAnimation.h diff --git a/thirdparty/assimp/code/Version.cpp b/thirdparty/assimp/code/Common/Version.cpp index 0381037ff1..cc94340ac8 100644 --- a/thirdparty/assimp/code/Version.cpp +++ b/thirdparty/assimp/code/Common/Version.cpp @@ -134,7 +134,7 @@ ASSIMP_API aiScene::aiScene() , mCameras(nullptr) , mMetaData(nullptr) , mPrivate(new Assimp::ScenePrivateData()) { - // empty + // empty } // ------------------------------------------------------------------------------------------------ diff --git a/thirdparty/assimp/code/VertexTriangleAdjacency.cpp b/thirdparty/assimp/code/Common/VertexTriangleAdjacency.cpp index 7cfd1a3505..7cfd1a3505 100644 --- a/thirdparty/assimp/code/VertexTriangleAdjacency.cpp +++ b/thirdparty/assimp/code/Common/VertexTriangleAdjacency.cpp diff --git a/thirdparty/assimp/code/VertexTriangleAdjacency.h b/thirdparty/assimp/code/Common/VertexTriangleAdjacency.h index f3be47612d..f3be47612d 100644 --- a/thirdparty/assimp/code/VertexTriangleAdjacency.h +++ b/thirdparty/assimp/code/Common/VertexTriangleAdjacency.h diff --git a/thirdparty/assimp/code/Win32DebugLogStream.h b/thirdparty/assimp/code/Common/Win32DebugLogStream.h index a6063a261e..a6063a261e 100644 --- a/thirdparty/assimp/code/Win32DebugLogStream.h +++ b/thirdparty/assimp/code/Common/Win32DebugLogStream.h diff --git a/thirdparty/assimp/code/Common/assbin_chunks.h b/thirdparty/assimp/code/Common/assbin_chunks.h new file mode 100644 index 0000000000..15e4af5e7d --- /dev/null +++ b/thirdparty/assimp/code/Common/assbin_chunks.h @@ -0,0 +1,196 @@ +#ifndef INCLUDED_ASSBIN_CHUNKS_H +#define INCLUDED_ASSBIN_CHUNKS_H + +#define ASSBIN_VERSION_MAJOR 1 +#define ASSBIN_VERSION_MINOR 0 + +/** +@page assfile .ASS File formats + +@section over Overview +Assimp provides its own interchange format, which is intended to applications which need +to serialize 3D-models and to reload them quickly. Assimp's file formats are designed to +be read by Assimp itself. They encode additional information needed by Assimp to optimize +its postprocessing pipeline. If you once apply specific steps to a scene, then save it +and reread it from an ASS format using the same post processing settings, they won't +be executed again. + +The format comes in two flavours: XML and binary - both of them hold a complete dump of +the 'aiScene' data structure returned by the APIs. The focus for the binary format +(<tt>.assbin</tt>) is fast loading. Optional deflate compression helps reduce file size. The XML +flavour, <tt>.assxml</tt> or simply .xml, is just a plain-to-xml conversion of aiScene. + +ASSBIN is Assimp's binary interchange format. assimp_cmd (<tt><root>/tools/assimp_cmd</tt>) is able to +write it and the core library provides a loader for it. + +@section assxml XML File format + +The format is pretty much self-explanatory due to its similarity to the in-memory aiScene structure. +With few exceptions, C structures are wrapped in XML elements. + +The DTD for ASSXML can be found in <tt><root>/doc/AssXML_Scheme.xml</tt>. Or have look +at the output files generated by assimp_cmd. + +@section assbin Binary file format + +The ASSBIN file format is composed of chunks to represent the hierarchical aiScene data structure. +This makes the format extensible and allows backward-compatibility with future data structure +versions. The <tt><root>/code/assbin_chunks.h</tt> header contains some magic constants +for use by stand-alone ASSBIN loaders. Also, Assimp's own file writer can be found +in <tt><root>/tools/assimp_cmd/WriteDumb.cpp</tt> (yes, the 'b' is no typo ...). + +@verbatim + +------------------------------------------------------------------------------- +1. File structure: +------------------------------------------------------------------------------- + +---------------------- +| Header (512 bytes) | +---------------------- +| Variable chunks | +---------------------- + +------------------------------------------------------------------------------- +2. Definitions: +------------------------------------------------------------------------------- + +integer is four bytes wide, stored in little-endian byte order. +short is two bytes wide, stored in little-endian byte order. +byte is a single byte. +string is an integer n followed by n UTF-8 characters, not terminated by zero +float is an IEEE 754 single-precision floating-point value +double is an IEEE 754 double-precision floating-point value +t[n] is an array of n elements of type t + +------------------------------------------------------------------------------- +2. Header: +------------------------------------------------------------------------------- + +byte[44] Magic identification string for ASSBIN files. + 'ASSIMP.binary' + +integer Major version of the Assimp library which wrote the file +integer Minor version of the Assimp library which wrote the file + match these against ASSBIN_VERSION_MAJOR and ASSBIN_VERSION_MINOR + +integer SVN revision of the Assimp library (intended for our internal + debugging - if you write Ass files from your own APPs, set this value to 0. +integer Assimp compile flags + +short 0 for normal files, 1 for shortened dumps for regression tests + these should have the file extension assbin.regress + +short 1 if the data after the header is compressed with the DEFLATE algorithm, + 0 for uncompressed files. + For compressed files, the first integer after the header is + always the uncompressed data size + +byte[256] Zero-terminated source file name, UTF-8 +byte[128] Zero-terminated command line parameters passed to assimp_cmd, UTF-8 + +byte[64] Reserved for future use +---> Total length: 512 bytes + +------------------------------------------------------------------------------- +3. Chunks: +------------------------------------------------------------------------------- + +integer Magic chunk ID (ASSBIN_CHUNK_XXX) +integer Chunk data length, in bytes + (unknown chunks are possible, a good reader skips over them) + (chunk-data-length does not include the first two integers) + +byte[n] chunk-data-length bytes of data, depending on the chunk type + +Chunks can contain nested chunks. Nested chunks are ALWAYS at the end of the chunk, +their size is included in chunk-data-length. + +The chunk layout for all ASSIMP data structures is derived from their C declarations. +The general 'rule' to get from Assimp headers to the serialized layout is: + + 1. POD members (i.e. aiMesh::mPrimitiveTypes, aiMesh::mNumVertices), + in order of declaration. + + 2. Array-members (aiMesh::mFaces, aiMesh::mVertices, aiBone::mWeights), + in order of declaration. + + 2. Object array members (i.e aiMesh::mBones, aiScene::mMeshes) are stored in + subchunks directly following the data written in 1.) and 2.) + + + Of course, there are some exceptions to this general order: + +[[aiScene]] + + - The root node holding the scene structure is naturally stored in + a ASSBIN_CHUNK_AINODE subchunk following 1.) and 2.) (which is + empty for aiScene). + +[[aiMesh]] + + - mTextureCoords and mNumUVComponents are serialized as follows: + + [number of used uv channels times] + integer mNumUVComponents[n] + float mTextureCoords[n][3] + + -> more than AI_MAX_TEXCOORD_CHANNELS can be stored. This allows Assimp + builds with different settings for AI_MAX_TEXCOORD_CHANNELS to exchange + data. + -> the on-disk format always uses 3 floats to write UV coordinates. + If mNumUVComponents[0] is 1, the corresponding mTextureCoords array + consists of 3 floats. + + - The array member block of aiMesh is prefixed with an integer that specifies + the kinds of vertex components actually present in the mesh. This is a + bitwise combination of the ASSBIN_MESH_HAS_xxx constants. + +[[aiFace]] + + - mNumIndices is stored as short + - mIndices are written as short, if aiMesh::mNumVertices<65536 + +[[aiNode]] + + - mParent is omitted + +[[aiLight]] + + - mAttenuationXXX not written if aiLight::mType == aiLightSource_DIRECTIONAL + - mAngleXXX not written if aiLight::mType != aiLightSource_SPOT + +[[aiMaterial]] + + - mNumAllocated is omitted, for obvious reasons :-) + + + @endverbatim*/ + + +#define ASSBIN_HEADER_LENGTH 512 + +// these are the magic chunk identifiers for the binary ASS file format +#define ASSBIN_CHUNK_AICAMERA 0x1234 +#define ASSBIN_CHUNK_AILIGHT 0x1235 +#define ASSBIN_CHUNK_AITEXTURE 0x1236 +#define ASSBIN_CHUNK_AIMESH 0x1237 +#define ASSBIN_CHUNK_AINODEANIM 0x1238 +#define ASSBIN_CHUNK_AISCENE 0x1239 +#define ASSBIN_CHUNK_AIBONE 0x123a +#define ASSBIN_CHUNK_AIANIMATION 0x123b +#define ASSBIN_CHUNK_AINODE 0x123c +#define ASSBIN_CHUNK_AIMATERIAL 0x123d +#define ASSBIN_CHUNK_AIMATERIALPROPERTY 0x123e + +#define ASSBIN_MESH_HAS_POSITIONS 0x1 +#define ASSBIN_MESH_HAS_NORMALS 0x2 +#define ASSBIN_MESH_HAS_TANGENTS_AND_BITANGENTS 0x4 +#define ASSBIN_MESH_HAS_TEXCOORD_BASE 0x100 +#define ASSBIN_MESH_HAS_COLOR_BASE 0x10000 + +#define ASSBIN_MESH_HAS_TEXCOORD(n) (ASSBIN_MESH_HAS_TEXCOORD_BASE << n) +#define ASSBIN_MESH_HAS_COLOR(n) (ASSBIN_MESH_HAS_COLOR_BASE << n) + + +#endif // INCLUDED_ASSBIN_CHUNKS_H diff --git a/thirdparty/assimp/code/scene.cpp b/thirdparty/assimp/code/Common/scene.cpp index 2acb348d81..2acb348d81 100644 --- a/thirdparty/assimp/code/scene.cpp +++ b/thirdparty/assimp/code/Common/scene.cpp diff --git a/thirdparty/assimp/code/simd.cpp b/thirdparty/assimp/code/Common/simd.cpp index 04615f408e..04615f408e 100644 --- a/thirdparty/assimp/code/simd.cpp +++ b/thirdparty/assimp/code/Common/simd.cpp diff --git a/thirdparty/assimp/code/simd.h b/thirdparty/assimp/code/Common/simd.h index 3eecdd4581..3eecdd4581 100644 --- a/thirdparty/assimp/code/simd.h +++ b/thirdparty/assimp/code/Common/simd.h diff --git a/thirdparty/assimp/code/FBXAnimation.cpp b/thirdparty/assimp/code/FBX/FBXAnimation.cpp index 874914431b..874914431b 100644 --- a/thirdparty/assimp/code/FBXAnimation.cpp +++ b/thirdparty/assimp/code/FBX/FBXAnimation.cpp diff --git a/thirdparty/assimp/code/FBXBinaryTokenizer.cpp b/thirdparty/assimp/code/FBX/FBXBinaryTokenizer.cpp index 7138df4315..a4a2bc8e79 100644 --- a/thirdparty/assimp/code/FBXBinaryTokenizer.cpp +++ b/thirdparty/assimp/code/FBX/FBXBinaryTokenizer.cpp @@ -98,7 +98,7 @@ namespace FBX { // return (flags & to_check) != 0; //} // ------------------------------------------------------------------------------------------------ -Token::Token(const char* sbegin, const char* send, TokenType type, unsigned int offset) +Token::Token(const char* sbegin, const char* send, TokenType type, size_t offset) : #ifdef DEBUG contents(sbegin, static_cast<size_t>(send-sbegin)), @@ -122,18 +122,18 @@ namespace { // ------------------------------------------------------------------------------------------------ // signal tokenization error, this is always unrecoverable. Throws DeadlyImportError. -AI_WONT_RETURN void TokenizeError(const std::string& message, unsigned int offset) AI_WONT_RETURN_SUFFIX; -AI_WONT_RETURN void TokenizeError(const std::string& message, unsigned int offset) +AI_WONT_RETURN void TokenizeError(const std::string& message, size_t offset) AI_WONT_RETURN_SUFFIX; +AI_WONT_RETURN void TokenizeError(const std::string& message, size_t offset) { throw DeadlyImportError(Util::AddOffset("FBX-Tokenize",message,offset)); } // ------------------------------------------------------------------------------------------------ -uint32_t Offset(const char* begin, const char* cursor) { +size_t Offset(const char* begin, const char* cursor) { ai_assert(begin <= cursor); - return static_cast<unsigned int>(cursor - begin); + return cursor - begin; } // ------------------------------------------------------------------------------------------------ @@ -424,7 +424,7 @@ bool ReadScope(TokenList& output_tokens, const char* input, const char*& cursor, // ------------------------------------------------------------------------------------------------ // TODO: Test FBX Binary files newer than the 7500 version to check if the 64 bits address behaviour is consistent -void TokenizeBinary(TokenList& output_tokens, const char* input, unsigned int length) +void TokenizeBinary(TokenList& output_tokens, const char* input, size_t length) { ai_assert(input); diff --git a/thirdparty/assimp/code/FBXCommon.h b/thirdparty/assimp/code/FBX/FBXCommon.h index fcb20a5cad..e516449130 100644 --- a/thirdparty/assimp/code/FBXCommon.h +++ b/thirdparty/assimp/code/FBX/FBXCommon.h @@ -47,7 +47,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef ASSIMP_BUILD_NO_FBX_EXPORTER - +namespace Assimp { namespace FBX { const std::string NULL_RECORD = { // 13 null bytes @@ -80,7 +80,7 @@ namespace FBX TransformInheritance_MAX // end-of-enum sentinel }; } - +} #endif // ASSIMP_BUILD_NO_FBX_EXPORTER #endif // AI_FBXCOMMON_H_INC diff --git a/thirdparty/assimp/code/FBXCompileConfig.h b/thirdparty/assimp/code/FBX/FBXCompileConfig.h index 3a3841fa5b..3a3841fa5b 100644 --- a/thirdparty/assimp/code/FBXCompileConfig.h +++ b/thirdparty/assimp/code/FBX/FBXCompileConfig.h diff --git a/thirdparty/assimp/code/FBXConverter.cpp b/thirdparty/assimp/code/FBX/FBXConverter.cpp index 09ae06a64f..9f940d3226 100644 --- a/thirdparty/assimp/code/FBXConverter.cpp +++ b/thirdparty/assimp/code/FBX/FBXConverter.cpp @@ -67,6 +67,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <sstream> #include <iomanip> + namespace Assimp { namespace FBX { @@ -76,20 +77,21 @@ namespace Assimp { #define CONVERT_FBX_TIME(time) static_cast<double>(time) / 46186158000L - FBXConverter::FBXConverter(aiScene* out, const Document& doc) - : defaultMaterialIndex() - , lights() - , cameras() - , textures() - , materials_converted() - , textures_converted() - , meshes_converted() - , node_anim_chain_bits() - , mNodeNameInstances() - , mNodeNames() - , anim_fps() - , out(out) - , doc(doc) { + FBXConverter::FBXConverter(aiScene* out, const Document& doc, bool removeEmptyBones, FbxUnit unit ) + : defaultMaterialIndex() + , lights() + , cameras() + , textures() + , materials_converted() + , textures_converted() + , meshes_converted() + , node_anim_chain_bits() + , mNodeNames() + , anim_fps() + , out(out) + , doc(doc) + , mRemoveEmptyBones( removeEmptyBones ) + , mCurrentUnit(FbxUnit::cm) { // animations need to be converted first since this will // populate the node_anim_chain_bits map, which is needed // to determine which nodes need to be generated. @@ -117,6 +119,7 @@ namespace Assimp { ConvertGlobalSettings(); TransferDataToScene(); + ConvertToUnitScale(unit); // if we didn't read any meshes set the AI_SCENE_FLAGS_INCOMPLETE // to make sure the scene passes assimp's validation. FBX files @@ -138,12 +141,46 @@ namespace Assimp { void FBXConverter::ConvertRootNode() { out->mRootNode = new aiNode(); - out->mRootNode->mName.Set("RootNode"); + std::string unique_name; + GetUniqueName("RootNode", unique_name); + out->mRootNode->mName.Set(unique_name); // root has ID 0 ConvertNodes(0L, *out->mRootNode); } + static std::string getAncestorBaseName(const aiNode* node) + { + const char* nodeName = nullptr; + size_t length = 0; + while (node && (!nodeName || length == 0)) + { + nodeName = node->mName.C_Str(); + length = node->mName.length; + node = node->mParent; + } + + if (!nodeName || length == 0) + { + return {}; + } + // could be std::string_view if c++17 available + return std::string(nodeName, length); + } + + // Make unique name + std::string FBXConverter::MakeUniqueNodeName(const Model* const model, const aiNode& parent) + { + std::string original_name = FixNodeName(model->Name()); + if (original_name.empty()) + { + original_name = getAncestorBaseName(&parent); + } + std::string unique_name; + GetUniqueName(original_name, unique_name); + return unique_name; + } + void FBXConverter::ConvertNodes(uint64_t id, aiNode& parent, const aiMatrix4x4& parent_transform) { const std::vector<const Connection*>& conns = doc.GetConnectionsByDestinationSequenced(id, "Model"); @@ -175,35 +212,18 @@ namespace Assimp { aiMatrix4x4 new_abs_transform = parent_transform; + std::string unique_name = MakeUniqueNodeName(model, parent); + // even though there is only a single input node, the design of // assimp (or rather: the complicated transformation chain that // is employed by fbx) means that we may need multiple aiNode's // to represent a fbx node's transformation. - GenerateTransformationNodeChain(*model, nodes_chain, post_nodes_chain); + const bool need_additional_node = GenerateTransformationNodeChain(*model, unique_name, nodes_chain, post_nodes_chain); ai_assert(nodes_chain.size()); - std::string original_name = FixNodeName(model->Name()); - - // check if any of the nodes in the chain has the name the fbx node - // is supposed to have. If there is none, add another node to - // preserve the name - people might have scripts etc. that rely - // on specific node names. - aiNode* name_carrier = NULL; - for (aiNode* prenode : nodes_chain) { - if (!strcmp(prenode->mName.C_Str(), original_name.c_str())) { - name_carrier = prenode; - break; - } - } - - if (!name_carrier) { - std::string old_original_name = original_name; - GetUniqueName(old_original_name, original_name); - nodes_chain.push_back(new aiNode(original_name)); - } - else { - original_name = nodes_chain.back()->mName.C_Str(); + if (need_additional_node) { + nodes_chain.push_back(new aiNode(unique_name)); } //setup metadata on newest node @@ -265,11 +285,11 @@ namespace Assimp { ConvertNodes(model->ID(), *last_parent, new_abs_transform); if (doc.Settings().readLights) { - ConvertLights(*model, original_name); + ConvertLights(*model, unique_name); } if (doc.Settings().readCameras) { - ConvertCameras(*model, original_name); + ConvertCameras(*model, unique_name); } nodes.push_back(nodes_chain.front()); @@ -387,6 +407,7 @@ namespace Assimp { break; default: ai_assert(false); + break; } } @@ -399,11 +420,6 @@ namespace Assimp { out_camera->mAspect = cam.AspectWidth() / cam.AspectHeight(); - //cameras are defined along positive x direction - /*out_camera->mPosition = cam.Position(); - out_camera->mLookAt = (cam.InterestPosition() - out_camera->mPosition).Normalize(); - out_camera->mUp = cam.UpVector();*/ - out_camera->mPosition = aiVector3D(0.0f); out_camera->mLookAt = aiVector3D(1.0f, 0.0f, 0.0f); out_camera->mUp = aiVector3D(0.0f, 1.0f, 0.0f); @@ -421,21 +437,16 @@ namespace Assimp { void FBXConverter::GetUniqueName(const std::string &name, std::string &uniqueName) { uniqueName = name; - int i = 0; - auto it = mNodeNameInstances.find(name); // duplicate node name instance count - if (it != mNodeNameInstances.end()) + auto it_pair = mNodeNames.insert({ name, 0 }); // duplicate node name instance count + unsigned int& i = it_pair.first->second; + while (!it_pair.second) { - i = it->second; - while (mNodeNames.find(uniqueName) != mNodeNames.end()) - { - i++; - std::stringstream ext; - ext << name << std::setfill('0') << std::setw(3) << i; - uniqueName = ext.str(); - } + i++; + std::ostringstream ext; + ext << name << std::setfill('0') << std::setw(3) << i; + uniqueName = ext.str(); + it_pair = mNodeNames.insert({ uniqueName, 0 }); } - mNodeNameInstances[name] = i; - mNodeNames.insert(uniqueName); } const char* FBXConverter::NameTransformationComp(TransformationComp comp) { @@ -651,8 +662,7 @@ namespace Assimp { if ((v - all_ones).SquareLength() > zero_epsilon) { return true; } - } - else if (ok) { + } else if (ok) { if (v.SquareLength() > zero_epsilon) { return true; } @@ -667,7 +677,7 @@ namespace Assimp { return name + std::string(MAGIC_NODE_TAG) + "_" + NameTransformationComp(comp); } - void FBXConverter::GenerateTransformationNodeChain(const Model& model, std::vector<aiNode*>& output_nodes, + bool FBXConverter::GenerateTransformationNodeChain(const Model& model, const std::string& name, std::vector<aiNode*>& output_nodes, std::vector<aiNode*>& post_output_nodes) { const PropertyTable& props = model.Props(); const Model::RotOrder rot = model.RotationOrder(); @@ -782,8 +792,6 @@ namespace Assimp { // not be guaranteed. ai_assert(NeedsComplexTransformationChain(model) == is_complex); - std::string name = FixNodeName(model.Name()); - // now, if we have more than just Translation, Scaling and Rotation, // we need to generate a full node chain to accommodate for assimp's // lack to express pivots and offsets. @@ -825,20 +833,20 @@ namespace Assimp { } ai_assert(output_nodes.size()); - return; + return true; } // else, we can just multiply the matrices together aiNode* nd = new aiNode(); output_nodes.push_back(nd); - std::string uniqueName; - GetUniqueName(name, uniqueName); - nd->mName.Set(uniqueName); + // name passed to the method is already unique + nd->mName.Set(name); for (const auto &transform : chain) { nd->mTransformation = nd->mTransformation * transform; } + return false; } void FBXConverter::SetupNodeMetadata(const Model& model, aiNode& nd) @@ -977,7 +985,9 @@ namespace Assimp { unsigned int epcount = 0; for (unsigned i = 0; i < indices.size(); i++) { - if (indices[i] < 0) epcount++; + if (indices[i] < 0) { + epcount++; + } } unsigned int pcount = static_cast<unsigned int>( indices.size() ); unsigned int scount = out_mesh->mNumFaces = pcount - epcount; @@ -1237,10 +1247,10 @@ namespace Assimp { ai_assert(count_faces); ai_assert(count_vertices); - // mapping from output indices to DOM indexing, needed to resolve weights + // mapping from output indices to DOM indexing, needed to resolve weights or blendshapes std::vector<unsigned int> reverseMapping; - - if (process_weights) { + std::map<unsigned int, unsigned int> translateIndexMap; + if (process_weights || mesh.GetBlendShapes().size() > 0) { reverseMapping.resize(count_vertices); } @@ -1347,6 +1357,7 @@ namespace Assimp { if (reverseMapping.size()) { reverseMapping[cursor] = in_cursor; + translateIndexMap[in_cursor] = cursor; } out_mesh->mVertices[cursor] = vertices[in_cursor]; @@ -1378,6 +1389,50 @@ namespace Assimp { ConvertWeights(out_mesh, model, mesh, node_global_transform, index, &reverseMapping); } + std::vector<aiAnimMesh*> animMeshes; + for (const BlendShape* blendShape : mesh.GetBlendShapes()) { + for (const BlendShapeChannel* blendShapeChannel : blendShape->BlendShapeChannels()) { + const std::vector<const ShapeGeometry*>& shapeGeometries = blendShapeChannel->GetShapeGeometries(); + for (size_t i = 0; i < shapeGeometries.size(); i++) { + aiAnimMesh* animMesh = aiCreateAnimMesh(out_mesh); + const ShapeGeometry* shapeGeometry = shapeGeometries.at(i); + const std::vector<aiVector3D>& vertices = shapeGeometry->GetVertices(); + const std::vector<aiVector3D>& normals = shapeGeometry->GetNormals(); + const std::vector<unsigned int>& indices = shapeGeometry->GetIndices(); + animMesh->mName.Set(FixAnimMeshName(shapeGeometry->Name())); + for (size_t j = 0; j < indices.size(); j++) { + unsigned int index = indices.at(j); + aiVector3D vertex = vertices.at(j); + aiVector3D normal = normals.at(j); + unsigned int count = 0; + const unsigned int* outIndices = mesh.ToOutputVertexIndex(index, count); + for (unsigned int k = 0; k < count; k++) { + unsigned int outIndex = outIndices[k]; + if (translateIndexMap.find(outIndex) == translateIndexMap.end()) + continue; + unsigned int index = translateIndexMap[outIndex]; + animMesh->mVertices[index] += vertex; + if (animMesh->mNormals != nullptr) { + animMesh->mNormals[index] += normal; + animMesh->mNormals[index].NormalizeSafe(); + } + } + } + animMesh->mWeight = shapeGeometries.size() > 1 ? blendShapeChannel->DeformPercent() / 100.0f : 1.0f; + animMeshes.push_back(animMesh); + } + } + } + + const size_t numAnimMeshes = animMeshes.size(); + if (numAnimMeshes > 0) { + out_mesh->mNumAnimMeshes = static_cast<unsigned int>(numAnimMeshes); + out_mesh->mAnimMeshes = new aiAnimMesh*[numAnimMeshes]; + for (size_t i = 0; i < numAnimMeshes; i++) { + out_mesh->mAnimMeshes[i] = animMeshes.at(i); + } + } + return static_cast<unsigned int>(meshes.size() - 1); } @@ -1407,7 +1462,7 @@ namespace Assimp { const WeightIndexArray& indices = cluster->GetIndices(); - if (indices.empty()) { + if (indices.empty() && mRemoveEmptyBones ) { continue; } @@ -1439,13 +1494,11 @@ namespace Assimp { if (index_out_indices.back() == no_index_sentinel) { index_out_indices.back() = out_indices.size(); - } if (no_mat_check) { out_indices.push_back(out_idx[i]); - } - else { + } else { // this extra lookup is in O(logn), so the entire algorithm becomes O(nlogn) const std::vector<unsigned int>::iterator it = std::lower_bound( outputVertStartIndices->begin(), @@ -1461,11 +1514,11 @@ namespace Assimp { } } } - + // if we found at least one, generate the output bones // XXX this could be heavily simplified by collecting the bone // data in a single step. - if (ok) { + if (ok && mRemoveEmptyBones) { ConvertCluster(bones, model, *cluster, out_indices, index_out_indices, count_out_indices, node_global_transform); } @@ -1596,6 +1649,13 @@ namespace Assimp { out_mat->AddProperty(&str, AI_MATKEY_NAME); } + // Set the shading mode as best we can: The FBX specification only mentions Lambert and Phong, and only Phong is mentioned in Assimp's aiShadingMode enum. + if (material.GetShadingModel() == "phong") + { + aiShadingMode shadingMode = aiShadingMode_Phong; + out_mat->AddProperty<aiShadingMode>(&shadingMode, 1, AI_MATKEY_SHADING_MODEL); + } + // shading stuff and colors SetShadingPropertiesCommon(out_mat, props); SetShadingPropertiesRaw( out_mat, props, material.Textures(), mesh ); @@ -1621,7 +1681,7 @@ namespace Assimp { out_tex->pcData = reinterpret_cast<aiTexel*>(const_cast<Video&>(video).RelinquishContent()); // try to extract a hint from the file extension - const std::string& filename = video.FileName().empty() ? video.RelativeFilename() : video.FileName(); + const std::string& filename = video.RelativeFilename().empty() ? video.FileName() : video.RelativeFilename(); std::string ext = BaseImporter::GetExtension(filename); if (ext == "jpeg") { @@ -1632,7 +1692,7 @@ namespace Assimp { memcpy(out_tex->achFormatHint, ext.c_str(), ext.size()); } - out_tex->mFilename.Set(video.FileName().c_str()); + out_tex->mFilename.Set(filename.c_str()); return static_cast<unsigned int>(textures.size() - 1); } @@ -1678,9 +1738,8 @@ namespace Assimp { } void FBXConverter::TrySetTextureProperties(aiMaterial* out_mat, const TextureMap& textures, - const std::string& propName, - aiTextureType target, const MeshGeometry* const mesh) - { + const std::string& propName, + aiTextureType target, const MeshGeometry* const mesh) { TextureMap::const_iterator it = textures.find(propName); if (it == textures.end()) { return; @@ -3407,8 +3466,9 @@ void FBXConverter::SetShadingPropertiesRaw(aiMaterial* out_mat, const PropertyTa na->mNumScalingKeys = static_cast<unsigned int>(keys.size()); na->mScalingKeys = new aiVectorKey[keys.size()]; - if (keys.size() > 0) + if (keys.size() > 0) { InterpolateKeys(na->mScalingKeys, keys, inputs, aiVector3D(1.0f, 1.0f, 1.0f), maxTime, minTime); + } } void FBXConverter::ConvertTranslationKeys(aiNodeAnim* na, const std::vector<const AnimationCurveNode*>& nodes, @@ -3472,6 +3532,46 @@ void FBXConverter::SetShadingPropertiesRaw(aiMaterial* out_mat, const PropertyTa out->mMetaData->Set(14, "CustomFrameRate", doc.GlobalSettings().CustomFrameRate()); } + void FBXConverter::ConvertToUnitScale( FbxUnit unit ) { + if (mCurrentUnit == unit) { + return; + } + + ai_real scale = 1.0; + if (mCurrentUnit == FbxUnit::cm) { + if (unit == FbxUnit::m) { + scale = (ai_real)0.01; + } else if (unit == FbxUnit::km) { + scale = (ai_real)0.00001; + } + } else if (mCurrentUnit == FbxUnit::m) { + if (unit == FbxUnit::cm) { + scale = (ai_real)100.0; + } else if (unit == FbxUnit::km) { + scale = (ai_real)0.001; + } + } else if (mCurrentUnit == FbxUnit::km) { + if (unit == FbxUnit::cm) { + scale = (ai_real)100000.0; + } else if (unit == FbxUnit::m) { + scale = (ai_real)1000.0; + } + } + + for (auto mesh : meshes) { + if (nullptr == mesh) { + continue; + } + + if (mesh->HasPositions()) { + for (unsigned int i = 0; i < mesh->mNumVertices; ++i) { + aiVector3D &pos = mesh->mVertices[i]; + pos *= scale; + } + } + } + } + void FBXConverter::TransferDataToScene() { ai_assert(!out->mMeshes); @@ -3525,9 +3625,9 @@ void FBXConverter::SetShadingPropertiesRaw(aiMaterial* out_mat, const PropertyTa } // ------------------------------------------------------------------------------------------------ - void ConvertToAssimpScene(aiScene* out, const Document& doc) + void ConvertToAssimpScene(aiScene* out, const Document& doc, bool removeEmptyBones, FbxUnit unit) { - FBXConverter converter(out, doc); + FBXConverter converter(out, doc, removeEmptyBones, unit); } } // !FBX diff --git a/thirdparty/assimp/code/FBXConverter.h b/thirdparty/assimp/code/FBX/FBXConverter.h index 50637468b9..17a7bc56b7 100644 --- a/thirdparty/assimp/code/FBXConverter.h +++ b/thirdparty/assimp/code/FBX/FBXConverter.h @@ -52,6 +52,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "FBXUtil.h" #include "FBXProperties.h" #include "FBXImporter.h" + #include <assimp/anim.h> #include <assimp/material.h> #include <assimp/light.h> @@ -76,12 +77,22 @@ namespace FBX { class Document; +enum class FbxUnit { + cm = 0, + m, + km, + NumUnits, + + Undefined +}; + /** * Convert a FBX #Document to #aiScene * @param out Empty scene to be populated - * @param doc Parsed FBX document + * @param doc Parsed FBX document + * @param removeEmptyBones Will remove bones, which do not have any references to vertices. */ -void ConvertToAssimpScene(aiScene* out, const Document& doc); +void ConvertToAssimpScene(aiScene* out, const Document& doc, bool removeEmptyBones, FbxUnit unit); /** Dummy class to encapsulate the conversion process */ class FBXConverter { @@ -112,7 +123,7 @@ public: }; public: - FBXConverter(aiScene* out, const Document& doc); + FBXConverter(aiScene* out, const Document& doc, bool removeEmptyBones, FbxUnit unit); ~FBXConverter(); private: @@ -145,6 +156,11 @@ private: const char* NameTransformationComp(TransformationComp comp); // ------------------------------------------------------------------------------------------------ + // Returns an unique name for a node or traverses up a hierarchy until a non-empty name is found and + // then makes this name unique + std::string MakeUniqueNodeName(const Model* const model, const aiNode& parent); + + // ------------------------------------------------------------------------------------------------ // note: this returns the REAL fbx property names const char* NameTransformationCompProperty(TransformationComp comp); @@ -167,7 +183,7 @@ private: /** * note: memory for output_nodes will be managed by the caller */ - void GenerateTransformationNodeChain(const Model& model, std::vector<aiNode*>& output_nodes, std::vector<aiNode*>& post_output_nodes); + bool GenerateTransformationNodeChain(const Model& model, const std::string& name, std::vector<aiNode*>& output_nodes, std::vector<aiNode*>& post_output_nodes); // ------------------------------------------------------------------------------------------------ void SetupNodeMetadata(const Model& model, aiNode& nd); @@ -415,6 +431,10 @@ private: void ConvertGlobalSettings(); // ------------------------------------------------------------------------------------------------ + // Will perform the conversion from a given unit to the requested unit. + void ConvertToUnitScale(FbxUnit unit); + + // ------------------------------------------------------------------------------------------------ // copy generated meshes, animations, lights, cameras and textures to the output scene void TransferDataToScene(); @@ -443,16 +463,17 @@ private: NodeAnimBitMap node_anim_chain_bits; // number of nodes with the same name - using NodeAnimNameMap = std::unordered_map<std::string, unsigned int>; - NodeAnimNameMap mNodeNameInstances; - - using NodeNameCache = std::unordered_set<std::string>; + using NodeNameCache = std::unordered_map<std::string, unsigned int>; NodeNameCache mNodeNames; double anim_fps; aiScene* const out; const FBX::Document& doc; + + bool mRemoveEmptyBones; + + FbxUnit mCurrentUnit; }; } diff --git a/thirdparty/assimp/code/FBXDeformer.cpp b/thirdparty/assimp/code/FBX/FBXDeformer.cpp index 6927553450..6927553450 100644 --- a/thirdparty/assimp/code/FBXDeformer.cpp +++ b/thirdparty/assimp/code/FBX/FBXDeformer.cpp diff --git a/thirdparty/assimp/code/FBXDocument.cpp b/thirdparty/assimp/code/FBX/FBXDocument.cpp index 1af08fe6d8..1af08fe6d8 100644 --- a/thirdparty/assimp/code/FBXDocument.cpp +++ b/thirdparty/assimp/code/FBX/FBXDocument.cpp diff --git a/thirdparty/assimp/code/FBXDocument.h b/thirdparty/assimp/code/FBX/FBXDocument.h index c849defdcd..18e5c38f13 100644 --- a/thirdparty/assimp/code/FBXDocument.h +++ b/thirdparty/assimp/code/FBX/FBXDocument.h @@ -627,7 +627,7 @@ public: return content; } - uint32_t ContentLength() const { + uint64_t ContentLength() const { return contentLength; } @@ -643,7 +643,7 @@ private: std::string fileName; std::shared_ptr<const PropertyTable> props; - uint32_t contentLength; + uint64_t contentLength; uint8_t* content; }; diff --git a/thirdparty/assimp/code/FBXDocumentUtil.cpp b/thirdparty/assimp/code/FBX/FBXDocumentUtil.cpp index f84691479a..f84691479a 100644 --- a/thirdparty/assimp/code/FBXDocumentUtil.cpp +++ b/thirdparty/assimp/code/FBX/FBXDocumentUtil.cpp diff --git a/thirdparty/assimp/code/FBXDocumentUtil.h b/thirdparty/assimp/code/FBX/FBXDocumentUtil.h index 2450109e59..2450109e59 100644 --- a/thirdparty/assimp/code/FBXDocumentUtil.h +++ b/thirdparty/assimp/code/FBX/FBXDocumentUtil.h diff --git a/thirdparty/assimp/code/FBXExportNode.cpp b/thirdparty/assimp/code/FBX/FBXExportNode.cpp index e5215466a1..06c89cee46 100644 --- a/thirdparty/assimp/code/FBXExportNode.cpp +++ b/thirdparty/assimp/code/FBX/FBXExportNode.cpp @@ -54,6 +54,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <sstream> // ostringstream #include <memory> // shared_ptr +namespace Assimp { // AddP70<type> helpers... there's no usable pattern here, // so all are defined as separate functions. // Even "animatable" properties are often completely different @@ -252,7 +253,8 @@ void FBX::Node::DumpChildren( } else { std::ostringstream ss; DumpChildrenAscii(ss, indent); - s.PutString(ss.str()); + if (ss.tellp() > 0) + s.PutString(ss.str()); } } @@ -266,7 +268,8 @@ void FBX::Node::End( } else { std::ostringstream ss; EndAscii(ss, indent, has_children); - s.PutString(ss.str()); + if (ss.tellp() > 0) + s.PutString(ss.str()); } } @@ -367,7 +370,7 @@ void FBX::Node::EndBinary( bool has_children ) { // if there were children, add a null record - if (has_children) { s.PutString(FBX::NULL_RECORD); } + if (has_children) { s.PutString(Assimp::FBX::NULL_RECORD); } // now go back and write initial pos this->end_pos = s.Tell(); @@ -563,6 +566,6 @@ void FBX::Node::WritePropertyNode( FBX::Node::WritePropertyNodeAscii(name, v, s, indent); } } - +} #endif // ASSIMP_BUILD_NO_FBX_EXPORTER #endif // ASSIMP_BUILD_NO_EXPORT diff --git a/thirdparty/assimp/code/FBXExportNode.h b/thirdparty/assimp/code/FBX/FBXExportNode.h index e1ebc36969..ef3bc781a4 100644 --- a/thirdparty/assimp/code/FBXExportNode.h +++ b/thirdparty/assimp/code/FBX/FBXExportNode.h @@ -54,16 +54,16 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <string> #include <vector> +namespace Assimp { namespace FBX { class Node; } -class FBX::Node -{ -public: // public data members +class FBX::Node { +public: // TODO: accessors std::string name; // node name - std::vector<FBX::Property> properties; // node properties + std::vector<FBX::FBXExportProperty> properties; // node properties std::vector<FBX::Node> children; // child nodes // some nodes always pretend they have children... @@ -214,7 +214,7 @@ public: // static member functions Assimp::StreamWriterLE& s, bool binary, int indent ) { - FBX::Property p(value); + FBX::FBXExportProperty p(value); FBX::Node node(name, p); node.Dump(s, binary, indent); } @@ -264,7 +264,7 @@ private: // static helper functions ); }; - +} #endif // ASSIMP_BUILD_NO_FBX_EXPORTER diff --git a/thirdparty/assimp/code/FBXExportProperty.cpp b/thirdparty/assimp/code/FBX/FBXExportProperty.cpp index 9981d6b1c6..f8593e6295 100644 --- a/thirdparty/assimp/code/FBXExportProperty.cpp +++ b/thirdparty/assimp/code/FBX/FBXExportProperty.cpp @@ -52,187 +52,210 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <locale> #include <sstream> // ostringstream +namespace Assimp { +namespace FBX { // constructors for single element properties -FBX::Property::Property(bool v) - : type('C'), data(1) -{ - data = {uint8_t(v)}; +FBXExportProperty::FBXExportProperty(bool v) +: type('C') +, data(1) { + data = { + uint8_t(v) + }; } -FBX::Property::Property(int16_t v) : type('Y'), data(2) -{ +FBXExportProperty::FBXExportProperty(int16_t v) +: type('Y') +, data(2) { uint8_t* d = data.data(); (reinterpret_cast<int16_t*>(d))[0] = v; } -FBX::Property::Property(int32_t v) : type('I'), data(4) -{ +FBXExportProperty::FBXExportProperty(int32_t v) +: type('I') +, data(4) { uint8_t* d = data.data(); (reinterpret_cast<int32_t*>(d))[0] = v; } -FBX::Property::Property(float v) : type('F'), data(4) -{ +FBXExportProperty::FBXExportProperty(float v) +: type('F') +, data(4) { uint8_t* d = data.data(); (reinterpret_cast<float*>(d))[0] = v; } -FBX::Property::Property(double v) : type('D'), data(8) -{ +FBXExportProperty::FBXExportProperty(double v) +: type('D') +, data(8) { uint8_t* d = data.data(); (reinterpret_cast<double*>(d))[0] = v; } -FBX::Property::Property(int64_t v) : type('L'), data(8) -{ +FBXExportProperty::FBXExportProperty(int64_t v) +: type('L') +, data(8) { uint8_t* d = data.data(); (reinterpret_cast<int64_t*>(d))[0] = v; } - // constructors for array-type properties -FBX::Property::Property(const char* c, bool raw) - : Property(std::string(c), raw) -{} +FBXExportProperty::FBXExportProperty(const char* c, bool raw) +: FBXExportProperty(std::string(c), raw) { + // empty +} // strings can either be saved as "raw" (R) data, or "string" (S) data -FBX::Property::Property(const std::string& s, bool raw) - : type(raw ? 'R' : 'S'), data(s.size()) -{ +FBXExportProperty::FBXExportProperty(const std::string& s, bool raw) +: type(raw ? 'R' : 'S') +, data(s.size()) { for (size_t i = 0; i < s.size(); ++i) { data[i] = uint8_t(s[i]); } } -FBX::Property::Property(const std::vector<uint8_t>& r) - : type('R'), data(r) -{} +FBXExportProperty::FBXExportProperty(const std::vector<uint8_t>& r) +: type('R') +, data(r) { + // empty +} -FBX::Property::Property(const std::vector<int32_t>& va) - : type('i'), data(4*va.size()) -{ +FBXExportProperty::FBXExportProperty(const std::vector<int32_t>& va) +: type('i') +, data(4 * va.size() ) { int32_t* d = reinterpret_cast<int32_t*>(data.data()); - for (size_t i = 0; i < va.size(); ++i) { d[i] = va[i]; } + for (size_t i = 0; i < va.size(); ++i) { + d[i] = va[i]; + } } -FBX::Property::Property(const std::vector<int64_t>& va) - : type('l'), data(8*va.size()) -{ +FBXExportProperty::FBXExportProperty(const std::vector<int64_t>& va) +: type('l') +, data(8 * va.size()) { int64_t* d = reinterpret_cast<int64_t*>(data.data()); - for (size_t i = 0; i < va.size(); ++i) { d[i] = va[i]; } + for (size_t i = 0; i < va.size(); ++i) { + d[i] = va[i]; + } } -FBX::Property::Property(const std::vector<float>& va) - : type('f'), data(4*va.size()) -{ +FBXExportProperty::FBXExportProperty(const std::vector<float>& va) +: type('f') +, data(4 * va.size()) { float* d = reinterpret_cast<float*>(data.data()); - for (size_t i = 0; i < va.size(); ++i) { d[i] = va[i]; } + for (size_t i = 0; i < va.size(); ++i) { + d[i] = va[i]; + } } -FBX::Property::Property(const std::vector<double>& va) - : type('d'), data(8*va.size()) -{ +FBXExportProperty::FBXExportProperty(const std::vector<double>& va) +: type('d') +, data(8 * va.size()) { double* d = reinterpret_cast<double*>(data.data()); - for (size_t i = 0; i < va.size(); ++i) { d[i] = va[i]; } + for (size_t i = 0; i < va.size(); ++i) { + d[i] = va[i]; + } } -FBX::Property::Property(const aiMatrix4x4& vm) - : type('d'), data(8*16) -{ +FBXExportProperty::FBXExportProperty(const aiMatrix4x4& vm) +: type('d') +, data(8 * 16) { double* d = reinterpret_cast<double*>(data.data()); for (unsigned int c = 0; c < 4; ++c) { for (unsigned int r = 0; r < 4; ++r) { - d[4*c+r] = vm[r][c]; + d[4 * c + r] = vm[r][c]; } } } // public member functions -size_t FBX::Property::size() -{ +size_t FBXExportProperty::size() { switch (type) { - case 'C': case 'Y': case 'I': case 'F': case 'D': case 'L': - return data.size() + 1; - case 'S': case 'R': - return data.size() + 5; - case 'i': case 'd': - return data.size() + 13; - default: - throw DeadlyExportError("Requested size on property of unknown type"); + case 'C': + case 'Y': + case 'I': + case 'F': + case 'D': + case 'L': + return data.size() + 1; + case 'S': + case 'R': + return data.size() + 5; + case 'i': + case 'd': + return data.size() + 13; + default: + throw DeadlyExportError("Requested size on property of unknown type"); } } -void FBX::Property::DumpBinary(Assimp::StreamWriterLE &s) -{ +void FBXExportProperty::DumpBinary(Assimp::StreamWriterLE& s) { s.PutU1(type); uint8_t* d = data.data(); size_t N; switch (type) { - case 'C': s.PutU1(*(reinterpret_cast<uint8_t*>(d))); return; - case 'Y': s.PutI2(*(reinterpret_cast<int16_t*>(d))); return; - case 'I': s.PutI4(*(reinterpret_cast<int32_t*>(d))); return; - case 'F': s.PutF4(*(reinterpret_cast<float*>(d))); return; - case 'D': s.PutF8(*(reinterpret_cast<double*>(d))); return; - case 'L': s.PutI8(*(reinterpret_cast<int64_t*>(d))); return; - case 'S': - case 'R': - s.PutU4(uint32_t(data.size())); - for (size_t i = 0; i < data.size(); ++i) { s.PutU1(data[i]); } - return; - case 'i': - N = data.size() / 4; - s.PutU4(uint32_t(N)); // number of elements - s.PutU4(0); // no encoding (1 would be zip-compressed) - // TODO: compress if large? - s.PutU4(uint32_t(data.size())); // data size - for (size_t i = 0; i < N; ++i) { - s.PutI4((reinterpret_cast<int32_t*>(d))[i]); - } - return; - case 'l': - N = data.size() / 8; - s.PutU4(uint32_t(N)); // number of elements - s.PutU4(0); // no encoding (1 would be zip-compressed) - // TODO: compress if large? - s.PutU4(uint32_t(data.size())); // data size - for (size_t i = 0; i < N; ++i) { - s.PutI8((reinterpret_cast<int64_t*>(d))[i]); - } - return; - case 'f': - N = data.size() / 4; - s.PutU4(uint32_t(N)); // number of elements - s.PutU4(0); // no encoding (1 would be zip-compressed) - // TODO: compress if large? - s.PutU4(uint32_t(data.size())); // data size - for (size_t i = 0; i < N; ++i) { - s.PutF4((reinterpret_cast<float*>(d))[i]); - } - return; - case 'd': - N = data.size() / 8; - s.PutU4(uint32_t(N)); // number of elements - s.PutU4(0); // no encoding (1 would be zip-compressed) - // TODO: compress if large? - s.PutU4(uint32_t(data.size())); // data size - for (size_t i = 0; i < N; ++i) { - s.PutF8((reinterpret_cast<double*>(d))[i]); - } - return; - default: - std::ostringstream err; - err << "Tried to dump property with invalid type '"; - err << type << "'!"; - throw DeadlyExportError(err.str()); + case 'C': s.PutU1(*(reinterpret_cast<uint8_t*>(d))); return; + case 'Y': s.PutI2(*(reinterpret_cast<int16_t*>(d))); return; + case 'I': s.PutI4(*(reinterpret_cast<int32_t*>(d))); return; + case 'F': s.PutF4(*(reinterpret_cast<float*>(d))); return; + case 'D': s.PutF8(*(reinterpret_cast<double*>(d))); return; + case 'L': s.PutI8(*(reinterpret_cast<int64_t*>(d))); return; + case 'S': + case 'R': + s.PutU4(uint32_t(data.size())); + for (size_t i = 0; i < data.size(); ++i) { s.PutU1(data[i]); } + return; + case 'i': + N = data.size() / 4; + s.PutU4(uint32_t(N)); // number of elements + s.PutU4(0); // no encoding (1 would be zip-compressed) + // TODO: compress if large? + s.PutU4(uint32_t(data.size())); // data size + for (size_t i = 0; i < N; ++i) { + s.PutI4((reinterpret_cast<int32_t*>(d))[i]); + } + return; + case 'l': + N = data.size() / 8; + s.PutU4(uint32_t(N)); // number of elements + s.PutU4(0); // no encoding (1 would be zip-compressed) + // TODO: compress if large? + s.PutU4(uint32_t(data.size())); // data size + for (size_t i = 0; i < N; ++i) { + s.PutI8((reinterpret_cast<int64_t*>(d))[i]); + } + return; + case 'f': + N = data.size() / 4; + s.PutU4(uint32_t(N)); // number of elements + s.PutU4(0); // no encoding (1 would be zip-compressed) + // TODO: compress if large? + s.PutU4(uint32_t(data.size())); // data size + for (size_t i = 0; i < N; ++i) { + s.PutF4((reinterpret_cast<float*>(d))[i]); + } + return; + case 'd': + N = data.size() / 8; + s.PutU4(uint32_t(N)); // number of elements + s.PutU4(0); // no encoding (1 would be zip-compressed) + // TODO: compress if large? + s.PutU4(uint32_t(data.size())); // data size + for (size_t i = 0; i < N; ++i) { + s.PutF8((reinterpret_cast<double*>(d))[i]); + } + return; + default: + std::ostringstream err; + err << "Tried to dump property with invalid type '"; + err << type << "'!"; + throw DeadlyExportError(err.str()); } } -void FBX::Property::DumpAscii(Assimp::StreamWriterLE &outstream, int indent) -{ +void FBXExportProperty::DumpAscii(Assimp::StreamWriterLE& outstream, int indent) { std::ostringstream ss; ss.imbue(std::locale::classic()); ss.precision(15); // this seems to match official FBX SDK exports @@ -240,8 +263,7 @@ void FBX::Property::DumpAscii(Assimp::StreamWriterLE &outstream, int indent) outstream.PutString(ss.str()); } -void FBX::Property::DumpAscii(std::ostream& s, int indent) -{ +void FBXExportProperty::DumpAscii(std::ostream& s, int indent) { // no writing type... or anything. just shove it into the stream. uint8_t* d = data.data(); size_t N; @@ -360,5 +382,8 @@ void FBX::Property::DumpAscii(std::ostream& s, int indent) } } +} // Namespace FBX +} // Namespace Assimp + #endif // ASSIMP_BUILD_NO_FBX_EXPORTER #endif // ASSIMP_BUILD_NO_EXPORT diff --git a/thirdparty/assimp/code/FBXExportProperty.h b/thirdparty/assimp/code/FBX/FBXExportProperty.h index 9c9d37c362..d692fe6ee3 100644 --- a/thirdparty/assimp/code/FBXExportProperty.h +++ b/thirdparty/assimp/code/FBX/FBXExportProperty.h @@ -47,7 +47,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef ASSIMP_BUILD_NO_FBX_EXPORTER - #include <assimp/types.h> // aiMatrix4x4 #include <assimp/StreamWriter.h> // StreamWriterLE @@ -56,11 +55,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <ostream> #include <type_traits> // is_void +namespace Assimp { namespace FBX { - class Property; -} -/** FBX::Property +/** @brief FBX::Property * * Holds a value of any of FBX's recognized types, * each represented by a particular one-character code. @@ -78,35 +76,34 @@ namespace FBX { * S : string (array of 1-byte char) * R : raw data (array of bytes) */ -class FBX::Property -{ +class FBXExportProperty { public: // constructors for basic types. // all explicit to avoid accidental typecasting - explicit Property(bool v); + explicit FBXExportProperty(bool v); // TODO: determine if there is actually a byte type, // or if this always means <bool>. 'C' seems to imply <char>, // so possibly the above was intended to represent both. - explicit Property(int16_t v); - explicit Property(int32_t v); - explicit Property(float v); - explicit Property(double v); - explicit Property(int64_t v); + explicit FBXExportProperty(int16_t v); + explicit FBXExportProperty(int32_t v); + explicit FBXExportProperty(float v); + explicit FBXExportProperty(double v); + explicit FBXExportProperty(int64_t v); // strings can either be stored as 'R' (raw) or 'S' (string) type - explicit Property(const char* c, bool raw=false); - explicit Property(const std::string& s, bool raw=false); - explicit Property(const std::vector<uint8_t>& r); - explicit Property(const std::vector<int32_t>& va); - explicit Property(const std::vector<int64_t>& va); - explicit Property(const std::vector<double>& va); - explicit Property(const std::vector<float>& va); - explicit Property(const aiMatrix4x4& vm); + explicit FBXExportProperty(const char* c, bool raw = false); + explicit FBXExportProperty(const std::string& s, bool raw = false); + explicit FBXExportProperty(const std::vector<uint8_t>& r); + explicit FBXExportProperty(const std::vector<int32_t>& va); + explicit FBXExportProperty(const std::vector<int64_t>& va); + explicit FBXExportProperty(const std::vector<double>& va); + explicit FBXExportProperty(const std::vector<float>& va); + explicit FBXExportProperty(const aiMatrix4x4& vm); // this will catch any type not defined above, // so that we don't accidentally convert something we don't want. // for example (const char*) --> (bool)... seriously wtf C++ template <class T> - explicit Property(T v) : type('X') { + explicit FBXExportProperty(T v) : type('X') { static_assert(std::is_void<T>::value, "TRIED TO CREATE FBX PROPERTY WITH UNSUPPORTED TYPE, CHECK YOUR PROPERTY INSTANTIATION"); } // note: no line wrap so it appears verbatim on the compiler error @@ -114,9 +111,9 @@ public: size_t size(); // write this property node as binary data to the given stream - void DumpBinary(Assimp::StreamWriterLE &s); - void DumpAscii(Assimp::StreamWriterLE &s, int indent=0); - void DumpAscii(std::ostream &s, int indent=0); + void DumpBinary(Assimp::StreamWriterLE& s); + void DumpAscii(Assimp::StreamWriterLE& s, int indent = 0); + void DumpAscii(std::ostream& s, int indent = 0); // note: make sure the ostream is in classic "C" locale private: @@ -124,6 +121,9 @@ private: std::vector<uint8_t> data; }; +} // Namespace FBX +} // Namespace Assimp + #endif // ASSIMP_BUILD_NO_FBX_EXPORTER #endif // AI_FBXEXPORTPROPERTY_H_INC diff --git a/thirdparty/assimp/code/FBXExporter.cpp b/thirdparty/assimp/code/FBX/FBXExporter.cpp index acb1227144..153e676506 100644 --- a/thirdparty/assimp/code/FBXExporter.cpp +++ b/thirdparty/assimp/code/FBX/FBXExporter.cpp @@ -45,6 +45,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "FBXExportNode.h" #include "FBXExportProperty.h" #include "FBXCommon.h" +#include "FBXUtil.h" #include <assimp/version.h> // aiGetVersion #include <assimp/IOSystem.hpp> @@ -73,7 +74,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. const ai_real DEG = ai_real( 57.29577951308232087679815481 ); // degrees per radian +using namespace Assimp; +using namespace Assimp::FBX; + // some constants that we'll use for writing metadata +namespace Assimp { namespace FBX { const std::string EXPORT_VERSION_STR = "7.4.0"; const uint32_t EXPORT_VERSION_INT = 7400; // 7.4 == 2014/2015 @@ -92,11 +97,6 @@ namespace FBX { ";------------------------------------------------------------------"; } -using namespace Assimp; -using namespace FBX; - -namespace Assimp { - // --------------------------------------------------------------------- // Worker function for exporting a scene to binary FBX. // Prototyped and registered in Exporter.cpp @@ -121,6 +121,7 @@ namespace Assimp { IOSystem* pIOSystem, const aiScene* pScene, const ExportProperties* pProperties + ){ // initialize the exporter FBXExporter exporter(pScene, pProperties); @@ -1218,6 +1219,16 @@ void FBXExporter::WriteObjects () layer.AddChild(le); layer.Dump(outstream, binary, indent); + for(unsigned int lr = 1; lr < m->GetNumUVChannels(); ++ lr) + { + FBX::Node layerExtra("Layer", int32_t(1)); + layerExtra.AddChild("Version", int32_t(100)); + FBX::Node leExtra("LayerElement"); + leExtra.AddChild("Type", "LayerElementUV"); + leExtra.AddChild("TypedIndex", int32_t(lr)); + layerExtra.AddChild(leExtra); + layerExtra.Dump(outstream, binary, indent); + } // finish the node record indent = 1; n.End(outstream, binary, indent, true); @@ -1393,10 +1404,6 @@ void FBXExporter::WriteObjects () // FbxVideo - stores images used by textures. for (const auto &it : uid_by_image) { - if (it.first.compare(0, 1, "*") == 0) { - // TODO: embedded textures - continue; - } FBX::Node n("Video"); const int64_t& uid = it.second; const std::string name = ""; // TODO: ... name??? @@ -1406,7 +1413,33 @@ void FBXExporter::WriteObjects () // TODO: get full path... relative path... etc... ugh... // for now just use the same path for everything, // and hopefully one of them will work out. - const std::string& path = it.first; + std::string path = it.first; + // try get embedded texture + const aiTexture* embedded_texture = mScene->GetEmbeddedTexture(it.first.c_str()); + if (embedded_texture != nullptr) { + // change the path (use original filename, if available. If name is empty, concatenate texture index with file extension) + std::stringstream newPath; + if (embedded_texture->mFilename.length > 0) { + newPath << embedded_texture->mFilename.C_Str(); + } else if (embedded_texture->achFormatHint[0]) { + int texture_index = std::stoi(path.substr(1, path.size() - 1)); + newPath << texture_index << "." << embedded_texture->achFormatHint; + } + path = newPath.str(); + // embed the texture + size_t texture_size = static_cast<size_t>(embedded_texture->mWidth * std::max(embedded_texture->mHeight, 1u)); + if (binary) { + // embed texture as binary data + std::vector<uint8_t> tex_data; + tex_data.resize(texture_size); + memcpy(&tex_data[0], (char*)embedded_texture->pcData, texture_size); + n.AddChild("Content", tex_data); + } else { + // embed texture in base64 encoding + std::string encoded_texture = FBX::Util::EncodeBase64((char*)embedded_texture->pcData, texture_size); + n.AddChild("Content", encoded_texture); + } + } p.AddP70("Path", "KString", "XRefUrl", "", path); n.AddChild(p); n.AddChild("UseMipMap", int32_t(0)); @@ -1419,17 +1452,17 @@ void FBXExporter::WriteObjects () // referenced by material_index/texture_type pairs. std::map<std::pair<size_t,size_t>,int64_t> texture_uids; const std::map<aiTextureType,std::string> prop_name_by_tt = { - {aiTextureType_DIFFUSE, "DiffuseColor"}, - {aiTextureType_SPECULAR, "SpecularColor"}, - {aiTextureType_AMBIENT, "AmbientColor"}, - {aiTextureType_EMISSIVE, "EmissiveColor"}, - {aiTextureType_HEIGHT, "Bump"}, - {aiTextureType_NORMALS, "NormalMap"}, - {aiTextureType_SHININESS, "ShininessExponent"}, - {aiTextureType_OPACITY, "TransparentColor"}, + {aiTextureType_DIFFUSE, "DiffuseColor"}, + {aiTextureType_SPECULAR, "SpecularColor"}, + {aiTextureType_AMBIENT, "AmbientColor"}, + {aiTextureType_EMISSIVE, "EmissiveColor"}, + {aiTextureType_HEIGHT, "Bump"}, + {aiTextureType_NORMALS, "NormalMap"}, + {aiTextureType_SHININESS, "ShininessExponent"}, + {aiTextureType_OPACITY, "TransparentColor"}, {aiTextureType_DISPLACEMENT, "DisplacementColor"}, //{aiTextureType_LIGHTMAP, "???"}, - {aiTextureType_REFLECTION, "ReflectionColor"} + {aiTextureType_REFLECTION, "ReflectionColor"} //{aiTextureType_UNKNOWN, ""} }; for (size_t i = 0; i < mScene->mNumMaterials; ++i) { @@ -1575,19 +1608,41 @@ void FBXExporter::WriteObjects () // one sticky point is that the number of vertices may not match, // because assimp splits vertices by normal, uv, etc. + // functor for aiNode sorting + struct SortNodeByName + { + bool operator()(const aiNode *lhs, const aiNode *rhs) const + { + return strcmp(lhs->mName.C_Str(), rhs->mName.C_Str()) < 0; + } + }; + // first we should mark the skeleton for each mesh. // the skeleton must include not only the aiBones, // but also all their parent nodes. // anything that affects the position of any bone node must be included. - std::vector<std::set<const aiNode*>> skeleton_by_mesh(mScene->mNumMeshes); + // Use SorNodeByName to make sure the exported result will be the same across all systems + // Otherwise the aiNodes of the skeleton would be sorted based on the pointer address, which isn't consistent + std::vector<std::set<const aiNode*, SortNodeByName>> skeleton_by_mesh(mScene->mNumMeshes); // at the same time we can build a list of all the skeleton nodes, // which will be used later to mark them as type "limbNode". std::unordered_set<const aiNode*> limbnodes; + + //actual bone nodes in fbx, without parenting-up + std::unordered_set<std::string> setAllBoneNamesInScene; + for(unsigned int m = 0; m < mScene->mNumMeshes; ++ m) + { + aiMesh* pMesh = mScene->mMeshes[m]; + for(unsigned int b = 0; b < pMesh->mNumBones; ++ b) + setAllBoneNamesInScene.insert(pMesh->mBones[b]->mName.data); + } + aiMatrix4x4 mxTransIdentity; + // and a map of nodes by bone name, as finding them is annoying. std::map<std::string,aiNode*> node_by_bone; for (size_t mi = 0; mi < mScene->mNumMeshes; ++mi) { const aiMesh* m = mScene->mMeshes[mi]; - std::set<const aiNode*> skeleton; + std::set<const aiNode*, SortNodeByName> skeleton; for (size_t bi =0; bi < m->mNumBones; ++bi) { const aiBone* b = m->mBones[bi]; const std::string name(b->mName.C_Str()); @@ -1626,6 +1681,11 @@ void FBXExporter::WriteObjects () if (node_name.find(MAGIC_NODE_TAG) != std::string::npos) { continue; } + //not a bone in scene && no effect in transform + if(setAllBoneNamesInScene.find(node_name)==setAllBoneNamesInScene.end() + && parent->mTransformation == mxTransIdentity) { + continue; + } // otherwise check if this is the root of the skeleton bool end = false; // is the mesh part of this node? @@ -1728,7 +1788,7 @@ void FBXExporter::WriteObjects () aiMatrix4x4 mesh_xform = get_world_transform(mesh_node, mScene); // now make a subdeformer for each bone in the skeleton - const std::set<const aiNode*> &skeleton = skeleton_by_mesh[mi]; + const std::set<const aiNode*, SortNodeByName> skeleton= skeleton_by_mesh[mi]; for (const aiNode* bone_node : skeleton) { // if there's a bone for this node, find it const aiBone* b = nullptr; @@ -1790,7 +1850,10 @@ void FBXExporter::WriteObjects () // this should be the same as the bone's mOffsetMatrix. // if it's not the same, the skeleton isn't in the bind pose. - const float epsilon = 1e-4f; // some error is to be expected + float epsilon = 1e-4f; // some error is to be expected + float epsilon_custom = mProperties->GetPropertyFloat("BINDPOSE_EPSILON", -1); + if(epsilon_custom > 0) + epsilon = epsilon_custom; bool bone_xform_okay = true; if (b && ! tr.Equal(b->mOffsetMatrix, epsilon)) { not_in_bind_pose.insert(b); @@ -2237,8 +2300,8 @@ void FBXExporter::WriteModelNode( // not sure what these are for, // but they seem to be omnipresent - m.AddChild("Shading", Property(true)); - m.AddChild("Culling", Property("CullingOff")); + m.AddChild("Shading", FBXExportProperty(true)); + m.AddChild("Culling", FBXExportProperty("CullingOff")); m.Dump(outstream, binary, 1); } @@ -2351,7 +2414,7 @@ void FBXExporter::WriteModelNodes( na.AddProperties( node_attribute_uid, FBX::SEPARATOR + "NodeAttribute", "LimbNode" ); - na.AddChild("TypeFlags", Property("Skeleton")); + na.AddChild("TypeFlags", FBXExportProperty("Skeleton")); na.Dump(outstream, binary, 1); // and connect them connections.emplace_back("C", "OO", node_attribute_uid, node_uid); diff --git a/thirdparty/assimp/code/FBXExporter.h b/thirdparty/assimp/code/FBX/FBXExporter.h index 71fb55c57f..71fb55c57f 100644 --- a/thirdparty/assimp/code/FBXExporter.h +++ b/thirdparty/assimp/code/FBX/FBXExporter.h diff --git a/thirdparty/assimp/code/FBXImportSettings.h b/thirdparty/assimp/code/FBX/FBXImportSettings.h index d5e1c20608..1a4c80f8b2 100644 --- a/thirdparty/assimp/code/FBXImportSettings.h +++ b/thirdparty/assimp/code/FBX/FBXImportSettings.h @@ -53,19 +53,22 @@ namespace FBX { struct ImportSettings { ImportSettings() - : strictMode(true) - , readAllLayers(true) - , readAllMaterials(false) - , readMaterials(true) - , readTextures(true) - , readCameras(true) - , readLights(true) - , readAnimations(true) - , readWeights(true) - , preservePivots(true) - , optimizeEmptyAnimationCurves(true) - , useLegacyEmbeddedTextureNaming(false) - {} + : strictMode(true) + , readAllLayers(true) + , readAllMaterials(false) + , readMaterials(true) + , readTextures(true) + , readCameras(true) + , readLights(true) + , readAnimations(true) + , readWeights(true) + , preservePivots(true) + , optimizeEmptyAnimationCurves(true) + , useLegacyEmbeddedTextureNaming(false) + , removeEmptyBones( true ) + , convertToMeters( false ) { + // empty + } /** enable strict mode: @@ -141,8 +144,16 @@ struct ImportSettings bool optimizeEmptyAnimationCurves; /** use legacy naming for embedded textures eg: (*0, *1, *2) - **/ + */ bool useLegacyEmbeddedTextureNaming; + + /** Empty bones shall be removed + */ + bool removeEmptyBones; + + /** Set to true to perform a conversion from cm to meter after the import + */ + bool convertToMeters; }; diff --git a/thirdparty/assimp/code/FBXImporter.cpp b/thirdparty/assimp/code/FBX/FBXImporter.cpp index 2cc8bffc29..ec8bbd2b47 100644 --- a/thirdparty/assimp/code/FBXImporter.cpp +++ b/thirdparty/assimp/code/FBX/FBXImporter.cpp @@ -140,6 +140,8 @@ void FBXImporter::SetupProperties(const Importer* pImp) settings.preservePivots = pImp->GetPropertyBool(AI_CONFIG_IMPORT_FBX_PRESERVE_PIVOTS, true); settings.optimizeEmptyAnimationCurves = pImp->GetPropertyBool(AI_CONFIG_IMPORT_FBX_OPTIMIZE_EMPTY_ANIMATION_CURVES, true); settings.useLegacyEmbeddedTextureNaming = pImp->GetPropertyBool(AI_CONFIG_IMPORT_FBX_EMBEDDED_TEXTURES_LEGACY_NAMING, false); + settings.removeEmptyBones = pImp->GetPropertyBool(AI_CONFIG_IMPORT_REMOVE_EMPTY_BONES, true); + settings.convertToMeters = pImp->GetPropertyBool(AI_CONFIG_FBX_CONVERT_TO_M, false); } // ------------------------------------------------------------------------------------------------ @@ -170,7 +172,7 @@ void FBXImporter::InternReadFile( const std::string& pFile, aiScene* pScene, IOS bool is_binary = false; if (!strncmp(begin,"Kaydara FBX Binary",18)) { is_binary = true; - TokenizeBinary(tokens,begin,static_cast<unsigned int>(contents.size())); + TokenizeBinary(tokens,begin,contents.size()); } else { Tokenize(tokens,begin); @@ -183,8 +185,12 @@ void FBXImporter::InternReadFile( const std::string& pFile, aiScene* pScene, IOS // take the raw parse-tree and convert it to a FBX DOM Document doc(parser,settings); + FbxUnit unit(FbxUnit::cm); + if (settings.convertToMeters) { + unit = FbxUnit::m; + } // convert the FBX DOM to aiScene - ConvertToAssimpScene(pScene,doc); + ConvertToAssimpScene(pScene,doc, settings.removeEmptyBones, unit); std::for_each(tokens.begin(),tokens.end(),Util::delete_fun<Token>()); } diff --git a/thirdparty/assimp/code/FBXImporter.h b/thirdparty/assimp/code/FBX/FBXImporter.h index c365b2cddf..c365b2cddf 100644 --- a/thirdparty/assimp/code/FBXImporter.h +++ b/thirdparty/assimp/code/FBX/FBXImporter.h diff --git a/thirdparty/assimp/code/FBXMaterial.cpp b/thirdparty/assimp/code/FBX/FBXMaterial.cpp index f16f134404..f43a8b84b0 100644 --- a/thirdparty/assimp/code/FBXMaterial.cpp +++ b/thirdparty/assimp/code/FBX/FBXMaterial.cpp @@ -316,7 +316,7 @@ Video::Video(uint64_t id, const Element& element, const Document& doc, const std relativeFileName = ParseTokenAsString(GetRequiredToken(*RelativeFilename,0)); } - if(Content) { + if(Content && !Content->Tokens().empty()) { //this field is omitted when the embedded texture is already loaded, let's ignore if it's not found try { const Token& token = GetRequiredToken(*Content, 0); @@ -326,16 +326,40 @@ Video::Video(uint64_t id, const Element& element, const Document& doc, const std DOMError("embedded content is not surrounded by quotation marks", &element); } else { - const char* encodedData = data + 1; - size_t encodedDataLen = static_cast<size_t>(token.end() - token.begin()); - // search for last quotation mark - while (encodedDataLen > 1 && encodedData[encodedDataLen] != '"') - encodedDataLen--; - if (encodedDataLen % 4 != 0) { - DOMError("embedded content is invalid, needs to be in base64", &element); + size_t targetLength = 0; + auto numTokens = Content->Tokens().size(); + // First time compute size (it could be large like 64Gb and it is good to allocate it once) + for (uint32_t tokenIdx = 0; tokenIdx < numTokens; ++tokenIdx) + { + const Token& dataToken = GetRequiredToken(*Content, tokenIdx); + size_t tokenLength = dataToken.end() - dataToken.begin() - 2; // ignore double quotes + const char* base64data = dataToken.begin() + 1; + const size_t outLength = Util::ComputeDecodedSizeBase64(base64data, tokenLength); + if (outLength == 0) + { + DOMError("Corrupted embedded content found", &element); + } + targetLength += outLength; } - else { - contentLength = Util::DecodeBase64(encodedData, encodedDataLen, content); + if (targetLength == 0) + { + DOMError("Corrupted embedded content found", &element); + } + content = new uint8_t[targetLength]; + contentLength = static_cast<uint64_t>(targetLength); + size_t dst_offset = 0; + for (uint32_t tokenIdx = 0; tokenIdx < numTokens; ++tokenIdx) + { + const Token& dataToken = GetRequiredToken(*Content, tokenIdx); + size_t tokenLength = dataToken.end() - dataToken.begin() - 2; // ignore double quotes + const char* base64data = dataToken.begin() + 1; + dst_offset += Util::DecodeBase64(base64data, tokenLength, content + dst_offset, targetLength - dst_offset); + } + if (targetLength != dst_offset) + { + delete[] content; + contentLength = 0; + DOMError("Corrupted embedded content found", &element); } } } diff --git a/thirdparty/assimp/code/FBXMeshGeometry.cpp b/thirdparty/assimp/code/FBX/FBXMeshGeometry.cpp index d75476b826..44a0264ca0 100644 --- a/thirdparty/assimp/code/FBXMeshGeometry.cpp +++ b/thirdparty/assimp/code/FBX/FBXMeshGeometry.cpp @@ -568,15 +568,15 @@ void MeshGeometry::ReadVertexDataColors(std::vector<aiColor4D>& colors_out, cons } // ------------------------------------------------------------------------------------------------ -static const std::string TangentIndexToken = "TangentIndex"; -static const std::string TangentsIndexToken = "TangentsIndex"; +static const char *TangentIndexToken = "TangentIndex"; +static const char *TangentsIndexToken = "TangentsIndex"; void MeshGeometry::ReadVertexDataTangents(std::vector<aiVector3D>& tangents_out, const Scope& source, const std::string& MappingInformationType, const std::string& ReferenceInformationType) { const char * str = source.Elements().count( "Tangents" ) > 0 ? "Tangents" : "Tangent"; - const char * strIdx = source.Elements().count( "Tangents" ) > 0 ? TangentsIndexToken.c_str() : TangentIndexToken.c_str(); + const char * strIdx = source.Elements().count( "Tangents" ) > 0 ? TangentsIndexToken : TangentIndexToken; ResolveVertexDataArray(tangents_out,source,MappingInformationType,ReferenceInformationType, str, strIdx, @@ -630,10 +630,11 @@ void MeshGeometry::ReadVertexDataMaterials(std::vector<int>& materials_out, cons materials_out.clear(); } - m_materials.assign(m_vertices.size(),materials_out[0]); + materials_out.resize(m_vertices.size()); + std::fill(materials_out.begin(), materials_out.end(), materials_out.at(0)); } else if (MappingInformationType == "ByPolygon" && ReferenceInformationType == "IndexToDirect") { - m_materials.resize(face_count); + materials_out.resize(face_count); if(materials_out.size() != face_count) { FBXImporter::LogError(Formatter::format("length of input data unexpected for ByPolygon mapping: ") diff --git a/thirdparty/assimp/code/FBXMeshGeometry.h b/thirdparty/assimp/code/FBX/FBXMeshGeometry.h index d6d4512177..d6d4512177 100644 --- a/thirdparty/assimp/code/FBXMeshGeometry.h +++ b/thirdparty/assimp/code/FBX/FBXMeshGeometry.h diff --git a/thirdparty/assimp/code/FBXModel.cpp b/thirdparty/assimp/code/FBX/FBXModel.cpp index 589af36ac7..589af36ac7 100644 --- a/thirdparty/assimp/code/FBXModel.cpp +++ b/thirdparty/assimp/code/FBX/FBXModel.cpp diff --git a/thirdparty/assimp/code/FBXNodeAttribute.cpp b/thirdparty/assimp/code/FBX/FBXNodeAttribute.cpp index b72e5637ee..b72e5637ee 100644 --- a/thirdparty/assimp/code/FBXNodeAttribute.cpp +++ b/thirdparty/assimp/code/FBX/FBXNodeAttribute.cpp diff --git a/thirdparty/assimp/code/FBXParser.cpp b/thirdparty/assimp/code/FBX/FBXParser.cpp index b255c47347..4a9346040d 100644 --- a/thirdparty/assimp/code/FBXParser.cpp +++ b/thirdparty/assimp/code/FBX/FBXParser.cpp @@ -117,7 +117,7 @@ namespace FBX { Element::Element(const Token& key_token, Parser& parser) : key_token(key_token) { - TokenPtr n = NULL; + TokenPtr n = nullptr; do { n = parser.AdvanceToNextToken(); if(!n) { @@ -643,9 +643,9 @@ void ParseVectorDataArray(std::vector<aiVector3D>& out, const Element& el) if (type == 'd') { const double* d = reinterpret_cast<const double*>(&buff[0]); for (unsigned int i = 0; i < count3; ++i, d += 3) { - out.push_back(aiVector3D(static_cast<float>(d[0]), - static_cast<float>(d[1]), - static_cast<float>(d[2]))); + out.push_back(aiVector3D(static_cast<ai_real>(d[0]), + static_cast<ai_real>(d[1]), + static_cast<ai_real>(d[2]))); } // for debugging /*for ( size_t i = 0; i < out.size(); i++ ) { @@ -963,7 +963,6 @@ void ParseVectorDataArray(std::vector<float>& out, const Element& el) } } - // ------------------------------------------------------------------------------------------------ // read an array of uints void ParseVectorDataArray(std::vector<unsigned int>& out, const Element& el) @@ -1280,7 +1279,6 @@ float ParseTokenAsFloat(const Token& t) return i; } - // ------------------------------------------------------------------------------------------------ // wrapper around ParseTokenAsInt() with ParseError handling int ParseTokenAsInt(const Token& t) @@ -1293,8 +1291,6 @@ int ParseTokenAsInt(const Token& t) return i; } - - // ------------------------------------------------------------------------------------------------ // wrapper around ParseTokenAsInt64() with ParseError handling int64_t ParseTokenAsInt64(const Token& t) diff --git a/thirdparty/assimp/code/FBXParser.h b/thirdparty/assimp/code/FBX/FBXParser.h index 7b0cf72039..7b0cf72039 100644 --- a/thirdparty/assimp/code/FBXParser.h +++ b/thirdparty/assimp/code/FBX/FBXParser.h diff --git a/thirdparty/assimp/code/FBXProperties.cpp b/thirdparty/assimp/code/FBX/FBXProperties.cpp index 8d7036b6a9..8d7036b6a9 100644 --- a/thirdparty/assimp/code/FBXProperties.cpp +++ b/thirdparty/assimp/code/FBX/FBXProperties.cpp diff --git a/thirdparty/assimp/code/FBXProperties.h b/thirdparty/assimp/code/FBX/FBXProperties.h index 58755542fc..58755542fc 100644 --- a/thirdparty/assimp/code/FBXProperties.h +++ b/thirdparty/assimp/code/FBX/FBXProperties.h diff --git a/thirdparty/assimp/code/FBXTokenizer.cpp b/thirdparty/assimp/code/FBX/FBXTokenizer.cpp index 252cce3557..252cce3557 100644 --- a/thirdparty/assimp/code/FBXTokenizer.cpp +++ b/thirdparty/assimp/code/FBX/FBXTokenizer.cpp diff --git a/thirdparty/assimp/code/FBXTokenizer.h b/thirdparty/assimp/code/FBX/FBXTokenizer.h index 2af29743f4..afa588a470 100644 --- a/thirdparty/assimp/code/FBXTokenizer.h +++ b/thirdparty/assimp/code/FBX/FBXTokenizer.h @@ -93,7 +93,7 @@ public: Token(const char* sbegin, const char* send, TokenType type, unsigned int line, unsigned int column); /** construct a binary token */ - Token(const char* sbegin, const char* send, TokenType type, unsigned int offset); + Token(const char* sbegin, const char* send, TokenType type, size_t offset); ~Token(); @@ -118,14 +118,14 @@ public: return type; } - unsigned int Offset() const { + size_t Offset() const { ai_assert(IsBinary()); return offset; } unsigned int Line() const { ai_assert(!IsBinary()); - return line; + return static_cast<unsigned int>(line); } unsigned int Column() const { @@ -147,8 +147,8 @@ private: const TokenType type; union { - const unsigned int line; - unsigned int offset; + size_t line; + size_t offset; }; const unsigned int column; }; @@ -178,7 +178,7 @@ void Tokenize(TokenList& output_tokens, const char* input); * @param input_buffer Binary input buffer to be processed. * @param length Length of input buffer, in bytes. There is no 0-terminal. * @throw DeadlyImportError if something goes wrong */ -void TokenizeBinary(TokenList& output_tokens, const char* input, unsigned int length); +void TokenizeBinary(TokenList& output_tokens, const char* input, size_t length); } // ! FBX diff --git a/thirdparty/assimp/code/FBXUtil.cpp b/thirdparty/assimp/code/FBX/FBXUtil.cpp index fb483161b2..c10e057c8c 100644 --- a/thirdparty/assimp/code/FBXUtil.cpp +++ b/thirdparty/assimp/code/FBX/FBXUtil.cpp @@ -86,7 +86,7 @@ const char* TokenTypeString(TokenType t) // ------------------------------------------------------------------------------------------------ -std::string AddOffset(const std::string& prefix, const std::string& text, unsigned int offset) +std::string AddOffset(const std::string& prefix, const std::string& text, size_t offset) { return static_cast<std::string>( (Formatter::format() << prefix << " (offset 0x" << std::hex << offset << ") " << text) ); } @@ -114,47 +114,126 @@ std::string AddTokenText(const std::string& prefix, const std::string& text, con text) ); } +// Generated by this formula: T["ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"[i]] = i; static const uint8_t base64DecodeTable[128] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 63, - 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 0, 0, 64, 0, 0, - 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 0, - 0, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, - 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 0, 0, 0, 0, 0 + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, 255, 255, 63, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, 255, 255, 255, 255, + 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 255, 255, 255, 255, 255, + 255, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 255, 255, 255, 255, 255 }; uint8_t DecodeBase64(char ch) { - return base64DecodeTable[size_t(ch)]; + const auto idx = static_cast<uint8_t>(ch); + if (idx > 127) + return 255; + return base64DecodeTable[idx]; } -size_t DecodeBase64(const char* in, size_t inLength, uint8_t*& out) +size_t ComputeDecodedSizeBase64(const char* in, size_t inLength) { - if (inLength < 4) { - out = 0; + if (inLength < 2) + { + return 0; + } + const size_t equals = size_t(in[inLength - 1] == '=') + size_t(in[inLength - 2] == '='); + const size_t full_length = (inLength * 3) >> 2; // div by 4 + if (full_length < equals) + { + return 0; + } + return full_length - equals; +} + +size_t DecodeBase64(const char* in, size_t inLength, uint8_t* out, size_t maxOutLength) +{ + if (maxOutLength == 0 || inLength < 2) { return 0; } + const size_t realLength = inLength - size_t(in[inLength - 1] == '=') - size_t(in[inLength - 2] == '='); + size_t dst_offset = 0; + int val = 0, valb = -8; + for (size_t src_offset = 0; src_offset < realLength; ++src_offset) + { + const uint8_t table_value = Util::DecodeBase64(in[src_offset]); + if (table_value == 255) + { + return 0; + } + val = (val << 6) + table_value; + valb += 6; + if (valb >= 0) + { + out[dst_offset++] = static_cast<uint8_t>((val >> valb) & 0xFF); + valb -= 8; + val &= 0xFFF; + } + } + return dst_offset; +} - const size_t outLength = (inLength * 3) / 4; - out = new uint8_t[outLength]; - memset(out, 0, outLength); +static const char to_base64_string[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +char EncodeBase64(char byte) +{ + return to_base64_string[(size_t)byte]; +} - size_t i = 0; - size_t j = 0; - for (i = 0; i < inLength - 4; i += 4) +/** Encodes a block of 4 bytes to base64 encoding +* +* @param bytes Bytes to encode. +* @param out_string String to write encoded values to. +* @param string_pos Position in out_string.*/ +void EncodeByteBlock(const char* bytes, std::string& out_string, size_t string_pos) +{ + char b0 = (bytes[0] & 0xFC) >> 2; + char b1 = (bytes[0] & 0x03) << 4 | ((bytes[1] & 0xF0) >> 4); + char b2 = (bytes[1] & 0x0F) << 2 | ((bytes[2] & 0xC0) >> 6); + char b3 = (bytes[2] & 0x3F); + + out_string[string_pos + 0] = EncodeBase64(b0); + out_string[string_pos + 1] = EncodeBase64(b1); + out_string[string_pos + 2] = EncodeBase64(b2); + out_string[string_pos + 3] = EncodeBase64(b3); +} + +std::string EncodeBase64(const char* data, size_t length) +{ + // calculate extra bytes needed to get a multiple of 3 + size_t extraBytes = 3 - length % 3; + + // number of base64 bytes + size_t encodedBytes = 4 * (length + extraBytes) / 3; + + std::string encoded_string(encodedBytes, '='); + + // read blocks of 3 bytes + for (size_t ib3 = 0; ib3 < length / 3; ib3++) { - uint8_t b0 = Util::DecodeBase64(in[i]); - uint8_t b1 = Util::DecodeBase64(in[i + 1]); - uint8_t b2 = Util::DecodeBase64(in[i + 2]); - uint8_t b3 = Util::DecodeBase64(in[i + 3]); - - out[j++] = (uint8_t)((b0 << 2) | (b1 >> 4)); - out[j++] = (uint8_t)((b1 << 4) | (b2 >> 2)); - out[j++] = (uint8_t)((b2 << 6) | b3); + const size_t iByte = ib3 * 3; + const size_t iEncodedByte = ib3 * 4; + const char* currData = &data[iByte]; + + EncodeByteBlock(currData, encoded_string, iEncodedByte); + } + + // if size of data is not a multiple of 3, also encode the final bytes (and add zeros where needed) + if (extraBytes > 0) + { + char finalBytes[4] = { 0,0,0,0 }; + memcpy(&finalBytes[0], &data[length - length % 3], length % 3); + + const size_t iEncodedByte = encodedBytes - 4; + EncodeByteBlock(&finalBytes[0], encoded_string, iEncodedByte); + + // add '=' at the end + for (size_t i = 0; i < 4 * extraBytes / 3; i++) + encoded_string[encodedBytes - i - 1] = '='; } - return outLength; + return encoded_string; } } // !Util diff --git a/thirdparty/assimp/code/FBXUtil.h b/thirdparty/assimp/code/FBX/FBXUtil.h index 6890e015ba..b634418858 100644 --- a/thirdparty/assimp/code/FBXUtil.h +++ b/thirdparty/assimp/code/FBX/FBXUtil.h @@ -78,7 +78,7 @@ const char* TokenTypeString(TokenType t); * @param line Line index, 1-based * @param column Column index, 1-based * @return A string of the following format: {prefix} (offset 0x{offset}) {text}*/ -std::string AddOffset(const std::string& prefix, const std::string& text, unsigned int offset); +std::string AddOffset(const std::string& prefix, const std::string& text, size_t offset); /** Format log/error messages using a given line location in the source file. @@ -105,13 +105,30 @@ std::string AddTokenText(const std::string& prefix, const std::string& text, con * @return decoded byte value*/ uint8_t DecodeBase64(char ch); +/** Compute decoded size of a Base64-encoded string +* +* @param in Characters to decode. +* @param inLength Number of characters to decode. +* @return size of the decoded data (number of bytes)*/ +size_t ComputeDecodedSizeBase64(const char* in, size_t inLength); + /** Decode a Base64-encoded string * * @param in Characters to decode. * @param inLength Number of characters to decode. -* @param out Reference to pointer where we will store the decoded data. +* @param out Pointer where we will store the decoded data. +* @param maxOutLength Size of output buffer. * @return size of the decoded data (number of bytes)*/ -size_t DecodeBase64(const char* in, size_t inLength, uint8_t*& out); +size_t DecodeBase64(const char* in, size_t inLength, uint8_t* out, size_t maxOutLength); + +char EncodeBase64(char byte); + +/** Encode bytes in base64-encoding +* +* @param data Binary data to encode. +* @param inLength Number of bytes to encode. +* @return base64-encoded string*/ +std::string EncodeBase64(const char* data, size_t length); } } diff --git a/thirdparty/assimp/code/FIReader.cpp b/thirdparty/assimp/code/FIReader.cpp deleted file mode 100644 index 2116316ca3..0000000000 --- a/thirdparty/assimp/code/FIReader.cpp +++ /dev/null @@ -1,1834 +0,0 @@ -/* -Open Asset Import Library (assimp) ----------------------------------------------------------------------- - -Copyright (c) 2006-2019, assimp team - - -All rights reserved. - -Redistribution and use of this software in source and binary forms, -with or without modification, are permitted provided that the -following conditions are met: - -* Redistributions of source code must retain the above -copyright notice, this list of conditions and the -following disclaimer. - -* Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the -following disclaimer in the documentation and/or other -materials provided with the distribution. - -* Neither the name of the assimp team, nor the names of its -contributors may be used to endorse or promote products -derived from this software without specific prior -written permission of the assimp team. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - ----------------------------------------------------------------------- -*/ -/// \file FIReader.cpp -/// \brief Reader for Fast Infoset encoded binary XML files. -/// \date 2017 -/// \author Patrick Daehne - -#ifndef ASSIMP_BUILD_NO_X3D_IMPORTER - -#include "FIReader.hpp" -#include <assimp/StringUtils.h> - -// Workaround for issue #1361 -// https://github.com/assimp/assimp/issues/1361 -#ifdef __ANDROID__ -# define _GLIBCXX_USE_C99 1 -#endif - -#include <assimp/Exceptional.h> -#include <assimp/IOStream.hpp> -#include <assimp/types.h> -#include <assimp/MemoryIOWrapper.h> -#include <assimp/irrXMLWrapper.h> -#include "../contrib/utf8cpp/source/utf8.h" -#include <assimp/fast_atof.h> -#include <stack> -#include <map> -#include <iostream> -#include <sstream> -#include <iomanip> - -namespace Assimp { - -static const std::string parseErrorMessage = "Fast Infoset parse error"; - -static const char *xmlDeclarations[] = { - "<?xml encoding='finf'?>", - "<?xml encoding='finf' standalone='yes'?>", - "<?xml encoding='finf' standalone='no'?>", - "<?xml version='1.0' encoding='finf'?>", - "<?xml version='1.0' encoding='finf' standalone='yes'?>", - "<?xml version='1.0' encoding='finf' standalone='no'?>", - "<?xml version='1.1' encoding='finf'?>", - "<?xml version='1.1' encoding='finf' standalone='yes'?>", - "<?xml version='1.1' encoding='finf' standalone='no'?>" -}; - -static size_t parseMagic(const uint8_t *data, const uint8_t *dataEnd) { - if (dataEnd - data < 4) { - return 0; - } - uint32_t magic = (data[0] << 24) | (data[1] << 16) | (data[2] << 8) | data[3]; - switch (magic) { - case 0xe0000001: - return 4; - case 0x3c3f786d: // "<?xm" - { - size_t xmlDeclarationsLength = sizeof(xmlDeclarations) / sizeof(xmlDeclarations[0]); - for (size_t i = 0; i < xmlDeclarationsLength; ++i) { - auto xmlDeclaration = xmlDeclarations[i]; - ptrdiff_t xmlDeclarationLength = strlen(xmlDeclaration); - if ((dataEnd - data >= xmlDeclarationLength) && (memcmp(xmlDeclaration, data, xmlDeclarationLength) == 0)) { - data += xmlDeclarationLength; - if (dataEnd - data < 4) { - return 0; - } - magic = (data[0] << 24) | (data[1] << 16) | (data[2] << 8) | data[3]; - return magic == 0xe0000001 ? xmlDeclarationLength + 4 : 0; - } - } - return 0; - } - default: - return 0; - } -} - -static std::string parseUTF8String(const uint8_t *data, size_t len) { - return std::string((char*)data, len); -} - -static std::string parseUTF16String(const uint8_t *data, size_t len) { - if (len & 1) { - throw DeadlyImportError(parseErrorMessage); - } - size_t numShorts = len / 2; - std::vector<short> utf16; - utf16.reserve(numShorts); - for (size_t i = 0; i < numShorts; ++i) { - short v = (data[0] << 8) | data[1]; - utf16.push_back(v); - data += 2; - } - std::string result; - utf8::utf16to8(utf16.begin(), utf16.end(), back_inserter(result)); - return result; -} - -struct FIStringValueImpl: public FIStringValue { - inline FIStringValueImpl(std::string &&value_) { value = std::move(value_); } - virtual const std::string &toString() const /*override*/ { return value; } -}; - -std::shared_ptr<FIStringValue> FIStringValue::create(std::string &&value) { - return std::make_shared<FIStringValueImpl>(std::move(value)); -} - -struct FIHexValueImpl: public FIHexValue { - mutable std::string strValue; - mutable bool strValueValid; - inline FIHexValueImpl(std::vector<uint8_t> &&value_): strValueValid(false) { value = std::move(value_); } - virtual const std::string &toString() const /*override*/ { - if (!strValueValid) { - strValueValid = true; - std::ostringstream os; - os << std::hex << std::uppercase << std::setfill('0'); - std::for_each(value.begin(), value.end(), [&](uint8_t c) { os << std::setw(2) << static_cast<int>(c); }); - strValue = os.str(); - } - return strValue; - }; -}; - -std::shared_ptr<FIHexValue> FIHexValue::create(std::vector<uint8_t> &&value) { - return std::make_shared<FIHexValueImpl>(std::move(value)); -} - -struct FIBase64ValueImpl: public FIBase64Value { - mutable std::string strValue; - mutable bool strValueValid; - inline FIBase64ValueImpl(std::vector<uint8_t> &&value_): strValueValid(false) { value = std::move(value_); } - virtual const std::string &toString() const /*override*/ { - if (!strValueValid) { - strValueValid = true; - std::ostringstream os; - uint8_t c1 = 0, c2; - int imod3 = 0; - std::vector<uint8_t>::size_type valueSize = value.size(); - for (std::vector<uint8_t>::size_type i = 0; i < valueSize; ++i) { - c2 = value[i]; - switch (imod3) { - case 0: - os << basis_64[c2 >> 2]; - imod3 = 1; - break; - case 1: - os << basis_64[((c1 & 0x03) << 4) | ((c2 & 0xf0) >> 4)]; - imod3 = 2; - break; - case 2: - os << basis_64[((c1 & 0x0f) << 2) | ((c2 & 0xc0) >> 6)] << basis_64[c2 & 0x3f]; - imod3 = 0; - break; - } - c1 = c2; - } - switch (imod3) { - case 1: - os << basis_64[(c1 & 0x03) << 4] << "=="; - break; - case 2: - os << basis_64[(c1 & 0x0f) << 2] << '='; - break; - } - strValue = os.str(); - } - return strValue; - }; - static const char basis_64[]; -}; - -const char FIBase64ValueImpl::basis_64[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - -std::shared_ptr<FIBase64Value> FIBase64Value::create(std::vector<uint8_t> &&value) { - return std::make_shared<FIBase64ValueImpl>(std::move(value)); -} - -struct FIShortValueImpl: public FIShortValue { - mutable std::string strValue; - mutable bool strValueValid; - inline FIShortValueImpl(std::vector<int16_t> &&value_): strValueValid(false) { value = std::move(value_); } - virtual const std::string &toString() const /*override*/ { - if (!strValueValid) { - strValueValid = true; - std::ostringstream os; - int n = 0; - std::for_each(value.begin(), value.end(), [&](int16_t s) { if (++n > 1) os << ' '; os << s; }); - strValue = os.str(); - } - return strValue; - } -}; - -std::shared_ptr<FIShortValue> FIShortValue::create(std::vector<int16_t> &&value) { - return std::make_shared<FIShortValueImpl>(std::move(value)); -} - -struct FIIntValueImpl: public FIIntValue { - mutable std::string strValue; - mutable bool strValueValid; - inline FIIntValueImpl(std::vector<int32_t> &&value_): strValueValid(false) { value = std::move(value_); } - virtual const std::string &toString() const /*override*/ { - if (!strValueValid) { - strValueValid = true; - std::ostringstream os; - int n = 0; - std::for_each(value.begin(), value.end(), [&](int32_t i) { if (++n > 1) os << ' '; os << i; }); - strValue = os.str(); - } - return strValue; - }; -}; - -std::shared_ptr<FIIntValue> FIIntValue::create(std::vector<int32_t> &&value) { - return std::make_shared<FIIntValueImpl>(std::move(value)); -} - -struct FILongValueImpl: public FILongValue { - mutable std::string strValue; - mutable bool strValueValid; - inline FILongValueImpl(std::vector<int64_t> &&value_): strValueValid(false) { value = std::move(value_); } - virtual const std::string &toString() const /*override*/ { - if (!strValueValid) { - strValueValid = true; - std::ostringstream os; - int n = 0; - std::for_each(value.begin(), value.end(), [&](int64_t l) { if (++n > 1) os << ' '; os << l; }); - strValue = os.str(); - } - return strValue; - }; -}; - -std::shared_ptr<FILongValue> FILongValue::create(std::vector<int64_t> &&value) { - return std::make_shared<FILongValueImpl>(std::move(value)); -} - -struct FIBoolValueImpl: public FIBoolValue { - mutable std::string strValue; - mutable bool strValueValid; - inline FIBoolValueImpl(std::vector<bool> &&value_): strValueValid(false) { value = std::move(value_); } - virtual const std::string &toString() const /*override*/ { - if (!strValueValid) { - strValueValid = true; - std::ostringstream os; - os << std::boolalpha; - int n = 0; - std::for_each(value.begin(), value.end(), [&](bool b) { if (++n > 1) os << ' '; os << b; }); - strValue = os.str(); - } - return strValue; - }; -}; - -std::shared_ptr<FIBoolValue> FIBoolValue::create(std::vector<bool> &&value) { - return std::make_shared<FIBoolValueImpl>(std::move(value)); -} - -struct FIFloatValueImpl: public FIFloatValue { - mutable std::string strValue; - mutable bool strValueValid; - inline FIFloatValueImpl(std::vector<float> &&value_): strValueValid(false) { value = std::move(value_); } - virtual const std::string &toString() const /*override*/ { - if (!strValueValid) { - strValueValid = true; - std::ostringstream os; - int n = 0; - std::for_each(value.begin(), value.end(), [&](float f) { if (++n > 1) os << ' '; os << f; }); - strValue = os.str(); - } - return strValue; - } -}; - -std::shared_ptr<FIFloatValue> FIFloatValue::create(std::vector<float> &&value) { - return std::make_shared<FIFloatValueImpl>(std::move(value)); -} - -struct FIDoubleValueImpl: public FIDoubleValue { - mutable std::string strValue; - mutable bool strValueValid; - inline FIDoubleValueImpl(std::vector<double> &&value_): strValueValid(false) { value = std::move(value_); } - virtual const std::string &toString() const /*override*/ { - if (!strValueValid) { - strValueValid = true; - std::ostringstream os; - int n = 0; - std::for_each(value.begin(), value.end(), [&](double d) { if (++n > 1) os << ' '; os << d; }); - strValue = os.str(); - } - return strValue; - } -}; - -std::shared_ptr<FIDoubleValue> FIDoubleValue::create(std::vector<double> &&value) { - return std::make_shared<FIDoubleValueImpl>(std::move(value)); -} - -struct FIUUIDValueImpl: public FIUUIDValue { - mutable std::string strValue; - mutable bool strValueValid; - inline FIUUIDValueImpl(std::vector<uint8_t> &&value_): strValueValid(false) { value = std::move(value_); } - virtual const std::string &toString() const /*override*/ { - if (!strValueValid) { - strValueValid = true; - std::ostringstream os; - os << std::hex << std::uppercase << std::setfill('0'); - std::vector<uint8_t>::size_type valueSize = value.size(); - for (std::vector<uint8_t>::size_type i = 0; i < valueSize; ++i) { - switch (i & 15) { - case 0: - if (i > 0) { - os << ' '; - } - os << std::setw(2) << static_cast<int>(value[i]); - break; - case 4: - case 6: - case 8: - case 10: - os << '-'; - // intentionally fall through! - case 1: - case 2: - case 3: - case 5: - case 7: - case 9: - case 11: - case 12: - case 13: - case 14: - case 15: - os << std::setw(2) << static_cast<int>(value[i]); - break; - } - } - strValue = os.str(); - } - return strValue; - }; -}; - -std::shared_ptr<FIUUIDValue> FIUUIDValue::create(std::vector<uint8_t> &&value) { - return std::make_shared<FIUUIDValueImpl>(std::move(value)); -} - -struct FICDATAValueImpl: public FICDATAValue { - inline FICDATAValueImpl(std::string &&value_) { value = std::move(value_); } - virtual const std::string &toString() const /*override*/ { return value; } -}; - -std::shared_ptr<FICDATAValue> FICDATAValue::create(std::string &&value) { - return std::make_shared<FICDATAValueImpl>(std::move(value)); -} - -struct FIHexDecoder: public FIDecoder { - virtual std::shared_ptr<const FIValue> decode(const uint8_t *data, size_t len) /*override*/ { - return FIHexValue::create(std::vector<uint8_t>(data, data + len)); - } -}; - -struct FIBase64Decoder: public FIDecoder { - virtual std::shared_ptr<const FIValue> decode(const uint8_t *data, size_t len) /*override*/ { - return FIBase64Value::create(std::vector<uint8_t>(data, data + len)); - } -}; - -struct FIShortDecoder: public FIDecoder { - virtual std::shared_ptr<const FIValue> decode(const uint8_t *data, size_t len) /*override*/ { - if (len & 1) { - throw DeadlyImportError(parseErrorMessage); - } - std::vector<int16_t> value; - size_t numShorts = len / 2; - value.reserve(numShorts); - for (size_t i = 0; i < numShorts; ++i) { - int16_t v = (data[0] << 8) | data[1]; - value.push_back(v); - data += 2; - } - return FIShortValue::create(std::move(value)); - } -}; - -struct FIIntDecoder: public FIDecoder { - virtual std::shared_ptr<const FIValue> decode(const uint8_t *data, size_t len) /*override*/ { - if (len & 3) { - throw DeadlyImportError(parseErrorMessage); - } - std::vector<int32_t> value; - size_t numInts = len / 4; - value.reserve(numInts); - for (size_t i = 0; i < numInts; ++i) { - int32_t v = (data[0] << 24) | (data[1] << 16) | (data[2] << 8) | data[3]; - value.push_back(v); - data += 4; - } - return FIIntValue::create(std::move(value)); - } -}; - -struct FILongDecoder: public FIDecoder { - virtual std::shared_ptr<const FIValue> decode(const uint8_t *data, size_t len) /*override*/ { - if (len & 7) { - throw DeadlyImportError(parseErrorMessage); - } - std::vector<int64_t> value; - size_t numLongs = len / 8; - value.reserve(numLongs); - for (size_t i = 0; i < numLongs; ++i) { - int64_t b0 = data[0], b1 = data[1], b2 = data[2], b3 = data[3], b4 = data[4], b5 = data[5], b6 = data[6], b7 = data[7]; - int64_t v = (b0 << 56) | (b1 << 48) | (b2 << 40) | (b3 << 32) | (b4 << 24) | (b5 << 16) | (b6 << 8) | b7; - value.push_back(v); - data += 8; - } - return FILongValue::create(std::move(value)); - } -}; - -struct FIBoolDecoder: public FIDecoder { - virtual std::shared_ptr<const FIValue> decode(const uint8_t *data, size_t len) /*override*/ { - if (len < 1) { - throw DeadlyImportError(parseErrorMessage); - } - std::vector<bool> value; - uint8_t b = *data++; - size_t unusedBits = b >> 4; - size_t numBools = (len * 8) - 4 - unusedBits; - value.reserve(numBools); - uint8_t mask = 1 << 3; - for (size_t i = 0; i < numBools; ++i) { - if (!mask) { - mask = 1 << 7; - b = *data++; - } - value.push_back((b & mask) != 0); - } - return FIBoolValue::create(std::move(value)); - } -}; - -struct FIFloatDecoder: public FIDecoder { - virtual std::shared_ptr<const FIValue> decode(const uint8_t *data, size_t len) /*override*/ { - if (len & 3) { - throw DeadlyImportError(parseErrorMessage); - } - std::vector<float> value; - size_t numFloats = len / 4; - value.reserve(numFloats); - for (size_t i = 0; i < numFloats; ++i) { - int v = (data[0] << 24) | (data[1] << 16) | (data[2] << 8) | data[3]; - float f; - memcpy(&f, &v, 4); - value.push_back(f); - data += 4; - } - return FIFloatValue::create(std::move(value)); - } -}; - -struct FIDoubleDecoder: public FIDecoder { - virtual std::shared_ptr<const FIValue> decode(const uint8_t *data, size_t len) /*override*/ { - if (len & 7) { - throw DeadlyImportError(parseErrorMessage); - } - std::vector<double> value; - size_t numDoubles = len / 8; - value.reserve(numDoubles); - for (size_t i = 0; i < numDoubles; ++i) { - long long b0 = data[0], b1 = data[1], b2 = data[2], b3 = data[3], b4 = data[4], b5 = data[5], b6 = data[6], b7 = data[7]; - long long v = (b0 << 56) | (b1 << 48) | (b2 << 40) | (b3 << 32) | (b4 << 24) | (b5 << 16) | (b6 << 8) | b7; - double f; - memcpy(&f, &v, 8); - value.push_back(f); - data += 8; - } - return FIDoubleValue::create(std::move(value)); - } -}; - -struct FIUUIDDecoder: public FIDecoder { - virtual std::shared_ptr<const FIValue> decode(const uint8_t *data, size_t len) /*override*/ { - if (len & 15) { - throw DeadlyImportError(parseErrorMessage); - } - return FIUUIDValue::create(std::vector<uint8_t>(data, data + len)); - } -}; - -struct FICDATADecoder: public FIDecoder { - virtual std::shared_ptr<const FIValue> decode(const uint8_t *data, size_t len) /*override*/ { - return FICDATAValue::create(parseUTF8String(data, len)); - } -}; - -class CFIReaderImpl: public FIReader { -public: - - CFIReaderImpl(std::unique_ptr<uint8_t[]> data_, size_t size): - data(std::move(data_)), dataP(data.get()), dataEnd(data.get() + size), currentNodeType(irr::io::EXN_NONE), - emptyElement(false), headerPending(true), terminatorPending(false) - {} - - virtual ~CFIReaderImpl() {} - - virtual bool read() /*override*/ { - if (headerPending) { - headerPending = false; - parseHeader(); - } - if (terminatorPending) { - terminatorPending = false; - if (elementStack.empty()) { - return false; - } - else { - nodeName = elementStack.top(); - elementStack.pop(); - currentNodeType = nodeName.empty() ? irr::io::EXN_UNKNOWN : irr::io::EXN_ELEMENT_END; - return true; - } - } - if (dataP >= dataEnd) { - return false; - } - uint8_t b = *dataP; - if (b < 0x80) { // Element (C.2.11.2, C.3.7.2) - // C.3 - parseElement(); - return true; - } - else if (b < 0xc0) { // Characters (C.3.7.5) - // C.7 - auto chars = parseNonIdentifyingStringOrIndex3(vocabulary.charactersTable); - nodeName = chars->toString(); - currentNodeType = irr::io::EXN_TEXT; - return true; - } - else if (b < 0xe0) { - if ((b & 0xfc) == 0xc4) { // DTD (C.2.11.5) - // C.9 - ++dataP; - if (b & 0x02) { - /*const std::string &systemID =*/ parseIdentifyingStringOrIndex(vocabulary.otherURITable); - } - if (b & 0x01) { - /*const std::string &publicID =*/ parseIdentifyingStringOrIndex(vocabulary.otherURITable); - } - elementStack.push(EmptyString); - currentNodeType = irr::io::EXN_UNKNOWN; - return true; - } - else if ((b & 0xfc) == 0xc8) { // Unexpanded entity reference (C.3.7.4) - // C.6 - ++dataP; - /*const std::string &name =*/ parseIdentifyingStringOrIndex(vocabulary.otherNCNameTable); - if (b & 0x02) { - /*const std::string &systemID =*/ parseIdentifyingStringOrIndex(vocabulary.otherURITable); - } - if (b & 0x01) { - /*const std::string &publicID =*/ parseIdentifyingStringOrIndex(vocabulary.otherURITable); - } - currentNodeType = irr::io::EXN_UNKNOWN; - return true; - } - } - else if (b < 0xf0) { - if (b == 0xe1) { // Processing instruction (C.2.11.3, C.3.7.3) - // C.5 - ++dataP; - /*const std::string &target =*/ parseIdentifyingStringOrIndex(vocabulary.otherNCNameTable); - if (dataEnd - dataP < 1) { - throw DeadlyImportError(parseErrorMessage); - } - /*std::shared_ptr<const FIValue> data =*/ parseNonIdentifyingStringOrIndex1(vocabulary.otherStringTable); - currentNodeType = irr::io::EXN_UNKNOWN; - return true; - } - else if (b == 0xe2) { // Comment (C.2.11.4, C.3.7.6) - // C.8 - ++dataP; - if (dataEnd - dataP < 1) { - throw DeadlyImportError(parseErrorMessage); - } - std::shared_ptr<const FIValue> comment = parseNonIdentifyingStringOrIndex1(vocabulary.otherStringTable); - nodeName = comment->toString(); - currentNodeType = irr::io::EXN_COMMENT; - return true; - } - } - else { // Terminator (C.2.12, C.3.8) - ++dataP; - if (b == 0xff) { - terminatorPending = true; - } - if (elementStack.empty()) { - return false; - } - else { - nodeName = elementStack.top(); - elementStack.pop(); - currentNodeType = nodeName.empty() ? irr::io::EXN_UNKNOWN : irr::io::EXN_ELEMENT_END; - return true; - } - } - throw DeadlyImportError(parseErrorMessage); - } - - virtual irr::io::EXML_NODE getNodeType() const /*override*/ { - return currentNodeType; - } - - virtual int getAttributeCount() const /*override*/ { - return static_cast<int>(attributes.size()); - } - - virtual const char* getAttributeName(int idx) const /*override*/ { - if (idx < 0 || idx >= (int)attributes.size()) { - return nullptr; - } - return attributes[idx].name.c_str(); - } - - virtual const char* getAttributeValue(int idx) const /*override*/ { - if (idx < 0 || idx >= (int)attributes.size()) { - return nullptr; - } - return attributes[idx].value->toString().c_str(); - } - - virtual const char* getAttributeValue(const char* name) const /*override*/ { - const Attribute* attr = getAttributeByName(name); - if (!attr) { - return nullptr; - } - return attr->value->toString().c_str(); - } - - virtual const char* getAttributeValueSafe(const char* name) const /*override*/ { - const Attribute* attr = getAttributeByName(name); - if (!attr) { - return EmptyString.c_str(); - } - return attr->value->toString().c_str(); - } - - virtual int getAttributeValueAsInt(const char* name) const /*override*/ { - const Attribute* attr = getAttributeByName(name); - if (!attr) { - return 0; - } - std::shared_ptr<const FIIntValue> intValue = std::dynamic_pointer_cast<const FIIntValue>(attr->value); - if (intValue) { - return intValue->value.size() == 1 ? intValue->value.front() : 0; - } - return atoi(attr->value->toString().c_str()); - } - - virtual int getAttributeValueAsInt(int idx) const /*override*/ { - if (idx < 0 || idx >= (int)attributes.size()) { - return 0; - } - std::shared_ptr<const FIIntValue> intValue = std::dynamic_pointer_cast<const FIIntValue>(attributes[idx].value); - if (intValue) { - return intValue->value.size() == 1 ? intValue->value.front() : 0; - } - return atoi(attributes[idx].value->toString().c_str()); - } - - virtual float getAttributeValueAsFloat(const char* name) const /*override*/ { - const Attribute* attr = getAttributeByName(name); - if (!attr) { - return 0; - } - std::shared_ptr<const FIFloatValue> floatValue = std::dynamic_pointer_cast<const FIFloatValue>(attr->value); - if (floatValue) { - return floatValue->value.size() == 1 ? floatValue->value.front() : 0; - } - - return fast_atof(attr->value->toString().c_str()); - } - - virtual float getAttributeValueAsFloat(int idx) const /*override*/ { - if (idx < 0 || idx >= (int)attributes.size()) { - return 0; - } - std::shared_ptr<const FIFloatValue> floatValue = std::dynamic_pointer_cast<const FIFloatValue>(attributes[idx].value); - if (floatValue) { - return floatValue->value.size() == 1 ? floatValue->value.front() : 0; - } - return fast_atof(attributes[idx].value->toString().c_str()); - } - - virtual const char* getNodeName() const /*override*/ { - return nodeName.c_str(); - } - - virtual const char* getNodeData() const /*override*/ { - return nodeName.c_str(); - } - - virtual bool isEmptyElement() const /*override*/ { - return emptyElement; - } - - virtual irr::io::ETEXT_FORMAT getSourceFormat() const /*override*/ { - return irr::io::ETF_UTF8; - } - - virtual irr::io::ETEXT_FORMAT getParserFormat() const /*override*/ { - return irr::io::ETF_UTF8; - } - - virtual std::shared_ptr<const FIValue> getAttributeEncodedValue(int idx) const /*override*/ { - if (idx < 0 || idx >= (int)attributes.size()) { - return nullptr; - } - return attributes[idx].value; - } - - virtual std::shared_ptr<const FIValue> getAttributeEncodedValue(const char* name) const /*override*/ { - const Attribute* attr = getAttributeByName(name); - if (!attr) { - return nullptr; - } - return attr->value; - } - - virtual void registerDecoder(const std::string &algorithmUri, std::unique_ptr<FIDecoder> decoder) /*override*/ { - decoderMap[algorithmUri] = std::move(decoder); - } - - virtual void registerVocabulary(const std::string &vocabularyUri, const FIVocabulary *vocabulary) /*override*/ { - vocabularyMap[vocabularyUri] = vocabulary; - } - -private: - - struct QName { - std::string prefix; - std::string uri; - std::string name; - inline QName() {} - inline QName(const FIQName &qname): prefix(qname.prefix ? qname.prefix : ""), uri(qname.uri ? qname.uri : ""), name(qname.name) {} - }; - - struct Attribute { - QName qname; - std::string name; - std::shared_ptr<const FIValue> value; - }; - - struct Vocabulary { - std::vector<std::string> restrictedAlphabetTable; - std::vector<std::string> encodingAlgorithmTable; - std::vector<std::string> prefixTable; - std::vector<std::string> namespaceNameTable; - std::vector<std::string> localNameTable; - std::vector<std::string> otherNCNameTable; - std::vector<std::string> otherURITable; - std::vector<std::shared_ptr<const FIValue>> attributeValueTable; - std::vector<std::shared_ptr<const FIValue>> charactersTable; - std::vector<std::shared_ptr<const FIValue>> otherStringTable; - std::vector<QName> elementNameTable; - std::vector<QName> attributeNameTable; - Vocabulary() { - prefixTable.push_back("xml"); - namespaceNameTable.push_back("http://www.w3.org/XML/1998/namespace"); - } - }; - - const Attribute* getAttributeByName(const char* name) const { - if (!name) { - return 0; - } - std::string n = name; - for (int i=0; i<(int)attributes.size(); ++i) { - if (attributes[i].name == n) { - return &attributes[i]; - } - } - return 0; - } - - size_t parseInt2() { // C.25 - uint8_t b = *dataP++; - if (!(b & 0x40)) { // x0...... (C.25.2) - return b & 0x3f; - } - else if ((b & 0x60) == 0x40) { // x10..... ........ (C.25.3) - if (dataEnd - dataP > 0) { - return (((b & 0x1f) << 8) | *dataP++) + 0x40; - } - } - else if ((b & 0x70) == 0x60) { // x110.... ........ ........ (C.25.4) - if (dataEnd - dataP > 1) { - size_t result = (((b & 0x0f) << 16) | (dataP[0] << 8) | dataP[1]) + 0x2040; - dataP += 2; - return result; - } - } - throw DeadlyImportError(parseErrorMessage); - } - - size_t parseInt3() { // C.27 - uint8_t b = *dataP++; - if (!(b & 0x20)) { // xx0..... (C.27.2) - return b & 0x1f; - } - else if ((b & 0x38) == 0x20) { // xx100... ........ (C.27.3) - if (dataEnd - dataP > 0) { - return (((b & 0x07) << 8) | *dataP++) + 0x20; - } - } - else if ((b & 0x38) == 0x28) { // xx101... ........ ........ (C.27.4) - if (dataEnd - dataP > 1) { - size_t result = (((b & 0x07) << 16) | (dataP[0] << 8) | dataP[1]) + 0x820; - dataP += 2; - return result; - } - } - else if ((b & 0x3f) == 0x30) { // xx110000 0000.... ........ ........ (C.27.5) - if ((dataEnd - dataP > 2) && !(dataP[0] & 0xf0)) { - size_t result = (((dataP[0] & 0x0f) << 16) | (dataP[1] << 8) | dataP[2]) + 0x80820; - dataP += 3; - return result; - } - } - throw DeadlyImportError(parseErrorMessage); - } - - size_t parseInt4() { // C.28 - uint8_t b = *dataP++; - if (!(b & 0x10)) { // xxx0.... (C.28.2) - return b & 0x0f; - } - else if ((b & 0x1c) == 0x10) { // xxx100.. ........ (C.28.3) - if (dataEnd - dataP > 0) { - return (((b & 0x03) << 8) | *dataP++) + 0x10; - } - } - else if ((b & 0x1c) == 0x14) { // xxx101.. ........ ........ (C.28.4) - if (dataEnd - dataP > 1) { - size_t result = (((b & 0x03) << 16) | (dataP[0] << 8) | dataP[1]) + 0x410; - dataP += 2; - return result; - } - } - else if ((b & 0x1f) == 0x18) { // xxx11000 0000.... ........ ........ (C.28.5) - if ((dataEnd - dataP > 2) && !(dataP[0] & 0xf0)) { - size_t result = (((dataP[0] & 0x0f) << 16) | (dataP[1] << 8) | dataP[2]) + 0x40410; - dataP += 3; - return result; - } - } - throw DeadlyImportError(parseErrorMessage); - } - - size_t parseSequenceLen() { // C.21 - if (dataEnd - dataP > 0) { - uint8_t b = *dataP++; - if (b < 0x80) { // 0....... (C.21.2) - return b; - } - else if ((b & 0xf0) == 0x80) { // 1000.... ........ ........ (C.21.3) - if (dataEnd - dataP > 1) { - size_t result = (((b & 0x0f) << 16) | (dataP[0] << 8) | dataP[1]) + 0x80; - dataP += 2; - return result; - } - } - } - throw DeadlyImportError(parseErrorMessage); - } - - std::string parseNonEmptyOctetString2() { // C.22 - // Parse the length of the string - uint8_t b = *dataP++ & 0x7f; - size_t len; - if (!(b & 0x40)) { // x0...... (C.22.3.1) - len = b + 1; - } - else if (b == 0x40) { // x1000000 ........ (C.22.3.2) - if (dataEnd - dataP < 1) { - throw DeadlyImportError(parseErrorMessage); - } - len = *dataP++ + 0x41; - } - else if (b == 0x60) { // x1100000 ........ ........ ........ ........ (C.22.3.3) - if (dataEnd - dataP < 4) { - throw DeadlyImportError(parseErrorMessage); - } - len = ((dataP[0] << 24) | (dataP[1] << 16) | (dataP[2] << 8) | dataP[3]) + 0x141; - dataP += 4; - } - else { - throw DeadlyImportError(parseErrorMessage); - } - - // Parse the string (C.22.4) - if (dataEnd - dataP < static_cast<ptrdiff_t>(len)) { - throw DeadlyImportError(parseErrorMessage); - } - std::string s = parseUTF8String(dataP, len); - dataP += len; - - return s; - } - - size_t parseNonEmptyOctetString5Length() { // C.23 - // Parse the length of the string - size_t b = *dataP++ & 0x0f; - if (!(b & 0x08)) { // xxxx0... (C.23.3.1) - return b + 1; - } - else if (b == 0x08) { // xxxx1000 ........ (C.23.3.2) - if (dataEnd - dataP > 0) { - return *dataP++ + 0x09; - } - } - else if (b == 0x0c) { // xxxx1100 ........ ........ ........ ........ (C.23.3.3) - if (dataEnd - dataP > 3) { - size_t result = ((dataP[0] << 24) | (dataP[1] << 16) | (dataP[2] << 8) | dataP[3]) + 0x109; - dataP += 4; - return result; - } - } - throw DeadlyImportError(parseErrorMessage); - } - - size_t parseNonEmptyOctetString7Length() { // C.24 - // Parse the length of the string - size_t b = *dataP++ & 0x03; - if (!(b & 0x02)) { // xxxxxx0. (C.24.3.1) - return b + 1; - } - else if (b == 0x02) { // xxxxxx10 ........ (C.24.3.2) - if (dataEnd - dataP > 0) { - return *dataP++ + 0x3; - } - } - else if (b == 0x03) { // xxxxxx11 ........ ........ ........ ........ (C.24.3.3) - if (dataEnd - dataP > 3) { - size_t result = ((dataP[0] << 24) | (dataP[1] << 16) | (dataP[2] << 8) | dataP[3]) + 0x103; - dataP += 4; - return result; - } - } - throw DeadlyImportError(parseErrorMessage); - } - - std::shared_ptr<const FIValue> parseEncodedData(size_t index, size_t len) { - if (index < 32) { - FIDecoder *decoder = defaultDecoder[index]; - if (!decoder) { - throw DeadlyImportError("Invalid encoding algorithm index " + to_string(index)); - } - return decoder->decode(dataP, len); - } - else { - if (index - 32 >= vocabulary.encodingAlgorithmTable.size()) { - throw DeadlyImportError("Invalid encoding algorithm index " + to_string(index)); - } - std::string uri = vocabulary.encodingAlgorithmTable[index - 32]; - auto it = decoderMap.find(uri); - if (it == decoderMap.end()) { - throw DeadlyImportError("Unsupported encoding algorithm " + uri); - } - else { - return it->second->decode(dataP, len); - } - } - } - - std::shared_ptr<const FIValue> parseRestrictedAlphabet(size_t index, size_t len) { - std::string alphabet; - if (index < 16) { - switch (index) { - case 0: // numeric - alphabet = "0123456789-+.e "; - break; - case 1: // date and time - alphabet = "0123456789-:TZ "; - break; - default: - throw DeadlyImportError("Invalid restricted alphabet index " + to_string(index)); - } - } - else { - if (index - 16 >= vocabulary.restrictedAlphabetTable.size()) { - throw DeadlyImportError("Invalid restricted alphabet index " + to_string(index)); - } - alphabet = vocabulary.restrictedAlphabetTable[index - 16]; - } - std::vector<uint32_t> alphabetUTF32; - utf8::utf8to32(alphabet.begin(), alphabet.end(), back_inserter(alphabetUTF32)); - std::string::size_type alphabetLength = alphabetUTF32.size(); - if (alphabetLength < 2) { - throw DeadlyImportError("Invalid restricted alphabet length " + to_string(alphabetLength)); - } - std::string::size_type bitsPerCharacter = 1; - while ((1ull << bitsPerCharacter) <= alphabetLength) { - ++bitsPerCharacter; - } - size_t bitsAvail = 0; - uint8_t mask = (1 << bitsPerCharacter) - 1; - uint32_t bits = 0; - std::string s; - for (size_t i = 0; i < len; ++i) { - bits = (bits << 8) | dataP[i]; - bitsAvail += 8; - while (bitsAvail >= bitsPerCharacter) { - bitsAvail -= bitsPerCharacter; - size_t charIndex = (bits >> bitsAvail) & mask; - if (charIndex < alphabetLength) { - s.push_back(alphabetUTF32[charIndex]); - } - else if (charIndex != mask) { - throw DeadlyImportError(parseErrorMessage); - } - } - } - return FIStringValue::create(std::move(s)); - } - - std::shared_ptr<const FIValue> parseEncodedCharacterString3() { // C.19 - std::shared_ptr<const FIValue> result; - size_t len; - uint8_t b = *dataP; - if (b & 0x20) { - ++dataP; - if (dataEnd - dataP < 1) { - throw DeadlyImportError(parseErrorMessage); - } - size_t index = ((b & 0x0f) << 4) | ((*dataP & 0xf0) >> 4); // C.29 - len = parseNonEmptyOctetString5Length(); - if (dataEnd - dataP < static_cast<ptrdiff_t>(len)) { - throw DeadlyImportError(parseErrorMessage); - } - if (b & 0x10) { - // encoding algorithm (C.19.3.4) - result = parseEncodedData(index, len); - } - else { - // Restricted alphabet (C.19.3.3) - result = parseRestrictedAlphabet(index, len); - } - } - else { - len = parseNonEmptyOctetString5Length(); - if (dataEnd - dataP < static_cast<ptrdiff_t>(len)) { - throw DeadlyImportError(parseErrorMessage); - } - if (b & 0x10) { - // UTF-16 (C.19.3.2) - if (len & 1) { - throw DeadlyImportError(parseErrorMessage); - } - result = FIStringValue::create(parseUTF16String(dataP, len)); - } - else { - // UTF-8 (C.19.3.1) - result = FIStringValue::create(parseUTF8String(dataP, len)); - } - } - dataP += len; - return result; - } - - std::shared_ptr<const FIValue> parseEncodedCharacterString5() { // C.20 - std::shared_ptr<const FIValue> result; - size_t len; - uint8_t b = *dataP; - if (b & 0x08) { - ++dataP; - if (dataEnd - dataP < 1) { - throw DeadlyImportError(parseErrorMessage); - } - size_t index = ((b & 0x03) << 6) | ((*dataP & 0xfc) >> 2); /* C.29 */ - len = parseNonEmptyOctetString7Length(); - if (dataEnd - dataP < static_cast<ptrdiff_t>(len)) { - throw DeadlyImportError(parseErrorMessage); - } - if (b & 0x04) { - // encoding algorithm (C.20.3.4) - result = parseEncodedData(index, len); - } - else { - // Restricted alphabet (C.20.3.3) - result = parseRestrictedAlphabet(index, len); - } - } - else { - len = parseNonEmptyOctetString7Length(); - if (dataEnd - dataP < static_cast<ptrdiff_t>(len)) { - throw DeadlyImportError(parseErrorMessage); - } - if (b & 0x04) { - // UTF-16 (C.20.3.2) - if (len & 1) { - throw DeadlyImportError(parseErrorMessage); - } - result = FIStringValue::create(parseUTF16String(dataP, len)); - } - else { - // UTF-8 (C.20.3.1) - result = FIStringValue::create(parseUTF8String(dataP, len)); - } - } - dataP += len; - return result; - } - - const std::string &parseIdentifyingStringOrIndex(std::vector<std::string> &stringTable) { // C.13 - if (dataEnd - dataP < 1) { - throw DeadlyImportError(parseErrorMessage); - } - uint8_t b = *dataP; - if (b & 0x80) { - // We have an index (C.13.4) - size_t index = parseInt2(); - if (index >= stringTable.size()) { - throw DeadlyImportError(parseErrorMessage); - } - return stringTable[index]; - } - else { - // We have a string (C.13.3) - stringTable.push_back(parseNonEmptyOctetString2()); - return stringTable.back(); - } - } - - QName parseNameSurrogate() { // C.16 - if (dataEnd - dataP < 1) { - throw DeadlyImportError(parseErrorMessage); - } - uint8_t b = *dataP++; - if (b & 0xfc) { // Padding '000000' C.2.5.5 - throw DeadlyImportError(parseErrorMessage); - } - QName result; - size_t index; - if (b & 0x02) { // prefix (C.16.3) - if ((dataEnd - dataP < 1) || (*dataP & 0x80)) { - throw DeadlyImportError(parseErrorMessage); - } - index = parseInt2(); - if (index >= vocabulary.prefixTable.size()) { - throw DeadlyImportError(parseErrorMessage); - } - result.prefix = vocabulary.prefixTable[index]; - } - if (b & 0x01) { // namespace-name (C.16.4) - if ((dataEnd - dataP < 1) || (*dataP & 0x80)) { - throw DeadlyImportError(parseErrorMessage); - } - index = parseInt2(); - if (index >= vocabulary.namespaceNameTable.size()) { - throw DeadlyImportError(parseErrorMessage); - } - result.uri = vocabulary.namespaceNameTable[index]; - } - // local-name - if ((dataEnd - dataP < 1) || (*dataP & 0x80)) { - throw DeadlyImportError(parseErrorMessage); - } - index = parseInt2(); - if (index >= vocabulary.localNameTable.size()) { - throw DeadlyImportError(parseErrorMessage); - } - result.name = vocabulary.localNameTable[index]; - return result; - } - - const QName &parseQualifiedNameOrIndex2(std::vector<QName> &qNameTable) { // C.17 - uint8_t b = *dataP; - if ((b & 0x7c) == 0x78) { // x11110.. - // We have a literal (C.17.3) - ++dataP; - QName result; - // prefix (C.17.3.1) - result.prefix = b & 0x02 ? parseIdentifyingStringOrIndex(vocabulary.prefixTable) : std::string(); - // namespace-name (C.17.3.1) - result.uri = b & 0x01 ? parseIdentifyingStringOrIndex(vocabulary.namespaceNameTable) : std::string(); - // local-name - result.name = parseIdentifyingStringOrIndex(vocabulary.localNameTable); - qNameTable.push_back(result); - return qNameTable.back(); - } - else { - // We have an index (C.17.4) - size_t index = parseInt2(); - if (index >= qNameTable.size()) { - throw DeadlyImportError(parseErrorMessage); - } - return qNameTable[index]; - } - } - - const QName &parseQualifiedNameOrIndex3(std::vector<QName> &qNameTable) { // C.18 - uint8_t b = *dataP; - if ((b & 0x3c) == 0x3c) { // xx1111.. - // We have a literal (C.18.3) - ++dataP; - QName result; - // prefix (C.18.3.1) - result.prefix = b & 0x02 ? parseIdentifyingStringOrIndex(vocabulary.prefixTable) : std::string(); - // namespace-name (C.18.3.1) - result.uri = b & 0x01 ? parseIdentifyingStringOrIndex(vocabulary.namespaceNameTable) : std::string(); - // local-name - result.name = parseIdentifyingStringOrIndex(vocabulary.localNameTable); - qNameTable.push_back(result); - return qNameTable.back(); - } - else { - // We have an index (C.18.4) - size_t index = parseInt3(); - if (index >= qNameTable.size()) { - throw DeadlyImportError(parseErrorMessage); - } - return qNameTable[index]; - } - } - - std::shared_ptr<const FIValue> parseNonIdentifyingStringOrIndex1(std::vector<std::shared_ptr<const FIValue>> &valueTable) { // C.14 - uint8_t b = *dataP; - if (b == 0xff) { // C.26.2 - // empty string - ++dataP; - return EmptyFIString; - } - else if (b & 0x80) { // C.14.4 - // We have an index - size_t index = parseInt2(); - if (index >= valueTable.size()) { - throw DeadlyImportError(parseErrorMessage); - } - return valueTable[index]; - } - else { // C.14.3 - // We have a literal - std::shared_ptr<const FIValue> result = parseEncodedCharacterString3(); - if (b & 0x40) { // C.14.3.1 - valueTable.push_back(result); - } - return result; - } - } - - std::shared_ptr<const FIValue> parseNonIdentifyingStringOrIndex3(std::vector<std::shared_ptr<const FIValue>> &valueTable) { // C.15 - uint8_t b = *dataP; - if (b & 0x20) { // C.15.4 - // We have an index - size_t index = parseInt4(); - if (index >= valueTable.size()) { - throw DeadlyImportError(parseErrorMessage); - } - return valueTable[index]; - } - else { // C.15.3 - // We have a literal - std::shared_ptr<const FIValue> result = parseEncodedCharacterString5(); - if (b & 0x10) { // C.15.3.1 - valueTable.push_back(result); - } - return result; - } - } - - void parseElement() { - // C.3 - - attributes.clear(); - - uint8_t b = *dataP; - bool hasAttributes = (b & 0x40) != 0; // C.3.3 - if ((b & 0x3f) == 0x38) { // C.3.4.1 - // Parse namespaces - ++dataP; - for (;;) { - if (dataEnd - dataP < 1) { - throw DeadlyImportError(parseErrorMessage); - } - b = *dataP++; - if (b == 0xf0) { // C.3.4.3 - break; - } - if ((b & 0xfc) != 0xcc) { // C.3.4.2 - throw DeadlyImportError(parseErrorMessage); - } - // C.12 - Attribute attr; - attr.qname.prefix = "xmlns"; - attr.qname.name = b & 0x02 ? parseIdentifyingStringOrIndex(vocabulary.prefixTable) : std::string(); - attr.qname.uri = b & 0x01 ? parseIdentifyingStringOrIndex(vocabulary.namespaceNameTable) : std::string(); - attr.name = attr.qname.name.empty() ? "xmlns" : "xmlns:" + attr.qname.name; - attr.value = FIStringValue::create(std::string(attr.qname.uri)); - attributes.push_back(attr); - } - if ((dataEnd - dataP < 1) || (*dataP & 0xc0)) { - throw DeadlyImportError(parseErrorMessage); - } - } - - // Parse Element name (C.3.5) - const QName &elemName = parseQualifiedNameOrIndex3(vocabulary.elementNameTable); - nodeName = elemName.prefix.empty() ? elemName.name : elemName.prefix + ':' + elemName.name; - - if (hasAttributes) { - for (;;) { - if (dataEnd - dataP < 1) { - throw DeadlyImportError(parseErrorMessage); - } - b = *dataP; - if (b < 0x80) { // C.3.6.1 - // C.4 - Attribute attr; - attr.qname = parseQualifiedNameOrIndex2(vocabulary.attributeNameTable); - attr.name = attr.qname.prefix.empty() ? attr.qname.name : attr.qname.prefix + ':' + attr.qname.name; - if (dataEnd - dataP < 1) { - throw DeadlyImportError(parseErrorMessage); - } - attr.value = parseNonIdentifyingStringOrIndex1(vocabulary.attributeValueTable); - attributes.push_back(attr); - } - else { - if ((b & 0xf0) != 0xf0) { // C.3.6.2 - throw DeadlyImportError(parseErrorMessage); - } - emptyElement = b == 0xff; // C.3.6.2, C.3.8 - ++dataP; - break; - } - } - } - else { - if (dataEnd - dataP < 1) { - throw DeadlyImportError(parseErrorMessage); - } - b = *dataP; - switch (b) { - case 0xff: - terminatorPending = true; - // Intentionally fall through - case 0xf0: - emptyElement = true; - ++dataP; - break; - default: - emptyElement = false; - } - } - if (!emptyElement) { - elementStack.push(nodeName); - } - - currentNodeType = irr::io::EXN_ELEMENT; - } - - void parseHeader() { - // Parse header (C.1.3) - size_t magicSize = parseMagic(dataP, dataEnd); - if (!magicSize) { - throw DeadlyImportError(parseErrorMessage); - } - dataP += magicSize; - // C.2.3 - if (dataEnd - dataP < 1) { - throw DeadlyImportError(parseErrorMessage); - } - uint8_t b = *dataP++; - if (b & 0x40) { - // Parse additional data (C.2.4) - size_t len = parseSequenceLen(); - for (size_t i = 0; i < len; ++i) { - if (dataEnd - dataP < 1) { - throw DeadlyImportError(parseErrorMessage); - } - /*std::string id =*/ parseNonEmptyOctetString2(); - if (dataEnd - dataP < 1) { - throw DeadlyImportError(parseErrorMessage); - } - /*std::string data =*/ parseNonEmptyOctetString2(); - } - } - if (b & 0x20) { - // Parse initial vocabulary (C.2.5) - if (dataEnd - dataP < 2) { - throw DeadlyImportError(parseErrorMessage); - } - uint16_t b1 = (dataP[0] << 8) | dataP[1]; - dataP += 2; - if (b1 & 0x1000) { - // External vocabulary (C.2.5.2) - if (dataEnd - dataP < 1) { - throw DeadlyImportError(parseErrorMessage); - } - std::string uri = parseNonEmptyOctetString2(); - auto it = vocabularyMap.find(uri); - if (it == vocabularyMap.end()) { - throw DeadlyImportError("Unknown vocabulary " + uri); - } - const FIVocabulary *externalVocabulary = it->second; - if (externalVocabulary->restrictedAlphabetTable) { - std::copy(externalVocabulary->restrictedAlphabetTable, externalVocabulary->restrictedAlphabetTable + externalVocabulary->restrictedAlphabetTableSize, std::back_inserter(vocabulary.restrictedAlphabetTable)); - } - if (externalVocabulary->encodingAlgorithmTable) { - std::copy(externalVocabulary->encodingAlgorithmTable, externalVocabulary->encodingAlgorithmTable + externalVocabulary->encodingAlgorithmTableSize, std::back_inserter(vocabulary.encodingAlgorithmTable)); - } - if (externalVocabulary->prefixTable) { - std::copy(externalVocabulary->prefixTable, externalVocabulary->prefixTable + externalVocabulary->prefixTableSize, std::back_inserter(vocabulary.prefixTable)); - } - if (externalVocabulary->namespaceNameTable) { - std::copy(externalVocabulary->namespaceNameTable, externalVocabulary->namespaceNameTable + externalVocabulary->namespaceNameTableSize, std::back_inserter(vocabulary.namespaceNameTable)); - } - if (externalVocabulary->localNameTable) { - std::copy(externalVocabulary->localNameTable, externalVocabulary->localNameTable + externalVocabulary->localNameTableSize, std::back_inserter(vocabulary.localNameTable)); - } - if (externalVocabulary->otherNCNameTable) { - std::copy(externalVocabulary->otherNCNameTable, externalVocabulary->otherNCNameTable + externalVocabulary->otherNCNameTableSize, std::back_inserter(vocabulary.otherNCNameTable)); - } - if (externalVocabulary->otherURITable) { - std::copy(externalVocabulary->otherURITable, externalVocabulary->otherURITable + externalVocabulary->otherURITableSize, std::back_inserter(vocabulary.otherURITable)); - } - if (externalVocabulary->attributeValueTable) { - std::copy(externalVocabulary->attributeValueTable, externalVocabulary->attributeValueTable + externalVocabulary->attributeValueTableSize, std::back_inserter(vocabulary.attributeValueTable)); - } - if (externalVocabulary->charactersTable) { - std::copy(externalVocabulary->charactersTable, externalVocabulary->charactersTable + externalVocabulary->charactersTableSize, std::back_inserter(vocabulary.charactersTable)); - } - if (externalVocabulary->otherStringTable) { - std::copy(externalVocabulary->otherStringTable, externalVocabulary->otherStringTable + externalVocabulary->otherStringTableSize, std::back_inserter(vocabulary.otherStringTable)); - } - if (externalVocabulary->elementNameTable) { - std::copy(externalVocabulary->elementNameTable, externalVocabulary->elementNameTable + externalVocabulary->elementNameTableSize, std::back_inserter(vocabulary.elementNameTable)); - } - if (externalVocabulary->attributeNameTable) { - std::copy(externalVocabulary->attributeNameTable, externalVocabulary->attributeNameTable + externalVocabulary->attributeNameTableSize, std::back_inserter(vocabulary.attributeNameTable)); - } - } - if (b1 & 0x0800) { - // Parse restricted alphabets (C.2.5.3) - for (size_t len = parseSequenceLen(); len > 0; --len) { - if (dataEnd - dataP < 1) { - throw DeadlyImportError(parseErrorMessage); - } - vocabulary.restrictedAlphabetTable.push_back(parseNonEmptyOctetString2()); - } - } - if (b1 & 0x0400) { - // Parse encoding algorithms (C.2.5.3) - for (size_t len = parseSequenceLen(); len > 0; --len) { - if (dataEnd - dataP < 1) { - throw DeadlyImportError(parseErrorMessage); - } - vocabulary.encodingAlgorithmTable.push_back(parseNonEmptyOctetString2()); - } - } - if (b1 & 0x0200) { - // Parse prefixes (C.2.5.3) - for (size_t len = parseSequenceLen(); len > 0; --len) { - if (dataEnd - dataP < 1) { - throw DeadlyImportError(parseErrorMessage); - } - vocabulary.prefixTable.push_back(parseNonEmptyOctetString2()); - } - } - if (b1 & 0x0100) { - // Parse namespace names (C.2.5.3) - for (size_t len = parseSequenceLen(); len > 0; --len) { - if (dataEnd - dataP < 1) { - throw DeadlyImportError(parseErrorMessage); - } - vocabulary.namespaceNameTable.push_back(parseNonEmptyOctetString2()); - } - } - if (b1 & 0x0080) { - // Parse local names (C.2.5.3) - for (size_t len = parseSequenceLen(); len > 0; --len) { - if (dataEnd - dataP < 1) { - throw DeadlyImportError(parseErrorMessage); - } - vocabulary.localNameTable.push_back(parseNonEmptyOctetString2()); - } - } - if (b1 & 0x0040) { - // Parse other ncnames (C.2.5.3) - for (size_t len = parseSequenceLen(); len > 0; --len) { - if (dataEnd - dataP < 1) { - throw DeadlyImportError(parseErrorMessage); - } - vocabulary.otherNCNameTable.push_back(parseNonEmptyOctetString2()); - } - } - if (b1 & 0x0020) { - // Parse other uris (C.2.5.3) - for (size_t len = parseSequenceLen(); len > 0; --len) { - if (dataEnd - dataP < 1) { - throw DeadlyImportError(parseErrorMessage); - } - vocabulary.otherURITable.push_back(parseNonEmptyOctetString2()); - } - } - if (b1 & 0x0010) { - // Parse attribute values (C.2.5.4) - for (size_t len = parseSequenceLen(); len > 0; --len) { - if (dataEnd - dataP < 1) { - throw DeadlyImportError(parseErrorMessage); - } - vocabulary.attributeValueTable.push_back(parseEncodedCharacterString3()); - } - } - if (b1 & 0x0008) { - // Parse content character chunks (C.2.5.4) - for (size_t len = parseSequenceLen(); len > 0; --len) { - if (dataEnd - dataP < 1) { - throw DeadlyImportError(parseErrorMessage); - } - vocabulary.charactersTable.push_back(parseEncodedCharacterString3()); - } - } - if (b1 & 0x0004) { - // Parse other strings (C.2.5.4) - for (size_t len = parseSequenceLen(); len > 0; --len) { - if (dataEnd - dataP < 1) { - throw DeadlyImportError(parseErrorMessage); - } - vocabulary.otherStringTable.push_back(parseEncodedCharacterString3()); - } - } - if (b1 & 0x0002) { - // Parse element name surrogates (C.2.5.5) - for (size_t len = parseSequenceLen(); len > 0; --len) { - vocabulary.elementNameTable.push_back(parseNameSurrogate()); - } - } - if (b1 & 0x0001) { - // Parse attribute name surrogates (C.2.5.5) - for (size_t len = parseSequenceLen(); len > 0; --len) { - vocabulary.attributeNameTable.push_back(parseNameSurrogate()); - } - } - } - if (b & 0x10) { - // Parse notations (C.2.6) - for (;;) { - if (dataEnd - dataP < 1) { - throw DeadlyImportError(parseErrorMessage); - } - uint8_t b1 = *dataP++; - if (b1 == 0xf0) { - break; - } - if ((b1 & 0xfc) != 0xc0) { - throw DeadlyImportError(parseErrorMessage); - } - /* C.11 */ - /*const std::string &name =*/ parseIdentifyingStringOrIndex(vocabulary.otherNCNameTable); - if (b1 & 0x02) { - /*const std::string &systemId =*/ parseIdentifyingStringOrIndex(vocabulary.otherURITable); - } - if (b1 & 0x01) { - /*const std::string &publicId =*/ parseIdentifyingStringOrIndex(vocabulary.otherURITable); - } - } - } - if (b & 0x08) { - // Parse unparsed entities (C.2.7) - for (;;) { - if (dataEnd - dataP < 1) { - throw DeadlyImportError(parseErrorMessage); - } - uint8_t b1 = *dataP++; - if (b1 == 0xf0) { - break; - } - if ((b1 & 0xfe) != 0xd0) { - throw DeadlyImportError(parseErrorMessage); - } - /* C.10 */ - /*const std::string &name =*/ parseIdentifyingStringOrIndex(vocabulary.otherNCNameTable); - /*const std::string &systemId =*/ parseIdentifyingStringOrIndex(vocabulary.otherURITable); - if (b1 & 0x01) { - /*const std::string &publicId =*/ parseIdentifyingStringOrIndex(vocabulary.otherURITable); - } - /*const std::string ¬ationName =*/ parseIdentifyingStringOrIndex(vocabulary.otherNCNameTable); - } - } - if (b & 0x04) { - // Parse character encoding scheme (C.2.8) - if (dataEnd - dataP < 1) { - throw DeadlyImportError(parseErrorMessage); - } - /*std::string characterEncodingScheme =*/ parseNonEmptyOctetString2(); - } - if (b & 0x02) { - // Parse standalone flag (C.2.9) - if (dataEnd - dataP < 1) { - throw DeadlyImportError(parseErrorMessage); - } - uint8_t b1 = *dataP++; - if (b1 & 0xfe) { - throw DeadlyImportError(parseErrorMessage); - } - //bool standalone = b1 & 0x01; - } - if (b & 0x01) { - // Parse version (C.2.10) - if (dataEnd - dataP < 1) { - throw DeadlyImportError(parseErrorMessage); - } - /*std::shared_ptr<const FIValue> version =*/ parseNonIdentifyingStringOrIndex1(vocabulary.otherStringTable); - } - } - - std::unique_ptr<uint8_t[]> data; - uint8_t *dataP, *dataEnd; - irr::io::EXML_NODE currentNodeType; - bool emptyElement; - bool headerPending; - bool terminatorPending; - Vocabulary vocabulary; - std::vector<Attribute> attributes; - std::stack<std::string> elementStack; - std::string nodeName; - std::map<std::string, std::unique_ptr<FIDecoder>> decoderMap; - std::map<std::string, const FIVocabulary*> vocabularyMap; - - static const std::string EmptyString; - static std::shared_ptr<const FIValue> EmptyFIString; - - static FIHexDecoder hexDecoder; - static FIBase64Decoder base64Decoder; - static FIShortDecoder shortDecoder; - static FIIntDecoder intDecoder; - static FILongDecoder longDecoder; - static FIBoolDecoder boolDecoder; - static FIFloatDecoder floatDecoder; - static FIDoubleDecoder doubleDecoder; - static FIUUIDDecoder uuidDecoder; - static FICDATADecoder cdataDecoder; - static FIDecoder *defaultDecoder[32]; -}; - -const std::string CFIReaderImpl::EmptyString; -std::shared_ptr<const FIValue> CFIReaderImpl::EmptyFIString = FIStringValue::create(std::string()); - -FIHexDecoder CFIReaderImpl::hexDecoder; -FIBase64Decoder CFIReaderImpl::base64Decoder; -FIShortDecoder CFIReaderImpl::shortDecoder; -FIIntDecoder CFIReaderImpl::intDecoder; -FILongDecoder CFIReaderImpl::longDecoder; -FIBoolDecoder CFIReaderImpl::boolDecoder; -FIFloatDecoder CFIReaderImpl::floatDecoder; -FIDoubleDecoder CFIReaderImpl::doubleDecoder; -FIUUIDDecoder CFIReaderImpl::uuidDecoder; -FICDATADecoder CFIReaderImpl::cdataDecoder; - -FIDecoder *CFIReaderImpl::defaultDecoder[32] = { - &hexDecoder, - &base64Decoder, - &shortDecoder, - &intDecoder, - &longDecoder, - &boolDecoder, - &floatDecoder, - &doubleDecoder, - &uuidDecoder, - &cdataDecoder -}; - -class CXMLReaderImpl : public FIReader -{ -public: - - //! Constructor - CXMLReaderImpl(std::unique_ptr<irr::io::IIrrXMLReader<char, irr::io::IXMLBase>> reader_) - : reader(std::move(reader_)) - {} - - virtual ~CXMLReaderImpl() {} - - virtual bool read() /*override*/ { - return reader->read(); - } - - virtual irr::io::EXML_NODE getNodeType() const /*override*/ { - return reader->getNodeType(); - } - - virtual int getAttributeCount() const /*override*/ { - return reader->getAttributeCount(); - } - - virtual const char* getAttributeName(int idx) const /*override*/ { - return reader->getAttributeName(idx); - } - - virtual const char* getAttributeValue(int idx) const /*override*/ { - return reader->getAttributeValue(idx); - } - - virtual const char* getAttributeValue(const char* name) const /*override*/ { - return reader->getAttributeValue(name); - } - - virtual const char* getAttributeValueSafe(const char* name) const /*override*/ { - return reader->getAttributeValueSafe(name); - } - - virtual int getAttributeValueAsInt(const char* name) const /*override*/ { - return reader->getAttributeValueAsInt(name); - } - - virtual int getAttributeValueAsInt(int idx) const /*override*/ { - return reader->getAttributeValueAsInt(idx); - } - - virtual float getAttributeValueAsFloat(const char* name) const /*override*/ { - return reader->getAttributeValueAsFloat(name); - } - - virtual float getAttributeValueAsFloat(int idx) const /*override*/ { - return reader->getAttributeValueAsFloat(idx); - } - - virtual const char* getNodeName() const /*override*/ { - return reader->getNodeName(); - } - - virtual const char* getNodeData() const /*override*/ { - return reader->getNodeData(); - } - - virtual bool isEmptyElement() const /*override*/ { - return reader->isEmptyElement(); - } - - virtual irr::io::ETEXT_FORMAT getSourceFormat() const /*override*/ { - return reader->getSourceFormat(); - } - - virtual irr::io::ETEXT_FORMAT getParserFormat() const /*override*/ { - return reader->getParserFormat(); - } - - virtual std::shared_ptr<const FIValue> getAttributeEncodedValue(int /*idx*/) const /*override*/ { - return nullptr; - } - - virtual std::shared_ptr<const FIValue> getAttributeEncodedValue(const char* /*name*/) const /*override*/ { - return nullptr; - } - - virtual void registerDecoder(const std::string & /*algorithmUri*/, std::unique_ptr<FIDecoder> /*decoder*/) /*override*/ {} - - - virtual void registerVocabulary(const std::string &/*vocabularyUri*/, const FIVocabulary * /*vocabulary*/) /*override*/ {} - -private: - - std::unique_ptr<irr::io::IIrrXMLReader<char, irr::io::IXMLBase>> reader; -}; - -static std::unique_ptr<uint8_t[]> readFile(IOStream *stream, size_t &size, bool &isFI) { - size = stream->FileSize(); - std::unique_ptr<uint8_t[]> data = std::unique_ptr<uint8_t[]>(new uint8_t[size]); - if (stream->Read(data.get(), size, 1) != 1) { - size = 0; - data.reset(); - } - isFI = parseMagic(data.get(), data.get() + size) > 0; - return data; -} - -std::unique_ptr<FIReader> FIReader::create(IOStream *stream) -{ - size_t size; - bool isFI; - auto data = readFile(stream, size, isFI); - if (isFI) { - return std::unique_ptr<FIReader>(new CFIReaderImpl(std::move(data), size)); - } - else { - auto memios = std::unique_ptr<MemoryIOStream>(new MemoryIOStream(data.release(), size, true)); - auto callback = std::unique_ptr<CIrrXML_IOStreamReader>(new CIrrXML_IOStreamReader(memios.get())); - return std::unique_ptr<FIReader>(new CXMLReaderImpl(std::unique_ptr<irr::io::IIrrXMLReader<char, irr::io::IXMLBase>>(createIrrXMLReader(callback.get())))); - } -} - -}// namespace Assimp - -#endif // !ASSIMP_BUILD_NO_X3D_IMPORTER diff --git a/thirdparty/assimp/code/MMDCpp14.h b/thirdparty/assimp/code/MMD/MMDCpp14.h index 638b0bfd2f..638b0bfd2f 100644 --- a/thirdparty/assimp/code/MMDCpp14.h +++ b/thirdparty/assimp/code/MMD/MMDCpp14.h diff --git a/thirdparty/assimp/code/MMDImporter.cpp b/thirdparty/assimp/code/MMD/MMDImporter.cpp index 84b9e35a6b..e7744e4cd0 100644 --- a/thirdparty/assimp/code/MMDImporter.cpp +++ b/thirdparty/assimp/code/MMD/MMDImporter.cpp @@ -41,15 +41,17 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef ASSIMP_BUILD_NO_MMD_IMPORTER -#include "MMDImporter.h" -#include "MMDPmdParser.h" -#include "MMDPmxParser.h" -#include "MMDVmdParser.h" -#include "ConvertToLHProcess.h" +#include "MMD/MMDImporter.h" +#include "MMD/MMDPmdParser.h" +#include "MMD/MMDPmxParser.h" +#include "MMD/MMDVmdParser.h" +#include "PostProcessing/ConvertToLHProcess.h" + #include <assimp/DefaultIOSystem.h> #include <assimp/Importer.hpp> #include <assimp/ai_assert.h> #include <assimp/scene.h> + #include <fstream> #include <iomanip> #include <memory> diff --git a/thirdparty/assimp/code/MMDImporter.h b/thirdparty/assimp/code/MMD/MMDImporter.h index 4ee94eeb00..4ee94eeb00 100644 --- a/thirdparty/assimp/code/MMDImporter.h +++ b/thirdparty/assimp/code/MMD/MMDImporter.h diff --git a/thirdparty/assimp/code/MMDPmdParser.h b/thirdparty/assimp/code/MMD/MMDPmdParser.h index d2f2224aa1..d2f2224aa1 100644 --- a/thirdparty/assimp/code/MMDPmdParser.h +++ b/thirdparty/assimp/code/MMD/MMDPmdParser.h diff --git a/thirdparty/assimp/code/MMDPmxParser.cpp b/thirdparty/assimp/code/MMD/MMDPmxParser.cpp index 7425ceac22..80f0986dd7 100644 --- a/thirdparty/assimp/code/MMDPmxParser.cpp +++ b/thirdparty/assimp/code/MMD/MMDPmxParser.cpp @@ -42,7 +42,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <utility> #include "MMDPmxParser.h" #include <assimp/StringUtils.h> -#include "../contrib/utf8cpp/source/utf8.h" +#ifdef ASSIMP_USE_HUNTER +# include <utf8/utf8.h> +#else +# include "../contrib/utf8cpp/source/utf8.h" +#endif #include <assimp/Exceptional.h> namespace pmx diff --git a/thirdparty/assimp/code/MMDPmxParser.h b/thirdparty/assimp/code/MMD/MMDPmxParser.h index cf523a1298..cf523a1298 100644 --- a/thirdparty/assimp/code/MMDPmxParser.h +++ b/thirdparty/assimp/code/MMD/MMDPmxParser.h diff --git a/thirdparty/assimp/code/MMDVmdParser.h b/thirdparty/assimp/code/MMD/MMDVmdParser.h index 947c3a2422..947c3a2422 100644 --- a/thirdparty/assimp/code/MMDVmdParser.h +++ b/thirdparty/assimp/code/MMD/MMDVmdParser.h diff --git a/thirdparty/assimp/code/MaterialSystem.cpp b/thirdparty/assimp/code/Material/MaterialSystem.cpp index 03d5a18a34..d0b39093b6 100644 --- a/thirdparty/assimp/code/MaterialSystem.cpp +++ b/thirdparty/assimp/code/Material/MaterialSystem.cpp @@ -96,12 +96,12 @@ aiReturn aiGetMaterialFloatArray(const aiMaterial* pMat, ai_real* pOut, unsigned int* pMax) { - ai_assert( pOut != NULL ); - ai_assert( pMat != NULL ); + ai_assert( pOut != nullptr ); + ai_assert( pMat != nullptr ); const aiMaterialProperty* prop; aiGetMaterialProperty(pMat,pKey,type,index, (const aiMaterialProperty**) &prop); - if (!prop) { + if ( nullptr == prop) { return AI_FAILURE; } @@ -112,9 +112,11 @@ aiReturn aiGetMaterialFloatArray(const aiMaterial* pMat, if (pMax) { iWrite = std::min(*pMax,iWrite); ; } - for (unsigned int a = 0; a < iWrite;++a) { - pOut[a] = static_cast<ai_real> ( reinterpret_cast<float*>(prop->mData)[a] ); + + for (unsigned int a = 0; a < iWrite; ++a) { + pOut[ a ] = static_cast<ai_real> ( reinterpret_cast<float*>(prop->mData)[a] ); } + if (pMax) { *pMax = iWrite; } diff --git a/thirdparty/assimp/code/MaterialSystem.h b/thirdparty/assimp/code/Material/MaterialSystem.h index 67d53578cb..67d53578cb 100644 --- a/thirdparty/assimp/code/MaterialSystem.h +++ b/thirdparty/assimp/code/Material/MaterialSystem.h diff --git a/thirdparty/assimp/code/CalcTangentsProcess.cpp b/thirdparty/assimp/code/PostProcessing/CalcTangentsProcess.cpp index b30f39c274..b30f39c274 100644 --- a/thirdparty/assimp/code/CalcTangentsProcess.cpp +++ b/thirdparty/assimp/code/PostProcessing/CalcTangentsProcess.cpp diff --git a/thirdparty/assimp/code/CalcTangentsProcess.h b/thirdparty/assimp/code/PostProcessing/CalcTangentsProcess.h index 18775abcc7..3568a624f8 100644 --- a/thirdparty/assimp/code/CalcTangentsProcess.h +++ b/thirdparty/assimp/code/PostProcessing/CalcTangentsProcess.h @@ -42,11 +42,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /** @file Defines a post processing step to calculate tangents and - bitangents on all imported meshes.*/ + bi-tangents on all imported meshes.*/ #ifndef AI_CALCTANGENTSPROCESS_H_INC #define AI_CALCTANGENTSPROCESS_H_INC -#include "BaseProcess.h" +#include "Common/BaseProcess.h" struct aiMesh; diff --git a/thirdparty/assimp/code/ComputeUVMappingProcess.cpp b/thirdparty/assimp/code/PostProcessing/ComputeUVMappingProcess.cpp index bb571a551b..bb571a551b 100644 --- a/thirdparty/assimp/code/ComputeUVMappingProcess.cpp +++ b/thirdparty/assimp/code/PostProcessing/ComputeUVMappingProcess.cpp diff --git a/thirdparty/assimp/code/ComputeUVMappingProcess.h b/thirdparty/assimp/code/PostProcessing/ComputeUVMappingProcess.h index 24f6bb7218..a6d36e06ea 100644 --- a/thirdparty/assimp/code/ComputeUVMappingProcess.h +++ b/thirdparty/assimp/code/PostProcessing/ComputeUVMappingProcess.h @@ -45,7 +45,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_COMPUTEUVMAPPING_H_INC #define AI_COMPUTEUVMAPPING_H_INC -#include "BaseProcess.h" +#include "Common/BaseProcess.h" + #include <assimp/mesh.h> #include <assimp/material.h> #include <assimp/types.h> diff --git a/thirdparty/assimp/code/ConvertToLHProcess.cpp b/thirdparty/assimp/code/PostProcessing/ConvertToLHProcess.cpp index b7cd4f0bc6..b7cd4f0bc6 100644 --- a/thirdparty/assimp/code/ConvertToLHProcess.cpp +++ b/thirdparty/assimp/code/PostProcessing/ConvertToLHProcess.cpp diff --git a/thirdparty/assimp/code/ConvertToLHProcess.h b/thirdparty/assimp/code/PostProcessing/ConvertToLHProcess.h index 63351568d0..f32b91fc39 100644 --- a/thirdparty/assimp/code/ConvertToLHProcess.h +++ b/thirdparty/assimp/code/PostProcessing/ConvertToLHProcess.h @@ -52,7 +52,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define AI_CONVERTTOLHPROCESS_H_INC #include <assimp/types.h> -#include "BaseProcess.h" + +#include "Common/BaseProcess.h" struct aiMesh; struct aiNodeAnim; diff --git a/thirdparty/assimp/code/DeboneProcess.cpp b/thirdparty/assimp/code/PostProcessing/DeboneProcess.cpp index 83b8336bc9..83b8336bc9 100644 --- a/thirdparty/assimp/code/DeboneProcess.cpp +++ b/thirdparty/assimp/code/PostProcessing/DeboneProcess.cpp diff --git a/thirdparty/assimp/code/DeboneProcess.h b/thirdparty/assimp/code/PostProcessing/DeboneProcess.h index ba77aba70e..8b64c2acc6 100644 --- a/thirdparty/assimp/code/DeboneProcess.h +++ b/thirdparty/assimp/code/PostProcessing/DeboneProcess.h @@ -44,17 +44,18 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_DEBONEPROCESS_H_INC #define AI_DEBONEPROCESS_H_INC -#include <vector> -#include <utility> -#include "BaseProcess.h" +#include "Common/BaseProcess.h" #include <assimp/mesh.h> #include <assimp/scene.h> +#include <vector> +#include <utility> + +#// Forward declarations class DeboneTest; -namespace Assimp -{ +namespace Assimp { #if (!defined AI_DEBONE_THRESHOLD) # define AI_DEBONE_THRESHOLD 1.0f @@ -66,14 +67,11 @@ namespace Assimp * the bone are split from the mesh. The split off (new) mesh is boneless. At any * point in time, bones without affect upon a given mesh are to be removed. */ -class DeboneProcess : public BaseProcess -{ +class DeboneProcess : public BaseProcess { public: - DeboneProcess(); ~DeboneProcess(); -public: // ------------------------------------------------------------------- /** Returns whether the processing step is present in the given flag. * @param pFlags The processing flags the importer was called with. @@ -91,7 +89,6 @@ public: void SetupProperties(const Importer* pImp); protected: - // ------------------------------------------------------------------- /** Executes the post processing step on the given imported data. * At the moment a process is not supposed to fail. diff --git a/thirdparty/assimp/code/DropFaceNormalsProcess.cpp b/thirdparty/assimp/code/PostProcessing/DropFaceNormalsProcess.cpp index b11615bb82..b11615bb82 100644 --- a/thirdparty/assimp/code/DropFaceNormalsProcess.cpp +++ b/thirdparty/assimp/code/PostProcessing/DropFaceNormalsProcess.cpp diff --git a/thirdparty/assimp/code/DropFaceNormalsProcess.h b/thirdparty/assimp/code/PostProcessing/DropFaceNormalsProcess.h index 0d116663b7..c710c5a5ee 100644 --- a/thirdparty/assimp/code/DropFaceNormalsProcess.h +++ b/thirdparty/assimp/code/PostProcessing/DropFaceNormalsProcess.h @@ -44,23 +44,20 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_DROPFACENORMALPROCESS_H_INC #define AI_DROPFACENORMALPROCESS_H_INC -#include "BaseProcess.h" +#include "Common/BaseProcess.h" + #include <assimp/mesh.h> -namespace Assimp -{ +namespace Assimp { // --------------------------------------------------------------------------- /** The DropFaceNormalsProcess computes face normals for all faces of all meshes */ -class ASSIMP_API_WINONLY DropFaceNormalsProcess : public BaseProcess -{ +class ASSIMP_API_WINONLY DropFaceNormalsProcess : public BaseProcess { public: - DropFaceNormalsProcess(); ~DropFaceNormalsProcess(); -public: // ------------------------------------------------------------------- /** Returns whether the processing step is present in the given flag field. * @param pFlags The processing flags the importer was called with. A bitwise diff --git a/thirdparty/assimp/code/EmbedTexturesProcess.cpp b/thirdparty/assimp/code/PostProcessing/EmbedTexturesProcess.cpp index 739382a057..739382a057 100644 --- a/thirdparty/assimp/code/EmbedTexturesProcess.cpp +++ b/thirdparty/assimp/code/PostProcessing/EmbedTexturesProcess.cpp diff --git a/thirdparty/assimp/code/EmbedTexturesProcess.h b/thirdparty/assimp/code/PostProcessing/EmbedTexturesProcess.h index cdf40bef74..3c4b2eab4e 100644 --- a/thirdparty/assimp/code/EmbedTexturesProcess.h +++ b/thirdparty/assimp/code/PostProcessing/EmbedTexturesProcess.h @@ -42,7 +42,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once -#include "BaseProcess.h" +#include "Common/BaseProcess.h" #include <string> diff --git a/thirdparty/assimp/code/FindDegenerates.cpp b/thirdparty/assimp/code/PostProcessing/FindDegenerates.cpp index 365f5d7447..50fac46dba 100644 --- a/thirdparty/assimp/code/FindDegenerates.cpp +++ b/thirdparty/assimp/code/PostProcessing/FindDegenerates.cpp @@ -228,6 +228,7 @@ bool FindDegeneratesProcess::ExecuteOnMesh( aiMesh* mesh) { if ( area < 1e-6 ) { if ( mConfigRemoveDegenerates ) { remove_me[ a ] = true; + ++deg; goto evil_jump_outside; } diff --git a/thirdparty/assimp/code/FindDegenerates.h b/thirdparty/assimp/code/PostProcessing/FindDegenerates.h index 880f5f16a2..7a15e77cf1 100644 --- a/thirdparty/assimp/code/FindDegenerates.h +++ b/thirdparty/assimp/code/PostProcessing/FindDegenerates.h @@ -45,7 +45,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_FINDDEGENERATESPROCESS_H_INC #define AI_FINDDEGENERATESPROCESS_H_INC -#include "BaseProcess.h" +#include "Common/BaseProcess.h" + #include <assimp/mesh.h> class FindDegeneratesProcessTest; diff --git a/thirdparty/assimp/code/FindInstancesProcess.cpp b/thirdparty/assimp/code/PostProcessing/FindInstancesProcess.cpp index be1138116e..64907458a1 100644 --- a/thirdparty/assimp/code/FindInstancesProcess.cpp +++ b/thirdparty/assimp/code/PostProcessing/FindInstancesProcess.cpp @@ -137,6 +137,11 @@ void FindInstancesProcess::Execute( aiScene* pScene) aiMesh* inst = pScene->mMeshes[i]; hashes[i] = GetMeshHash(inst); + // Find an appropriate epsilon + // to compare position differences against + float epsilon = ComputePositionEpsilon(inst); + epsilon *= epsilon; + for (int a = i-1; a >= 0; --a) { if (hashes[i] == hashes[a]) { @@ -154,12 +159,7 @@ void FindInstancesProcess::Execute( aiScene* pScene) orig->mPrimitiveTypes != inst->mPrimitiveTypes) continue; - // up to now the meshes are equal. find an appropriate - // epsilon to compare position differences against - float epsilon = ComputePositionEpsilon(inst); - epsilon *= epsilon; - - // now compare vertex positions, normals, + // up to now the meshes are equal. Now compare vertex positions, normals, // tangents and bitangents using this epsilon. if (orig->HasPositions()) { if(!CompareArrays(orig->mVertices,inst->mVertices,orig->mNumVertices,epsilon)) diff --git a/thirdparty/assimp/code/FindInstancesProcess.h b/thirdparty/assimp/code/PostProcessing/FindInstancesProcess.h index ab4a371c71..64b838d7cc 100644 --- a/thirdparty/assimp/code/FindInstancesProcess.h +++ b/thirdparty/assimp/code/PostProcessing/FindInstancesProcess.h @@ -46,8 +46,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_FINDINSTANCES_H_INC #define AI_FINDINSTANCES_H_INC -#include "BaseProcess.h" -#include "ProcessHelper.h" +#include "Common/BaseProcess.h" +#include "PostProcessing/ProcessHelper.h" class FindInstancesProcessTest; namespace Assimp { diff --git a/thirdparty/assimp/code/FindInvalidDataProcess.cpp b/thirdparty/assimp/code/PostProcessing/FindInvalidDataProcess.cpp index 433f042448..433f042448 100644 --- a/thirdparty/assimp/code/FindInvalidDataProcess.cpp +++ b/thirdparty/assimp/code/PostProcessing/FindInvalidDataProcess.cpp diff --git a/thirdparty/assimp/code/FindInvalidDataProcess.h b/thirdparty/assimp/code/PostProcessing/FindInvalidDataProcess.h index 8504fb7b1f..ce7375f34f 100644 --- a/thirdparty/assimp/code/FindInvalidDataProcess.h +++ b/thirdparty/assimp/code/PostProcessing/FindInvalidDataProcess.h @@ -46,7 +46,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_FINDINVALIDDATA_H_INC #define AI_FINDINVALIDDATA_H_INC -#include "BaseProcess.h" +#include "Common/BaseProcess.h" + #include <assimp/types.h> #include <assimp/anim.h> diff --git a/thirdparty/assimp/code/FixNormalsStep.cpp b/thirdparty/assimp/code/PostProcessing/FixNormalsStep.cpp index bbbe6899b4..bbbe6899b4 100644 --- a/thirdparty/assimp/code/FixNormalsStep.cpp +++ b/thirdparty/assimp/code/PostProcessing/FixNormalsStep.cpp diff --git a/thirdparty/assimp/code/FixNormalsStep.h b/thirdparty/assimp/code/PostProcessing/FixNormalsStep.h index 6be27faef6..f60ce596a4 100644 --- a/thirdparty/assimp/code/FixNormalsStep.h +++ b/thirdparty/assimp/code/PostProcessing/FixNormalsStep.h @@ -45,7 +45,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_FIXNORMALSPROCESS_H_INC #define AI_FIXNORMALSPROCESS_H_INC -#include "BaseProcess.h" +#include "Common/BaseProcess.h" struct aiMesh; diff --git a/thirdparty/assimp/code/PostProcessing/GenBoundingBoxesProcess.cpp b/thirdparty/assimp/code/PostProcessing/GenBoundingBoxesProcess.cpp new file mode 100644 index 0000000000..c013454fc3 --- /dev/null +++ b/thirdparty/assimp/code/PostProcessing/GenBoundingBoxesProcess.cpp @@ -0,0 +1,115 @@ +/* +--------------------------------------------------------------------------- +Open Asset Import Library (assimp) +--------------------------------------------------------------------------- + +Copyright (c) 2006-2019, assimp team + +All rights reserved. + +Redistribution and use of this software in source and binary forms, +with or without modification, are permitted provided that the following +conditions are met: + +* Redistributions of source code must retain the above + copyright notice, this list of conditions and the + following disclaimer. + +* Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the + following disclaimer in the documentation and/or other + materials provided with the distribution. + +* Neither the name of the assimp team, nor the names of its + contributors may be used to endorse or promote products + derived from this software without specific prior + written permission of the assimp team. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +--------------------------------------------------------------------------- +*/ + +#ifndef ASSIMP_BUILD_NO_GENBOUNDINGBOXES_PROCESS + +#include "PostProcessing/GenBoundingBoxesProcess.h" + +#include <assimp/postprocess.h> +#include <assimp/scene.h> + +namespace Assimp { + +GenBoundingBoxesProcess::GenBoundingBoxesProcess() +: BaseProcess() { + +} + +GenBoundingBoxesProcess::~GenBoundingBoxesProcess() { + // empty +} + +bool GenBoundingBoxesProcess::IsActive(unsigned int pFlags) const { + return 0 != ( pFlags & aiProcess_GenBoundingBoxes ); +} + +void checkMesh(aiMesh* mesh, aiVector3D& min, aiVector3D& max) { + ai_assert(nullptr != mesh); + + if (0 == mesh->mNumVertices) { + return; + } + + for (unsigned int i = 0; i < mesh->mNumVertices; ++i) { + const aiVector3D &pos = mesh->mVertices[i]; + if (pos.x < min.x) { + min.x = pos.x; + } + if (pos.y < min.y) { + min.y = pos.y; + } + if (pos.z < min.z) { + min.z = pos.z; + } + + if (pos.x > max.x) { + max.x = pos.x; + } + if (pos.y > max.y) { + max.y = pos.y; + } + if (pos.z > max.z) { + max.z = pos.z; + } + } +} + +void GenBoundingBoxesProcess::Execute(aiScene* pScene) { + if (nullptr == pScene) { + return; + } + + for (unsigned int i = 0; i < pScene->mNumMeshes; ++i) { + aiMesh* mesh = pScene->mMeshes[i]; + if (nullptr == mesh) { + continue; + } + + aiVector3D min(999999, 999999, 999999), max(-999999, -999999, -999999); + checkMesh(mesh, min, max); + mesh->mAABB.mMin = min; + mesh->mAABB.mMax = max; + } +} + +} // Namespace Assimp + +#endif // ASSIMP_BUILD_NO_GENBOUNDINGBOXES_PROCESS diff --git a/thirdparty/assimp/code/PostProcessing/GenBoundingBoxesProcess.h b/thirdparty/assimp/code/PostProcessing/GenBoundingBoxesProcess.h new file mode 100644 index 0000000000..4b43c82a42 --- /dev/null +++ b/thirdparty/assimp/code/PostProcessing/GenBoundingBoxesProcess.h @@ -0,0 +1,76 @@ +/* +--------------------------------------------------------------------------- +Open Asset Import Library (assimp) +--------------------------------------------------------------------------- + +Copyright (c) 2006-2019, assimp team + +All rights reserved. + +Redistribution and use of this software in source and binary forms, +with or without modification, are permitted provided that the following +conditions are met: + +* Redistributions of source code must retain the above + copyright notice, this list of conditions and the + following disclaimer. + +* Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the + following disclaimer in the documentation and/or other + materials provided with the distribution. + +* Neither the name of the assimp team, nor the names of its + contributors may be used to endorse or promote products + derived from this software without specific prior + written permission of the assimp team. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +--------------------------------------------------------------------------- +*/ + +/** @file Defines a post-processing step to generate Axis-aligned bounding + * volumes for all meshes. + */ + +#pragma once + +#ifndef AI_GENBOUNDINGBOXESPROCESS_H_INC +#define AI_GENBOUNDINGBOXESPROCESS_H_INC + +#ifndef ASSIMP_BUILD_NO_GENBOUNDINGBOXES_PROCESS + +#include "Common/BaseProcess.h" + +namespace Assimp { + +/** Post-processing process to find axis-aligned bounding volumes for amm meshes + * used in a scene + */ +class ASSIMP_API GenBoundingBoxesProcess : public BaseProcess { +public: + /// The class constructor. + GenBoundingBoxesProcess(); + /// The class destructor. + ~GenBoundingBoxesProcess(); + /// Will return true, if aiProcess_GenBoundingBoxes is defined. + bool IsActive(unsigned int pFlags) const override; + /// The execution callback. + void Execute(aiScene* pScene) override; +}; + +} // Namespace Assimp + +#endif // #ifndef ASSIMP_BUILD_NO_GENBOUNDINGBOXES_PROCESS + +#endif // AI_GENBOUNDINGBOXESPROCESS_H_INC diff --git a/thirdparty/assimp/code/GenFaceNormalsProcess.cpp b/thirdparty/assimp/code/PostProcessing/GenFaceNormalsProcess.cpp index 028334dec7..028334dec7 100644 --- a/thirdparty/assimp/code/GenFaceNormalsProcess.cpp +++ b/thirdparty/assimp/code/PostProcessing/GenFaceNormalsProcess.cpp diff --git a/thirdparty/assimp/code/GenFaceNormalsProcess.h b/thirdparty/assimp/code/PostProcessing/GenFaceNormalsProcess.h index c80ec9fddc..c641fd6353 100644 --- a/thirdparty/assimp/code/GenFaceNormalsProcess.h +++ b/thirdparty/assimp/code/PostProcessing/GenFaceNormalsProcess.h @@ -44,7 +44,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_GENFACENORMALPROCESS_H_INC #define AI_GENFACENORMALPROCESS_H_INC -#include "BaseProcess.h" +#include "Common/BaseProcess.h" #include <assimp/mesh.h> namespace Assimp diff --git a/thirdparty/assimp/code/GenVertexNormalsProcess.cpp b/thirdparty/assimp/code/PostProcessing/GenVertexNormalsProcess.cpp index 3f6c2f86bd..3f6c2f86bd 100644 --- a/thirdparty/assimp/code/GenVertexNormalsProcess.cpp +++ b/thirdparty/assimp/code/PostProcessing/GenVertexNormalsProcess.cpp diff --git a/thirdparty/assimp/code/GenVertexNormalsProcess.h b/thirdparty/assimp/code/PostProcessing/GenVertexNormalsProcess.h index 9142ad26f5..2ceee17e85 100644 --- a/thirdparty/assimp/code/GenVertexNormalsProcess.h +++ b/thirdparty/assimp/code/PostProcessing/GenVertexNormalsProcess.h @@ -45,24 +45,24 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_GENVERTEXNORMALPROCESS_H_INC #define AI_GENVERTEXNORMALPROCESS_H_INC -#include "BaseProcess.h" +#include "Common/assbin_chunks.h" +#include "Common/BaseProcess.h" + #include <assimp/mesh.h> +// Forward declarations class GenNormalsTest; namespace Assimp { // --------------------------------------------------------------------------- -/** The GenFaceNormalsProcess computes vertex normals for all vertizes +/** The GenFaceNormalsProcess computes vertex normals for all vertices */ -class ASSIMP_API GenVertexNormalsProcess : public BaseProcess -{ +class ASSIMP_API GenVertexNormalsProcess : public BaseProcess { public: - GenVertexNormalsProcess(); ~GenVertexNormalsProcess(); -public: // ------------------------------------------------------------------- /** Returns whether the processing step is present in the given flag. * @param pFlags The processing flags the importer was called with. @@ -88,13 +88,10 @@ public: // setter for configMaxAngle - inline void SetMaxSmoothAngle(ai_real f) - { + inline void SetMaxSmoothAngle(ai_real f) { configMaxAngle =f; } -public: - // ------------------------------------------------------------------- /** Computes normals for a specific mesh * @param pcMesh Mesh @@ -104,7 +101,6 @@ public: bool GenMeshVertexNormals (aiMesh* pcMesh, unsigned int meshIndex); private: - /** Configuration option: maximum smoothing angle, in radians*/ ai_real configMaxAngle; mutable bool force_ = false; diff --git a/thirdparty/assimp/code/ImproveCacheLocality.cpp b/thirdparty/assimp/code/PostProcessing/ImproveCacheLocality.cpp index ace9d95ff8..d0a016fa42 100644 --- a/thirdparty/assimp/code/ImproveCacheLocality.cpp +++ b/thirdparty/assimp/code/PostProcessing/ImproveCacheLocality.cpp @@ -45,14 +45,13 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * <br> * The algorithm is roughly basing on this paper: * http://www.cs.princeton.edu/gfx/pubs/Sander_2007_%3ETR/tipsy.pdf - * .. although overdraw rduction isn't implemented yet ... + * .. although overdraw reduction isn't implemented yet ... */ - - // internal headers -#include "ImproveCacheLocality.h" -#include "VertexTriangleAdjacency.h" +#include "PostProcessing/ImproveCacheLocality.h" +#include "Common/VertexTriangleAdjacency.h" + #include <assimp/StringUtils.h> #include <assimp/postprocess.h> #include <assimp/scene.h> @@ -64,36 +63,33 @@ using namespace Assimp; // ------------------------------------------------------------------------------------------------ // Constructor to be privately used by Importer -ImproveCacheLocalityProcess::ImproveCacheLocalityProcess() { - configCacheDepth = PP_ICL_PTCACHE_SIZE; +ImproveCacheLocalityProcess::ImproveCacheLocalityProcess() +: mConfigCacheDepth(PP_ICL_PTCACHE_SIZE) { + // empty } // ------------------------------------------------------------------------------------------------ // Destructor, private as well -ImproveCacheLocalityProcess::~ImproveCacheLocalityProcess() -{ +ImproveCacheLocalityProcess::~ImproveCacheLocalityProcess() { // nothing to do here } // ------------------------------------------------------------------------------------------------ // Returns whether the processing step is present in the given flag field. -bool ImproveCacheLocalityProcess::IsActive( unsigned int pFlags) const -{ +bool ImproveCacheLocalityProcess::IsActive( unsigned int pFlags) const { return (pFlags & aiProcess_ImproveCacheLocality) != 0; } // ------------------------------------------------------------------------------------------------ // Setup configuration -void ImproveCacheLocalityProcess::SetupProperties(const Importer* pImp) -{ +void ImproveCacheLocalityProcess::SetupProperties(const Importer* pImp) { // AI_CONFIG_PP_ICL_PTCACHE_SIZE controls the target cache size for the optimizer - configCacheDepth = pImp->GetPropertyInteger(AI_CONFIG_PP_ICL_PTCACHE_SIZE,PP_ICL_PTCACHE_SIZE); + mConfigCacheDepth = pImp->GetPropertyInteger(AI_CONFIG_PP_ICL_PTCACHE_SIZE,PP_ICL_PTCACHE_SIZE); } // ------------------------------------------------------------------------------------------------ // Executes the post processing step on the given imported data. -void ImproveCacheLocalityProcess::Execute( aiScene* pScene) -{ +void ImproveCacheLocalityProcess::Execute( aiScene* pScene) { if (!pScene->mNumMeshes) { ASSIMP_LOG_DEBUG("ImproveCacheLocalityProcess skipped; there are no meshes"); return; @@ -103,7 +99,7 @@ void ImproveCacheLocalityProcess::Execute( aiScene* pScene) float out = 0.f; unsigned int numf = 0, numm = 0; - for( unsigned int a = 0; a < pScene->mNumMeshes; a++){ + for( unsigned int a = 0; a < pScene->mNumMeshes; ++a ){ const float res = ProcessMesh( pScene->mMeshes[a],a); if (res) { numf += pScene->mMeshes[a]->mNumFaces; @@ -121,44 +117,41 @@ void ImproveCacheLocalityProcess::Execute( aiScene* pScene) // ------------------------------------------------------------------------------------------------ // Improves the cache coherency of a specific mesh -float ImproveCacheLocalityProcess::ProcessMesh( aiMesh* pMesh, unsigned int meshNum) -{ +ai_real ImproveCacheLocalityProcess::ProcessMesh( aiMesh* pMesh, unsigned int meshNum) { // TODO: rewrite this to use std::vector or boost::shared_array - ai_assert(NULL != pMesh); + ai_assert(nullptr != pMesh); // Check whether the input data is valid // - there must be vertices and faces // - all faces must be triangulated or we can't operate on them if (!pMesh->HasFaces() || !pMesh->HasPositions()) - return 0.f; + return static_cast<ai_real>(0.f); if (pMesh->mPrimitiveTypes != aiPrimitiveType_TRIANGLE) { ASSIMP_LOG_ERROR("This algorithm works on triangle meshes only"); - return 0.f; + return static_cast<ai_real>(0.f); } - if(pMesh->mNumVertices <= configCacheDepth) { - return 0.f; + if(pMesh->mNumVertices <= mConfigCacheDepth) { + return static_cast<ai_real>(0.f); } - float fACMR = 3.f; + ai_real fACMR = 3.f; const aiFace* const pcEnd = pMesh->mFaces+pMesh->mNumFaces; // Input ACMR is for logging purposes only if (!DefaultLogger::isNullLogger()) { - unsigned int* piFIFOStack = new unsigned int[configCacheDepth]; - memset(piFIFOStack,0xff,configCacheDepth*sizeof(unsigned int)); + unsigned int* piFIFOStack = new unsigned int[mConfigCacheDepth]; + memset(piFIFOStack,0xff,mConfigCacheDepth*sizeof(unsigned int)); unsigned int* piCur = piFIFOStack; - const unsigned int* const piCurEnd = piFIFOStack + configCacheDepth; + const unsigned int* const piCurEnd = piFIFOStack + mConfigCacheDepth; // count the number of cache misses unsigned int iCacheMisses = 0; for (const aiFace* pcFace = pMesh->mFaces;pcFace != pcEnd;++pcFace) { - for (unsigned int qq = 0; qq < 3;++qq) { bool bInCache = false; - for (unsigned int* pp = piFIFOStack;pp < piCurEnd;++pp) { if (*pp == pcFace->mIndices[qq]) { // the vertex is in cache @@ -176,7 +169,7 @@ float ImproveCacheLocalityProcess::ProcessMesh( aiMesh* pMesh, unsigned int mesh } } delete[] piFIFOStack; - fACMR = (float)iCacheMisses / pMesh->mNumFaces; + fACMR = (ai_real) iCacheMisses / pMesh->mNumFaces; if (3.0 == fACMR) { char szBuff[128]; // should be sufficiently large in every case @@ -185,7 +178,7 @@ float ImproveCacheLocalityProcess::ProcessMesh( aiMesh* pMesh, unsigned int mesh // smaller than 3.0 ... ai_snprintf(szBuff,128,"Mesh %u: Not suitable for vcache optimization",meshNum); ASSIMP_LOG_WARN(szBuff); - return 0.f; + return static_cast<ai_real>(0.f); } } @@ -258,7 +251,7 @@ float ImproveCacheLocalityProcess::ProcessMesh( aiMesh* pMesh, unsigned int mesh int ivdx = 0; int ics = 1; - int iStampCnt = configCacheDepth+1; + int iStampCnt = mConfigCacheDepth+1; while (ivdx >= 0) { unsigned int icnt = piNumTriPtrNoModify[ivdx]; @@ -294,7 +287,7 @@ float ImproveCacheLocalityProcess::ProcessMesh( aiMesh* pMesh, unsigned int mesh *piCSIter++ = dp; // if the vertex is not yet in cache, set its cache count - if (iStampCnt-piCachingStamps[dp] > configCacheDepth) { + if (iStampCnt-piCachingStamps[dp] > mConfigCacheDepth) { piCachingStamps[dp] = iStampCnt++; ++iCacheMisses; } @@ -319,7 +312,7 @@ float ImproveCacheLocalityProcess::ProcessMesh( aiMesh* pMesh, unsigned int mesh // will the vertex be in cache, even after fanning occurs? unsigned int tmp; - if ((tmp = iStampCnt-piCachingStamps[dp]) + 2*piNumTriPtr[dp] <= configCacheDepth) { + if ((tmp = iStampCnt-piCachingStamps[dp]) + 2*piNumTriPtr[dp] <= mConfigCacheDepth) { priority = tmp; } @@ -356,7 +349,7 @@ float ImproveCacheLocalityProcess::ProcessMesh( aiMesh* pMesh, unsigned int mesh } } } - float fACMR2 = 0.0f; + ai_real fACMR2 = 0.0f; if (!DefaultLogger::isNullLogger()) { fACMR2 = (float)iCacheMisses / pMesh->mNumFaces; diff --git a/thirdparty/assimp/code/ImproveCacheLocality.h b/thirdparty/assimp/code/PostProcessing/ImproveCacheLocality.h index 1b29ee0d6e..de25ecd9fb 100644 --- a/thirdparty/assimp/code/ImproveCacheLocality.h +++ b/thirdparty/assimp/code/PostProcessing/ImproveCacheLocality.h @@ -45,7 +45,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_IMPROVECACHELOCALITY_H_INC #define AI_IMPROVECACHELOCALITY_H_INC -#include "BaseProcess.h" +#include "Common/BaseProcess.h" + #include <assimp/types.h> struct aiMesh; @@ -87,12 +88,12 @@ protected: * @param pMesh The mesh to process. * @param meshNum Index of the mesh to process */ - float ProcessMesh( aiMesh* pMesh, unsigned int meshNum); + ai_real ProcessMesh( aiMesh* pMesh, unsigned int meshNum); private: //! Configuration parameter: specifies the size of the cache to //! optimize the vertex data for. - unsigned int configCacheDepth; + unsigned int mConfigCacheDepth; }; } // end of namespace Assimp diff --git a/thirdparty/assimp/code/JoinVerticesProcess.cpp b/thirdparty/assimp/code/PostProcessing/JoinVerticesProcess.cpp index 914ec05b46..914ec05b46 100644 --- a/thirdparty/assimp/code/JoinVerticesProcess.cpp +++ b/thirdparty/assimp/code/PostProcessing/JoinVerticesProcess.cpp diff --git a/thirdparty/assimp/code/JoinVerticesProcess.h b/thirdparty/assimp/code/PostProcessing/JoinVerticesProcess.h index 66fa362de2..e017ae62db 100644 --- a/thirdparty/assimp/code/JoinVerticesProcess.h +++ b/thirdparty/assimp/code/PostProcessing/JoinVerticesProcess.h @@ -45,7 +45,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_JOINVERTICESPROCESS_H_INC #define AI_JOINVERTICESPROCESS_H_INC -#include "BaseProcess.h" +#include "Common/BaseProcess.h" + #include <assimp/types.h> struct aiMesh; @@ -61,13 +62,11 @@ namespace Assimp * erases all but one of the copies. This usually reduces the number of vertices * in a mesh by a serious amount and is the standard form to render a mesh. */ -class ASSIMP_API JoinVerticesProcess : public BaseProcess -{ +class ASSIMP_API JoinVerticesProcess : public BaseProcess { public: JoinVerticesProcess(); ~JoinVerticesProcess(); -public: // ------------------------------------------------------------------- /** Returns whether the processing step is present in the given flag field. * @param pFlags The processing flags the importer was called with. A bitwise @@ -83,15 +82,12 @@ public: */ void Execute( aiScene* pScene); -public: // ------------------------------------------------------------------- /** Unites identical vertices in the given mesh. * @param pMesh The mesh to process. * @param meshIndex Index of the mesh to process */ int ProcessMesh( aiMesh* pMesh, unsigned int meshIndex); - -private: }; } // end of namespace Assimp diff --git a/thirdparty/assimp/code/LimitBoneWeightsProcess.cpp b/thirdparty/assimp/code/PostProcessing/LimitBoneWeightsProcess.cpp index d560f19287..d560f19287 100644 --- a/thirdparty/assimp/code/LimitBoneWeightsProcess.cpp +++ b/thirdparty/assimp/code/PostProcessing/LimitBoneWeightsProcess.cpp diff --git a/thirdparty/assimp/code/LimitBoneWeightsProcess.h b/thirdparty/assimp/code/PostProcessing/LimitBoneWeightsProcess.h index 3602fd8edf..73c2a68d53 100644 --- a/thirdparty/assimp/code/LimitBoneWeightsProcess.h +++ b/thirdparty/assimp/code/PostProcessing/LimitBoneWeightsProcess.h @@ -44,14 +44,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_LIMITBONEWEIGHTSPROCESS_H_INC #define AI_LIMITBONEWEIGHTSPROCESS_H_INC -#include "BaseProcess.h" +#include "Common/BaseProcess.h" +// Forward declarations struct aiMesh; class LimitBoneWeightsTest; -namespace Assimp -{ +namespace Assimp { // NOTE: If you change these limits, don't forget to change the // corresponding values in all Assimp ports @@ -72,14 +72,11 @@ namespace Assimp * The other weights on this bone are then renormalized to assure the sum weight * to be 1. */ -class ASSIMP_API LimitBoneWeightsProcess : public BaseProcess -{ +class ASSIMP_API LimitBoneWeightsProcess : public BaseProcess { public: - LimitBoneWeightsProcess(); ~LimitBoneWeightsProcess(); -public: // ------------------------------------------------------------------- /** Returns whether the processing step is present in the given flag. * @param pFlags The processing flags the importer was called with. @@ -96,8 +93,6 @@ public: */ void SetupProperties(const Importer* pImp); -public: - // ------------------------------------------------------------------- /** Limits the bone weight count for all vertices in the given mesh. * @param pMesh The mesh to process. @@ -111,34 +106,29 @@ public: */ void Execute( aiScene* pScene); - -public: - // ------------------------------------------------------------------- /** Describes a bone weight on a vertex */ - struct Weight - { + struct Weight { unsigned int mBone; ///< Index of the bone float mWeight; ///< Weight of that bone on this vertex Weight() AI_NO_EXCEPT : mBone(0) - , mWeight(0.0f) - { } + , mWeight(0.0f) { + // empty + } Weight( unsigned int pBone, float pWeight) - { - mBone = pBone; - mWeight = pWeight; + : mBone(pBone) + , mWeight(pWeight) { + // empty } /** Comparison operator to sort bone weights by descending weight */ - bool operator < (const Weight& pWeight) const - { + bool operator < (const Weight& pWeight) const { return mWeight > pWeight.mWeight; } }; -public: /** Maximum number of bones influencing any single vertex. */ unsigned int mMaxWeights; }; diff --git a/thirdparty/assimp/code/MakeVerboseFormat.cpp b/thirdparty/assimp/code/PostProcessing/MakeVerboseFormat.cpp index 50ff5ed93d..50ff5ed93d 100644 --- a/thirdparty/assimp/code/MakeVerboseFormat.cpp +++ b/thirdparty/assimp/code/PostProcessing/MakeVerboseFormat.cpp diff --git a/thirdparty/assimp/code/MakeVerboseFormat.h b/thirdparty/assimp/code/PostProcessing/MakeVerboseFormat.h index d12db63ae1..1adf8e2f69 100644 --- a/thirdparty/assimp/code/MakeVerboseFormat.h +++ b/thirdparty/assimp/code/PostProcessing/MakeVerboseFormat.h @@ -46,7 +46,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_MAKEVERBOSEFORMAT_H_INC #define AI_MAKEVERBOSEFORMAT_H_INC -#include "BaseProcess.h" +#include "Common/BaseProcess.h" + struct aiMesh; namespace Assimp { diff --git a/thirdparty/assimp/code/OptimizeGraph.cpp b/thirdparty/assimp/code/PostProcessing/OptimizeGraph.cpp index add9ab79e1..5db51f58b6 100644 --- a/thirdparty/assimp/code/OptimizeGraph.cpp +++ b/thirdparty/assimp/code/PostProcessing/OptimizeGraph.cpp @@ -5,8 +5,6 @@ Open Asset Import Library (assimp) Copyright (c) 2006-2019, assimp team - - All rights reserved. Redistribution and use of this software in source and binary forms, diff --git a/thirdparty/assimp/code/OptimizeGraph.h b/thirdparty/assimp/code/PostProcessing/OptimizeGraph.h index e5bbed7679..82cc5db3fe 100644 --- a/thirdparty/assimp/code/OptimizeGraph.h +++ b/thirdparty/assimp/code/PostProcessing/OptimizeGraph.h @@ -46,13 +46,18 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_OPTIMIZEGRAPHPROCESS_H_INC #define AI_OPTIMIZEGRAPHPROCESS_H_INC -#include "BaseProcess.h" -#include "ProcessHelper.h" +#include "Common/BaseProcess.h" +#include "PostProcessing/ProcessHelper.h" + #include <assimp/types.h> + #include <set> +// Forward declarations struct aiMesh; + class OptimizeGraphProcessTest; + namespace Assimp { // ----------------------------------------------------------------------------- @@ -64,14 +69,11 @@ namespace Assimp { * @see aiProcess_OptimizeGraph for a detailed description of the * algorithm being applied. */ -class OptimizeGraphProcess : public BaseProcess -{ +class OptimizeGraphProcess : public BaseProcess { public: - OptimizeGraphProcess(); ~OptimizeGraphProcess(); -public: // ------------------------------------------------------------------- bool IsActive( unsigned int pFlags) const; @@ -81,14 +83,12 @@ public: // ------------------------------------------------------------------- void SetupProperties(const Importer* pImp); - // ------------------------------------------------------------------- /** @brief Add a list of node names to be locked and not modified. * @param in List of nodes. See #AI_CONFIG_PP_OG_EXCLUDE_LIST for * format explanations. */ - inline void AddLockedNodeList(std::string& in) - { + inline void AddLockedNodeList(std::string& in) { ConvertListToStrings (in,locked_nodes); } @@ -96,8 +96,7 @@ public: /** @brief Add another node to be locked and not modified. * @param name Name to be locked */ - inline void AddLockedNode(std::string& name) - { + inline void AddLockedNode(std::string& name) { locked_nodes.push_back(name); } @@ -105,25 +104,21 @@ public: /** @brief Remove a node from the list of locked nodes. * @param name Name to be unlocked */ - inline void RemoveLockedNode(std::string& name) - { + inline void RemoveLockedNode(std::string& name) { locked_nodes.remove(name); } protected: - void CollectNewChildren(aiNode* nd, std::list<aiNode*>& nodes); void FindInstancedMeshes (aiNode* pNode); private: - #ifdef AI_OG_USE_HASHING typedef std::set<unsigned int> LockedSetType; #else typedef std::set<std::string> LockedSetType; #endif - //! Scene we're working with aiScene* mScene; diff --git a/thirdparty/assimp/code/OptimizeMeshes.cpp b/thirdparty/assimp/code/PostProcessing/OptimizeMeshes.cpp index 3f6765f6ca..3f6765f6ca 100644 --- a/thirdparty/assimp/code/OptimizeMeshes.cpp +++ b/thirdparty/assimp/code/PostProcessing/OptimizeMeshes.cpp diff --git a/thirdparty/assimp/code/OptimizeMeshes.h b/thirdparty/assimp/code/PostProcessing/OptimizeMeshes.h index 9f46f349b4..dec4ab52de 100644 --- a/thirdparty/assimp/code/OptimizeMeshes.h +++ b/thirdparty/assimp/code/PostProcessing/OptimizeMeshes.h @@ -46,8 +46,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_OPTIMIZEMESHESPROCESS_H_INC #define AI_OPTIMIZEMESHESPROCESS_H_INC -#include "BaseProcess.h" +#include "Common/BaseProcess.h" + #include <assimp/types.h> + #include <vector> struct aiMesh; @@ -64,16 +66,14 @@ namespace Assimp { * * @note Instanced meshes are currently not processed. */ -class OptimizeMeshesProcess : public BaseProcess -{ +class OptimizeMeshesProcess : public BaseProcess { public: /// @brief The class constructor. OptimizeMeshesProcess(); - /// @brief The class destcructor, + /// @brief The class destructor. ~OptimizeMeshesProcess(); - /** @brief Internal utility to store additional mesh info */ struct MeshInfo { diff --git a/thirdparty/assimp/code/PretransformVertices.cpp b/thirdparty/assimp/code/PostProcessing/PretransformVertices.cpp index 52001a0578..52001a0578 100644 --- a/thirdparty/assimp/code/PretransformVertices.cpp +++ b/thirdparty/assimp/code/PostProcessing/PretransformVertices.cpp diff --git a/thirdparty/assimp/code/PretransformVertices.h b/thirdparty/assimp/code/PostProcessing/PretransformVertices.h index b7329af130..b2982951e0 100644 --- a/thirdparty/assimp/code/PretransformVertices.h +++ b/thirdparty/assimp/code/PostProcessing/PretransformVertices.h @@ -47,13 +47,18 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_PRETRANSFORMVERTICES_H_INC #define AI_PRETRANSFORMVERTICES_H_INC -#include "BaseProcess.h" +#include "Common/BaseProcess.h" + #include <assimp/mesh.h> + #include <list> #include <vector> +// Forward declarations struct aiNode; + class PretransformVerticesTest; + namespace Assimp { // --------------------------------------------------------------------------- @@ -80,10 +85,10 @@ public: // ------------------------------------------------------------------- /** @brief Toggle the 'keep hierarchy' option - * @param d hm ... difficult to guess what this means, hu!? + * @param keep true for keep configuration. */ - void KeepHierarchy(bool d) { - configKeepHierarchy = d; + void KeepHierarchy(bool keep) { + configKeepHierarchy = keep; } // ------------------------------------------------------------------- @@ -148,8 +153,6 @@ private: // Build reference counters for all meshes void BuildMeshRefCountArray(aiNode* nd, unsigned int * refs); - - //! Configuration option: keep scene hierarchy as long as possible bool configKeepHierarchy; bool configNormalize; diff --git a/thirdparty/assimp/code/ProcessHelper.cpp b/thirdparty/assimp/code/PostProcessing/ProcessHelper.cpp index 59869fdff7..59869fdff7 100644 --- a/thirdparty/assimp/code/ProcessHelper.cpp +++ b/thirdparty/assimp/code/PostProcessing/ProcessHelper.cpp diff --git a/thirdparty/assimp/code/ProcessHelper.h b/thirdparty/assimp/code/PostProcessing/ProcessHelper.h index c59f3217bf..0afcc41420 100644 --- a/thirdparty/assimp/code/ProcessHelper.h +++ b/thirdparty/assimp/code/PostProcessing/ProcessHelper.h @@ -51,7 +51,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <assimp/scene.h> #include <assimp/SpatialSort.h> -#include "BaseProcess.h" +#include "Common/BaseProcess.h" #include <assimp/ParsingUtils.h> #include <list> diff --git a/thirdparty/assimp/code/RemoveRedundantMaterials.cpp b/thirdparty/assimp/code/PostProcessing/RemoveRedundantMaterials.cpp index 632bdca3fe..49ec8f5c47 100644 --- a/thirdparty/assimp/code/RemoveRedundantMaterials.cpp +++ b/thirdparty/assimp/code/PostProcessing/RemoveRedundantMaterials.cpp @@ -49,7 +49,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "RemoveRedundantMaterials.h" #include <assimp/ParsingUtils.h> #include "ProcessHelper.h" -#include "MaterialSystem.h" +#include "Material/MaterialSystem.h" #include <stdio.h> using namespace Assimp; @@ -57,7 +57,7 @@ using namespace Assimp; // ------------------------------------------------------------------------------------------------ // Constructor to be privately used by Importer RemoveRedundantMatsProcess::RemoveRedundantMatsProcess() -: configFixedMaterials() { +: mConfigFixedMaterials() { // nothing to do here } @@ -80,7 +80,7 @@ bool RemoveRedundantMatsProcess::IsActive( unsigned int pFlags) const void RemoveRedundantMatsProcess::SetupProperties(const Importer* pImp) { // Get value of AI_CONFIG_PP_RRM_EXCLUDE_LIST - configFixedMaterials = pImp->GetPropertyString(AI_CONFIG_PP_RRM_EXCLUDE_LIST,""); + mConfigFixedMaterials = pImp->GetPropertyString(AI_CONFIG_PP_RRM_EXCLUDE_LIST,""); } // ------------------------------------------------------------------------------------------------ @@ -100,10 +100,10 @@ void RemoveRedundantMatsProcess::Execute( aiScene* pScene) // If a list of materials to be excluded was given, match the list with // our imported materials and 'salt' all positive matches to ensure that // we get unique hashes later. - if (configFixedMaterials.length()) { + if (mConfigFixedMaterials.length()) { std::list<std::string> strings; - ConvertListToStrings(configFixedMaterials,strings); + ConvertListToStrings(mConfigFixedMaterials,strings); for (unsigned int i = 0; i < pScene->mNumMaterials;++i) { aiMaterial* mat = pScene->mMaterials[i]; diff --git a/thirdparty/assimp/code/RemoveRedundantMaterials.h b/thirdparty/assimp/code/PostProcessing/RemoveRedundantMaterials.h index dbd4d44cc0..1f32a0abfb 100644 --- a/thirdparty/assimp/code/RemoveRedundantMaterials.h +++ b/thirdparty/assimp/code/PostProcessing/RemoveRedundantMaterials.h @@ -46,7 +46,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_REMOVEREDUNDANTMATERIALS_H_INC #define AI_REMOVEREDUNDANTMATERIALS_H_INC -#include "BaseProcess.h" +#include "Common/BaseProcess.h" #include <assimp/mesh.h> class RemoveRedundantMatsTest; @@ -57,8 +57,7 @@ namespace Assimp { /** RemoveRedundantMatsProcess: Post-processing step to remove redundant * materials from the imported scene. */ -class ASSIMP_API RemoveRedundantMatsProcess : public BaseProcess -{ +class ASSIMP_API RemoveRedundantMatsProcess : public BaseProcess { public: /// The default class constructor. RemoveRedundantMatsProcess(); @@ -66,7 +65,6 @@ public: /// The class destructor. ~RemoveRedundantMatsProcess(); -public: // ------------------------------------------------------------------- // Check whether step is active bool IsActive( unsigned int pFlags) const; @@ -79,27 +77,25 @@ public: // Setup import settings void SetupProperties(const Importer* pImp); - // ------------------------------------------------------------------- - /** @brief Set list of fixed (unmutable) materials + /** @brief Set list of fixed (inmutable) materials * @param fixed See #AI_CONFIG_PP_RRM_EXCLUDE_LIST */ void SetFixedMaterialsString(const std::string& fixed = "") { - configFixedMaterials = fixed; + mConfigFixedMaterials = fixed; } // ------------------------------------------------------------------- - /** @brief Get list of fixed (unmutable) materials + /** @brief Get list of fixed (inmutable) materials * @return See #AI_CONFIG_PP_RRM_EXCLUDE_LIST */ const std::string& GetFixedMaterialsString() const { - return configFixedMaterials; + return mConfigFixedMaterials; } private: - //! Configuration option: list of all fixed materials - std::string configFixedMaterials; + std::string mConfigFixedMaterials; }; } // end of namespace Assimp diff --git a/thirdparty/assimp/code/RemoveVCProcess.cpp b/thirdparty/assimp/code/PostProcessing/RemoveVCProcess.cpp index 99fd47a3aa..99fd47a3aa 100644 --- a/thirdparty/assimp/code/RemoveVCProcess.cpp +++ b/thirdparty/assimp/code/PostProcessing/RemoveVCProcess.cpp diff --git a/thirdparty/assimp/code/RemoveVCProcess.h b/thirdparty/assimp/code/PostProcessing/RemoveVCProcess.h index 617d7b9b20..7bb21a8330 100644 --- a/thirdparty/assimp/code/RemoveVCProcess.h +++ b/thirdparty/assimp/code/PostProcessing/RemoveVCProcess.h @@ -44,7 +44,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_REMOVEVCPROCESS_H_INCLUDED #define AI_REMOVEVCPROCESS_H_INCLUDED -#include "BaseProcess.h" +#include "Common/BaseProcess.h" + #include <assimp/mesh.h> class RemoveVCProcessTest; diff --git a/thirdparty/assimp/code/ScaleProcess.cpp b/thirdparty/assimp/code/PostProcessing/ScaleProcess.cpp index 6d458c4b11..6d458c4b11 100644 --- a/thirdparty/assimp/code/ScaleProcess.cpp +++ b/thirdparty/assimp/code/PostProcessing/ScaleProcess.cpp diff --git a/thirdparty/assimp/code/ScaleProcess.h b/thirdparty/assimp/code/PostProcessing/ScaleProcess.h index 55146ae064..2567378759 100644 --- a/thirdparty/assimp/code/ScaleProcess.h +++ b/thirdparty/assimp/code/PostProcessing/ScaleProcess.h @@ -41,7 +41,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #pragma once -#include "BaseProcess.h" +#include "Common/BaseProcess.h" struct aiNode; diff --git a/thirdparty/assimp/code/SortByPTypeProcess.cpp b/thirdparty/assimp/code/PostProcessing/SortByPTypeProcess.cpp index 2e0cc54004..be8405a17b 100644 --- a/thirdparty/assimp/code/SortByPTypeProcess.cpp +++ b/thirdparty/assimp/code/PostProcessing/SortByPTypeProcess.cpp @@ -57,8 +57,8 @@ using namespace Assimp; // ------------------------------------------------------------------------------------------------ // Constructor to be privately used by Importer SortByPTypeProcess::SortByPTypeProcess() -{ - configRemoveMeshes = 0; +: mConfigRemoveMeshes( 0 ) { + // empty } // ------------------------------------------------------------------------------------------------ @@ -78,7 +78,7 @@ bool SortByPTypeProcess::IsActive( unsigned int pFlags) const // ------------------------------------------------------------------------------------------------ void SortByPTypeProcess::SetupProperties(const Importer* pImp) { - configRemoveMeshes = pImp->GetPropertyInteger(AI_CONFIG_PP_SBP_REMOVE,0); + mConfigRemoveMeshes = pImp->GetPropertyInteger(AI_CONFIG_PP_SBP_REMOVE,0); } // ------------------------------------------------------------------------------------------------ @@ -172,7 +172,7 @@ void SortByPTypeProcess::Execute( aiScene* pScene) { } if (1 == num) { - if (!(configRemoveMeshes & mesh->mPrimitiveTypes)) { + if (!(mConfigRemoveMeshes & mesh->mPrimitiveTypes)) { *meshIdx = static_cast<unsigned int>( outMeshes.size() ); outMeshes.push_back(mesh); } else { @@ -206,7 +206,7 @@ void SortByPTypeProcess::Execute( aiScene* pScene) { VertexWeightTable* avw = ComputeVertexBoneWeightTable(mesh); for (unsigned int real = 0; real < 4; ++real,++meshIdx) { - if ( !aiNumPerPType[real] || configRemoveMeshes & (1u << real)) + if ( !aiNumPerPType[real] || mConfigRemoveMeshes & (1u << real)) { continue; } @@ -392,10 +392,10 @@ void SortByPTypeProcess::Execute( aiScene* pScene) { { char buffer[1024]; ::ai_snprintf(buffer,1024,"Points: %u%s, Lines: %u%s, Triangles: %u%s, Polygons: %u%s (Meshes, X = removed)", - aiNumMeshesPerPType[0], ((configRemoveMeshes & aiPrimitiveType_POINT) ? "X" : ""), - aiNumMeshesPerPType[1], ((configRemoveMeshes & aiPrimitiveType_LINE) ? "X" : ""), - aiNumMeshesPerPType[2], ((configRemoveMeshes & aiPrimitiveType_TRIANGLE) ? "X" : ""), - aiNumMeshesPerPType[3], ((configRemoveMeshes & aiPrimitiveType_POLYGON) ? "X" : "")); + aiNumMeshesPerPType[0], ((mConfigRemoveMeshes & aiPrimitiveType_POINT) ? "X" : ""), + aiNumMeshesPerPType[1], ((mConfigRemoveMeshes & aiPrimitiveType_LINE) ? "X" : ""), + aiNumMeshesPerPType[2], ((mConfigRemoveMeshes & aiPrimitiveType_TRIANGLE) ? "X" : ""), + aiNumMeshesPerPType[3], ((mConfigRemoveMeshes & aiPrimitiveType_POLYGON) ? "X" : "")); ASSIMP_LOG_INFO(buffer); ASSIMP_LOG_DEBUG("SortByPTypeProcess finished"); } diff --git a/thirdparty/assimp/code/SortByPTypeProcess.h b/thirdparty/assimp/code/PostProcessing/SortByPTypeProcess.h index c9d9924d8f..1d7ccfc152 100644 --- a/thirdparty/assimp/code/SortByPTypeProcess.h +++ b/thirdparty/assimp/code/PostProcessing/SortByPTypeProcess.h @@ -45,10 +45,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_SORTBYPTYPEPROCESS_H_INC #define AI_SORTBYPTYPEPROCESS_H_INC -#include "BaseProcess.h" +#include "Common/BaseProcess.h" #include <assimp/mesh.h> class SortByPTypeProcessTest; + namespace Assimp { @@ -57,14 +58,11 @@ namespace Assimp { * A mesh with 5 lines, 3 points and 145 triangles would be split in 3 * submeshes. */ -class ASSIMP_API SortByPTypeProcess : public BaseProcess -{ +class ASSIMP_API SortByPTypeProcess : public BaseProcess { public: - SortByPTypeProcess(); ~SortByPTypeProcess(); -public: // ------------------------------------------------------------------- bool IsActive( unsigned int pFlags) const; @@ -75,8 +73,7 @@ public: void SetupProperties(const Importer* pImp); private: - - int configRemoveMeshes; + int mConfigRemoveMeshes; }; diff --git a/thirdparty/assimp/code/SplitLargeMeshes.cpp b/thirdparty/assimp/code/PostProcessing/SplitLargeMeshes.cpp index 1797b28d5a..1797b28d5a 100644 --- a/thirdparty/assimp/code/SplitLargeMeshes.cpp +++ b/thirdparty/assimp/code/PostProcessing/SplitLargeMeshes.cpp diff --git a/thirdparty/assimp/code/SplitLargeMeshes.h b/thirdparty/assimp/code/PostProcessing/SplitLargeMeshes.h index 77f089ce7e..3f90576ea9 100644 --- a/thirdparty/assimp/code/SplitLargeMeshes.h +++ b/thirdparty/assimp/code/PostProcessing/SplitLargeMeshes.h @@ -4,7 +4,6 @@ Open Asset Import Library (assimp) Copyright (c) 2006-2019, assimp team - All rights reserved. Redistribution and use of this software in source and binary forms, @@ -40,21 +39,21 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ---------------------------------------------------------------------- */ -/** @file Defines a post processing step to split large meshes into submeshes +/** @file Defines a post processing step to split large meshes into sub-meshes */ #ifndef AI_SPLITLARGEMESHES_H_INC #define AI_SPLITLARGEMESHES_H_INC #include <vector> -#include "BaseProcess.h" +#include "Common/BaseProcess.h" #include <assimp/mesh.h> #include <assimp/scene.h> +// Forward declarations class SplitLargeMeshesTest; -namespace Assimp -{ +namespace Assimp { class SplitLargeMeshesProcess_Triangle; class SplitLargeMeshesProcess_Vertex; diff --git a/thirdparty/assimp/code/TextureTransform.cpp b/thirdparty/assimp/code/PostProcessing/TextureTransform.cpp index 8ae2ba7218..8ae2ba7218 100644 --- a/thirdparty/assimp/code/TextureTransform.cpp +++ b/thirdparty/assimp/code/PostProcessing/TextureTransform.cpp diff --git a/thirdparty/assimp/code/TextureTransform.h b/thirdparty/assimp/code/PostProcessing/TextureTransform.h index c556ff5d8c..2a5d623d7f 100644 --- a/thirdparty/assimp/code/TextureTransform.h +++ b/thirdparty/assimp/code/PostProcessing/TextureTransform.h @@ -45,7 +45,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define AI_TEXTURE_TRANSFORM_H_INCLUDED #include <assimp/BaseImporter.h> -#include "BaseProcess.h" +#include "Common/BaseProcess.h" #include <assimp/material.h> #include <list> diff --git a/thirdparty/assimp/code/TriangulateProcess.cpp b/thirdparty/assimp/code/PostProcessing/TriangulateProcess.cpp index 0f68f47ddb..1040836bbe 100644 --- a/thirdparty/assimp/code/TriangulateProcess.cpp +++ b/thirdparty/assimp/code/PostProcessing/TriangulateProcess.cpp @@ -5,8 +5,6 @@ Open Asset Import Library (assimp) Copyright (c) 2006-2019, assimp team - - All rights reserved. Redistribution and use of this software in source and binary forms, @@ -60,9 +58,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * a file */ #ifndef ASSIMP_BUILD_NO_TRIANGULATE_PROCESS -#include "TriangulateProcess.h" -#include "ProcessHelper.h" -#include "PolyTools.h" + +#include "PostProcessing/TriangulateProcess.h" +#include "PostProcessing/ProcessHelper.h" +#include "Common/PolyTools.h" + #include <memory> //#define AI_BUILD_TRIANGULATE_COLOR_FACE_WINDING diff --git a/thirdparty/assimp/code/TriangulateProcess.h b/thirdparty/assimp/code/PostProcessing/TriangulateProcess.h index 47bd2115ad..916b5103dd 100644 --- a/thirdparty/assimp/code/TriangulateProcess.h +++ b/thirdparty/assimp/code/PostProcessing/TriangulateProcess.h @@ -46,7 +46,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_TRIANGULATEPROCESS_H_INC #define AI_TRIANGULATEPROCESS_H_INC -#include "BaseProcess.h" +#include "Common/BaseProcess.h" struct aiMesh; @@ -59,14 +59,11 @@ namespace Assimp { * into triangles. You usually want this to happen because the graphics cards * need their data as triangles. */ -class ASSIMP_API TriangulateProcess : public BaseProcess -{ +class ASSIMP_API TriangulateProcess : public BaseProcess { public: - TriangulateProcess(); ~TriangulateProcess(); -public: // ------------------------------------------------------------------- /** Returns whether the processing step is present in the given flag field. * @param pFlags The processing flags the importer was called with. A bitwise @@ -82,7 +79,6 @@ public: */ void Execute( aiScene* pScene); -public: // ------------------------------------------------------------------- /** Triangulates the given mesh. * @param pMesh The mesh to triangulate. diff --git a/thirdparty/assimp/code/ValidateDataStructure.cpp b/thirdparty/assimp/code/PostProcessing/ValidateDataStructure.cpp index 657b0361b7..712fd6943d 100644 --- a/thirdparty/assimp/code/ValidateDataStructure.cpp +++ b/thirdparty/assimp/code/PostProcessing/ValidateDataStructure.cpp @@ -46,8 +46,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * the data structure returned by Assimp. */ - - // internal headers #include "ValidateDataStructure.h" #include <assimp/BaseImporter.h> @@ -110,8 +108,8 @@ void ValidateDSProcess::ReportWarning(const char* msg,...) } // ------------------------------------------------------------------------------------------------ -inline int HasNameMatch(const aiString& in, aiNode* node) -{ +inline +int HasNameMatch(const aiString& in, aiNode* node) { int result = (node->mName == in ? 1 : 0 ); for (unsigned int i = 0; i < node->mNumChildren;++i) { result += HasNameMatch(in,node->mChildren[i]); @@ -121,9 +119,8 @@ inline int HasNameMatch(const aiString& in, aiNode* node) // ------------------------------------------------------------------------------------------------ template <typename T> -inline void ValidateDSProcess::DoValidation(T** parray, unsigned int size, - const char* firstName, const char* secondName) -{ +inline +void ValidateDSProcess::DoValidation(T** parray, unsigned int size, const char* firstName, const char* secondName) { // validate all entries if (size) { @@ -181,7 +178,8 @@ inline void ValidateDSProcess::DoValidationEx(T** parray, unsigned int size, // ------------------------------------------------------------------------------------------------ template <typename T> inline -void ValidateDSProcess::DoValidationWithNameCheck(T** array, unsigned int size, const char* firstName, const char* secondName) { +void ValidateDSProcess::DoValidationWithNameCheck(T** array, unsigned int size, const char* firstName, + const char* secondName) { // validate all entries DoValidationEx(array,size,firstName,secondName); @@ -201,9 +199,8 @@ void ValidateDSProcess::DoValidationWithNameCheck(T** array, unsigned int size, // ------------------------------------------------------------------------------------------------ // Executes the post processing step on the given imported data. -void ValidateDSProcess::Execute( aiScene* pScene) -{ - this->mScene = pScene; +void ValidateDSProcess::Execute( aiScene* pScene) { + mScene = pScene; ASSIMP_LOG_DEBUG("ValidateDataStructureProcess begin"); // validate the node graph of the scene @@ -516,13 +513,11 @@ void ValidateDSProcess::Validate( const aiMesh* pMesh) } // ------------------------------------------------------------------------------------------------ -void ValidateDSProcess::Validate( const aiMesh* pMesh, - const aiBone* pBone,float* afSum) -{ +void ValidateDSProcess::Validate( const aiMesh* pMesh, const aiBone* pBone,float* afSum) { this->Validate(&pBone->mName); if (!pBone->mNumWeights) { - ReportError("aiBone::mNumWeights is zero"); + //ReportError("aiBone::mNumWeights is zero"); } // check whether all vertices affected by this bone are valid @@ -563,9 +558,6 @@ void ValidateDSProcess::Validate( const aiAnimation* pAnimation) else { ReportError("aiAnimation::mNumChannels is 0. At least one node animation channel must be there."); } - - // Animation duration is allowed to be zero in cases where the anim contains only a single key frame. - // if (!pAnimation->mDuration)this->ReportError("aiAnimation::mDuration is zero"); } // ------------------------------------------------------------------------------------------------ @@ -746,8 +738,9 @@ void ValidateDSProcess::Validate( const aiMaterial* pMaterial) "AI_MATKEY_SHININESS_STRENGTH key is 0.0"); } break; - default: ; - }; + default: + break; + } } if (AI_SUCCESS == aiGetMaterialFloat( pMaterial,AI_MATKEY_OPACITY,&fTemp) && (!fTemp || fTemp > 1.01)) { diff --git a/thirdparty/assimp/code/ValidateDataStructure.h b/thirdparty/assimp/code/PostProcessing/ValidateDataStructure.h index bd21e88545..0b891ef414 100644 --- a/thirdparty/assimp/code/ValidateDataStructure.h +++ b/thirdparty/assimp/code/PostProcessing/ValidateDataStructure.h @@ -48,7 +48,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <assimp/types.h> #include <assimp/material.h> -#include "BaseProcess.h" + +#include "Common/BaseProcess.h" struct aiBone; struct aiMesh; diff --git a/thirdparty/assimp/code/RawLoader.cpp b/thirdparty/assimp/code/RawLoader.cpp deleted file mode 100644 index d0da247e47..0000000000 --- a/thirdparty/assimp/code/RawLoader.cpp +++ /dev/null @@ -1,331 +0,0 @@ -/* ---------------------------------------------------------------------------- -Open Asset Import Library (assimp) ---------------------------------------------------------------------------- - -Copyright (c) 2006-2019, assimp team - - - -All rights reserved. - -Redistribution and use of this software in source and binary forms, -with or without modification, are permitted provided that the following -conditions are met: - -* Redistributions of source code must retain the above - copyright notice, this list of conditions and the - following disclaimer. - -* Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the - following disclaimer in the documentation and/or other - materials provided with the distribution. - -* Neither the name of the assimp team, nor the names of its - contributors may be used to endorse or promote products - derived from this software without specific prior - written permission of the assimp team. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ---------------------------------------------------------------------------- -*/ - -/** @file RawLoader.cpp - * @brief Implementation of the RAW importer class - */ - - -#ifndef ASSIMP_BUILD_NO_RAW_IMPORTER - -// internal headers -#include "RawLoader.h" -#include <assimp/ParsingUtils.h> -#include <assimp/fast_atof.h> -#include <memory> -#include <assimp/IOSystem.hpp> -#include <assimp/DefaultLogger.hpp> -#include <assimp/scene.h> -#include <assimp/importerdesc.h> - -using namespace Assimp; - -static const aiImporterDesc desc = { - "Raw Importer", - "", - "", - "", - aiImporterFlags_SupportTextFlavour, - 0, - 0, - 0, - 0, - "raw" -}; - -// ------------------------------------------------------------------------------------------------ -// Constructor to be privately used by Importer -RAWImporter::RAWImporter() -{} - -// ------------------------------------------------------------------------------------------------ -// Destructor, private as well -RAWImporter::~RAWImporter() -{} - -// ------------------------------------------------------------------------------------------------ -// Returns whether the class can handle the format of the given file. -bool RAWImporter::CanRead( const std::string& pFile, IOSystem* /*pIOHandler*/, bool /*checkSig*/) const -{ - return SimpleExtensionCheck(pFile,"raw"); -} - -// ------------------------------------------------------------------------------------------------ -const aiImporterDesc* RAWImporter::GetInfo () const -{ - return &desc; -} - -// ------------------------------------------------------------------------------------------------ -// Imports the given file into the given scene structure. -void RAWImporter::InternReadFile( const std::string& pFile, - aiScene* pScene, IOSystem* pIOHandler) -{ - std::unique_ptr<IOStream> file( pIOHandler->Open( pFile, "rb")); - - // Check whether we can read from the file - if( file.get() == NULL) { - throw DeadlyImportError( "Failed to open RAW file " + pFile + "."); - } - - // allocate storage and copy the contents of the file to a memory buffer - // (terminate it with zero) - std::vector<char> mBuffer2; - TextFileToBuffer(file.get(),mBuffer2); - const char* buffer = &mBuffer2[0]; - - // list of groups loaded from the file - std::vector< GroupInformation > outGroups(1,GroupInformation("<default>")); - std::vector< GroupInformation >::iterator curGroup = outGroups.begin(); - - // now read all lines - char line[4096]; - while (GetNextLine(buffer,line)) - { - // if the line starts with a non-numeric identifier, it marks - // the beginning of a new group - const char* sz = line;SkipSpaces(&sz); - if (IsLineEnd(*sz))continue; - if (!IsNumeric(*sz)) - { - const char* sz2 = sz; - while (!IsSpaceOrNewLine(*sz2))++sz2; - const unsigned int length = (unsigned int)(sz2-sz); - - // find an existing group with this name - for (std::vector< GroupInformation >::iterator it = outGroups.begin(), end = outGroups.end(); - it != end;++it) - { - if (length == (*it).name.length() && !::strcmp(sz,(*it).name.c_str())) - { - curGroup = it;sz2 = NULL; - break; - } - } - if (sz2) - { - outGroups.push_back(GroupInformation(std::string(sz,length))); - curGroup = outGroups.end()-1; - } - } - else - { - // there can be maximally 12 floats plus an extra texture file name - float data[12]; - unsigned int num; - for (num = 0; num < 12;++num) - { - if(!SkipSpaces(&sz) || !IsNumeric(*sz))break; - sz = fast_atoreal_move<float>(sz,data[num]); - } - if (num != 12 && num != 9) - { - ASSIMP_LOG_ERROR("A line may have either 9 or 12 floats and an optional texture"); - continue; - } - - MeshInformation* output = NULL; - - const char* sz2 = sz; - unsigned int length; - if (!IsLineEnd(*sz)) - { - while (!IsSpaceOrNewLine(*sz2))++sz2; - length = (unsigned int)(sz2-sz); - } - else if (9 == num) - { - sz = "%default%"; - length = 9; - } - else - { - sz = ""; - length = 0; - } - - // search in the list of meshes whether we have one with this texture - for (auto &mesh : (*curGroup).meshes) - { - if (length == mesh.name.length() && (length ? !::strcmp(sz, mesh.name.c_str()) : true)) - { - output = &mesh; - break; - } - } - // if we don't have the mesh, create it - if (!output) - { - (*curGroup).meshes.push_back(MeshInformation(std::string(sz,length))); - output = &((*curGroup).meshes.back()); - } - if (12 == num) - { - aiColor4D v(data[0],data[1],data[2],1.0f); - output->colors.push_back(v); - output->colors.push_back(v); - output->colors.push_back(v); - - output->vertices.push_back(aiVector3D(data[3],data[4],data[5])); - output->vertices.push_back(aiVector3D(data[6],data[7],data[8])); - output->vertices.push_back(aiVector3D(data[9],data[10],data[11])); - } - else - { - output->vertices.push_back(aiVector3D(data[0],data[1],data[2])); - output->vertices.push_back(aiVector3D(data[3],data[4],data[5])); - output->vertices.push_back(aiVector3D(data[6],data[7],data[8])); - } - } - } - - pScene->mRootNode = new aiNode(); - pScene->mRootNode->mName.Set("<RawRoot>"); - - // count the number of valid groups - // (meshes can't be empty) - for (auto & outGroup : outGroups) - { - if (!outGroup.meshes.empty()) - { - ++pScene->mRootNode->mNumChildren; - pScene->mNumMeshes += (unsigned int) outGroup.meshes.size(); - } - } - - if (!pScene->mNumMeshes) - { - throw DeadlyImportError("RAW: No meshes loaded. The file seems to be corrupt or empty."); - } - - pScene->mMeshes = new aiMesh*[pScene->mNumMeshes]; - aiNode** cc; - if (1 == pScene->mRootNode->mNumChildren) - { - cc = &pScene->mRootNode; - pScene->mRootNode->mNumChildren = 0; - } else { - cc = new aiNode*[pScene->mRootNode->mNumChildren]; - memset(cc, 0, sizeof(aiNode*) * pScene->mRootNode->mNumChildren); - pScene->mRootNode->mChildren = cc; - } - - pScene->mNumMaterials = pScene->mNumMeshes; - aiMaterial** mats = pScene->mMaterials = new aiMaterial*[pScene->mNumMaterials]; - - unsigned int meshIdx = 0; - for (auto & outGroup : outGroups) - { - if (outGroup.meshes.empty())continue; - - aiNode* node; - if (pScene->mRootNode->mNumChildren) - { - node = *cc = new aiNode(); - node->mParent = pScene->mRootNode; - } - else node = *cc; - node->mName.Set(outGroup.name); - - // add all meshes - node->mNumMeshes = (unsigned int) outGroup.meshes.size(); - unsigned int* pi = node->mMeshes = new unsigned int[ node->mNumMeshes ]; - for (std::vector< MeshInformation >::iterator it2 = outGroup.meshes.begin(), - end2 = outGroup.meshes.end(); it2 != end2; ++it2) - { - ai_assert(!(*it2).vertices.empty()); - - // allocate the mesh - *pi++ = meshIdx; - aiMesh* mesh = pScene->mMeshes[meshIdx] = new aiMesh(); - mesh->mMaterialIndex = meshIdx++; - - mesh->mPrimitiveTypes = aiPrimitiveType_TRIANGLE; - - // allocate storage for the vertex components and copy them - mesh->mNumVertices = (unsigned int)(*it2).vertices.size(); - mesh->mVertices = new aiVector3D[ mesh->mNumVertices ]; - ::memcpy(mesh->mVertices,&(*it2).vertices[0],sizeof(aiVector3D)*mesh->mNumVertices); - - if ((*it2).colors.size()) - { - ai_assert((*it2).colors.size() == mesh->mNumVertices); - - mesh->mColors[0] = new aiColor4D[ mesh->mNumVertices ]; - ::memcpy(mesh->mColors[0],&(*it2).colors[0],sizeof(aiColor4D)*mesh->mNumVertices); - } - - // generate triangles - ai_assert(0 == mesh->mNumVertices % 3); - aiFace* fc = mesh->mFaces = new aiFace[ mesh->mNumFaces = mesh->mNumVertices/3 ]; - aiFace* const fcEnd = fc + mesh->mNumFaces; - unsigned int n = 0; - while (fc != fcEnd) - { - aiFace& f = *fc++; - f.mIndices = new unsigned int[f.mNumIndices = 3]; - for (unsigned int m = 0; m < 3;++m) - f.mIndices[m] = n++; - } - - // generate a material for the mesh - aiMaterial* mat = new aiMaterial(); - - aiColor4D clr(1.0f,1.0f,1.0f,1.0f); - if ("%default%" == (*it2).name) // a gray default material - { - clr.r = clr.g = clr.b = 0.6f; - } - else if ((*it2).name.length() > 0) // a texture - { - aiString s; - s.Set((*it2).name); - mat->AddProperty(&s,AI_MATKEY_TEXTURE_DIFFUSE(0)); - } - mat->AddProperty<aiColor4D>(&clr,1,AI_MATKEY_COLOR_DIFFUSE); - *mats++ = mat; - } - } -} - -#endif // !! ASSIMP_BUILD_NO_RAW_IMPORTER diff --git a/thirdparty/assimp/include/assimp/Exporter.hpp b/thirdparty/assimp/include/assimp/Exporter.hpp index bf0096e7e9..ea0303e804 100644 --- a/thirdparty/assimp/include/assimp/Exporter.hpp +++ b/thirdparty/assimp/include/assimp/Exporter.hpp @@ -190,7 +190,7 @@ public: * @note Use aiCopyScene() to get a modifiable copy of a previously * imported scene. */ const aiExportDataBlob* ExportToBlob(const aiScene* pScene, const char* pFormatId, - unsigned int pPreprocessing = 0u, const ExportProperties* = nullptr); + unsigned int pPreprocessing = 0u, const ExportProperties* pProperties = nullptr); const aiExportDataBlob* ExportToBlob( const aiScene* pScene, const std::string& pFormatId, unsigned int pPreprocessing = 0u, const ExportProperties* pProperties = nullptr); diff --git a/thirdparty/assimp/include/assimp/ParsingUtils.h b/thirdparty/assimp/include/assimp/ParsingUtils.h index ca30ce13b0..6b9574fc67 100644 --- a/thirdparty/assimp/include/assimp/ParsingUtils.h +++ b/thirdparty/assimp/include/assimp/ParsingUtils.h @@ -196,8 +196,7 @@ bool GetNextLine( const char_t*& buffer, char_t out[ BufferSize ] ) { // --------------------------------------------------------------------------------- template <class char_t> -AI_FORCE_INLINE bool IsNumeric( char_t in) -{ +AI_FORCE_INLINE bool IsNumeric( char_t in) { return ( in >= '0' && in <= '9' ) || '-' == in || '+' == in; } diff --git a/thirdparty/assimp/include/assimp/aabb.h b/thirdparty/assimp/include/assimp/aabb.h new file mode 100644 index 0000000000..a20f317424 --- /dev/null +++ b/thirdparty/assimp/include/assimp/aabb.h @@ -0,0 +1,76 @@ +/* +--------------------------------------------------------------------------- +Open Asset Import Library (assimp) +--------------------------------------------------------------------------- + +Copyright (c) 2006-2019, assimp team + + + +All rights reserved. + +Redistribution and use of this software in source and binary forms, +with or without modification, are permitted provided that the following +conditions are met: + +* Redistributions of source code must retain the above +copyright notice, this list of conditions and the +following disclaimer. + +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the +following disclaimer in the documentation and/or other +materials provided with the distribution. + +* Neither the name of the assimp team, nor the names of its +contributors may be used to endorse or promote products +derived from this software without specific prior +written permission of the assimp team. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +--------------------------------------------------------------------------- +*/ + +#pragma once +#ifndef AI_AABB_H_INC +#define AI_AABB_H_INC + +#include <assimp/vector3.h> + +struct aiAABB { + C_STRUCT aiVector3D mMin; + C_STRUCT aiVector3D mMax; + +#ifdef __cplusplus + + aiAABB() + : mMin() + , mMax() { + // empty + } + + aiAABB(const aiVector3D &min, const aiVector3D &max ) + : mMin(min) + , mMax(max) { + // empty + } + + ~aiAABB() { + // empty + } + +#endif +}; + + +#endif diff --git a/thirdparty/assimp/include/assimp/camera.h b/thirdparty/assimp/include/assimp/camera.h index 99daf69934..e573eea5d1 100644 --- a/thirdparty/assimp/include/assimp/camera.h +++ b/thirdparty/assimp/include/assimp/camera.h @@ -5,8 +5,6 @@ Open Asset Import Library (assimp) Copyright (c) 2006-2019, assimp team - - All rights reserved. Redistribution and use of this software in source and binary forms, @@ -60,7 +58,7 @@ extern "C" { * * Cameras have a representation in the node graph and can be animated. * An important aspect is that the camera itself is also part of the - * scenegraph. This means, any values such as the look-at vector are not + * scene-graph. This means, any values such as the look-at vector are not * *absolute*, they're <b>relative</b> to the coordinate system defined * by the node which corresponds to the camera. This allows for camera * animations. For static cameras parameters like the 'look-at' or 'up' vectors @@ -162,7 +160,6 @@ struct aiCamera */ float mClipPlaneFar; - /** Screen aspect ratio. * * This is the ration between the width and the height of the diff --git a/thirdparty/assimp/include/assimp/color4.inl b/thirdparty/assimp/include/assimp/color4.inl index 3192d55f39..afa53dcb5b 100644 --- a/thirdparty/assimp/include/assimp/color4.inl +++ b/thirdparty/assimp/include/assimp/color4.inl @@ -85,6 +85,8 @@ AI_FORCE_INLINE TReal aiColor4t<TReal>::operator[](unsigned int i) const { return g; case 2: return b; + case 3: + return a; default: break; } @@ -100,6 +102,8 @@ AI_FORCE_INLINE TReal& aiColor4t<TReal>::operator[](unsigned int i) { return g; case 2: return b; + case 3: + return a; default: break; } diff --git a/thirdparty/assimp/include/assimp/config.h.in b/thirdparty/assimp/include/assimp/config.h.in index a37ff0b8c8..d08b929a10 100644 --- a/thirdparty/assimp/include/assimp/config.h.in +++ b/thirdparty/assimp/include/assimp/config.h.in @@ -142,7 +142,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /** @brief Specifies the maximum angle that may be between two vertex tangents * that their tangents and bi-tangents are smoothed. * - * This applies to the CalcTangentSpace-Step. The angle is specified + * This applies to the CalcTangentSpace-Step. TFvhe angle is specified * in degrees. The maximum value is 175. * Property type: float. Default value: 45 degrees */ @@ -651,13 +651,28 @@ enum aiComponent // --------------------------------------------------------------------------- /** @brief Set whether the fbx importer will use the legacy embedded texture naming. -* -* The default value is false (0) -* Property type: bool -*/ + * + * The default value is false (0) + * Property type: bool + */ #define AI_CONFIG_IMPORT_FBX_EMBEDDED_TEXTURES_LEGACY_NAMING \ "AI_CONFIG_IMPORT_FBX_EMBEDDED_TEXTURES_LEGACY_NAMING" - + +// --------------------------------------------------------------------------- +/** @brief Set wether the importer shall not remove empty bones. + * + * Empty bone are often used to define connections for other models. + */ +#define AI_CONFIG_IMPORT_REMOVE_EMPTY_BONES \ + "AI_CONFIG_IMPORT_REMOVE_EMPTY_BONES" + + +// --------------------------------------------------------------------------- +/** @brief Set wether the FBX importer shall convert the unit from cm to m. + */ +#define AI_CONFIG_FBX_CONVERT_TO_M \ + "AI_CONFIG_FBX_CONVERT_TO_M" + // --------------------------------------------------------------------------- /** @brief Set the vertex animation keyframe to be imported * @@ -966,8 +981,12 @@ enum aiComponent #define AI_CONFIG_EXPORT_XFILE_64BIT "EXPORT_XFILE_64BIT" -/** - * +/** @brief Specifies whether the assimp export shall be able to export point clouds + * + * When this flag is not defined the render data has to contain valid faces. + * Point clouds are only a collection of vertices which have nor spatial organization + * by a face and the validation process will remove them. Enabling this feature will + * switch off the flag and enable the functionality to export pure point clouds. */ #define AI_CONFIG_EXPORT_POINT_CLOUDS "EXPORT_POINT_CLOUDS" diff --git a/thirdparty/assimp/include/assimp/defs.h b/thirdparty/assimp/include/assimp/defs.h index 4a177e3c3e..05a5e3fd4b 100644 --- a/thirdparty/assimp/include/assimp/defs.h +++ b/thirdparty/assimp/include/assimp/defs.h @@ -122,7 +122,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * OPTIMIZEANIMS * OPTIMIZEGRAPH * GENENTITYMESHES - * FIXTEXTUREPATHS */ + * FIXTEXTUREPATHS + * GENBOUNDINGBOXES */ ////////////////////////////////////////////////////////////////////////// #ifdef _MSC_VER @@ -214,10 +215,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif #if (defined(__BORLANDC__) || defined (__BCPLUSPLUS__)) -#error Currently, Borland is unsupported. Feel free to port Assimp. - -// "W8059 Packgröße der Struktur geändert" - +# error Currently, Borland is unsupported. Feel free to port Assimp. #endif @@ -243,10 +241,16 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. typedef double ai_real; typedef signed long long int ai_int; typedef unsigned long long int ai_uint; +#ifndef ASSIMP_AI_REAL_TEXT_PRECISION +#define ASSIMP_AI_REAL_TEXT_PRECISION 16 +#endif // ASSIMP_AI_REAL_TEXT_PRECISION #else // ASSIMP_DOUBLE_PRECISION typedef float ai_real; typedef signed int ai_int; typedef unsigned int ai_uint; +#ifndef ASSIMP_AI_REAL_TEXT_PRECISION +#define ASSIMP_AI_REAL_TEXT_PRECISION 8 +#endif // ASSIMP_AI_REAL_TEXT_PRECISION #endif // ASSIMP_DOUBLE_PRECISION ////////////////////////////////////////////////////////////////////////// @@ -267,6 +271,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define AI_DEG_TO_RAD(x) ((x)*(ai_real)0.0174532925) #define AI_RAD_TO_DEG(x) ((x)*(ai_real)57.2957795) +/* Numerical limits */ +static const ai_real ai_epsilon = (ai_real) 0.00001; + /* Support for big-endian builds */ #if defined(__BYTE_ORDER__) # if (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) @@ -293,11 +300,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef _MSC_VER # define AI_NO_EXCEPT noexcept #else -# if (_MSC_VER == 1915 ) +# if (_MSC_VER >= 1915 ) # define AI_NO_EXCEPT noexcept # else # define AI_NO_EXCEPT # endif -#endif +#endif // _MSC_VER #endif // !! AI_DEFINES_H_INC diff --git a/thirdparty/assimp/include/assimp/mesh.h b/thirdparty/assimp/include/assimp/mesh.h index 36f3ed2afd..f1628f1f54 100644 --- a/thirdparty/assimp/include/assimp/mesh.h +++ b/thirdparty/assimp/include/assimp/mesh.h @@ -48,7 +48,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_MESH_H_INC #define AI_MESH_H_INC -#include "types.h" +#include <assimp/types.h> +#include <assimp/aabb.h> #ifdef __cplusplus extern "C" { @@ -714,6 +715,11 @@ struct aiMesh * Method of morphing when animeshes are specified. */ unsigned int mMethod; + + /** + * + */ + C_STRUCT aiAABB mAABB; #ifdef __cplusplus @@ -735,7 +741,8 @@ struct aiMesh , mMaterialIndex( 0 ) , mNumAnimMeshes( 0 ) , mAnimMeshes(nullptr) - , mMethod( 0 ) { + , mMethod( 0 ) + , mAABB() { for( unsigned int a = 0; a < AI_MAX_NUMBER_OF_TEXTURECOORDS; ++a ) { mNumUVComponents[a] = 0; mTextureCoords[a] = nullptr; diff --git a/thirdparty/assimp/include/assimp/postprocess.h b/thirdparty/assimp/include/assimp/postprocess.h index c23a5490a5..2a74414216 100644 --- a/thirdparty/assimp/include/assimp/postprocess.h +++ b/thirdparty/assimp/include/assimp/postprocess.h @@ -438,7 +438,7 @@ enum aiPostProcessSteps aiProcess_FindInstances = 0x100000, // ------------------------------------------------------------------------- - /** <hr>A postprocessing step to reduce the number of meshes. + /** <hr>A post-processing step to reduce the number of meshes. * * This will, in fact, reduce the number of draw calls. * @@ -450,7 +450,7 @@ enum aiPostProcessSteps // ------------------------------------------------------------------------- - /** <hr>A postprocessing step to optimize the scene hierarchy. + /** <hr>A post-processing step to optimize the scene hierarchy. * * Nodes without animations, bones, lights or cameras assigned are * collapsed and joined. @@ -514,7 +514,7 @@ enum aiPostProcessSteps // ------------------------------------------------------------------------- /** <hr>This step splits meshes with many bones into sub-meshes so that each - * su-bmesh has fewer or as many bones as a given limit. + * sub-mesh has fewer or as many bones as a given limit. */ aiProcess_SplitByBoneCount = 0x2000000, @@ -541,7 +541,7 @@ enum aiPostProcessSteps * global scaling from your importer settings like in FBX. Use the flag * AI_CONFIG_GLOBAL_SCALE_FACTOR_KEY from the global property table to configure this. * - * Use <tt>#AI_CONFIG_GLOBAL_SCALE_FACTOR_KEY</tt> to setup the global scaing factor. + * Use <tt>#AI_CONFIG_GLOBAL_SCALE_FACTOR_KEY</tt> to setup the global scaling factor. */ aiProcess_GlobalScale = 0x8000000, @@ -574,6 +574,11 @@ enum aiPostProcessSteps * This process gives sense back to aiProcess_JoinIdenticalVertices */ aiProcess_DropNormals = 0x40000000, + + // ------------------------------------------------------------------------- + /** + */ + aiProcess_GenBoundingBoxes = 0x80000000 }; diff --git a/thirdparty/assimp/include/assimp/scene.h b/thirdparty/assimp/include/assimp/scene.h index de0239702d..df5d6f3b5e 100644 --- a/thirdparty/assimp/include/assimp/scene.h +++ b/thirdparty/assimp/include/assimp/scene.h @@ -56,6 +56,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "material.h" #include "anim.h" #include "metadata.h" +#include <cstdlib> #ifdef __cplusplus extern "C" { @@ -389,6 +390,14 @@ struct aiScene //! Returns an embedded texture const aiTexture* GetEmbeddedTexture(const char* filename) const { + // lookup using texture ID (if referenced like: "*1", "*2", etc.) + if ('*' == *filename) { + int index = std::atoi(filename + 1); + if (0 > index || mNumTextures <= static_cast<unsigned>(index)) + return nullptr; + return mTextures[index]; + } + // lookup using filename const char* shortFilename = GetShortFilename(filename); for (unsigned int i = 0; i < mNumTextures; i++) { const char* shortTextureFilename = GetShortFilename(mTextures[i]->mFilename.C_Str()); diff --git a/thirdparty/jpeg-compressor/jpgd.cpp b/thirdparty/jpeg-compressor/jpgd.cpp index fad9a37a9a..62fbd1b72d 100644 --- a/thirdparty/jpeg-compressor/jpgd.cpp +++ b/thirdparty/jpeg-compressor/jpgd.cpp @@ -29,6 +29,10 @@ #define JPGD_MAX(a,b) (((a)>(b)) ? (a) : (b)) #define JPGD_MIN(a,b) (((a)<(b)) ? (a) : (b)) +// TODO: Move to header and use these constants when declaring the arrays. +#define JPGD_HUFF_TREE_MAX_LENGTH 512 +#define JPGD_HUFF_CODE_SIZE_MAX_LENGTH 256 + namespace jpgd { static inline void *jpgd_malloc(size_t nSize) { return malloc(nSize); } @@ -491,8 +495,9 @@ inline uint jpeg_decoder::get_bits_no_markers(int num_bits) // Decodes a Huffman encoded symbol. inline int jpeg_decoder::huff_decode(huff_tables *pH) { - int symbol; + JPGD_ASSERT(pH); + int symbol; // Check first 8-bits: do we have a complete symbol? if ((symbol = pH->look_up[m_bit_buf >> 24]) < 0) { @@ -500,14 +505,19 @@ inline int jpeg_decoder::huff_decode(huff_tables *pH) int ofs = 23; do { - symbol = pH->tree[-(int)(symbol + ((m_bit_buf >> ofs) & 1))]; + unsigned int idx = -(int)(symbol + ((m_bit_buf >> ofs) & 1)); + JPGD_ASSERT(idx < JPGD_HUFF_TREE_MAX_LENGTH); + symbol = pH->tree[idx]; ofs--; } while (symbol < 0); get_bits_no_markers(8 + (23 - ofs)); } else + { + JPGD_ASSERT(symbol < JPGD_HUFF_CODE_SIZE_MAX_LENGTH); get_bits_no_markers(pH->code_size[symbol]); + } return symbol; } @@ -517,6 +527,8 @@ inline int jpeg_decoder::huff_decode(huff_tables *pH, int& extra_bits) { int symbol; + JPGD_ASSERT(pH); + // Check first 8-bits: do we have a complete symbol? if ((symbol = pH->look_up2[m_bit_buf >> 24]) < 0) { @@ -524,7 +536,9 @@ inline int jpeg_decoder::huff_decode(huff_tables *pH, int& extra_bits) int ofs = 23; do { - symbol = pH->tree[-(int)(symbol + ((m_bit_buf >> ofs) & 1))]; + unsigned int idx = -(int)(symbol + ((m_bit_buf >> ofs) & 1)); + JPGD_ASSERT(idx < JPGD_HUFF_TREE_MAX_LENGTH); + symbol = pH->tree[idx]; ofs--; } while (symbol < 0); @@ -1495,6 +1509,12 @@ void jpeg_decoder::fix_in_buffer() void jpeg_decoder::transform_mcu(int mcu_row) { jpgd_block_t* pSrc_ptr = m_pMCU_coefficients; + if (m_freq_domain_chroma_upsample) { + JPGD_ASSERT(mcu_row * m_blocks_per_mcu < m_expanded_blocks_per_row); + } + else { + JPGD_ASSERT(mcu_row * m_blocks_per_mcu < m_max_blocks_per_row); + } uint8* pDst_ptr = m_pSample_buf + mcu_row * m_blocks_per_mcu * 64; for (int mcu_block = 0; mcu_block < m_blocks_per_mcu; mcu_block++) @@ -1650,6 +1670,7 @@ void jpeg_decoder::load_next_row() for (mcu_block = 0; mcu_block < m_blocks_per_mcu; mcu_block++) { component_id = m_mcu_org[mcu_block]; + JPGD_ASSERT(m_comp_quant[component_id] < JPGD_MAX_QUANT_TABLES); q = m_quant[m_comp_quant[component_id]]; p = m_pMCU_coefficients + 64 * mcu_block; @@ -1770,6 +1791,7 @@ void jpeg_decoder::decode_next_row() for (int mcu_block = 0; mcu_block < m_blocks_per_mcu; mcu_block++, p += 64) { int component_id = m_mcu_org[mcu_block]; + JPGD_ASSERT(m_comp_quant[component_id] < JPGD_MAX_QUANT_TABLES); jpgd_quant_t* q = m_quant[m_comp_quant[component_id]]; int r, s; @@ -2229,7 +2251,10 @@ void jpeg_decoder::make_huff_table(int index, huff_tables *pH) for (l = 1; l <= 16; l++) { for (i = 1; i <= m_huff_num[index][l]; i++) + { + JPGD_ASSERT(p < 257); huffsize[p++] = static_cast<uint8>(l); + } } huffsize[p] = 0; @@ -2244,6 +2269,7 @@ void jpeg_decoder::make_huff_table(int index, huff_tables *pH) { while (huffsize[p] == si) { + JPGD_ASSERT(p < 257); huffcode[p++] = code; code++; } @@ -2275,7 +2301,8 @@ void jpeg_decoder::make_huff_table(int index, huff_tables *pH) for (l = 1 << (8 - code_size); l > 0; l--) { - JPGD_ASSERT(i < 256); + JPGD_ASSERT(i < JPGD_HUFF_CODE_SIZE_MAX_LENGTH); + JPGD_ASSERT(code < JPGD_HUFF_CODE_SIZE_MAX_LENGTH); pH->look_up[code] = i; @@ -2325,16 +2352,19 @@ void jpeg_decoder::make_huff_table(int index, huff_tables *pH) if ((code & 0x8000) == 0) currententry--; - if (pH->tree[-currententry - 1] == 0) + unsigned int idx = -currententry - 1; + JPGD_ASSERT(idx < JPGD_HUFF_TREE_MAX_LENGTH); + if (pH->tree[idx] == 0) { - pH->tree[-currententry - 1] = nextfreeentry; + pH->tree[idx] = nextfreeentry; currententry = nextfreeentry; nextfreeentry -= 2; } - else - currententry = pH->tree[-currententry - 1]; + else { + currententry = pH->tree[idx]; + } code <<= 1; } @@ -2636,7 +2666,9 @@ void jpeg_decoder::decode_block_ac_first(jpeg_decoder *pD, int component_id, int for (k = pD->m_spectral_start; k <= pD->m_spectral_end; k++) { - s = pD->huff_decode(pD->m_pHuff_tabs[pD->m_comp_ac_tab[component_id]]); + unsigned int idx = pD->m_comp_ac_tab[component_id]; + JPGD_ASSERT(idx < JPGD_MAX_HUFF_TABLES); + s = pD->huff_decode(pD->m_pHuff_tabs[idx]); r = s >> 4; s &= 15; @@ -2679,7 +2711,6 @@ void jpeg_decoder::decode_block_ac_refine(jpeg_decoder *pD, int component_id, in int p1 = 1 << pD->m_successive_low; int m1 = (-1) << pD->m_successive_low; jpgd_block_t *p = pD->coeff_buf_getp(pD->m_ac_coeffs[component_id], block_x, block_y); - JPGD_ASSERT(pD->m_spectral_end <= 63); k = pD->m_spectral_start; @@ -2688,7 +2719,9 @@ void jpeg_decoder::decode_block_ac_refine(jpeg_decoder *pD, int component_id, in { for ( ; k <= pD->m_spectral_end; k++) { - s = pD->huff_decode(pD->m_pHuff_tabs[pD->m_comp_ac_tab[component_id]]); + unsigned int idx = pD->m_comp_ac_tab[component_id]; + JPGD_ASSERT(idx < JPGD_MAX_HUFF_TABLES); + s = pD->huff_decode(pD->m_pHuff_tabs[idx]); r = s >> 4; s &= 15; diff --git a/thirdparty/xatlas/xatlas.cpp b/thirdparty/xatlas/xatlas.cpp index c62be4e73a..1b30305cd4 100644 --- a/thirdparty/xatlas/xatlas.cpp +++ b/thirdparty/xatlas/xatlas.cpp @@ -299,29 +299,30 @@ static void *Realloc(void *ptr, size_t size, int /*tag*/, const char * /*file*/, #if XA_PROFILE #define XA_PROFILE_START(var) const clock_t var##Start = clock(); #define XA_PROFILE_END(var) internal::s_profile.var += clock() - var##Start; -#define XA_PROFILE_PRINT(label, var) XA_PRINT("%s%.2f seconds (%g ms)\n", label, internal::clockToSeconds(internal::s_profile.var), internal::clockToMs(internal::s_profile.var)); +#define XA_PROFILE_PRINT_AND_RESET(label, var) XA_PRINT("%s%.2f seconds (%g ms)\n", label, internal::clockToSeconds(internal::s_profile.var), internal::clockToMs(internal::s_profile.var)); internal::s_profile.var = 0; struct ProfileData { - clock_t addMeshConcurrent; - std::atomic<clock_t> addMesh; + clock_t addMeshReal; + std::atomic<clock_t> addMeshThread; std::atomic<clock_t> addMeshCreateColocals; std::atomic<clock_t> addMeshCreateFaceGroups; std::atomic<clock_t> addMeshCreateBoundaries; - std::atomic<clock_t> addMeshCreateChartGroupsConcurrent; - std::atomic<clock_t> addMeshCreateChartGroups; - clock_t computeChartsConcurrent; - std::atomic<clock_t> computeCharts; + std::atomic<clock_t> addMeshCreateChartGroupsReal; + std::atomic<clock_t> addMeshCreateChartGroupsThread; + clock_t computeChartsReal; + std::atomic<clock_t> computeChartsThread; std::atomic<clock_t> atlasBuilder; std::atomic<clock_t> atlasBuilderInit; std::atomic<clock_t> atlasBuilderCreateInitialCharts; std::atomic<clock_t> atlasBuilderGrowCharts; std::atomic<clock_t> atlasBuilderMergeCharts; - std::atomic<clock_t> createChartMeshes; + std::atomic<clock_t> createChartMeshesReal; + std::atomic<clock_t> createChartMeshesThread; std::atomic<clock_t> fixChartMeshTJunctions; std::atomic<clock_t> closeChartMeshHoles; - clock_t parameterizeChartsConcurrent; - std::atomic<clock_t> parameterizeCharts; + clock_t parameterizeChartsReal; + std::atomic<clock_t> parameterizeChartsThread; std::atomic<clock_t> parameterizeChartsOrthogonal; std::atomic<clock_t> parameterizeChartsLSCM; std::atomic<clock_t> parameterizeChartsEvaluateQuality; @@ -329,6 +330,7 @@ struct ProfileData clock_t packChartsRasterize; clock_t packChartsDilate; clock_t packChartsFindLocation; + std::atomic<clock_t> packChartsFindLocationThread; clock_t packChartsBlit; }; @@ -346,7 +348,7 @@ static double clockToSeconds(clock_t c) #else #define XA_PROFILE_START(var) #define XA_PROFILE_END(var) -#define XA_PROFILE_PRINT(label, var) +#define XA_PROFILE_PRINT_AND_RESET(label, var) #endif static constexpr float kPi = 3.14159265358979323846f; @@ -641,6 +643,7 @@ static bool linesIntersect(const Vector2 &a1, const Vector2 &a2, const Vector2 & struct Vector2i { + Vector2i() {} Vector2i(int32_t x, int32_t y) : x(x), y(y) {} int32_t x, y; @@ -3528,6 +3531,15 @@ private: std::mutex m_mutex; }; +struct Spinlock +{ + void lock() { while(m_lock.test_and_set(std::memory_order_acquire)) {} } + void unlock() { m_lock.clear(std::memory_order_release); } + +private: + std::atomic_flag m_lock = ATOMIC_FLAG_INIT; +}; + struct TaskGroupHandle { uint32_t value = UINT32_MAX; @@ -3545,6 +3557,14 @@ class TaskScheduler public: TaskScheduler() : m_shutdown(false) { + // Max with current task scheduler usage is 1 per thread + 1 deep nesting, but allow for some slop. + m_maxGroups = std::thread::hardware_concurrency() * 4; + m_groups = XA_ALLOC_ARRAY(MemTag::Default, TaskGroup, m_maxGroups); + for (uint32_t i = 0; i < m_maxGroups; i++) { + new (&m_groups[i]) TaskGroup(); + m_groups[i].free = true; + m_groups[i].ref = 0; + } m_workers.resize(std::thread::hardware_concurrency() <= 1 ? 1 : std::thread::hardware_concurrency() - 1); for (uint32_t i = 0; i < m_workers.size(); i++) { m_workers[i].wakeup = false; @@ -3565,36 +3585,42 @@ public: worker.thread->~thread(); XA_FREE(worker.thread); } - for (uint32_t i = 0; i < m_groups.size(); i++) - destroyGroup(i); + for (uint32_t i = 0; i < m_maxGroups; i++) + m_groups[i].~TaskGroup(); + XA_FREE(m_groups); } - void run(TaskGroupHandle *handle, Task task) + TaskGroupHandle createTaskGroup(uint32_t reserveSize = 0) { - // Allocate a task group if this is the first time using this handle. - TaskGroup *group; - if (handle->value == UINT32_MAX) { - group = XA_NEW(MemTag::Default, TaskGroup); - group->ref = 0; - std::lock_guard<std::mutex> lock(m_groupsMutex); - for (uint32_t i = 0; i < m_groups.size(); i++) { - if (!m_groups[i]) { - m_groups[i] = group; - handle->value = i; - break; - } - } - if (handle->value == UINT32_MAX) { - m_groups.push_back(group); - handle->value = m_groups.size() - 1; - } - } - group = m_groups[handle->value]; - { - std::lock_guard<std::mutex> lock(group->queueMutex); - group->queue.push_back(task); - } - group->ref++; + // Claim the first free group. + for (uint32_t i = 0; i < m_maxGroups; i++) { + TaskGroup &group = m_groups[i]; + bool expected = true; + if (!group.free.compare_exchange_strong(expected, false)) + continue; + group.queueLock.lock(); + group.queueHead = 0; + group.queue.clear(); + group.queue.reserve(reserveSize); + group.queueLock.unlock(); + TaskGroupHandle handle; + handle.value = i; + return handle; + } + XA_DEBUG_ASSERT(false); + TaskGroupHandle handle; + handle.value = UINT32_MAX; + return handle; + } + + void run(TaskGroupHandle handle, Task task) + { + XA_DEBUG_ASSERT(handle.value != UINT32_MAX); + TaskGroup &group = m_groups[handle.value]; + group.queueLock.lock(); + group.queue.push_back(task); + group.queueLock.unlock(); + group.ref++; // Wake up a worker to run this task. for (uint32_t i = 0; i < m_workers.size(); i++) { m_workers[i].wakeup = true; @@ -3609,33 +3635,32 @@ public: return; } // Run tasks from the group queue until empty. - TaskGroup *group = m_groups[handle->value]; + TaskGroup &group = m_groups[handle->value]; for (;;) { Task *task = nullptr; - { - std::lock_guard<std::mutex> lock(group->queueMutex); - if (group->queueHead < group->queue.size()) - task = &group->queue[group->queueHead++]; - } + group.queueLock.lock(); + if (group.queueHead < group.queue.size()) + task = &group.queue[group.queueHead++]; + group.queueLock.unlock(); if (!task) break; task->func(task->userData); - group->ref--; + group.ref--; } // Even though the task queue is empty, workers can still be running tasks. - while (group->ref > 0) + while (group.ref > 0) std::this_thread::yield(); - std::lock_guard<std::mutex> lock(m_groupsMutex); - destroyGroup(handle->value); + group.free = true; handle->value = UINT32_MAX; } private: struct TaskGroup { + std::atomic<bool> free; Array<Task> queue; // Items are never removed. queueHead is incremented to pop items. uint32_t queueHead = 0; - std::mutex queueMutex; + Spinlock queueLock; std::atomic<uint32_t> ref; // Increment when a task is enqueued, decrement when a task finishes. }; @@ -3647,21 +3672,11 @@ private: std::atomic<bool> wakeup; }; - Array<TaskGroup *> m_groups; - std::mutex m_groupsMutex; + TaskGroup *m_groups; + uint32_t m_maxGroups; Array<Worker> m_workers; std::atomic<bool> m_shutdown; - void destroyGroup(uint32_t index) - { - TaskGroup *group = m_groups[index]; - m_groups[index] = nullptr; - if (group) { - group->~TaskGroup(); - XA_FREE(group); - } - } - static void workerThread(TaskScheduler *scheduler, Worker *worker) { std::unique_lock<std::mutex> lock(worker->mutex); @@ -3674,18 +3689,17 @@ private: // Look for a task in any of the groups and run it. TaskGroup *group = nullptr; Task *task = nullptr; - { - std::lock_guard<std::mutex> groupsLock(scheduler->m_groupsMutex); - for (uint32_t i = 0; i < scheduler->m_groups.size(); i++) { - group = scheduler->m_groups[i]; - if (!group) - continue; - std::lock_guard<std::mutex> queueLock(group->queueMutex); - if (group->queueHead < group->queue.size()) { - task = &group->queue[group->queueHead++]; - break; - } + for (uint32_t i = 0; i < scheduler->m_maxGroups; i++) { + group = &scheduler->m_groups[i]; + if (group->free || group->ref == 0) + continue; + group->queueLock.lock(); + if (group->queueHead < group->queue.size()) { + task = &group->queue[group->queueHead++]; + group->queueLock.unlock(); + break; } + group->queueLock.unlock(); } if (!task) break; @@ -3705,23 +3719,19 @@ public: destroyGroup({ i }); } - void run(TaskGroupHandle *handle, Task task) + TaskGroupHandle createTaskGroup(uint32_t reserveSize = 0) { - if (handle->value == UINT32_MAX) { - TaskGroup *group = XA_NEW(MemTag::Default, TaskGroup); - for (uint32_t i = 0; i < m_groups.size(); i++) { - if (!m_groups[i]) { - m_groups[i] = group; - handle->value = i; - break; - } - } - if (handle->value == UINT32_MAX) { - m_groups.push_back(group); - handle->value = m_groups.size() - 1; - } - } - m_groups[handle->value]->queue.push_back(task); + TaskGroup *group = XA_NEW(MemTag::Default, TaskGroup); + group->queue.reserve(reserveSize); + m_groups.push_back(group); + TaskGroupHandle handle; + handle.value = m_groups.size() - 1; + return handle; + } + + void run(TaskGroupHandle handle, Task task) + { + m_groups[handle.value]->queue.push_back(task); } void wait(TaskGroupHandle *handle) @@ -5967,6 +5977,58 @@ private: #endif }; +struct CreateChartTaskArgs +{ + const Mesh *mesh; + const Array<uint32_t> *faceArray; + const Basis *basis; + uint32_t meshId; + uint32_t chartGroupId; + uint32_t chartId; + Chart **chart; +}; + +static void runCreateChartTask(void *userData) +{ + XA_PROFILE_START(createChartMeshesThread) + auto args = (CreateChartTaskArgs *)userData; + *(args->chart) = XA_NEW(MemTag::Default, Chart, args->mesh, *(args->faceArray), *(args->basis), args->meshId, args->chartGroupId, args->chartId); + XA_PROFILE_END(createChartMeshesThread) +} + +struct ParameterizeChartTaskArgs +{ + Chart *chart; + ParameterizeFunc func; +}; + +static void runParameterizeChartTask(void *userData) +{ + auto args = (ParameterizeChartTaskArgs *)userData; + Mesh *mesh = args->chart->unifiedMesh(); + XA_PROFILE_START(parameterizeChartsOrthogonal) +#if 1 + computeOrthogonalProjectionMap(mesh); +#else + for (uint32_t i = 0; i < vertexCount; i++) + mesh->texcoord(i) = Vector2(dot(args->chart->basis().tangent, mesh->position(i)), dot(args->chart->basis().bitangent, mesh->position(i))); +#endif + XA_PROFILE_END(parameterizeChartsOrthogonal) + args->chart->evaluateOrthoParameterizationQuality(); + if (!args->chart->isOrtho() && !args->chart->isPlanar()) { + XA_PROFILE_START(parameterizeChartsLSCM) + if (args->func) + args->func(&mesh->position(0).x, &mesh->texcoord(0).x, mesh->vertexCount(), mesh->indices(), mesh->indexCount()); + else if (args->chart->isDisk()) + computeLeastSquaresConformalMap(mesh); + XA_PROFILE_END(parameterizeChartsLSCM) + args->chart->evaluateParameterizationQuality(); + } + // @@ Check that parameterization quality is above a certain threshold. + // Transfer parameterization from unified mesh to chart mesh. + args->chart->transferParameterization(); +} + // Set of charts corresponding to mesh faces in the same face group. class ChartGroup { @@ -6107,7 +6169,7 @@ public: - emphasize roundness metrics to prevent those cases. - If interior self-overlaps: preserve boundary parameterization and use mean-value map. */ - void computeCharts(const ChartOptions &options) + void computeCharts(TaskScheduler *taskScheduler, const ChartOptions &options) { m_chartOptions = options; // This function may be called multiple times, so destroy existing charts. @@ -6128,13 +6190,30 @@ public: AtlasBuilder builder(m_mesh, nullptr, options); runAtlasBuilder(builder, options); XA_PROFILE_END(atlasBuilder) - XA_PROFILE_START(createChartMeshes) const uint32_t chartCount = builder.chartCount(); + m_chartArray.resize(chartCount); + Array<CreateChartTaskArgs> taskArgs; + taskArgs.resize(chartCount); + for (uint32_t i = 0; i < chartCount; i++) { + CreateChartTaskArgs &args = taskArgs[i]; + args.mesh = m_mesh; + args.faceArray = &builder.chartFaces(i); + args.basis = &builder.chartBasis(i); + args.meshId = m_sourceId; + args.chartGroupId = m_id; + args.chartId = i; + args.chart = &m_chartArray[i]; + } + XA_PROFILE_START(createChartMeshesReal) + TaskGroupHandle taskGroup = taskScheduler->createTaskGroup(chartCount); for (uint32_t i = 0; i < chartCount; i++) { - Chart *chart = XA_NEW(MemTag::Default, Chart, m_mesh, builder.chartFaces(i), builder.chartBasis(i), m_sourceId, m_id, i); - m_chartArray.push_back(chart); + Task task; + task.userData = &taskArgs[i]; + task.func = runCreateChartTask; + taskScheduler->run(taskGroup, task); } - XA_PROFILE_END(createChartMeshes) + taskScheduler->wait(&taskGroup); + XA_PROFILE_END(createChartMeshesReal) #endif #if XA_DEBUG_EXPORT_OBJ_CHARTS char filename[256]; @@ -6157,18 +6236,33 @@ public: #endif } - void parameterizeCharts(ParameterizeFunc func) + void parameterizeCharts(TaskScheduler *taskScheduler, ParameterizeFunc func) { + const uint32_t chartCount = m_chartArray.size(); + Array<ParameterizeChartTaskArgs> taskArgs; + taskArgs.resize(chartCount); + TaskGroupHandle taskGroup = taskScheduler->createTaskGroup(chartCount); + for (uint32_t i = 0; i < chartCount; i++) { + ParameterizeChartTaskArgs &args = taskArgs[i]; + args.chart = m_chartArray[i]; + args.func = func; + Task task; + task.userData = &args; + task.func = runParameterizeChartTask; + taskScheduler->run(taskGroup, task); + } + taskScheduler->wait(&taskGroup); #if XA_RECOMPUTE_CHARTS + // Find charts with invalid parameterizations. Array<Chart *> invalidCharts; - const uint32_t chartCount = m_chartArray.size(); for (uint32_t i = 0; i < chartCount; i++) { Chart *chart = m_chartArray[i]; - parameterizeChart(chart, func); const ParameterizationQuality &quality = chart->paramQuality(); if (quality.boundaryIntersection || quality.flippedTriangleCount > 0) invalidCharts.push_back(chart); } + if (invalidCharts.isEmpty()) + return; // Recompute charts with invalid parameterizations. Array<uint32_t> meshFaces; for (uint32_t i = 0; i < invalidCharts.size(); i++) { @@ -6211,8 +6305,18 @@ public: #endif } // Parameterize the new charts. - for (uint32_t i = chartCount; i < m_chartArray.size(); i++) - parameterizeChart(m_chartArray[i], func); + taskGroup = taskScheduler->createTaskGroup(m_chartArray.size() - chartCount); + taskArgs.resize(m_chartArray.size() - chartCount); + for (uint32_t i = chartCount; i < m_chartArray.size(); i++) { + ParameterizeChartTaskArgs &args = taskArgs[i - chartCount]; + args.chart = m_chartArray[i]; + args.func = func; + Task task; + task.userData = &args; + task.func = runParameterizeChartTask; + taskScheduler->run(taskGroup, task); + } + taskScheduler->wait(&taskGroup); // Remove and delete the invalid charts. for (uint32_t i = 0; i < invalidCharts.size(); i++) { Chart *chart = invalidCharts[i]; @@ -6221,12 +6325,6 @@ public: XA_FREE(chart); m_paramDeletedChartsCount++; } -#else - const uint32_t chartCount = m_chartArray.size(); - for (uint32_t i = 0; i < chartCount; i++) { - Chart *chart = m_chartArray[i]; - parameterizeChart(chart, func); - } #endif } @@ -6269,32 +6367,6 @@ private: XA_DEBUG_ASSERT(builder.facesLeft() == 0); } - void parameterizeChart(Chart *chart, ParameterizeFunc func) - { - Mesh *mesh = chart->unifiedMesh(); - XA_PROFILE_START(parameterizeChartsOrthogonal) -#if 1 - computeOrthogonalProjectionMap(mesh); -#else - for (uint32_t i = 0; i < vertexCount; i++) - mesh->texcoord(i) = Vector2(dot(chart->basis().tangent, mesh->position(i)), dot(chart->basis().bitangent, mesh->position(i))); -#endif - XA_PROFILE_END(parameterizeChartsOrthogonal) - chart->evaluateOrthoParameterizationQuality(); - if (!chart->isOrtho() && !chart->isPlanar()) { - XA_PROFILE_START(parameterizeChartsLSCM) - if (func) - func(&mesh->position(0).x, &mesh->texcoord(0).x, mesh->vertexCount(), mesh->indices(), mesh->indexCount()); - else if (chart->isDisk()) - computeLeastSquaresConformalMap(mesh); - XA_PROFILE_END(parameterizeChartsLSCM) - chart->evaluateParameterizationQuality(); - } - // @@ Check that parameterization quality is above a certain threshold. - // Transfer parameterization from unified mesh to chart mesh. - chart->transferParameterization(); - } - void removeChart(const Chart *chart) { for (uint32_t i = 0; i < m_chartArray.size(); i++) { @@ -6326,14 +6398,15 @@ struct CreateChartGroupTaskArgs static void runCreateChartGroupTask(void *userData) { - XA_PROFILE_START(addMeshCreateChartGroups) + XA_PROFILE_START(addMeshCreateChartGroupsThread) auto args = (CreateChartGroupTaskArgs *)userData; *(args->chartGroup) = XA_NEW(MemTag::Default, ChartGroup, args->groupId, args->mesh, args->faceGroup); - XA_PROFILE_END(addMeshCreateChartGroups) + XA_PROFILE_END(addMeshCreateChartGroupsThread) } struct ComputeChartsTaskArgs { + TaskScheduler *taskScheduler; ChartGroup *chartGroup; const ChartOptions *options; Progress *progress; @@ -6341,18 +6414,19 @@ struct ComputeChartsTaskArgs static void runComputeChartsJob(void *userData) { - ComputeChartsTaskArgs *args = (ComputeChartsTaskArgs *)userData; + auto args = (ComputeChartsTaskArgs *)userData; if (args->progress->cancel) return; - XA_PROFILE_START(computeCharts) - args->chartGroup->computeCharts(*args->options); - XA_PROFILE_END(computeCharts) + XA_PROFILE_START(computeChartsThread) + args->chartGroup->computeCharts(args->taskScheduler, *args->options); + XA_PROFILE_END(computeChartsThread) args->progress->value++; args->progress->update(); } struct ParameterizeChartsTaskArgs { + TaskScheduler *taskScheduler; ChartGroup *chartGroup; ParameterizeFunc func; Progress *progress; @@ -6360,12 +6434,12 @@ struct ParameterizeChartsTaskArgs static void runParameterizeChartsJob(void *userData) { - ParameterizeChartsTaskArgs *args = (ParameterizeChartsTaskArgs *)userData; + auto args = (ParameterizeChartsTaskArgs *)userData; if (args->progress->cancel) return; - XA_PROFILE_START(parameterizeCharts) - args->chartGroup->parameterizeCharts(args->func); - XA_PROFILE_END(parameterizeCharts) + XA_PROFILE_START(parameterizeChartsThread) + args->chartGroup->parameterizeCharts(args->taskScheduler, args->func); + XA_PROFILE_END(parameterizeChartsThread) args->progress->value++; args->progress->update(); } @@ -6460,12 +6534,12 @@ public: args.groupId = g; args.mesh = mesh; } - TaskGroupHandle taskGroup; + TaskGroupHandle taskGroup = taskScheduler->createTaskGroup(chartGroups.size()); for (uint32_t g = 0; g < chartGroups.size(); g++) { Task task; task.userData = &taskArgs[g]; task.func = runCreateChartGroupTask; - taskScheduler->run(&taskGroup, task); + taskScheduler->run(taskGroup, task); } taskScheduler->wait(&taskGroup); // Thread-safe append. @@ -6481,29 +6555,39 @@ public: { m_chartsComputed = false; m_chartsParameterized = false; - uint32_t taskCount = 0; + // Ignore vertex maps. + uint32_t chartGroupCount = 0; for (uint32_t i = 0; i < m_chartGroups.size(); i++) { if (!m_chartGroups[i]->isVertexMap()) - taskCount++; + chartGroupCount++; } - Progress progress(ProgressCategory::ComputeCharts, progressFunc, progressUserData, taskCount); + Progress progress(ProgressCategory::ComputeCharts, progressFunc, progressUserData, chartGroupCount); Array<ComputeChartsTaskArgs> taskArgs; - taskArgs.reserve(taskCount); + taskArgs.reserve(chartGroupCount); for (uint32_t i = 0; i < m_chartGroups.size(); i++) { if (!m_chartGroups[i]->isVertexMap()) { ComputeChartsTaskArgs args; + args.taskScheduler = taskScheduler; args.chartGroup = m_chartGroups[i]; args.options = &options; args.progress = &progress; taskArgs.push_back(args); } } - TaskGroupHandle taskGroup; - for (uint32_t i = 0; i < taskCount; i++) { + // Sort chart groups by mesh indexCount. + m_chartGroupsRadix = RadixSort(); + Array<float> chartGroupSortData; + chartGroupSortData.resize(chartGroupCount); + for (uint32_t i = 0; i < chartGroupCount; i++) + chartGroupSortData[i] = (float)taskArgs[i].chartGroup->mesh()->indexCount(); + m_chartGroupsRadix.sort(chartGroupSortData); + // Larger chart group meshes are added first to reduce the chance of thread starvation. + TaskGroupHandle taskGroup = taskScheduler->createTaskGroup(chartGroupCount); + for (uint32_t i = 0; i < chartGroupCount; i++) { Task task; - task.userData = &taskArgs[i]; + task.userData = &taskArgs[m_chartGroupsRadix.ranks()[chartGroupCount - i - 1]]; task.func = runComputeChartsJob; - taskScheduler->run(&taskGroup, task); + taskScheduler->run(taskGroup, task); } taskScheduler->wait(&taskGroup); if (progress.cancel) @@ -6515,29 +6599,32 @@ public: bool parameterizeCharts(TaskScheduler *taskScheduler, ParameterizeFunc func, ProgressFunc progressFunc, void *progressUserData) { m_chartsParameterized = false; - uint32_t taskCount = 0; + // Ignore vertex maps. + uint32_t chartGroupCount = 0; for (uint32_t i = 0; i < m_chartGroups.size(); i++) { if (!m_chartGroups[i]->isVertexMap()) - taskCount++; + chartGroupCount++; } - Progress progress(ProgressCategory::ParameterizeCharts, progressFunc, progressUserData, taskCount); + Progress progress(ProgressCategory::ParameterizeCharts, progressFunc, progressUserData, chartGroupCount); Array<ParameterizeChartsTaskArgs> taskArgs; - taskArgs.reserve(taskCount); + taskArgs.reserve(chartGroupCount); for (uint32_t i = 0; i < m_chartGroups.size(); i++) { if (!m_chartGroups[i]->isVertexMap()) { ParameterizeChartsTaskArgs args; + args.taskScheduler = taskScheduler; args.chartGroup = m_chartGroups[i]; args.func = func; args.progress = &progress; taskArgs.push_back(args); } } - TaskGroupHandle taskGroup; - for (uint32_t i = 0; i < taskCount; i++) { + // Larger chart group meshes are added first to reduce the chance of thread starvation. + TaskGroupHandle taskGroup = taskScheduler->createTaskGroup(chartGroupCount); + for (uint32_t i = 0; i < chartGroupCount; i++) { Task task; - task.userData = &taskArgs[i]; + task.userData = &taskArgs[m_chartGroupsRadix.ranks()[chartGroupCount - i - 1]]; task.func = runParameterizeChartsJob; - taskScheduler->run(&taskGroup, task); + taskScheduler->run(taskGroup, task); } taskScheduler->wait(&taskGroup); if (progress.cancel) @@ -6570,6 +6657,7 @@ private: bool m_chartsComputed; bool m_chartsParameterized; Array<ChartGroup *> m_chartGroups; + RadixSort m_chartGroupsRadix; // By mesh indexCount. Array<uint32_t> m_chartGroupSourceMeshes; Array<Array<Vector2> > m_originalChartTexcoords; }; @@ -6734,6 +6822,71 @@ struct Chart uint32_t uniqueVertexCount() const { return uniqueVertices.isEmpty() ? vertexCount : uniqueVertices.size(); } }; +struct FindChartLocationBruteForceTaskArgs +{ + std::atomic<bool> *finished; // One of the tasks found a location that doesn't expand the atlas. + Vector2i startPosition; + const BitImage *atlasBitImage; + const BitImage *chartBitImage; + const BitImage *chartBitImageRotated; + int w, h; + bool blockAligned, resizableAtlas, allowRotate; + // out + bool best_insideAtlas; + int best_metric, best_x, best_y, best_w, best_h, best_r; +}; + +static void runFindChartLocationBruteForceTask(void *userData) +{ + XA_PROFILE_START(packChartsFindLocationThread) + auto args = (FindChartLocationBruteForceTaskArgs *)userData; + args->best_metric = INT_MAX; + if (args->finished->load()) + return; + // Try two different orientations. + for (int r = 0; r < 2; r++) { + int cw = args->chartBitImage->width(); + int ch = args->chartBitImage->height(); + if (r == 1) { + if (args->allowRotate) + swap(cw, ch); + else + break; + } + const int y = args->startPosition.y; + const int stepSize = args->blockAligned ? 4 : 1; + for (int x = args->startPosition.x; x <= args->w + stepSize; x += stepSize) { // + 1 not really necessary here. + if (!args->resizableAtlas && (x > (int)args->atlasBitImage->width() - cw || y > (int)args->atlasBitImage->height() - ch)) + continue; + if (args->finished->load()) + break; + // Early out if metric not better. + const int area = max(args->w, x + cw) * max(args->h, y + ch); + const int extents = max(max(args->w, x + cw), max(args->h, y + ch)); + const int metric = extents * extents + area; + if (metric > args->best_metric) + continue; + // If metric is the same, pick the one closest to the origin. + if (metric == args->best_metric && max(x, y) >= max(args->best_x, args->best_y)) + continue; + if (!args->atlasBitImage->canBlit(r == 1 ? *(args->chartBitImageRotated) : *(args->chartBitImage), x, y)) + continue; + args->best_metric = metric; + args->best_insideAtlas = area == args->w * args->h; + args->best_x = x; + args->best_y = y; + args->best_w = cw; + args->best_h = ch; + args->best_r = r; + if (args->best_insideAtlas) { + args->finished->store(true); + break; + } + } + } + XA_PROFILE_END(packChartsFindLocationThread) +} + struct Atlas { ~Atlas() @@ -6854,7 +7007,7 @@ struct Atlas } // Pack charts in the smallest possible rectangle. - bool packCharts(const PackOptions &options, ProgressFunc progressFunc, void *progressUserData) + bool packCharts(TaskScheduler *taskScheduler, const PackOptions &options, ProgressFunc progressFunc, void *progressUserData) { if (progressFunc) { if (!progressFunc(ProgressCategory::PackCharts, 0, progressUserData)) @@ -7069,7 +7222,7 @@ struct Atlas chartStartPositions.push_back(Vector2i(0, 0)); } XA_PROFILE_START(packChartsFindLocation) - const bool foundLocation = findChartLocation(chartStartPositions[currentAtlas], options.bruteForce, m_bitImages[currentAtlas], &chartBitImage, &chartBitImageRotated, atlasWidth, atlasHeight, &best_x, &best_y, &best_cw, &best_ch, &best_r, options.blockAlign, resizableAtlas, chart->allowRotate); + const bool foundLocation = findChartLocation(taskScheduler, chartStartPositions[currentAtlas], options.bruteForce, m_bitImages[currentAtlas], &chartBitImage, &chartBitImageRotated, atlasWidth, atlasHeight, &best_x, &best_y, &best_cw, &best_ch, &best_r, options.blockAlign, resizableAtlas, chart->allowRotate); XA_PROFILE_END(packChartsFindLocation) if (firstChartInBitImage && !foundLocation) { // Chart doesn't fit in an empty, newly allocated bitImage. texelsPerUnit must be too large for the resolution. @@ -7181,65 +7334,66 @@ private: // is occupied at this point. At the end we have many small charts and a large atlas with sparse holes. Finding those holes randomly is slow. A better approach would be to // start stacking large charts as if they were tetris pieces. Once charts get small try to place them randomly. It may be interesting to try a intermediate strategy, first try // along one axis and then try exhaustively along that axis. - bool findChartLocation(const Vector2i &startPosition, bool bruteForce, const BitImage *atlasBitImage, const BitImage *chartBitImage, const BitImage *chartBitImageRotated, int w, int h, int *best_x, int *best_y, int *best_w, int *best_h, int *best_r, bool blockAligned, bool resizableAtlas, bool allowRotate) + bool findChartLocation(TaskScheduler *taskScheduler, const Vector2i &startPosition, bool bruteForce, const BitImage *atlasBitImage, const BitImage *chartBitImage, const BitImage *chartBitImageRotated, int w, int h, int *best_x, int *best_y, int *best_w, int *best_h, int *best_r, bool blockAligned, bool resizableAtlas, bool allowRotate) { const int attempts = 4096; if (bruteForce || attempts >= w * h) - return findChartLocation_bruteForce(startPosition, atlasBitImage, chartBitImage, chartBitImageRotated, w, h, best_x, best_y, best_w, best_h, best_r, blockAligned, resizableAtlas, allowRotate); + return findChartLocation_bruteForce(taskScheduler, startPosition, atlasBitImage, chartBitImage, chartBitImageRotated, w, h, best_x, best_y, best_w, best_h, best_r, blockAligned, resizableAtlas, allowRotate); return findChartLocation_random(atlasBitImage, chartBitImage, chartBitImageRotated, w, h, best_x, best_y, best_w, best_h, best_r, attempts, blockAligned, resizableAtlas, allowRotate); } - bool findChartLocation_bruteForce(const Vector2i &startPosition, const BitImage *atlasBitImage, const BitImage *chartBitImage, const BitImage *chartBitImageRotated, int w, int h, int *best_x, int *best_y, int *best_w, int *best_h, int *best_r, bool blockAligned, bool resizableAtlas, bool allowRotate) + bool findChartLocation_bruteForce(TaskScheduler *taskScheduler, const Vector2i &startPosition, const BitImage *atlasBitImage, const BitImage *chartBitImage, const BitImage *chartBitImageRotated, int w, int h, int *best_x, int *best_y, int *best_w, int *best_h, int *best_r, bool blockAligned, bool resizableAtlas, bool allowRotate) { - bool result = false; - const int BLOCK_SIZE = 4; + const int stepSize = blockAligned ? 4 : 1; + uint32_t taskCount = 0; + for (int y = startPosition.y; y <= h + stepSize; y += stepSize) + taskCount++; + Array<FindChartLocationBruteForceTaskArgs> taskArgs; + taskArgs.resize(taskCount); + TaskGroupHandle taskGroup = taskScheduler->createTaskGroup(taskCount); + std::atomic<bool> finished(false); // One of the tasks found a location that doesn't expand the atlas. + uint32_t i = 0; + for (int y = startPosition.y; y <= h + stepSize; y += stepSize) { + FindChartLocationBruteForceTaskArgs &args = taskArgs[i]; + args.finished = &finished; + args.startPosition = Vector2i(y == startPosition.y ? startPosition.x : 0, y); + args.atlasBitImage = atlasBitImage; + args.chartBitImage = chartBitImage; + args.chartBitImageRotated = chartBitImageRotated; + args.w = w; + args.h = h; + args.blockAligned = blockAligned; + args.resizableAtlas = resizableAtlas; + args.allowRotate = allowRotate; + Task task; + task.userData = &taskArgs[i]; + task.func = runFindChartLocationBruteForceTask; + taskScheduler->run(taskGroup, task); + i++; + } + taskScheduler->wait(&taskGroup); + // Find the task result with the best metric. int best_metric = INT_MAX; - int step_size = blockAligned ? BLOCK_SIZE : 1; - // Try two different orientations. - for (int r = 0; r < 2; r++) { - int cw = chartBitImage->width(); - int ch = chartBitImage->height(); - if (r == 1) { - if (allowRotate) - swap(cw, ch); - else - break; - } - for (int y = startPosition.y; y <= h + step_size; y += step_size) { // + 1 to extend atlas in case atlas full. - for (int x = (y == startPosition.y ? startPosition.x : 0); x <= w + step_size; x += step_size) { // + 1 not really necessary here. - if (!resizableAtlas && (x > (int)atlasBitImage->width() - cw || y > (int)atlasBitImage->height() - ch)) - continue; - // Early out. - int area = max(w, x + cw) * max(h, y + ch); - //int perimeter = max(w, x+cw) + max(h, y+ch); - int extents = max(max(w, x + cw), max(h, y + ch)); - int metric = extents * extents + area; - if (metric > best_metric) { - continue; - } - if (metric == best_metric && max(x, y) >= max(*best_x, *best_y)) { - // If metric is the same, pick the one closest to the origin. - continue; - } - if (atlasBitImage->canBlit(r == 1 ? *chartBitImageRotated : *chartBitImage, x, y)) { - result = true; - best_metric = metric; - *best_x = x; - *best_y = y; - *best_w = cw; - *best_h = ch; - *best_r = r; - if (area == w * h) { - // Chart is completely inside, do not look at any other location. - goto done; - } - } - } - } + bool best_insideAtlas = false; + for (i = 0; i < taskCount; i++) { + FindChartLocationBruteForceTaskArgs &args = taskArgs[i]; + if (args.best_metric > best_metric) + continue; + // A location that doesn't expand the atlas is always preferred. + if (!args.best_insideAtlas && best_insideAtlas) + continue; + // If metric is the same, pick the one closest to the origin. + if (args.best_insideAtlas == best_insideAtlas && args.best_metric == best_metric && max(args.best_x, args.best_y) >= max(*best_x, *best_y)) + continue; + best_metric = args.best_metric; + best_insideAtlas = args.best_insideAtlas; + *best_x = args.best_x; + *best_y = args.best_y; + *best_w = args.best_w; + *best_h = args.best_h; + *best_r = args.best_r; } - done: - XA_DEBUG_ASSERT (best_metric != INT_MAX); - return result; + return best_metric != INT_MAX; } bool findChartLocation_random(const BitImage *atlasBitImage, const BitImage *chartBitImage, const BitImage *chartBitImageRotated, int w, int h, int *best_x, int *best_y, int *best_w, int *best_h, int *best_r, int minTrialCount, bool blockAligned, bool resizableAtlas, bool allowRotate) @@ -7439,25 +7593,31 @@ struct AddMeshTaskArgs static void runAddMeshTask(void *userData) { - XA_PROFILE_START(addMesh) + XA_PROFILE_START(addMeshThread) auto args = (AddMeshTaskArgs *)userData; // Responsible for freeing this. internal::Mesh *mesh = args->mesh; internal::Progress *progress = args->ctx->addMeshProgress; if (progress->cancel) goto cleanup; - XA_PROFILE_START(addMeshCreateColocals) - mesh->createColocals(); - XA_PROFILE_END(addMeshCreateColocals) + { + XA_PROFILE_START(addMeshCreateColocals) + mesh->createColocals(); + XA_PROFILE_END(addMeshCreateColocals) + } if (progress->cancel) goto cleanup; - XA_PROFILE_START(addMeshCreateFaceGroups) - mesh->createFaceGroups(); - XA_PROFILE_END(addMeshCreateFaceGroups) + { + XA_PROFILE_START(addMeshCreateFaceGroups) + mesh->createFaceGroups(); + XA_PROFILE_END(addMeshCreateFaceGroups) + } if (progress->cancel) goto cleanup; - XA_PROFILE_START(addMeshCreateBoundaries) - mesh->createBoundaries(); - XA_PROFILE_END(addMeshCreateBoundaries) + { + XA_PROFILE_START(addMeshCreateBoundaries) + mesh->createBoundaries(); + XA_PROFILE_END(addMeshCreateBoundaries) + } if (progress->cancel) goto cleanup; #if XA_DEBUG_EXPORT_OBJ_SOURCE_MESHES @@ -7491,9 +7651,11 @@ static void runAddMeshTask(void *userData) fclose(file); } #endif - XA_PROFILE_START(addMeshCreateChartGroupsConcurrent) - args->ctx->paramAtlas.addMesh(args->ctx->taskScheduler, mesh); // addMesh is thread safe - XA_PROFILE_END(addMeshCreateChartGroupsConcurrent) + { + XA_PROFILE_START(addMeshCreateChartGroupsReal) + args->ctx->paramAtlas.addMesh(args->ctx->taskScheduler, mesh); // addMesh is thread safe + XA_PROFILE_END(addMeshCreateChartGroupsReal) + } if (progress->cancel) goto cleanup; progress->value++; @@ -7503,7 +7665,7 @@ cleanup: XA_FREE(mesh); args->~AddMeshTaskArgs(); XA_FREE(args); - XA_PROFILE_END(addMesh) + XA_PROFILE_END(addMeshThread) } static internal::Vector3 DecodePosition(const MeshDecl &meshDecl, uint32_t index) @@ -7547,12 +7709,13 @@ AddMeshError::Enum AddMesh(Atlas *atlas, const MeshDecl &meshDecl, uint32_t mesh XA_PRINT_WARNING("AddMesh: Meshes and UV meshes cannot be added to the same atlas.\n"); return AddMeshError::Error; } +#if XA_PROFILE + if (ctx->meshCount == 0) + internal::s_profile.addMeshReal = clock(); +#endif // Don't know how many times AddMesh will be called, so progress needs to adjusted each time. if (!ctx->addMeshProgress) { ctx->addMeshProgress = XA_NEW(internal::MemTag::Default, internal::Progress, ProgressCategory::AddMesh, ctx->progressFunc, ctx->progressUserData, 1); -#if XA_PROFILE - internal::s_profile.addMeshConcurrent = clock(); -#endif } else { ctx->addMeshProgress->setMaxValue(internal::max(ctx->meshCount + 1, meshCountHint)); @@ -7560,7 +7723,6 @@ AddMeshError::Enum AddMesh(Atlas *atlas, const MeshDecl &meshDecl, uint32_t mesh bool decoded = (meshDecl.indexCount <= 0); uint32_t indexCount = decoded ? meshDecl.vertexCount : meshDecl.indexCount; XA_PRINT("Adding mesh %d: %u vertices, %u triangles\n", ctx->meshCount, meshDecl.vertexCount, indexCount / 3); - XA_PROFILE_START(addMesh) // Expecting triangle faces. if ((indexCount % 3) != 0) return AddMeshError::InvalidIndexCount; @@ -7629,15 +7791,16 @@ AddMeshError::Enum AddMesh(Atlas *atlas, const MeshDecl &meshDecl, uint32_t mesh ignore = true; mesh->addFace(tri[0], tri[1], tri[2], ignore); } + if (ctx->addMeshTaskGroup.value == UINT32_MAX) + ctx->addMeshTaskGroup = ctx->taskScheduler->createTaskGroup(); AddMeshTaskArgs *taskArgs = XA_NEW(internal::MemTag::Default, AddMeshTaskArgs); // The task frees this. taskArgs->ctx = ctx; taskArgs->mesh = mesh; internal::Task task; task.userData = taskArgs; task.func = runAddMeshTask; - ctx->taskScheduler->run(&ctx->addMeshTaskGroup, task); + ctx->taskScheduler->run(ctx->addMeshTaskGroup, task); ctx->meshCount++; - XA_PROFILE_END(addMesh) return AddMeshError::Success; } @@ -7657,15 +7820,15 @@ void AddMeshJoin(Atlas *atlas) ctx->addMeshProgress = nullptr; #if XA_PROFILE XA_PRINT("Added %u meshes\n", ctx->meshCount); - internal::s_profile.addMeshConcurrent = clock() - internal::s_profile.addMeshConcurrent; + internal::s_profile.addMeshReal = clock() - internal::s_profile.addMeshReal; #endif - XA_PROFILE_PRINT(" Total (concurrent): ", addMeshConcurrent) - XA_PROFILE_PRINT(" Total: ", addMesh) - XA_PROFILE_PRINT(" Create colocals: ", addMeshCreateColocals) - XA_PROFILE_PRINT(" Create face groups: ", addMeshCreateFaceGroups) - XA_PROFILE_PRINT(" Create boundaries: ", addMeshCreateBoundaries) - XA_PROFILE_PRINT(" Create chart groups (concurrent): ", addMeshCreateChartGroupsConcurrent) - XA_PROFILE_PRINT(" Create chart groups: ", addMeshCreateChartGroups) + XA_PROFILE_PRINT_AND_RESET(" Total (real): ", addMeshReal) + XA_PROFILE_PRINT_AND_RESET(" Total (thread): ", addMeshThread) + XA_PROFILE_PRINT_AND_RESET(" Create colocals: ", addMeshCreateColocals) + XA_PROFILE_PRINT_AND_RESET(" Create face groups: ", addMeshCreateFaceGroups) + XA_PROFILE_PRINT_AND_RESET(" Create boundaries: ", addMeshCreateBoundaries) + XA_PROFILE_PRINT_AND_RESET(" Create chart groups (real): ", addMeshCreateChartGroupsReal) + XA_PROFILE_PRINT_AND_RESET(" Create chart groups (thread): ", addMeshCreateChartGroupsThread) XA_PRINT_MEM_USAGE } @@ -7815,12 +7978,12 @@ void ComputeCharts(Atlas *atlas, ChartOptions chartOptions) } XA_PRINT("Computing charts\n"); uint32_t chartCount = 0, chartsWithHolesCount = 0, holesCount = 0, chartsWithTJunctionsCount = 0, tJunctionsCount = 0; - XA_PROFILE_START(computeChartsConcurrent) + XA_PROFILE_START(computeChartsReal) if (!ctx->paramAtlas.computeCharts(ctx->taskScheduler, chartOptions, ctx->progressFunc, ctx->progressUserData)) { XA_PRINT(" Cancelled by user\n"); return; } - XA_PROFILE_END(computeChartsConcurrent) + XA_PROFILE_END(computeChartsReal) // Count charts and print warnings. for (uint32_t i = 0; i < ctx->meshCount; i++) { for (uint32_t j = 0; j < ctx->paramAtlas.chartGroupCount(i); j++) { @@ -7854,16 +8017,17 @@ void ComputeCharts(Atlas *atlas, ChartOptions chartOptions) if (tJunctionsCount > 0) XA_PRINT(" Fixed %u t-junctions in %u charts\n", tJunctionsCount, chartsWithTJunctionsCount); XA_PRINT(" %u charts\n", chartCount); - XA_PROFILE_PRINT(" Total (concurrent): ", computeChartsConcurrent) - XA_PROFILE_PRINT(" Total: ", computeCharts) - XA_PROFILE_PRINT(" Atlas builder: ", atlasBuilder) - XA_PROFILE_PRINT(" Init: ", atlasBuilderInit) - XA_PROFILE_PRINT(" Create initial charts: ", atlasBuilderCreateInitialCharts) - XA_PROFILE_PRINT(" Grow charts: ", atlasBuilderGrowCharts) - XA_PROFILE_PRINT(" Merge charts: ", atlasBuilderMergeCharts) - XA_PROFILE_PRINT(" Create chart meshes: ", createChartMeshes) - XA_PROFILE_PRINT(" Fix t-junctions: ", fixChartMeshTJunctions); - XA_PROFILE_PRINT(" Close holes: ", closeChartMeshHoles) + XA_PROFILE_PRINT_AND_RESET(" Total (real): ", computeChartsReal) + XA_PROFILE_PRINT_AND_RESET(" Total (thread): ", computeChartsThread) + XA_PROFILE_PRINT_AND_RESET(" Atlas builder: ", atlasBuilder) + XA_PROFILE_PRINT_AND_RESET(" Init: ", atlasBuilderInit) + XA_PROFILE_PRINT_AND_RESET(" Create initial charts: ", atlasBuilderCreateInitialCharts) + XA_PROFILE_PRINT_AND_RESET(" Grow charts: ", atlasBuilderGrowCharts) + XA_PROFILE_PRINT_AND_RESET(" Merge charts: ", atlasBuilderMergeCharts) + XA_PROFILE_PRINT_AND_RESET(" Create chart meshes (real): ", createChartMeshesReal) + XA_PROFILE_PRINT_AND_RESET(" Create chart meshes (thread): ", createChartMeshesThread) + XA_PROFILE_PRINT_AND_RESET(" Fix t-junctions: ", fixChartMeshTJunctions) + XA_PROFILE_PRINT_AND_RESET(" Close holes: ", closeChartMeshHoles) XA_PRINT_MEM_USAGE } @@ -7896,12 +8060,12 @@ void ParameterizeCharts(Atlas *atlas, ParameterizeFunc func) } DestroyOutputMeshes(ctx); XA_PRINT("Parameterizing charts\n"); - XA_PROFILE_START(parameterizeChartsConcurrent) + XA_PROFILE_START(parameterizeChartsReal) if (!ctx->paramAtlas.parameterizeCharts(ctx->taskScheduler, func, ctx->progressFunc, ctx->progressUserData)) { XA_PRINT(" Cancelled by user\n"); return; } - XA_PROFILE_END(parameterizeChartsConcurrent) + XA_PROFILE_END(parameterizeChartsReal) uint32_t chartCount = 0, orthoChartsCount = 0, planarChartsCount = 0, chartsAddedCount = 0, chartsDeletedCount = 0; for (uint32_t i = 0; i < ctx->meshCount; i++) { for (uint32_t j = 0; j < ctx->paramAtlas.chartGroupCount(i); j++) { @@ -7982,11 +8146,11 @@ void ParameterizeCharts(Atlas *atlas, ParameterizeFunc func) } if (invalidParamCount > 0) XA_PRINT_WARNING(" %u charts with invalid parameterizations\n", invalidParamCount); - XA_PROFILE_PRINT(" Total (concurrent): ", parameterizeChartsConcurrent) - XA_PROFILE_PRINT(" Total: ", parameterizeCharts) - XA_PROFILE_PRINT(" Orthogonal: ", parameterizeChartsOrthogonal) - XA_PROFILE_PRINT(" LSCM: ", parameterizeChartsLSCM) - XA_PROFILE_PRINT(" Evaluate quality: ", parameterizeChartsEvaluateQuality) + XA_PROFILE_PRINT_AND_RESET(" Total (real): ", parameterizeChartsReal) + XA_PROFILE_PRINT_AND_RESET(" Total (thread): ", parameterizeChartsThread) + XA_PROFILE_PRINT_AND_RESET(" Orthogonal: ", parameterizeChartsOrthogonal) + XA_PROFILE_PRINT_AND_RESET(" LSCM: ", parameterizeChartsLSCM) + XA_PROFILE_PRINT_AND_RESET(" Evaluate quality: ", parameterizeChartsEvaluateQuality) XA_PRINT_MEM_USAGE } @@ -8039,7 +8203,7 @@ void PackCharts(Atlas *atlas, PackOptions packOptions) packAtlas.addChart(ctx->paramAtlas.chartAt(i)); } XA_PROFILE_START(packCharts) - if (!packAtlas.packCharts(packOptions, ctx->progressFunc, ctx->progressUserData)) + if (!packAtlas.packCharts(ctx->taskScheduler, packOptions, ctx->progressFunc, ctx->progressUserData)) return; XA_PROFILE_END(packCharts) // Populate atlas object with pack results. @@ -8058,19 +8222,13 @@ void PackCharts(Atlas *atlas, PackOptions packOptions) for (uint32_t i = 0; i < atlas->atlasCount; i++) packAtlas.getImages()[i]->copyTo(&atlas->image[atlas->width * atlas->height * i], atlas->width, atlas->height); } - XA_PROFILE_PRINT(" Total: ", packCharts) - XA_PROFILE_PRINT(" Rasterize: ", packChartsRasterize) - XA_PROFILE_PRINT(" Dilate (padding): ", packChartsDilate) - XA_PROFILE_PRINT(" Find location: ", packChartsFindLocation) - XA_PROFILE_PRINT(" Blit: ", packChartsBlit) + XA_PROFILE_PRINT_AND_RESET(" Total: ", packCharts) + XA_PROFILE_PRINT_AND_RESET(" Rasterize: ", packChartsRasterize) + XA_PROFILE_PRINT_AND_RESET(" Dilate (padding): ", packChartsDilate) + XA_PROFILE_PRINT_AND_RESET(" Find location (real): ", packChartsFindLocation) + XA_PROFILE_PRINT_AND_RESET(" Find location (thread): ", packChartsFindLocationThread) + XA_PROFILE_PRINT_AND_RESET(" Blit: ", packChartsBlit) XA_PRINT_MEM_USAGE -#if XA_PROFILE - internal::s_profile.packCharts = 0; - internal::s_profile.packChartsRasterize = 0; - internal::s_profile.packChartsDilate = 0; - internal::s_profile.packChartsFindLocation = 0; - internal::s_profile.packChartsBlit = 0; -#endif XA_PRINT("Building output meshes\n"); int progress = 0; if (ctx->progressFunc) { diff --git a/thirdparty/zstd/common/compiler.h b/thirdparty/zstd/common/compiler.h index 0836e3ed27..87bf51ae8c 100644 --- a/thirdparty/zstd/common/compiler.h +++ b/thirdparty/zstd/common/compiler.h @@ -127,6 +127,13 @@ } \ } +/* vectorization */ +#if !defined(__clang__) && defined(__GNUC__) +# define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize"))) +#else +# define DONT_VECTORIZE +#endif + /* disable warnings */ #ifdef _MSC_VER /* Visual Studio */ # include <intrin.h> /* For Visual 2005 */ diff --git a/thirdparty/zstd/common/zstd_internal.h b/thirdparty/zstd/common/zstd_internal.h index 31f756ab58..81b16eac2e 100644 --- a/thirdparty/zstd/common/zstd_internal.h +++ b/thirdparty/zstd/common/zstd_internal.h @@ -34,7 +34,6 @@ #endif #include "xxhash.h" /* XXH_reset, update, digest */ - #if defined (__cplusplus) extern "C" { #endif @@ -193,19 +192,72 @@ static const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG; * Shared functions to include for inlining *********************************************/ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); } + #define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; } +static void ZSTD_copy16(void* dst, const void* src) { memcpy(dst, src, 16); } +#define COPY16(d,s) { ZSTD_copy16(d,s); d+=16; s+=16; } + +#define WILDCOPY_OVERLENGTH 8 +#define VECLEN 16 + +typedef enum { + ZSTD_no_overlap, + ZSTD_overlap_src_before_dst, + /* ZSTD_overlap_dst_before_src, */ +} ZSTD_overlap_e; /*! ZSTD_wildcopy() : * custom version of memcpy(), can overwrite up to WILDCOPY_OVERLENGTH bytes (if length==0) */ -#define WILDCOPY_OVERLENGTH 8 -MEM_STATIC void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length) +MEM_STATIC FORCE_INLINE_ATTR DONT_VECTORIZE +void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e ovtype) { + ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src; const BYTE* ip = (const BYTE*)src; BYTE* op = (BYTE*)dst; BYTE* const oend = op + length; - do - COPY8(op, ip) - while (op < oend); + + assert(diff >= 8 || (ovtype == ZSTD_no_overlap && diff < -8)); + if (length < VECLEN || (ovtype == ZSTD_overlap_src_before_dst && diff < VECLEN)) { + do + COPY8(op, ip) + while (op < oend); + } + else { + if ((length & 8) == 0) + COPY8(op, ip); + do { + COPY16(op, ip); + } + while (op < oend); + } +} + +/*! ZSTD_wildcopy_16min() : + * same semantics as ZSTD_wilcopy() except guaranteed to be able to copy 16 bytes at the start */ +MEM_STATIC FORCE_INLINE_ATTR DONT_VECTORIZE +void ZSTD_wildcopy_16min(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e ovtype) +{ + ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src; + const BYTE* ip = (const BYTE*)src; + BYTE* op = (BYTE*)dst; + BYTE* const oend = op + length; + + assert(length >= 8); + assert(diff >= 8 || (ovtype == ZSTD_no_overlap && diff < -8)); + + if (ovtype == ZSTD_overlap_src_before_dst && diff < VECLEN) { + do + COPY8(op, ip) + while (op < oend); + } + else { + if ((length & 8) == 0) + COPY8(op, ip); + do { + COPY16(op, ip); + } + while (op < oend); + } } MEM_STATIC void ZSTD_wildcopy_e(void* dst, const void* src, void* dstEnd) /* should be faster for decoding, but strangely, not verified on all platform */ diff --git a/thirdparty/zstd/compress/zstd_compress.c b/thirdparty/zstd/compress/zstd_compress.c index 2e163c8bf3..1476512580 100644 --- a/thirdparty/zstd/compress/zstd_compress.c +++ b/thirdparty/zstd/compress/zstd_compress.c @@ -385,6 +385,11 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) bounds.upperBound = ZSTD_lcm_uncompressed; return bounds; + case ZSTD_c_targetCBlockSize: + bounds.lowerBound = ZSTD_TARGETCBLOCKSIZE_MIN; + bounds.upperBound = ZSTD_TARGETCBLOCKSIZE_MAX; + return bounds; + default: { ZSTD_bounds const boundError = { ERROR(parameter_unsupported), 0, 0 }; return boundError; @@ -452,6 +457,7 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param) case ZSTD_c_ldmHashRateLog: case ZSTD_c_forceAttachDict: case ZSTD_c_literalCompressionMode: + case ZSTD_c_targetCBlockSize: default: return 0; } @@ -497,6 +503,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value) case ZSTD_c_ldmHashLog: case ZSTD_c_ldmMinMatch: case ZSTD_c_ldmBucketSizeLog: + case ZSTD_c_targetCBlockSize: break; default: RETURN_ERROR(parameter_unsupported); @@ -671,6 +678,12 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, CCtxParams->ldmParams.hashRateLog = value; return CCtxParams->ldmParams.hashRateLog; + case ZSTD_c_targetCBlockSize : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_targetCBlockSize, value); + CCtxParams->targetCBlockSize = value; + return CCtxParams->targetCBlockSize; + default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); } } @@ -692,13 +705,13 @@ size_t ZSTD_CCtxParams_getParameter( *value = CCtxParams->compressionLevel; break; case ZSTD_c_windowLog : - *value = CCtxParams->cParams.windowLog; + *value = (int)CCtxParams->cParams.windowLog; break; case ZSTD_c_hashLog : - *value = CCtxParams->cParams.hashLog; + *value = (int)CCtxParams->cParams.hashLog; break; case ZSTD_c_chainLog : - *value = CCtxParams->cParams.chainLog; + *value = (int)CCtxParams->cParams.chainLog; break; case ZSTD_c_searchLog : *value = CCtxParams->cParams.searchLog; @@ -773,6 +786,9 @@ size_t ZSTD_CCtxParams_getParameter( case ZSTD_c_ldmHashRateLog : *value = CCtxParams->ldmParams.hashRateLog; break; + case ZSTD_c_targetCBlockSize : + *value = (int)CCtxParams->targetCBlockSize; + break; default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); } return 0; @@ -930,12 +946,12 @@ size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset) @return : 0, or an error code if one value is beyond authorized range */ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) { - BOUNDCHECK(ZSTD_c_windowLog, cParams.windowLog); - BOUNDCHECK(ZSTD_c_chainLog, cParams.chainLog); - BOUNDCHECK(ZSTD_c_hashLog, cParams.hashLog); - BOUNDCHECK(ZSTD_c_searchLog, cParams.searchLog); - BOUNDCHECK(ZSTD_c_minMatch, cParams.minMatch); - BOUNDCHECK(ZSTD_c_targetLength,cParams.targetLength); + BOUNDCHECK(ZSTD_c_windowLog, (int)cParams.windowLog); + BOUNDCHECK(ZSTD_c_chainLog, (int)cParams.chainLog); + BOUNDCHECK(ZSTD_c_hashLog, (int)cParams.hashLog); + BOUNDCHECK(ZSTD_c_searchLog, (int)cParams.searchLog); + BOUNDCHECK(ZSTD_c_minMatch, (int)cParams.minMatch); + BOUNDCHECK(ZSTD_c_targetLength,(int)cParams.targetLength); BOUNDCHECK(ZSTD_c_strategy, cParams.strategy); return 0; } @@ -951,7 +967,7 @@ ZSTD_clampCParams(ZSTD_compressionParameters cParams) if ((int)val<bounds.lowerBound) val=(type)bounds.lowerBound; \ else if ((int)val>bounds.upperBound) val=(type)bounds.upperBound; \ } -# define CLAMP(cParam, val) CLAMP_TYPE(cParam, val, int) +# define CLAMP(cParam, val) CLAMP_TYPE(cParam, val, unsigned) CLAMP(ZSTD_c_windowLog, cParams.windowLog); CLAMP(ZSTD_c_chainLog, cParams.chainLog); CLAMP(ZSTD_c_hashLog, cParams.hashLog); @@ -1282,15 +1298,14 @@ static void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs) } /*! ZSTD_invalidateMatchState() - * Invalidate all the matches in the match finder tables. - * Requires nextSrc and base to be set (can be NULL). + * Invalidate all the matches in the match finder tables. + * Requires nextSrc and base to be set (can be NULL). */ static void ZSTD_invalidateMatchState(ZSTD_matchState_t* ms) { ZSTD_window_clear(&ms->window); ms->nextToUpdate = ms->window.dictLimit; - ms->nextToUpdate3 = ms->window.dictLimit; ms->loadedDictEnd = 0; ms->opt.litLengthSum = 0; /* force reset of btopt stats */ ms->dictMatchState = NULL; @@ -1327,15 +1342,17 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_CCtx_params params, U64 pl typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset } ZSTD_compResetPolicy_e; +typedef enum { ZSTD_resetTarget_CDict, ZSTD_resetTarget_CCtx } ZSTD_resetTarget_e; + static void* ZSTD_reset_matchState(ZSTD_matchState_t* ms, void* ptr, const ZSTD_compressionParameters* cParams, - ZSTD_compResetPolicy_e const crp, U32 const forCCtx) + ZSTD_compResetPolicy_e const crp, ZSTD_resetTarget_e const forWho) { size_t const chainSize = (cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cParams->chainLog); size_t const hSize = ((size_t)1) << cParams->hashLog; - U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; + U32 const hashLog3 = ((forWho == ZSTD_resetTarget_CCtx) && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; size_t const h3Size = ((size_t)1) << hashLog3; size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); @@ -1349,7 +1366,7 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, ZSTD_invalidateMatchState(ms); /* opt parser space */ - if (forCCtx && (cParams->strategy >= ZSTD_btopt)) { + if ((forWho == ZSTD_resetTarget_CCtx) && (cParams->strategy >= ZSTD_btopt)) { DEBUGLOG(4, "reserving optimal parser space"); ms->opt.litFreq = (unsigned*)ptr; ms->opt.litLengthFreq = ms->opt.litFreq + (1<<Litbits); @@ -1377,6 +1394,19 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, return ptr; } +/* ZSTD_indexTooCloseToMax() : + * minor optimization : prefer memset() rather than reduceIndex() + * which is measurably slow in some circumstances (reported for Visual Studio). + * Works when re-using a context for a lot of smallish inputs : + * if all inputs are smaller than ZSTD_INDEXOVERFLOW_MARGIN, + * memset() will be triggered before reduceIndex(). + */ +#define ZSTD_INDEXOVERFLOW_MARGIN (16 MB) +static int ZSTD_indexTooCloseToMax(ZSTD_window_t w) +{ + return (size_t)(w.nextSrc - w.base) > (ZSTD_CURRENT_MAX - ZSTD_INDEXOVERFLOW_MARGIN); +} + #define ZSTD_WORKSPACETOOLARGE_FACTOR 3 /* define "workspace is too large" as this number of times larger than needed */ #define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 /* when workspace is continuously too large * during at least this number of times, @@ -1388,7 +1418,7 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, note : `params` are assumed fully validated at this stage */ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ZSTD_CCtx_params params, - U64 pledgedSrcSize, + U64 const pledgedSrcSize, ZSTD_compResetPolicy_e const crp, ZSTD_buffered_policy_e const zbuff) { @@ -1400,13 +1430,21 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, if (ZSTD_equivalentParams(zc->appliedParams, params, zc->inBuffSize, zc->seqStore.maxNbSeq, zc->seqStore.maxNbLit, - zbuff, pledgedSrcSize)) { - DEBUGLOG(4, "ZSTD_equivalentParams()==1 -> continue mode (wLog1=%u, blockSize1=%zu)", - zc->appliedParams.cParams.windowLog, zc->blockSize); + zbuff, pledgedSrcSize) ) { + DEBUGLOG(4, "ZSTD_equivalentParams()==1 -> consider continue mode"); zc->workSpaceOversizedDuration += (zc->workSpaceOversizedDuration > 0); /* if it was too large, it still is */ - if (zc->workSpaceOversizedDuration <= ZSTD_WORKSPACETOOLARGE_MAXDURATION) + if (zc->workSpaceOversizedDuration <= ZSTD_WORKSPACETOOLARGE_MAXDURATION) { + DEBUGLOG(4, "continue mode confirmed (wLog1=%u, blockSize1=%zu)", + zc->appliedParams.cParams.windowLog, zc->blockSize); + if (ZSTD_indexTooCloseToMax(zc->blockState.matchState.window)) { + /* prefer a reset, faster than a rescale */ + ZSTD_reset_matchState(&zc->blockState.matchState, + zc->entropyWorkspace + HUF_WORKSPACE_SIZE_U32, + ¶ms.cParams, + crp, ZSTD_resetTarget_CCtx); + } return ZSTD_continueCCtx(zc, params, pledgedSrcSize); - } } + } } } DEBUGLOG(4, "ZSTD_equivalentParams()==0 -> reset CCtx"); if (params.ldmParams.enableLdm) { @@ -1449,7 +1487,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, DEBUGLOG(4, "windowSize: %zu - blockSize: %zu", windowSize, blockSize); if (workSpaceTooSmall || workSpaceWasteful) { - DEBUGLOG(4, "Need to resize workSpaceSize from %zuKB to %zuKB", + DEBUGLOG(4, "Resize workSpaceSize from %zuKB to %zuKB", zc->workSpaceSize >> 10, neededSpace >> 10); @@ -1491,7 +1529,10 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ZSTD_reset_compressedBlockState(zc->blockState.prevCBlock); - ptr = zc->entropyWorkspace + HUF_WORKSPACE_SIZE_U32; + ptr = ZSTD_reset_matchState(&zc->blockState.matchState, + zc->entropyWorkspace + HUF_WORKSPACE_SIZE_U32, + ¶ms.cParams, + crp, ZSTD_resetTarget_CCtx); /* ldm hash table */ /* initialize bucketOffsets table later for pointer alignment */ @@ -1509,8 +1550,6 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, } assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */ - ptr = ZSTD_reset_matchState(&zc->blockState.matchState, ptr, ¶ms.cParams, crp, /* forCCtx */ 1); - /* sequences storage */ zc->seqStore.maxNbSeq = maxNbSeq; zc->seqStore.sequencesStart = (seqDef*)ptr; @@ -1587,15 +1626,14 @@ static int ZSTD_shouldAttachDict(const ZSTD_CDict* cdict, * handled in _enforceMaxDist */ } -static size_t ZSTD_resetCCtx_byAttachingCDict( - ZSTD_CCtx* cctx, - const ZSTD_CDict* cdict, - ZSTD_CCtx_params params, - U64 pledgedSrcSize, - ZSTD_buffered_policy_e zbuff) +static size_t +ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx, + const ZSTD_CDict* cdict, + ZSTD_CCtx_params params, + U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) { - { - const ZSTD_compressionParameters *cdict_cParams = &cdict->matchState.cParams; + { const ZSTD_compressionParameters* const cdict_cParams = &cdict->matchState.cParams; unsigned const windowLog = params.cParams.windowLog; assert(windowLog != 0); /* Resize working context table params for input only, since the dict @@ -1607,8 +1645,7 @@ static size_t ZSTD_resetCCtx_byAttachingCDict( assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy); } - { - const U32 cdictEnd = (U32)( cdict->matchState.window.nextSrc + { const U32 cdictEnd = (U32)( cdict->matchState.window.nextSrc - cdict->matchState.window.base); const U32 cdictLen = cdictEnd - cdict->matchState.window.dictLimit; if (cdictLen == 0) { @@ -1625,9 +1662,9 @@ static size_t ZSTD_resetCCtx_byAttachingCDict( cctx->blockState.matchState.window.base + cdictEnd; ZSTD_window_clear(&cctx->blockState.matchState.window); } + /* loadedDictEnd is expressed within the referential of the active context */ cctx->blockState.matchState.loadedDictEnd = cctx->blockState.matchState.window.dictLimit; - } - } + } } cctx->dictID = cdict->dictID; @@ -1681,7 +1718,6 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, ZSTD_matchState_t* dstMatchState = &cctx->blockState.matchState; dstMatchState->window = srcMatchState->window; dstMatchState->nextToUpdate = srcMatchState->nextToUpdate; - dstMatchState->nextToUpdate3= srcMatchState->nextToUpdate3; dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd; } @@ -1761,7 +1797,6 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, ZSTD_matchState_t* dstMatchState = &dstCCtx->blockState.matchState; dstMatchState->window = srcMatchState->window; dstMatchState->nextToUpdate = srcMatchState->nextToUpdate; - dstMatchState->nextToUpdate3= srcMatchState->nextToUpdate3; dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd; } dstCCtx->dictID = srcCCtx->dictID; @@ -1831,16 +1866,15 @@ static void ZSTD_reduceTable_btlazy2(U32* const table, U32 const size, U32 const /*! ZSTD_reduceIndex() : * rescale all indexes to avoid future overflow (indexes are U32) */ -static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue) +static void ZSTD_reduceIndex (ZSTD_matchState_t* ms, ZSTD_CCtx_params const* params, const U32 reducerValue) { - ZSTD_matchState_t* const ms = &zc->blockState.matchState; - { U32 const hSize = (U32)1 << zc->appliedParams.cParams.hashLog; + { U32 const hSize = (U32)1 << params->cParams.hashLog; ZSTD_reduceTable(ms->hashTable, hSize, reducerValue); } - if (zc->appliedParams.cParams.strategy != ZSTD_fast) { - U32 const chainSize = (U32)1 << zc->appliedParams.cParams.chainLog; - if (zc->appliedParams.cParams.strategy == ZSTD_btlazy2) + if (params->cParams.strategy != ZSTD_fast) { + U32 const chainSize = (U32)1 << params->cParams.chainLog; + if (params->cParams.strategy == ZSTD_btlazy2) ZSTD_reduceTable_btlazy2(ms->chainTable, chainSize, reducerValue); else ZSTD_reduceTable(ms->chainTable, chainSize, reducerValue); @@ -2524,6 +2558,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2; else op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3; + assert(op <= oend); if (nbSeq==0) { /* Copy the old tables over as if we repeated them */ memcpy(&nextEntropy->fse, &prevEntropy->fse, sizeof(prevEntropy->fse)); @@ -2532,6 +2567,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, /* seqHead : flags for FSE encoding type */ seqHead = op++; + assert(op <= oend); /* convert length/distances into codes */ ZSTD_seqToCodes(seqStorePtr); @@ -2555,6 +2591,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, if (LLtype == set_compressed) lastNCount = op; op += countSize; + assert(op <= oend); } } /* build CTable for Offsets */ { unsigned max = MaxOff; @@ -2577,6 +2614,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, if (Offtype == set_compressed) lastNCount = op; op += countSize; + assert(op <= oend); } } /* build CTable for MatchLengths */ { unsigned max = MaxML; @@ -2597,6 +2635,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, if (MLtype == set_compressed) lastNCount = op; op += countSize; + assert(op <= oend); } } *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2)); @@ -2610,6 +2649,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, longOffsets, bmi2); FORWARD_IF_ERROR(bitstreamSize); op += bitstreamSize; + assert(op <= oend); /* zstd versions <= 1.3.4 mistakenly report corruption when * FSE_readNCount() receives a buffer < 4 bytes. * Fixed by https://github.com/facebook/zstd/pull/1146. @@ -2721,30 +2761,24 @@ void ZSTD_resetSeqStore(seqStore_t* ssPtr) ssPtr->longLengthID = 0; } -static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize) +typedef enum { ZSTDbss_compress, ZSTDbss_noCompress } ZSTD_buildSeqStore_e; + +static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) { ZSTD_matchState_t* const ms = &zc->blockState.matchState; - size_t cSize; - DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)", - (unsigned)dstCapacity, (unsigned)ms->window.dictLimit, (unsigned)ms->nextToUpdate); + DEBUGLOG(5, "ZSTD_buildSeqStore (srcSize=%zu)", srcSize); assert(srcSize <= ZSTD_BLOCKSIZE_MAX); - /* Assert that we have correctly flushed the ctx params into the ms's copy */ ZSTD_assertEqualCParams(zc->appliedParams.cParams, ms->cParams); - if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) { ZSTD_ldm_skipSequences(&zc->externSeqStore, srcSize, zc->appliedParams.cParams.minMatch); - cSize = 0; - goto out; /* don't even attempt compression below a certain srcSize */ + return ZSTDbss_noCompress; /* don't even attempt compression below a certain srcSize */ } ZSTD_resetSeqStore(&(zc->seqStore)); /* required for optimal parser to read stats from dictionary */ ms->opt.symbolCosts = &zc->blockState.prevCBlock->entropy; /* tell the optimal parser how we expect to compress literals */ ms->opt.literalCompressionMode = zc->appliedParams.literalCompressionMode; - /* a gap between an attached dict and the current window is not safe, * they must remain adjacent, * and when that stops being the case, the dict must be unset */ @@ -2798,6 +2832,21 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, { const BYTE* const lastLiterals = (const BYTE*)src + srcSize - lastLLSize; ZSTD_storeLastLiterals(&zc->seqStore, lastLiterals, lastLLSize); } } + return ZSTDbss_compress; +} + +static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + size_t cSize; + DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)", + (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, (unsigned)zc->blockState.matchState.nextToUpdate); + + { const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); + FORWARD_IF_ERROR(bss); + if (bss == ZSTDbss_noCompress) { cSize = 0; goto out; } + } /* encode sequences and literals */ cSize = ZSTD_compressSequences(&zc->seqStore, @@ -2826,6 +2875,25 @@ out: } +static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, ZSTD_CCtx_params const* params, void const* ip, void const* iend) +{ + if (ZSTD_window_needOverflowCorrection(ms->window, iend)) { + U32 const maxDist = (U32)1 << params->cParams.windowLog; + U32 const cycleLog = ZSTD_cycleLog(params->cParams.chainLog, params->cParams.strategy); + U32 const correction = ZSTD_window_correctOverflow(&ms->window, cycleLog, maxDist, ip); + ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30); + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30); + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); + ZSTD_reduceIndex(ms, params, correction); + if (ms->nextToUpdate < correction) ms->nextToUpdate = 0; + else ms->nextToUpdate -= correction; + /* invalidate dictionaries on overflow correction */ + ms->loadedDictEnd = 0; + ms->dictMatchState = NULL; + } +} + + /*! ZSTD_compress_frameChunk() : * Compress a chunk of data into one or multiple blocks. * All blocks will be terminated, all input will be consumed. @@ -2844,7 +2912,7 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog; - assert(cctx->appliedParams.cParams.windowLog <= 31); + assert(cctx->appliedParams.cParams.windowLog <= ZSTD_WINDOWLOG_MAX); DEBUGLOG(5, "ZSTD_compress_frameChunk (blockSize=%u)", (unsigned)blockSize); if (cctx->appliedParams.fParams.checksumFlag && srcSize) @@ -2859,19 +2927,10 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, "not enough space to store compressed block"); if (remaining < blockSize) blockSize = remaining; - if (ZSTD_window_needOverflowCorrection(ms->window, ip + blockSize)) { - U32 const cycleLog = ZSTD_cycleLog(cctx->appliedParams.cParams.chainLog, cctx->appliedParams.cParams.strategy); - U32 const correction = ZSTD_window_correctOverflow(&ms->window, cycleLog, maxDist, ip); - ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30); - ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30); - ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); - ZSTD_reduceIndex(cctx, correction); - if (ms->nextToUpdate < correction) ms->nextToUpdate = 0; - else ms->nextToUpdate -= correction; - ms->loadedDictEnd = 0; - ms->dictMatchState = NULL; - } - ZSTD_window_enforceMaxDist(&ms->window, ip + blockSize, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); + ZSTD_overflowCorrectIfNeeded(ms, &cctx->appliedParams, ip, ip + blockSize); + ZSTD_checkDictValidity(&ms->window, ip + blockSize, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); + + /* Ensure hash/chain table insertion resumes no sooner than lowlimit */ if (ms->nextToUpdate < ms->window.lowLimit) ms->nextToUpdate = ms->window.lowLimit; { size_t cSize = ZSTD_compressBlock_internal(cctx, @@ -2899,7 +2958,7 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, } } if (lastFrameChunk && (op>ostart)) cctx->stage = ZSTDcs_ending; - return op-ostart; + return (size_t)(op-ostart); } @@ -2991,6 +3050,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->appliedParams, cctx->pledgedSrcSizePlusOne-1, cctx->dictID); FORWARD_IF_ERROR(fhSize); + assert(fhSize <= dstCapacity); dstCapacity -= fhSize; dst = (char*)dst + fhSize; cctx->stage = ZSTDcs_ongoing; @@ -3007,18 +3067,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, if (!frame) { /* overflow check and correction for block mode */ - if (ZSTD_window_needOverflowCorrection(ms->window, (const char*)src + srcSize)) { - U32 const cycleLog = ZSTD_cycleLog(cctx->appliedParams.cParams.chainLog, cctx->appliedParams.cParams.strategy); - U32 const correction = ZSTD_window_correctOverflow(&ms->window, cycleLog, 1 << cctx->appliedParams.cParams.windowLog, src); - ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30); - ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30); - ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); - ZSTD_reduceIndex(cctx, correction); - if (ms->nextToUpdate < correction) ms->nextToUpdate = 0; - else ms->nextToUpdate -= correction; - ms->loadedDictEnd = 0; - ms->dictMatchState = NULL; - } + ZSTD_overflowCorrectIfNeeded(ms, &cctx->appliedParams, src, (BYTE const*)src + srcSize); } DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (unsigned)cctx->blockSize); @@ -3074,7 +3123,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, const void* src, size_t srcSize, ZSTD_dictTableLoadMethod_e dtlm) { - const BYTE* const ip = (const BYTE*) src; + const BYTE* ip = (const BYTE*) src; const BYTE* const iend = ip + srcSize; ZSTD_window_update(&ms->window, src, srcSize); @@ -3085,32 +3134,42 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, if (srcSize <= HASH_READ_SIZE) return 0; - switch(params->cParams.strategy) - { - case ZSTD_fast: - ZSTD_fillHashTable(ms, iend, dtlm); - break; - case ZSTD_dfast: - ZSTD_fillDoubleHashTable(ms, iend, dtlm); - break; + while (iend - ip > HASH_READ_SIZE) { + size_t const remaining = iend - ip; + size_t const chunk = MIN(remaining, ZSTD_CHUNKSIZE_MAX); + const BYTE* const ichunk = ip + chunk; - case ZSTD_greedy: - case ZSTD_lazy: - case ZSTD_lazy2: - if (srcSize >= HASH_READ_SIZE) - ZSTD_insertAndFindFirstIndex(ms, iend-HASH_READ_SIZE); - break; + ZSTD_overflowCorrectIfNeeded(ms, params, ip, ichunk); - case ZSTD_btlazy2: /* we want the dictionary table fully sorted */ - case ZSTD_btopt: - case ZSTD_btultra: - case ZSTD_btultra2: - if (srcSize >= HASH_READ_SIZE) - ZSTD_updateTree(ms, iend-HASH_READ_SIZE, iend); - break; + switch(params->cParams.strategy) + { + case ZSTD_fast: + ZSTD_fillHashTable(ms, ichunk, dtlm); + break; + case ZSTD_dfast: + ZSTD_fillDoubleHashTable(ms, ichunk, dtlm); + break; - default: - assert(0); /* not possible : not a valid strategy id */ + case ZSTD_greedy: + case ZSTD_lazy: + case ZSTD_lazy2: + if (chunk >= HASH_READ_SIZE) + ZSTD_insertAndFindFirstIndex(ms, ichunk-HASH_READ_SIZE); + break; + + case ZSTD_btlazy2: /* we want the dictionary table fully sorted */ + case ZSTD_btopt: + case ZSTD_btultra: + case ZSTD_btultra2: + if (chunk >= HASH_READ_SIZE) + ZSTD_updateTree(ms, ichunk-HASH_READ_SIZE, ichunk); + break; + + default: + assert(0); /* not possible : not a valid strategy id */ + } + + ip = ichunk; } ms->nextToUpdate = (U32)(iend - ms->window.base); @@ -3297,12 +3356,11 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, ZSTDcrp_continue, zbuff) ); - { - size_t const dictID = ZSTD_compress_insertDictionary( + { size_t const dictID = ZSTD_compress_insertDictionary( cctx->blockState.prevCBlock, &cctx->blockState.matchState, ¶ms, dict, dictSize, dictContentType, dtlm, cctx->entropyWorkspace); FORWARD_IF_ERROR(dictID); - assert(dictID <= (size_t)(U32)-1); + assert(dictID <= UINT_MAX); cctx->dictID = (U32)dictID; } return 0; @@ -3555,10 +3613,10 @@ static size_t ZSTD_initCDict_internal( /* Reset the state to no dictionary */ ZSTD_reset_compressedBlockState(&cdict->cBlockState); - { void* const end = ZSTD_reset_matchState( - &cdict->matchState, - (U32*)cdict->workspace + HUF_WORKSPACE_SIZE_U32, - &cParams, ZSTDcrp_continue, /* forCCtx */ 0); + { void* const end = ZSTD_reset_matchState(&cdict->matchState, + (U32*)cdict->workspace + HUF_WORKSPACE_SIZE_U32, + &cParams, + ZSTDcrp_continue, ZSTD_resetTarget_CDict); assert(end == (char*)cdict->workspace + cdict->workspaceSize); (void)end; } @@ -4068,7 +4126,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, case zcss_flush: DEBUGLOG(5, "flush stage"); { size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; - size_t const flushed = ZSTD_limitCopy(op, oend-op, + size_t const flushed = ZSTD_limitCopy(op, (size_t)(oend-op), zcs->outBuff + zcs->outBuffFlushedSize, toFlush); DEBUGLOG(5, "toFlush: %u into %u ==> flushed: %u", (unsigned)toFlush, (unsigned)(oend-op), (unsigned)flushed); @@ -4262,7 +4320,7 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) if (zcs->appliedParams.nbWorkers > 0) return remainingToFlush; /* minimal estimation */ /* single thread mode : attempt to calculate remaining to flush more precisely */ { size_t const lastBlockSize = zcs->frameEnded ? 0 : ZSTD_BLOCKHEADERSIZE; - size_t const checksumSize = zcs->frameEnded ? 0 : zcs->appliedParams.fParams.checksumFlag * 4; + size_t const checksumSize = (size_t)(zcs->frameEnded ? 0 : zcs->appliedParams.fParams.checksumFlag * 4); size_t const toFlush = remainingToFlush + lastBlockSize + checksumSize; DEBUGLOG(4, "ZSTD_endStream : remaining to flush : %u", (unsigned)toFlush); return toFlush; diff --git a/thirdparty/zstd/compress/zstd_compress_internal.h b/thirdparty/zstd/compress/zstd_compress_internal.h index cc3cbb9da9..5495899be3 100644 --- a/thirdparty/zstd/compress/zstd_compress_internal.h +++ b/thirdparty/zstd/compress/zstd_compress_internal.h @@ -33,13 +33,13 @@ extern "C" { ***************************************/ #define kSearchStrength 8 #define HASH_READ_SIZE 8 -#define ZSTD_DUBT_UNSORTED_MARK 1 /* For btlazy2 strategy, index 1 now means "unsorted". +#define ZSTD_DUBT_UNSORTED_MARK 1 /* For btlazy2 strategy, index ZSTD_DUBT_UNSORTED_MARK==1 means "unsorted". It could be confused for a real successor at index "1", if sorted as larger than its predecessor. It's not a big deal though : candidate will just be sorted again. Additionally, candidate position 1 will be lost. But candidate 1 cannot hide a large tree of candidates, so it's a minimal loss. - The benefit is that ZSTD_DUBT_UNSORTED_MARK cannot be mishandled after table re-use with a different strategy - Constant required by ZSTD_compressBlock_btlazy2() and ZSTD_reduceTable_internal() */ + The benefit is that ZSTD_DUBT_UNSORTED_MARK cannot be mishandled after table re-use with a different strategy. + This constant is required by ZSTD_compressBlock_btlazy2() and ZSTD_reduceTable_internal() */ /*-************************************* @@ -128,21 +128,20 @@ typedef struct { BYTE const* base; /* All regular indexes relative to this position */ BYTE const* dictBase; /* extDict indexes relative to this position */ U32 dictLimit; /* below that point, need extDict */ - U32 lowLimit; /* below that point, no more data */ + U32 lowLimit; /* below that point, no more valid data */ } ZSTD_window_t; typedef struct ZSTD_matchState_t ZSTD_matchState_t; struct ZSTD_matchState_t { ZSTD_window_t window; /* State for window round buffer management */ - U32 loadedDictEnd; /* index of end of dictionary */ + U32 loadedDictEnd; /* index of end of dictionary, within context's referential. When dict referential is copied into active context (i.e. not attached), effectively same value as dictSize, since referential starts from zero */ U32 nextToUpdate; /* index from which to continue table update */ - U32 nextToUpdate3; /* index from which to continue table update */ U32 hashLog3; /* dispatch table : larger == faster, more memory */ U32* hashTable; U32* hashTable3; U32* chainTable; optState_t opt; /* optimal parser state */ - const ZSTD_matchState_t * dictMatchState; + const ZSTD_matchState_t* dictMatchState; ZSTD_compressionParameters cParams; }; @@ -195,6 +194,9 @@ struct ZSTD_CCtx_params_s { int compressionLevel; int forceWindow; /* force back-references to respect limit of * 1<<wLog, even for dictionary */ + size_t targetCBlockSize; /* Tries to fit compressed block size to be around targetCBlockSize. + * No target when targetCBlockSize == 0. + * There is no guarantee on compressed block size */ ZSTD_dictAttachPref_e attachDictPref; ZSTD_literalCompressionMode_e literalCompressionMode; @@ -324,7 +326,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v /* copy Literals */ assert(seqStorePtr->maxNbLit <= 128 KB); assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + seqStorePtr->maxNbLit); - ZSTD_wildcopy(seqStorePtr->lit, literals, litLength); + ZSTD_wildcopy(seqStorePtr->lit, literals, litLength, ZSTD_no_overlap); seqStorePtr->lit += litLength; /* literal Length */ @@ -564,6 +566,9 @@ MEM_STATIC U64 ZSTD_rollingHash_rotate(U64 hash, BYTE toRemove, BYTE toAdd, U64 /*-************************************* * Round buffer management ***************************************/ +#if (ZSTD_WINDOWLOG_MAX_64 > 31) +# error "ZSTD_WINDOWLOG_MAX is too large : would overflow ZSTD_CURRENT_MAX" +#endif /* Max current allowed */ #define ZSTD_CURRENT_MAX ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX)) /* Maximum chunk size before overflow correction needs to be called again */ @@ -675,31 +680,49 @@ MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog, * Updates lowLimit so that: * (srcEnd - base) - lowLimit == maxDist + loadedDictEnd * - * This allows a simple check that index >= lowLimit to see if index is valid. - * This must be called before a block compression call, with srcEnd as the block - * source end. + * It ensures index is valid as long as index >= lowLimit. + * This must be called before a block compression call. + * + * loadedDictEnd is only defined if a dictionary is in use for current compression. + * As the name implies, loadedDictEnd represents the index at end of dictionary. + * The value lies within context's referential, it can be directly compared to blockEndIdx. * - * If loadedDictEndPtr is not NULL, we set it to zero once we update lowLimit. - * This is because dictionaries are allowed to be referenced as long as the last - * byte of the dictionary is in the window, but once they are out of range, - * they cannot be referenced. If loadedDictEndPtr is NULL, we use - * loadedDictEnd == 0. + * If loadedDictEndPtr is NULL, no dictionary is in use, and we use loadedDictEnd == 0. + * If loadedDictEndPtr is not NULL, we set it to zero after updating lowLimit. + * This is because dictionaries are allowed to be referenced fully + * as long as the last byte of the dictionary is in the window. + * Once input has progressed beyond window size, dictionary cannot be referenced anymore. * - * In normal dict mode, the dict is between lowLimit and dictLimit. In - * dictMatchState mode, lowLimit and dictLimit are the same, and the dictionary - * is below them. forceWindow and dictMatchState are therefore incompatible. + * In normal dict mode, the dictionary lies between lowLimit and dictLimit. + * In dictMatchState mode, lowLimit and dictLimit are the same, + * and the dictionary is below them. + * forceWindow and dictMatchState are therefore incompatible. */ MEM_STATIC void ZSTD_window_enforceMaxDist(ZSTD_window_t* window, - void const* srcEnd, - U32 maxDist, - U32* loadedDictEndPtr, + const void* blockEnd, + U32 maxDist, + U32* loadedDictEndPtr, const ZSTD_matchState_t** dictMatchStatePtr) { - U32 const blockEndIdx = (U32)((BYTE const*)srcEnd - window->base); - U32 loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0; - DEBUGLOG(5, "ZSTD_window_enforceMaxDist: blockEndIdx=%u, maxDist=%u", - (unsigned)blockEndIdx, (unsigned)maxDist); + U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base); + U32 const loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0; + DEBUGLOG(5, "ZSTD_window_enforceMaxDist: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u", + (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd); + + /* - When there is no dictionary : loadedDictEnd == 0. + In which case, the test (blockEndIdx > maxDist) is merely to avoid + overflowing next operation `newLowLimit = blockEndIdx - maxDist`. + - When there is a standard dictionary : + Index referential is copied from the dictionary, + which means it starts from 0. + In which case, loadedDictEnd == dictSize, + and it makes sense to compare `blockEndIdx > maxDist + dictSize` + since `blockEndIdx` also starts from zero. + - When there is an attached dictionary : + loadedDictEnd is expressed within the referential of the context, + so it can be directly compared against blockEndIdx. + */ if (blockEndIdx > maxDist + loadedDictEnd) { U32 const newLowLimit = blockEndIdx - maxDist; if (window->lowLimit < newLowLimit) window->lowLimit = newLowLimit; @@ -708,10 +731,31 @@ ZSTD_window_enforceMaxDist(ZSTD_window_t* window, (unsigned)window->dictLimit, (unsigned)window->lowLimit); window->dictLimit = window->lowLimit; } - if (loadedDictEndPtr) - *loadedDictEndPtr = 0; - if (dictMatchStatePtr) - *dictMatchStatePtr = NULL; + /* On reaching window size, dictionaries are invalidated */ + if (loadedDictEndPtr) *loadedDictEndPtr = 0; + if (dictMatchStatePtr) *dictMatchStatePtr = NULL; + } +} + +/* Similar to ZSTD_window_enforceMaxDist(), + * but only invalidates dictionary + * when input progresses beyond window size. */ +MEM_STATIC void +ZSTD_checkDictValidity(ZSTD_window_t* window, + const void* blockEnd, + U32 maxDist, + U32* loadedDictEndPtr, + const ZSTD_matchState_t** dictMatchStatePtr) +{ + U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base); + U32 const loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0; + DEBUGLOG(5, "ZSTD_checkDictValidity: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u", + (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd); + + if (loadedDictEnd && (blockEndIdx > maxDist + loadedDictEnd)) { + /* On reaching window size, dictionaries are invalidated */ + if (loadedDictEndPtr) *loadedDictEndPtr = 0; + if (dictMatchStatePtr) *dictMatchStatePtr = NULL; } } diff --git a/thirdparty/zstd/compress/zstd_double_fast.c b/thirdparty/zstd/compress/zstd_double_fast.c index 47faf6d641..5957255d90 100644 --- a/thirdparty/zstd/compress/zstd_double_fast.c +++ b/thirdparty/zstd/compress/zstd_double_fast.c @@ -43,8 +43,7 @@ void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms, /* Only load extra positions for ZSTD_dtlm_full */ if (dtlm == ZSTD_dtlm_fast) break; - } - } + } } } @@ -63,7 +62,10 @@ size_t ZSTD_compressBlock_doubleFast_generic( const BYTE* const istart = (const BYTE*)src; const BYTE* ip = istart; const BYTE* anchor = istart; - const U32 prefixLowestIndex = ms->window.dictLimit; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + const U32 lowestValid = ms->window.dictLimit; + const U32 maxDistance = 1U << cParams->windowLog; + const U32 prefixLowestIndex = (endIndex - lowestValid > maxDistance) ? endIndex - maxDistance : lowestValid; const BYTE* const prefixLowest = base + prefixLowestIndex; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - HASH_READ_SIZE; @@ -95,8 +97,15 @@ size_t ZSTD_compressBlock_doubleFast_generic( dictCParams->chainLog : hBitsS; const U32 dictAndPrefixLength = (U32)(ip - prefixLowest + dictEnd - dictStart); + DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_generic"); + assert(dictMode == ZSTD_noDict || dictMode == ZSTD_dictMatchState); + /* if a dictionary is attached, it must be within window range */ + if (dictMode == ZSTD_dictMatchState) { + assert(lowestValid + maxDistance >= endIndex); + } + /* init */ ip += (dictAndPrefixLength == 0); if (dictMode == ZSTD_noDict) { @@ -138,7 +147,7 @@ size_t ZSTD_compressBlock_doubleFast_generic( const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; ip++; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH); goto _match_stored; } @@ -147,7 +156,7 @@ size_t ZSTD_compressBlock_doubleFast_generic( && ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)))) { mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; ip++; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH); goto _match_stored; } @@ -170,8 +179,7 @@ size_t ZSTD_compressBlock_doubleFast_generic( offset = (U32)(current - dictMatchIndexL - dictIndexDelta); while (((ip>anchor) & (dictMatchL>dictStart)) && (ip[-1] == dictMatchL[-1])) { ip--; dictMatchL--; mLength++; } /* catch up */ goto _match_found; - } - } + } } if (matchIndexS > prefixLowestIndex) { /* check prefix short match */ @@ -186,16 +194,14 @@ size_t ZSTD_compressBlock_doubleFast_generic( if (match > dictStart && MEM_read32(match) == MEM_read32(ip)) { goto _search_next_long; - } - } + } } ip += ((ip-anchor) >> kSearchStrength) + 1; continue; _search_next_long: - { - size_t const hl3 = ZSTD_hashPtr(ip+1, hBitsL, 8); + { size_t const hl3 = ZSTD_hashPtr(ip+1, hBitsL, 8); size_t const dictHLNext = ZSTD_hashPtr(ip+1, dictHBitsL, 8); U32 const matchIndexL3 = hashLong[hl3]; const BYTE* matchL3 = base + matchIndexL3; @@ -221,9 +227,7 @@ _search_next_long: offset = (U32)(current + 1 - dictMatchIndexL3 - dictIndexDelta); while (((ip>anchor) & (dictMatchL3>dictStart)) && (ip[-1] == dictMatchL3[-1])) { ip--; dictMatchL3--; mLength++; } /* catch up */ goto _match_found; - } - } - } + } } } /* if no long +1 match, explore the short match we found */ if (dictMode == ZSTD_dictMatchState && matchIndexS < prefixLowestIndex) { @@ -242,7 +246,7 @@ _match_found: offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); _match_stored: /* match found */ @@ -250,11 +254,14 @@ _match_stored: anchor = ip; if (ip <= ilimit) { - /* Fill Table */ - hashLong[ZSTD_hashPtr(base+current+2, hBitsL, 8)] = - hashSmall[ZSTD_hashPtr(base+current+2, hBitsS, mls)] = current+2; /* here because current+2 could be > iend-8 */ - hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = - hashSmall[ZSTD_hashPtr(ip-2, hBitsS, mls)] = (U32)(ip-2-base); + /* Complementary insertion */ + /* done after iLimit test, as candidates could be > iend-8 */ + { U32 const indexToInsert = current+2; + hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert; + hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); + hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert; + hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base); + } /* check immediate repcode */ if (dictMode == ZSTD_dictMatchState) { @@ -278,8 +285,7 @@ _match_stored: continue; } break; - } - } + } } if (dictMode == ZSTD_noDict) { while ( (ip <= ilimit) @@ -294,14 +300,15 @@ _match_stored: ip += rLength; anchor = ip; continue; /* faster when present ... (?) */ - } } } } + } } } + } /* while (ip < ilimit) */ /* save reps for next block */ rep[0] = offset_1 ? offset_1 : offsetSaved; rep[1] = offset_2 ? offset_2 : offsetSaved; /* Return the last literals size */ - return iend - anchor; + return (size_t)(iend - anchor); } @@ -360,10 +367,15 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( const BYTE* anchor = istart; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - 8; - const U32 prefixStartIndex = ms->window.dictLimit; const BYTE* const base = ms->window.base; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + const U32 maxDistance = 1U << cParams->windowLog; + const U32 lowestValid = ms->window.lowLimit; + const U32 lowLimit = (endIndex - lowestValid > maxDistance) ? endIndex - maxDistance : lowestValid; + const U32 dictStartIndex = lowLimit; + const U32 dictLimit = ms->window.dictLimit; + const U32 prefixStartIndex = (dictLimit > lowLimit) ? dictLimit : lowLimit; const BYTE* const prefixStart = base + prefixStartIndex; - const U32 dictStartIndex = ms->window.lowLimit; const BYTE* const dictBase = ms->window.dictBase; const BYTE* const dictStart = dictBase + dictStartIndex; const BYTE* const dictEnd = dictBase + prefixStartIndex; @@ -371,6 +383,10 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_extDict_generic (srcSize=%zu)", srcSize); + /* if extDict is invalidated due to maxDistance, switch to "regular" variant */ + if (prefixStartIndex == dictStartIndex) + return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, mls, ZSTD_noDict); + /* Search Loop */ while (ip < ilimit) { /* < instead of <=, because (ip+1) */ const size_t hSmall = ZSTD_hashPtr(ip, hBitsS, mls); @@ -396,7 +412,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; ip++; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH); } else { if ((matchLongIndex > dictStartIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) { const BYTE* const matchEnd = matchLongIndex < prefixStartIndex ? dictEnd : iend; @@ -407,7 +423,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); } else if ((matchIndex > dictStartIndex) && (MEM_read32(match) == MEM_read32(ip))) { size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8); @@ -432,23 +448,27 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( } offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); } else { ip += ((ip-anchor) >> kSearchStrength) + 1; continue; } } - /* found a match : store it */ + /* move to next sequence start */ ip += mLength; anchor = ip; if (ip <= ilimit) { - /* Fill Table */ - hashSmall[ZSTD_hashPtr(base+current+2, hBitsS, mls)] = current+2; - hashLong[ZSTD_hashPtr(base+current+2, hBitsL, 8)] = current+2; - hashSmall[ZSTD_hashPtr(ip-2, hBitsS, mls)] = (U32)(ip-2-base); - hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); + /* Complementary insertion */ + /* done after iLimit test, as candidates could be > iend-8 */ + { U32 const indexToInsert = current+2; + hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert; + hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); + hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert; + hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base); + } + /* check immediate repcode */ while (ip <= ilimit) { U32 const current2 = (U32)(ip-base); @@ -475,7 +495,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( rep[1] = offset_2; /* Return the last literals size */ - return iend - anchor; + return (size_t)(iend - anchor); } diff --git a/thirdparty/zstd/compress/zstd_fast.c b/thirdparty/zstd/compress/zstd_fast.c index ed997b441c..a05b8a47f1 100644 --- a/thirdparty/zstd/compress/zstd_fast.c +++ b/thirdparty/zstd/compress/zstd_fast.c @@ -13,7 +13,8 @@ void ZSTD_fillHashTable(ZSTD_matchState_t* ms, - void const* end, ZSTD_dictTableLoadMethod_e dtlm) + const void* const end, + ZSTD_dictTableLoadMethod_e dtlm) { const ZSTD_compressionParameters* const cParams = &ms->cParams; U32* const hashTable = ms->hashTable; @@ -41,6 +42,7 @@ void ZSTD_fillHashTable(ZSTD_matchState_t* ms, } } } } } + FORCE_INLINE_TEMPLATE size_t ZSTD_compressBlock_fast_generic( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], @@ -58,7 +60,10 @@ size_t ZSTD_compressBlock_fast_generic( const BYTE* ip0 = istart; const BYTE* ip1; const BYTE* anchor = istart; - const U32 prefixStartIndex = ms->window.dictLimit; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + const U32 maxDistance = 1U << cParams->windowLog; + const U32 validStartIndex = ms->window.dictLimit; + const U32 prefixStartIndex = (endIndex - validStartIndex > maxDistance) ? endIndex - maxDistance : validStartIndex; const BYTE* const prefixStart = base + prefixStartIndex; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - HASH_READ_SIZE; @@ -165,7 +170,7 @@ _match: /* Requires: ip0, match0, offcode */ rep[1] = offset_2 ? offset_2 : offsetSaved; /* Return the last literals size */ - return iend - anchor; + return (size_t)(iend - anchor); } @@ -222,8 +227,15 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( const U32 dictAndPrefixLength = (U32)(ip - prefixStart + dictEnd - dictStart); const U32 dictHLog = dictCParams->hashLog; - /* otherwise, we would get index underflow when translating a dict index - * into a local index */ + /* if a dictionary is still attached, it necessarily means that + * it is within window size. So we just check it. */ + const U32 maxDistance = 1U << cParams->windowLog; + const U32 endIndex = (U32)((size_t)(ip - base) + srcSize); + assert(endIndex - prefixStartIndex <= maxDistance); + (void)maxDistance; (void)endIndex; /* these variables are not used when assert() is disabled */ + + /* ensure there will be no no underflow + * when translating a dict index into a local index */ assert(prefixStartIndex >= (U32)(dictEnd - dictBase)); /* init */ @@ -251,7 +263,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; ip++; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH); } else if ( (matchIndex <= prefixStartIndex) ) { size_t const dictHash = ZSTD_hashPtr(ip, dictHLog, mls); U32 const dictMatchIndex = dictHashTable[dictHash]; @@ -271,7 +283,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( } /* catch up */ offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); } } else if (MEM_read32(match) != MEM_read32(ip)) { /* it's not a match, and we're not going to check the dictionary */ @@ -286,7 +298,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); } /* match found */ @@ -327,7 +339,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( rep[1] = offset_2 ? offset_2 : offsetSaved; /* Return the last literals size */ - return iend - anchor; + return (size_t)(iend - anchor); } size_t ZSTD_compressBlock_fast_dictMatchState( @@ -366,15 +378,24 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( const BYTE* const istart = (const BYTE*)src; const BYTE* ip = istart; const BYTE* anchor = istart; - const U32 dictStartIndex = ms->window.lowLimit; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + const U32 maxDistance = 1U << cParams->windowLog; + const U32 validLow = ms->window.lowLimit; + const U32 lowLimit = (endIndex - validLow > maxDistance) ? endIndex - maxDistance : validLow; + const U32 dictStartIndex = lowLimit; const BYTE* const dictStart = dictBase + dictStartIndex; - const U32 prefixStartIndex = ms->window.dictLimit; + const U32 dictLimit = ms->window.dictLimit; + const U32 prefixStartIndex = dictLimit < lowLimit ? lowLimit : dictLimit; const BYTE* const prefixStart = base + prefixStartIndex; const BYTE* const dictEnd = dictBase + prefixStartIndex; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - 8; U32 offset_1=rep[0], offset_2=rep[1]; + /* switch to "regular" variant if extDict is invalidated due to maxDistance */ + if (prefixStartIndex == dictStartIndex) + return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, mls); + /* Search Loop */ while (ip < ilimit) { /* < instead of <=, because (ip+1) */ const size_t h = ZSTD_hashPtr(ip, hlog, mls); @@ -394,7 +415,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; ip++; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH); } else { if ( (matchIndex < dictStartIndex) || (MEM_read32(match) != MEM_read32(ip)) ) { @@ -410,7 +431,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( offset = current - matchIndex; offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); } } /* found a match : store it */ @@ -445,7 +466,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( rep[1] = offset_2; /* Return the last literals size */ - return iend - anchor; + return (size_t)(iend - anchor); } diff --git a/thirdparty/zstd/compress/zstd_lazy.c b/thirdparty/zstd/compress/zstd_lazy.c index 53f998a437..94d906c01f 100644 --- a/thirdparty/zstd/compress/zstd_lazy.c +++ b/thirdparty/zstd/compress/zstd_lazy.c @@ -83,7 +83,10 @@ ZSTD_insertDUBT1(ZSTD_matchState_t* ms, U32* largerPtr = smallerPtr + 1; U32 matchIndex = *smallerPtr; /* this candidate is unsorted : next sorted candidate is reached through *smallerPtr, while *largerPtr contains previous unsorted candidate (which is already saved and can be overwritten) */ U32 dummy32; /* to be nullified at the end */ - U32 const windowLow = ms->window.lowLimit; + U32 const windowValid = ms->window.lowLimit; + U32 const maxDistance = 1U << cParams->windowLog; + U32 const windowLow = (current - windowValid > maxDistance) ? current - maxDistance : windowValid; + DEBUGLOG(8, "ZSTD_insertDUBT1(%u) (dictLimit=%u, lowLimit=%u)", current, dictLimit, windowLow); @@ -239,7 +242,9 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms, const BYTE* const base = ms->window.base; U32 const current = (U32)(ip-base); - U32 const windowLow = ms->window.lowLimit; + U32 const maxDistance = 1U << cParams->windowLog; + U32 const windowValid = ms->window.lowLimit; + U32 const windowLow = (current - windowValid > maxDistance) ? current - maxDistance : windowValid; U32* const bt = ms->chainTable; U32 const btLog = cParams->chainLog - 1; @@ -490,8 +495,10 @@ size_t ZSTD_HcFindBestMatch_generic ( const U32 dictLimit = ms->window.dictLimit; const BYTE* const prefixStart = base + dictLimit; const BYTE* const dictEnd = dictBase + dictLimit; - const U32 lowLimit = ms->window.lowLimit; const U32 current = (U32)(ip-base); + const U32 maxDistance = 1U << cParams->windowLog; + const U32 lowValid = ms->window.lowLimit; + const U32 lowLimit = (current - lowValid > maxDistance) ? current - maxDistance : lowValid; const U32 minChain = current > chainSize ? current - chainSize : 0; U32 nbAttempts = 1U << cParams->searchLog; size_t ml=4-1; @@ -653,7 +660,6 @@ size_t ZSTD_compressBlock_lazy_generic( /* init */ ip += (dictAndPrefixLength == 0); - ms->nextToUpdate3 = ms->nextToUpdate; if (dictMode == ZSTD_noDict) { U32 const maxRep = (U32)(ip - prefixLowest); if (offset_2 > maxRep) savedOffset = offset_2, offset_2 = 0; @@ -933,7 +939,6 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( U32 offset_1 = rep[0], offset_2 = rep[1]; /* init */ - ms->nextToUpdate3 = ms->nextToUpdate; ip += (ip == prefixStart); /* Match Loop */ diff --git a/thirdparty/zstd/compress/zstd_ldm.c b/thirdparty/zstd/compress/zstd_ldm.c index 784d20f3ab..3dcf86e6e8 100644 --- a/thirdparty/zstd/compress/zstd_ldm.c +++ b/thirdparty/zstd/compress/zstd_ldm.c @@ -447,7 +447,7 @@ size_t ZSTD_ldm_generateSequences( if (ZSTD_window_needOverflowCorrection(ldmState->window, chunkEnd)) { U32 const ldmHSize = 1U << params->hashLog; U32 const correction = ZSTD_window_correctOverflow( - &ldmState->window, /* cycleLog */ 0, maxDist, src); + &ldmState->window, /* cycleLog */ 0, maxDist, chunkStart); ZSTD_ldm_reduceTable(ldmState->hashTable, ldmHSize, correction); } /* 2. We enforce the maximum offset allowed. diff --git a/thirdparty/zstd/compress/zstd_opt.c b/thirdparty/zstd/compress/zstd_opt.c index efb69d3267..e32e542e02 100644 --- a/thirdparty/zstd/compress/zstd_opt.c +++ b/thirdparty/zstd/compress/zstd_opt.c @@ -255,13 +255,13 @@ static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optP * to provide a cost which is directly comparable to a match ending at same position */ static int ZSTD_litLengthContribution(U32 const litLength, const optState_t* const optPtr, int optLevel) { - if (optPtr->priceType >= zop_predef) return WEIGHT(litLength, optLevel); + if (optPtr->priceType >= zop_predef) return (int)WEIGHT(litLength, optLevel); /* dynamic statistics */ { U32 const llCode = ZSTD_LLcode(litLength); - int const contribution = (LL_bits[llCode] * BITCOST_MULTIPLIER) - + WEIGHT(optPtr->litLengthFreq[0], optLevel) /* note: log2litLengthSum cancel out */ - - WEIGHT(optPtr->litLengthFreq[llCode], optLevel); + int const contribution = (int)(LL_bits[llCode] * BITCOST_MULTIPLIER) + + (int)WEIGHT(optPtr->litLengthFreq[0], optLevel) /* note: log2litLengthSum cancel out */ + - (int)WEIGHT(optPtr->litLengthFreq[llCode], optLevel); #if 1 return contribution; #else @@ -278,7 +278,7 @@ static int ZSTD_literalsContribution(const BYTE* const literals, U32 const litLe const optState_t* const optPtr, int optLevel) { - int const contribution = ZSTD_rawLiteralsCost(literals, litLength, optPtr, optLevel) + int const contribution = (int)ZSTD_rawLiteralsCost(literals, litLength, optPtr, optLevel) + ZSTD_litLengthContribution(litLength, optPtr, optLevel); return contribution; } @@ -372,13 +372,15 @@ MEM_STATIC U32 ZSTD_readMINMATCH(const void* memPtr, U32 length) /* Update hashTable3 up to ip (excluded) Assumption : always within prefix (i.e. not within extDict) */ -static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_matchState_t* ms, const BYTE* const ip) +static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_matchState_t* ms, + U32* nextToUpdate3, + const BYTE* const ip) { U32* const hashTable3 = ms->hashTable3; U32 const hashLog3 = ms->hashLog3; const BYTE* const base = ms->window.base; - U32 idx = ms->nextToUpdate3; - U32 const target = ms->nextToUpdate3 = (U32)(ip - base); + U32 idx = *nextToUpdate3; + U32 const target = (U32)(ip - base); size_t const hash3 = ZSTD_hash3Ptr(ip, hashLog3); assert(hashLog3 > 0); @@ -387,6 +389,7 @@ static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_matchState_t* ms, const BYTE* idx++; } + *nextToUpdate3 = target; return hashTable3[hash3]; } @@ -503,9 +506,11 @@ static U32 ZSTD_insertBt1( } } *smallerPtr = *largerPtr = 0; - if (bestLength > 384) return MIN(192, (U32)(bestLength - 384)); /* speed optimization */ - assert(matchEndIdx > current + 8); - return matchEndIdx - (current + 8); + { U32 positions = 0; + if (bestLength > 384) positions = MIN(192, (U32)(bestLength - 384)); /* speed optimization */ + assert(matchEndIdx > current + 8); + return MAX(positions, matchEndIdx - (current + 8)); + } } FORCE_INLINE_TEMPLATE @@ -520,8 +525,13 @@ void ZSTD_updateTree_internal( DEBUGLOG(6, "ZSTD_updateTree_internal, from %u to %u (dictMode:%u)", idx, target, dictMode); - while(idx < target) - idx += ZSTD_insertBt1(ms, base+idx, iend, mls, dictMode == ZSTD_extDict); + while(idx < target) { + U32 const forward = ZSTD_insertBt1(ms, base+idx, iend, mls, dictMode == ZSTD_extDict); + assert(idx < (U32)(idx + forward)); + idx += forward; + } + assert((size_t)(ip - base) <= (size_t)(U32)(-1)); + assert((size_t)(iend - base) <= (size_t)(U32)(-1)); ms->nextToUpdate = target; } @@ -531,16 +541,18 @@ void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend) { FORCE_INLINE_TEMPLATE U32 ZSTD_insertBtAndGetAllMatches ( + ZSTD_match_t* matches, /* store result (found matches) in this table (presumed large enough) */ ZSTD_matchState_t* ms, + U32* nextToUpdate3, const BYTE* const ip, const BYTE* const iLimit, const ZSTD_dictMode_e dictMode, - U32 rep[ZSTD_REP_NUM], + const U32 rep[ZSTD_REP_NUM], U32 const ll0, /* tells if associated literal length is 0 or not. This value must be 0 or 1 */ - ZSTD_match_t* matches, const U32 lengthToBeat, U32 const mls /* template */) { const ZSTD_compressionParameters* const cParams = &ms->cParams; U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); + U32 const maxDistance = 1U << cParams->windowLog; const BYTE* const base = ms->window.base; U32 const current = (U32)(ip-base); U32 const hashLog = cParams->hashLog; @@ -556,8 +568,9 @@ U32 ZSTD_insertBtAndGetAllMatches ( U32 const dictLimit = ms->window.dictLimit; const BYTE* const dictEnd = dictBase + dictLimit; const BYTE* const prefixStart = base + dictLimit; - U32 const btLow = btMask >= current ? 0 : current - btMask; - U32 const windowLow = ms->window.lowLimit; + U32 const btLow = (btMask >= current) ? 0 : current - btMask; + U32 const windowValid = ms->window.lowLimit; + U32 const windowLow = ((current - windowValid) > maxDistance) ? current - maxDistance : windowValid; U32 const matchLow = windowLow ? windowLow : 1; U32* smallerPtr = bt + 2*(current&btMask); U32* largerPtr = bt + 2*(current&btMask) + 1; @@ -627,7 +640,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( /* HC3 match finder */ if ((mls == 3) /*static*/ && (bestLength < mls)) { - U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(ms, ip); + U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(ms, nextToUpdate3, ip); if ((matchIndex3 >= matchLow) & (current - matchIndex3 < (1<<18)) /*heuristic : longer distance likely too expensive*/ ) { size_t mlen; @@ -653,9 +666,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( (ip+mlen == iLimit) ) { /* best possible length */ ms->nextToUpdate = current+1; /* skip insertion */ return 1; - } - } - } + } } } /* no dictMatchState lookup: dicts don't have a populated HC3 table */ } @@ -760,10 +771,13 @@ U32 ZSTD_insertBtAndGetAllMatches ( FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches ( + ZSTD_match_t* matches, /* store result (match found, increasing size) in this table */ ZSTD_matchState_t* ms, + U32* nextToUpdate3, const BYTE* ip, const BYTE* const iHighLimit, const ZSTD_dictMode_e dictMode, - U32 rep[ZSTD_REP_NUM], U32 const ll0, - ZSTD_match_t* matches, U32 const lengthToBeat) + const U32 rep[ZSTD_REP_NUM], + U32 const ll0, + U32 const lengthToBeat) { const ZSTD_compressionParameters* const cParams = &ms->cParams; U32 const matchLengthSearch = cParams->minMatch; @@ -772,12 +786,12 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches ( ZSTD_updateTree_internal(ms, ip, iHighLimit, matchLengthSearch, dictMode); switch(matchLengthSearch) { - case 3 : return ZSTD_insertBtAndGetAllMatches(ms, ip, iHighLimit, dictMode, rep, ll0, matches, lengthToBeat, 3); + case 3 : return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, 3); default : - case 4 : return ZSTD_insertBtAndGetAllMatches(ms, ip, iHighLimit, dictMode, rep, ll0, matches, lengthToBeat, 4); - case 5 : return ZSTD_insertBtAndGetAllMatches(ms, ip, iHighLimit, dictMode, rep, ll0, matches, lengthToBeat, 5); + case 4 : return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, 4); + case 5 : return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, 5); case 7 : - case 6 : return ZSTD_insertBtAndGetAllMatches(ms, ip, iHighLimit, dictMode, rep, ll0, matches, lengthToBeat, 6); + case 6 : return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, 6); } } @@ -853,6 +867,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); U32 const minMatch = (cParams->minMatch == 3) ? 3 : 4; + U32 nextToUpdate3 = ms->nextToUpdate; ZSTD_optimal_t* const opt = optStatePtr->priceTable; ZSTD_match_t* const matches = optStatePtr->matchTable; @@ -862,7 +877,6 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u", (U32)(ip - base), ms->window.dictLimit, ms->nextToUpdate); assert(optLevel <= 2); - ms->nextToUpdate3 = ms->nextToUpdate; ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize, optLevel); ip += (ip==prefixStart); @@ -873,7 +887,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, /* find first match */ { U32 const litlen = (U32)(ip - anchor); U32 const ll0 = !litlen; - U32 const nbMatches = ZSTD_BtGetAllMatches(ms, ip, iend, dictMode, rep, ll0, matches, minMatch); + U32 const nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, ip, iend, dictMode, rep, ll0, minMatch); if (!nbMatches) { ip++; continue; } /* initialize opt[0] */ @@ -970,7 +984,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 const litlen = (opt[cur].mlen == 0) ? opt[cur].litlen : 0; U32 const previousPrice = opt[cur].price; U32 const basePrice = previousPrice + ZSTD_litLengthPrice(0, optStatePtr, optLevel); - U32 const nbMatches = ZSTD_BtGetAllMatches(ms, inr, iend, dictMode, opt[cur].rep, ll0, matches, minMatch); + U32 const nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, inr, iend, dictMode, opt[cur].rep, ll0, minMatch); U32 matchNb; if (!nbMatches) { DEBUGLOG(7, "rPos:%u : no match found", cur); @@ -1094,7 +1108,7 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ } /* while (ip < ilimit) */ /* Return the last literals size */ - return iend - anchor; + return (size_t)(iend - anchor); } @@ -1158,7 +1172,6 @@ ZSTD_initStats_ultra(ZSTD_matchState_t* ms, ms->window.dictLimit += (U32)srcSize; ms->window.lowLimit = ms->window.dictLimit; ms->nextToUpdate = ms->window.dictLimit; - ms->nextToUpdate3 = ms->window.dictLimit; /* re-inforce weight of collected statistics */ ZSTD_upscaleStats(&ms->opt); diff --git a/thirdparty/zstd/compress/zstdmt_compress.c b/thirdparty/zstd/compress/zstdmt_compress.c index 38fbb90768..9e537b8848 100644 --- a/thirdparty/zstd/compress/zstdmt_compress.c +++ b/thirdparty/zstd/compress/zstdmt_compress.c @@ -1129,9 +1129,14 @@ size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx) size_t const produced = ZSTD_isError(cResult) ? 0 : cResult; size_t const flushed = ZSTD_isError(cResult) ? 0 : jobPtr->dstFlushed; assert(flushed <= produced); + assert(jobPtr->consumed <= jobPtr->src.size); toFlush = produced - flushed; - if (toFlush==0 && (jobPtr->consumed >= jobPtr->src.size)) { - /* doneJobID is not-fully-flushed, but toFlush==0 : doneJobID should be compressing some more data */ + /* if toFlush==0, nothing is available to flush. + * However, jobID is expected to still be active: + * if jobID was already completed and fully flushed, + * ZSTDMT_flushProduced() should have already moved onto next job. + * Therefore, some input has not yet been consumed. */ + if (toFlush==0) { assert(jobPtr->consumed < jobPtr->src.size); } } @@ -1148,12 +1153,16 @@ size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx) static unsigned ZSTDMT_computeTargetJobLog(ZSTD_CCtx_params const params) { - if (params.ldmParams.enableLdm) + unsigned jobLog; + if (params.ldmParams.enableLdm) { /* In Long Range Mode, the windowLog is typically oversized. * In which case, it's preferable to determine the jobSize * based on chainLog instead. */ - return MAX(21, params.cParams.chainLog + 4); - return MAX(20, params.cParams.windowLog + 2); + jobLog = MAX(21, params.cParams.chainLog + 4); + } else { + jobLog = MAX(20, params.cParams.windowLog + 2); + } + return MIN(jobLog, (unsigned)ZSTDMT_JOBLOG_MAX); } static int ZSTDMT_overlapLog_default(ZSTD_strategy strat) @@ -1197,7 +1206,7 @@ static size_t ZSTDMT_computeOverlapSize(ZSTD_CCtx_params const params) ovLog = MIN(params.cParams.windowLog, ZSTDMT_computeTargetJobLog(params) - 2) - overlapRLog; } - assert(0 <= ovLog && ovLog <= 30); + assert(0 <= ovLog && ovLog <= ZSTD_WINDOWLOG_MAX); DEBUGLOG(4, "overlapLog : %i", params.overlapLog); DEBUGLOG(4, "overlap size : %i", 1 << ovLog); return (ovLog==0) ? 0 : (size_t)1 << ovLog; @@ -1391,7 +1400,7 @@ size_t ZSTDMT_initCStream_internal( FORWARD_IF_ERROR( ZSTDMT_resize(mtctx, params.nbWorkers) ); if (params.jobSize != 0 && params.jobSize < ZSTDMT_JOBSIZE_MIN) params.jobSize = ZSTDMT_JOBSIZE_MIN; - if (params.jobSize > (size_t)ZSTDMT_JOBSIZE_MAX) params.jobSize = ZSTDMT_JOBSIZE_MAX; + if (params.jobSize > (size_t)ZSTDMT_JOBSIZE_MAX) params.jobSize = (size_t)ZSTDMT_JOBSIZE_MAX; mtctx->singleBlockingThread = (pledgedSrcSize <= ZSTDMT_JOBSIZE_MIN); /* do not trigger multi-threading when srcSize is too small */ if (mtctx->singleBlockingThread) { @@ -1432,6 +1441,8 @@ size_t ZSTDMT_initCStream_internal( if (mtctx->targetSectionSize == 0) { mtctx->targetSectionSize = 1ULL << ZSTDMT_computeTargetJobLog(params); } + assert(mtctx->targetSectionSize <= (size_t)ZSTDMT_JOBSIZE_MAX); + if (params.rsyncable) { /* Aim for the targetsectionSize as the average job size. */ U32 const jobSizeMB = (U32)(mtctx->targetSectionSize >> 20); diff --git a/thirdparty/zstd/compress/zstdmt_compress.h b/thirdparty/zstd/compress/zstdmt_compress.h index 12e6bcb3a3..12a526087d 100644 --- a/thirdparty/zstd/compress/zstdmt_compress.h +++ b/thirdparty/zstd/compress/zstdmt_compress.h @@ -50,6 +50,7 @@ #ifndef ZSTDMT_JOBSIZE_MIN # define ZSTDMT_JOBSIZE_MIN (1 MB) #endif +#define ZSTDMT_JOBLOG_MAX (MEM_32bits() ? 29 : 30) #define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (1024 MB)) diff --git a/thirdparty/zstd/decompress/zstd_decompress.c b/thirdparty/zstd/decompress/zstd_decompress.c index 675596f5aa..e42872ad96 100644 --- a/thirdparty/zstd/decompress/zstd_decompress.c +++ b/thirdparty/zstd/decompress/zstd_decompress.c @@ -360,8 +360,11 @@ static size_t readSkippableFrameSize(void const* src, size_t srcSize) sizeU32 = MEM_readLE32((BYTE const*)src + ZSTD_FRAMEIDSIZE); RETURN_ERROR_IF((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32, frameParameter_unsupported); - - return skippableHeaderSize + sizeU32; + { + size_t const skippableSize = skippableHeaderSize + sizeU32; + RETURN_ERROR_IF(skippableSize > srcSize, srcSize_wrong); + return skippableSize; + } } /** ZSTD_findDecompressedSize() : @@ -378,11 +381,10 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { size_t const skippableSize = readSkippableFrameSize(src, srcSize); - if (ZSTD_isError(skippableSize)) - return skippableSize; - if (srcSize < skippableSize) { + if (ZSTD_isError(skippableSize)) { return ZSTD_CONTENTSIZE_ERROR; } + assert(skippableSize <= srcSize); src = (const BYTE *)src + skippableSize; srcSize -= skippableSize; @@ -467,6 +469,8 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize if ((srcSize >= ZSTD_SKIPPABLEHEADERSIZE) && (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { frameSizeInfo.compressedSize = readSkippableFrameSize(src, srcSize); + assert(ZSTD_isError(frameSizeInfo.compressedSize) || + frameSizeInfo.compressedSize <= srcSize); return frameSizeInfo; } else { const BYTE* ip = (const BYTE*)src; @@ -529,7 +533,6 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) return frameSizeInfo.compressedSize; } - /** ZSTD_decompressBound() : * compatible with legacy mode * `src` must point to the start of a ZSTD frame or a skippeable frame @@ -546,6 +549,7 @@ unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize) unsigned long long const decompressedBound = frameSizeInfo.decompressedBound; if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR) return ZSTD_CONTENTSIZE_ERROR; + assert(srcSize >= compressedSize); src = (const BYTE*)src + compressedSize; srcSize -= compressedSize; bound += decompressedBound; @@ -738,9 +742,8 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, (unsigned)magicNumber, ZSTD_MAGICNUMBER); if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { size_t const skippableSize = readSkippableFrameSize(src, srcSize); - if (ZSTD_isError(skippableSize)) - return skippableSize; - RETURN_ERROR_IF(srcSize < skippableSize, srcSize_wrong); + FORWARD_IF_ERROR(skippableSize); + assert(skippableSize <= srcSize); src = (const BYTE *)src + skippableSize; srcSize -= skippableSize; diff --git a/thirdparty/zstd/decompress/zstd_decompress_block.c b/thirdparty/zstd/decompress/zstd_decompress_block.c index a2a7eedcf2..24f4859c56 100644 --- a/thirdparty/zstd/decompress/zstd_decompress_block.c +++ b/thirdparty/zstd/decompress/zstd_decompress_block.c @@ -505,7 +505,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, *nbSeqPtr = nbSeq; /* FSE table descriptors */ - RETURN_ERROR_IF(ip+4 > iend, srcSize_wrong); /* minimum possible size */ + RETURN_ERROR_IF(ip+1 > iend, srcSize_wrong); /* minimum possible size: 1 byte for symbol encoding types */ { symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6); symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3); symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3); @@ -637,9 +637,10 @@ size_t ZSTD_execSequence(BYTE* op, if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); /* copy Literals */ - ZSTD_copy8(op, *litPtr); if (sequence.litLength > 8) - ZSTD_wildcopy(op+8, (*litPtr)+8, sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ + ZSTD_wildcopy_16min(op, (*litPtr), sequence.litLength, ZSTD_no_overlap); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ + else + ZSTD_copy8(op, *litPtr); op = oLitEnd; *litPtr = iLitEnd; /* update for next sequence */ @@ -686,13 +687,13 @@ size_t ZSTD_execSequence(BYTE* op, if (oMatchEnd > oend-(16-MINMATCH)) { if (op < oend_w) { - ZSTD_wildcopy(op, match, oend_w - op); + ZSTD_wildcopy(op, match, oend_w - op, ZSTD_overlap_src_before_dst); match += oend_w - op; op = oend_w; } while (op < oMatchEnd) *op++ = *match++; } else { - ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */ + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst); /* works even if matchLength < 8 */ } return sequenceLength; } @@ -717,9 +718,11 @@ size_t ZSTD_execSequenceLong(BYTE* op, if (oLitEnd > oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, prefixStart, dictStart, dictEnd); /* copy Literals */ - ZSTD_copy8(op, *litPtr); /* note : op <= oLitEnd <= oend_w == oend - 8 */ if (sequence.litLength > 8) - ZSTD_wildcopy(op+8, (*litPtr)+8, sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ + ZSTD_wildcopy_16min(op, *litPtr, sequence.litLength, ZSTD_no_overlap); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ + else + ZSTD_copy8(op, *litPtr); /* note : op <= oLitEnd <= oend_w == oend - 8 */ + op = oLitEnd; *litPtr = iLitEnd; /* update for next sequence */ @@ -766,13 +769,13 @@ size_t ZSTD_execSequenceLong(BYTE* op, if (oMatchEnd > oend-(16-MINMATCH)) { if (op < oend_w) { - ZSTD_wildcopy(op, match, oend_w - op); + ZSTD_wildcopy(op, match, oend_w - op, ZSTD_overlap_src_before_dst); match += oend_w - op; op = oend_w; } while (op < oMatchEnd) *op++ = *match++; } else { - ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */ + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst); /* works even if matchLength < 8 */ } return sequenceLength; } @@ -889,6 +892,7 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets) } FORCE_INLINE_TEMPLATE size_t +DONT_VECTORIZE ZSTD_decompressSequences_body( ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, @@ -918,6 +922,11 @@ ZSTD_decompressSequences_body( ZSTD_DCtx* dctx, ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + ZSTD_STATIC_ASSERT( + BIT_DStream_unfinished < BIT_DStream_completed && + BIT_DStream_endOfBuffer < BIT_DStream_completed && + BIT_DStream_completed < BIT_DStream_overflow); + for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) { nbSeq--; { seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset); @@ -930,6 +939,7 @@ ZSTD_decompressSequences_body( ZSTD_DCtx* dctx, /* check if reached exact end */ DEBUGLOG(5, "ZSTD_decompressSequences_body: after decode loop, remaining nbSeq : %i", nbSeq); RETURN_ERROR_IF(nbSeq, corruption_detected); + RETURN_ERROR_IF(BIT_reloadDStream(&seqState.DStream) < BIT_DStream_completed, corruption_detected); /* save reps for next block */ { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); } } @@ -1131,6 +1141,7 @@ ZSTD_decompressSequencesLong_default(ZSTD_DCtx* dctx, #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG static TARGET_ATTRIBUTE("bmi2") size_t +DONT_VECTORIZE ZSTD_decompressSequences_bmi2(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, diff --git a/thirdparty/zstd/zstd.h b/thirdparty/zstd/zstd.h index 53470c18f3..a1910ee223 100644 --- a/thirdparty/zstd/zstd.h +++ b/thirdparty/zstd/zstd.h @@ -71,7 +71,7 @@ extern "C" { /*------ Version ------*/ #define ZSTD_VERSION_MAJOR 1 #define ZSTD_VERSION_MINOR 4 -#define ZSTD_VERSION_RELEASE 0 +#define ZSTD_VERSION_RELEASE 1 #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< to check runtime library version */ @@ -82,16 +82,16 @@ ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< to check runtime library v #define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) ZSTDLIB_API const char* ZSTD_versionString(void); /* requires v1.3.0+ */ -/*************************************** -* Default constant -***************************************/ +/* ************************************* + * Default constant + ***************************************/ #ifndef ZSTD_CLEVEL_DEFAULT # define ZSTD_CLEVEL_DEFAULT 3 #endif -/*************************************** -* Constants -***************************************/ +/* ************************************* + * Constants + ***************************************/ /* All magic numbers are supposed read/written to/from files/memory using little-endian convention */ #define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */ @@ -183,9 +183,14 @@ ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compres ***************************************/ /*= Compression context * When compressing many times, - * it is recommended to allocate a context just once, and re-use it for each successive compression operation. + * it is recommended to allocate a context just once, + * and re-use it for each successive compression operation. * This will make workload friendlier for system's memory. - * Use one context per thread for parallel execution in multi-threaded environments. */ + * Note : re-using context is just a speed / resource optimization. + * It doesn't change the compression ratio, which remains identical. + * Note 2 : In multi-threaded environments, + * use one different context per thread for parallel execution. + */ typedef struct ZSTD_CCtx_s ZSTD_CCtx; ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); @@ -380,6 +385,7 @@ typedef enum { * ZSTD_c_forceMaxWindow * ZSTD_c_forceAttachDict * ZSTD_c_literalCompressionMode + * ZSTD_c_targetCBlockSize * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. * note : never ever use experimentalParam? names directly; * also, the enums values themselves are unstable and can still change. @@ -389,6 +395,7 @@ typedef enum { ZSTD_c_experimentalParam3=1000, ZSTD_c_experimentalParam4=1001, ZSTD_c_experimentalParam5=1002, + ZSTD_c_experimentalParam6=1003, } ZSTD_cParameter; typedef struct { @@ -657,17 +664,33 @@ ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, ZSTD_inBuffer* input, ZSTD_EndDirective endOp); + +/* These buffer sizes are softly recommended. + * They are not required : ZSTD_compressStream*() happily accepts any buffer size, for both input and output. + * Respecting the recommended size just makes it a bit easier for ZSTD_compressStream*(), + * reducing the amount of memory shuffling and buffering, resulting in minor performance savings. + * + * However, note that these recommendations are from the perspective of a C caller program. + * If the streaming interface is invoked from some other language, + * especially managed ones such as Java or Go, through a foreign function interface such as jni or cgo, + * a major performance rule is to reduce crossing such interface to an absolute minimum. + * It's not rare that performance ends being spent more into the interface, rather than compression itself. + * In which cases, prefer using large buffers, as large as practical, + * for both input and output, to reduce the nb of roundtrips. + */ ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ -ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */ +ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */ -/******************************************************************************* - * This is a legacy streaming API, and can be replaced by ZSTD_CCtx_reset() and - * ZSTD_compressStream2(). It is redundant, but is still fully supported. + +/* ***************************************************************************** + * This following is a legacy streaming API. + * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2(). + * It is redundant, but remains fully supported. * Advanced parameters and dictionary compression can only be used through the * new API. ******************************************************************************/ -/** +/*! * Equivalent to: * * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); @@ -675,16 +698,16 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); */ ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); -/** +/*! * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue). * NOTE: The return value is different. ZSTD_compressStream() returns a hint for * the next read size (if non-zero and not an error). ZSTD_compressStream2() - * returns the number of bytes left to flush (if non-zero and not an error). + * returns the minimum nb of bytes left to flush (if non-zero and not an error). */ ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); -/** Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */ +/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */ ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); -/** Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */ +/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */ ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); @@ -969,7 +992,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); #endif /* ZSTD_H_235446 */ -/**************************************************************************************** +/* ************************************************************************************** * ADVANCED AND EXPERIMENTAL FUNCTIONS **************************************************************************************** * The definitions in the following section are considered experimental. @@ -1037,6 +1060,10 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); #define ZSTD_LDM_HASHRATELOG_MIN 0 #define ZSTD_LDM_HASHRATELOG_MAX (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN) +/* Advanced parameter bounds */ +#define ZSTD_TARGETCBLOCKSIZE_MIN 64 +#define ZSTD_TARGETCBLOCKSIZE_MAX ZSTD_BLOCKSIZE_MAX + /* internal */ #define ZSTD_HASHLOG3_MAX 17 @@ -1162,7 +1189,7 @@ typedef enum { * however it does mean that all frame data must be present and valid. */ ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); -/** ZSTD_decompressBound() : +/*! ZSTD_decompressBound() : * `src` should point to the start of a series of ZSTD encoded and/or skippable frames * `srcSize` must be the _exact_ size of this series * (i.e. there should be a frame boundary at `src + srcSize`) @@ -1409,6 +1436,11 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre */ #define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5 +/* Tries to fit compressed block size to be around targetCBlockSize. + * No target when targetCBlockSize == 0. + * There is no guarantee on compressed block size (default:0) */ +#define ZSTD_c_targetCBlockSize ZSTD_c_experimentalParam6 + /*! ZSTD_CCtx_getParameter() : * Get the requested compression parameter value, selected by enum ZSTD_cParameter, * and store it into int* value. @@ -1843,7 +1875,7 @@ typedef struct { unsigned checksumFlag; } ZSTD_frameHeader; -/** ZSTD_getFrameHeader() : +/*! ZSTD_getFrameHeader() : * decode Frame Header, or requires larger `srcSize`. * @return : 0, `zfhPtr` is correctly filled, * >0, `srcSize` is too small, value is wanted `srcSize` amount, |