Options to clean/simplify convex hull generated from mesh

Clean: remove duplicate and interior vertices (uses Bullet algorithm)
Simplify: modify the geometry for further simplification (uses VHACD
algorithm)

In the editor, single convex hull now uses the clean option.
Added a new editor entry to create a simplified convex hull, can be
useful for creating convex hull from highly tessellated triangle meshes.

4 files changed, 36 insertions, 15 deletions

diff --git a/scene/3d/mesh_instance_3d.cpp b/scene/3d/mesh_instance_3d.cpp
index 279a1fb7de..28ccbd3e68 100644
--- a/scene/3d/mesh_instance_3d.cpp
+++ b/scene/3d/mesh_instance_3d.cpp
@@ -241,12 +241,12 @@ void MeshInstance3D::create_trimesh_collision() {
 	}
 }
 
-Node *MeshInstance3D::create_convex_collision_node() {
+Node *MeshInstance3D::create_convex_collision_node(bool p_clean, bool p_simplify) {
 	if (mesh.is_null()) {
 		return nullptr;
 	}
 
-	Ref<Shape3D> shape = mesh->create_convex_shape();
+	Ref<Shape3D> shape = mesh->create_convex_shape(p_clean, p_simplify);
 	if (shape.is_null()) {
 		return nullptr;
 	}
@@ -258,8 +258,8 @@ Node *MeshInstance3D::create_convex_collision_node() {
 	return static_body;
 }
 
-void MeshInstance3D::create_convex_collision() {
-	StaticBody3D *static_body = Object::cast_to<StaticBody3D>(create_convex_collision_node());
+void MeshInstance3D::create_convex_collision(bool p_clean, bool p_simplify) {
+	StaticBody3D *static_body = Object::cast_to<StaticBody3D>(create_convex_collision_node(p_clean, p_simplify));
 	ERR_FAIL_COND(!static_body);
 	static_body->set_name(String(get_name()) + "_col");
 
@@ -451,7 +451,7 @@ void MeshInstance3D::_bind_methods() {
 
 	ClassDB::bind_method(D_METHOD("create_trimesh_collision"), &MeshInstance3D::create_trimesh_collision);
 	ClassDB::set_method_flags("MeshInstance3D", "create_trimesh_collision", METHOD_FLAGS_DEFAULT);
-	ClassDB::bind_method(D_METHOD("create_convex_collision"), &MeshInstance3D::create_convex_collision);
+	ClassDB::bind_method(D_METHOD("create_convex_collision", "clean", "simplify"), &MeshInstance3D::create_convex_collision, DEFVAL(true), DEFVAL(false));
 	ClassDB::set_method_flags("MeshInstance3D", "create_convex_collision", METHOD_FLAGS_DEFAULT);
 	ClassDB::bind_method(D_METHOD("create_multiple_convex_collisions"), &MeshInstance3D::create_multiple_convex_collisions);
 	ClassDB::set_method_flags("MeshInstance3D", "create_multiple_convex_collisions", METHOD_FLAGS_DEFAULT);
diff --git a/scene/3d/mesh_instance_3d.h b/scene/3d/mesh_instance_3d.h
index 9dea5804e0..e2d20d0a90 100644
--- a/scene/3d/mesh_instance_3d.h
+++ b/scene/3d/mesh_instance_3d.h
@@ -83,8 +83,8 @@ public:
 	Node *create_trimesh_collision_node();
 	void create_trimesh_collision();
 
-	Node *create_convex_collision_node();
-	void create_convex_collision();
+	Node *create_convex_collision_node(bool p_clean = true, bool p_simplify = false);
+	void create_convex_collision(bool p_clean = true, bool p_simplify = false);
 
 	Node *create_multiple_convex_collisions_node();
 	void create_multiple_convex_collisions();
diff --git a/scene/resources/mesh.cpp b/scene/resources/mesh.cpp
index f44c0c3ee2..2f92872ad5 100644
--- a/scene/resources/mesh.cpp
+++ b/scene/resources/mesh.cpp
@@ -30,6 +30,7 @@
 
 #include "mesh.h"
 
+#include "core/math/convex_hull.h"
 #include "core/templates/pair.h"
 #include "scene/resources/concave_polygon_shape_3d.h"
 #include "scene/resources/convex_polygon_shape_3d.h"
@@ -221,9 +222,17 @@ Vector<Face3> Mesh::get_faces() const {
 */
 }
 
-Ref<Shape3D> Mesh::create_convex_shape() const {
-	Vector<Vector3> vertices;
+Ref<Shape3D> Mesh::create_convex_shape(bool p_clean, bool p_simplify) const {
+	if (p_simplify) {
+		Vector<Ref<Shape3D>> decomposed = convex_decompose(1);
+		if (decomposed.size() == 1) {
+			return decomposed[0];
+		} else {
+			ERR_PRINT("Convex shape simplification failed, falling back to simpler process.");
+		}
+	}
 
+	Vector<Vector3> vertices;
 	for (int i = 0; i < get_surface_count(); i++) {
 		Array a = surface_get_arrays(i);
 		ERR_FAIL_COND_V(a.is_empty(), Ref<ConvexPolygonShape3D>());
@@ -232,6 +241,18 @@ Ref<Shape3D> Mesh::create_convex_shape() const {
 	}
 
 	Ref<ConvexPolygonShape3D> shape = memnew(ConvexPolygonShape3D);
+
+	if (p_clean) {
+		Geometry3D::MeshData md;
+		Error err = ConvexHullComputer::convex_hull(vertices, md);
+		if (err == OK) {
+			shape->set_points(md.vertices);
+			return shape;
+		} else {
+			ERR_PRINT("Convex shape cleaning failed, falling back to simpler process.");
+		}
+	}
+
 	shape->set_points(vertices);
 	return shape;
 }
@@ -543,12 +564,12 @@ void Mesh::clear_cache() const {
 	debug_lines.clear();
 }
 
-Vector<Ref<Shape3D>> Mesh::convex_decompose() const {
+Vector<Ref<Shape3D>> Mesh::convex_decompose(int p_max_convex_hulls) const {
 	ERR_FAIL_COND_V(!convex_composition_function, Vector<Ref<Shape3D>>());
 
 	const Vector<Face3> faces = get_faces();
 
-	Vector<Vector<Face3>> decomposed = convex_composition_function(faces);
+	Vector<Vector<Face3>> decomposed = convex_composition_function(faces, p_max_convex_hulls);
 
 	Vector<Ref<Shape3D>> ret;
 
@@ -1852,7 +1873,7 @@ void ArrayMesh::_bind_methods() {
 	ClassDB::bind_method(D_METHOD("surface_set_name", "surf_idx", "name"), &ArrayMesh::surface_set_name);
 	ClassDB::bind_method(D_METHOD("surface_get_name", "surf_idx"), &ArrayMesh::surface_get_name);
 	ClassDB::bind_method(D_METHOD("create_trimesh_shape"), &ArrayMesh::create_trimesh_shape);
-	ClassDB::bind_method(D_METHOD("create_convex_shape"), &ArrayMesh::create_convex_shape);
+	ClassDB::bind_method(D_METHOD("create_convex_shape", "clean", "simplify"), &ArrayMesh::create_convex_shape, DEFVAL(true), DEFVAL(false));
 	ClassDB::bind_method(D_METHOD("create_outline", "margin"), &ArrayMesh::create_outline);
 	ClassDB::bind_method(D_METHOD("regen_normal_maps"), &ArrayMesh::regen_normal_maps);
 	ClassDB::set_method_flags(get_class_static(), _scs_create("regen_normal_maps"), METHOD_FLAGS_DEFAULT | METHOD_FLAG_EDITOR);
diff --git a/scene/resources/mesh.h b/scene/resources/mesh.h
index 02cab9a5e1..27b0eb098b 100644
--- a/scene/resources/mesh.h
+++ b/scene/resources/mesh.h
@@ -149,7 +149,7 @@ public:
 	void generate_debug_mesh_indices(Vector<Vector3> &r_points);
 
 	Ref<Shape3D> create_trimesh_shape() const;
-	Ref<Shape3D> create_convex_shape() const;
+	Ref<Shape3D> create_convex_shape(bool p_clean = true, bool p_simplify = false) const;
 
 	Ref<Mesh> create_outline(float p_margin) const;
 
@@ -159,11 +159,11 @@ public:
 	Size2i get_lightmap_size_hint() const;
 	void clear_cache() const;
 
-	typedef Vector<Vector<Face3>> (*ConvexDecompositionFunc)(const Vector<Face3> &);
+	typedef Vector<Vector<Face3>> (*ConvexDecompositionFunc)(const Vector<Face3> &p_faces, int p_max_convex_hulls);
 
 	static ConvexDecompositionFunc convex_composition_function;
 
-	Vector<Ref<Shape3D>> convex_decompose() const;
+	Vector<Ref<Shape3D>> convex_decompose(int p_max_convex_hulls = -1) const;
 
 	virtual int get_builtin_bind_pose_count() const;
 	virtual Transform3D get_builtin_bind_pose(int p_index) const;