From f7dadc47969740cb4d48379146962d94325424b7 Mon Sep 17 00:00:00 2001 From: JFonS Date: Wed, 22 Apr 2020 15:34:00 +0200 Subject: Add caching the lightmap unwrapping on import This commit adds caching to the lightmap mesh unwraps generated on import. This speeds up re-imports of meshes that haven't changed and also makes sure that the unwraps are consistent across imports. The unwrapping process is not deterministic, so one could end up with a different mapping every time the scene was imported, breaking any previously baked lightmaps. The changes in this commit prevent that from happening. --- editor/import/resource_importer_scene.cpp | 111 ++++++++++++++++++++---- modules/xatlas_unwrap/register_types.cpp | 138 +++++++++++++++++++++++++++--- scene/resources/mesh.cpp | 78 ++++++++++------- scene/resources/mesh.h | 1 + 4 files changed, 271 insertions(+), 57 deletions(-) diff --git a/editor/import/resource_importer_scene.cpp b/editor/import/resource_importer_scene.cpp index b5766a48a0..239fae2268 100644 --- a/editor/import/resource_importer_scene.cpp +++ b/editor/import/resource_importer_scene.cpp @@ -1429,29 +1429,110 @@ Error ResourceImporterScene::import(const String &p_source_file, const String &p Map, Transform> meshes; _find_meshes(scene, meshes); - if (light_bake_mode == 2) { + String file_id = src_path.get_file(); + String cache_file_path = base_path.plus_file(file_id + ".unwrap_cache"); - float texel_size = p_options["meshes/lightmap_texel_size"]; - texel_size = MAX(0.001, texel_size); + Vector cache_data; - EditorProgress progress2("gen_lightmaps", TTR("Generating Lightmaps"), meshes.size()); - int step = 0; - for (Map, Transform>::Element *E = meshes.front(); E; E = E->next()) { + if (FileAccess::exists(cache_file_path)) { + Error err2; + FileAccess *file = FileAccess::open(cache_file_path, FileAccess::READ, &err2); - Ref mesh = E->key(); - String name = mesh->get_name(); - if (name == "") { //should not happen but.. - name = "Mesh " + itos(step); + if (err2) { + if (file) + memdelete(file); + } else { + int cache_size = file->get_len(); + cache_data.resize(cache_size); + file->get_buffer(cache_data.ptrw(), cache_size); + } + } + + float texel_size = p_options["meshes/lightmap_texel_size"]; + texel_size = MAX(0.001, texel_size); + + Map used_unwraps; + + EditorProgress progress2("gen_lightmaps", TTR("Generating Lightmaps"), meshes.size()); + int step = 0; + for (Map, Transform>::Element *E = meshes.front(); E; E = E->next()) { + + Ref mesh = E->key(); + String name = mesh->get_name(); + if (name == "") { //should not happen but.. + name = "Mesh " + itos(step); + } + + progress2.step(TTR("Generating for Mesh: ") + name + " (" + itos(step) + "/" + itos(meshes.size()) + ")", step); + + int *ret_cache_data = (int *)cache_data.ptrw(); + unsigned int ret_cache_size = cache_data.size(); + bool ret_used_cache = true; // Tell the unwrapper to use the cache + Error err2 = mesh->lightmap_unwrap_cached(ret_cache_data, ret_cache_size, ret_used_cache, E->get(), texel_size); + + if (err2 != OK) { + EditorNode::add_io_error("Mesh '" + name + "' failed lightmap generation. Please fix geometry."); + } else { + + String hash = String::md5((unsigned char *)ret_cache_data); + used_unwraps.insert(hash, ret_cache_size); + + if (!ret_used_cache) { + // Cache was not used, add the generated entry to the current cache + if (cache_data.empty()) { + cache_data.resize(4 + ret_cache_size); + int *data = (int *)cache_data.ptrw(); + data[0] = 1; + memcpy(&data[1], ret_cache_data, ret_cache_size); + } else { + int current_size = cache_data.size(); + cache_data.resize(cache_data.size() + ret_cache_size); + unsigned char *ptrw = cache_data.ptrw(); + memcpy(&ptrw[current_size], ret_cache_data, ret_cache_size); + int *data = (int *)ptrw; + data[0] += 1; + } } + } + step++; + } - progress2.step(TTR("Generating for Mesh: ") + name + " (" + itos(step) + "/" + itos(meshes.size()) + ")", step); + Error err2; + FileAccess *file = FileAccess::open(cache_file_path, FileAccess::WRITE, &err2); - Error err2 = mesh->lightmap_unwrap(E->get(), texel_size); - if (err2 != OK) { - EditorNode::add_io_error("Mesh '" + name + "' failed lightmap generation. Please fix geometry."); + if (err2) { + if (file) + memdelete(file); + } else { + + // Store number of entries + file->store_32(used_unwraps.size()); + + // Store cache entries + const int *cache = (int *)cache_data.ptr(); + unsigned int r_idx = 1; + for (int i = 0; i < cache[0]; ++i) { + unsigned char *entry_start = (unsigned char *)&cache[r_idx]; + String entry_hash = String::md5(entry_start); + if (used_unwraps.has(entry_hash)) { + unsigned int entry_size = used_unwraps[entry_hash]; + file->store_buffer(entry_start, entry_size); } - step++; + + r_idx += 4; // hash + r_idx += 2; // size hint + + int vertex_count = cache[r_idx]; + r_idx += 1; // vertex count + r_idx += vertex_count; // vertex + r_idx += vertex_count * 2; // uvs + + int index_count = cache[r_idx]; + r_idx += 1; // index count + r_idx += index_count; // indices } + + file->close(); } } diff --git a/modules/xatlas_unwrap/register_types.cpp b/modules/xatlas_unwrap/register_types.cpp index e293dfd50c..8c5525bed3 100644 --- a/modules/xatlas_unwrap/register_types.cpp +++ b/modules/xatlas_unwrap/register_types.cpp @@ -32,14 +32,91 @@ #include "core/error_macros.h" +#include "core/crypto/crypto_core.h" + #include "thirdparty/xatlas/xatlas.h" #include #include -extern bool (*array_mesh_lightmap_unwrap_callback)(float p_texel_size, const float *p_vertices, const float *p_normals, int p_vertex_count, const int *p_indices, const int *p_face_materials, int p_index_count, float **r_uv, int **r_vertex, int *r_vertex_count, int **r_index, int *r_index_count, int *r_size_hint_x, int *r_size_hint_y); +extern bool (*array_mesh_lightmap_unwrap_callback)(float p_texel_size, const float *p_vertices, const float *p_normals, int p_vertex_count, const int *p_indices, int p_index_count, float **r_uv, int **r_vertex, int *r_vertex_count, int **r_index, int *r_index_count, int *r_size_hint_x, int *r_size_hint_y, int *&r_cache_data, unsigned int &r_cache_size, bool &r_used_cache); + +bool xatlas_mesh_lightmap_unwrap_callback(float p_texel_size, const float *p_vertices, const float *p_normals, int p_vertex_count, const int *p_indices, int p_index_count, float **r_uvs, int **r_vertices, int *r_vertex_count, int **r_indices, int *r_index_count, int *r_size_hint_x, int *r_size_hint_y, int *&r_cache_data, unsigned int &r_cache_size, bool &r_used_cache) { + + CryptoCore::MD5Context ctx; + ctx.start(); + + ctx.update((unsigned char *)&p_texel_size, sizeof(float)); + ctx.update((unsigned char *)p_indices, sizeof(int) * p_index_count); + ctx.update((unsigned char *)p_vertices, sizeof(float) * p_vertex_count); + ctx.update((unsigned char *)p_normals, sizeof(float) * p_vertex_count); + + unsigned char hash[16]; + ctx.finish(hash); + + bool cached = false; + unsigned int cache_idx = 0; + + if (r_used_cache && r_cache_size) { + //Check if hash is in cache data + + int *cache_data = r_cache_data; + int n_entries = cache_data[0]; + unsigned int r_idx = 1; + for (int i = 0; i < n_entries; ++i) { + if (memcmp(&cache_data[r_idx], hash, 16) == 0) { + cached = true; + cache_idx = r_idx; + break; + } + + r_idx += 4; // hash + r_idx += 2; // size hint + + int vertex_count = cache_data[r_idx]; + r_idx += 1; // vertex count + r_idx += vertex_count; // vertex + r_idx += vertex_count * 2; // uvs + + int index_count = cache_data[r_idx]; + r_idx += 1; // index count + r_idx += index_count; // indices + } + } -bool xatlas_mesh_lightmap_unwrap_callback(float p_texel_size, const float *p_vertices, const float *p_normals, int p_vertex_count, const int *p_indices, const int *p_face_materials, int p_index_count, float **r_uv, int **r_vertex, int *r_vertex_count, int **r_index, int *r_index_count, int *r_size_hint_x, int *r_size_hint_y) { + if (r_used_cache && cached) { + int *cache_data = r_cache_data; + + // Return cache data pointer to the caller + r_cache_data = &cache_data[cache_idx]; + + cache_idx += 4; + + // Load size + *r_size_hint_x = cache_data[cache_idx]; + *r_size_hint_y = cache_data[cache_idx + 1]; + cache_idx += 2; + + // Load vertices + *r_vertex_count = cache_data[cache_idx]; + cache_idx++; + *r_vertices = &cache_data[cache_idx]; + cache_idx += *r_vertex_count; + + // Load UVs + *r_uvs = (float *)&cache_data[cache_idx]; + cache_idx += *r_vertex_count * 2; + + // Load indices + *r_index_count = cache_data[cache_idx]; + cache_idx++; + *r_indices = &cache_data[cache_idx]; + + // Return cache data size to the caller + r_cache_size = sizeof(int) * (4 + 2 + 1 + *r_vertex_count + (*r_vertex_count * 2) + 1 + *r_index_count); // hash + size hint + vertex_count + vertices + uvs + index_count + indices + r_used_cache = true; + return true; + } //set up input mesh xatlas::MeshDecl input_mesh; @@ -82,16 +159,16 @@ bool xatlas_mesh_lightmap_unwrap_callback(float p_texel_size, const float *p_ver const xatlas::Mesh &output = atlas->meshes[0]; - *r_vertex = (int *)malloc(sizeof(int) * output.vertexCount); - *r_uv = (float *)malloc(sizeof(float) * output.vertexCount * 2); - *r_index = (int *)malloc(sizeof(int) * output.indexCount); + *r_vertices = (int *)malloc(sizeof(int) * output.vertexCount); + *r_uvs = (float *)malloc(sizeof(float) * output.vertexCount * 2); + *r_indices = (int *)malloc(sizeof(int) * output.indexCount); float max_x = 0; float max_y = 0; for (uint32_t i = 0; i < output.vertexCount; i++) { - (*r_vertex)[i] = output.vertexArray[i].xref; - (*r_uv)[i * 2 + 0] = output.vertexArray[i].uv[0] / w; - (*r_uv)[i * 2 + 1] = output.vertexArray[i].uv[1] / h; + (*r_vertices)[i] = output.vertexArray[i].xref; + (*r_uvs)[i * 2 + 0] = output.vertexArray[i].uv[0] / w; + (*r_uvs)[i * 2 + 1] = output.vertexArray[i].uv[1] / h; max_x = MAX(max_x, output.vertexArray[i].uv[0]); max_y = MAX(max_y, output.vertexArray[i].uv[1]); } @@ -100,13 +177,54 @@ bool xatlas_mesh_lightmap_unwrap_callback(float p_texel_size, const float *p_ver *r_vertex_count = output.vertexCount; for (uint32_t i = 0; i < output.indexCount; i++) { - (*r_index)[i] = output.indexArray[i]; + (*r_indices)[i] = output.indexArray[i]; } *r_index_count = output.indexCount; xatlas::Destroy(atlas); - printf("Done\n"); + + if (r_used_cache) { + unsigned int new_cache_size = 4 + 2 + 1 + *r_vertex_count + (*r_vertex_count * 2) + 1 + *r_index_count; // hash + size hint + vertex_count + vertices + uvs + index_count + indices + new_cache_size *= sizeof(int); + int *new_cache_data = (int *)memalloc(new_cache_size); + unsigned int new_cache_idx = 0; + + // hash + memcpy(&new_cache_data[new_cache_idx], hash, 16); + new_cache_idx += 4; + + // size hint + new_cache_data[new_cache_idx] = *r_size_hint_x; + new_cache_data[new_cache_idx + 1] = *r_size_hint_y; + new_cache_idx += 2; + + // vertex count + new_cache_data[new_cache_idx] = *r_vertex_count; + new_cache_idx++; + + // vertices + memcpy(&new_cache_data[new_cache_idx], *r_vertices, sizeof(int) * *r_vertex_count); + new_cache_idx += *r_vertex_count; + + // uvs + memcpy(&new_cache_data[new_cache_idx], *r_uvs, sizeof(float) * *r_vertex_count * 2); + new_cache_idx += *r_vertex_count * 2; + + // index count + new_cache_data[new_cache_idx] = *r_index_count; + new_cache_idx++; + + // indices + memcpy(&new_cache_data[new_cache_idx], *r_indices, sizeof(int) * *r_index_count); + new_cache_idx += *r_index_count; + + // Return cache data to the caller + r_cache_data = new_cache_data; + r_cache_size = new_cache_size; + r_used_cache = false; + } + return true; } diff --git a/scene/resources/mesh.cpp b/scene/resources/mesh.cpp index 6bb5be15f3..401b689145 100644 --- a/scene/resources/mesh.cpp +++ b/scene/resources/mesh.cpp @@ -1359,7 +1359,7 @@ void ArrayMesh::regen_normalmaps() { } //dirty hack -bool (*array_mesh_lightmap_unwrap_callback)(float p_texel_size, const float *p_vertices, const float *p_normals, int p_vertex_count, const int *p_indices, const int *p_face_materials, int p_index_count, float **r_uv, int **r_vertex, int *r_vertex_count, int **r_index, int *r_index_count, int *r_size_hint_x, int *r_size_hint_y) = nullptr; +bool (*array_mesh_lightmap_unwrap_callback)(float p_texel_size, const float *p_vertices, const float *p_normals, int p_vertex_count, const int *p_indices, int p_index_count, float **r_uv, int **r_vertex, int *r_vertex_count, int **r_index, int *r_index_count, int *r_size_hint_x, int *r_size_hint_y, int *&r_cache_data, unsigned int &r_cache_size, bool &r_used_cache); struct ArrayMeshLightmapSurface { @@ -1370,6 +1370,13 @@ struct ArrayMeshLightmapSurface { }; Error ArrayMesh::lightmap_unwrap(const Transform &p_base_transform, float p_texel_size) { + int *cache_data = nullptr; + unsigned int cache_size = 0; + bool use_cache = false; // Don't use cache + return lightmap_unwrap_cached(cache_data, cache_size, use_cache, p_base_transform, p_texel_size); +} + +Error ArrayMesh::lightmap_unwrap_cached(int *&r_cache_data, unsigned int &r_cache_size, bool &r_used_cache, const Transform &p_base_transform, float p_texel_size) { ERR_FAIL_COND_V(!array_mesh_lightmap_unwrap_callback, ERR_UNCONFIGURED); ERR_FAIL_COND_V_MSG(blend_shapes.size() != 0, ERR_UNAVAILABLE, "Can't unwrap mesh with blend shapes."); @@ -1377,11 +1384,18 @@ Error ArrayMesh::lightmap_unwrap(const Transform &p_base_transform, float p_texe Vector vertices; Vector normals; Vector indices; - Vector face_materials; Vector uv; - Vector> uv_index; + Vector> uv_indices; + + Vector lightmap_surfaces; + + // Keep only the scale + Transform transform = p_base_transform; + transform.origin = Vector3(); + transform.looking_at(Vector3(1, 0, 0), Vector3(0, 1, 0)); + + Basis normal_basis = transform.basis.inverse().transposed(); - Vector surfaces; for (int i = 0; i < get_surface_count(); i++) { ArrayMeshLightmapSurface s; s.primitive = surface_get_primitive_type(i); @@ -1405,12 +1419,12 @@ Error ArrayMesh::lightmap_unwrap(const Transform &p_base_transform, float p_texe vertices.resize((vertex_ofs + vc) * 3); normals.resize((vertex_ofs + vc) * 3); - uv_index.resize(vertex_ofs + vc); + uv_indices.resize(vertex_ofs + vc); for (int j = 0; j < vc; j++) { - Vector3 v = p_base_transform.xform(r[j]); - Vector3 n = p_base_transform.basis.xform(rn[j]).normalized(); + Vector3 v = transform.xform(r[j]); + Vector3 n = normal_basis.xform(rn[j]).normalized(); vertices.write[(j + vertex_ofs) * 3 + 0] = v.x; vertices.write[(j + vertex_ofs) * 3 + 1] = v.y; @@ -1418,7 +1432,7 @@ Error ArrayMesh::lightmap_unwrap(const Transform &p_base_transform, float p_texe normals.write[(j + vertex_ofs) * 3 + 0] = n.x; normals.write[(j + vertex_ofs) * 3 + 1] = n.y; normals.write[(j + vertex_ofs) * 3 + 2] = n.z; - uv_index.write[j + vertex_ofs] = Pair(i, j); + uv_indices.write[j + vertex_ofs] = Pair(i, j); } Vector rindices = arrays[Mesh::ARRAY_INDEX]; @@ -1433,7 +1447,6 @@ Error ArrayMesh::lightmap_unwrap(const Transform &p_base_transform, float p_texe indices.push_back(vertex_ofs + j * 3 + 0); indices.push_back(vertex_ofs + j * 3 + 1); indices.push_back(vertex_ofs + j * 3 + 2); - face_materials.push_back(i); } } else { @@ -1445,11 +1458,10 @@ Error ArrayMesh::lightmap_unwrap(const Transform &p_base_transform, float p_texe indices.push_back(vertex_ofs + ri[j * 3 + 0]); indices.push_back(vertex_ofs + ri[j * 3 + 1]); indices.push_back(vertex_ofs + ri[j * 3 + 2]); - face_materials.push_back(i); } } - surfaces.push_back(s); + lightmap_surfaces.push_back(s); } //unwrap @@ -1462,7 +1474,7 @@ Error ArrayMesh::lightmap_unwrap(const Transform &p_base_transform, float p_texe int size_x; int size_y; - bool ok = array_mesh_lightmap_unwrap_callback(p_texel_size, vertices.ptr(), normals.ptr(), vertices.size() / 3, indices.ptr(), face_materials.ptr(), indices.size(), &gen_uvs, &gen_vertices, &gen_vertex_count, &gen_indices, &gen_index_count, &size_x, &size_y); + bool ok = array_mesh_lightmap_unwrap_callback(p_texel_size, vertices.ptr(), normals.ptr(), vertices.size() / 3, indices.ptr(), indices.size(), &gen_uvs, &gen_vertices, &gen_vertex_count, &gen_indices, &gen_index_count, &size_x, &size_y, r_cache_data, r_cache_size, r_used_cache); if (!ok) { return ERR_CANT_CREATE; @@ -1474,11 +1486,11 @@ Error ArrayMesh::lightmap_unwrap(const Transform &p_base_transform, float p_texe //create surfacetools for each surface.. Vector> surfaces_tools; - for (int i = 0; i < surfaces.size(); i++) { + for (int i = 0; i < lightmap_surfaces.size(); i++) { Ref st; st.instance(); st->begin(Mesh::PRIMITIVE_TRIANGLES); - st->set_material(surfaces[i].material); + st->set_material(lightmap_surfaces[i].material); surfaces_tools.push_back(st); //stay there } @@ -1486,37 +1498,37 @@ Error ArrayMesh::lightmap_unwrap(const Transform &p_base_transform, float p_texe //go through all indices for (int i = 0; i < gen_index_count; i += 3) { - ERR_FAIL_INDEX_V(gen_vertices[gen_indices[i + 0]], uv_index.size(), ERR_BUG); - ERR_FAIL_INDEX_V(gen_vertices[gen_indices[i + 1]], uv_index.size(), ERR_BUG); - ERR_FAIL_INDEX_V(gen_vertices[gen_indices[i + 2]], uv_index.size(), ERR_BUG); + ERR_FAIL_INDEX_V(gen_vertices[gen_indices[i + 0]], uv_indices.size(), ERR_BUG); + ERR_FAIL_INDEX_V(gen_vertices[gen_indices[i + 1]], uv_indices.size(), ERR_BUG); + ERR_FAIL_INDEX_V(gen_vertices[gen_indices[i + 2]], uv_indices.size(), ERR_BUG); - ERR_FAIL_COND_V(uv_index[gen_vertices[gen_indices[i + 0]]].first != uv_index[gen_vertices[gen_indices[i + 1]]].first || uv_index[gen_vertices[gen_indices[i + 0]]].first != uv_index[gen_vertices[gen_indices[i + 2]]].first, ERR_BUG); + ERR_FAIL_COND_V(uv_indices[gen_vertices[gen_indices[i + 0]]].first != uv_indices[gen_vertices[gen_indices[i + 1]]].first || uv_indices[gen_vertices[gen_indices[i + 0]]].first != uv_indices[gen_vertices[gen_indices[i + 2]]].first, ERR_BUG); - int surface = uv_index[gen_vertices[gen_indices[i + 0]]].first; + int surface = uv_indices[gen_vertices[gen_indices[i + 0]]].first; for (int j = 0; j < 3; j++) { - SurfaceTool::Vertex v = surfaces[surface].vertices[uv_index[gen_vertices[gen_indices[i + j]]].second]; + SurfaceTool::Vertex v = lightmap_surfaces[surface].vertices[uv_indices[gen_vertices[gen_indices[i + j]]].second]; - if (surfaces[surface].format & ARRAY_FORMAT_COLOR) { + if (lightmap_surfaces[surface].format & ARRAY_FORMAT_COLOR) { surfaces_tools.write[surface]->add_color(v.color); } - if (surfaces[surface].format & ARRAY_FORMAT_TEX_UV) { + if (lightmap_surfaces[surface].format & ARRAY_FORMAT_TEX_UV) { surfaces_tools.write[surface]->add_uv(v.uv); } - if (surfaces[surface].format & ARRAY_FORMAT_NORMAL) { + if (lightmap_surfaces[surface].format & ARRAY_FORMAT_NORMAL) { surfaces_tools.write[surface]->add_normal(v.normal); } - if (surfaces[surface].format & ARRAY_FORMAT_TANGENT) { + if (lightmap_surfaces[surface].format & ARRAY_FORMAT_TANGENT) { Plane t; t.normal = v.tangent; t.d = v.binormal.dot(v.normal.cross(v.tangent)) < 0 ? -1 : 1; surfaces_tools.write[surface]->add_tangent(t); } - if (surfaces[surface].format & ARRAY_FORMAT_BONES) { + if (lightmap_surfaces[surface].format & ARRAY_FORMAT_BONES) { surfaces_tools.write[surface]->add_bones(v.bones); } - if (surfaces[surface].format & ARRAY_FORMAT_WEIGHTS) { + if (lightmap_surfaces[surface].format & ARRAY_FORMAT_WEIGHTS) { surfaces_tools.write[surface]->add_weights(v.weights); } @@ -1527,20 +1539,22 @@ Error ArrayMesh::lightmap_unwrap(const Transform &p_base_transform, float p_texe } } - //free stuff - ::free(gen_vertices); - ::free(gen_indices); - ::free(gen_uvs); - //generate surfaces for (int i = 0; i < surfaces_tools.size(); i++) { surfaces_tools.write[i]->index(); - surfaces_tools.write[i]->commit(Ref((ArrayMesh *)this), surfaces[i].format); + surfaces_tools.write[i]->commit(Ref((ArrayMesh *)this), lightmap_surfaces[i].format); } set_lightmap_size_hint(Size2(size_x, size_y)); + if (!r_used_cache) { + //free stuff + ::free(gen_vertices); + ::free(gen_indices); + ::free(gen_uvs); + } + return OK; } diff --git a/scene/resources/mesh.h b/scene/resources/mesh.h index 25a9722046..a65cf0a928 100644 --- a/scene/resources/mesh.h +++ b/scene/resources/mesh.h @@ -238,6 +238,7 @@ public: void regen_normalmaps(); Error lightmap_unwrap(const Transform &p_base_transform = Transform(), float p_texel_size = 0.05); + Error lightmap_unwrap_cached(int *&r_cache_data, unsigned int &r_cache_size, bool &r_used_cache, const Transform &p_base_transform = Transform(), float p_texel_size = 0.05); virtual void reload_from_file(); -- cgit v1.2.3