Add caching the lightmap unwrapping on import

This commit adds caching to the lightmap mesh unwraps generated on
import. This speeds up re-imports of meshes that haven't changed and
also makes sure that the unwraps are consistent across imports.

The unwrapping process is not deterministic, so one could end up with
a different mapping every time the scene was imported, breaking any
previously baked lightmaps. The changes in this commit prevent that
from happening.
This commit is contained in:
JFonS 2020-04-22 15:34:00 +02:00
parent f989a43135
commit f7dadc4796
4 changed files with 275 additions and 61 deletions

View file

@ -1429,30 +1429,111 @@ Error ResourceImporterScene::import(const String &p_source_file, const String &p
Map<Ref<ArrayMesh>, Transform> meshes;
_find_meshes(scene, meshes);
if (light_bake_mode == 2) {
String file_id = src_path.get_file();
String cache_file_path = base_path.plus_file(file_id + ".unwrap_cache");
float texel_size = p_options["meshes/lightmap_texel_size"];
texel_size = MAX(0.001, texel_size);
Vector<unsigned char> cache_data;
EditorProgress progress2("gen_lightmaps", TTR("Generating Lightmaps"), meshes.size());
int step = 0;
for (Map<Ref<ArrayMesh>, Transform>::Element *E = meshes.front(); E; E = E->next()) {
if (FileAccess::exists(cache_file_path)) {
Error err2;
FileAccess *file = FileAccess::open(cache_file_path, FileAccess::READ, &err2);
Ref<ArrayMesh> mesh = E->key();
String name = mesh->get_name();
if (name == "") { //should not happen but..
name = "Mesh " + itos(step);
}
progress2.step(TTR("Generating for Mesh: ") + name + " (" + itos(step) + "/" + itos(meshes.size()) + ")", step);
Error err2 = mesh->lightmap_unwrap(E->get(), texel_size);
if (err2 != OK) {
EditorNode::add_io_error("Mesh '" + name + "' failed lightmap generation. Please fix geometry.");
}
step++;
if (err2) {
if (file)
memdelete(file);
} else {
int cache_size = file->get_len();
cache_data.resize(cache_size);
file->get_buffer(cache_data.ptrw(), cache_size);
}
}
float texel_size = p_options["meshes/lightmap_texel_size"];
texel_size = MAX(0.001, texel_size);
Map<String, unsigned int> used_unwraps;
EditorProgress progress2("gen_lightmaps", TTR("Generating Lightmaps"), meshes.size());
int step = 0;
for (Map<Ref<ArrayMesh>, Transform>::Element *E = meshes.front(); E; E = E->next()) {
Ref<ArrayMesh> mesh = E->key();
String name = mesh->get_name();
if (name == "") { //should not happen but..
name = "Mesh " + itos(step);
}
progress2.step(TTR("Generating for Mesh: ") + name + " (" + itos(step) + "/" + itos(meshes.size()) + ")", step);
int *ret_cache_data = (int *)cache_data.ptrw();
unsigned int ret_cache_size = cache_data.size();
bool ret_used_cache = true; // Tell the unwrapper to use the cache
Error err2 = mesh->lightmap_unwrap_cached(ret_cache_data, ret_cache_size, ret_used_cache, E->get(), texel_size);
if (err2 != OK) {
EditorNode::add_io_error("Mesh '" + name + "' failed lightmap generation. Please fix geometry.");
} else {
String hash = String::md5((unsigned char *)ret_cache_data);
used_unwraps.insert(hash, ret_cache_size);
if (!ret_used_cache) {
// Cache was not used, add the generated entry to the current cache
if (cache_data.empty()) {
cache_data.resize(4 + ret_cache_size);
int *data = (int *)cache_data.ptrw();
data[0] = 1;
memcpy(&data[1], ret_cache_data, ret_cache_size);
} else {
int current_size = cache_data.size();
cache_data.resize(cache_data.size() + ret_cache_size);
unsigned char *ptrw = cache_data.ptrw();
memcpy(&ptrw[current_size], ret_cache_data, ret_cache_size);
int *data = (int *)ptrw;
data[0] += 1;
}
}
}
step++;
}
Error err2;
FileAccess *file = FileAccess::open(cache_file_path, FileAccess::WRITE, &err2);
if (err2) {
if (file)
memdelete(file);
} else {
// Store number of entries
file->store_32(used_unwraps.size());
// Store cache entries
const int *cache = (int *)cache_data.ptr();
unsigned int r_idx = 1;
for (int i = 0; i < cache[0]; ++i) {
unsigned char *entry_start = (unsigned char *)&cache[r_idx];
String entry_hash = String::md5(entry_start);
if (used_unwraps.has(entry_hash)) {
unsigned int entry_size = used_unwraps[entry_hash];
file->store_buffer(entry_start, entry_size);
}
r_idx += 4; // hash
r_idx += 2; // size hint
int vertex_count = cache[r_idx];
r_idx += 1; // vertex count
r_idx += vertex_count; // vertex
r_idx += vertex_count * 2; // uvs
int index_count = cache[r_idx];
r_idx += 1; // index count
r_idx += index_count; // indices
}
file->close();
}
}
if (external_animations || external_materials || external_meshes) {

View file

@ -32,14 +32,91 @@
#include "core/error_macros.h"
#include "core/crypto/crypto_core.h"
#include "thirdparty/xatlas/xatlas.h"
#include <stdio.h>
#include <stdlib.h>
extern bool (*array_mesh_lightmap_unwrap_callback)(float p_texel_size, const float *p_vertices, const float *p_normals, int p_vertex_count, const int *p_indices, const int *p_face_materials, int p_index_count, float **r_uv, int **r_vertex, int *r_vertex_count, int **r_index, int *r_index_count, int *r_size_hint_x, int *r_size_hint_y);
extern bool (*array_mesh_lightmap_unwrap_callback)(float p_texel_size, const float *p_vertices, const float *p_normals, int p_vertex_count, const int *p_indices, int p_index_count, float **r_uv, int **r_vertex, int *r_vertex_count, int **r_index, int *r_index_count, int *r_size_hint_x, int *r_size_hint_y, int *&r_cache_data, unsigned int &r_cache_size, bool &r_used_cache);
bool xatlas_mesh_lightmap_unwrap_callback(float p_texel_size, const float *p_vertices, const float *p_normals, int p_vertex_count, const int *p_indices, const int *p_face_materials, int p_index_count, float **r_uv, int **r_vertex, int *r_vertex_count, int **r_index, int *r_index_count, int *r_size_hint_x, int *r_size_hint_y) {
bool xatlas_mesh_lightmap_unwrap_callback(float p_texel_size, const float *p_vertices, const float *p_normals, int p_vertex_count, const int *p_indices, int p_index_count, float **r_uvs, int **r_vertices, int *r_vertex_count, int **r_indices, int *r_index_count, int *r_size_hint_x, int *r_size_hint_y, int *&r_cache_data, unsigned int &r_cache_size, bool &r_used_cache) {
CryptoCore::MD5Context ctx;
ctx.start();
ctx.update((unsigned char *)&p_texel_size, sizeof(float));
ctx.update((unsigned char *)p_indices, sizeof(int) * p_index_count);
ctx.update((unsigned char *)p_vertices, sizeof(float) * p_vertex_count);
ctx.update((unsigned char *)p_normals, sizeof(float) * p_vertex_count);
unsigned char hash[16];
ctx.finish(hash);
bool cached = false;
unsigned int cache_idx = 0;
if (r_used_cache && r_cache_size) {
//Check if hash is in cache data
int *cache_data = r_cache_data;
int n_entries = cache_data[0];
unsigned int r_idx = 1;
for (int i = 0; i < n_entries; ++i) {
if (memcmp(&cache_data[r_idx], hash, 16) == 0) {
cached = true;
cache_idx = r_idx;
break;
}
r_idx += 4; // hash
r_idx += 2; // size hint
int vertex_count = cache_data[r_idx];
r_idx += 1; // vertex count
r_idx += vertex_count; // vertex
r_idx += vertex_count * 2; // uvs
int index_count = cache_data[r_idx];
r_idx += 1; // index count
r_idx += index_count; // indices
}
}
if (r_used_cache && cached) {
int *cache_data = r_cache_data;
// Return cache data pointer to the caller
r_cache_data = &cache_data[cache_idx];
cache_idx += 4;
// Load size
*r_size_hint_x = cache_data[cache_idx];
*r_size_hint_y = cache_data[cache_idx + 1];
cache_idx += 2;
// Load vertices
*r_vertex_count = cache_data[cache_idx];
cache_idx++;
*r_vertices = &cache_data[cache_idx];
cache_idx += *r_vertex_count;
// Load UVs
*r_uvs = (float *)&cache_data[cache_idx];
cache_idx += *r_vertex_count * 2;
// Load indices
*r_index_count = cache_data[cache_idx];
cache_idx++;
*r_indices = &cache_data[cache_idx];
// Return cache data size to the caller
r_cache_size = sizeof(int) * (4 + 2 + 1 + *r_vertex_count + (*r_vertex_count * 2) + 1 + *r_index_count); // hash + size hint + vertex_count + vertices + uvs + index_count + indices
r_used_cache = true;
return true;
}
//set up input mesh
xatlas::MeshDecl input_mesh;
@ -82,16 +159,16 @@ bool xatlas_mesh_lightmap_unwrap_callback(float p_texel_size, const float *p_ver
const xatlas::Mesh &output = atlas->meshes[0];
*r_vertex = (int *)malloc(sizeof(int) * output.vertexCount);
*r_uv = (float *)malloc(sizeof(float) * output.vertexCount * 2);
*r_index = (int *)malloc(sizeof(int) * output.indexCount);
*r_vertices = (int *)malloc(sizeof(int) * output.vertexCount);
*r_uvs = (float *)malloc(sizeof(float) * output.vertexCount * 2);
*r_indices = (int *)malloc(sizeof(int) * output.indexCount);
float max_x = 0;
float max_y = 0;
for (uint32_t i = 0; i < output.vertexCount; i++) {
(*r_vertex)[i] = output.vertexArray[i].xref;
(*r_uv)[i * 2 + 0] = output.vertexArray[i].uv[0] / w;
(*r_uv)[i * 2 + 1] = output.vertexArray[i].uv[1] / h;
(*r_vertices)[i] = output.vertexArray[i].xref;
(*r_uvs)[i * 2 + 0] = output.vertexArray[i].uv[0] / w;
(*r_uvs)[i * 2 + 1] = output.vertexArray[i].uv[1] / h;
max_x = MAX(max_x, output.vertexArray[i].uv[0]);
max_y = MAX(max_y, output.vertexArray[i].uv[1]);
}
@ -100,13 +177,54 @@ bool xatlas_mesh_lightmap_unwrap_callback(float p_texel_size, const float *p_ver
*r_vertex_count = output.vertexCount;
for (uint32_t i = 0; i < output.indexCount; i++) {
(*r_index)[i] = output.indexArray[i];
(*r_indices)[i] = output.indexArray[i];
}
*r_index_count = output.indexCount;
xatlas::Destroy(atlas);
printf("Done\n");
if (r_used_cache) {
unsigned int new_cache_size = 4 + 2 + 1 + *r_vertex_count + (*r_vertex_count * 2) + 1 + *r_index_count; // hash + size hint + vertex_count + vertices + uvs + index_count + indices
new_cache_size *= sizeof(int);
int *new_cache_data = (int *)memalloc(new_cache_size);
unsigned int new_cache_idx = 0;
// hash
memcpy(&new_cache_data[new_cache_idx], hash, 16);
new_cache_idx += 4;
// size hint
new_cache_data[new_cache_idx] = *r_size_hint_x;
new_cache_data[new_cache_idx + 1] = *r_size_hint_y;
new_cache_idx += 2;
// vertex count
new_cache_data[new_cache_idx] = *r_vertex_count;
new_cache_idx++;
// vertices
memcpy(&new_cache_data[new_cache_idx], *r_vertices, sizeof(int) * *r_vertex_count);
new_cache_idx += *r_vertex_count;
// uvs
memcpy(&new_cache_data[new_cache_idx], *r_uvs, sizeof(float) * *r_vertex_count * 2);
new_cache_idx += *r_vertex_count * 2;
// index count
new_cache_data[new_cache_idx] = *r_index_count;
new_cache_idx++;
// indices
memcpy(&new_cache_data[new_cache_idx], *r_indices, sizeof(int) * *r_index_count);
new_cache_idx += *r_index_count;
// Return cache data to the caller
r_cache_data = new_cache_data;
r_cache_size = new_cache_size;
r_used_cache = false;
}
return true;
}

View file

@ -1359,7 +1359,7 @@ void ArrayMesh::regen_normalmaps() {
}
//dirty hack
bool (*array_mesh_lightmap_unwrap_callback)(float p_texel_size, const float *p_vertices, const float *p_normals, int p_vertex_count, const int *p_indices, const int *p_face_materials, int p_index_count, float **r_uv, int **r_vertex, int *r_vertex_count, int **r_index, int *r_index_count, int *r_size_hint_x, int *r_size_hint_y) = nullptr;
bool (*array_mesh_lightmap_unwrap_callback)(float p_texel_size, const float *p_vertices, const float *p_normals, int p_vertex_count, const int *p_indices, int p_index_count, float **r_uv, int **r_vertex, int *r_vertex_count, int **r_index, int *r_index_count, int *r_size_hint_x, int *r_size_hint_y, int *&r_cache_data, unsigned int &r_cache_size, bool &r_used_cache);
struct ArrayMeshLightmapSurface {
@ -1370,6 +1370,13 @@ struct ArrayMeshLightmapSurface {
};
Error ArrayMesh::lightmap_unwrap(const Transform &p_base_transform, float p_texel_size) {
int *cache_data = nullptr;
unsigned int cache_size = 0;
bool use_cache = false; // Don't use cache
return lightmap_unwrap_cached(cache_data, cache_size, use_cache, p_base_transform, p_texel_size);
}
Error ArrayMesh::lightmap_unwrap_cached(int *&r_cache_data, unsigned int &r_cache_size, bool &r_used_cache, const Transform &p_base_transform, float p_texel_size) {
ERR_FAIL_COND_V(!array_mesh_lightmap_unwrap_callback, ERR_UNCONFIGURED);
ERR_FAIL_COND_V_MSG(blend_shapes.size() != 0, ERR_UNAVAILABLE, "Can't unwrap mesh with blend shapes.");
@ -1377,11 +1384,18 @@ Error ArrayMesh::lightmap_unwrap(const Transform &p_base_transform, float p_texe
Vector<float> vertices;
Vector<float> normals;
Vector<int> indices;
Vector<int> face_materials;
Vector<float> uv;
Vector<Pair<int, int>> uv_index;
Vector<Pair<int, int>> uv_indices;
Vector<ArrayMeshLightmapSurface> lightmap_surfaces;
// Keep only the scale
Transform transform = p_base_transform;
transform.origin = Vector3();
transform.looking_at(Vector3(1, 0, 0), Vector3(0, 1, 0));
Basis normal_basis = transform.basis.inverse().transposed();
Vector<ArrayMeshLightmapSurface> surfaces;
for (int i = 0; i < get_surface_count(); i++) {
ArrayMeshLightmapSurface s;
s.primitive = surface_get_primitive_type(i);
@ -1405,12 +1419,12 @@ Error ArrayMesh::lightmap_unwrap(const Transform &p_base_transform, float p_texe
vertices.resize((vertex_ofs + vc) * 3);
normals.resize((vertex_ofs + vc) * 3);
uv_index.resize(vertex_ofs + vc);
uv_indices.resize(vertex_ofs + vc);
for (int j = 0; j < vc; j++) {
Vector3 v = p_base_transform.xform(r[j]);
Vector3 n = p_base_transform.basis.xform(rn[j]).normalized();
Vector3 v = transform.xform(r[j]);
Vector3 n = normal_basis.xform(rn[j]).normalized();
vertices.write[(j + vertex_ofs) * 3 + 0] = v.x;
vertices.write[(j + vertex_ofs) * 3 + 1] = v.y;
@ -1418,7 +1432,7 @@ Error ArrayMesh::lightmap_unwrap(const Transform &p_base_transform, float p_texe
normals.write[(j + vertex_ofs) * 3 + 0] = n.x;
normals.write[(j + vertex_ofs) * 3 + 1] = n.y;
normals.write[(j + vertex_ofs) * 3 + 2] = n.z;
uv_index.write[j + vertex_ofs] = Pair<int, int>(i, j);
uv_indices.write[j + vertex_ofs] = Pair<int, int>(i, j);
}
Vector<int> rindices = arrays[Mesh::ARRAY_INDEX];
@ -1433,7 +1447,6 @@ Error ArrayMesh::lightmap_unwrap(const Transform &p_base_transform, float p_texe
indices.push_back(vertex_ofs + j * 3 + 0);
indices.push_back(vertex_ofs + j * 3 + 1);
indices.push_back(vertex_ofs + j * 3 + 2);
face_materials.push_back(i);
}
} else {
@ -1445,11 +1458,10 @@ Error ArrayMesh::lightmap_unwrap(const Transform &p_base_transform, float p_texe
indices.push_back(vertex_ofs + ri[j * 3 + 0]);
indices.push_back(vertex_ofs + ri[j * 3 + 1]);
indices.push_back(vertex_ofs + ri[j * 3 + 2]);
face_materials.push_back(i);
}
}
surfaces.push_back(s);
lightmap_surfaces.push_back(s);
}
//unwrap
@ -1462,7 +1474,7 @@ Error ArrayMesh::lightmap_unwrap(const Transform &p_base_transform, float p_texe
int size_x;
int size_y;
bool ok = array_mesh_lightmap_unwrap_callback(p_texel_size, vertices.ptr(), normals.ptr(), vertices.size() / 3, indices.ptr(), face_materials.ptr(), indices.size(), &gen_uvs, &gen_vertices, &gen_vertex_count, &gen_indices, &gen_index_count, &size_x, &size_y);
bool ok = array_mesh_lightmap_unwrap_callback(p_texel_size, vertices.ptr(), normals.ptr(), vertices.size() / 3, indices.ptr(), indices.size(), &gen_uvs, &gen_vertices, &gen_vertex_count, &gen_indices, &gen_index_count, &size_x, &size_y, r_cache_data, r_cache_size, r_used_cache);
if (!ok) {
return ERR_CANT_CREATE;
@ -1474,11 +1486,11 @@ Error ArrayMesh::lightmap_unwrap(const Transform &p_base_transform, float p_texe
//create surfacetools for each surface..
Vector<Ref<SurfaceTool>> surfaces_tools;
for (int i = 0; i < surfaces.size(); i++) {
for (int i = 0; i < lightmap_surfaces.size(); i++) {
Ref<SurfaceTool> st;
st.instance();
st->begin(Mesh::PRIMITIVE_TRIANGLES);
st->set_material(surfaces[i].material);
st->set_material(lightmap_surfaces[i].material);
surfaces_tools.push_back(st); //stay there
}
@ -1486,37 +1498,37 @@ Error ArrayMesh::lightmap_unwrap(const Transform &p_base_transform, float p_texe
//go through all indices
for (int i = 0; i < gen_index_count; i += 3) {
ERR_FAIL_INDEX_V(gen_vertices[gen_indices[i + 0]], uv_index.size(), ERR_BUG);
ERR_FAIL_INDEX_V(gen_vertices[gen_indices[i + 1]], uv_index.size(), ERR_BUG);
ERR_FAIL_INDEX_V(gen_vertices[gen_indices[i + 2]], uv_index.size(), ERR_BUG);
ERR_FAIL_INDEX_V(gen_vertices[gen_indices[i + 0]], uv_indices.size(), ERR_BUG);
ERR_FAIL_INDEX_V(gen_vertices[gen_indices[i + 1]], uv_indices.size(), ERR_BUG);
ERR_FAIL_INDEX_V(gen_vertices[gen_indices[i + 2]], uv_indices.size(), ERR_BUG);
ERR_FAIL_COND_V(uv_index[gen_vertices[gen_indices[i + 0]]].first != uv_index[gen_vertices[gen_indices[i + 1]]].first || uv_index[gen_vertices[gen_indices[i + 0]]].first != uv_index[gen_vertices[gen_indices[i + 2]]].first, ERR_BUG);
ERR_FAIL_COND_V(uv_indices[gen_vertices[gen_indices[i + 0]]].first != uv_indices[gen_vertices[gen_indices[i + 1]]].first || uv_indices[gen_vertices[gen_indices[i + 0]]].first != uv_indices[gen_vertices[gen_indices[i + 2]]].first, ERR_BUG);
int surface = uv_index[gen_vertices[gen_indices[i + 0]]].first;
int surface = uv_indices[gen_vertices[gen_indices[i + 0]]].first;
for (int j = 0; j < 3; j++) {
SurfaceTool::Vertex v = surfaces[surface].vertices[uv_index[gen_vertices[gen_indices[i + j]]].second];
SurfaceTool::Vertex v = lightmap_surfaces[surface].vertices[uv_indices[gen_vertices[gen_indices[i + j]]].second];
if (surfaces[surface].format & ARRAY_FORMAT_COLOR) {
if (lightmap_surfaces[surface].format & ARRAY_FORMAT_COLOR) {
surfaces_tools.write[surface]->add_color(v.color);
}
if (surfaces[surface].format & ARRAY_FORMAT_TEX_UV) {
if (lightmap_surfaces[surface].format & ARRAY_FORMAT_TEX_UV) {
surfaces_tools.write[surface]->add_uv(v.uv);
}
if (surfaces[surface].format & ARRAY_FORMAT_NORMAL) {
if (lightmap_surfaces[surface].format & ARRAY_FORMAT_NORMAL) {
surfaces_tools.write[surface]->add_normal(v.normal);
}
if (surfaces[surface].format & ARRAY_FORMAT_TANGENT) {
if (lightmap_surfaces[surface].format & ARRAY_FORMAT_TANGENT) {
Plane t;
t.normal = v.tangent;
t.d = v.binormal.dot(v.normal.cross(v.tangent)) < 0 ? -1 : 1;
surfaces_tools.write[surface]->add_tangent(t);
}
if (surfaces[surface].format & ARRAY_FORMAT_BONES) {
if (lightmap_surfaces[surface].format & ARRAY_FORMAT_BONES) {
surfaces_tools.write[surface]->add_bones(v.bones);
}
if (surfaces[surface].format & ARRAY_FORMAT_WEIGHTS) {
if (lightmap_surfaces[surface].format & ARRAY_FORMAT_WEIGHTS) {
surfaces_tools.write[surface]->add_weights(v.weights);
}
@ -1527,20 +1539,22 @@ Error ArrayMesh::lightmap_unwrap(const Transform &p_base_transform, float p_texe
}
}
//free stuff
::free(gen_vertices);
::free(gen_indices);
::free(gen_uvs);
//generate surfaces
for (int i = 0; i < surfaces_tools.size(); i++) {
surfaces_tools.write[i]->index();
surfaces_tools.write[i]->commit(Ref<ArrayMesh>((ArrayMesh *)this), surfaces[i].format);
surfaces_tools.write[i]->commit(Ref<ArrayMesh>((ArrayMesh *)this), lightmap_surfaces[i].format);
}
set_lightmap_size_hint(Size2(size_x, size_y));
if (!r_used_cache) {
//free stuff
::free(gen_vertices);
::free(gen_indices);
::free(gen_uvs);
}
return OK;
}

View file

@ -238,6 +238,7 @@ public:
void regen_normalmaps();
Error lightmap_unwrap(const Transform &p_base_transform = Transform(), float p_texel_size = 0.05);
Error lightmap_unwrap_cached(int *&r_cache_data, unsigned int &r_cache_size, bool &r_used_cache, const Transform &p_base_transform = Transform(), float p_texel_size = 0.05);
virtual void reload_from_file();