Skip to content

Commit

Permalink
Merge pull request #782 from zeux/gltf-dedup
Browse files Browse the repository at this point in the history 
gltfpack: Deduplicate mesh geometry
  • Loading branch information
@zeux
zeux authored Oct 7, 2024
2 parents 5d7b0e6 + 82e0332 commit 2df0a25
Show file tree
Hide file tree
Showing 4 changed files with 220 additions and 29 deletions.
63 changes: 39 additions & 24 deletions gltf/gltfpack.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
#include "gltfpack.h"

#include <algorithm>
#include <map>

#include <locale.h>
#include <stdint.h>
Expand Down Expand Up @@ -295,6 +296,10 @@ static void detachMesh(Mesh& mesh, cgltf_data* data, const std::vector<NodeInfo>
if (mesh.nodes.size() > 1 && !settings.mesh_merge && !settings.mesh_instancing)
return;

// mesh has duplicate geometry; detaching it would increase the size due to unique world-space transforms
if (mesh.nodes.size() == 1 && mesh.geometry_duplicate && !settings.mesh_merge)
return;

// prefer instancing if possible, use merging otherwise
if (mesh.nodes.size() > 1 && settings.mesh_instancing)
{
Expand Down Expand Up @@ -341,6 +346,10 @@ static void process(cgltf_data* data, const char* input_path, const char* output
markScenes(data, nodes);
markAnimated(data, nodes, animations);

mergeMeshMaterials(data, meshes, settings);
if (settings.mesh_dedup)
dedupMeshes(meshes);

for (size_t i = 0; i < meshes.size(); ++i)
detachMesh(meshes[i], data, nodes, settings);

Expand Down Expand Up @@ -375,7 +384,6 @@ static void process(cgltf_data* data, const char* input_path, const char* output
filterStreams(mesh, mi);
}

mergeMeshMaterials(data, meshes, settings);
mergeMeshes(meshes, settings);
filterEmptyMeshes(meshes);

Expand Down Expand Up @@ -408,7 +416,13 @@ static void process(cgltf_data* data, const char* input_path, const char* output
#endif

for (size_t i = 0; i < meshes.size(); ++i)
processMesh(meshes[i], settings);
{
Mesh& mesh = meshes[i];
processMesh(mesh, settings);

if (mesh.geometry_duplicate)
hashMesh(mesh);
}

#ifndef NDEBUG
meshes.insert(meshes.end(), debug_meshes.begin(), debug_meshes.end());
Expand Down Expand Up @@ -551,6 +565,8 @@ static void process(cgltf_data* data, const char* input_path, const char* output
ext_texture_transform = ext_texture_transform || mi.uses_texture_transform;
}

std::map<std::pair<uint64_t, uint64_t>, std::pair<size_t, size_t> > primitive_cache;

for (size_t i = 0; i < meshes.size(); ++i)
{
const Mesh& mesh = meshes[i];
Expand Down Expand Up @@ -578,33 +594,26 @@ static void process(cgltf_data* data, const char* input_path, const char* output
const QuantizationTexture& qt = qt_meshes[pi] == size_t(-1) ? qt_dummy : qt_materials[qt_meshes[pi]];

comma(json_meshes);
append(json_meshes, "{\"attributes\":{");
writeMeshAttributes(json_meshes, views, json_accessors, accr_offset, prim, 0, qp, qt, settings);
append(json_meshes, "}");
if (prim.type != cgltf_primitive_type_triangles)
{
append(json_meshes, ",\"mode\":");
append(json_meshes, size_t(prim.type - cgltf_primitive_type_points));
}
if (mesh.targets)

if (prim.geometry_duplicate)
{
append(json_meshes, ",\"targets\":[");
for (size_t j = 0; j < mesh.targets; ++j)
std::pair<size_t, size_t>& primitive_json = primitive_cache[std::make_pair(prim.geometry_hash[0], prim.geometry_hash[1])];

if (primitive_json.second)
{
comma(json_meshes);
append(json_meshes, "{");
writeMeshAttributes(json_meshes, views, json_accessors, accr_offset, prim, int(1 + j), qp, qt, settings);
append(json_meshes, "}");
// reuse previously written accessors
json_meshes.append(json_meshes, primitive_json.first, primitive_json.second);
}
else
{
primitive_json.first = json_meshes.size();
writeMeshGeometry(json_meshes, views, json_accessors, accr_offset, prim, qp, qt, settings);
primitive_json.second = json_meshes.size() - primitive_json.first;
}
append(json_meshes, "]");
}

if (!prim.indices.empty())
else
{
size_t index_accr = writeMeshIndices(views, json_accessors, accr_offset, prim, settings);

append(json_meshes, ",\"indices\":");
append(json_meshes, index_accr);
writeMeshGeometry(json_meshes, views, json_accessors, accr_offset, prim, qp, qt, settings);
}

if (prim.material)
Expand Down Expand Up @@ -1177,6 +1186,7 @@ Settings defaults()
settings.rot_bits = 12;
settings.scl_bits = 16;
settings.anim_freq = 30;
settings.mesh_dedup = true;
settings.simplify_ratio = 1.f;
settings.simplify_error = 1e-2f;
settings.texture_scale = 1.f;
Expand Down Expand Up @@ -1314,6 +1324,11 @@ int main(int argc, char** argv)
{
settings.keep_attributes = true;
}
else if (strcmp(arg, "-mdd") == 0)
{
fprintf(stderr, "Warning: option -mdd disables mesh deduplication and is only provided as a safety measure; it will be removed in the future\n");
settings.mesh_dedup = false;
}
else if (strcmp(arg, "-mm") == 0)
{
settings.mesh_merge = true;
Expand Down
9 changes: 8 additions & 1 deletion gltf/gltfpack.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,9 @@ struct Mesh
std::vector<Stream> streams;
std::vector<unsigned int> indices;

bool geometry_duplicate;
uint64_t geometry_hash[2];

size_t targets;
std::vector<float> target_weights;
std::vector<const char*> target_names;
Expand Down Expand Up @@ -131,6 +134,7 @@ struct Settings
bool keep_extras;
bool keep_attributes;

bool mesh_dedup;
bool mesh_merge;
bool mesh_instancing;

Expand Down Expand Up @@ -314,6 +318,8 @@ bool compareMeshTargets(const Mesh& lhs, const Mesh& rhs);
bool compareMeshVariants(const Mesh& lhs, const Mesh& rhs);
bool compareMeshNodes(const Mesh& lhs, const Mesh& rhs);

void hashMesh(Mesh& mesh);
void dedupMeshes(std::vector<Mesh>& meshes);
void mergeMeshInstances(Mesh& mesh);
void mergeMeshes(std::vector<Mesh>& meshes, const Settings& settings);
void filterEmptyMeshes(std::vector<Mesh>& meshes);
Expand Down Expand Up @@ -376,7 +382,8 @@ void writeSampler(std::string& json, const cgltf_sampler& sampler);
void writeImage(std::string& json, std::vector<BufferView>& views, const cgltf_image& image, const ImageInfo& info, const std::string* encoded, size_t index, const char* input_path, const char* output_path, const Settings& settings);
void writeTexture(std::string& json, const cgltf_texture& texture, const ImageInfo* info, cgltf_data* data, const Settings& settings);
void writeMeshAttributes(std::string& json, std::vector<BufferView>& views, std::string& json_accessors, size_t& accr_offset, const Mesh& mesh, int target, const QuantizationPosition& qp, const QuantizationTexture& qt, const Settings& settings);
size_t writeMeshIndices(std::vector<BufferView>& views, std::string& json_accessors, size_t& accr_offset, const Mesh& mesh, const Settings& settings);
size_t writeMeshIndices(std::vector<BufferView>& views, std::string& json_accessors, size_t& accr_offset, const std::vector<unsigned int>& indices, cgltf_primitive_type type, const Settings& settings);
void writeMeshGeometry(std::string& json, std::vector<BufferView>& views, std::string& json_accessors, size_t& accr_offset, const Mesh& mesh, const QuantizationPosition& qp, const QuantizationTexture& qt, const Settings& settings);
size_t writeJointBindMatrices(std::vector<BufferView>& views, std::string& json_accessors, size_t& accr_offset, const cgltf_skin& skin, const QuantizationPosition& qp, const Settings& settings);
size_t writeInstances(std::vector<BufferView>& views, std::string& json_accessors, size_t& accr_offset, const std::vector<Transform>& transforms, const QuantizationPosition& qp, const Settings& settings);
void writeMeshNode(std::string& json, size_t mesh_offset, cgltf_node* node, cgltf_skin* skin, cgltf_data* data, const QuantizationPosition* qp);
Expand Down
137 changes: 137 additions & 0 deletions gltf/mesh.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,143 @@ static void mergeMeshes(Mesh& target, const Mesh& mesh)
target.indices[index_offset + i] = unsigned(vertex_offset + mesh.indices[i]);
}

static void hashUpdate(uint64_t hash[2], const void* data, size_t size)
{
#define ROTL64(x, r) (((x) << (r)) | ((x) >> (64 - (r))))

// MurMurHash3 128-bit
const uint64_t c1 = 0x87c37b91114253d5ull;
const uint64_t c2 = 0x4cf5ad432745937full;

uint64_t h1 = hash[0], h2 = hash[1];

size_t offset = 0;

// body
for (; offset + 16 <= size; offset += 16)
{
uint64_t k1, k2;
memcpy(&k1, static_cast<const char*>(data) + offset + 0, 8);
memcpy(&k2, static_cast<const char*>(data) + offset + 8, 8);

k1 *= c1, k1 = ROTL64(k1, 31), k1 *= c2;
h1 ^= k1, h1 = ROTL64(h1, 27), h1 += h2;
h1 = h1 * 5 + 0x52dce729;
k2 *= c2, k2 = ROTL64(k2, 33), k2 *= c1;
h2 ^= k2, h2 = ROTL64(h2, 31), h2 += h1;
h2 = h2 * 5 + 0x38495ab5;
}

// tail
if (offset < size)
{
uint64_t tail[2] = {};
memcpy(tail, static_cast<const char*>(data) + offset, size - offset);

uint64_t k1 = tail[0], k2 = tail[1];

k1 *= c1, k1 = ROTL64(k1, 31), k1 *= c2;
h1 ^= k1;
k2 *= c2, k2 = ROTL64(k2, 33), k2 *= c1;
h2 ^= k2;
}

h1 ^= size;
h2 ^= size;

hash[0] = h1;
hash[1] = h2;

#undef ROTL64
}

void hashMesh(Mesh& mesh)
{
mesh.geometry_hash[0] = mesh.geometry_hash[1] = 41;

for (size_t i = 0; i < mesh.streams.size(); ++i)
{
const Stream& stream = mesh.streams[i];

int meta[3] = {stream.type, stream.index, stream.target};
hashUpdate(mesh.geometry_hash, meta, sizeof(meta));

if (stream.custom_name)
hashUpdate(mesh.geometry_hash, stream.custom_name, strlen(stream.custom_name));

hashUpdate(mesh.geometry_hash, stream.data.data(), stream.data.size() * sizeof(Attr));
}

if (!mesh.indices.empty())
hashUpdate(mesh.geometry_hash, mesh.indices.data(), mesh.indices.size() * sizeof(unsigned int));

int meta[4] = {int(mesh.streams.size()), mesh.streams.empty() ? 0 : int(mesh.streams[0].data.size()), int(mesh.indices.size()), mesh.type};
hashUpdate(mesh.geometry_hash, meta, sizeof(meta));
}

static bool canDedupMesh(const Mesh& mesh)
{
// empty mesh
if (mesh.streams.empty())
return false;

// world-space mesh
if (mesh.nodes.empty() && mesh.instances.empty())
return false;

// to simplify dedup we ignore complex target setups for now
if (!mesh.target_weights.empty() || !mesh.target_names.empty() || !mesh.variants.empty())
return false;

return true;
}

void dedupMeshes(std::vector<Mesh>& meshes)
{
for (size_t i = 0; i < meshes.size(); ++i)
hashMesh(meshes[i]);

for (size_t i = 0; i < meshes.size(); ++i)
{
Mesh& target = meshes[i];

if (!canDedupMesh(target))
continue;

for (size_t j = i + 1; j < meshes.size(); ++j)
{
Mesh& mesh = meshes[j];

if (mesh.geometry_hash[0] != target.geometry_hash[0] || mesh.geometry_hash[1] != target.geometry_hash[1])
continue;

if (!canDedupMesh(mesh))
continue;

if (mesh.scene != target.scene || mesh.material != target.material || mesh.skin != target.skin)
{
// mark both meshes as having duplicate geometry; we don't use this in dedupMeshes but it's useful later in the pipeline
target.geometry_duplicate = true;
mesh.geometry_duplicate = true;
continue;
}

// basic sanity test; these should be included in geometry hash
assert(mesh.streams.size() == target.streams.size());
assert(mesh.streams[0].data.size() == target.streams[0].data.size());
assert(mesh.indices.size() == target.indices.size());

target.nodes.insert(target.nodes.end(), mesh.nodes.begin(), mesh.nodes.end());
target.instances.insert(target.instances.end(), mesh.instances.begin(), mesh.instances.end());

mesh.streams.clear();
mesh.indices.clear();
mesh.nodes.clear();
mesh.instances.clear();
}
}
}

void mergeMeshInstances(Mesh& mesh)
{
if (mesh.nodes.empty())
Expand Down
40 changes: 36 additions & 4 deletions gltf/write.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1046,24 +1046,56 @@ void writeMeshAttributes(std::string& json, std::vector<BufferView>& views, std:
}
}

size_t writeMeshIndices(std::vector<BufferView>& views, std::string& json_accessors, size_t& accr_offset, const Mesh& mesh, const Settings& settings)
size_t writeMeshIndices(std::vector<BufferView>& views, std::string& json_accessors, size_t& accr_offset, const std::vector<unsigned int>& indices, cgltf_primitive_type type, const Settings& settings)
{
std::string scratch;
StreamFormat format = writeIndexStream(scratch, mesh.indices);
BufferView::Compression compression = settings.compress ? (mesh.type == cgltf_primitive_type_triangles ? BufferView::Compression_Index : BufferView::Compression_IndexSequence) : BufferView::Compression_None;
StreamFormat format = writeIndexStream(scratch, indices);
BufferView::Compression compression = settings.compress ? (type == cgltf_primitive_type_triangles ? BufferView::Compression_Index : BufferView::Compression_IndexSequence) : BufferView::Compression_None;

size_t view = getBufferView(views, BufferView::Kind_Index, StreamFormat::Filter_None, compression, format.stride);
size_t offset = views[view].data.size();
views[view].data += scratch;

comma(json_accessors);
writeAccessor(json_accessors, view, offset, format.type, format.component_type, format.normalized, mesh.indices.size());
writeAccessor(json_accessors, view, offset, format.type, format.component_type, format.normalized, indices.size());

size_t index_accr = accr_offset++;

return index_accr;
}

void writeMeshGeometry(std::string& json, std::vector<BufferView>& views, std::string& json_accessors, size_t& accr_offset, const Mesh& mesh, const QuantizationPosition& qp, const QuantizationTexture& qt, const Settings& settings)
{
append(json, "{\"attributes\":{");
writeMeshAttributes(json, views, json_accessors, accr_offset, mesh, 0, qp, qt, settings);
append(json, "}");
if (mesh.type != cgltf_primitive_type_triangles)
{
append(json, ",\"mode\":");
append(json, size_t(mesh.type - cgltf_primitive_type_points));
}
if (mesh.targets)
{
append(json, ",\"targets\":[");
for (size_t j = 0; j < mesh.targets; ++j)
{
comma(json);
append(json, "{");
writeMeshAttributes(json, views, json_accessors, accr_offset, mesh, int(1 + j), qp, qt, settings);
append(json, "}");
}
append(json, "]");
}

if (!mesh.indices.empty())
{
size_t index_accr = writeMeshIndices(views, json_accessors, accr_offset, mesh.indices, mesh.type, settings);

append(json, ",\"indices\":");
append(json, index_accr);
}
}

static size_t writeAnimationTime(std::vector<BufferView>& views, std::string& json_accessors, size_t& accr_offset, float mint, int frames, float period, const Settings& settings)
{
std::vector<float> time(frames);
Expand Down

0 comments on commit 2df0a25

Please sign in to comment.