Skip to content

Commit

Permalink
Batch uploads of meshlet instances, updated glslang version, and fixe…
Browse files Browse the repository at this point in the history 
…d some validation errors.
  • Loading branch information
@JuanDiegoMontoya
JuanDiegoMontoya committed Jul 18, 2024
1 parent 8d78378 commit 8c4ffe8
Show file tree
Hide file tree
Showing 6 changed files with 117 additions and 71 deletions.
2 changes: 1 addition & 1 deletion external/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ option(ENABLE_OPT "" OFF)
FetchContent_Declare(
glslang
GIT_REPOSITORY https://github.com/KhronosGroup/glslang.git
GIT_TAG 14.0.0
GIT_TAG 14.3.0
#SYSTEM
)

Expand Down
164 changes: 106 additions & 58 deletions src/FrogRenderer2.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,16 +209,16 @@ FrogRenderer2::FrogRenderer2(const Application::CreateInfo& createInfo)
mainCamera.position.y = 1;

//scene.ImportLoadedScene(*this, Utility::LoadModelFromFileMeshlet(*device_, "models/simple_scene.glb", glm::scale(glm::vec3{.5})));
//scene.ImportLoadedScene(*this, Utility::LoadModelFromFileMeshlet(*device_, "H:/Repositories/glTF-Sample-Models/downloaded schtuff/cube.glb", glm::scale(glm::vec3{1})));
scene.ImportLoadedScene(*this, Utility::LoadModelFromFileMeshlet(*device_, "H:/Repositories/glTF-Sample-Models/downloaded schtuff/cube.glb", glm::scale(glm::vec3{1})));
//Utility::LoadModelFromFileMeshlet(*device_, scene, "H:\\Repositories\\glTF-Sample-Models\\2.0\\BoomBox\\glTF/BoomBox.gltf", glm::scale(glm::vec3{10.0f}));
//Utility::LoadModelFromFileMeshlet(*device_, scene, "H:/Repositories/glTF-Sample-Models/2.0/Sponza/glTF/Sponza.gltf", glm::scale(glm::vec3{.5}));
//Utility::LoadModelFromFileMeshlet(*device_, scene, "H:/Repositories/glTF-Sample-Models/downloaded schtuff/Main/NewSponza_Main_Blender_glTF.gltf", glm::scale(glm::vec3{1}));
//Utility::LoadModelFromFileMeshlet(*device_, scene, "H:/Repositories/glTF-Sample-Models/downloaded schtuff/bistro_compressed.glb", glm::scale(glm::vec3{.5}));
//scene.ImportLoadedScene(*this, Utility::LoadModelFromFileMeshlet(*device_, "H:/Repositories/glTF-Sample-Models/downloaded schtuff/bistro_compressed.glb", glm::scale(glm::vec3{.5})));
//Utility::LoadModelFromFileMeshlet(*device_, scene, "H:/Repositories/glTF-Sample-Models/downloaded schtuff/sponza_compressed.glb", glm::scale(glm::vec3{1}));
//scene.ImportLoadedScene(*this, Utility::LoadModelFromFileMeshlet(*device_, "H:/Repositories/glTF-Sample-Models/downloaded schtuff/sponza_compressed_tu.glb", glm::scale(glm::vec3{1})));
//Utility::LoadModelFromFileMeshlet(*device_, scene, "H:/Repositories/glTF-Sample-Models/downloaded schtuff/subdiv_deccer_cubes.glb", glm::scale(glm::vec3{1}));
//Utility::LoadModelFromFileMeshlet(*device_, scene, "H:/Repositories/glTF-Sample-Models/downloaded schtuff/SM_Deccer_Cubes_Textured.glb", glm::scale(glm::vec3{1}));
scene.ImportLoadedScene(*this, Utility::LoadModelFromFileMeshlet(*device_, "H:/Repositories/glTF-Sample-Models/downloaded schtuff/small_city.glb", glm::scale(glm::vec3{1})));
//scene.ImportLoadedScene(*this, Utility::LoadModelFromFileMeshlet(*device_, "H:/Repositories/glTF-Sample-Models/downloaded schtuff/small_city.glb", glm::scale(glm::vec3{1})));

meshletIndirectCommand = Fvog::TypedBuffer<Fvog::DrawIndexedIndirectCommand>(*device_, {}, "Meshlet Indirect Command");
cullTrianglesDispatchParams = Fvog::TypedBuffer<Fvog::DispatchIndirectCommand>(*device_, {}, "Cull Triangles Dispatch Params");
Expand Down Expand Up @@ -423,28 +423,6 @@ void FrogRenderer2::OnUpdate([[maybe_unused]] double dt)

shadingUniforms.numberOfLights = NumLights();

// A few of these buffers are really slow (2-3ms) to create and destroy every frame (large ones hit vkAllocateMemory), so
// this scheme is still not ideal as e.g. adding geometry every frame will cause reallocs.
// The current scheme works fine when the scene is mostly static.

// Soft cap of 1 billion indices should prevent oversubscribing memory (on my system) when loading huge scenes.
// This limit should be OK as it only limits post-culling geometry.
const auto maxIndices = glm::min(1'000'000'000u, NumMeshletInstances() * Utility::maxMeshletPrimitives * 3);
if (!instancedMeshletBuffer || instancedMeshletBuffer->Size() < maxIndices)
{
instancedMeshletBuffer = Fvog::TypedBuffer<uint32_t>(*device_, {.count = maxIndices}, "Instanced Meshlets");
}

if (!persistentVisibleMeshletIds || persistentVisibleMeshletIds->Size() < NumMeshletInstances())
{
persistentVisibleMeshletIds = Fvog::TypedBuffer<uint32_t>(*device_, {.count = NumMeshletInstances()}, "Persistent Visible Meshlet IDs");
}

if (!transientVisibleMeshletIds || transientVisibleMeshletIds->Size() < NumMeshletInstances())
{
transientVisibleMeshletIds = Fvog::TypedBuffer<uint32_t>(*device_, {.count = NumMeshletInstances()}, "Transient Visible Meshlet IDs");
}

// TODO: perhaps this logic belongs in Application
if (shouldResizeNextFrame)
{
Expand Down Expand Up @@ -562,6 +540,28 @@ void FrogRenderer2::OnRender([[maybe_unused]] double dt, VkCommandBuffer command

FlushUpdatedSceneData(commandBuffer);

// A few of these buffers are really slow (2-3ms) to create and destroy every frame (large ones hit vkAllocateMemory), so
// this scheme is still not ideal as e.g. adding geometry every frame will cause reallocs.
// The current scheme works fine when the scene is mostly static.

// Soft cap of 1 billion indices should prevent oversubscribing memory (on my system) when loading huge scenes.
// This limit should be OK as it only limits post-culling geometry.
const auto maxIndices = glm::min(1'000'000'000u, NumMeshletInstances() * Utility::maxMeshletPrimitives * 3);
if (!instancedMeshletBuffer || instancedMeshletBuffer->Size() < maxIndices)
{
instancedMeshletBuffer = Fvog::TypedBuffer<uint32_t>(*device_, {.count = maxIndices}, "Instanced Meshlets");
}

if (!persistentVisibleMeshletIds || persistentVisibleMeshletIds->Size() < NumMeshletInstances())
{
persistentVisibleMeshletIds = Fvog::TypedBuffer<uint32_t>(*device_, {.count = std::max(1u, NumMeshletInstances())}, "Persistent Visible Meshlet IDs");
}

if (!transientVisibleMeshletIds || transientVisibleMeshletIds->Size() < NumMeshletInstances())
{
transientVisibleMeshletIds = Fvog::TypedBuffer<uint32_t>(*device_, {.count = std::max(1u, NumMeshletInstances())}, "Transient Visible Meshlet IDs");
}

if (clearDebugAabbsEachFrame)
{
debugGpuAabbsBuffer->FillData(commandBuffer, {.offset = offsetof(Fvog::DrawIndirectCommand, instanceCount), .size = sizeof(uint32_t), .data = 0});
Expand Down Expand Up @@ -1312,47 +1312,18 @@ void FrogRenderer2::UnregisterMeshInstance(Render::MeshInstanceID meshInstance)
meshInstanceInfos.erase(meshInstance.id);
}

Render::MeshID FrogRenderer2::SpawnMesh(Render::MeshInstanceID meshInstance, VkCommandBuffer commandBuffer)
Render::MeshID FrogRenderer2::SpawnMesh(Render::MeshInstanceID meshInstance)
{
ZoneScoped;
auto [meshGeometryId, materialId] = meshInstanceInfos.at(meshInstance.id);
auto& meshletsAlloc = meshGeometryAllocations.at(meshGeometryId.id).meshletsAlloc;
auto& materialAlloc = materialAllocations.at(materialId.id).materialAlloc;

const auto meshletInstanceCount = meshletsAlloc.GetSize() / sizeof(Render::Meshlet);
auto meshletInstances = std::vector<Render::MeshletInstance>();
meshletInstances.reserve(meshletInstanceCount);

auto instanceAlloc = geometryBuffer.Allocate(sizeof(Render::ObjectUniforms), sizeof(Render::ObjectUniforms));

// Spawn a set of meshlet instances referring to the mesh's meshlets, with the correct offsets.
auto baseMeshletIndex = meshletsAlloc.GetOffset() / sizeof(Render::Meshlet);
auto instanceIndex = instanceAlloc.GetOffset() / sizeof(Render::ObjectUniforms);
auto materialIndex = materialAlloc.GetOffset() / sizeof(Render::GpuMaterial);
for (size_t i = 0; i < meshletInstanceCount; i++)
{
meshletInstances.emplace_back(uint32_t(baseMeshletIndex + i), (uint32_t)instanceIndex, (uint32_t)materialIndex);
}

const auto meshletInstancesAlloc = meshletInstancesBuffer.Allocate(std::span(meshletInstances).size_bytes());

meshletInstancesBuffer.GetBuffer().UpdateDataExpensive(commandBuffer, std::span(meshletInstances), meshletInstancesAlloc.offset);

auto myId = nextId++;
meshAllocations.emplace(myId,
MeshAllocs{
.meshletInstancesAlloc = meshletInstancesAlloc,
.instanceAlloc = std::move(instanceAlloc),
});
spawnedMeshes.emplace_back(myId, meshInstance);
return {myId};
}

void FrogRenderer2::DeleteMesh(Render::MeshID mesh, VkCommandBuffer commandBuffer)
void FrogRenderer2::DeleteMesh(Render::MeshID mesh)
{
ZoneScoped;
auto it = meshAllocations.find(mesh.id);
meshletInstancesBuffer.Free(it->second.meshletInstancesAlloc, commandBuffer);
meshAllocations.erase(it);
deletedMeshes.emplace_back(mesh.id);
}

// Having the data payload in the alloc function is kinda quirky and inconsistent, but I'll keep it for now.
Expand Down Expand Up @@ -1420,6 +1391,81 @@ void FrogRenderer2::FlushUpdatedSceneData(VkCommandBuffer commandBuffer)

auto marker = ctx.MakeScopedDebugMarker("Flush updated scene data");

// Deleted meshes
for (auto id : deletedMeshes)
{
auto it = meshAllocations.find(id);
meshletInstancesBuffer.Free(it->second.meshletInstancesAlloc, commandBuffer);
meshAllocations.erase(it);
}

struct MeshletInstancesUpload
{
size_t srcOffset;
size_t dstOffset;
size_t sizeBytes;
};

auto meshletInstancesUploads = std::vector<MeshletInstancesUpload>();
auto meshletInstances = std::vector<Render::MeshletInstance>();
meshletInstancesUploads.reserve(spawnedMeshes.size());

// Spawned meshes
for (const auto& [id, meshInstance] : spawnedMeshes)
{
auto [meshGeometryId, materialId] = meshInstanceInfos.at(meshInstance.id);
const auto& meshletsAlloc = meshGeometryAllocations.at(meshGeometryId.id).meshletsAlloc;
const auto& materialAlloc = materialAllocations.at(materialId.id).materialAlloc;

const auto meshletInstanceCount = meshletsAlloc.GetSize() / sizeof(Render::Meshlet);

auto instanceAlloc = geometryBuffer.Allocate(sizeof(Render::ObjectUniforms), sizeof(Render::ObjectUniforms));

// Spawn a set of meshlet instances referring to the mesh's meshlets, with the correct offsets.
auto baseMeshletIndex = meshletsAlloc.GetOffset() / sizeof(Render::Meshlet);
auto instanceIndex = instanceAlloc.GetOffset() / sizeof(Render::ObjectUniforms);
auto materialIndex = materialAlloc.GetOffset() / sizeof(Render::GpuMaterial);
auto srcOffset = meshletInstances.size() * sizeof(Render::MeshletInstance);
for (size_t i = 0; i < meshletInstanceCount; i++)
{
meshletInstances.emplace_back(uint32_t(baseMeshletIndex + i), (uint32_t)instanceIndex, (uint32_t)materialIndex);
}

const auto meshletInstancesAlloc = meshletInstancesBuffer.Allocate(meshletInstanceCount * sizeof(Render::MeshletInstance));
meshletInstancesUploads.emplace_back(srcOffset, meshletInstancesAlloc.offset, meshletInstancesAlloc.size);

//meshletInstancesBuffer.GetBuffer().UpdateDataExpensive(commandBuffer, std::span(meshletInstances), meshletInstancesAlloc.offset);

meshAllocations.emplace(id,
MeshAllocs{
.meshletInstancesAlloc = meshletInstancesAlloc,
.instanceAlloc = std::move(instanceAlloc),
});
}

// Upload meshlet instances of spawned meshes.
// TODO: This should be a scatter-write compute shader
if (!meshletInstancesUploads.empty())
{
auto uploadBuffer = Fvog::TypedBuffer<Render::MeshletInstance>(*device_,
{
.count = (uint32_t)meshletInstances.size(),
.flag = Fvog::BufferFlagThingy::MAP_SEQUENTIAL_WRITE | Fvog::BufferFlagThingy::NO_DESCRIPTOR,
},
"Meshlet Upload Staging Buffer");

std::memcpy(uploadBuffer.GetMappedMemory(), meshletInstances.data(), meshletInstances.size() * sizeof(Render::MeshletInstance));

for (auto [srcOffset, dstOffset, size] : meshletInstancesUploads)
{
ctx.CopyBuffer(uploadBuffer, meshletInstancesBuffer.GetBuffer(), {
.srcOffset = srcOffset,
.dstOffset = dstOffset,
.size = size,
});
}
}

for (const auto& [id, uniforms] : modifiedMeshUniforms)
{
const auto offset = meshAllocations.at(id).instanceAlloc.GetOffset();
Expand All @@ -1445,4 +1491,6 @@ void FrogRenderer2::FlushUpdatedSceneData(VkCommandBuffer commandBuffer)
modifiedMeshUniforms.clear();
modifiedLights.clear();
modifiedMaterials.clear();
deletedMeshes.clear();
spawnedMeshes.clear();
}
7 changes: 4 additions & 3 deletions src/FrogRenderer2.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,8 +165,8 @@ class FrogRenderer2 final : public Application
[[nodiscard]] Render::MeshInstanceID RegisterMeshInstance(const Render::MeshInstanceInfo& meshInstance);
void UnregisterMeshInstance(Render::MeshInstanceID meshInstance);

[[nodiscard]] Render::MeshID SpawnMesh(Render::MeshInstanceID meshInstance, VkCommandBuffer commandBuffer);
void DeleteMesh(Render::MeshID mesh, VkCommandBuffer commandBuffer);
[[nodiscard]] Render::MeshID SpawnMesh(Render::MeshInstanceID meshInstance);
void DeleteMesh(Render::MeshID mesh);

[[nodiscard]] Render::LightID SpawnLight(const Render::GpuLight& lightData, VkCommandBuffer commandBuffer);
void DeleteLight(Render::LightID light, VkCommandBuffer commandBuffer);
Expand Down Expand Up @@ -467,7 +467,8 @@ class FrogRenderer2 final : public Application
std::unordered_map<uint64_t, Render::ObjectUniforms> modifiedMeshUniforms;
std::unordered_map<uint64_t, Render::GpuLight> modifiedLights;
std::unordered_map<uint64_t, Render::GpuMaterial> modifiedMaterials;
std::vector<Render::MeshInstanceID> spawnedMeshes;
std::vector<std::pair<uint64_t, Render::MeshInstanceID>> spawnedMeshes;
std::vector<uint64_t> deletedMeshes;

void FlushUpdatedSceneData(VkCommandBuffer commandBuffer);

Expand Down
1 change: 1 addition & 0 deletions src/Fvog/Buffer2.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,7 @@ namespace Fvog
ContiguousManagedBuffer::Alloc ContiguousManagedBuffer::Allocate(size_t size)
{
assert(currentSize_ + size <= buffer_.SizeBytes());
assert(size > 0);
const auto alloc = Alloc{currentSize_, size};

currentSize_ += size;
Expand Down
2 changes: 1 addition & 1 deletion src/Fvog/Buffer2.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ namespace Fvog
struct TypedBufferCreateInfo
{
uint32_t count{1};
BufferFlagThingy flag{};
BufferFlags flag{};
};

template<typename T = std::byte>
Expand Down
12 changes: 4 additions & 8 deletions src/Scene.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,14 +86,10 @@ namespace Scene
.meshGeometry = meshGeometryIds[baseMeshGeometryIndex + meshIndex],
.material = materialIds[baseMaterialIndex + materialIndex],
}));

// TODO: HACK
renderer.GetDevice().ImmediateSubmit([&](VkCommandBuffer cmd)
{
auto meshId = meshIds.emplace_back(renderer.SpawnMesh(meshInstanceId, cmd));
// TODO: make a new node instead of putting a bunch of meshes on one node (or not, honestly this is fine)
newNode->meshIds.push_back(meshId);
});

auto meshId = meshIds.emplace_back(renderer.SpawnMesh(meshInstanceId));
// TODO: make a new node instead of putting a bunch of meshes on one node (or not, honestly this is fine)
newNode->meshIds.push_back(meshId);
}

if (node->light)
Expand Down

0 comments on commit 8c4ffe8

Please sign in to comment.