OpenGL: Monotonic UBO allocator (#1911)
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Bart Louwers <bart@emeel.net>
This commit is contained in:
mwilsnd
518
src/mbgl/gl/buffer_allocator.cpp
Normal file
518
src/mbgl/gl/buffer_allocator.cpp
Normal file
@@ -0,0 +1,518 @@
|
||||
#include <mbgl/gl/buffer_allocator.hpp>
|
||||
#include <mbgl/gl/defines.hpp>
|
||||
#include <mbgl/gl/context.hpp>
|
||||
#include <mbgl/gl/uniform_buffer_gl.hpp>
|
||||
#include <utility>
|
||||
|
||||
namespace mbgl {
|
||||
|
||||
using namespace platform;
|
||||
|
||||
namespace gl {
|
||||
|
||||
/// @brief BufferAllocator is a semi-generic allocation strategy for uniform buffer objects.
|
||||
/// This allocator works by streaming buffer writes into sub-allocated sections of actual UBOs.
|
||||
/// As UBOs fill up, they are marked 'in-flight' and placed in a waiting area for references to
|
||||
/// be released. Once all references to a UBO are gone (Or the buffer is defragmented), the
|
||||
/// buffer is recycled.
|
||||
/// @tparam OwnerClass The class type having allocations managed by this allocator.
|
||||
/// @tparam type GL_UNIFORM_BUFFER, no other type is currently valid.
|
||||
/// @tparam PageSizeKB Size of underlying UBO allocations, in KB. 8KB has been observed to work well.
|
||||
/// @tparam MaxFragmentationOccupancy Buffers under this percentage of occupancy are considered fragmented
|
||||
/// and the defragmentation routine will attempt to relocate living references to them to other buffers.
|
||||
/// @tparam MaxFreeBuffers When buffers are made free for recycling, buffers in excess of this amount are
|
||||
/// released to reduce memory consumption.
|
||||
/// @tparam InitialBufferSize Initial size of the reference list. 128 has been observed to work well.
|
||||
template <typename OwnerClass, // UniformBufferGL
|
||||
GLenum type, // GL_UNIFORM_BUFFER
|
||||
size_t PageSizeKB = 8,
|
||||
size_t MaxFragmentationOccupancy = 10,
|
||||
size_t MaxFreeBuffers = 48,
|
||||
size_t InitialBufferSize = 256>
|
||||
class BufferAllocator : public IBufferAllocator {
|
||||
static_assert(type == GL_UNIFORM_BUFFER);
|
||||
|
||||
public:
|
||||
static constexpr const size_t PageSize = 1024 * PageSizeKB;
|
||||
static constexpr const size_t FragmentedOccupancyLevel = static_cast<size_t>(
|
||||
PageSize * (static_cast<double>(MaxFragmentationOccupancy) / 100.0));
|
||||
using BufferTy =
|
||||
BufferAllocator<OwnerClass, type, PageSizeKB, MaxFragmentationOccupancy, MaxFreeBuffers, InitialBufferSize>;
|
||||
using RefTy = TypedBufferRef<OwnerClass>;
|
||||
|
||||
~BufferAllocator() override = default;
|
||||
|
||||
private:
|
||||
/// @brief A buffer in-flight is one that is being used by the GPU. Such buffers become free
|
||||
/// once their `frameSync` fence becomes signaled.
|
||||
struct InFlightBuffer {
|
||||
size_t bufferIndex; // Index into buffers list
|
||||
std::shared_ptr<gl::Fence> frameSync;
|
||||
|
||||
InFlightBuffer(size_t buf, std::shared_ptr<gl::Fence> sync)
|
||||
: bufferIndex(buf),
|
||||
frameSync(std::move(sync)) {}
|
||||
};
|
||||
|
||||
public:
|
||||
/// @brief An allocated buffer the allocator manages. These buffers are sub-allocated
|
||||
/// in monotonic fashion. Once a buffer's capacity has been reached, it is stored until
|
||||
/// it can be recycled and a new buffer is acquired to replace it.
|
||||
struct Buffer {
|
||||
BufferTy& allocator;
|
||||
|
||||
// The pointer points to the next unused region of memory in the buffer
|
||||
ptrdiff_t pointer = 0;
|
||||
|
||||
// References to buffer objects allocating from this block of memory.
|
||||
// For performance reasons, we use a vector here. To deal with vector storage reallocation,
|
||||
// vector resizes are handled explicitly and Refs are made aware of relocation events.
|
||||
std::vector<RefTy> refs;
|
||||
|
||||
// Backing GL object for this buffer
|
||||
GLuint id = 0;
|
||||
|
||||
// Tombstones indicating a removed ref. When a Ref is removed, a tombstone is added to this
|
||||
// counter. This is much faster vs. erasing from our vector. This works because we enforce
|
||||
// a monotonic allocation scheme on our buffers and recycle the whole buffer at once.
|
||||
size_t tombstones = 0;
|
||||
|
||||
// Used to drive defragmentation, occupancy is a measure of the number of bytes held by
|
||||
// living references
|
||||
size_t occupancyBytes = 0;
|
||||
|
||||
// We need to know the index to ourselves in the main buffer list
|
||||
size_t bufferIndex = 0;
|
||||
|
||||
// If true, the buffer's memory has been released to compact memory
|
||||
bool reclaimed = false;
|
||||
|
||||
Buffer(BufferAllocator& allocator_)
|
||||
: allocator(allocator_) {
|
||||
refs.reserve(InitialBufferSize);
|
||||
MBGL_CHECK_ERROR(glGenBuffers(1, &id));
|
||||
MBGL_CHECK_ERROR(glBindBuffer(type, id));
|
||||
MBGL_CHECK_ERROR(glBufferData(type, PageSize, nullptr, GL_DYNAMIC_DRAW));
|
||||
}
|
||||
|
||||
~Buffer() {
|
||||
if (id != 0) {
|
||||
glDeleteBuffers(1, &id);
|
||||
id = 0;
|
||||
}
|
||||
}
|
||||
|
||||
void reclaim() {
|
||||
refs.clear();
|
||||
refs = decltype(refs)();
|
||||
|
||||
if (id != 0) {
|
||||
glDeleteBuffers(1, &id);
|
||||
id = 0;
|
||||
}
|
||||
}
|
||||
|
||||
Buffer(const Buffer&) = delete;
|
||||
Buffer(Buffer&& rhs) noexcept
|
||||
: allocator(rhs.allocator),
|
||||
pointer(rhs.pointer),
|
||||
refs(std::move(rhs.refs)),
|
||||
id(rhs.id),
|
||||
tombstones(rhs.tombstones),
|
||||
occupancyBytes(rhs.occupancyBytes),
|
||||
bufferIndex(rhs.bufferIndex) {
|
||||
rhs.id = 0;
|
||||
}
|
||||
|
||||
Buffer& operator=(const Buffer&) = delete;
|
||||
Buffer& operator=(Buffer&& rhs) noexcept {
|
||||
allocator = rhs.allocator;
|
||||
pointer = rhs.pointer;
|
||||
refs = std::move(rhs.refs);
|
||||
id = rhs.id;
|
||||
rhs.id = 0;
|
||||
tombstones = rhs.tombstones;
|
||||
occupancyBytes = rhs.occupancyBytes;
|
||||
bufferIndex = rhs.bufferIndex;
|
||||
return *this;
|
||||
}
|
||||
|
||||
void setBufferIndex(size_t index) noexcept { bufferIndex = index; }
|
||||
|
||||
// Reset this buffer for a fresh recording session
|
||||
void reset() {
|
||||
assert(refs.size() - tombstones == 0);
|
||||
pointer = tombstones = occupancyBytes = 0;
|
||||
refs.clear();
|
||||
}
|
||||
|
||||
/// @brief Once the buffer is full, we mark it 'in-flight', storing it in another location
|
||||
/// in the allocator. The defragmentation routine will traverse this location for buffers
|
||||
/// that can be recycled.
|
||||
/// @param index The buffer index that maps to this buffer instance.
|
||||
void markInFlight(size_t index) noexcept { allocator.inFlightBuffers.emplace_back(index); }
|
||||
|
||||
/// @brief Number of living references to this buffer
|
||||
size_t numRefs() const noexcept { return refs.size() - tombstones; }
|
||||
|
||||
/// @brief Add a living reference to a sub-allocated region of this buffer
|
||||
/// @param bufObj Managed instance that owns this sub-allocation
|
||||
/// @param bufPtr Offset into the buffer the allocation begins at
|
||||
/// @param size_ Size of the allocated region
|
||||
/// @return BufferRef instance
|
||||
RefTy* addRef(OwnerClass* bufObj, ptrdiff_t bufPtr, size_t size_) {
|
||||
// Important: Check if our list needs to reallocate, if so we need to handle it manually
|
||||
// so we can update the references with their new memory locations
|
||||
if (refs.size() == refs.capacity()) {
|
||||
auto oldList = std::exchange(refs, {});
|
||||
refs.reserve(oldList.size() * 2);
|
||||
|
||||
for (size_t i = 0; i < oldList.size(); ++i) {
|
||||
if (!oldList[i].getOwner()) {
|
||||
// Released ref, skip it
|
||||
continue;
|
||||
}
|
||||
|
||||
// Relocate the reference
|
||||
refs.push_back(std::move(oldList[i]));
|
||||
refs.back().getOwner()->getManagedBuffer().relocRef(&refs.back());
|
||||
}
|
||||
|
||||
// Since we dropped released refs, we can clear the tombstone count
|
||||
tombstones = 0;
|
||||
}
|
||||
|
||||
refs.emplace_back(bufObj, bufferIndex, bufPtr, size_);
|
||||
occupancyBytes += size_;
|
||||
return &refs.back();
|
||||
}
|
||||
|
||||
/// @brief Remove a living reference from this buffer.
|
||||
/// @note: Remember buffers are only permitted to grow monotonically. A buffer can only be
|
||||
/// reused once all living references are removed or relocated to other buffers.
|
||||
/// @param bufObj BufferRef to remove
|
||||
void decRef(RefTy* bufObj) {
|
||||
#ifndef NDEBUG
|
||||
assert(bufObj->getOwner());
|
||||
// Sanity check this ref actually belongs
|
||||
auto it = std::find(refs.begin(), refs.end(), *bufObj);
|
||||
if (it != refs.end()) {
|
||||
tombstones++;
|
||||
} else {
|
||||
assert(0);
|
||||
}
|
||||
#else
|
||||
tombstones++;
|
||||
#endif
|
||||
bufObj->setOwner(nullptr);
|
||||
|
||||
assert(occupancyBytes >= bufObj->getSize());
|
||||
occupancyBytes -= bufObj->getSize();
|
||||
}
|
||||
|
||||
// Align a pointer on a set block size
|
||||
static size_t align(size_t ptr, size_t alignment) {
|
||||
if (ptr == alignment) {
|
||||
return ptr;
|
||||
}
|
||||
return ((ptr + alignment) / alignment) * alignment;
|
||||
}
|
||||
};
|
||||
|
||||
public:
|
||||
// Write a block of memory to the recording buffer
|
||||
// Returns the base index of written memory in the buffer
|
||||
bool write(const void* data, size_t size, BufferRef*& residentBuffer) noexcept override {
|
||||
Buffer* buffer = nullptr;
|
||||
if (buffers.size() == 0) {
|
||||
// Query for the alignment we must use
|
||||
int32_t alignment = 0;
|
||||
MBGL_CHECK_ERROR(glGetIntegerv(GL_UNIFORM_BUFFER_OFFSET_ALIGNMENT, &alignment));
|
||||
|
||||
sharedBufferAlignment = static_cast<size_t>(alignment);
|
||||
recordingBuffer = newBuffer();
|
||||
}
|
||||
buffer = &buffers[recordingBuffer];
|
||||
|
||||
const auto alignedSize = Buffer::align(size, sharedBufferAlignment);
|
||||
|
||||
// Reached past the buffer range
|
||||
if ((buffer->pointer + alignedSize) > PageSize) {
|
||||
// Put the buffer in the pending list
|
||||
buffer->markInFlight(recordingBuffer);
|
||||
buffer = nullptr;
|
||||
|
||||
// Look for a free buffer
|
||||
const auto freeIndex = getFreeBuffer();
|
||||
if (freeIndex != -1) {
|
||||
recordingBuffer = freeIndex;
|
||||
} else {
|
||||
// Otherwise, allocate a fresh buffer
|
||||
recordingBuffer = newBuffer();
|
||||
}
|
||||
buffer = &buffers[recordingBuffer];
|
||||
}
|
||||
|
||||
assert(buffer);
|
||||
if (!buffer) {
|
||||
assert(0);
|
||||
return false;
|
||||
}
|
||||
MBGL_CHECK_ERROR(glBindBuffer(type, buffer->id));
|
||||
|
||||
// Map the next available slice of memory
|
||||
auto* buf = MBGL_CHECK_ERROR(
|
||||
glMapBufferRange(type,
|
||||
buffer->pointer,
|
||||
alignedSize,
|
||||
GL_MAP_INVALIDATE_RANGE_BIT | GL_MAP_UNSYNCHRONIZED_BIT | GL_MAP_WRITE_BIT));
|
||||
const auto writtenIndex = buffer->pointer;
|
||||
|
||||
if (buf) {
|
||||
std::memcpy(buf, data, size);
|
||||
MBGL_CHECK_ERROR(glUnmapBuffer(type));
|
||||
|
||||
residentBuffer = buffer->addRef(nullptr, writtenIndex, size);
|
||||
buffer->pointer += alignedSize;
|
||||
} else {
|
||||
assert(0);
|
||||
return false;
|
||||
}
|
||||
|
||||
#ifndef NDEBUG
|
||||
MBGL_CHECK_ERROR(glBindBuffer(type, 0));
|
||||
#endif
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Release a living reference
|
||||
void release(BufferRef* ref) noexcept override {
|
||||
assert(ref);
|
||||
if (!ref) {
|
||||
return;
|
||||
}
|
||||
buffers[ref->getBufferIndex()].decRef(static_cast<RefTy*>(ref));
|
||||
}
|
||||
|
||||
// Look for buffers that either have zero living references or equal or under FragmentationThresh
|
||||
// references. In the latter case, attempt to relocate these allocations to more utilized buffers.
|
||||
void defragment(const std::shared_ptr<gl::Fence>& fence) noexcept override {
|
||||
if (buffers.size() == 0) {
|
||||
return;
|
||||
}
|
||||
intptr_t recycleIndex = recordingBuffer;
|
||||
|
||||
// Phase one: Mark free buffers
|
||||
updateInFlight();
|
||||
|
||||
// Phase two: Consolidate fragmentation
|
||||
MBGL_CHECK_ERROR(glBindBuffer(type, buffers[recycleIndex].id));
|
||||
|
||||
for (auto it = inFlightBuffers.begin(); it != inFlightBuffers.end();) {
|
||||
auto& fragBuffer = buffers[*it];
|
||||
if (fragBuffer.occupancyBytes > FragmentedOccupancyLevel) {
|
||||
++it;
|
||||
continue;
|
||||
}
|
||||
|
||||
// This buffer is fragmented. Move allocations from this buffer and append to fresh ones
|
||||
// so that this one may be recycled.
|
||||
|
||||
// For each ref, append to new buffer and update that ref
|
||||
for (auto refIt = fragBuffer.refs.begin(); refIt != fragBuffer.refs.end();) {
|
||||
if (!refIt->getOwner()) {
|
||||
++refIt;
|
||||
continue; // This is a released reference
|
||||
}
|
||||
|
||||
const auto alignedSize = Buffer::align(refIt->getOwner()->getSize(), sharedBufferAlignment);
|
||||
|
||||
// 2.a: Get recycle buffer
|
||||
if (recycleIndex == -1 || (buffers[recycleIndex].pointer + alignedSize) > PageSize) {
|
||||
if (recycleIndex != -1) {
|
||||
// We filled this one up, mark it in-flight and get a fresh one.
|
||||
buffers[recycleIndex].markInFlight(recycleIndex);
|
||||
}
|
||||
|
||||
// We need to get a new buffer.
|
||||
recycleIndex = recordingBuffer = getFreeBuffer();
|
||||
if (recycleIndex == -1) {
|
||||
// No more buffers available, we're done.
|
||||
break;
|
||||
}
|
||||
|
||||
MBGL_CHECK_ERROR(glBindBuffer(type, buffers[recycleIndex].id));
|
||||
}
|
||||
|
||||
// 2.b: Copy into recycle buffer
|
||||
if (!moveRef(*refIt, recycleIndex, alignedSize)) {
|
||||
// Relocation failure, stop trying to defrag.
|
||||
#ifndef NDEBUG
|
||||
MBGL_CHECK_ERROR(glBindBuffer(GL_UNIFORM_BUFFER, 0));
|
||||
#endif
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// All refs have relocated, clear this buffer.
|
||||
if (fragBuffer.numRefs() == 0) {
|
||||
fragBuffer.reset();
|
||||
// Store this buffer in a waiting area until the GPU is done using it
|
||||
waitingFree.emplace_back(fragBuffer.bufferIndex, fence);
|
||||
it = inFlightBuffers.erase(it);
|
||||
} else {
|
||||
// We only managed a partial (or no) defrag. Do nothing with this buffer.
|
||||
++it;
|
||||
}
|
||||
|
||||
// No more buffers to write into, stop.
|
||||
if (recycleIndex == -1) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
#ifndef NDEBUG
|
||||
MBGL_CHECK_ERROR(glBindBuffer(GL_UNIFORM_BUFFER, 0));
|
||||
#endif
|
||||
|
||||
if (recordingBuffer == -1) {
|
||||
recordingBuffer = newBuffer();
|
||||
}
|
||||
}
|
||||
|
||||
size_t pageSize() const noexcept override { return PageSize; }
|
||||
|
||||
int32_t getBufferID(size_t bufferIndex) const noexcept override { return buffers[bufferIndex].id; }
|
||||
|
||||
private:
|
||||
size_t newBuffer() noexcept {
|
||||
buffers.emplace_back(*this);
|
||||
const auto index = recordingBuffer = buffers.size() - 1;
|
||||
buffers[index].setBufferIndex(index);
|
||||
return index;
|
||||
}
|
||||
|
||||
intptr_t getFreeBuffer() {
|
||||
if (freeList.size() > 0) {
|
||||
assert(buffers[freeList.back()].pointer == 0);
|
||||
const auto index = freeList.back();
|
||||
freeList.pop_back();
|
||||
return index;
|
||||
} else if (reclaimedList.size() > 0) {
|
||||
const auto index = reclaimedList.back();
|
||||
reclaimedList.pop_back();
|
||||
new (&buffers[index]) Buffer(*this);
|
||||
buffers[index].setBufferIndex(index);
|
||||
return index;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
// Check waiting in-flight buffers and move them to the free list
|
||||
void updateInFlight() {
|
||||
for (auto it = waitingFree.begin(); it != waitingFree.end();) {
|
||||
if (it->frameSync->isSignaled()) {
|
||||
freeList.push_back(it->bufferIndex);
|
||||
it = waitingFree.erase(it);
|
||||
} else {
|
||||
++it;
|
||||
}
|
||||
}
|
||||
|
||||
// And check the free list for an excess of buffers, reclaim if we need to
|
||||
while (freeList.size() > MaxFreeBuffers) {
|
||||
reclaimedList.push_back(freeList.back());
|
||||
freeList.pop_back();
|
||||
buffers[reclaimedList.back()].reclaim();
|
||||
}
|
||||
}
|
||||
|
||||
// Move a buffer reference (An allocation from a UniformBufferGL instance) to a new buffer
|
||||
bool moveRef(RefTy& ref, size_t toIndex, size_t alignedSize) {
|
||||
auto& destBuffer = buffers[toIndex];
|
||||
const auto recycledWriteIndex = destBuffer.pointer;
|
||||
|
||||
auto* buf = MBGL_CHECK_ERROR(
|
||||
glMapBufferRange(type,
|
||||
recycledWriteIndex,
|
||||
alignedSize,
|
||||
GL_MAP_INVALIDATE_RANGE_BIT | GL_MAP_UNSYNCHRONIZED_BIT | GL_MAP_WRITE_BIT));
|
||||
|
||||
if (buf) {
|
||||
std::memcpy(buf, ref.getOwner()->getManagedBuffer().getContents().data(), ref.getOwner()->getSize());
|
||||
MBGL_CHECK_ERROR(glUnmapBuffer(type));
|
||||
destBuffer.pointer += alignedSize;
|
||||
} else {
|
||||
assert(0);
|
||||
return false;
|
||||
}
|
||||
|
||||
// 2.c: Now the ref must be made aware of the relocation of its contents.
|
||||
// Note that this means defragmentation can never run on refs that are
|
||||
// expected to remain bound after defragmentation. They must be re-bound
|
||||
// to become valid again.
|
||||
auto owner = ref.getOwner();
|
||||
const auto refSize = ref.getSize();
|
||||
const auto oldIndex = ref.getBufferIndex();
|
||||
|
||||
const auto newRef = destBuffer.addRef(owner, recycledWriteIndex, refSize);
|
||||
buffers[oldIndex].decRef(&ref);
|
||||
owner->getManagedBuffer().relocRef(newRef);
|
||||
return true;
|
||||
}
|
||||
|
||||
private:
|
||||
// GL requires us to satisfy this alignment for sub-allocations in our buffers
|
||||
size_t sharedBufferAlignment = 0;
|
||||
// Our active recording buffer
|
||||
intptr_t recordingBuffer = 0;
|
||||
|
||||
// Buffers free for re-use
|
||||
std::vector<size_t> freeList;
|
||||
// Buffers reclaimed that can be recreated if needed
|
||||
std::vector<size_t> reclaimedList;
|
||||
// Buffers recorded and currently in-use
|
||||
std::list<size_t> inFlightBuffers;
|
||||
// When we recycle a buffer, we copy from an in-flight buffer to a fresh one.
|
||||
// Before we can convert the in-flight buffer to a free one however, we must wait
|
||||
// after the copy operation(s) to ensure the GPU is done with it before we reuse it.
|
||||
std::list<InFlightBuffer> waitingFree;
|
||||
|
||||
public:
|
||||
// All buffers allocated so far
|
||||
std::vector<Buffer> buffers;
|
||||
|
||||
friend Buffer;
|
||||
};
|
||||
|
||||
class UniformBufferAllocator::Impl : public gl::BufferAllocator<gl::UniformBufferGL, GL_UNIFORM_BUFFER> {};
|
||||
|
||||
UniformBufferAllocator::~UniformBufferAllocator() = default;
|
||||
|
||||
UniformBufferAllocator::UniformBufferAllocator() {
|
||||
impl = std::make_unique<UniformBufferAllocator::Impl>();
|
||||
}
|
||||
|
||||
bool UniformBufferAllocator::write(const void* data, size_t size, BufferRef*& ref) noexcept {
|
||||
return impl->write(data, size, ref);
|
||||
}
|
||||
|
||||
void UniformBufferAllocator::release(BufferRef* ref) noexcept {
|
||||
impl->release(ref);
|
||||
}
|
||||
|
||||
void UniformBufferAllocator::defragment(const std::shared_ptr<gl::Fence>& fence) noexcept {
|
||||
impl->defragment(fence);
|
||||
}
|
||||
|
||||
size_t UniformBufferAllocator::pageSize() const noexcept {
|
||||
return impl->pageSize();
|
||||
}
|
||||
|
||||
int32_t UniformBufferAllocator::getBufferID(size_t bufferIndex) const noexcept {
|
||||
return impl->getBufferID(bufferIndex);
|
||||
}
|
||||
|
||||
} // namespace gl
|
||||
} // namespace mbgl
|
||||
Reference in New Issue
Block a user