#include #include #include #include #include #include namespace mbgl { using namespace platform; namespace gl { /// @brief BufferAllocator is a semi-generic allocation strategy for uniform buffer objects. /// This allocator works by streaming buffer writes into sub-allocated sections of actual UBOs. /// As UBOs fill up, they are marked 'in-flight' and placed in a waiting area for references to /// be released. Once all references to a UBO are gone (Or the buffer is defragmented), the /// buffer is recycled. /// @tparam OwnerClass The class type having allocations managed by this allocator. /// @tparam type GL_UNIFORM_BUFFER, no other type is currently valid. /// @tparam PageSizeKB Size of underlying UBO allocations, in KB. 8KB has been observed to work well. /// @tparam MaxFragmentationOccupancy Buffers under this percentage of occupancy are considered fragmented /// and the defragmentation routine will attempt to relocate living references to them to other buffers. /// @tparam MaxFreeBuffers When buffers are made free for recycling, buffers in excess of this amount are /// released to reduce memory consumption. /// @tparam InitialBufferSize Initial size of the reference list. 128 has been observed to work well. template class BufferAllocator : public IBufferAllocator { static_assert(type == GL_UNIFORM_BUFFER); public: static constexpr const size_t PageSize = 1024 * PageSizeKB; static constexpr const size_t FragmentedOccupancyLevel = static_cast( PageSize * (static_cast(MaxFragmentationOccupancy) / 100.0)); using BufferTy = BufferAllocator; using RefTy = TypedBufferRef; ~BufferAllocator() override = default; private: /// @brief A buffer in-flight is one that is being used by the GPU. Such buffers become free /// once their `frameSync` fence becomes signaled. struct InFlightBuffer { size_t bufferIndex; // Index into buffers list std::shared_ptr frameSync; InFlightBuffer(size_t buf, std::shared_ptr sync) : bufferIndex(buf), frameSync(std::move(sync)) {} }; public: /// @brief An allocated buffer the allocator manages. These buffers are sub-allocated /// in monotonic fashion. Once a buffer's capacity has been reached, it is stored until /// it can be recycled and a new buffer is acquired to replace it. struct Buffer { BufferTy& allocator; // The pointer points to the next unused region of memory in the buffer ptrdiff_t pointer = 0; // References to buffer objects allocating from this block of memory. // For performance reasons, we use a vector here. To deal with vector storage reallocation, // vector resizes are handled explicitly and Refs are made aware of relocation events. std::vector refs; // Backing GL object for this buffer GLuint id = 0; // Tombstones indicating a removed ref. When a Ref is removed, a tombstone is added to this // counter. This is much faster vs. erasing from our vector. This works because we enforce // a monotonic allocation scheme on our buffers and recycle the whole buffer at once. size_t tombstones = 0; // Used to drive defragmentation, occupancy is a measure of the number of bytes held by // living references size_t occupancyBytes = 0; // We need to know the index to ourselves in the main buffer list size_t bufferIndex = 0; // If true, the buffer's memory has been released to compact memory bool reclaimed = false; Buffer(BufferAllocator& allocator_) : allocator(allocator_) { refs.reserve(InitialBufferSize); MBGL_CHECK_ERROR(glGenBuffers(1, &id)); MBGL_CHECK_ERROR(glBindBuffer(type, id)); MBGL_CHECK_ERROR(glBufferData(type, PageSize, nullptr, GL_DYNAMIC_DRAW)); } ~Buffer() { if (id != 0) { glDeleteBuffers(1, &id); id = 0; } } void reclaim() { refs.clear(); refs = decltype(refs)(); if (id != 0) { glDeleteBuffers(1, &id); id = 0; } } Buffer(const Buffer&) = delete; Buffer(Buffer&& rhs) noexcept : allocator(rhs.allocator), pointer(rhs.pointer), refs(std::move(rhs.refs)), id(rhs.id), tombstones(rhs.tombstones), occupancyBytes(rhs.occupancyBytes), bufferIndex(rhs.bufferIndex) { rhs.id = 0; } Buffer& operator=(const Buffer&) = delete; Buffer& operator=(Buffer&& rhs) noexcept { allocator = rhs.allocator; pointer = rhs.pointer; refs = std::move(rhs.refs); id = rhs.id; rhs.id = 0; tombstones = rhs.tombstones; occupancyBytes = rhs.occupancyBytes; bufferIndex = rhs.bufferIndex; return *this; } void setBufferIndex(size_t index) noexcept { bufferIndex = index; } // Reset this buffer for a fresh recording session void reset() { assert(refs.size() - tombstones == 0); pointer = tombstones = occupancyBytes = 0; refs.clear(); } /// @brief Once the buffer is full, we mark it 'in-flight', storing it in another location /// in the allocator. The defragmentation routine will traverse this location for buffers /// that can be recycled. /// @param index The buffer index that maps to this buffer instance. void markInFlight(size_t index) noexcept { allocator.inFlightBuffers.emplace_back(index); } /// @brief Number of living references to this buffer size_t numRefs() const noexcept { return refs.size() - tombstones; } /// @brief Add a living reference to a sub-allocated region of this buffer /// @param bufObj Managed instance that owns this sub-allocation /// @param bufPtr Offset into the buffer the allocation begins at /// @param size_ Size of the allocated region /// @return BufferRef instance RefTy* addRef(OwnerClass* bufObj, ptrdiff_t bufPtr, size_t size_) { // Important: Check if our list needs to reallocate, if so we need to handle it manually // so we can update the references with their new memory locations if (refs.size() == refs.capacity()) { auto oldList = std::exchange(refs, {}); refs.reserve(oldList.size() * 2); for (size_t i = 0; i < oldList.size(); ++i) { if (!oldList[i].getOwner()) { // Released ref, skip it continue; } // Relocate the reference refs.push_back(std::move(oldList[i])); refs.back().getOwner()->getManagedBuffer().relocRef(&refs.back()); } // Since we dropped released refs, we can clear the tombstone count tombstones = 0; } refs.emplace_back(bufObj, bufferIndex, bufPtr, size_); occupancyBytes += size_; return &refs.back(); } /// @brief Remove a living reference from this buffer. /// @note: Remember buffers are only permitted to grow monotonically. A buffer can only be /// reused once all living references are removed or relocated to other buffers. /// @param bufObj BufferRef to remove void decRef(RefTy* bufObj) { #ifndef NDEBUG assert(bufObj->getOwner()); // Sanity check this ref actually belongs auto it = std::find(refs.begin(), refs.end(), *bufObj); if (it != refs.end()) { tombstones++; } else { assert(0); } #else tombstones++; #endif bufObj->setOwner(nullptr); assert(occupancyBytes >= bufObj->getSize()); occupancyBytes -= bufObj->getSize(); } // Align a pointer on a set block size static size_t align(size_t ptr, size_t alignment) { if (ptr == alignment) { return ptr; } return ((ptr + alignment) / alignment) * alignment; } }; public: // Write a block of memory to the recording buffer // Returns the base index of written memory in the buffer bool write(const void* data, size_t size, BufferRef*& residentBuffer) noexcept override { Buffer* buffer = nullptr; if (buffers.size() == 0) { // Query for the alignment we must use int32_t alignment = 0; MBGL_CHECK_ERROR(glGetIntegerv(GL_UNIFORM_BUFFER_OFFSET_ALIGNMENT, &alignment)); sharedBufferAlignment = static_cast(alignment); recordingBuffer = newBuffer(); } buffer = &buffers[recordingBuffer]; const auto alignedSize = Buffer::align(size, sharedBufferAlignment); // Reached past the buffer range if ((buffer->pointer + alignedSize) > PageSize) { // Put the buffer in the pending list buffer->markInFlight(recordingBuffer); buffer = nullptr; // Look for a free buffer const auto freeIndex = getFreeBuffer(); if (freeIndex != -1) { recordingBuffer = freeIndex; } else { // Otherwise, allocate a fresh buffer recordingBuffer = newBuffer(); } buffer = &buffers[recordingBuffer]; } assert(buffer); if (!buffer) { assert(0); return false; } MBGL_CHECK_ERROR(glBindBuffer(type, buffer->id)); // Map the next available slice of memory auto* buf = MBGL_CHECK_ERROR( glMapBufferRange(type, buffer->pointer, alignedSize, GL_MAP_INVALIDATE_RANGE_BIT | GL_MAP_UNSYNCHRONIZED_BIT | GL_MAP_WRITE_BIT)); const auto writtenIndex = buffer->pointer; if (buf) { std::memcpy(buf, data, size); MBGL_CHECK_ERROR(glUnmapBuffer(type)); residentBuffer = buffer->addRef(nullptr, writtenIndex, size); buffer->pointer += alignedSize; } else { assert(0); return false; } #ifndef NDEBUG MBGL_CHECK_ERROR(glBindBuffer(type, 0)); #endif return true; } /// Release a living reference void release(BufferRef* ref) noexcept override { assert(ref); if (!ref) { return; } buffers[ref->getBufferIndex()].decRef(static_cast(ref)); } // Look for buffers that either have zero living references or equal or under FragmentationThresh // references. In the latter case, attempt to relocate these allocations to more utilized buffers. void defragment(const std::shared_ptr& fence) override { MLN_TRACE_FUNC(); if (buffers.size() == 0) { return; } intptr_t recycleIndex = recordingBuffer; // Phase one: Mark free buffers updateInFlight(); // Phase two: Consolidate fragmentation MBGL_CHECK_ERROR(glBindBuffer(type, buffers[recycleIndex].id)); for (auto it = inFlightBuffers.begin(); it != inFlightBuffers.end();) { auto& fragBuffer = buffers[*it]; if (fragBuffer.occupancyBytes > FragmentedOccupancyLevel) { ++it; continue; } // This buffer is fragmented. Move allocations from this buffer and append to fresh ones // so that this one may be recycled. // For each ref, append to new buffer and update that ref for (auto refIt = fragBuffer.refs.begin(); refIt != fragBuffer.refs.end();) { if (!refIt->getOwner()) { ++refIt; continue; // This is a released reference } const auto alignedSize = Buffer::align(refIt->getOwner()->getSize(), sharedBufferAlignment); // 2.a: Get recycle buffer if (recycleIndex == -1 || (buffers[recycleIndex].pointer + alignedSize) > PageSize) { if (recycleIndex != -1) { // We filled this one up, mark it in-flight and get a fresh one. buffers[recycleIndex].markInFlight(recycleIndex); } // We need to get a new buffer. recycleIndex = recordingBuffer = getFreeBuffer(); if (recycleIndex == -1) { // No more buffers available, we're done. break; } MBGL_CHECK_ERROR(glBindBuffer(type, buffers[recycleIndex].id)); } // 2.b: Copy into recycle buffer if (!moveRef(*refIt, recycleIndex, alignedSize)) { // Relocation failure, stop trying to defrag. #ifndef NDEBUG MBGL_CHECK_ERROR(glBindBuffer(GL_UNIFORM_BUFFER, 0)); #endif return; } } // All refs have relocated, clear this buffer. if (fragBuffer.numRefs() == 0) { fragBuffer.reset(); // Store this buffer in a waiting area until the GPU is done using it waitingFree.emplace_back(fragBuffer.bufferIndex, fence); it = inFlightBuffers.erase(it); } else { // We only managed a partial (or no) defrag. Do nothing with this buffer. ++it; } // No more buffers to write into, stop. if (recycleIndex == -1) { break; } } #ifndef NDEBUG MBGL_CHECK_ERROR(glBindBuffer(GL_UNIFORM_BUFFER, 0)); #endif if (recordingBuffer == -1) { recordingBuffer = newBuffer(); } } size_t pageSize() const noexcept override { return PageSize; } int32_t getBufferID(size_t bufferIndex) const noexcept override { return buffers[bufferIndex].id; } private: size_t newBuffer() noexcept { buffers.emplace_back(*this); const auto index = recordingBuffer = buffers.size() - 1; buffers[index].setBufferIndex(index); return index; } intptr_t getFreeBuffer() { if (freeList.size() > 0) { assert(buffers[freeList.back()].pointer == 0); const auto index = freeList.back(); freeList.pop_back(); return index; } else if (reclaimedList.size() > 0) { const auto index = reclaimedList.back(); reclaimedList.pop_back(); new (&buffers[index]) Buffer(*this); buffers[index].setBufferIndex(index); return index; } return -1; } // Check waiting in-flight buffers and move them to the free list void updateInFlight() { for (auto it = waitingFree.begin(); it != waitingFree.end();) { if (it->frameSync->isSignaled()) { freeList.push_back(it->bufferIndex); it = waitingFree.erase(it); } else { ++it; } } // And check the free list for an excess of buffers, reclaim if we need to while (freeList.size() > MaxFreeBuffers) { reclaimedList.push_back(freeList.back()); freeList.pop_back(); buffers[reclaimedList.back()].reclaim(); } } // Move a buffer reference (An allocation from a UniformBufferGL instance) to a new buffer bool moveRef(RefTy& ref, size_t toIndex, size_t alignedSize) { auto& destBuffer = buffers[toIndex]; const auto recycledWriteIndex = destBuffer.pointer; auto* buf = MBGL_CHECK_ERROR( glMapBufferRange(type, recycledWriteIndex, alignedSize, GL_MAP_INVALIDATE_RANGE_BIT | GL_MAP_UNSYNCHRONIZED_BIT | GL_MAP_WRITE_BIT)); if (buf) { std::memcpy(buf, ref.getOwner()->getManagedBuffer().getContents().data(), ref.getOwner()->getSize()); MBGL_CHECK_ERROR(glUnmapBuffer(type)); destBuffer.pointer += alignedSize; } else { assert(0); return false; } // 2.c: Now the ref must be made aware of the relocation of its contents. // Note that this means defragmentation can never run on refs that are // expected to remain bound after defragmentation. They must be re-bound // to become valid again. auto owner = ref.getOwner(); const auto refSize = ref.getSize(); const auto oldIndex = ref.getBufferIndex(); const auto newRef = destBuffer.addRef(owner, recycledWriteIndex, refSize); buffers[oldIndex].decRef(&ref); owner->getManagedBuffer().relocRef(newRef); return true; } private: // GL requires us to satisfy this alignment for sub-allocations in our buffers size_t sharedBufferAlignment = 0; // Our active recording buffer intptr_t recordingBuffer = 0; // Buffers free for re-use std::vector freeList; // Buffers reclaimed that can be recreated if needed std::vector reclaimedList; // Buffers recorded and currently in-use std::list inFlightBuffers; // When we recycle a buffer, we copy from an in-flight buffer to a fresh one. // Before we can convert the in-flight buffer to a free one however, we must wait // after the copy operation(s) to ensure the GPU is done with it before we reuse it. std::list waitingFree; public: // All buffers allocated so far std::vector buffers; friend Buffer; }; class UniformBufferAllocator::Impl : public gl::BufferAllocator {}; UniformBufferAllocator::~UniformBufferAllocator() = default; UniformBufferAllocator::UniformBufferAllocator() { impl = std::make_unique(); } bool UniformBufferAllocator::write(const void* data, size_t size, BufferRef*& ref) noexcept { return impl->write(data, size, ref); } void UniformBufferAllocator::release(BufferRef* ref) noexcept { impl->release(ref); } void UniformBufferAllocator::defragment(const std::shared_ptr& fence) { impl->defragment(fence); } size_t UniformBufferAllocator::pageSize() const noexcept { return impl->pageSize(); } int32_t UniformBufferAllocator::getBufferID(size_t bufferIndex) const noexcept { return impl->getBufferID(bufferIndex); } } // namespace gl } // namespace mbgl