diff --git a/CMakeLists.txt b/CMakeLists.txt index 95610e261..b8df248e3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -159,8 +159,8 @@ if(MLN_DRAWABLE_RENDERER) ${PROJECT_SOURCE_DIR}/include/mbgl/shaders/shader_program_base.hpp ${PROJECT_SOURCE_DIR}/include/mbgl/util/identity.hpp ${PROJECT_SOURCE_DIR}/include/mbgl/util/suppress_copies.hpp - ${PROJECT_SOURCE_DIR}/include/mbgl/shaders/gl/shader_program_gl.hpp + ${PROJECT_SOURCE_DIR}/include/mbgl/gl/buffer_allocator.hpp ${PROJECT_SOURCE_DIR}/include/mbgl/gl/drawable_gl.hpp ${PROJECT_SOURCE_DIR}/include/mbgl/gl/drawable_gl_builder.hpp ${PROJECT_SOURCE_DIR}/include/mbgl/gl/layer_group_gl.hpp @@ -216,6 +216,7 @@ if(MLN_DRAWABLE_RENDERER) ${PROJECT_SOURCE_DIR}/src/mbgl/shaders/shader_program_base.cpp ${PROJECT_SOURCE_DIR}/src/mbgl/util/identity.cpp ${PROJECT_SOURCE_DIR}/src/mbgl/shaders/gl/shader_program_gl.cpp + ${PROJECT_SOURCE_DIR}/src/mbgl/gl/buffer_allocator.cpp ${PROJECT_SOURCE_DIR}/src/mbgl/gl/drawable_gl.cpp ${PROJECT_SOURCE_DIR}/src/mbgl/gl/drawable_gl_builder.cpp ${PROJECT_SOURCE_DIR}/src/mbgl/gl/drawable_gl_impl.hpp @@ -1067,6 +1068,8 @@ if(MLN_WITH_OPENGL) ${PROJECT_SOURCE_DIR}/src/mbgl/gl/command_encoder.hpp ${PROJECT_SOURCE_DIR}/src/mbgl/gl/context.cpp ${PROJECT_SOURCE_DIR}/src/mbgl/gl/context.hpp + ${PROJECT_SOURCE_DIR}/src/mbgl/gl/fence.cpp + ${PROJECT_SOURCE_DIR}/src/mbgl/gl/fence.hpp ${PROJECT_SOURCE_DIR}/src/mbgl/style/layers/custom_layer.cpp ${PROJECT_SOURCE_DIR}/src/mbgl/layermanager/custom_layer_factory.cpp ${PROJECT_SOURCE_DIR}/src/mbgl/style/layers/custom_layer_impl.cpp diff --git a/bazel/core.bzl b/bazel/core.bzl index 8b5f2c4b5..21341a2da 100644 --- a/bazel/core.bzl +++ b/bazel/core.bzl @@ -854,6 +854,8 @@ MLN_OPENGL_SOURCE = [ "src/mbgl/gl/command_encoder.hpp", "src/mbgl/gl/context.cpp", "src/mbgl/gl/context.hpp", + "src/mbgl/gl/fence.cpp", + "src/mbgl/gl/fence.hpp", "src/mbgl/gl/debugging_extension.cpp", "src/mbgl/gl/debugging_extension.hpp", "src/mbgl/gl/defines.hpp", @@ -995,6 +997,7 @@ MLN_DRAWABLES_HEADERS = [ ] MLN_DRAWABLES_GL_SOURCE = [ + "src/mbgl/gl/buffer_allocator.cpp", "src/mbgl/gl/drawable_gl.cpp", "src/mbgl/gl/drawable_gl_builder.cpp", "src/mbgl/gl/drawable_gl_impl.hpp", @@ -1007,6 +1010,7 @@ MLN_DRAWABLES_GL_SOURCE = [ ] MLN_DRAWABLES_GL_HEADERS = [ + "include/mbgl/gl/buffer_allocator.hpp", "include/mbgl/gl/drawable_gl.hpp", "include/mbgl/gl/drawable_gl_builder.hpp", "include/mbgl/gl/layer_group_gl.hpp", diff --git a/include/mbgl/gl/buffer_allocator.hpp b/include/mbgl/gl/buffer_allocator.hpp new file mode 100644 index 000000000..246a0f4ff --- /dev/null +++ b/include/mbgl/gl/buffer_allocator.hpp @@ -0,0 +1,177 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace mbgl { +namespace gl { + +class Context; +class Fence; + +class BufferRef { +public: + BufferRef(size_t pointer, size_t offset, size_t size_) + : bufferIndex(pointer), + bufferOffset(offset), + size(size_) {} + + // Size of our allocation + size_t getSize() const noexcept { return size; } + // Index into the array of buffers indicating our current buffer + size_t getBufferIndex() const noexcept { return bufferIndex; } + // Offset into the buffer where our allocation begins + intptr_t getBufferOffset() const noexcept { return bufferOffset; } + +private: + size_t bufferIndex = 0; + intptr_t bufferOffset = 0; + size_t size = 0; +}; + +/// @brief IBufferAllocator hides the underlying implementation of the buffer allocator scheme used. +class IBufferAllocator { +public: + virtual ~IBufferAllocator(){}; + + /// @brief Write data into a buffer managed by the allocator + /// @param data Pointer to data to write into the buffer + /// @param size Size in bytes of contents to copy from `data` + /// @param ref Buffer reference keeping the new allocation alive. + /// Note: The concrete type of BufferRef is assumed based on the class type of + /// IBufferAllocator. + /// @return False on allocation failure. + virtual bool write(const void* data, size_t size, BufferRef*& ref) noexcept = 0; + + /// @brief Release the allocation held by the given reference + /// @param ref Allocation reference to release + virtual void release(BufferRef* ref) noexcept = 0; + + /// Defragment the allocator's underlying buffer pool. + virtual void defragment(const std::shared_ptr& fence) noexcept = 0; + + /// Return the buffer page size used by the allocator. This is the maximum size a + /// single buffer can possible be. + virtual size_t pageSize() const noexcept = 0; + + /// Get the OpenGL object ID for the buffer at the given index. + virtual int32_t getBufferID(size_t bufferIndex) const noexcept = 0; +}; + +/// @brief A BufferRef holds a strong reference on a buffer sub-allocation, managed by a RelocatableBuffer. +/// @tparam OwnerClass The class type holding a reference on the buffer +template +class TypedBufferRef : public BufferRef { +public: + TypedBufferRef() = default; + TypedBufferRef(OwnerClass* buffer, size_t pointer, size_t offset, size_t size_) + : BufferRef(pointer, offset, size_), + ownerPtr(buffer) {} + TypedBufferRef(OwnerClass* buffer) + : ownerPtr(buffer) {} + + bool operator==(const TypedBufferRef& rhs) const noexcept { return ownerPtr == rhs.ownerPtr; } + + // The owner of this allocation + OwnerClass* getOwner() const noexcept { return ownerPtr; } + void setOwner(OwnerClass* owner) { ownerPtr = owner; } + +private: + OwnerClass* ownerPtr = nullptr; +}; + +/// A RelocatableBuffer is in essence a view onto an actual buffer. These actual buffers may move around +/// over time as fragmentation is managed and buffers are recycled. A RelocatableBuffer is aware of this +/// and actively participates in this memory relocation. +template +class RelocatableBuffer { +public: + RelocatableBuffer(IBufferAllocator& allocator_, OwnerClass* owner_) + : allocator(allocator_), + owner(owner_) { + assert(owner); + } + RelocatableBuffer(const RelocatableBuffer& rhs) + : allocator(rhs.allocator), + owner(rhs.owner), + contents(rhs.contents) { + allocate(contents.data(), contents.size()); + } + RelocatableBuffer(RelocatableBuffer&& rhs) noexcept + : allocator(rhs.allocator), + owner(rhs.owner), + ref(std::move(rhs.ref)), + contents(std::move(rhs.contents)) {} + ~RelocatableBuffer() { + if (ref) { + assert(ref->getOwner()); + allocator.release(ref); + } + } + + // As references are added to a buffer, reallocation of the underlying reference + // storage may occur. When that happens, we will be informed of the new memory location + // of our reference here. + void relocRef(TypedBufferRef* newRef) noexcept { ref = newRef; } + + // Get the current OpenGL buffer ID. Do not store this, bind it and discard after the active frame. + int32_t getBufferID() const noexcept { return ref ? allocator.getBufferID(ref->getBufferIndex()) : 0; } + + intptr_t getBindingOffset() const noexcept { return ref ? ref->getBufferOffset() : 0; } + + const std::vector& getContents() const noexcept { return contents; } + + void setOwner(OwnerClass* owner_) noexcept { owner = owner_; } + + /// Allocate buffer memory and copy `size` bytes into the allocation from `data` + void allocate(const void* data, size_t size) noexcept { + assert(owner); + + if (ref && ref->getOwner()) { + // If we're writing new data, we need to remove our ref from our old buffer. + allocator.release(ref); + ref = nullptr; + } + + BufferRef* reference = ref; + allocator.write(data, size, reference); + + ref = std::move(static_cast(reference)); + ref->setOwner(owner); + + contents.resize(size); + std::memcpy(contents.data(), data, size); + }; + + IBufferAllocator& allocator; + +private: + OwnerClass* owner = nullptr; + TypedBufferRef* ref = nullptr; // A strong reference to the active allocation backing this buffer + std::vector contents; // CPU-side buffer contents +}; + +class UniformBufferAllocator : public IBufferAllocator { +public: + UniformBufferAllocator(); + ~UniformBufferAllocator() override; + + bool write(const void* data, size_t size, BufferRef*& ref) noexcept override; + void release(BufferRef* ref) noexcept override; + void defragment(const std::shared_ptr& fence) noexcept override; + size_t pageSize() const noexcept override; + int32_t getBufferID(size_t bufferIndex) const noexcept override; + +private: + class Impl; + std::unique_ptr impl; +}; + +} // namespace gl +} // namespace mbgl diff --git a/include/mbgl/gl/uniform_buffer_gl.hpp b/include/mbgl/gl/uniform_buffer_gl.hpp index 579f5f523..9775dc9ad 100644 --- a/include/mbgl/gl/uniform_buffer_gl.hpp +++ b/include/mbgl/gl/uniform_buffer_gl.hpp @@ -2,6 +2,7 @@ #include #include +#include namespace mbgl { namespace gl { @@ -10,19 +11,28 @@ class UniformBufferGL final : public gfx::UniformBuffer { UniformBufferGL(const UniformBufferGL&); public: - UniformBufferGL(const void* data, std::size_t size_); - UniformBufferGL(UniformBufferGL&& other) - : UniformBuffer(std::move(other)) {} + UniformBufferGL(const void* data, std::size_t size_, IBufferAllocator& allocator); ~UniformBufferGL() override; - BufferID getID() const { return id; } - void update(const void* data, std::size_t size_) override; + UniformBufferGL(UniformBufferGL&& rhs) noexcept; + UniformBufferGL& operator=(const UniformBufferGL& rhs) = delete; + + BufferID getID() const; + gl::RelocatableBuffer& getManagedBuffer() noexcept { return managedBuffer; } + const gl::RelocatableBuffer& getManagedBuffer() const noexcept { return managedBuffer; } UniformBufferGL clone() const { return {*this}; } -protected: - BufferID id = 0; - uint32_t hash; + // gfx::UniformBuffer + void update(const void* data, std::size_t size_) override; + +private: + // If the requested UBO size is too large for the allocator, the UBO will manage its own allocation + bool isManagedAllocation = false; + BufferID localID; + gl::RelocatableBuffer managedBuffer; + + friend class UniformBufferArrayGL; }; /// Stores a collection of uniform buffers by name diff --git a/include/mbgl/mtl/context.hpp b/include/mbgl/mtl/context.hpp index 24d3b8e16..83e73d3b5 100644 --- a/include/mbgl/mtl/context.hpp +++ b/include/mbgl/mtl/context.hpp @@ -50,6 +50,9 @@ public: const RendererBackend& getBackend() const { return backend; } + void beginFrame() override; + void endFrame() override; + std::unique_ptr createCommandEncoder() override; /// Create a new buffer object diff --git a/include/mbgl/shaders/gl/drawable_fill_outline.hpp b/include/mbgl/shaders/gl/drawable_fill_outline.hpp index 2e90ca776..ebbd70345 100644 --- a/include/mbgl/shaders/gl/drawable_fill_outline.hpp +++ b/include/mbgl/shaders/gl/drawable_fill_outline.hpp @@ -11,6 +11,7 @@ struct ShaderSource { static constexpr const char* vertex = R"(layout (std140) uniform FillOutlineDrawableUBO { highp mat4 u_matrix; highp vec2 u_world; + highp vec2 pad; }; layout (std140) uniform FillOutlineEvaluatedPropsUBO { highp vec4 u_outline_color; diff --git a/src/mbgl/gfx/context.hpp b/src/mbgl/gfx/context.hpp index 154d35908..3ec22e7b7 100644 --- a/src/mbgl/gfx/context.hpp +++ b/src/mbgl/gfx/context.hpp @@ -65,6 +65,9 @@ public: Context& operator=(const Context& other) = delete; virtual ~Context() = default; + virtual void beginFrame() = 0; + virtual void endFrame() = 0; + /// Called at the end of a frame. virtual void performCleanup() = 0; diff --git a/src/mbgl/gl/buffer_allocator.cpp b/src/mbgl/gl/buffer_allocator.cpp new file mode 100644 index 000000000..ced99efb4 --- /dev/null +++ b/src/mbgl/gl/buffer_allocator.cpp @@ -0,0 +1,518 @@ +#include +#include +#include +#include +#include + +namespace mbgl { + +using namespace platform; + +namespace gl { + +/// @brief BufferAllocator is a semi-generic allocation strategy for uniform buffer objects. +/// This allocator works by streaming buffer writes into sub-allocated sections of actual UBOs. +/// As UBOs fill up, they are marked 'in-flight' and placed in a waiting area for references to +/// be released. Once all references to a UBO are gone (Or the buffer is defragmented), the +/// buffer is recycled. +/// @tparam OwnerClass The class type having allocations managed by this allocator. +/// @tparam type GL_UNIFORM_BUFFER, no other type is currently valid. +/// @tparam PageSizeKB Size of underlying UBO allocations, in KB. 8KB has been observed to work well. +/// @tparam MaxFragmentationOccupancy Buffers under this percentage of occupancy are considered fragmented +/// and the defragmentation routine will attempt to relocate living references to them to other buffers. +/// @tparam MaxFreeBuffers When buffers are made free for recycling, buffers in excess of this amount are +/// released to reduce memory consumption. +/// @tparam InitialBufferSize Initial size of the reference list. 128 has been observed to work well. +template +class BufferAllocator : public IBufferAllocator { + static_assert(type == GL_UNIFORM_BUFFER); + +public: + static constexpr const size_t PageSize = 1024 * PageSizeKB; + static constexpr const size_t FragmentedOccupancyLevel = static_cast( + PageSize * (static_cast(MaxFragmentationOccupancy) / 100.0)); + using BufferTy = + BufferAllocator; + using RefTy = TypedBufferRef; + + ~BufferAllocator() override = default; + +private: + /// @brief A buffer in-flight is one that is being used by the GPU. Such buffers become free + /// once their `frameSync` fence becomes signaled. + struct InFlightBuffer { + size_t bufferIndex; // Index into buffers list + std::shared_ptr frameSync; + + InFlightBuffer(size_t buf, std::shared_ptr sync) + : bufferIndex(buf), + frameSync(std::move(sync)) {} + }; + +public: + /// @brief An allocated buffer the allocator manages. These buffers are sub-allocated + /// in monotonic fashion. Once a buffer's capacity has been reached, it is stored until + /// it can be recycled and a new buffer is acquired to replace it. + struct Buffer { + BufferTy& allocator; + + // The pointer points to the next unused region of memory in the buffer + ptrdiff_t pointer = 0; + + // References to buffer objects allocating from this block of memory. + // For performance reasons, we use a vector here. To deal with vector storage reallocation, + // vector resizes are handled explicitly and Refs are made aware of relocation events. + std::vector refs; + + // Backing GL object for this buffer + GLuint id = 0; + + // Tombstones indicating a removed ref. When a Ref is removed, a tombstone is added to this + // counter. This is much faster vs. erasing from our vector. This works because we enforce + // a monotonic allocation scheme on our buffers and recycle the whole buffer at once. + size_t tombstones = 0; + + // Used to drive defragmentation, occupancy is a measure of the number of bytes held by + // living references + size_t occupancyBytes = 0; + + // We need to know the index to ourselves in the main buffer list + size_t bufferIndex = 0; + + // If true, the buffer's memory has been released to compact memory + bool reclaimed = false; + + Buffer(BufferAllocator& allocator_) + : allocator(allocator_) { + refs.reserve(InitialBufferSize); + MBGL_CHECK_ERROR(glGenBuffers(1, &id)); + MBGL_CHECK_ERROR(glBindBuffer(type, id)); + MBGL_CHECK_ERROR(glBufferData(type, PageSize, nullptr, GL_DYNAMIC_DRAW)); + } + + ~Buffer() { + if (id != 0) { + glDeleteBuffers(1, &id); + id = 0; + } + } + + void reclaim() { + refs.clear(); + refs = decltype(refs)(); + + if (id != 0) { + glDeleteBuffers(1, &id); + id = 0; + } + } + + Buffer(const Buffer&) = delete; + Buffer(Buffer&& rhs) noexcept + : allocator(rhs.allocator), + pointer(rhs.pointer), + refs(std::move(rhs.refs)), + id(rhs.id), + tombstones(rhs.tombstones), + occupancyBytes(rhs.occupancyBytes), + bufferIndex(rhs.bufferIndex) { + rhs.id = 0; + } + + Buffer& operator=(const Buffer&) = delete; + Buffer& operator=(Buffer&& rhs) noexcept { + allocator = rhs.allocator; + pointer = rhs.pointer; + refs = std::move(rhs.refs); + id = rhs.id; + rhs.id = 0; + tombstones = rhs.tombstones; + occupancyBytes = rhs.occupancyBytes; + bufferIndex = rhs.bufferIndex; + return *this; + } + + void setBufferIndex(size_t index) noexcept { bufferIndex = index; } + + // Reset this buffer for a fresh recording session + void reset() { + assert(refs.size() - tombstones == 0); + pointer = tombstones = occupancyBytes = 0; + refs.clear(); + } + + /// @brief Once the buffer is full, we mark it 'in-flight', storing it in another location + /// in the allocator. The defragmentation routine will traverse this location for buffers + /// that can be recycled. + /// @param index The buffer index that maps to this buffer instance. + void markInFlight(size_t index) noexcept { allocator.inFlightBuffers.emplace_back(index); } + + /// @brief Number of living references to this buffer + size_t numRefs() const noexcept { return refs.size() - tombstones; } + + /// @brief Add a living reference to a sub-allocated region of this buffer + /// @param bufObj Managed instance that owns this sub-allocation + /// @param bufPtr Offset into the buffer the allocation begins at + /// @param size_ Size of the allocated region + /// @return BufferRef instance + RefTy* addRef(OwnerClass* bufObj, ptrdiff_t bufPtr, size_t size_) { + // Important: Check if our list needs to reallocate, if so we need to handle it manually + // so we can update the references with their new memory locations + if (refs.size() == refs.capacity()) { + auto oldList = std::exchange(refs, {}); + refs.reserve(oldList.size() * 2); + + for (size_t i = 0; i < oldList.size(); ++i) { + if (!oldList[i].getOwner()) { + // Released ref, skip it + continue; + } + + // Relocate the reference + refs.push_back(std::move(oldList[i])); + refs.back().getOwner()->getManagedBuffer().relocRef(&refs.back()); + } + + // Since we dropped released refs, we can clear the tombstone count + tombstones = 0; + } + + refs.emplace_back(bufObj, bufferIndex, bufPtr, size_); + occupancyBytes += size_; + return &refs.back(); + } + + /// @brief Remove a living reference from this buffer. + /// @note: Remember buffers are only permitted to grow monotonically. A buffer can only be + /// reused once all living references are removed or relocated to other buffers. + /// @param bufObj BufferRef to remove + void decRef(RefTy* bufObj) { +#ifndef NDEBUG + assert(bufObj->getOwner()); + // Sanity check this ref actually belongs + auto it = std::find(refs.begin(), refs.end(), *bufObj); + if (it != refs.end()) { + tombstones++; + } else { + assert(0); + } +#else + tombstones++; +#endif + bufObj->setOwner(nullptr); + + assert(occupancyBytes >= bufObj->getSize()); + occupancyBytes -= bufObj->getSize(); + } + + // Align a pointer on a set block size + static size_t align(size_t ptr, size_t alignment) { + if (ptr == alignment) { + return ptr; + } + return ((ptr + alignment) / alignment) * alignment; + } + }; + +public: + // Write a block of memory to the recording buffer + // Returns the base index of written memory in the buffer + bool write(const void* data, size_t size, BufferRef*& residentBuffer) noexcept override { + Buffer* buffer = nullptr; + if (buffers.size() == 0) { + // Query for the alignment we must use + int32_t alignment = 0; + MBGL_CHECK_ERROR(glGetIntegerv(GL_UNIFORM_BUFFER_OFFSET_ALIGNMENT, &alignment)); + + sharedBufferAlignment = static_cast(alignment); + recordingBuffer = newBuffer(); + } + buffer = &buffers[recordingBuffer]; + + const auto alignedSize = Buffer::align(size, sharedBufferAlignment); + + // Reached past the buffer range + if ((buffer->pointer + alignedSize) > PageSize) { + // Put the buffer in the pending list + buffer->markInFlight(recordingBuffer); + buffer = nullptr; + + // Look for a free buffer + const auto freeIndex = getFreeBuffer(); + if (freeIndex != -1) { + recordingBuffer = freeIndex; + } else { + // Otherwise, allocate a fresh buffer + recordingBuffer = newBuffer(); + } + buffer = &buffers[recordingBuffer]; + } + + assert(buffer); + if (!buffer) { + assert(0); + return false; + } + MBGL_CHECK_ERROR(glBindBuffer(type, buffer->id)); + + // Map the next available slice of memory + auto* buf = MBGL_CHECK_ERROR( + glMapBufferRange(type, + buffer->pointer, + alignedSize, + GL_MAP_INVALIDATE_RANGE_BIT | GL_MAP_UNSYNCHRONIZED_BIT | GL_MAP_WRITE_BIT)); + const auto writtenIndex = buffer->pointer; + + if (buf) { + std::memcpy(buf, data, size); + MBGL_CHECK_ERROR(glUnmapBuffer(type)); + + residentBuffer = buffer->addRef(nullptr, writtenIndex, size); + buffer->pointer += alignedSize; + } else { + assert(0); + return false; + } + +#ifndef NDEBUG + MBGL_CHECK_ERROR(glBindBuffer(type, 0)); +#endif + + return true; + } + + /// Release a living reference + void release(BufferRef* ref) noexcept override { + assert(ref); + if (!ref) { + return; + } + buffers[ref->getBufferIndex()].decRef(static_cast(ref)); + } + + // Look for buffers that either have zero living references or equal or under FragmentationThresh + // references. In the latter case, attempt to relocate these allocations to more utilized buffers. + void defragment(const std::shared_ptr& fence) noexcept override { + if (buffers.size() == 0) { + return; + } + intptr_t recycleIndex = recordingBuffer; + + // Phase one: Mark free buffers + updateInFlight(); + + // Phase two: Consolidate fragmentation + MBGL_CHECK_ERROR(glBindBuffer(type, buffers[recycleIndex].id)); + + for (auto it = inFlightBuffers.begin(); it != inFlightBuffers.end();) { + auto& fragBuffer = buffers[*it]; + if (fragBuffer.occupancyBytes > FragmentedOccupancyLevel) { + ++it; + continue; + } + + // This buffer is fragmented. Move allocations from this buffer and append to fresh ones + // so that this one may be recycled. + + // For each ref, append to new buffer and update that ref + for (auto refIt = fragBuffer.refs.begin(); refIt != fragBuffer.refs.end();) { + if (!refIt->getOwner()) { + ++refIt; + continue; // This is a released reference + } + + const auto alignedSize = Buffer::align(refIt->getOwner()->getSize(), sharedBufferAlignment); + + // 2.a: Get recycle buffer + if (recycleIndex == -1 || (buffers[recycleIndex].pointer + alignedSize) > PageSize) { + if (recycleIndex != -1) { + // We filled this one up, mark it in-flight and get a fresh one. + buffers[recycleIndex].markInFlight(recycleIndex); + } + + // We need to get a new buffer. + recycleIndex = recordingBuffer = getFreeBuffer(); + if (recycleIndex == -1) { + // No more buffers available, we're done. + break; + } + + MBGL_CHECK_ERROR(glBindBuffer(type, buffers[recycleIndex].id)); + } + + // 2.b: Copy into recycle buffer + if (!moveRef(*refIt, recycleIndex, alignedSize)) { + // Relocation failure, stop trying to defrag. +#ifndef NDEBUG + MBGL_CHECK_ERROR(glBindBuffer(GL_UNIFORM_BUFFER, 0)); +#endif + return; + } + } + + // All refs have relocated, clear this buffer. + if (fragBuffer.numRefs() == 0) { + fragBuffer.reset(); + // Store this buffer in a waiting area until the GPU is done using it + waitingFree.emplace_back(fragBuffer.bufferIndex, fence); + it = inFlightBuffers.erase(it); + } else { + // We only managed a partial (or no) defrag. Do nothing with this buffer. + ++it; + } + + // No more buffers to write into, stop. + if (recycleIndex == -1) { + break; + } + } + +#ifndef NDEBUG + MBGL_CHECK_ERROR(glBindBuffer(GL_UNIFORM_BUFFER, 0)); +#endif + + if (recordingBuffer == -1) { + recordingBuffer = newBuffer(); + } + } + + size_t pageSize() const noexcept override { return PageSize; } + + int32_t getBufferID(size_t bufferIndex) const noexcept override { return buffers[bufferIndex].id; } + +private: + size_t newBuffer() noexcept { + buffers.emplace_back(*this); + const auto index = recordingBuffer = buffers.size() - 1; + buffers[index].setBufferIndex(index); + return index; + } + + intptr_t getFreeBuffer() { + if (freeList.size() > 0) { + assert(buffers[freeList.back()].pointer == 0); + const auto index = freeList.back(); + freeList.pop_back(); + return index; + } else if (reclaimedList.size() > 0) { + const auto index = reclaimedList.back(); + reclaimedList.pop_back(); + new (&buffers[index]) Buffer(*this); + buffers[index].setBufferIndex(index); + return index; + } + return -1; + } + + // Check waiting in-flight buffers and move them to the free list + void updateInFlight() { + for (auto it = waitingFree.begin(); it != waitingFree.end();) { + if (it->frameSync->isSignaled()) { + freeList.push_back(it->bufferIndex); + it = waitingFree.erase(it); + } else { + ++it; + } + } + + // And check the free list for an excess of buffers, reclaim if we need to + while (freeList.size() > MaxFreeBuffers) { + reclaimedList.push_back(freeList.back()); + freeList.pop_back(); + buffers[reclaimedList.back()].reclaim(); + } + } + + // Move a buffer reference (An allocation from a UniformBufferGL instance) to a new buffer + bool moveRef(RefTy& ref, size_t toIndex, size_t alignedSize) { + auto& destBuffer = buffers[toIndex]; + const auto recycledWriteIndex = destBuffer.pointer; + + auto* buf = MBGL_CHECK_ERROR( + glMapBufferRange(type, + recycledWriteIndex, + alignedSize, + GL_MAP_INVALIDATE_RANGE_BIT | GL_MAP_UNSYNCHRONIZED_BIT | GL_MAP_WRITE_BIT)); + + if (buf) { + std::memcpy(buf, ref.getOwner()->getManagedBuffer().getContents().data(), ref.getOwner()->getSize()); + MBGL_CHECK_ERROR(glUnmapBuffer(type)); + destBuffer.pointer += alignedSize; + } else { + assert(0); + return false; + } + + // 2.c: Now the ref must be made aware of the relocation of its contents. + // Note that this means defragmentation can never run on refs that are + // expected to remain bound after defragmentation. They must be re-bound + // to become valid again. + auto owner = ref.getOwner(); + const auto refSize = ref.getSize(); + const auto oldIndex = ref.getBufferIndex(); + + const auto newRef = destBuffer.addRef(owner, recycledWriteIndex, refSize); + buffers[oldIndex].decRef(&ref); + owner->getManagedBuffer().relocRef(newRef); + return true; + } + +private: + // GL requires us to satisfy this alignment for sub-allocations in our buffers + size_t sharedBufferAlignment = 0; + // Our active recording buffer + intptr_t recordingBuffer = 0; + + // Buffers free for re-use + std::vector freeList; + // Buffers reclaimed that can be recreated if needed + std::vector reclaimedList; + // Buffers recorded and currently in-use + std::list inFlightBuffers; + // When we recycle a buffer, we copy from an in-flight buffer to a fresh one. + // Before we can convert the in-flight buffer to a free one however, we must wait + // after the copy operation(s) to ensure the GPU is done with it before we reuse it. + std::list waitingFree; + +public: + // All buffers allocated so far + std::vector buffers; + + friend Buffer; +}; + +class UniformBufferAllocator::Impl : public gl::BufferAllocator {}; + +UniformBufferAllocator::~UniformBufferAllocator() = default; + +UniformBufferAllocator::UniformBufferAllocator() { + impl = std::make_unique(); +} + +bool UniformBufferAllocator::write(const void* data, size_t size, BufferRef*& ref) noexcept { + return impl->write(data, size, ref); +} + +void UniformBufferAllocator::release(BufferRef* ref) noexcept { + impl->release(ref); +} + +void UniformBufferAllocator::defragment(const std::shared_ptr& fence) noexcept { + impl->defragment(fence); +} + +size_t UniformBufferAllocator::pageSize() const noexcept { + return impl->pageSize(); +} + +int32_t UniformBufferAllocator::getBufferID(size_t bufferIndex) const noexcept { + return impl->getBufferID(bufferIndex); +} + +} // namespace gl +} // namespace mbgl diff --git a/src/mbgl/gl/context.cpp b/src/mbgl/gl/context.cpp index 1ece3d10c..1fc9ccf58 100644 --- a/src/mbgl/gl/context.cpp +++ b/src/mbgl/gl/context.cpp @@ -73,7 +73,11 @@ GLint getMaxVertexAttribs() { Context::Context(RendererBackend& backend_) : gfx::Context(/*maximumVertexBindingCount=*/getMaxVertexAttribs()), - backend(backend_) {} + backend(backend_) { +#if MLN_DRAWABLE_RENDERER + uboAllocator = std::make_unique(); +#endif +} Context::~Context() noexcept { if (cleanupOnDestruction) { @@ -85,6 +89,32 @@ Context::~Context() noexcept { } } +void Context::beginFrame() { +#if MLN_DRAWABLE_RENDERER + frameInFlightFence = std::make_shared(); + + // Run allocator defragmentation on this frame interval. + constexpr auto defragFreq = 4; + + if (frameNum == defragFreq) { + uboAllocator->defragment(frameInFlightFence); + frameNum = 0; + } else { + frameNum++; + } +#endif +} + +void Context::endFrame() { +#if MLN_DRAWABLE_RENDERER + if (!frameInFlightFence) { + return; + } + + frameInFlightFence->insert(); +#endif +} + void Context::initializeExtensions(const std::function& getProcAddress) { if (const auto* extensions = reinterpret_cast(MBGL_CHECK_ERROR(glGetString(GL_EXTENSIONS)))) { auto fn = [&](std::initializer_list> probes) -> ProcAddress { @@ -494,7 +524,7 @@ gfx::UniqueDrawableBuilder Context::createDrawableBuilder(std::string name) { } gfx::UniformBufferPtr Context::createUniformBuffer(const void* data, std::size_t size, bool /*persistent*/) { - return std::make_shared(data, size); + return std::make_shared(data, size, *uboAllocator); } gfx::ShaderProgramBasePtr Context::getGenericShader(gfx::ShaderRegistry& shaders, const std::string& name) { @@ -634,6 +664,12 @@ void Context::finish() { MBGL_CHECK_ERROR(glFinish()); } +#if MLN_DRAWABLE_RENDERER +std::shared_ptr Context::getCurrentFrameFence() const { + return frameInFlightFence; +} +#endif + void Context::draw(const gfx::DrawMode& drawMode, std::size_t indexOffset, std::size_t indexLength) { switch (drawMode.type) { case gfx::DrawModeType::Points: diff --git a/src/mbgl/gl/context.hpp b/src/mbgl/gl/context.hpp index 810c9033c..1c9002002 100644 --- a/src/mbgl/gl/context.hpp +++ b/src/mbgl/gl/context.hpp @@ -16,6 +16,8 @@ #include #if MLN_DRAWABLE_RENDERER +#include +#include #include #endif @@ -44,6 +46,9 @@ public: std::unique_ptr createCommandEncoder() override; + void beginFrame() override; + void endFrame() override; + void initializeExtensions(const std::function&); void enableDebugging(); @@ -81,6 +86,10 @@ public: void finish(); +#if MLN_DRAWABLE_RENDERER + std::shared_ptr getCurrentFrameFence() const; +#endif + // Actually remove the objects we marked as abandoned with the above methods. // Only call this while the OpenGL context is exclusive to this thread. void performCleanup() override; @@ -134,6 +143,11 @@ private: bool cleanupOnDestruction = true; std::unique_ptr debugging; +#if MLN_DRAWABLE_RENDERER + std::shared_ptr frameInFlightFence; + std::unique_ptr uboAllocator; + size_t frameNum = 0; +#endif public: State activeTextureUnit; diff --git a/src/mbgl/gl/drawable_gl.cpp b/src/mbgl/gl/drawable_gl.cpp index 7c368069a..f9b2728d3 100644 --- a/src/mbgl/gl/drawable_gl.cpp +++ b/src/mbgl/gl/drawable_gl.cpp @@ -71,10 +71,12 @@ void DrawableGL::draw(PaintParameters& parameters) const { } } +#ifndef NDEBUG context.bindVertexArray = value::BindVertexArray::Default; unbindTextures(); unbindUniformBuffers(); +#endif } void DrawableGL::setIndexData(gfx::IndexVectorBasePtr indexes, std::vector segments) { diff --git a/src/mbgl/gl/fence.cpp b/src/mbgl/gl/fence.cpp new file mode 100644 index 000000000..eb8622da5 --- /dev/null +++ b/src/mbgl/gl/fence.cpp @@ -0,0 +1,32 @@ +#include +#include +#include + +namespace mbgl { +namespace gl { + +using namespace platform; + +Fence::Fence() {} + +Fence::~Fence() { + if (fence) { + glDeleteSync(fence); + } +} + +void Fence::insert() noexcept { + assert(!fence); + fence = glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0); +} + +bool Fence::isSignaled() const noexcept { + if (!fence) { + return false; + } + + return glClientWaitSync(fence, GL_SYNC_FLUSH_COMMANDS_BIT, 0); +} + +} // namespace gl +} // namespace mbgl diff --git a/src/mbgl/gl/fence.hpp b/src/mbgl/gl/fence.hpp new file mode 100644 index 000000000..989cad8b2 --- /dev/null +++ b/src/mbgl/gl/fence.hpp @@ -0,0 +1,22 @@ +#pragma once + +#include +#include + +namespace mbgl { +namespace gl { + +class Fence { +public: + Fence(); + ~Fence(); + + void insert() noexcept; + bool isSignaled() const noexcept; + +private: + platform::GLsync fence{nullptr}; +}; + +} // namespace gl +} // namespace mbgl diff --git a/src/mbgl/gl/uniform_block_gl.cpp b/src/mbgl/gl/uniform_block_gl.cpp index 8b93c87e0..c53a9cd86 100644 --- a/src/mbgl/gl/uniform_block_gl.cpp +++ b/src/mbgl/gl/uniform_block_gl.cpp @@ -1,4 +1,4 @@ - +#include #include #include #include @@ -15,7 +15,11 @@ void UniformBlockGL::bindBuffer(const gfx::UniformBuffer& uniformBuffer) { assert(size == uniformBuffer.getSize()); GLint binding = index; const auto& uniformBufferGL = static_cast(uniformBuffer); - MBGL_CHECK_ERROR(glBindBufferBase(GL_UNIFORM_BUFFER, binding, uniformBufferGL.getID())); + MBGL_CHECK_ERROR(glBindBufferRange(GL_UNIFORM_BUFFER, + binding, + uniformBufferGL.getID(), + uniformBufferGL.getManagedBuffer().getBindingOffset(), + uniformBufferGL.getSize())); } void UniformBlockGL::unbindBuffer() { diff --git a/src/mbgl/gl/uniform_buffer_gl.cpp b/src/mbgl/gl/uniform_buffer_gl.cpp index 209ddc1ec..6c6ab4adc 100644 --- a/src/mbgl/gl/uniform_buffer_gl.cpp +++ b/src/mbgl/gl/uniform_buffer_gl.cpp @@ -1,9 +1,10 @@ +#include #include #include -#include #include #include +#include #include namespace mbgl { @@ -11,45 +12,82 @@ namespace gl { using namespace platform; -UniformBufferGL::UniformBufferGL(const void* data_, std::size_t size_) +UniformBufferGL::UniformBufferGL(const void* data_, std::size_t size_, IBufferAllocator& allocator_) : UniformBuffer(size_), - hash(util::crc32(data_, size_)) { - MBGL_CHECK_ERROR(glGenBuffers(1, &id)); - MBGL_CHECK_ERROR(glBindBuffer(GL_UNIFORM_BUFFER, id)); - MBGL_CHECK_ERROR(glBufferData(GL_UNIFORM_BUFFER, size, data_, GL_DYNAMIC_DRAW)); - MBGL_CHECK_ERROR(glBindBuffer(GL_UNIFORM_BUFFER, 0)); + managedBuffer(allocator_, this) { + if (size_ > managedBuffer.allocator.pageSize()) { + // Buffer is very large, won't fit in the provided allocator + MBGL_CHECK_ERROR(glGenBuffers(1, &localID)); + MBGL_CHECK_ERROR(glBindBuffer(GL_UNIFORM_BUFFER, localID)); + MBGL_CHECK_ERROR(glBufferData(GL_UNIFORM_BUFFER, size, data_, GL_DYNAMIC_DRAW)); + MBGL_CHECK_ERROR(glBindBuffer(GL_UNIFORM_BUFFER, 0)); + return; + } + + isManagedAllocation = true; + managedBuffer.allocate(data_, size_); +} + +UniformBufferGL::UniformBufferGL(UniformBufferGL&& rhs) noexcept + : UniformBuffer(rhs.size), + isManagedAllocation(rhs.isManagedAllocation), + localID(rhs.localID), + managedBuffer(std::move(rhs.managedBuffer)) { + managedBuffer.setOwner(this); } UniformBufferGL::UniformBufferGL(const UniformBufferGL& other) : UniformBuffer(other), - hash(other.hash) { - MBGL_CHECK_ERROR(glGenBuffers(1, &id)); - MBGL_CHECK_ERROR(glCopyBufferSubData(other.id, id, 0, 0, size)); - MBGL_CHECK_ERROR(glBindBuffer(GL_COPY_READ_BUFFER, other.id)); - MBGL_CHECK_ERROR(glBindBuffer(GL_COPY_WRITE_BUFFER, id)); - MBGL_CHECK_ERROR(glCopyBufferSubData(GL_COPY_READ_BUFFER, GL_COPY_WRITE_BUFFER, 0, 0, size)); + managedBuffer(other.managedBuffer.allocator, this) { + managedBuffer.setOwner(this); + if (other.isManagedAllocation) { + managedBuffer.allocate(other.managedBuffer.getContents().data(), other.size); + } else { + MBGL_CHECK_ERROR(glGenBuffers(1, &localID)); + MBGL_CHECK_ERROR(glCopyBufferSubData(other.localID, localID, 0, 0, size)); + MBGL_CHECK_ERROR(glBindBuffer(GL_COPY_READ_BUFFER, other.localID)); + MBGL_CHECK_ERROR(glBindBuffer(GL_COPY_WRITE_BUFFER, localID)); + MBGL_CHECK_ERROR(glCopyBufferSubData(GL_COPY_READ_BUFFER, GL_COPY_WRITE_BUFFER, 0, 0, size)); + } } UniformBufferGL::~UniformBufferGL() { - if (id) { - MBGL_CHECK_ERROR(glDeleteBuffers(1, &id)); - id = 0; + if (isManagedAllocation) { + return; + } + + if (localID) { + MBGL_CHECK_ERROR(glDeleteBuffers(1, &localID)); + localID = 0; + } +} + +BufferID UniformBufferGL::getID() const { + if (isManagedAllocation) { + return managedBuffer.getBufferID(); + } else { + return localID; } } void UniformBufferGL::update(const void* data_, std::size_t size_) { - assert(size == size_); - if (size != size_) { + assert(isManagedAllocation ? managedBuffer.getContents().size() == size_ : size == size_); + + if (size != size_ || (isManagedAllocation && managedBuffer.getContents().size() != size_)) { Log::Error( Event::General, "Mismatched size given to UBO update, expected " + std::to_string(size) + ", got " + std::to_string(size_)); return; } - const uint32_t newHash = util::crc32(data_, size_); - if (newHash != hash) { - hash = newHash; - MBGL_CHECK_ERROR(glBindBuffer(GL_UNIFORM_BUFFER, id)); + if (std::memcmp(data_, managedBuffer.getContents().data(), managedBuffer.getContents().size()) == 0) { + return; + } + + if (isManagedAllocation) { + managedBuffer.allocate(data_, size); + } else { + MBGL_CHECK_ERROR(glBindBuffer(GL_UNIFORM_BUFFER, localID)); MBGL_CHECK_ERROR(glBufferSubData(GL_UNIFORM_BUFFER, 0, size_, data_)); MBGL_CHECK_ERROR(glBindBuffer(GL_UNIFORM_BUFFER, 0)); } diff --git a/src/mbgl/mtl/context.cpp b/src/mbgl/mtl/context.cpp index bb075bfc7..ee4ad6995 100644 --- a/src/mbgl/mtl/context.cpp +++ b/src/mbgl/mtl/context.cpp @@ -60,6 +60,9 @@ Context::~Context() noexcept { } } +void Context::beginFrame() {} +void Context::endFrame() {} + std::unique_ptr Context::createCommandEncoder() { return std::make_unique(*this); } diff --git a/src/mbgl/renderer/renderer_impl.cpp b/src/mbgl/renderer/renderer_impl.cpp index f74cf2e6e..22d970476 100644 --- a/src/mbgl/renderer/renderer_impl.cpp +++ b/src/mbgl/renderer/renderer_impl.cpp @@ -137,6 +137,7 @@ void Renderer::Impl::render(const RenderTree& renderTree, // Blocks execution until the renderable is available. backend.getDefaultRenderable().wait(); + context.beginFrame(); if (!staticData) { staticData = std::make_unique(pixelRatio, std::make_unique()); @@ -471,6 +472,7 @@ void Renderer::Impl::render(const RenderTree& renderTree, // CommandEncoder destructor submits render commands. parameters.encoder.reset(); + context.endFrame(); #if MLN_RENDER_BACKEND_METAL if constexpr (EnableMetalCapture) {