Add TaggedScheduler, couple lifetime of tasks and orchestrator (#2398)

include/mbgl/actor/actor.hpp

@@ -14,6 +14,8 @@
 
 namespace mbgl {
 
+class TaggedScheduler;
+
 /**
     An `Actor<O>` is an owning reference to an asynchronous object of type `O`:
    an "actor". Communication with an actor happens via message passing: you send
@@ -61,6 +63,11 @@ public:
     Actor(Scheduler& scheduler, Args&&... args)
         : target(scheduler, parent, std::forward<Args>(args)...) {}
 
+    template <class... Args>
+    Actor(const TaggedScheduler& scheduler, Args&&... args)
+        : retainer(scheduler.get()),
+          target(scheduler, parent, std::forward<Args>(args)...) {}
+
     template <class... Args>
     Actor(std::shared_ptr<Scheduler> scheduler, Args&&... args)
         : retainer(std::move(scheduler)),

include/mbgl/actor/established_actor.hpp

@@ -39,6 +39,17 @@ public:
         parent.mailbox->open(scheduler);
     }
 
+    // Construct the Object from a parameter pack `args` (i.e. `Object(args...)`)
+    template <typename U = Object,
+              class... Args,
+              typename std::enable_if_t<std::is_constructible_v<U, Args...> ||
+                                        std::is_constructible_v<U, ActorRef<U>, Args...>>* = nullptr>
+    EstablishedActor(const TaggedScheduler& scheduler, AspiringActor<Object>& parent_, Args&&... args)
+        : parent(parent_) {
+        emplaceObject(std::forward<Args>(args)...);
+        parent.mailbox->open(scheduler);
+    }
+
     // Construct the `Object` from a tuple containing the constructor arguments
     // (i.e. `Object(std::get<0>(args), std::get<1>(args), ...)`)
     template <class ArgsTuple, std::size_t ArgCount = std::tuple_size<std::decay_t<ArgsTuple>>::value>
@@ -48,6 +59,11 @@ public:
         parent.mailbox->open(scheduler);
     }
 
+    template <class ArgsTuple, std::size_t ArgCount = std::tuple_size<std::decay_t<ArgsTuple>>::value>
+    EstablishedActor(const TaggedScheduler& scheduler, AspiringActor<Object>& parent_, ArgsTuple&& args) {
+        EstablishedActor(*scheduler.get(), parent_, std::forward<ArgsTuple>(args));
+    }
+
     EstablishedActor(const EstablishedActor&) = delete;
 
     ~EstablishedActor() {

include/mbgl/actor/mailbox.hpp

@@ -6,6 +6,7 @@
 #include <queue>
 
 #include <mapbox/std/weak.hpp>
+#include <mbgl/actor/scheduler.hpp>
 
 namespace mbgl {
 
@@ -21,11 +22,13 @@ public:
     Mailbox();
 
     Mailbox(Scheduler&);
+    Mailbox(const TaggedScheduler&);
 
     /// Attach the given scheduler to this mailbox and begin processing messages
     /// sent to it. The mailbox must be a "holding" mailbox, as created by the
     /// default constructor Mailbox().
-    void open(Scheduler& scheduler_);
+    void open(const TaggedScheduler& scheduler_);
+    void open(Scheduler&);
     void close();
 
     // Indicate this mailbox will no longer be checked for messages
@@ -46,6 +49,7 @@ private:
         Abandoned
     };
 
+    util::SimpleIdentity schedulerTag = util::SimpleIdentity::Empty;
     mapbox::base::WeakPtr<Scheduler> weakScheduler;
 
     std::recursive_mutex receivingMutex;

include/mbgl/actor/scheduler.hpp

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <mbgl/util/chrono.hpp>
+#include <mbgl/util/identity.hpp>
 
 #include <mapbox/std/weak.hpp>
 
@@ -39,12 +40,17 @@ public:
 
     /// Enqueues a function for execution.
     virtual void schedule(std::function<void()>&&) = 0;
+    virtual void schedule(const util::SimpleIdentity, std::function<void()>&&) = 0;
+
     /// Makes a weak pointer to this Scheduler.
     virtual mapbox::base::WeakPtr<Scheduler> makeWeakPtr() = 0;
-    /// Enqueues a function for execution on the render thread.
-    virtual void runOnRenderThread(std::function<void()>&&) {};
-    virtual void runRenderJobs() {}
-
+    /// Enqueues a function for execution on the render thread owned by the given tag.
+    virtual void runOnRenderThread(const util::SimpleIdentity, std::function<void()>&&) {}
+    /// Run render thread jobs for the given tag
+    /// @param tag Tag of owner
+    /// @param closeQueue Runs all render jobs and then removes the internal queue.
+    virtual void runRenderJobs([[maybe_unused]] const util::SimpleIdentity tag,
+                               [[maybe_unused]] bool closeQueue = false) {}
     /// Returns a closure wrapping the given one.
     ///
     /// When the returned closure is invoked for the first time, it schedules
@@ -62,15 +68,14 @@ public:
     /// The |TaskFn| return type must be compatible with the |ReplyFn| argument
     /// type. Note: the task result is copied and passed by value.
     template <typename TaskFn, typename ReplyFn>
-    void scheduleAndReplyValue(const TaskFn& task, const ReplyFn& reply) {
+    void scheduleAndReplyValue(const util::SimpleIdentity tag, TaskFn&& task, ReplyFn&& reply) {
         assert(GetCurrent());
-        scheduleAndReplyValue(task, reply, GetCurrent()->makeWeakPtr());
+        scheduleAndReplyValue(tag, task, reply, GetCurrent()->makeWeakPtr());
     }
 
     /// Wait until there's nothing pending or in process
     /// Must not be called from a task provided to this scheduler.
-    /// @param timeout Time to wait, or zero to wait forever.
-    virtual std::size_t waitForEmpty(Milliseconds timeout = Milliseconds{0}) = 0;
+    virtual void waitForEmpty(const util::SimpleIdentity = util::SimpleIdentity::Empty) = 0;
 
     /// Set/Get the current Scheduler for this thread
     static Scheduler* GetCurrent();
@@ -99,21 +104,50 @@ public:
 
 protected:
     template <typename TaskFn, typename ReplyFn>
-    void scheduleAndReplyValue(const TaskFn& task,
+    void scheduleAndReplyValue(const util::SimpleIdentity tag,
+                               const TaskFn& task,
                                const ReplyFn& reply,
                                mapbox::base::WeakPtr<Scheduler> replyScheduler) {
-        auto scheduled = [replyScheduler = std::move(replyScheduler), task, reply] {
+        schedule(tag, [replyScheduler = std::move(replyScheduler), tag, task, reply] {
             auto lock = replyScheduler.lock();
             if (!replyScheduler) return;
-            auto scheduledReply = [reply, result = task()] {
-                reply(result);
-            };
-            replyScheduler->schedule(std::move(scheduledReply));
-        };
-        schedule(std::move(scheduled));
+            replyScheduler->schedule(tag, [reply, result = task()] { reply(result); });
+        });
     }
 
     std::function<void(const std::exception_ptr)> handler;
 };
 
+/// @brief A TaggedScheduler pairs a scheduler with an identifier. Tasklets submitted via a TaggedScheduler
+/// are bucketed with the tag to enable queries on tasks related to that tag. This allows multiple map
+/// instances to all use the same scheduler and await processing of all their tasks prior to map deletion.
+class TaggedScheduler {
+public:
+    TaggedScheduler() = delete;
+    TaggedScheduler(std::shared_ptr<Scheduler> scheduler_, const util::SimpleIdentity tag_)
+        : tag(tag_),
+          scheduler(std::move(scheduler_)) {}
+    TaggedScheduler(const TaggedScheduler&) = default;
+
+    /// @brief Get the wrapped scheduler
+    /// @return
+    const std::shared_ptr<Scheduler>& get() const noexcept { return scheduler; }
+
+    void schedule(std::function<void()>&& fn) { scheduler->schedule(tag, std::move(fn)); }
+    void runOnRenderThread(std::function<void()>&& fn) { scheduler->runOnRenderThread(tag, std::move(fn)); }
+    void runRenderJobs(bool closeQueue = false) { scheduler->runRenderJobs(tag, closeQueue); }
+    void waitForEmpty() const noexcept { scheduler->waitForEmpty(tag); }
+
+    /// type. Note: the task result is copied and passed by value.
+    template <typename TaskFn, typename ReplyFn>
+    void scheduleAndReplyValue(TaskFn&& task, ReplyFn&& reply) {
+        scheduler->scheduleAndReplyValue(tag, task, reply);
+    }
+
+    const mbgl::util::SimpleIdentity tag;
+
+private:
+    std::shared_ptr<Scheduler> scheduler;
+};
+
 } // namespace mbgl

include/mbgl/gfx/context.hpp

@@ -54,8 +54,7 @@ using VertexAttributeArrayPtr = std::shared_ptr<VertexAttributeArray>;
 class Context {
 protected:
     Context(uint32_t maximumVertexBindingCount_)
-        : maximumVertexBindingCount(maximumVertexBindingCount_),
-          backgroundScheduler(Scheduler::GetBackground()) {}
+        : maximumVertexBindingCount(maximumVertexBindingCount_) {}
 
 public:
     static constexpr const uint32_t minimumRequiredVertexBindingCount = 8;
@@ -174,8 +173,6 @@ protected:
     virtual std::unique_ptr<DrawScopeResource> createDrawScopeResource() = 0;
 
     gfx::RenderingStats stats;
-
-    std::shared_ptr<Scheduler> backgroundScheduler;
 };
 
 } // namespace gfx

include/mbgl/gfx/renderer_backend.hpp

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <mbgl/actor/scheduler.hpp>
 #include <mbgl/util/util.hpp>
 
 #include <memory>
@@ -8,6 +9,7 @@
 namespace mbgl {
 
 class ProgramParameters;
+class Map;
 
 namespace gfx {
 
@@ -25,12 +27,16 @@ enum class ContextMode : bool {
 class RendererBackend {
 protected:
     explicit RendererBackend(ContextMode);
+    RendererBackend(ContextMode, const TaggedScheduler&);
 
 public:
     virtual ~RendererBackend();
     RendererBackend(const RendererBackend&) = delete;
     RendererBackend& operator=(const RendererBackend&) = delete;
 
+    // Return the background thread pool assigned to this backend
+    TaggedScheduler& getThreadPool() noexcept { return threadPool; }
+
     /// Returns the device's context.
     Context& getContext();
 
@@ -48,6 +54,7 @@ public:
     /// One-time shader initialization
     virtual void initShaders(gfx::ShaderRegistry&, const ProgramParameters&) = 0;
 #endif
+    const mbgl::util::SimpleIdentity uniqueID;
 
 protected:
     virtual std::unique_ptr<Context> createContext() = 0;
@@ -70,6 +77,7 @@ protected:
     std::unique_ptr<Context> context;
     const ContextMode contextMode;
     std::once_flag initialized;
+    TaggedScheduler threadPool;
 
     friend class BackendScope;
 };

include/mbgl/gl/renderer_backend.hpp

@@ -17,6 +17,7 @@ using FramebufferID = uint32_t;
 class RendererBackend : public gfx::RendererBackend {
 public:
     RendererBackend(gfx::ContextMode);
+    RendererBackend(gfx::ContextMode, const TaggedScheduler&);
     ~RendererBackend() override;
 
     /// Called prior to rendering to update the internally assumed OpenGL state.

include/mbgl/renderer/renderer_frontend.hpp

@@ -4,6 +4,7 @@
 #include <mbgl/gfx/drawable.hpp>
 #endif
 
+#include <mbgl/actor/scheduler.hpp>
 #include <memory>
 
 namespace mbgl {
@@ -30,6 +31,8 @@ public:
     /// Coalescing updates is up to the implementer
     virtual void update(std::shared_ptr<UpdateParameters>) = 0;
 
+    virtual const TaggedScheduler& getThreadPool() const = 0;
+
 protected:
 };
 

include/mbgl/style/sources/geojson_source.hpp

@@ -41,7 +41,7 @@ public:
     using TileFeatures = mapbox::feature::feature_collection<int16_t>;
     using Features = mapbox::feature::feature_collection<double>;
     static std::shared_ptr<GeoJSONData> create(const GeoJSON&,
-                                               std::shared_ptr<Scheduler> scheduler,
+                                               std::shared_ptr<Scheduler> sequencedScheduler,
                                                const Immutable<GeoJSONOptions>& = GeoJSONOptions::defaultOptions());
 
     virtual ~GeoJSONData() = default;
@@ -80,7 +80,6 @@ protected:
 private:
     std::optional<std::string> url;
     std::unique_ptr<AsyncRequest> req;
-    std::shared_ptr<Scheduler> threadPool;
     std::shared_ptr<Scheduler> sequencedScheduler;
     mapbox::base::WeakPtrFactory<Source> weakFactory{this};
     // Do not add members here, see `WeakPtrFactory`

include/mbgl/style/style.hpp

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <mbgl/actor/scheduler.hpp>
 #include <mbgl/map/camera.hpp>
 #include <mbgl/style/image.hpp>
 #include <mbgl/style/transition_options.hpp>
@@ -22,7 +23,7 @@ class Layer;
 
 class Style {
 public:
-    Style(std::shared_ptr<FileSource>, float pixelRatio);
+    Style(std::shared_ptr<FileSource>, float pixelRatio, const TaggedScheduler& threadPool_);
     ~Style();
 
     void loadJSON(const std::string&);

include/mbgl/util/identity.hpp

@@ -27,7 +27,7 @@ public:
     bool operator!=(const SimpleIdentity& other) const noexcept { return uniqueID != other.uniqueID; }
 
     std::int64_t id() const noexcept { return uniqueID; }
-    bool isEmpty() const noexcept { return uniqueID != emptyID; }
+    bool isEmpty() const noexcept { return uniqueID == emptyID; }
 
     operator bool() const noexcept { return isEmpty(); }
     bool operator!() const noexcept { return !isEmpty(); }
@@ -58,3 +58,10 @@ inline std::string toString(const util::SimpleIdentity& ident) {
 
 } // namespace util
 } // namespace mbgl
+
+template <>
+struct std::hash<mbgl::util::SimpleIdentity> {
+    std::size_t operator()(const mbgl::util::SimpleIdentity& s) const noexcept {
+        return std::hash<std::int64_t>{}(s.id());
+    }
+};

include/mbgl/util/run_loop.hpp

@@ -79,9 +79,10 @@ public:
     }
 
     void schedule(std::function<void()>&& fn) override { invoke(std::move(fn)); }
+    void schedule(const util::SimpleIdentity, std::function<void()>&& fn) override { schedule(std::move(fn)); }
     ::mapbox::base::WeakPtr<Scheduler> makeWeakPtr() override { return weakFactory.makeWeakPtr(); }
 
-    std::size_t waitForEmpty(Milliseconds timeout) override;
+    void waitForEmpty(const util::SimpleIdentity = util::SimpleIdentity::Empty) override;
 
     class Impl;