core: do not use JSON serialization for layer grouping key (#4075)

src/mbgl/renderer/group_by_layout.cpp

@@ -1,32 +1,86 @@
 #include <mbgl/renderer/group_by_layout.hpp>
+#include <mbgl/util/hash.hpp>
+#include <mbgl/util/instrumentation.hpp>
+#include <mbgl/style/filter.hpp>
+
 #include <mbgl/style/conversion/stringify.hpp>
 #include <mbgl/util/rapidjson.hpp>
 
 #include <rapidjson/writer.h>
 #include <rapidjson/stringbuffer.h>
 
-#include <unordered_map>
-
 namespace mbgl {
 
-std::string layoutKey(const style::Layer::Impl& impl) {
-    using namespace style::conversion;
+bool LayoutGroupKey::operator==(const LayoutGroupKey& other) const {
+    // Fast path: compare pointers and scalars first
+    if (typeInfo != other.typeInfo) return false;
+    if (minZoom != other.minZoom) return false;
+    if (maxZoom != other.maxZoom) return false;
+    if (visibility != other.visibility) return false;
 
-    rapidjson::StringBuffer s;
-    rapidjson::Writer<rapidjson::StringBuffer> writer(s);
+    // Compare source strings
+    if (source != other.source) return false;
+    if (sourceLayer != other.sourceLayer) return false;
 
-    writer.StartArray();
-    writer.Uint64(reinterpret_cast<uint64_t>(impl.getTypeInfo()));
-    writer.String(impl.source);
-    writer.String(impl.sourceLayer);
-    writer.Double(impl.minZoom);
-    writer.Double(impl.maxZoom);
-    writer.Uint(static_cast<uint32_t>(impl.visibility));
-    stringify(writer, impl.filter);
-    impl.stringifyLayout(writer);
-    writer.EndArray();
+    // Filter comparison - prefer pointer equality
+    if (filterExpression != other.filterExpression) {
+        if (!filterExpression || !other.filterExpression) return false;
+        // Deep comparison if pointers differ
+        if (*filterExpression != *other.filterExpression) return false;
+    }
 
-    return s.GetString();
+    return !impl->hasLayoutDifference(*other.impl);
+}
+
+namespace {
+
+LayoutGroupKey createGroupKey(const style::Layer::Impl& impl) {
+    LayoutGroupKey key;
+    key.typeInfo = impl.getTypeInfo();
+    key.source = impl.source;
+    key.sourceLayer = impl.sourceLayer;
+    key.minZoom = impl.minZoom;
+    key.maxZoom = impl.maxZoom;
+    key.visibility = impl.visibility;
+    key.filterExpression = impl.filter.expression.value_or(nullptr);
+    key.impl = &impl;
+    return key;
+}
+
+} // anonymous namespace
+
+GroupMap groupLayers(const std::vector<Immutable<style::LayerProperties>>& layers) {
+    MLN_TRACE_FUNC();
+
+    GroupMap groupMap;
+    groupMap.reserve(layers.size());
+
+    for (auto layer : layers) {
+        groupMap[createGroupKey(*layer->baseImpl)].push_back(std::move(layer));
+    }
+    return groupMap;
 }
 
 } // namespace mbgl
+
+// std::hash specialization (outside namespace)
+std::size_t std::hash<mbgl::LayoutGroupKey>::operator()(const mbgl::LayoutGroupKey& key) const noexcept {
+    using mbgl::util::hash_combine;
+
+    std::size_t seed = 0;
+
+    // Hash stable pointers and scalars only (NOT layout)
+    hash_combine(seed, reinterpret_cast<std::size_t>(key.typeInfo));
+    hash_combine(seed, std::hash<std::string>{}(key.source));
+    hash_combine(seed, std::hash<std::string>{}(key.sourceLayer));
+    hash_combine(seed, std::hash<float>{}(key.minZoom));
+    hash_combine(seed, std::hash<float>{}(key.maxZoom));
+    hash_combine(seed, static_cast<bool>(key.visibility));
+    hash_combine(seed, reinterpret_cast<std::size_t>(key.filterExpression.get()));
+
+    // Note: We intentionally DON'T hash the layout here
+    // This means keys with same source/zoom/type/filter but different layouts
+    // will hash to the same bucket, then operator== distinguishes them
+
+    return seed;
+}

src/mbgl/renderer/group_by_layout.hpp

@@ -1,12 +1,40 @@
 #pragma once
 
 #include <mbgl/style/layer_impl.hpp>
+#include <mbgl/util/containers.hpp>
+#include <mbgl/style/layer_properties.hpp>
 
-#include <vector>
 #include <memory>
+#include <vector>
 
 namespace mbgl {
 
-std::string layoutKey(const style::Layer::Impl& impl);
+struct LayoutGroupKey {
+    // Direct comparable fields (fast path)
+    const style::LayerTypeInfo* typeInfo;
+    std::string source;
+    std::string sourceLayer;
+    float minZoom;
+    float maxZoom;
+    style::VisibilityType visibility;
+
+    // Filter - use shared_ptr for identity comparison
+    std::shared_ptr<const style::expression::Expression> filterExpression;
+
+    // Reference to impl for layout comparison
+    const style::Layer::Impl* impl;
+
+    // Comparison operator
+    bool operator==(const LayoutGroupKey& other) const;
+};
+
+using GroupMap = mbgl::unordered_map<LayoutGroupKey, std::vector<Immutable<style::LayerProperties>>>;
+
+GroupMap groupLayers(const std::vector<Immutable<style::LayerProperties>>&);
 
 } // namespace mbgl
+
+template <>
+struct std::hash<mbgl::LayoutGroupKey> {
+    std::size_t operator()(const mbgl::LayoutGroupKey& key) const noexcept;
+};

src/mbgl/tile/geometry_tile_worker.cpp

@@ -431,12 +431,7 @@ void GeometryTileWorker::parse() {
     ImageDependencies imageDependencies;
 
     // Create render layers and group by layout
-    mbgl::unordered_map<std::string, std::vector<Immutable<style::LayerProperties>>> groupMap;
-    groupMap.reserve(layers->size());
-
-    for (auto layer : *layers) {
-        groupMap[layoutKey(*layer->baseImpl)].push_back(std::move(layer));
-    }
+    GroupMap groupMap = groupLayers(*layers);
 
     for (auto& pair : groupMap) {
         const auto& group = pair.second;