diff --git a/platform/android/docs/data/PMTiles.md b/platform/android/docs/data/PMTiles.md index d490319a2..1800c8ce7 100644 --- a/platform/android/docs/data/PMTiles.md +++ b/platform/android/docs/data/PMTiles.md @@ -2,7 +2,7 @@ Starting MapLibre Android 11.7.0, using [PMTiles](https://docs.protomaps.com/pmtiles/) as a data source is supported. You can prefix your vector tile source with `pmtiles://` to load a PMTiles file. The rest of the URL continue with be `https://` to load a remote PMTiles file, `asset://` to load an asset or `file://` to load a local PMTiles file. -> Note: PMTiles sources currently do not support caching or offline pack downloads. +> Note: PMTiles sources do not currently support offline pack downloads. Oliver Wipfli has made a style available that combines a [Protomaps]() basemap togehter with Foursquare's POI dataset. It is available in the [wipfli/foursquare-os-places-pmtiles](https://github.com/wipfli/foursquare-os-places-pmtiles) repository on GitHub. The style to use is diff --git a/platform/default/include/mbgl/storage/offline_database.hpp b/platform/default/include/mbgl/storage/offline_database.hpp index 7c3af65f3..d34ec79ef 100644 --- a/platform/default/include/mbgl/storage/offline_database.hpp +++ b/platform/default/include/mbgl/storage/offline_database.hpp @@ -97,6 +97,9 @@ public: void reopenDatabaseReadOnly(bool readOnly); + // Builds the offline-cache key for a resource. Exposed for testing. + static std::string cacheKey(const Resource&); + private: class DatabaseSizeChangeStats; diff --git a/platform/default/src/mbgl/storage/offline_database.cpp b/platform/default/src/mbgl/storage/offline_database.cpp index ec4291244..10a76fe70 100644 --- a/platform/default/src/mbgl/storage/offline_database.cpp +++ b/platform/default/src/mbgl/storage/offline_database.cpp @@ -13,6 +13,17 @@ namespace mbgl { +// Single-point transformer for offline storage keys. For sources where the URL is +// not enough to guarantee uniqueness (eg. PMTiles where all requests resolve to +// the same URL), this function creates a synthetic key based on the requested +// data range. Can be expanded in the future to accommodate other formats. +std::string OfflineDatabase::cacheKey(const Resource& resource) { + if (!resource.dataRange) return resource.url; + const auto sep = resource.url.find('?') == std::string::npos ? '?' : '&'; + return resource.url + sep + "_mlnRange=" + std::to_string(resource.dataRange->first) + "-" + + std::to_string(resource.dataRange->second); +} + OfflineDatabase::OfflineDatabase(std::string path_, const TileServerOptions& options) : path(std::move(path_)), tileServerOptions(options) { @@ -354,7 +365,7 @@ std::optional> OfflineDatabase::getResource(const try { mapbox::sqlite::Query accessedQuery{getStatement("UPDATE resources SET accessed = ?1 WHERE url = ?2")}; accessedQuery.bind(1, util::now()); - accessedQuery.bind(2, resource.url); + accessedQuery.bind(2, cacheKey(resource)); accessedQuery.run(); } catch (const mapbox::sqlite::Exception& ex) { if (ex.code == mapbox::sqlite::ResultCode::NotADB || ex.code == mapbox::sqlite::ResultCode::Corrupt) { @@ -375,7 +386,7 @@ std::optional> OfflineDatabase::getResource(const "WHERE url = ?") }; // clang-format on - query.bind(1, resource.url); + query.bind(1, cacheKey(resource)); if (!query.run()) { return std::nullopt; @@ -405,7 +416,7 @@ std::optional> OfflineDatabase::getResource(const std::optional OfflineDatabase::hasResource(const Resource& resource) { mapbox::sqlite::Query query{getStatement("SELECT length(data) FROM resources WHERE url = ?")}; - query.bind(1, resource.url); + query.bind(1, cacheKey(resource)); if (!query.run()) { return std::nullopt; } @@ -432,7 +443,7 @@ bool OfflineDatabase::putResource(const Resource& resource, notModifiedQuery.bind(1, util::now()); notModifiedQuery.bind(2, response.expires); notModifiedQuery.bind(3, response.mustRevalidate); - notModifiedQuery.bind(4, resource.url); + notModifiedQuery.bind(4, cacheKey(resource)); notModifiedQuery.run(); return false; } @@ -458,7 +469,7 @@ bool OfflineDatabase::putResource(const Resource& resource, updateQuery.bind(4, response.mustRevalidate); updateQuery.bind(5, response.modified); updateQuery.bind(6, util::now()); - updateQuery.bind(9, resource.url); + updateQuery.bind(9, cacheKey(resource)); if (response.noContent) { updateQuery.bind(7, nullptr); @@ -479,7 +490,7 @@ bool OfflineDatabase::putResource(const Resource& resource, "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9) ") }; // clang-format on - insertQuery.bind(1, resource.url); + insertQuery.bind(1, cacheKey(resource)); insertQuery.bind(2, int(resource.kind)); insertQuery.bind(3, response.etag); insertQuery.bind(4, response.expires); @@ -1155,7 +1166,7 @@ bool OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) { // clang-format on insertQuery.bind(1, regionID); - insertQuery.bind(2, resource.url); + insertQuery.bind(2, cacheKey(resource)); insertQuery.run(); if (insertQuery.changes() == 0) { @@ -1172,7 +1183,7 @@ bool OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) { // clang-format on selectQuery.bind(1, regionID); - selectQuery.bind(2, resource.url); + selectQuery.bind(2, cacheKey(resource)); return !selectQuery.run(); } } diff --git a/platform/default/src/mbgl/storage/pmtiles_file_source.cpp b/platform/default/src/mbgl/storage/pmtiles_file_source.cpp index 438c66fd7..6991e9c1a 100644 --- a/platform/default/src/mbgl/storage/pmtiles_file_source.cpp +++ b/platform/default/src/mbgl/storage/pmtiles_file_source.cpp @@ -143,7 +143,7 @@ public: } Resource tileResource(Resource::Kind::Source, url); - tileResource.loadingMethod = Resource::LoadingMethod::Network; + tileResource.loadingMethod = Resource::LoadingMethod::All; tileResource.dataRange = std::make_pair(tileAddress.first, tileAddress.first + tileAddress.second - 1); @@ -159,6 +159,17 @@ public: return; } + if (tileResponse.notModified) { + return; + } + + if (!tileResponse.data) { + response.error = std::make_unique( + Response::Error::Reason::Other, "Error fetching PMTiles tile: response has no data"); + ref.invoke(&FileSourceRequest::setResponse, response); + return; + } + response.data = tileResponse.data; response.noContent = false; response.modified = tileResponse.modified; @@ -232,10 +243,11 @@ private: void getHeader(const std::string& url, AsyncRequest* req, AsyncCallback callback) { if (header_cache.contains(url)) { callback(std::unique_ptr()); + return; } Resource resource(Resource::Kind::Source, url); - resource.loadingMethod = Resource::LoadingMethod::Network; + resource.loadingMethod = Resource::LoadingMethod::All; resource.dataRange = std::make_pair(pmtilesHeaderOffset, pmtilesHeaderOffset + pmtilesHeaderLength - 1); @@ -260,6 +272,16 @@ private: return; } + if (response.notModified) { + return; + } + + if (!response.data) { + callback(std::make_unique(Response::Error::Reason::Other, + "PMTiles header response has no data")); + return; + } + try { pmtiles::headerv3 header = pmtiles::deserialize_header(response.data->substr(0, 127)); @@ -283,6 +305,7 @@ private: void getMetadata(std::string& url, AsyncRequest* req, AsyncCallback callback) { if (metadata_cache.contains(url)) { callback(std::unique_ptr()); + return; } getHeader( @@ -397,7 +420,7 @@ private: if (header.json_metadata_bytes > 0) { Resource resource(Resource::Kind::Source, url); - resource.loadingMethod = Resource::LoadingMethod::Network; + resource.loadingMethod = Resource::LoadingMethod::All; resource.dataRange = std::make_pair(header.json_metadata_offset, header.json_metadata_offset + header.json_metadata_bytes - 1); @@ -410,6 +433,20 @@ private: return; } + if (responseMetadata.notModified) { + return; + } + + if (!responseMetadata.data) { + callback(std::make_unique( + Response::Error::Reason::Other, + "PMTiles metadata response has no data (range=" + + std::to_string(header.json_metadata_offset) + "-" + + std::to_string(header.json_metadata_offset + header.json_metadata_bytes - 1) + + ")")); + return; + } + std::string data = *responseMetadata.data; if (header.internal_compression == pmtiles::COMPRESSION_GZIP) { @@ -480,7 +517,7 @@ private: pmtiles::headerv3 header = header_cache.at(url); Resource resource(Resource::Kind::Source, url); - resource.loadingMethod = Resource::LoadingMethod::Network; + resource.loadingMethod = Resource::LoadingMethod::All; resource.dataRange = std::make_pair(directoryOffset, directoryOffset + directoryLength - 1); tasks[req] = getFileSource()->request(resource, [=, this](const Response& response) { @@ -492,6 +529,16 @@ private: return; } + if (response.notModified) { + return; + } + + if (!response.data) { + callback(std::make_unique(Response::Error::Reason::Other, + "PMTiles directory response has no data")); + return; + } + try { std::string directoryData = *response.data; diff --git a/platform/ios/MapLibre.docc/PMTiles.md b/platform/ios/MapLibre.docc/PMTiles.md index fbaa4aa4a..f5b4585ff 100644 --- a/platform/ios/MapLibre.docc/PMTiles.md +++ b/platform/ios/MapLibre.docc/PMTiles.md @@ -4,7 +4,7 @@ Working with PMTiles Starting MapLibre iOS 6.10.0, using [PMTiles](https://docs.protomaps.com/pmtiles/) as a data source is supported. You can prefix your vector tile source with `pmtiles://` to load a PMTiles file. The rest of the URL continue with be `https://` to load a remote PMTiles file, `asset://` to load an asset or `file://` to load a local PMTiles file. -> Note: PMTiles sources currently do not support caching or offline pack downloads. +> Note: PMTiles sources do not currently support offline pack downloads. Oliver Wipfli has made a style available that combines a [Protomaps]() basemap togehter with Foursquare's POI dataset. It is available in the [wipfli/foursquare-os-places-pmtiles](https://github.com/wipfli/foursquare-os-places-pmtiles) repository on GitHub. The style to use is diff --git a/test/storage/offline_database.test.cpp b/test/storage/offline_database.test.cpp index abf4485e2..57a497c0a 100644 --- a/test/storage/offline_database.test.cpp +++ b/test/storage/offline_database.test.cpp @@ -396,6 +396,22 @@ TEST(OfflineDatabase, PutTile) { EXPECT_EQ(0u, log.uncheckedCount()); } +// Cache-key encoding for byte-range sources (eg. PMTiles). The range is appended +// to the URL so distinct ranges occupy distinct cache rows under the resources +// table's UNIQUE(url) constraint. +TEST(OfflineDatabase, CacheKey) { + Resource bare{Resource::Kind::Source, "https://example.com/x.pmtiles"}; + EXPECT_EQ("https://example.com/x.pmtiles", OfflineDatabase::cacheKey(bare)); + + Resource ranged{Resource::Kind::Source, "https://example.com/x.pmtiles"}; + ranged.dataRange = std::make_pair(0, 126); + EXPECT_EQ("https://example.com/x.pmtiles?_mlnRange=0-126", OfflineDatabase::cacheKey(ranged)); + + Resource rangedWithQuery{Resource::Kind::Source, "https://example.com/x.pmtiles?token=abc"}; + rangedWithQuery.dataRange = std::make_pair(200, 399); + EXPECT_EQ("https://example.com/x.pmtiles?token=abc&_mlnRange=200-399", OfflineDatabase::cacheKey(rangedWithQuery)); +} + TEST(OfflineDatabase, PutResourceNoContent) { FixtureLog log; OfflineDatabase db(":memory:", fixture::tileServerOptions); diff --git a/test/storage/pmtiles_file_source.test.cpp b/test/storage/pmtiles_file_source.test.cpp index a7cf411c3..fcccafa55 100644 --- a/test/storage/pmtiles_file_source.test.cpp +++ b/test/storage/pmtiles_file_source.test.cpp @@ -124,6 +124,41 @@ TEST(PMTilesFileSource, CorruptGzipTile) { loop.run(); } +// Regression test for https://github.com/maplibre/maplibre-native/issues/3690 — the loading +// method for PMTiles sub-requests changed from `Network` to `All` so that http(s)-hosted +// archives flow through the ambient cache. The file:// path bypasses the cache by design, +// so this test can only confirm that issuing the same tile twice still succeeds; a true +// cache-hit assertion requires an http(s) PMTiles fixture and a mock server. +TEST(PMTilesFileSource, RepeatedTileRequest) { + util::RunLoop loop; + + PMTilesFileSource pmtiles(ResourceOptions::Default(), ClientOptions()); + + std::unique_ptr req1; + std::unique_ptr req2; + + req1 = pmtiles.request( + Resource::tile(toAbsoluteURL("geography-class-png.pmtiles"), 1.0, 0, 0, 0, Tileset::Scheme::XYZ), + [&](Response res1) { + req1.reset(); + ASSERT_EQ(nullptr, res1.error); + ASSERT_TRUE(res1.data.get()); + const auto firstSize = res1.data->size(); + + req2 = pmtiles.request( + Resource::tile(toAbsoluteURL("geography-class-png.pmtiles"), 1.0, 0, 0, 0, Tileset::Scheme::XYZ), + [&, firstSize](Response res2) { + req2.reset(); + EXPECT_EQ(nullptr, res2.error); + ASSERT_TRUE(res2.data.get()); + EXPECT_EQ(firstSize, res2.data->size()); + loop.stop(); + }); + }); + + loop.run(); +} + // An archive whose header flags tile_compression=GZIP but whose individual tiles are stored // without compression must be served without error (uncompressed bytes passed through as-is). TEST(PMTilesFileSource, UncompressedTile) {