tei/MapLibreParticleLab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

core: ambient cache for pmtile sources (#4290)

Browse Source
This commit is contained in:
twopastmidnight
2026-06-06 21:10:10 +02:00
committed by GitHub
parent 1b6ed35426
commit a2eb4409aa
7 changed files with 126 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
platform/android/docs/data/PMTiles.md
View File

@@ -2,7 +2,7 @@
Starting MapLibre Android 11.7.0, using [PMTiles](https://docs.protomaps.com/pmtiles/) as a data source is supported. You can prefix your vector tile source with `pmtiles://` to load a PMTiles file. The rest of the URL continue with be `https://` to load a remote PMTiles file, `asset://` to load an asset or `file://` to load a local PMTiles file.
> Note: PMTiles sources currently do not support caching or offline pack downloads.
> Note: PMTiles sources do not currently support offline pack downloads.
Oliver Wipfli has made a style available that combines a [Protomaps]() basemap togehter with Foursquare's POI dataset. It is available in the [wipfli/foursquare-os-places-pmtiles](https://github.com/wipfli/foursquare-os-places-pmtiles) repository on GitHub. The style to use is

3
platform/default/include/mbgl/storage/offline_database.hpp
View File

@@ -97,6 +97,9 @@ public:
void reopenDatabaseReadOnly(bool readOnly);
// Builds the offline-cache key for a resource. Exposed for testing.
static std::string cacheKey(const Resource&);
private:
class DatabaseSizeChangeStats;

27
platform/default/src/mbgl/storage/offline_database.cpp
View File

@@ -13,6 +13,17 @@
namespace mbgl {
// Single-point transformer for offline storage keys. For sources where the URL is
// not enough to guarantee uniqueness (eg. PMTiles where all requests resolve to
// the same URL), this function creates a synthetic key based on the requested
// data range. Can be expanded in the future to accommodate other formats.
std::string OfflineDatabase::cacheKey(const Resource& resource) {
if (!resource.dataRange) return resource.url;
const auto sep = resource.url.find('?') == std::string::npos ? '?' : '&';
return resource.url + sep + "_mlnRange=" + std::to_string(resource.dataRange->first) + "-" +
std::to_string(resource.dataRange->second);
}
OfflineDatabase::OfflineDatabase(std::string path_, const TileServerOptions& options)
: path(std::move(path_)),
tileServerOptions(options) {
@@ -354,7 +365,7 @@ std::optional<std::pair<Response, uint64_t>> OfflineDatabase::getResource(const
try {
mapbox::sqlite::Query accessedQuery{getStatement("UPDATE resources SET accessed = ?1 WHERE url = ?2")};
accessedQuery.bind(1, util::now());
accessedQuery.bind(2, resource.url);
accessedQuery.bind(2, cacheKey(resource));
accessedQuery.run();
} catch (const mapbox::sqlite::Exception& ex) {
if (ex.code == mapbox::sqlite::ResultCode::NotADB || ex.code == mapbox::sqlite::ResultCode::Corrupt) {
@@ -375,7 +386,7 @@ std::optional<std::pair<Response, uint64_t>> OfflineDatabase::getResource(const
"WHERE url = ?") };
// clang-format on
query.bind(1, resource.url);
query.bind(1, cacheKey(resource));
if (!query.run()) {
return std::nullopt;
@@ -405,7 +416,7 @@ std::optional<std::pair<Response, uint64_t>> OfflineDatabase::getResource(const
std::optional<int64_t> OfflineDatabase::hasResource(const Resource& resource) {
mapbox::sqlite::Query query{getStatement("SELECT length(data) FROM resources WHERE url = ?")};
query.bind(1, resource.url);
query.bind(1, cacheKey(resource));
if (!query.run()) {
return std::nullopt;
}
@@ -432,7 +443,7 @@ bool OfflineDatabase::putResource(const Resource& resource,
notModifiedQuery.bind(1, util::now());
notModifiedQuery.bind(2, response.expires);
notModifiedQuery.bind(3, response.mustRevalidate);
notModifiedQuery.bind(4, resource.url);
notModifiedQuery.bind(4, cacheKey(resource));
notModifiedQuery.run();
return false;
}
@@ -458,7 +469,7 @@ bool OfflineDatabase::putResource(const Resource& resource,
updateQuery.bind(4, response.mustRevalidate);
updateQuery.bind(5, response.modified);
updateQuery.bind(6, util::now());
updateQuery.bind(9, resource.url);
updateQuery.bind(9, cacheKey(resource));
if (response.noContent) {
updateQuery.bind(7, nullptr);
@@ -479,7 +490,7 @@ bool OfflineDatabase::putResource(const Resource& resource,
"VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9) ") };
// clang-format on
insertQuery.bind(1, resource.url);
insertQuery.bind(1, cacheKey(resource));
insertQuery.bind(2, int(resource.kind));
insertQuery.bind(3, response.etag);
insertQuery.bind(4, response.expires);
@@ -1155,7 +1166,7 @@ bool OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) {
// clang-format on
insertQuery.bind(1, regionID);
insertQuery.bind(2, resource.url);
insertQuery.bind(2, cacheKey(resource));
insertQuery.run();
if (insertQuery.changes() == 0) {
@@ -1172,7 +1183,7 @@ bool OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) {
// clang-format on
selectQuery.bind(1, regionID);
selectQuery.bind(2, resource.url);
selectQuery.bind(2, cacheKey(resource));
return !selectQuery.run();
}
}

55
platform/default/src/mbgl/storage/pmtiles_file_source.cpp
View File

@@ -143,7 +143,7 @@ public:
}
Resource tileResource(Resource::Kind::Source, url);
tileResource.loadingMethod = Resource::LoadingMethod::Network;
tileResource.loadingMethod = Resource::LoadingMethod::All;
tileResource.dataRange = std::make_pair(tileAddress.first,
tileAddress.first + tileAddress.second - 1);
@@ -159,6 +159,17 @@ public:
return;
}
if (tileResponse.notModified) {
return;
}
if (!tileResponse.data) {
response.error = std::make_unique<Response::Error>(
Response::Error::Reason::Other, "Error fetching PMTiles tile: response has no data");
ref.invoke(&FileSourceRequest::setResponse, response);
return;
}
response.data = tileResponse.data;
response.noContent = false;
response.modified = tileResponse.modified;
@@ -232,10 +243,11 @@ private:
void getHeader(const std::string& url, AsyncRequest* req, AsyncCallback callback) {
if (header_cache.contains(url)) {
callback(std::unique_ptr<Response::Error>());
return;
}
Resource resource(Resource::Kind::Source, url);
resource.loadingMethod = Resource::LoadingMethod::Network;
resource.loadingMethod = Resource::LoadingMethod::All;
resource.dataRange = std::make_pair<uint64_t, uint64_t>(pmtilesHeaderOffset,
pmtilesHeaderOffset + pmtilesHeaderLength - 1);
@@ -260,6 +272,16 @@ private:
return;
}
if (response.notModified) {
return;
}
if (!response.data) {
callback(std::make_unique<Response::Error>(Response::Error::Reason::Other,
"PMTiles header response has no data"));
return;
}
try {
pmtiles::headerv3 header = pmtiles::deserialize_header(response.data->substr(0, 127));
@@ -283,6 +305,7 @@ private:
void getMetadata(std::string& url, AsyncRequest* req, AsyncCallback callback) {
if (metadata_cache.contains(url)) {
callback(std::unique_ptr<Response::Error>());
return;
}
getHeader(
@@ -397,7 +420,7 @@ private:
if (header.json_metadata_bytes > 0) {
Resource resource(Resource::Kind::Source, url);
resource.loadingMethod = Resource::LoadingMethod::Network;
resource.loadingMethod = Resource::LoadingMethod::All;
resource.dataRange = std::make_pair(header.json_metadata_offset,
header.json_metadata_offset + header.json_metadata_bytes - 1);
@@ -410,6 +433,20 @@ private:
return;
}
if (responseMetadata.notModified) {
return;
}
if (!responseMetadata.data) {
callback(std::make_unique<Response::Error>(
Response::Error::Reason::Other,
"PMTiles metadata response has no data (range=" +
std::to_string(header.json_metadata_offset) + "-" +
std::to_string(header.json_metadata_offset + header.json_metadata_bytes - 1) +
")"));
return;
}
std::string data = *responseMetadata.data;
if (header.internal_compression == pmtiles::COMPRESSION_GZIP) {
@@ -480,7 +517,7 @@ private:
pmtiles::headerv3 header = header_cache.at(url);
Resource resource(Resource::Kind::Source, url);
resource.loadingMethod = Resource::LoadingMethod::Network;
resource.loadingMethod = Resource::LoadingMethod::All;
resource.dataRange = std::make_pair(directoryOffset, directoryOffset + directoryLength - 1);
tasks[req] = getFileSource()->request(resource, [=, this](const Response& response) {
@@ -492,6 +529,16 @@ private:
return;
}
if (response.notModified) {
return;
}
if (!response.data) {
callback(std::make_unique<Response::Error>(Response::Error::Reason::Other,
"PMTiles directory response has no data"));
return;
}
try {
std::string directoryData = *response.data;

2
platform/ios/MapLibre.docc/PMTiles.md
View File

@@ -4,7 +4,7 @@ Working with PMTiles
Starting MapLibre iOS 6.10.0, using [PMTiles](https://docs.protomaps.com/pmtiles/) as a data source is supported. You can prefix your vector tile source with `pmtiles://` to load a PMTiles file. The rest of the URL continue with be `https://` to load a remote PMTiles file, `asset://` to load an asset or `file://` to load a local PMTiles file.
> Note: PMTiles sources currently do not support caching or offline pack downloads.
> Note: PMTiles sources do not currently support offline pack downloads.
Oliver Wipfli has made a style available that combines a [Protomaps]() basemap togehter with Foursquare's POI dataset. It is available in the [wipfli/foursquare-os-places-pmtiles](https://github.com/wipfli/foursquare-os-places-pmtiles) repository on GitHub. The style to use is

16
test/storage/offline_database.test.cpp
View File

@@ -396,6 +396,22 @@ TEST(OfflineDatabase, PutTile) {
EXPECT_EQ(0u, log.uncheckedCount());
}
// Cache-key encoding for byte-range sources (eg. PMTiles). The range is appended
// to the URL so distinct ranges occupy distinct cache rows under the resources
// table's UNIQUE(url) constraint.
TEST(OfflineDatabase, CacheKey) {
Resource bare{Resource::Kind::Source, "https://example.com/x.pmtiles"};
EXPECT_EQ("https://example.com/x.pmtiles", OfflineDatabase::cacheKey(bare));
Resource ranged{Resource::Kind::Source, "https://example.com/x.pmtiles"};
ranged.dataRange = std::make_pair<uint64_t, uint64_t>(0, 126);
EXPECT_EQ("https://example.com/x.pmtiles?_mlnRange=0-126", OfflineDatabase::cacheKey(ranged));
Resource rangedWithQuery{Resource::Kind::Source, "https://example.com/x.pmtiles?token=abc"};
rangedWithQuery.dataRange = std::make_pair<uint64_t, uint64_t>(200, 399);
EXPECT_EQ("https://example.com/x.pmtiles?token=abc&_mlnRange=200-399", OfflineDatabase::cacheKey(rangedWithQuery));
}
TEST(OfflineDatabase, PutResourceNoContent) {
FixtureLog log;
OfflineDatabase db(":memory:", fixture::tileServerOptions);

35
test/storage/pmtiles_file_source.test.cpp
View File

@@ -124,6 +124,41 @@ TEST(PMTilesFileSource, CorruptGzipTile) {
loop.run();
}
// Regression test for https://github.com/maplibre/maplibre-native/issues/3690 — the loading
// method for PMTiles sub-requests changed from `Network` to `All` so that http(s)-hosted
// archives flow through the ambient cache. The file:// path bypasses the cache by design,
// so this test can only confirm that issuing the same tile twice still succeeds; a true
// cache-hit assertion requires an http(s) PMTiles fixture and a mock server.
TEST(PMTilesFileSource, RepeatedTileRequest) {
util::RunLoop loop;
PMTilesFileSource pmtiles(ResourceOptions::Default(), ClientOptions());
std::unique_ptr<AsyncRequest> req1;
std::unique_ptr<AsyncRequest> req2;
req1 = pmtiles.request(
Resource::tile(toAbsoluteURL("geography-class-png.pmtiles"), 1.0, 0, 0, 0, Tileset::Scheme::XYZ),
[&](Response res1) {
req1.reset();
ASSERT_EQ(nullptr, res1.error);
ASSERT_TRUE(res1.data.get());
const auto firstSize = res1.data->size();
req2 = pmtiles.request(
Resource::tile(toAbsoluteURL("geography-class-png.pmtiles"), 1.0, 0, 0, 0, Tileset::Scheme::XYZ),
[&, firstSize](Response res2) {
req2.reset();
EXPECT_EQ(nullptr, res2.error);
ASSERT_TRUE(res2.data.get());
EXPECT_EQ(firstSize, res2.data->size());
loop.stop();
});
});
loop.run();
}
// An archive whose header flags tile_compression=GZIP but whose individual tiles are stored
// without compression must be served without error (uncompressed bytes passed through as-is).
TEST(PMTilesFileSource, UncompressedTile) {