tei/MapLibreParticleLab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Initial PMTiles support (#2882)

Browse Source 
Co-authored-by: Bart Louwers <bart.louwers@gmail.com>
This commit is contained in:
tdcosta100
2025-01-07 10:32:17 -03:00
committed by GitHub
parent c7e6b0c796
commit cf9f41cf61
41 changed files with 969 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitmodules vendored
View File

@@ -55,3 +55,6 @@
[submodule "vendor/glslang"]
path = vendor/glslang
url = https://github.com/KhronosGroup/glslang.git
[submodule "vendor/PMTiles"]
path = vendor/PMTiles
url = https://github.com/protomaps/PMTiles.git

1
BUILD.bazel
View File

@@ -129,6 +129,7 @@ cc_library(
"//vendor:eternal",
"//vendor:mapbox-base",
"//vendor:parsedate",
"//vendor:pmtiles",
"//vendor:polylabel",
"//vendor:protozero",
"//vendor:unique_resource",

4
CMakeLists.txt
View File

@@ -11,6 +11,7 @@ option(MLN_WITH_OPENGL "Build with OpenGL renderer" ON)
option(MLN_WITH_EGL "Build with EGL renderer" OFF)
option(MLN_WITH_VULKAN "Build with Vulkan renderer" OFF)
option(MLN_WITH_OSMESA "Build with OSMesa (Software) renderer" OFF)
option(MLN_WITH_PMTILES "Build with PMTiles support" ON)
option(MLN_WITH_WERROR "Make all compilation warnings errors" ON)
option(MLN_LEGACY_RENDERER "Include the legacy rendering pathway" ON)
option(MLN_DRAWABLE_RENDERER "Include the drawable rendering pathway" OFF)
@@ -1450,6 +1451,7 @@ include(${PROJECT_SOURCE_DIR}/vendor/earcut.hpp.cmake)
include(${PROJECT_SOURCE_DIR}/vendor/eternal.cmake)
include(${PROJECT_SOURCE_DIR}/vendor/mapbox-base.cmake)
include(${PROJECT_SOURCE_DIR}/vendor/parsedate.cmake)
include(${PROJECT_SOURCE_DIR}/vendor/pmtiles.cmake)
include(${PROJECT_SOURCE_DIR}/vendor/polylabel.cmake)
include(${PROJECT_SOURCE_DIR}/vendor/protozero.cmake)
include(${PROJECT_SOURCE_DIR}/vendor/tracy.cmake)
@@ -1473,6 +1475,7 @@ target_link_libraries(
mbgl-vendor-earcut.hpp
mbgl-vendor-eternal
mbgl-vendor-parsedate
mbgl-vendor-pmtiles
mbgl-vendor-polylabel
mbgl-vendor-protozero
mbgl-vendor-unique_resource
@@ -1512,6 +1515,7 @@ export(TARGETS
mbgl-vendor-earcut.hpp
mbgl-vendor-eternal
mbgl-vendor-parsedate
mbgl-vendor-pmtiles
mbgl-vendor-polylabel
mbgl-vendor-protozero
mbgl-vendor-unique_resource

1
bazel/core.bzl
View File

@@ -377,6 +377,7 @@ MLN_CORE_SOURCE = [
"src/mbgl/storage/local_file_source.hpp",
"src/mbgl/storage/main_resource_loader.hpp",
"src/mbgl/storage/network_status.cpp",
"src/mbgl/storage/pmtiles_file_source.hpp",
"src/mbgl/storage/resource.cpp",
"src/mbgl/storage/resource_options.cpp",
"src/mbgl/storage/resource_transform.cpp",

1
include/mbgl/storage/file_source.hpp
View File

@@ -25,6 +25,7 @@ enum FileSourceType : uint8_t {
FileSystem,
Network,
Mbtiles,
Pmtiles,
ResourceLoader ///< %Resource loader acts as a proxy and has logic
/// for request delegation to Asset, Cache, and other
/// file sources.

1
include/mbgl/storage/resource.hpp
View File

@@ -95,6 +95,7 @@ public:
// Includes auxiliary data if this is a tile request.
std::optional<TileData> tileData;
std::optional<std::pair<uint64_t, uint64_t>> dataRange = std::nullopt;
std::optional<Timestamp> priorModified = std::nullopt;
std::optional<Timestamp> priorExpires = std::nullopt;
std::optional<std::string> priorEtag = std::nullopt;

1
include/mbgl/util/constants.hpp
View File

@@ -60,6 +60,7 @@ constexpr int DEFAULT_RATE_LIMIT_TIMEOUT = 5;
constexpr const char* ASSET_PROTOCOL = "asset://";
constexpr const char* FILE_PROTOCOL = "file://";
constexpr const char* MBTILES_PROTOCOL = "mbtiles://";
constexpr const char* PMTILES_PROTOCOL = "pmtiles://";
constexpr uint32_t DEFAULT_MAXIMUM_CONCURRENT_REQUESTS = 20;
constexpr uint8_t TERRAIN_RGB_MAXZOOM = 15;

9
platform/android/MapLibreAndroid/src/cpp/http_file_source.cpp
View File

@@ -91,9 +91,15 @@ void RegisterNativeHTTPRequest(jni::JNIEnv& env) {
HTTPRequest::HTTPRequest(jni::JNIEnv& env, const Resource& resource_, FileSource::Callback callback_)
: resource(resource_),
callback(callback_) {
std::string dataRangeStr;
std::string etagStr;
std::string modifiedStr;
if (resource.dataRange) {
dataRangeStr = std::string("bytes=") + std::to_string(resource.dataRange->first) + std::string("-") +
std::to_string(resource.dataRange->second);
}
if (resource.priorEtag) {
etagStr = *resource.priorEtag;
} else if (resource.priorModified) {
@@ -104,13 +110,14 @@ HTTPRequest::HTTPRequest(jni::JNIEnv& env, const Resource& resource_, FileSource
static auto& javaClass = jni::Class<HTTPRequest>::Singleton(env);
static auto constructor =
javaClass.GetConstructor<jni::jlong, jni::String, jni::String, jni::String, jni::jboolean>(env);
javaClass.GetConstructor<jni::jlong, jni::String, jni::String, jni::String, jni::String, jni::jboolean>(env);
javaRequest = jni::NewGlobal(env,
javaClass.New(env,
constructor,
reinterpret_cast<jlong>(this),
jni::Make<jni::String>(env, resource.url),
jni::Make<jni::String>(env, dataRangeStr),
jni::Make<jni::String>(env, etagStr),
jni::Make<jni::String>(env, modifiedStr),
(jboolean)(resource_.usage == Resource::Usage::Offline)));

3
platform/android/MapLibreAndroid/src/main/java/org/maplibre/android/http/HttpRequest.java
View File

@@ -18,12 +18,13 @@ public interface HttpRequest {
* @param httpRequest callback to be invoked when we receive a response
* @param nativePtr the pointer associated to the request
* @param resourceUrl the resource url to download
* @param dataRange http header, used to indicate the part of a resource that the server should return
* @param etag http header, identifier for a specific version of a resource
* @param modified http header, used to determine if a resource hasn't been modified since
* @param offlineUsage flag to indicate a resource will be used for offline, appends offline=true as a query parameter
*/
void executeRequest(HttpResponder httpRequest, long nativePtr, String resourceUrl,
String etag, String modified, boolean offlineUsage);
String dataRange, String etag, String modified, boolean offlineUsage);
/**
* Cancels the request.

5
platform/android/MapLibreAndroid/src/main/java/org/maplibre/android/http/NativeHttpRequest.java
View File

@@ -19,7 +19,8 @@ public class NativeHttpRequest implements HttpResponder {
private long nativePtr;
@Keep
private NativeHttpRequest(long nativePtr, String resourceUrl, String etag, String modified, boolean offlineUsage) {
private NativeHttpRequest(long nativePtr, String resourceUrl, String dataRange, String etag, String modified,
boolean offlineUsage) {
this.nativePtr = nativePtr;
if (resourceUrl.startsWith("local://")) {
@@ -27,7 +28,7 @@ public class NativeHttpRequest implements HttpResponder {
executeLocalRequest(resourceUrl);
return;
}
httpRequest.executeRequest(this, nativePtr, resourceUrl, etag, modified, offlineUsage);
httpRequest.executeRequest(this, nativePtr, resourceUrl, dataRange, etag, modified, offlineUsage);
}
public void cancel() {

8
platform/android/MapLibreAndroid/src/main/java/org/maplibre/android/module/http/HttpRequestImpl.java
View File

@@ -58,7 +58,8 @@ public class HttpRequestImpl implements HttpRequest {
@Override
public void executeRequest(HttpResponder httpRequest, long nativePtr, @NonNull String resourceUrl,
@NonNull String etag, @NonNull String modified, boolean offlineUsage) {
@NonNull String dataRange, @NonNull String etag, @NonNull String modified,
boolean offlineUsage) {
OkHttpCallback callback = new OkHttpCallback(httpRequest);
try {
HttpUrl httpUrl = HttpUrl.parse(resourceUrl);
@@ -74,6 +75,11 @@ public class HttpRequestImpl implements HttpRequest {
.url(resourceUrl)
.tag(resourceUrl.toLowerCase(MapLibreConstants.MAPLIBRE_LOCALE))
.addHeader("User-Agent", userAgentString);
if (dataRange.length() > 0) {
builder.addHeader("Range", dataRange);
}
if (etag.length() > 0) {
builder.addHeader("If-None-Match", etag);
} else if (modified.length() > 0) {

4
platform/android/MapLibreAndroidTestApp/src/main/java/org/maplibre/android/testapp/utils/ExampleHttpRequestImpl.kt
View File

@@ -11,11 +11,11 @@ import org.maplibre.android.module.http.HttpRequestImpl
*/
class ExampleHttpRequestImpl : HttpRequest {
override fun executeRequest(httpRequest: HttpResponder, nativePtr: Long, resourceUrl: String,
etag: String, modified: String, offlineUsage: Boolean)
dataRange: String, etag: String, modified: String, offlineUsage: Boolean)
{
// Load all json documents and any pbf ending with a 0.
if (resourceUrl.endsWith(".json") || resourceUrl.endsWith("0.pbf")) {
impl.executeRequest(httpRequest, nativePtr, resourceUrl, etag, modified, offlineUsage)
impl.executeRequest(httpRequest, nativePtr, resourceUrl, dataRange, etag, modified, offlineUsage)
} else {
// All other requests get an instant 404!
httpRequest.onResponse(

1
platform/android/android.cmake
View File

@@ -61,6 +61,7 @@ target_sources(
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/offline_database.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/offline_download.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/online_file_source.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/$<IF:$<BOOL:${MLN_WITH_PMTILES}>,pmtiles_file_source.cpp,pmtiles_file_source_stub.cpp>
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/sqlite3.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/text/bidi.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/util/compression.cpp

9
platform/darwin/src/http_file_source.mm
View File

@@ -265,6 +265,13 @@ std::unique_ptr<AsyncRequest> HTTPFileSource::request(const Resource& resource,
forHTTPHeaderField:@"If-Modified-Since"];
}
if (resource.dataRange) {
NSString *rangeHeader = [NSString stringWithFormat:@"bytes=%lld-%lld",
static_cast<long long>(resource.dataRange->first),
static_cast<long long>(resource.dataRange->second)];
[req setValue:rangeHeader forHTTPHeaderField:@"Range"];
}
[req addValue:impl->userAgent forHTTPHeaderField:@"User-Agent"];
const bool isTile = resource.kind == mbgl::Resource::Kind::Tile;
@@ -360,7 +367,7 @@ std::unique_ptr<AsyncRequest> HTTPFileSource::request(const Resource& resource,
response.etag = std::string([etag UTF8String]);
}
if (responseCode == 200) {
if (responseCode == 200 || responseCode == 206) {
response.data = std::make_shared<std::string>((const char *)[data bytes], [data length]);
} else if (responseCode == 204 || (responseCode == 404 && isTile)) {
response.noContent = true;

1
platform/default/BUILD.bazel
View File

@@ -57,6 +57,7 @@ cc_library(
"src/mbgl/storage/offline_database.cpp",
"src/mbgl/storage/offline_download.cpp",
"src/mbgl/storage/online_file_source.cpp",
"src/mbgl/storage/pmtiles_file_source.cpp",
"src/mbgl/storage/sqlite3.cpp",
"src/mbgl/text/bidi.cpp",
"src/mbgl/util/compression.cpp",

4
platform/default/include/mbgl/storage/local_file_request.hpp
View File

@@ -8,6 +8,8 @@ template <typename>
class ActorRef;
class FileSourceRequest;
void requestLocalFile(const std::string&, const ActorRef<FileSourceRequest>&);
void requestLocalFile(const std::string& path,
const ActorRef<FileSourceRequest>& req,
const std::optional<std::pair<uint64_t, uint64_t>>& dataRange = std::nullopt);
} // namespace mbgl

6
platform/default/src/mbgl/storage/file_source_manager.cpp
View File

@@ -5,6 +5,7 @@
#include <mbgl/storage/main_resource_loader.hpp>
#include <mbgl/storage/online_file_source.hpp>
#include <mbgl/storage/mbtiles_file_source.hpp>
#include <mbgl/storage/pmtiles_file_source.hpp>
#include <mbgl/storage/resource_options.hpp>
namespace mbgl {
@@ -37,6 +38,11 @@ public:
return std::make_unique<MBTilesFileSource>(resourceOptions, clientOptions);
});
registerFileSourceFactory(FileSourceType::Pmtiles,
[](const ResourceOptions& resourceOptions, const ClientOptions& clientOptions) {
return std::make_unique<PMTilesFileSource>(resourceOptions, clientOptions);
});
registerFileSourceFactory(FileSourceType::Network,
[](const ResourceOptions& resourceOptions, const ClientOptions& clientOptions) {
return std::make_unique<OnlineFileSource>(resourceOptions, clientOptions);

8
platform/default/src/mbgl/storage/http_file_source.cpp
View File

@@ -269,6 +269,12 @@ HTTPRequest::HTTPRequest(HTTPFileSource::Impl *context_, Resource resource_, Fil
resource(std::move(resource_)),
callback(std::move(callback_)),
handle(context->getHandle()) {
if (resource.dataRange) {
const std::string header = std::string("Range: bytes=") + std::to_string(resource.dataRange->first) +
std::string("-") + std::to_string(resource.dataRange->second);
headers = curl_slist_append(headers, header.c_str());
}
// If there's already a response, set the correct etags/modified headers to
// make sure we are getting a 304 response if possible. This avoids
// redownloading unchanged data.
@@ -417,7 +423,7 @@ void HTTPRequest::handleResult(CURLcode code) {
long responseCode = 0;
curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &responseCode);
if (responseCode == 200) {
if (responseCode == 200 || responseCode == 206) {
if (data) {
response->data = std::move(data);
} else {

6
platform/default/src/mbgl/storage/local_file_request.cpp
View File

@@ -11,7 +11,9 @@
namespace mbgl {
void requestLocalFile(const std::string& path, const ActorRef<FileSourceRequest>& req) {
void requestLocalFile(const std::string& path,
const ActorRef<FileSourceRequest>& req,
const std::optional<std::pair<uint64_t, uint64_t>>& dataRange) {
Response response;
struct stat buf;
int result = stat(path.c_str(), &buf);
@@ -21,7 +23,7 @@ void requestLocalFile(const std::string& path, const ActorRef<FileSourceRequest>
} else if (result == -1 && errno == ENOENT) {
response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound);
} else {
auto data = util::readFile(path);
auto data = util::readFile(path, dataRange);
if (!data) {
response.error = std::make_unique<Response::Error>(Response::Error::Reason::Other,
std::string("Cannot read file ") + path);

11
platform/default/src/mbgl/storage/local_file_source.cpp
View File

@@ -25,8 +25,8 @@ public:
: resourceOptions(resourceOptions_.clone()),
clientOptions(clientOptions_.clone()) {}
void request(const std::string& url, const ActorRef<FileSourceRequest>& req) {
if (!acceptsURL(url)) {
void request(const Resource& resource, const ActorRef<FileSourceRequest>& req) {
if (!acceptsURL(resource.url)) {
Response response;
response.error = std::make_unique<Response::Error>(Response::Error::Reason::Other, "Invalid file URL");
req.invoke(&FileSourceRequest::setResponse, response);
@@ -34,8 +34,9 @@ public:
}
// Cut off the protocol and prefix with path.
const auto path = mbgl::util::percentDecode(url.substr(std::char_traits<char>::length(util::FILE_PROTOCOL)));
requestLocalFile(path, req);
const auto path = mbgl::util::percentDecode(
resource.url.substr(std::char_traits<char>::length(util::FILE_PROTOCOL)));
requestLocalFile(path, req, resource.dataRange);
}
void setResourceOptions(ResourceOptions options) {
@@ -77,7 +78,7 @@ LocalFileSource::~LocalFileSource() = default;
std::unique_ptr<AsyncRequest> LocalFileSource::request(const Resource& resource, Callback callback) {
auto req = std::make_unique<FileSourceRequest>(std::move(callback));
impl->actor().invoke(&Impl::request, resource.url, req->actor());
impl->actor().invoke(&Impl::request, resource, req->actor());
return req;
}

26
platform/default/src/mbgl/storage/main_resource_loader.cpp
View File

@@ -21,12 +21,14 @@ public:
std::shared_ptr<FileSource> databaseFileSource_,
std::shared_ptr<FileSource> localFileSource_,
std::shared_ptr<FileSource> onlineFileSource_,
std::shared_ptr<FileSource> mbtilesFileSource_)
std::shared_ptr<FileSource> mbtilesFileSource_,
std::shared_ptr<FileSource> pmtilesFileSource_)
: assetFileSource(std::move(assetFileSource_)),
databaseFileSource(std::move(databaseFileSource_)),
localFileSource(std::move(localFileSource_)),
onlineFileSource(std::move(onlineFileSource_)),
mbtilesFileSource(std::move(mbtilesFileSource_)) {}
mbtilesFileSource(std::move(mbtilesFileSource_)),
pmtilesFileSource(std::move(pmtilesFileSource_)) {}
void request(AsyncRequest* req, const Resource& resource, const ActorRef<FileSourceRequest>& ref) {
auto callback = [ref](const Response& res) {
@@ -70,6 +72,9 @@ public:
} else if (mbtilesFileSource && mbtilesFileSource->canRequest(resource)) {
// Local file request
tasks[req] = mbtilesFileSource->request(resource, callback);
} else if (pmtilesFileSource && pmtilesFileSource->canRequest(resource)) {
// Local file request
tasks[req] = pmtilesFileSource->request(resource, callback);
} else if (localFileSource && localFileSource->canRequest(resource)) {
// Local file request
tasks[req] = localFileSource->request(resource, callback);
@@ -131,6 +136,7 @@ private:
const std::shared_ptr<FileSource> localFileSource;
const std::shared_ptr<FileSource> onlineFileSource;
const std::shared_ptr<FileSource> mbtilesFileSource;
const std::shared_ptr<FileSource> pmtilesFileSource;
std::map<AsyncRequest*, std::unique_ptr<AsyncRequest>> tasks;
};
@@ -142,12 +148,14 @@ public:
std::shared_ptr<FileSource> databaseFileSource_,
std::shared_ptr<FileSource> localFileSource_,
std::shared_ptr<FileSource> onlineFileSource_,
std::shared_ptr<FileSource> mbtilesFileSource_)
std::shared_ptr<FileSource> mbtilesFileSource_,
std::shared_ptr<FileSource> pmtilesFileSource_)
: assetFileSource(std::move(assetFileSource_)),
databaseFileSource(std::move(databaseFileSource_)),
localFileSource(std::move(localFileSource_)),
onlineFileSource(std::move(onlineFileSource_)),
mbtilesFileSource(std::move(mbtilesFileSource_)),
pmtilesFileSource(std::move(pmtilesFileSource_)),
supportsCacheOnlyRequests_(bool(databaseFileSource)),
thread(std::make_unique<util::Thread<MainResourceLoaderThread>>(
util::makeThreadPrioritySetter(platform::EXPERIMENTAL_THREAD_PRIORITY_WORKER),
@@ -156,7 +164,8 @@ public:
databaseFileSource,
localFileSource,
onlineFileSource,
mbtilesFileSource)),
mbtilesFileSource,
pmtilesFileSource)),
resourceOptions(resourceOptions_.clone()),
clientOptions(clientOptions_.clone()) {}
@@ -175,7 +184,8 @@ public:
(localFileSource && localFileSource->canRequest(resource)) ||
(databaseFileSource && databaseFileSource->canRequest(resource)) ||
(onlineFileSource && onlineFileSource->canRequest(resource)) ||
(mbtilesFileSource && mbtilesFileSource->canRequest(resource));
(mbtilesFileSource && mbtilesFileSource->canRequest(resource)) ||
(pmtilesFileSource && pmtilesFileSource->canRequest(resource));
}
bool supportsCacheOnlyRequests() const { return supportsCacheOnlyRequests_; }
@@ -192,6 +202,7 @@ public:
localFileSource->setResourceOptions(options.clone());
onlineFileSource->setResourceOptions(options.clone());
mbtilesFileSource->setResourceOptions(options.clone());
pmtilesFileSource->setResourceOptions(options.clone());
}
ResourceOptions getResourceOptions() {
@@ -207,6 +218,7 @@ public:
localFileSource->setClientOptions(options.clone());
onlineFileSource->setClientOptions(options.clone());
mbtilesFileSource->setClientOptions(options.clone());
pmtilesFileSource->setClientOptions(options.clone());
}
ClientOptions getClientOptions() {
@@ -220,6 +232,7 @@ private:
const std::shared_ptr<FileSource> localFileSource;
const std::shared_ptr<FileSource> onlineFileSource;
const std::shared_ptr<FileSource> mbtilesFileSource;
const std::shared_ptr<FileSource> pmtilesFileSource;
const bool supportsCacheOnlyRequests_;
const std::unique_ptr<util::Thread<MainResourceLoaderThread>> thread;
mutable std::mutex resourceOptionsMutex;
@@ -236,7 +249,8 @@ MainResourceLoader::MainResourceLoader(const ResourceOptions& resourceOptions, c
FileSourceManager::get()->getFileSource(FileSourceType::Database, resourceOptions, clientOptions),
FileSourceManager::get()->getFileSource(FileSourceType::FileSystem, resourceOptions, clientOptions),
FileSourceManager::get()->getFileSource(FileSourceType::Network, resourceOptions, clientOptions),
FileSourceManager::get()->getFileSource(FileSourceType::Mbtiles, resourceOptions, clientOptions))) {}
FileSourceManager::get()->getFileSource(FileSourceType::Mbtiles, resourceOptions, clientOptions),
FileSourceManager::get()->getFileSource(FileSourceType::Pmtiles, resourceOptions, clientOptions))) {}
MainResourceLoader::~MainResourceLoader() = default;

586
platform/default/src/mbgl/storage/pmtiles_file_source.cpp Normal file
View File

@@ -0,0 +1,586 @@
#include <sstream>
#include <map>
#include <mbgl/platform/settings.hpp>
#include <mbgl/storage/file_source_manager.hpp>
#include <mbgl/storage/file_source_request.hpp>
#include <mbgl/storage/pmtiles_file_source.hpp>
#include <mbgl/storage/resource.hpp>
#include <rapidjson/document.h>
#include <rapidjson/rapidjson.h>
#include <rapidjson/stringbuffer.h>
#include <rapidjson/writer.h>
#include <mbgl/util/thread.hpp>
#include <mbgl/util/url.hpp>
#include <mbgl/util/chrono.hpp>
#include <mbgl/util/compression.hpp>
#include <mbgl/util/filesystem.hpp>
#include <pmtiles.hpp>
#include <sys/types.h>
#include <sys/stat.h>
#if defined(__QT__) && (defined(_WIN32) || defined(__EMSCRIPTEN__))
#include <QtZlib/zlib.h>
#else
#include <zlib.h>
#endif
namespace {
// https://github.com/protomaps/PMTiles/blob/main/spec/v3/spec.md#3-header
constexpr int pmtilesHeaderOffset = 0;
constexpr int pmtilesHeaderLength = 127;
// To avoid allocating lots of memory with PMTiles directory caching,
// set a limit so it doesn't grow unlimited
constexpr int MAX_DIRECTORY_CACHE_ENTRIES = 100;
bool acceptsURL(const std::string& url) {
return url.starts_with(mbgl::util::PMTILES_PROTOCOL);
}
std::string extract_url(const std::string& url) {
return url.substr(std::char_traits<char>::length(mbgl::util::PMTILES_PROTOCOL));
}
} // namespace
namespace mbgl {
using namespace rapidjson;
using AsyncCallback = std::function<void(std::unique_ptr<Response::Error>)>;
using AsyncTileCallback = std::function<void(std::pair<uint64_t, uint32_t>, std::unique_ptr<Response::Error>)>;
class PMTilesFileSource::Impl {
public:
explicit Impl(const ActorRef<Impl>&, const ResourceOptions& resourceOptions_, const ClientOptions& clientOptions_)
: resourceOptions(resourceOptions_.clone()),
clientOptions(clientOptions_.clone()) {}
// Generate a tilejson resource from .pmtiles file
void request_tilejson(AsyncRequest* req, const Resource& resource, const ActorRef<FileSourceRequest>& ref) {
auto url = extract_url(resource.url);
getMetadata(url, req, [=, this](std::unique_ptr<Response::Error> error) {
Response response;
if (error) {
response.error = std::move(error);
ref.invoke(&FileSourceRequest::setResponse, response);
return;
}
response.data = std::make_shared<std::string>(metadata_cache.at(url));
ref.invoke(&FileSourceRequest::setResponse, response);
});
}
// Load data for specific tile
void request_tile(AsyncRequest* req, const Resource& resource, ActorRef<FileSourceRequest> ref) {
auto url = extract_url(resource.url);
getHeader(url, req, [=, this](std::unique_ptr<Response::Error> error) {
if (error) {
Response response;
response.noContent = true;
response.error = std::move(error);
ref.invoke(&FileSourceRequest::setResponse, response);
return;
}
pmtiles::headerv3 header = header_cache.at(url);
if (resource.tileData->z < header.min_zoom || resource.tileData->z > header.max_zoom) {
Response response;
response.noContent = true;
ref.invoke(&FileSourceRequest::setResponse, response);
return;
}
uint64_t tileID;
try {
tileID = pmtiles::zxy_to_tileid(static_cast<uint8_t>(resource.tileData->z),
static_cast<uint32_t>(resource.tileData->x),
static_cast<uint32_t>(resource.tileData->y));
} catch (const std::exception& e) {
Response response;
response.noContent = true;
response.error = std::make_unique<Response::Error>(Response::Error::Reason::Other,
std::string("Invalid tile: ") + e.what());
ref.invoke(&FileSourceRequest::setResponse, response);
return;
}
getTileAddress(
url,
req,
tileID,
header.root_dir_offset,
header.root_dir_bytes,
0,
[=, this](std::pair<uint64_t, uint32_t> tileAddress, std::unique_ptr<Response::Error> tileError) {
if (tileError) {
Response response;
response.noContent = true;
response.error = std::move(tileError);
ref.invoke(&FileSourceRequest::setResponse, response);
return;
}
if (tileAddress.first == 0 && tileAddress.second == 0) {
Response response;
response.noContent = true;
ref.invoke(&FileSourceRequest::setResponse, response);
return;
}
Resource tileResource(Resource::Kind::Source, url);
tileResource.loadingMethod = Resource::LoadingMethod::Network;
tileResource.dataRange = std::make_pair(tileAddress.first,
tileAddress.first + tileAddress.second - 1);
tasks[req] = getFileSource()->request(tileResource, [=](const Response& tileResponse) {
Response response;
response.noContent = true;
if (tileResponse.error) {
response.error = std::make_unique<Response::Error>(
tileResponse.error->reason,
std::string("Error fetching PMTiles tile: ") + tileResponse.error->message);
ref.invoke(&FileSourceRequest::setResponse, response);
return;
}
response.data = tileResponse.data;
response.noContent = false;
response.modified = tileResponse.modified;
response.expires = tileResponse.expires;
response.etag = tileResponse.etag;
if (header.tile_compression == pmtiles::COMPRESSION_GZIP) {
response.data = std::make_shared<std::string>(util::decompress(*tileResponse.data));
}
ref.invoke(&FileSourceRequest::setResponse, response);
return;
});
});
});
}
void setResourceOptions(ResourceOptions options) {
std::lock_guard<std::mutex> lock(resourceOptionsMutex);
resourceOptions = options;
}
ResourceOptions getResourceOptions() {
std::lock_guard<std::mutex> lock(resourceOptionsMutex);
return resourceOptions.clone();
}
void setClientOptions(ClientOptions options) {
std::lock_guard<std::mutex> lock(clientOptionsMutex);
clientOptions = options;
}
ClientOptions getClientOptions() {
std::lock_guard<std::mutex> lock(clientOptionsMutex);
return clientOptions.clone();
}
private:
mutable std::mutex resourceOptionsMutex;
mutable std::mutex clientOptionsMutex;
ResourceOptions resourceOptions;
ClientOptions clientOptions;
std::shared_ptr<FileSource> fileSource;
std::map<std::string, pmtiles::headerv3> header_cache;
std::map<std::string, std::string> metadata_cache;
std::map<std::string, std::map<std::string, std::vector<pmtiles::entryv3>>> directory_cache;
std::map<std::string, std::vector<std::string>> directory_cache_control;
std::map<AsyncRequest*, std::unique_ptr<AsyncRequest>> tasks;
std::shared_ptr<FileSource> getFileSource() {
if (!fileSource) {
fileSource = FileSourceManager::get()->getFileSource(
FileSourceType::ResourceLoader, resourceOptions, clientOptions);
}
return fileSource;
}
void getHeader(const std::string& url, AsyncRequest* req, AsyncCallback callback) {
if (header_cache.find(url) != header_cache.end()) {
callback(std::unique_ptr<Response::Error>());
}
Resource resource(Resource::Kind::Source, url);
resource.loadingMethod = Resource::LoadingMethod::Network;
resource.dataRange = std::make_pair<uint64_t, uint64_t>(pmtilesHeaderOffset,
pmtilesHeaderOffset + pmtilesHeaderLength - 1);
tasks[req] = getFileSource()->request(resource, [=, this](const Response& response) {
if (response.error) {
std::string message = std::string("Error fetching PMTiles header: ") + response.error->message;
if (response.error->message.empty() && response.error->reason == Response::Error::Reason::NotFound) {
if (url.starts_with(mbgl::util::FILE_PROTOCOL)) {
message += "path not found: " +
url.substr(std::char_traits<char>::length(mbgl::util::FILE_PROTOCOL));
} else {
message += "url not found: " + url;
}
}
callback(std::make_unique<Response::Error>(response.error->reason, message));
return;
}
try {
pmtiles::headerv3 header = pmtiles::deserialize_header(response.data->substr(0, 127));
if ((header.internal_compression != pmtiles::COMPRESSION_NONE &&
header.internal_compression != pmtiles::COMPRESSION_GZIP) ||
(header.tile_compression != pmtiles::COMPRESSION_NONE &&
header.tile_compression != pmtiles::COMPRESSION_GZIP)) {
throw std::runtime_error("Compression method not supported");
}
header_cache.emplace(url, header);
callback(std::unique_ptr<Response::Error>());
} catch (const std::exception& e) {
callback(std::make_unique<Response::Error>(Response::Error::Reason::Other,
std::string("Error parsing PMTiles header: ") + e.what()));
}
});
}
void getMetadata(std::string& url, AsyncRequest* req, AsyncCallback callback) {
if (metadata_cache.find(url) != metadata_cache.end()) {
callback(std::unique_ptr<Response::Error>());
}
getHeader(url, req, [=, this](std::unique_ptr<Response::Error> error) {
if (error) {
callback(std::move(error));
return;
}
pmtiles::headerv3 header = header_cache.at(url);
auto parse_callback = [=, this](const std::string& data) {
Document doc;
auto& allocator = doc.GetAllocator();
if (!data.empty()) {
doc.Parse(data);
}
if (!doc.IsObject()) {
doc.SetObject();
}
doc.AddMember("tilejson", "3.0.0", allocator);
if (!doc.HasMember("scheme")) {
doc.AddMember("scheme", rapidjson::Value().SetString("xyz"), allocator);
}
if (!doc.HasMember("tiles")) {
doc.AddMember("tiles", rapidjson::Value(), allocator);
}
if (!doc["tiles"].IsArray()) {
doc["tiles"] = rapidjson::Value().SetArray().PushBack(
rapidjson::Value().SetString(std::string(util::PMTILES_PROTOCOL + url), allocator), allocator);
}
if (!doc.HasMember("bounds")) {
doc.AddMember("bounds", rapidjson::Value(), allocator);
}
if (!doc["bounds"].IsArray()) {
doc["bounds"] = rapidjson::Value()
.SetArray()
.PushBack(static_cast<double>(header.min_lon_e7) / 1e7, allocator)
.PushBack(static_cast<double>(header.min_lat_e7) / 1e7, allocator)
.PushBack(static_cast<double>(header.max_lon_e7) / 1e7, allocator)
.PushBack(static_cast<double>(header.max_lat_e7) / 1e7, allocator);
}
if (!doc.HasMember("center")) {
doc.AddMember("center", rapidjson::Value(), allocator);
}
if (!doc["center"].IsArray()) {
doc["center"] = rapidjson::Value()
.SetArray()
.PushBack(static_cast<double>(header.center_lon_e7) / 1e7, allocator)
.PushBack(static_cast<double>(header.center_lat_e7) / 1e7, allocator)
.PushBack(header.center_zoom, allocator);
}
if (!doc.HasMember("minzoom")) {
doc.AddMember("minzoom", rapidjson::Value(), allocator);
}
auto& minzoom = doc["minzoom"];
if (minzoom.IsString()) {
minzoom.SetInt(std::atoi(minzoom.GetString()));
}
if (!minzoom.IsNumber()) {
minzoom = rapidjson::Value().SetUint(header.min_zoom);
}
doc["minzoom"] = minzoom;
if (!doc.HasMember("maxzoom")) {
doc.AddMember("maxzoom", rapidjson::Value(), allocator);
}
auto& maxzoom = doc["maxzoom"];
if (maxzoom.IsString()) {
maxzoom.SetInt(std::atoi(maxzoom.GetString()));
}
if (!maxzoom.IsNumber()) {
maxzoom = rapidjson::Value().SetUint(header.max_zoom);
}
doc["maxzoom"] = maxzoom;
std::string metadata = serialize(doc);
metadata_cache.emplace(url, metadata);
callback(std::unique_ptr<Response::Error>());
};
if (header.json_metadata_bytes > 0) {
Resource resource(Resource::Kind::Source, url);
resource.loadingMethod = Resource::LoadingMethod::Network;
resource.dataRange = std::make_pair(header.json_metadata_offset,
header.json_metadata_offset + header.json_metadata_bytes - 1);
tasks[req] = getFileSource()->request(resource, [=](const Response& responseMetadata) {
if (responseMetadata.error) {
callback(std::make_unique<Response::Error>(
responseMetadata.error->reason,
std::string("Error fetching PMTiles metadata: ") + responseMetadata.error->message));
return;
}
std::string data = *responseMetadata.data;
if (header.internal_compression == pmtiles::COMPRESSION_GZIP) {
data = util::decompress(data);
}
parse_callback(data);
});
return;
}
parse_callback(std::string());
});
}
void storeDirectory(const std::string& url,
uint64_t directoryOffset,
uint64_t directoryLength,
const std::string& directoryData) {
if (directory_cache.find(url) == directory_cache.end()) {
directory_cache.emplace(url, std::map<std::string, std::vector<pmtiles::entryv3>>());
directory_cache_control.emplace(url, std::vector<std::string>());
}
std::string directory_cache_key = url + "|" + std::to_string(directoryOffset) + "|" +
std::to_string(directoryLength);
directory_cache.at(url).emplace(directory_cache_key, pmtiles::deserialize_directory(directoryData));
directory_cache_control.at(url).emplace_back(directory_cache_key);
if (directory_cache_control.at(url).size() > MAX_DIRECTORY_CACHE_ENTRIES) {
directory_cache.at(url).erase(directory_cache_control.at(url).front());
directory_cache_control.at(url).erase(directory_cache_control.at(url).begin());
}
}
void getDirectory(const std::string& url,
AsyncRequest* req,
uint64_t directoryOffset,
uint32_t directoryLength,
AsyncCallback callback) {
std::string directory_cache_key = url + "|" + std::to_string(directoryOffset) + "|" +
std::to_string(directoryLength);
if (directory_cache.find(url) != directory_cache.end() &&
directory_cache.at(url).find(directory_cache_key) != directory_cache.at(url).end()) {
if (directory_cache_control.at(url).back() != directory_cache_key) {
directory_cache_control.at(url).emplace_back(directory_cache_key);
for (auto it = directory_cache_control.at(url).begin(); it != directory_cache_control.at(url).end();
++it) {
if (*it == directory_cache_key) {
directory_cache_control.at(url).erase(it);
break;
}
}
}
callback(std::unique_ptr<Response::Error>());
return;
}
getHeader(url, req, [=, this](std::unique_ptr<Response::Error> error) {
if (error) {
callback(std::move(error));
return;
}
pmtiles::headerv3 header = header_cache.at(url);
Resource resource(Resource::Kind::Source, url);
resource.loadingMethod = Resource::LoadingMethod::Network;
resource.dataRange = std::make_pair(directoryOffset, directoryOffset + directoryLength - 1);
tasks[req] = getFileSource()->request(resource, [=, this](const Response& response) {
if (response.error) {
callback(std::make_unique<Response::Error>(
response.error->reason,
std::string("Error fetching PMTiles directory: ") + response.error->message));
return;
}
try {
std::string directoryData = *response.data;
if (header.internal_compression == pmtiles::COMPRESSION_GZIP) {
directoryData = util::decompress(directoryData);
}
storeDirectory(url, directoryOffset, directoryLength, directoryData);
callback(std::unique_ptr<Response::Error>());
} catch (const std::exception& e) {
callback(std::make_unique<Response::Error>(
Response::Error::Reason::Other,
std::string(std::string("Error parsing PMTiles directory: ") + e.what())));
}
});
});
}
void getTileAddress(const std::string& url,
AsyncRequest* req,
uint64_t tileID,
uint64_t directoryOffset,
uint32_t directoryLength,
uint32_t directoryDepth,
AsyncTileCallback callback) {
if (directoryDepth > 3) {
callback(std::make_pair<uint64_t, uint32_t>(0, 0),
std::make_unique<Response::Error>(
Response::Error::Reason::Other,
std::string("Error fetching PMTiles tile address: Maximum directory depth exceeded")));
return;
}
getDirectory(url, req, directoryOffset, directoryLength, [=, this](std::unique_ptr<Response::Error> error) {
if (error) {
callback(std::make_pair(0, 0), std::move(error));
return;
}
pmtiles::headerv3 header = header_cache.at(url);
std::vector<pmtiles::entryv3> directory = directory_cache.at(url).at(
url + "|" + std::to_string(directoryOffset) + "|" + std::to_string(directoryLength));
pmtiles::entryv3 entry = pmtiles::find_tile(directory, tileID);
if (entry.length > 0) {
if (entry.run_length > 0) {
callback(std::make_pair(header.tile_data_offset + entry.offset, entry.length), {});
return;
}
getTileAddress(url,
req,
tileID,
header.leaf_dirs_offset + entry.offset,
entry.length,
directoryDepth + 1,
std::move(callback));
return;
}
callback(std::make_pair(0, 0), {});
});
}
std::string serialize(Document& doc) {
StringBuffer buffer;
Writer<StringBuffer> writer(buffer);
doc.Accept(writer);
return std::string(buffer.GetString(), buffer.GetSize());
}
};
PMTilesFileSource::PMTilesFileSource(const ResourceOptions& resourceOptions, const ClientOptions& clientOptions)
: thread(std::make_unique<util::Thread<Impl>>(
util::makeThreadPrioritySetter(platform::EXPERIMENTAL_THREAD_PRIORITY_FILE),
"PMTilesFileSource",
resourceOptions.clone(),
clientOptions.clone())) {}
std::unique_ptr<AsyncRequest> PMTilesFileSource::request(const Resource& resource, FileSource::Callback callback) {
auto req = std::make_unique<FileSourceRequest>(std::move(callback));
// assume if there is a tile request, that the pmtiles file has been validated
if (resource.kind == Resource::Tile) {
thread->actor().invoke(&Impl::request_tile, req.get(), resource, req->actor());
return req;
}
// return TileJSON
thread->actor().invoke(&Impl::request_tilejson, req.get(), resource, req->actor());
return req;
}
bool PMTilesFileSource::canRequest(const Resource& resource) const {
return acceptsURL(resource.url);
}
PMTilesFileSource::~PMTilesFileSource() = default;
void PMTilesFileSource::setResourceOptions(ResourceOptions options) {
thread->actor().invoke(&Impl::setResourceOptions, options.clone());
}
ResourceOptions PMTilesFileSource::getResourceOptions() {
return thread->actor().ask(&Impl::getResourceOptions).get();
}
void PMTilesFileSource::setClientOptions(ClientOptions options) {
thread->actor().invoke(&Impl::setClientOptions, options.clone());
}
ClientOptions PMTilesFileSource::getClientOptions() {
return thread->actor().ask(&Impl::getClientOptions).get();
}
} // namespace mbgl

38
platform/default/src/mbgl/storage/pmtiles_file_source_stub.cpp Normal file
View File

@@ -0,0 +1,38 @@
#include <mbgl/storage/file_source_request.hpp>
#include <mbgl/storage/pmtiles_file_source.hpp>
#include <mbgl/storage/resource.hpp>
#include <mbgl/util/async_request.hpp>
namespace mbgl {
class PMTilesFileSource::Impl {
public:
Impl() = default;
~Impl() = default;
};
PMTilesFileSource::PMTilesFileSource(const ResourceOptions& resourceOptions, const ClientOptions& clientOptions) {}
std::unique_ptr<AsyncRequest> PMTilesFileSource::request(const Resource& resource, FileSource::Callback callback) {
return nullptr;
}
bool PMTilesFileSource::canRequest(const Resource& resource) const {
return false;
}
PMTilesFileSource::~PMTilesFileSource() = default;
void PMTilesFileSource::setResourceOptions(ResourceOptions options) {}
ResourceOptions PMTilesFileSource::getResourceOptions() {
return {};
}
void PMTilesFileSource::setClientOptions(ClientOptions options) {}
ClientOptions PMTilesFileSource::getClientOptions() {
return {};
}
} // namespace mbgl

1
platform/linux/linux.cmake
View File

@@ -50,6 +50,7 @@ target_sources(
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/offline_database.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/offline_download.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/online_file_source.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/$<IF:$<BOOL:${MLN_WITH_PMTILES}>,pmtiles_file_source.cpp,pmtiles_file_source_stub.cpp>
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/sqlite3.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/text/bidi.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/text/local_glyph_rasterizer.cpp

1
platform/macos/macos.cmake
View File

@@ -92,6 +92,7 @@ target_sources(
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/offline_database.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/offline_download.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/online_file_source.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/$<IF:$<BOOL:${MLN_WITH_PMTILES}>,pmtiles_file_source.cpp,pmtiles_file_source_stub.cpp>
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/sqlite3.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/text/bidi.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/util/compression.cpp

1
platform/qt/qt.cmake
View File

@@ -84,6 +84,7 @@ target_sources(
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/offline_database.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/offline_download.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/online_file_source.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/$<IF:$<BOOL:${MLN_WITH_PMTILES}>,pmtiles_file_source.cpp,pmtiles_file_source_stub.cpp>
${PROJECT_SOURCE_DIR}/platform/$<IF:$<BOOL:${MLN_QT_WITH_INTERNAL_SQLITE}>,default/src/mbgl/storage/sqlite3.cpp,qt/src/mbgl/sqlite3.cpp>
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/util/compression.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/util/filesystem.cpp

9
platform/qt/src/mbgl/http_request.cpp
View File

@@ -49,6 +49,12 @@ QNetworkRequest HTTPRequest::networkRequest() const {
req.setRawHeader("User-Agent", agent);
#endif
if (m_resource.dataRange) {
std::string range = std::string("bytes=") + std::to_string(m_resource.dataRange->first) + std::string("-") +
std::to_string(m_resource.dataRange->second);
req.setRawHeader("Range", QByteArray(range.data(), static_cast<int>(range.size())));
}
if (m_resource.priorEtag) {
const auto etag = m_resource.priorEtag;
req.setRawHeader("If-None-Match", QByteArray(etag->data(), static_cast<int>(etag->size())));
@@ -112,7 +118,8 @@ void HTTPRequest::handleNetworkReply(QNetworkReply* reply, const QByteArray& dat
int responseCode = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
switch (responseCode) {
case 200: {
case 200:
case 206: {
if (data.isEmpty()) {
response.data = std::make_shared<std::string>();
} else {

1
platform/windows/windows.cmake
View File

@@ -51,6 +51,7 @@ target_sources(
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/offline_database.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/offline_download.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/online_file_source.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/$<IF:$<BOOL:${MLN_WITH_PMTILES}>,pmtiles_file_source.cpp,pmtiles_file_source_stub.cpp>
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/sqlite3.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/text/bidi.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/text/local_glyph_rasterizer.cpp

29
src/mbgl/storage/pmtiles_file_source.hpp Normal file
View File

@@ -0,0 +1,29 @@
#pragma once
#include <mbgl/storage/file_source.hpp>
#include <mbgl/storage/resource_options.hpp>
#include <mbgl/util/client_options.hpp>
#include <mbgl/util/thread.hpp>
namespace mbgl {
// File source for supporting .pmtiles maps
class PMTilesFileSource : public FileSource {
public:
PMTilesFileSource(const ResourceOptions& resourceOptions, const ClientOptions& clientOptions);
~PMTilesFileSource() override;
std::unique_ptr<AsyncRequest> request(const Resource&, Callback) override;
bool canRequest(const Resource&) const override;
void setResourceOptions(ResourceOptions) override;
ResourceOptions getResourceOptions() override;
void setClientOptions(ClientOptions) override;
ClientOptions getClientOptions() override;
private:
class Impl;
std::unique_ptr<util::Thread<Impl>> thread; // impl
};
} // namespace mbgl

11
src/mbgl/util/io.cpp
View File

@@ -46,15 +46,24 @@ std::string read_file(const std::string &filename) {
}
}
std::optional<std::string> readFile(const std::string &filename) {
std::optional<std::string> readFile(const std::string &filename,
const std::optional<std::pair<uint64_t, uint64_t>> &dataRange) {
MLN_TRACE_FUNC();
std::ifstream file(filename, std::ios::binary);
if (file.good()) {
if (dataRange) {
size_t size = static_cast<size_t>(dataRange->second - dataRange->first + 1);
std::string data(size, '\0');
file.seekg(static_cast<std::streampos>(dataRange->first));
file.read(&data[0], static_cast<std::streamsize>(size));
return data;
} else {
std::stringstream data;
data << file.rdbuf();
return data.str();
}
}
return {};
}

4
src/mbgl/util/io.hpp
View File

@@ -1,5 +1,6 @@
#pragma once
#include <cstdint>
#include <string>
#include <stdexcept>
#include <optional>
@@ -15,7 +16,8 @@ struct IOException : std::runtime_error {
void write_file(const std::string& filename, const std::string& data);
std::string read_file(const std::string& filename);
std::optional<std::string> readFile(const std::string& filename);
std::optional<std::string> readFile(const std::string& filename,
const std::optional<std::pair<uint64_t, uint64_t>>& dataRange = std::nullopt);
void deleteFile(const std::string& filename);
void copyFile(const std::string& destination, const std::string& source);

1
test/CMakeLists.txt
View File

@@ -41,6 +41,7 @@ add_library(
${PROJECT_SOURCE_DIR}/test/storage/offline_database.test.cpp
${PROJECT_SOURCE_DIR}/test/storage/offline_download.test.cpp
${PROJECT_SOURCE_DIR}/test/storage/online_file_source.test.cpp
${PROJECT_SOURCE_DIR}/test/storage/pmtiles_file_source.test.cpp
${PROJECT_SOURCE_DIR}/test/storage/resource.test.cpp
${PROJECT_SOURCE_DIR}/test/storage/sqlite.test.cpp
${PROJECT_SOURCE_DIR}/test/style/conversion/conversion_impl.test.cpp

BIN
test/fixtures/storage/pmtiles/geography-class-png.pmtiles vendored Normal file
View File

Binary file not shown.

13
test/src/mbgl/test/http_server.cpp
View File

@@ -9,6 +9,8 @@
#include <httplib.h>
#include <atomic>
#include <cstdint>
#include <string>
using namespace std::literals::string_literals;
@@ -19,6 +21,7 @@ void runServer(std::unique_ptr<httplib::Server>& server) {
using namespace httplib;
server->Get("/test", [](const Request& req, Response& res) {
std::string content = "Hello World!";
if (req.has_param("modified")) {
std::string str = util::rfc1123(util::parseTimestamp(std::stoi(req.get_param_value("modified"))));
res.set_header("Last-Modified", str);
@@ -33,7 +36,15 @@ void runServer(std::unique_ptr<httplib::Server>& server) {
if (req.has_param("cachecontrol")) {
res.set_header("Cache-Control", "max-age=" + req.get_param_value("cachecontrol"));
}
res.set_content("Hello World!", "text/plain");
if (req.has_param("range")) {
std::string str = req.get_param_value("range");
str = str.substr(std::char_traits<char>::length("bytes="));
uint64_t start = std::strtoull(str.substr(0, str.find("-")).c_str(), nullptr, 10);
uint64_t end = std::strtoull(str.substr(str.find("-") + 1).c_str(), nullptr, 10);
content = content.substr(start, end - start + 1);
res.status = 206;
}
res.set_content(content, "text/plain");
});
server->Get("/stale", [](const Request&, Response&) {

21
test/storage/http_file_source.test.cpp
View File

@@ -37,6 +37,27 @@ TEST(HTTPFileSource, TEST_REQUIRES_SERVER(HTTP200)) {
loop.run();
}
TEST(HTTPFileSource, TEST_REQUIRES_SERVER(HTTP206)) {
util::RunLoop loop;
HTTPFileSource fs(ResourceOptions::Default(), ClientOptions());
Resource resource(Resource::Unknown, "http://127.0.0.1:3000/test");
resource.dataRange = std::make_pair<uint64_t, uint64_t>(3, 8);
auto req = fs.request(resource, [&](Response res) {
EXPECT_EQ(nullptr, res.error);
ASSERT_TRUE(res.data.get());
EXPECT_EQ("lo Wor", *res.data);
EXPECT_FALSE(bool(res.expires));
EXPECT_FALSE(res.mustRevalidate);
EXPECT_FALSE(bool(res.modified));
EXPECT_FALSE(bool(res.etag));
loop.stop();
});
loop.run();
}
TEST(HTTPFileSource, TEST_REQUIRES_SERVER(HTTP404)) {
util::RunLoop loop;
HTTPFileSource fs(ResourceOptions::Default(), ClientOptions());

22
test/storage/local_file_source.test.cpp
View File

@@ -5,6 +5,7 @@
#include <mbgl/util/run_loop.hpp>
#include <climits>
#include <cstdint>
#include <gtest/gtest.h>
#if defined(WIN32)
@@ -25,7 +26,7 @@ std::string toAbsoluteURL(const std::string& fileName) {
#else
char* cwd = getcwd(buff, PATH_MAX + 1);
#endif
std::string url = {"file://" + std::string(cwd) + "/test/fixtures/storage/assets/" + fileName};
std::string url = {mbgl::util::FILE_PROTOCOL + std::string(cwd) + "/test/fixtures/storage/assets/" + fileName};
assert(url.size() <= PATH_MAX);
return url;
}
@@ -76,6 +77,25 @@ TEST(LocalFileSource, NonEmptyFile) {
loop.run();
}
TEST(LocalFileSource, PartialFile) {
util::RunLoop loop;
LocalFileSource fs(ResourceOptions::Default(), ClientOptions());
Resource resource(Resource::Unknown, toAbsoluteURL("nonempty"));
resource.dataRange = std::make_pair<uint64_t, uint64_t>(4, 12);
std::unique_ptr<AsyncRequest> req = fs.request(resource, [&](Response res) {
req.reset();
EXPECT_EQ(nullptr, res.error);
ASSERT_TRUE(res.data.get());
EXPECT_EQ("ent is he", *res.data);
loop.stop();
});
loop.run();
}
TEST(LocalFileSource, NonExistentFile) {
util::RunLoop loop;

105
test/storage/pmtiles_file_source.test.cpp Normal file
View File

@@ -0,0 +1,105 @@
#include <mbgl/storage/pmtiles_file_source.hpp>
#include <mbgl/storage/resource.hpp>
#include <mbgl/storage/resource_options.hpp>
#include <mbgl/util/platform.hpp>
#include <mbgl/util/run_loop.hpp>
#include <filesystem>
#include <climits>
#include <gtest/gtest.h>
namespace {
std::string toAbsoluteURL(const std::string &fileName) {
auto path = std::filesystem::current_path() / "test/fixtures/storage/pmtiles" / fileName;
return std::string(mbgl::util::PMTILES_PROTOCOL) + std::string(mbgl::util::FILE_PROTOCOL) + path.string();
}
} // namespace
using namespace mbgl;
TEST(PMTilesFileSource, AcceptsURL) {
PMTilesFileSource pmtiles(ResourceOptions::Default(), ClientOptions());
EXPECT_TRUE(pmtiles.canRequest(Resource::style("pmtiles:///test")));
EXPECT_FALSE(pmtiles.canRequest(Resource::style("pmtile://test")));
EXPECT_FALSE(pmtiles.canRequest(Resource::style("pmtiles:")));
EXPECT_FALSE(pmtiles.canRequest(Resource::style("")));
}
// Nonexistent pmtiles file raises error
TEST(PMTilesFileSource, NonExistentFile) {
util::RunLoop loop;
PMTilesFileSource pmtiles(ResourceOptions::Default(), ClientOptions());
std::unique_ptr<AsyncRequest> req = pmtiles.request(
{Resource::Unknown, toAbsoluteURL("does_not_exist")}, [&](Response res) {
req.reset();
ASSERT_NE(nullptr, res.error);
EXPECT_EQ(Response::Error::Reason::NotFound, res.error->reason);
EXPECT_NE((res.error->message).find("path not found"), std::string::npos);
ASSERT_FALSE(res.data.get());
loop.stop();
});
loop.run();
}
// Existing pmtiles file default request returns TileJSON
TEST(PMTilesFileSource, TileJSON) {
util::RunLoop loop;
PMTilesFileSource pmtiles(ResourceOptions::Default(), ClientOptions());
std::unique_ptr<AsyncRequest> req = pmtiles.request(
{Resource::Unknown, toAbsoluteURL("geography-class-png.pmtiles")}, [&](Response res) {
req.reset();
EXPECT_EQ(nullptr, res.error);
ASSERT_TRUE(res.data.get());
// basic test that TileJSON included a tile URL
EXPECT_NE((*res.data).find("geography-class-png.pmtiles"), std::string::npos);
loop.stop();
});
loop.run();
}
// Existing tiles return tile data
TEST(PMTilesFileSource, Tile) {
util::RunLoop loop;
PMTilesFileSource pmtiles(ResourceOptions::Default(), ClientOptions());
std::unique_ptr<AsyncRequest> req = pmtiles.request(
Resource::tile(toAbsoluteURL("geography-class-png.pmtiles"), 1.0, 0, 0, 0, Tileset::Scheme::XYZ),
[&](Response res) {
req.reset();
EXPECT_EQ(nullptr, res.error);
ASSERT_TRUE(res.data.get());
ASSERT_EQ(res.noContent, false);
loop.stop();
});
loop.run();
}
// Nonexistent tiles do not raise errors, they simply return no content
TEST(PMTilesFileSource, NonExistentTile) {
util::RunLoop loop;
PMTilesFileSource pmtiles(ResourceOptions::Default(), ClientOptions());
std::unique_ptr<AsyncRequest> req = pmtiles.request(
Resource::tile(toAbsoluteURL("geography-class-png.pmtiles"), 1.0, 0, 0, 4, Tileset::Scheme::XYZ),
[&](Response res) {
req.reset();
EXPECT_EQ(nullptr, res.error);
ASSERT_FALSE(res.data.get());
ASSERT_EQ(res.noContent, true);
loop.stop();
});
loop.run();
}

8
test/storage/server.js
View File

@@ -15,6 +15,7 @@ var app = express();
app.disable('etag');
app.get('/test', function (req, res) {
var content = 'Hello World!';
if (req.query.modified) {
res.setHeader('Last-Modified', (new Date(req.query.modified * 1000)).toUTCString());
}
@@ -27,7 +28,12 @@ app.get('/test', function (req, res) {
if (req.query.cachecontrol) {
res.setHeader('Cache-Control', req.query.cachecontrol);
}
res.send('Hello World!');
if (req.range()) {
const [ range ] = req.range();
content = content.substring(range.start, range.end + 1);
res.status(206);
}
res.send(content);
});
app.get('/stale/*', function() {

9
vendor/BUILD.bazel vendored
View File

@@ -154,6 +154,15 @@ cc_library(
visibility = ["//visibility:public"],
)
# vendor/pmtiles-files.json
cc_library(
name = "pmtiles",
hdrs = ["PMTiles/cpp/pmtiles.hpp"],
copts = CPP_FLAGS,
includes = ["PMTiles/cpp"],
visibility = ["//visibility:public"],
)
# vendor/polylabel-files.json
cc_library(
name = "polylabel",

1
vendor/PMTiles vendored Submodule

Submodule vendor/PMTiles added at 60f3331ca9

21
vendor/pmtiles.cmake vendored Normal file
View File

@@ -0,0 +1,21 @@
if(TARGET mbgl-vendor-pmtiles)
return()
endif()
add_library(
mbgl-vendor-pmtiles INTERFACE
)
target_include_directories(
mbgl-vendor-pmtiles SYSTEM
INTERFACE ${CMAKE_CURRENT_LIST_DIR}/PMTiles/cpp
)
set_target_properties(
mbgl-vendor-pmtiles
PROPERTIES
INTERFACE_MAPLIBRE_NAME "pmtiles"
INTERFACE_MAPLIBRE_URL "https://github.com/protomaps/PMTiles"
INTERFACE_MAPLIBRE_AUTHOR "Protomaps LLC"
INTERFACE_MAPLIBRE_LICENSE ${CMAKE_CURRENT_LIST_DIR}/PMTiles/LICENSE
)