tei/MapLibreParticleLab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added the mbtiles file source for rendering vector tiles from file stored locally on the device.

Browse Source
This commit is contained in:
Petr Pokorny
2021-01-06 14:06:03 +01:00
parent 943e7c22dc
commit 2f67fd30d8
14 changed files with 406 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
.gitignore vendored
View File

@@ -3,3 +3,12 @@
/platform/android/MapboxGLAndroidSDKTestApp/src/main/res/values/developer-config.xml
/platform/android/local.properties
/platform/android/gradle/configuration.gradle
/platform/android/arm64-v8a
/platform/android/armeabi-v7a
/platform/android/MapboxGLAndroidSDK/.cxx
/platform/android/MapboxGLAndroidSDK/build
/platform/android/MapboxGLAndroidSDKTestApp/.cxx
/platform/android/MapboxGLAndroidSDKTestApp/build
/platform/android/.gradle
/platform/android/x86
/platform/android/x86_64

1
CMakeLists.txt
View File

@@ -529,6 +529,7 @@ add_library(
${PROJECT_SOURCE_DIR}/src/mbgl/sprite/sprite_parser.cpp
${PROJECT_SOURCE_DIR}/src/mbgl/sprite/sprite_parser.hpp
${PROJECT_SOURCE_DIR}/src/mbgl/storage/asset_file_source.hpp
${PROJECT_SOURCE_DIR}/src/mbgl/storage/mbtiles_file_source.hpp
${PROJECT_SOURCE_DIR}/src/mbgl/storage/file_source_manager.cpp
${PROJECT_SOURCE_DIR}/src/mbgl/storage/http_file_source.hpp
${PROJECT_SOURCE_DIR}/src/mbgl/storage/local_file_source.hpp

1
include/mbgl/storage/file_source.hpp
View File

@@ -22,6 +22,7 @@ enum FileSourceType : uint8_t {
Database,
FileSystem,
Network,
Mbtiles,
// Resource loader acts as a proxy and has logic
// for request delegation to Asset, Cache, and other
// file sources.

1
include/mbgl/util/constants.hpp
View File

@@ -62,6 +62,7 @@ constexpr int DEFAULT_RATE_LIMIT_TIMEOUT = 5;
constexpr const char* API_BASE_URL = "https://api.mapbox.com";
constexpr const char* ASSET_PROTOCOL = "asset://";
constexpr const char* FILE_PROTOCOL = "file://";
constexpr const char* MBTILES_PROTOCOL = "mbtiles://";
constexpr uint32_t DEFAULT_MAXIMUM_CONCURRENT_REQUESTS = 20;
constexpr uint8_t TERRAIN_RGB_MAXZOOM = 15;

4
platform/android/CHANGELOG.md
View File

@@ -2,6 +2,10 @@
Mapbox welcomes participation and contributions from everyone. Please read [`Contributing Guide`](https://github.com/mapbox/mapbox-gl-native/blob/master/CONTRIBUTING.md) to get started.
## 9.3.0 - January 6, 2021
### Features
- Added the mbtiles file source for rendering vector tiles from file stored locally on the device.
## 9.2.1 - May 15, 2020
### Improvements and bug fixes
- Added compiler and linker options to reduce binary size for release build. ([#382](https://github.com/mapbox/mapbox-gl-native-android/pull/382))

1
platform/android/android.cmake
View File

@@ -57,6 +57,7 @@ target_sources(
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/file_source_request.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/local_file_request.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/local_file_source.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/mbtiles_file_source.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/main_resource_loader.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/offline.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/offline_database.cpp

4
platform/default/src/mbgl/storage/file_source_manager.cpp
View File

@@ -4,6 +4,7 @@
#include <mbgl/storage/local_file_source.hpp>
#include <mbgl/storage/main_resource_loader.hpp>
#include <mbgl/storage/online_file_source.hpp>
#include <mbgl/storage/mbtiles_file_source.hpp>
#include <mbgl/storage/resource_options.hpp>
namespace mbgl {
@@ -26,6 +27,9 @@ public:
registerFileSourceFactory(FileSourceType::FileSystem,
[](const ResourceOptions&) { return std::make_unique<LocalFileSource>(); });
registerFileSourceFactory(FileSourceType::Mbtiles,
[](const ResourceOptions&) { return std::make_unique<MaptilerFileSource>(); });
registerFileSourceFactory(FileSourceType::Network, [](const ResourceOptions& options) {
std::unique_ptr<FileSource> networkSource = std::make_unique<OnlineFileSource>();
networkSource->setProperty(ACCESS_TOKEN_KEY, options.accessToken());

24
platform/default/src/mbgl/storage/main_resource_loader.cpp
View File

@@ -19,11 +19,13 @@ public:
MainResourceLoaderThread(std::shared_ptr<FileSource> assetFileSource_,
std::shared_ptr<FileSource> databaseFileSource_,
std::shared_ptr<FileSource> localFileSource_,
std::shared_ptr<FileSource> onlineFileSource_)
std::shared_ptr<FileSource> onlineFileSource_,
std::shared_ptr<FileSource> maptilerFileSource_)
: assetFileSource(std::move(assetFileSource_)),
databaseFileSource(std::move(databaseFileSource_)),
localFileSource(std::move(localFileSource_)),
onlineFileSource(std::move(onlineFileSource_)) {}
onlineFileSource(std::move(onlineFileSource_)),
maptilerFileSource(std::move(maptilerFileSource_)) {}
void request(AsyncRequest* req, const Resource& resource, const ActorRef<FileSourceRequest>& ref) {
auto callback = [ref](const Response& res) { ref.invoke(&FileSourceRequest::setResponse, res); };
@@ -63,6 +65,9 @@ public:
if (assetFileSource && assetFileSource->canRequest(resource)) {
// Asset request
tasks[req] = assetFileSource->request(resource, callback);
} else if (maptilerFileSource && maptilerFileSource->canRequest(resource)) {
// Local file request
tasks[req] = maptilerFileSource->request(resource, callback);
} else if (localFileSource && localFileSource->canRequest(resource)) {
// Local file request
tasks[req] = localFileSource->request(resource, callback);
@@ -122,6 +127,7 @@ private:
const std::shared_ptr<FileSource> databaseFileSource;
const std::shared_ptr<FileSource> localFileSource;
const std::shared_ptr<FileSource> onlineFileSource;
const std::shared_ptr<FileSource> maptilerFileSource;
std::map<AsyncRequest*, std::unique_ptr<AsyncRequest>> tasks;
};
@@ -130,11 +136,13 @@ public:
Impl(std::shared_ptr<FileSource> assetFileSource_,
std::shared_ptr<FileSource> databaseFileSource_,
std::shared_ptr<FileSource> localFileSource_,
std::shared_ptr<FileSource> onlineFileSource_)
std::shared_ptr<FileSource> onlineFileSource_,
std::shared_ptr<FileSource> maptilerFileSource_)
: assetFileSource(std::move(assetFileSource_)),
databaseFileSource(std::move(databaseFileSource_)),
localFileSource(std::move(localFileSource_)),
onlineFileSource(std::move(onlineFileSource_)),
maptilerFileSource(std::move(maptilerFileSource_)),
supportsCacheOnlyRequests_(bool(databaseFileSource)),
thread(std::make_unique<util::Thread<MainResourceLoaderThread>>(
util::makeThreadPrioritySetter(platform::EXPERIMENTAL_THREAD_PRIORITY_WORKER),
@@ -142,7 +150,8 @@ public:
assetFileSource,
databaseFileSource,
localFileSource,
onlineFileSource)) {}
onlineFileSource,
maptilerFileSource)) {}
std::unique_ptr<AsyncRequest> request(const Resource& resource, Callback callback) {
auto req = std::make_unique<FileSourceRequest>(std::move(callback));
@@ -158,7 +167,8 @@ public:
return (assetFileSource && assetFileSource->canRequest(resource)) ||
(localFileSource && localFileSource->canRequest(resource)) ||
(databaseFileSource && databaseFileSource->canRequest(resource)) ||
(onlineFileSource && onlineFileSource->canRequest(resource));
(onlineFileSource && onlineFileSource->canRequest(resource)) ||
(maptilerFileSource && maptilerFileSource->canRequest(resource));
}
bool supportsCacheOnlyRequests() const { return supportsCacheOnlyRequests_; }
@@ -172,6 +182,7 @@ private:
const std::shared_ptr<FileSource> databaseFileSource;
const std::shared_ptr<FileSource> localFileSource;
const std::shared_ptr<FileSource> onlineFileSource;
const std::shared_ptr<FileSource> maptilerFileSource;
const bool supportsCacheOnlyRequests_;
const std::unique_ptr<util::Thread<MainResourceLoaderThread>> thread;
};
@@ -180,7 +191,8 @@ MainResourceLoader::MainResourceLoader(const ResourceOptions& options)
: impl(std::make_unique<Impl>(FileSourceManager::get()->getFileSource(FileSourceType::Asset, options),
FileSourceManager::get()->getFileSource(FileSourceType::Database, options),
FileSourceManager::get()->getFileSource(FileSourceType::FileSystem, options),
FileSourceManager::get()->getFileSource(FileSourceType::Network, options))) {}
FileSourceManager::get()->getFileSource(FileSourceType::Network, options),
FileSourceManager::get()->getFileSource(FileSourceType::Mbtiles, options))) {}
MainResourceLoader::~MainResourceLoader() = default;

323
platform/default/src/mbgl/storage/mbtiles_file_source.cpp Normal file
View File

@@ -0,0 +1,323 @@
#include <sstream>
#include <map>
#include <mbgl/platform/settings.hpp>
#include <mbgl/storage/mbtiles_file_source.hpp>
#include <mbgl/storage/file_source_request.hpp>
#include <rapidjson/document.h>
#include <rapidjson/rapidjson.h>
#include <rapidjson/stringbuffer.h>
#include <rapidjson/writer.h>
#include <mbgl/util/thread.hpp>
#include <mbgl/util/url.hpp>
#include <mbgl/util/chrono.hpp>
#include <mbgl/storage/sqlite3.hpp>
#include <zlib.h>
namespace {
//TODO: replace by mbgl::util::MBTILES_PROTOCOL
const std::string maptilerProtocol = "mbtiles://";
bool acceptsURL(const std::string& url) {
return 0 == url.rfind(mbgl::util::MBTILES_PROTOCOL, 0);
}
} // namespace
namespace mbgl {
using namespace rapidjson;
//using namespace mapbox::sqlite;
class MaptilerFileSource::Impl {
public:
explicit Impl(const ActorRef<Impl>&) {}
std::vector<double> &split(const std::string &s, char delim, std::vector<double> &elems) {
std::stringstream ss(s);
std::string item;
while (std::getline(ss, item, delim)) {
double value = atof(item.c_str());
elems.push_back(value);
}
return elems;
}
std::vector<double> split(const std::string &s, char delim) {
std::vector<double> elems;
split(s, delim, elems);
return elems;
}
std::string serialize(Document &doc) {
StringBuffer buffer;
Writer<StringBuffer> writer(buffer);
doc.Accept(writer);
return std::string(buffer.GetString(), buffer.GetSize());
}
std::string url_to_path(const std::string &url) {
return mbgl::util::percentDecode(url.substr(maptilerProtocol.size()));
}
std::string db_path(const std::string &path) {
return path.substr(0, path.find('?'));
}
bool is_compressed(const std::string &v) {
return (((uint8_t) v[0]) == 0x1f) && (((uint8_t) v[1]) == 0x8b);
}
// Some Mbtiles store GZIP-ed tile data, this function is used for decompression
std::string decompress_string(const std::string &data) {
uint32_t OUT_SIZE = 8192;
int ret;
char outbuffer[OUT_SIZE];
std::string outstring;
z_stream zs{};
// Init inflate to gzip mode
if (inflateInit2(&zs, (16 + MAX_WBITS)) != Z_OK)
return "";
zs.next_in = (Bytef *) data.data();
zs.avail_in = (unsigned int) data.size();
// get the decompressed bytes blockwise using repeated calls to inflate
do {
zs.next_out = reinterpret_cast<Bytef *>(outbuffer);
zs.avail_out = OUT_SIZE;
ret = inflate(&zs, 0);
if (outstring.size() < zs.total_out) {
outstring.append(outbuffer, zs.total_out - outstring.size());
}
} while (ret == Z_OK);
inflateEnd(&zs);
if (ret != Z_STREAM_END) { // an error occurred that was not EOF
return "";
}
return outstring;
}
// Generate a tilejson resource from .mbtiles file
void request_tilejson(const Resource &resource, ActorRef<FileSourceRequest> req) {
auto path = url_to_path(resource.url);
Response response;
Document doc;
auto &allocator = doc.GetAllocator();
std::map<std::string, std::string> values;
auto db = mapbox::sqlite::Database::open(path.c_str(), mapbox::sqlite::ReadOnly);
mapbox::sqlite::Statement meta(db, "SELECT * from metadata");
for (mapbox::sqlite::Query q(meta); q.run();) {
auto name = q.get<std::string>(0);
auto val = q.get<std::string>(1);
if (name == "json") {
doc.Parse(val.c_str());
} else {
values[name] = val;
}
}
if (!doc.IsObject()) {
doc.SetObject();
}
values["tilejson"] = "2.0.0";
values["scheme"] = "xyz";
auto format_ptr = values.find("format");
std::string format = format_ptr == values.end() ? "png" : format_ptr->second;
if (format != "pbf" && values.count("scale") == 0) {
values["scale"] = "1";
}
// We use file location with appended parameter query parameter as URL for actual tile data
std::string tile_url = std::string(resource.url + "?file={x}/{y}/{z}." + format);
rapidjson::Value arr(kArrayType);
rapidjson::Value tile_url_val(tile_url, allocator);
arr.PushBack(tile_url_val, allocator);
doc.AddMember("tiles", arr, allocator);
std::string minz;
std::string maxz;
auto minzoom_ptr = values.find("minzoom");
auto maxzoom_ptr = values.find("maxzoom");
if (minzoom_ptr == values.end() || maxzoom_ptr == values.end()) {
mapbox::sqlite::Statement zoom_stmt(db, "SELECT MIN(zoom_level),MAX(zoom_level) from tiles");
for (mapbox::sqlite::Query q(zoom_stmt); q.run();) {
minz = q.get<std::string>(0);
maxz = q.get<std::string>(1);
}
values["minzoom"] = minz;
values["maxzoom"] = maxz;
}
else {
minz = minzoom_ptr->second;
maxz = maxzoom_ptr->second;
}
auto minZoom = std::stoi(minz);
auto maxZoom = std::stoi(maxz);
for (auto const &entry : values) {
if (entry.first == "scale") {
doc.AddMember("scale", std::stod(entry.second), allocator);
} else if (entry.first == "minzoom" || entry.first == "maxzoom") {
auto name = rapidjson::Value(entry.first, allocator);
doc.AddMember(name, std::stoi(entry.second), allocator);
} else if (entry.first == "bounds") {
std::vector<double> x = split(entry.second, ',');
if (x.size() != 4) {
response.error = std::make_unique<Response::Error>(Response::Error::Reason::Other);
req.invoke(&FileSourceRequest::setResponse, response);
return;
}
double cLon = (x[0] + x[2]) / 2;
double cLat = (x[1] + x[3]) / 2;
int cZoom = (minZoom + maxZoom) / 2;
rapidjson::Value bounds_arr(kArrayType);
for (auto vv : x) {
bounds_arr.PushBack(vv, allocator);
}
if (format != "pbf") {
doc.AddMember("bounds", bounds_arr, allocator);
}
rapidjson::Value center_arr(kArrayType);
center_arr.PushBack(cLon, allocator);
center_arr.PushBack(cLat, allocator);
center_arr.PushBack(cZoom, allocator);
doc.AddMember("center", center_arr, allocator);
} else {
rapidjson::Value name = rapidjson::Value(entry.first, allocator);
doc.AddMember(name, rapidjson::Value(entry.second, allocator), allocator);
}
}
response.data = std::make_shared<std::string>(serialize(doc));
req.invoke(&FileSourceRequest::setResponse, response);
}
// Load data for specific tile
void request_tile(const Resource &resource, ActorRef<FileSourceRequest> req) {
std::string base_path = url_to_path(resource.url);
std::string path = db_path(base_path);
auto &db = get_db(path);
int iy = resource.tileData->y;
int iz = resource.tileData->z;
auto x = std::to_string(resource.tileData->x);
auto y = std::to_string((int)(pow(2, iz) - 1) - iy);
auto z = std::to_string(iz);
std::string sql = "SELECT tile_data FROM tiles where zoom_level = " + z +
" AND tile_column = " + x + " AND tile_row = " + y;
mapbox::sqlite::Statement stmt(db, sql.c_str());
Response response;
response.noContent = true;
response.error = std::make_unique<Response::Error>(Response::Error::Reason::Connection,
"Not found in mbtile database");
for (mapbox::sqlite::Query q(stmt); q.run();) {
optional<std::string> data = q.get<optional<std::string>>(0);
if (data) {
response.data = std::make_shared<std::string>(*data);
response.noContent = false;
response.expires = Timestamp::max();
response.etag = resource.url;
response.error.release();
if (is_compressed(*response.data)) {
response.data = std::make_shared<std::string>(decompress_string(*response.data));
}
}
}
req.invoke(&FileSourceRequest::setResponse, response);
}
private:
std::map<std::string, mapbox::sqlite::Database> db_cache;
void close_db(const std::string &path) {
auto ptr = db_cache.find(path);
if (ptr != db_cache.end()) {
db_cache.erase(path);
}
}
void close_all() {
db_cache.clear();
}
// Multiple databases open simultaneoulsy, to effectively support multiple .mbtiles maps
mapbox::sqlite::Database &get_db(const std::string &path) {
auto ptr = db_cache.find(path);
if (ptr != db_cache.end()) {
return ptr->second;
};
auto ptr2 = db_cache.insert(std::pair<std::string, mapbox::sqlite::Database>(path, mapbox::sqlite::Database::open(path.c_str(), mapbox::sqlite::ReadOnly)));
return ptr2.first->second;
}
};
MaptilerFileSource::MaptilerFileSource() :
thread(std::make_unique<util::Thread<Impl>>(
util::makeThreadPrioritySetter(platform::EXPERIMENTAL_THREAD_PRIORITY_FILE), "MaptilerFileSource")) {}
std::unique_ptr<AsyncRequest> MaptilerFileSource::request(const Resource &resource, FileSource::Callback callback) {
auto req = std::make_unique<FileSourceRequest>(std::move(callback));
if (resource.url.find(":///") == std::string::npos) {
Response response;
response.noContent = true;
response.error = std::make_unique<Response::Error>(Response::Error::Reason::Other,
"MaptilerFileSource only supports absolute path urls");
req->actor().invoke(&FileSourceRequest::setResponse, response);
} else {
if (resource.kind == Resource::Tile) {
thread->actor().invoke(&Impl::request_tile, resource, req->actor());
} else {
thread->actor().invoke(&Impl::request_tilejson, resource, req->actor());
}
}
return std::move(req);
}
bool MaptilerFileSource::canRequest(const Resource& resource) const {
return acceptsURL(resource.url);
}
MaptilerFileSource::~MaptilerFileSource() = default;
}

1
platform/ios/ios.cmake
View File

@@ -51,6 +51,7 @@ target_sources(
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/map/map_snapshotter.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/platform/time.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/asset_file_source.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/mbtiles_file_source.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/database_file_source.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/file_source_manager.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/file_source_request.cpp

7
platform/ios/platform/ios/CHANGELOG.md
View File

@@ -2,6 +2,13 @@
Mapbox welcomes participation and contributions from everyone. Please read [CONTRIBUTING.md](../../CONTRIBUTING.md) to get started.
## 5.10.0 - January 6, 2021
### Features
* Added the mbtiles file source for rendering vector tiles from file stored locally on the device.
## 5.9.0
### Styles and rendering

6
platform/ios/platform/macos/CHANGELOG.md
View File

@@ -1,5 +1,11 @@
# Changelog for Mapbox Maps SDK for macOS
## 0.17.0 - January 6, 2021
### Features
* Added the mbtiles file source for rendering vector tiles from file stored locally on the device.
## 0.16.0
### Styles and rendering

1
platform/macos/macos.cmake
View File

@@ -42,6 +42,7 @@ target_sources(
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/map/map_snapshotter.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/platform/time.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/asset_file_source.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/mbtiles_file_source.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/database_file_source.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/file_source_manager.cpp
${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/file_source_request.cpp

29
src/mbgl/storage/mbtiles_file_source.hpp Normal file
View File

@@ -0,0 +1,29 @@
#include <mbgl/storage/file_source.hpp>
#include <mbgl/util/logging.hpp>
#include <mbgl/util/thread.hpp>
#ifndef MBGL_MAPTILER_FILE_SOURCE_H
#define MBGL_MAPTILER_FILE_SOURCE_H
namespace mbgl {
// File source for supporting .mbtiles maps.
// can only load resource URLS that are absolute paths to local files
class MaptilerFileSource : public FileSource {
public:
MaptilerFileSource();
~MaptilerFileSource() override;
std::unique_ptr <AsyncRequest> request(const Resource &, Callback) override;
//static bool acceptsURL(const std::string &url);
bool canRequest(const Resource&) const override;
private:
class Impl;
std::unique_ptr <util::Thread<Impl>> thread; //impl
};
} // namespace mbgl
#endif //MBGL_MAPTILER_FILE_SOURCE_H