diff --git a/.gitignore b/.gitignore index 01147ed88..00b179df6 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,12 @@ /platform/android/MapboxGLAndroidSDKTestApp/src/main/res/values/developer-config.xml /platform/android/local.properties /platform/android/gradle/configuration.gradle +/platform/android/arm64-v8a +/platform/android/armeabi-v7a +/platform/android/MapboxGLAndroidSDK/.cxx +/platform/android/MapboxGLAndroidSDK/build +/platform/android/MapboxGLAndroidSDKTestApp/.cxx +/platform/android/MapboxGLAndroidSDKTestApp/build +/platform/android/.gradle +/platform/android/x86 +/platform/android/x86_64 diff --git a/CMakeLists.txt b/CMakeLists.txt index c9ebb3631..c8eea2361 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -529,6 +529,7 @@ add_library( ${PROJECT_SOURCE_DIR}/src/mbgl/sprite/sprite_parser.cpp ${PROJECT_SOURCE_DIR}/src/mbgl/sprite/sprite_parser.hpp ${PROJECT_SOURCE_DIR}/src/mbgl/storage/asset_file_source.hpp + ${PROJECT_SOURCE_DIR}/src/mbgl/storage/mbtiles_file_source.hpp ${PROJECT_SOURCE_DIR}/src/mbgl/storage/file_source_manager.cpp ${PROJECT_SOURCE_DIR}/src/mbgl/storage/http_file_source.hpp ${PROJECT_SOURCE_DIR}/src/mbgl/storage/local_file_source.hpp diff --git a/include/mbgl/storage/file_source.hpp b/include/mbgl/storage/file_source.hpp index 562fd8da0..b236ff9ee 100644 --- a/include/mbgl/storage/file_source.hpp +++ b/include/mbgl/storage/file_source.hpp @@ -22,6 +22,7 @@ enum FileSourceType : uint8_t { Database, FileSystem, Network, + Mbtiles, // Resource loader acts as a proxy and has logic // for request delegation to Asset, Cache, and other // file sources. diff --git a/include/mbgl/util/constants.hpp b/include/mbgl/util/constants.hpp index bb026817c..345733f22 100644 --- a/include/mbgl/util/constants.hpp +++ b/include/mbgl/util/constants.hpp @@ -62,6 +62,7 @@ constexpr int DEFAULT_RATE_LIMIT_TIMEOUT = 5; constexpr const char* API_BASE_URL = "https://api.mapbox.com"; constexpr const char* ASSET_PROTOCOL = "asset://"; constexpr const char* FILE_PROTOCOL = "file://"; +constexpr const char* MBTILES_PROTOCOL = "mbtiles://"; constexpr uint32_t DEFAULT_MAXIMUM_CONCURRENT_REQUESTS = 20; constexpr uint8_t TERRAIN_RGB_MAXZOOM = 15; diff --git a/platform/android/CHANGELOG.md b/platform/android/CHANGELOG.md index f1fc6c3ba..cd977856f 100644 --- a/platform/android/CHANGELOG.md +++ b/platform/android/CHANGELOG.md @@ -2,6 +2,10 @@ Mapbox welcomes participation and contributions from everyone. Please read [`Contributing Guide`](https://github.com/mapbox/mapbox-gl-native/blob/master/CONTRIBUTING.md) to get started. +## 9.3.0 - January 6, 2021 +### Features + - Added the mbtiles file source for rendering vector tiles from file stored locally on the device. + ## 9.2.1 - May 15, 2020 ### Improvements and bug fixes - Added compiler and linker options to reduce binary size for release build. ([#382](https://github.com/mapbox/mapbox-gl-native-android/pull/382)) diff --git a/platform/android/android.cmake b/platform/android/android.cmake index 625bf3a17..fd9814cac 100644 --- a/platform/android/android.cmake +++ b/platform/android/android.cmake @@ -57,6 +57,7 @@ target_sources( ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/file_source_request.cpp ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/local_file_request.cpp ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/local_file_source.cpp + ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/mbtiles_file_source.cpp ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/main_resource_loader.cpp ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/offline.cpp ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/offline_database.cpp diff --git a/platform/default/src/mbgl/storage/file_source_manager.cpp b/platform/default/src/mbgl/storage/file_source_manager.cpp index 1a13f0556..ff5c65e56 100644 --- a/platform/default/src/mbgl/storage/file_source_manager.cpp +++ b/platform/default/src/mbgl/storage/file_source_manager.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include namespace mbgl { @@ -26,6 +27,9 @@ public: registerFileSourceFactory(FileSourceType::FileSystem, [](const ResourceOptions&) { return std::make_unique(); }); + registerFileSourceFactory(FileSourceType::Mbtiles, + [](const ResourceOptions&) { return std::make_unique(); }); + registerFileSourceFactory(FileSourceType::Network, [](const ResourceOptions& options) { std::unique_ptr networkSource = std::make_unique(); networkSource->setProperty(ACCESS_TOKEN_KEY, options.accessToken()); diff --git a/platform/default/src/mbgl/storage/main_resource_loader.cpp b/platform/default/src/mbgl/storage/main_resource_loader.cpp index 1658ca4a5..813e11fa4 100644 --- a/platform/default/src/mbgl/storage/main_resource_loader.cpp +++ b/platform/default/src/mbgl/storage/main_resource_loader.cpp @@ -19,11 +19,13 @@ public: MainResourceLoaderThread(std::shared_ptr assetFileSource_, std::shared_ptr databaseFileSource_, std::shared_ptr localFileSource_, - std::shared_ptr onlineFileSource_) + std::shared_ptr onlineFileSource_, + std::shared_ptr maptilerFileSource_) : assetFileSource(std::move(assetFileSource_)), databaseFileSource(std::move(databaseFileSource_)), localFileSource(std::move(localFileSource_)), - onlineFileSource(std::move(onlineFileSource_)) {} + onlineFileSource(std::move(onlineFileSource_)), + maptilerFileSource(std::move(maptilerFileSource_)) {} void request(AsyncRequest* req, const Resource& resource, const ActorRef& ref) { auto callback = [ref](const Response& res) { ref.invoke(&FileSourceRequest::setResponse, res); }; @@ -63,6 +65,9 @@ public: if (assetFileSource && assetFileSource->canRequest(resource)) { // Asset request tasks[req] = assetFileSource->request(resource, callback); + } else if (maptilerFileSource && maptilerFileSource->canRequest(resource)) { + // Local file request + tasks[req] = maptilerFileSource->request(resource, callback); } else if (localFileSource && localFileSource->canRequest(resource)) { // Local file request tasks[req] = localFileSource->request(resource, callback); @@ -122,6 +127,7 @@ private: const std::shared_ptr databaseFileSource; const std::shared_ptr localFileSource; const std::shared_ptr onlineFileSource; + const std::shared_ptr maptilerFileSource; std::map> tasks; }; @@ -130,11 +136,13 @@ public: Impl(std::shared_ptr assetFileSource_, std::shared_ptr databaseFileSource_, std::shared_ptr localFileSource_, - std::shared_ptr onlineFileSource_) + std::shared_ptr onlineFileSource_, + std::shared_ptr maptilerFileSource_) : assetFileSource(std::move(assetFileSource_)), databaseFileSource(std::move(databaseFileSource_)), localFileSource(std::move(localFileSource_)), onlineFileSource(std::move(onlineFileSource_)), + maptilerFileSource(std::move(maptilerFileSource_)), supportsCacheOnlyRequests_(bool(databaseFileSource)), thread(std::make_unique>( util::makeThreadPrioritySetter(platform::EXPERIMENTAL_THREAD_PRIORITY_WORKER), @@ -142,7 +150,8 @@ public: assetFileSource, databaseFileSource, localFileSource, - onlineFileSource)) {} + onlineFileSource, + maptilerFileSource)) {} std::unique_ptr request(const Resource& resource, Callback callback) { auto req = std::make_unique(std::move(callback)); @@ -158,7 +167,8 @@ public: return (assetFileSource && assetFileSource->canRequest(resource)) || (localFileSource && localFileSource->canRequest(resource)) || (databaseFileSource && databaseFileSource->canRequest(resource)) || - (onlineFileSource && onlineFileSource->canRequest(resource)); + (onlineFileSource && onlineFileSource->canRequest(resource)) || + (maptilerFileSource && maptilerFileSource->canRequest(resource)); } bool supportsCacheOnlyRequests() const { return supportsCacheOnlyRequests_; } @@ -172,6 +182,7 @@ private: const std::shared_ptr databaseFileSource; const std::shared_ptr localFileSource; const std::shared_ptr onlineFileSource; + const std::shared_ptr maptilerFileSource; const bool supportsCacheOnlyRequests_; const std::unique_ptr> thread; }; @@ -180,7 +191,8 @@ MainResourceLoader::MainResourceLoader(const ResourceOptions& options) : impl(std::make_unique(FileSourceManager::get()->getFileSource(FileSourceType::Asset, options), FileSourceManager::get()->getFileSource(FileSourceType::Database, options), FileSourceManager::get()->getFileSource(FileSourceType::FileSystem, options), - FileSourceManager::get()->getFileSource(FileSourceType::Network, options))) {} + FileSourceManager::get()->getFileSource(FileSourceType::Network, options), + FileSourceManager::get()->getFileSource(FileSourceType::Mbtiles, options))) {} MainResourceLoader::~MainResourceLoader() = default; diff --git a/platform/default/src/mbgl/storage/mbtiles_file_source.cpp b/platform/default/src/mbgl/storage/mbtiles_file_source.cpp new file mode 100644 index 000000000..c2056ab00 --- /dev/null +++ b/platform/default/src/mbgl/storage/mbtiles_file_source.cpp @@ -0,0 +1,323 @@ +#include +#include + +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + +namespace { +//TODO: replace by mbgl::util::MBTILES_PROTOCOL +const std::string maptilerProtocol = "mbtiles://"; +bool acceptsURL(const std::string& url) { + return 0 == url.rfind(mbgl::util::MBTILES_PROTOCOL, 0); +} +} // namespace + +namespace mbgl { +using namespace rapidjson; +//using namespace mapbox::sqlite; + +class MaptilerFileSource::Impl { +public: + explicit Impl(const ActorRef&) {} + + std::vector &split(const std::string &s, char delim, std::vector &elems) { + std::stringstream ss(s); + std::string item; + while (std::getline(ss, item, delim)) { + double value = atof(item.c_str()); + elems.push_back(value); + } + return elems; + } + + std::vector split(const std::string &s, char delim) { + std::vector elems; + split(s, delim, elems); + return elems; + } + + std::string serialize(Document &doc) { + + StringBuffer buffer; + Writer writer(buffer); + doc.Accept(writer); + + return std::string(buffer.GetString(), buffer.GetSize()); + } + + std::string url_to_path(const std::string &url) { + return mbgl::util::percentDecode(url.substr(maptilerProtocol.size())); + } + + std::string db_path(const std::string &path) { + return path.substr(0, path.find('?')); + } + + bool is_compressed(const std::string &v) { + return (((uint8_t) v[0]) == 0x1f) && (((uint8_t) v[1]) == 0x8b); + } + + // Some Mbtiles store GZIP-ed tile data, this function is used for decompression + std::string decompress_string(const std::string &data) { + + uint32_t OUT_SIZE = 8192; + + int ret; + char outbuffer[OUT_SIZE]; + std::string outstring; + + z_stream zs{}; + + // Init inflate to gzip mode + if (inflateInit2(&zs, (16 + MAX_WBITS)) != Z_OK) + return ""; + + zs.next_in = (Bytef *) data.data(); + zs.avail_in = (unsigned int) data.size(); + + + // get the decompressed bytes blockwise using repeated calls to inflate + do { + zs.next_out = reinterpret_cast(outbuffer); + zs.avail_out = OUT_SIZE; + + ret = inflate(&zs, 0); + + if (outstring.size() < zs.total_out) { + outstring.append(outbuffer, zs.total_out - outstring.size()); + } + + } while (ret == Z_OK); + + inflateEnd(&zs); + + if (ret != Z_STREAM_END) { // an error occurred that was not EOF + return ""; + } + + return outstring; + } + + // Generate a tilejson resource from .mbtiles file + void request_tilejson(const Resource &resource, ActorRef req) { + auto path = url_to_path(resource.url); + + Response response; + + Document doc; + auto &allocator = doc.GetAllocator(); + + std::map values; + auto db = mapbox::sqlite::Database::open(path.c_str(), mapbox::sqlite::ReadOnly); + + mapbox::sqlite::Statement meta(db, "SELECT * from metadata"); + for (mapbox::sqlite::Query q(meta); q.run();) { + auto name = q.get(0); + auto val = q.get(1); + if (name == "json") { + doc.Parse(val.c_str()); + } else { + values[name] = val; + } + } + + if (!doc.IsObject()) { + doc.SetObject(); + } + + values["tilejson"] = "2.0.0"; + values["scheme"] = "xyz"; + + auto format_ptr = values.find("format"); + std::string format = format_ptr == values.end() ? "png" : format_ptr->second; + + if (format != "pbf" && values.count("scale") == 0) { + values["scale"] = "1"; + } + + // We use file location with appended parameter query parameter as URL for actual tile data + std::string tile_url = std::string(resource.url + "?file={x}/{y}/{z}." + format); + rapidjson::Value arr(kArrayType); + rapidjson::Value tile_url_val(tile_url, allocator); + arr.PushBack(tile_url_val, allocator); + doc.AddMember("tiles", arr, allocator); + + std::string minz; + std::string maxz; + + auto minzoom_ptr = values.find("minzoom"); + auto maxzoom_ptr = values.find("maxzoom"); + + if (minzoom_ptr == values.end() || maxzoom_ptr == values.end()) { + mapbox::sqlite::Statement zoom_stmt(db, "SELECT MIN(zoom_level),MAX(zoom_level) from tiles"); + + for (mapbox::sqlite::Query q(zoom_stmt); q.run();) { + minz = q.get(0); + maxz = q.get(1); + } + + values["minzoom"] = minz; + values["maxzoom"] = maxz; + } + else { + minz = minzoom_ptr->second; + maxz = maxzoom_ptr->second; + } + + auto minZoom = std::stoi(minz); + auto maxZoom = std::stoi(maxz); + + for (auto const &entry : values) { + if (entry.first == "scale") { + doc.AddMember("scale", std::stod(entry.second), allocator); + } else if (entry.first == "minzoom" || entry.first == "maxzoom") { + auto name = rapidjson::Value(entry.first, allocator); + doc.AddMember(name, std::stoi(entry.second), allocator); + } else if (entry.first == "bounds") { + std::vector x = split(entry.second, ','); + if (x.size() != 4) { + response.error = std::make_unique(Response::Error::Reason::Other); + req.invoke(&FileSourceRequest::setResponse, response); + return; + } + double cLon = (x[0] + x[2]) / 2; + double cLat = (x[1] + x[3]) / 2; + int cZoom = (minZoom + maxZoom) / 2; + rapidjson::Value bounds_arr(kArrayType); + for (auto vv : x) { + bounds_arr.PushBack(vv, allocator); + } + + if (format != "pbf") { + doc.AddMember("bounds", bounds_arr, allocator); + } + + rapidjson::Value center_arr(kArrayType); + center_arr.PushBack(cLon, allocator); + center_arr.PushBack(cLat, allocator); + center_arr.PushBack(cZoom, allocator); + doc.AddMember("center", center_arr, allocator); + } else { + rapidjson::Value name = rapidjson::Value(entry.first, allocator); + doc.AddMember(name, rapidjson::Value(entry.second, allocator), allocator); + } + } + + response.data = std::make_shared(serialize(doc)); + req.invoke(&FileSourceRequest::setResponse, response); + } + + // Load data for specific tile + void request_tile(const Resource &resource, ActorRef req) { + std::string base_path = url_to_path(resource.url); + std::string path = db_path(base_path); + auto &db = get_db(path); + + int iy = resource.tileData->y; + int iz = resource.tileData->z; + + auto x = std::to_string(resource.tileData->x); + auto y = std::to_string((int)(pow(2, iz) - 1) - iy); + auto z = std::to_string(iz); + + std::string sql = "SELECT tile_data FROM tiles where zoom_level = " + z + + " AND tile_column = " + x + " AND tile_row = " + y; + mapbox::sqlite::Statement stmt(db, sql.c_str()); + + Response response; + response.noContent = true; + response.error = std::make_unique(Response::Error::Reason::Connection, + "Not found in mbtile database"); + + for (mapbox::sqlite::Query q(stmt); q.run();) { + + optional data = q.get>(0); + if (data) { + response.data = std::make_shared(*data); + response.noContent = false; + response.expires = Timestamp::max(); + response.etag = resource.url; + response.error.release(); + + if (is_compressed(*response.data)) { + response.data = std::make_shared(decompress_string(*response.data)); + } + } + } + req.invoke(&FileSourceRequest::setResponse, response); + } + +private: + std::map db_cache; + + void close_db(const std::string &path) { + auto ptr = db_cache.find(path); + if (ptr != db_cache.end()) { + db_cache.erase(path); + } + } + + void close_all() { + db_cache.clear(); + } + + // Multiple databases open simultaneoulsy, to effectively support multiple .mbtiles maps + mapbox::sqlite::Database &get_db(const std::string &path) { + auto ptr = db_cache.find(path); + if (ptr != db_cache.end()) { + return ptr->second; + }; + + auto ptr2 = db_cache.insert(std::pair(path, mapbox::sqlite::Database::open(path.c_str(), mapbox::sqlite::ReadOnly))); + return ptr2.first->second; + } +}; + + + +MaptilerFileSource::MaptilerFileSource() : + thread(std::make_unique>( + util::makeThreadPrioritySetter(platform::EXPERIMENTAL_THREAD_PRIORITY_FILE), "MaptilerFileSource")) {} + + +std::unique_ptr MaptilerFileSource::request(const Resource &resource, FileSource::Callback callback) { + auto req = std::make_unique(std::move(callback)); + + if (resource.url.find(":///") == std::string::npos) { + Response response; + response.noContent = true; + response.error = std::make_unique(Response::Error::Reason::Other, + "MaptilerFileSource only supports absolute path urls"); + req->actor().invoke(&FileSourceRequest::setResponse, response); + + } else { + if (resource.kind == Resource::Tile) { + thread->actor().invoke(&Impl::request_tile, resource, req->actor()); + } else { + thread->actor().invoke(&Impl::request_tilejson, resource, req->actor()); + } + } + return std::move(req); +} + +bool MaptilerFileSource::canRequest(const Resource& resource) const { + return acceptsURL(resource.url); +} + +MaptilerFileSource::~MaptilerFileSource() = default; + + +} \ No newline at end of file diff --git a/platform/ios/ios.cmake b/platform/ios/ios.cmake index 2f79188b1..d00b80267 100644 --- a/platform/ios/ios.cmake +++ b/platform/ios/ios.cmake @@ -51,6 +51,7 @@ target_sources( ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/map/map_snapshotter.cpp ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/platform/time.cpp ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/asset_file_source.cpp + ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/mbtiles_file_source.cpp ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/database_file_source.cpp ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/file_source_manager.cpp ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/file_source_request.cpp diff --git a/platform/ios/platform/ios/CHANGELOG.md b/platform/ios/platform/ios/CHANGELOG.md index 0ed033ea4..c3484e68e 100644 --- a/platform/ios/platform/ios/CHANGELOG.md +++ b/platform/ios/platform/ios/CHANGELOG.md @@ -2,6 +2,13 @@ Mapbox welcomes participation and contributions from everyone. Please read [CONTRIBUTING.md](../../CONTRIBUTING.md) to get started. + +## 5.10.0 - January 6, 2021 + +### Features + +* Added the mbtiles file source for rendering vector tiles from file stored locally on the device. + ## 5.9.0 ### Styles and rendering diff --git a/platform/ios/platform/macos/CHANGELOG.md b/platform/ios/platform/macos/CHANGELOG.md index 9ad4e9385..5d2cd6b23 100644 --- a/platform/ios/platform/macos/CHANGELOG.md +++ b/platform/ios/platform/macos/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog for Mapbox Maps SDK for macOS +## 0.17.0 - January 6, 2021 + +### Features + +* Added the mbtiles file source for rendering vector tiles from file stored locally on the device. + ## 0.16.0 ### Styles and rendering diff --git a/platform/macos/macos.cmake b/platform/macos/macos.cmake index 4331852aa..bb140d02c 100644 --- a/platform/macos/macos.cmake +++ b/platform/macos/macos.cmake @@ -42,6 +42,7 @@ target_sources( ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/map/map_snapshotter.cpp ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/platform/time.cpp ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/asset_file_source.cpp + ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/mbtiles_file_source.cpp ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/database_file_source.cpp ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/file_source_manager.cpp ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/file_source_request.cpp diff --git a/src/mbgl/storage/mbtiles_file_source.hpp b/src/mbgl/storage/mbtiles_file_source.hpp new file mode 100644 index 000000000..b4d40c960 --- /dev/null +++ b/src/mbgl/storage/mbtiles_file_source.hpp @@ -0,0 +1,29 @@ +#include +#include +#include + +#ifndef MBGL_MAPTILER_FILE_SOURCE_H +#define MBGL_MAPTILER_FILE_SOURCE_H + +namespace mbgl { +// File source for supporting .mbtiles maps. +// can only load resource URLS that are absolute paths to local files +class MaptilerFileSource : public FileSource { +public: + MaptilerFileSource(); + ~MaptilerFileSource() override; + + std::unique_ptr request(const Resource &, Callback) override; + + //static bool acceptsURL(const std::string &url); + bool canRequest(const Resource&) const override; + +private: + class Impl; + std::unique_ptr > thread; //impl +}; + +} // namespace mbgl + + +#endif //MBGL_MAPTILER_FILE_SOURCE_H \ No newline at end of file