tei/MapLibreParticleLab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[core] Add DefaultFileSource::mergeRegions API and CLI support in the mbgl-offline tool.

Browse Source
This commit is contained in:
Asheem Mamoowala
2018-08-21 14:11:44 -07:00
parent 48381efa11
commit 065f5ca87e
5 changed files with 122 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
bin/offline.cpp
View File

@@ -62,18 +62,21 @@ int main(int argc, char *argv[]) {
args::ValueFlag<std::string> outputValue(argumentParser, "file", "Output database file name", {'o', "output"});
args::ValueFlag<std::string> apiBaseValue(argumentParser, "URL", "API Base URL", {'a', "apiBaseURL"});
args::Group mergeGroup(argumentParser, "Merge databases:", args::Group::Validators::AllOrNone);
args::ValueFlag<std::string> mergePathValue(mergeGroup, "merge", "Database to merge from", {'m', "merge"});
args::ValueFlag<std::string> inputValue(mergeGroup, "input", "Database to merge into. Use with --merge option.", {'i', "input"});
// LatLngBounds
args::Group latLngBoundsGroup(argumentParser, "LatLng bounds:", args::Group::Validators::AllOrNone);
args::ValueFlag<double> northValue(latLngBoundsGroup, "degrees", "North latitude", {"north"});
args::ValueFlag<double> westValue(latLngBoundsGroup, "degrees", "West longitude", {"west"});
args::ValueFlag<double> southValue(latLngBoundsGroup, "degrees", "South latitude", {"south"});
args::ValueFlag<double> eastValue(latLngBoundsGroup, "degrees", "East longitude", {"east"});
// Geometry
args::Group geoJSONGroup(argumentParser, "GeoJson geometry:", args::Group::Validators::AllOrNone);
args::ValueFlag<std::string> geometryValue(geoJSONGroup, "file", "GeoJSON file containing the region geometry", {"geojson"});
args::ValueFlag<double> minZoomValue(argumentParser, "number", "Min zoom level", {"minZoom"});
args::ValueFlag<double> maxZoomValue(argumentParser, "number", "Max zoom level", {"maxZoom"});
args::ValueFlag<double> pixelRatioValue(argumentParser, "number", "Pixel ratio", {"pixelRatio"});
@@ -95,6 +98,11 @@ int main(int argc, char *argv[]) {
std::string style = styleValue ? args::get(styleValue) : mbgl::util::default_styles::streets.url;
mbgl::optional<std::string> mergePath = {};
if (mergePathValue) mergePath = args::get(mergePathValue);
mbgl::optional<std::string> inputDb = {};
if (inputValue) inputDb = args::get(inputValue);
const double minZoom = minZoomValue ? args::get(minZoomValue) : 0.0;
const double maxZoom = maxZoomValue ? args::get(maxZoomValue) : 15.0;
const double pixelRatio = pixelRatioValue ? args::get(pixelRatioValue) : 1.0;
@@ -136,15 +144,37 @@ int main(int argc, char *argv[]) {
fileSource.setAccessToken(token);
fileSource.setAPIBaseURL(apiBaseURL);
if (inputDb && mergePath) {
DefaultFileSource inputSource(*inputDb, ".");
inputSource.setAccessToken(token);
inputSource.setAPIBaseURL(apiBaseURL);
int retCode = 0;
std::cout << "Start Merge" << std::endl;
inputSource.mergeOfflineRegions(*mergePath, [&] (mbgl::expected<std::vector<OfflineRegion>, std::exception_ptr> result) {
if (!result) {
std::cerr << "Error merging database: " << util::toString(result.error()) << std::endl;
retCode = 1;
} else {
std::cout << " Added " << result->size() << " Regions" << std::endl;
std::cout << "Finished Merge" << std::endl;
}
loop.stop();
});
loop.run();
return retCode;
}
OfflineRegionMetadata metadata;
class Observer : public OfflineRegionObserver {
public:
Observer(OfflineRegion& region_, DefaultFileSource& fileSource_, util::RunLoop& loop_)
Observer(OfflineRegion& region_, DefaultFileSource& fileSource_, util::RunLoop& loop_, mbgl::optional<std::string> mergePath_)
: region(region_),
fileSource(fileSource_),
loop(loop_),
mergePath(mergePath_),
start(util::now()) {
}
@@ -170,7 +200,7 @@ int main(int argc, char *argv[]) {
<< std::endl;
if (status.complete()) {
std::cout << "Finished" << std::endl;
std::cout << "Finished Download" << std::endl;
loop.stop();
}
}
@@ -186,6 +216,7 @@ int main(int argc, char *argv[]) {
OfflineRegion& region;
DefaultFileSource& fileSource;
util::RunLoop& loop;
mbgl::optional<std::string> mergePath;
Timestamp start;
};
@@ -206,7 +237,7 @@ int main(int argc, char *argv[]) {
} else {
assert(region_);
region = std::make_unique<OfflineRegion>(std::move(*region_));
fileSource.setOfflineRegionObserver(*region, std::make_unique<Observer>(*region, fileSource, loop));
fileSource.setOfflineRegionObserver(*region, std::make_unique<Observer>(*region, fileSource, loop, mergePath));
fileSource.setOfflineRegionDownloadState(*region, OfflineRegionDownloadState::Active);
}
});

17
include/mbgl/storage/default_file_source.hpp
View File

@@ -99,6 +99,23 @@ public:
OfflineRegion&,
std::function<void (expected<OfflineRegionStatus, std::exception_ptr>)>) const;
/*
* Merge offline regions from a secondary database into the main offline database.
*
* When the database merge is completed, the provided callback will be
* executed on the database thread; it is the responsibility of the SDK bindings
* to re-execute a user-provided callback on the main thread.
*
* Only resources and tiles that belong to a region will be copied over. Identical
* regions will be flattened into a single new region in the main database.
*
* Note that the resulting new regions may not be in a completed status if the
* secondary database does not contain all the tiles or resources required by the
* region definition.
*/
void mergeOfflineRegions(const std::string& sideDatabasePath,
std::function<void (expected<OfflineRegions, std::exception_ptr>)>);
/*
* Remove an offline region from the database and perform any resources evictions
* necessary as a result.

13
platform/default/default_file_source.cpp
View File

@@ -55,6 +55,11 @@ public:
callback(offlineDatabase->createRegion(definition, metadata));
}
void mergeOfflineRegions(const std::string& sideDatabasePath,
std::function<void (expected<OfflineRegions, std::exception_ptr>)> callback) {
callback(offlineDatabase->mergeDatabase(sideDatabasePath));
}
void updateMetadata(const int64_t regionID,
const OfflineRegionMetadata& metadata,
std::function<void (expected<OfflineRegionMetadata, std::exception_ptr>)> callback) {
@@ -258,9 +263,15 @@ void DefaultFileSource::createOfflineRegion(const OfflineRegionDefinition& defin
impl->actor().invoke(&Impl::createRegion, definition, metadata, callback);
}
void DefaultFileSource::mergeOfflineRegions(const std::string& sideDatabasePath,
std::function<void (expected<OfflineRegions, std::exception_ptr>)> callback) {
impl->actor().invoke(&Impl::mergeOfflineRegions, sideDatabasePath, callback);
}
void DefaultFileSource::updateOfflineMetadata(const int64_t regionID,
const OfflineRegionMetadata& metadata,
std::function<void (expected<OfflineRegionMetadata, std::exception_ptr>)> callback) {
std::function<void (expected<OfflineRegionMetadata,
std::exception_ptr>)> callback) {
impl->actor().invoke(&Impl::updateMetadata, regionID, metadata, callback);
}

53
platform/default/mbgl/storage/offline_database.cpp
View File

@@ -7,6 +7,7 @@
#include <mbgl/util/logging.hpp>
#include "offline_schema.hpp"
#include "merge_sideloaded.hpp"
#include "sqlite3.hpp"
@@ -637,6 +638,58 @@ OfflineDatabase::createRegion(const OfflineRegionDefinition& definition,
return unexpected<std::exception_ptr>(std::current_exception());
}
expected<OfflineRegions, std::exception_ptr>
OfflineDatabase::mergeDatabase(const std::string& sideDatabasePath) {
try {
// clang-format off
mapbox::sqlite::Query query{ getStatement("ATTACH DATABASE ?1 AS side") };
// clang-format on
query.bind(1, sideDatabasePath);
query.run();
} catch (const mapbox::sqlite::Exception& ex) {
handleError(ex, "merge databse attach");
return unexpected<std::exception_ptr>(std::current_exception());
}
try {
// Attaching an accessible path without a db file creates a new temporary
//database and attaches it. Check for matching schema version.
auto sideUserVersion = static_cast<int>(getPragma<int64_t>("PRAGMA side.user_version"));
if (sideUserVersion != 6) {
Log::Warning(Event::Database, "Merge database does not match user_version of main database");
throw std::runtime_error("merge database does not match schema or has incorrect user_version");
}
mapbox::sqlite::Transaction transaction(*db);
db->exec(mergeSideloadedDatabaseSQL);
transaction.commit();
// clang-format off
mapbox::sqlite::Query query{ getStatement(
"SELECT r.id, r.definition, r.description "
"FROM side.regions sr "
"JOIN regions r ON sr.definition = r.definition") };
// clang-format on
OfflineRegions result;
while (query.run()) {
// Construct, then move because this constructor is private.
OfflineRegion region(query.get<int64_t>(0),
decodeOfflineRegionDefinition(query.get<std::string>(1)),
query.get<std::vector<uint8_t>>(2));
result.emplace_back(std::move(region));
}
db->exec("DETACH DATABASE side");
// Explicit move to avoid triggering the copy constructor.
return { std::move(result) };
} catch (const mapbox::sqlite::Exception& ex) {
db->exec("DETACH DATABASE side");
handleError(ex, "merge databse post merge sql");
return unexpected<std::exception_ptr>(std::current_exception());
}
return {};
}
expected<OfflineRegionMetadata, std::exception_ptr>
OfflineDatabase::updateMetadata(const int64_t regionID, const OfflineRegionMetadata& metadata) try {
// clang-format off

3
platform/default/mbgl/storage/offline_database.hpp
View File

@@ -53,6 +53,9 @@ public:
expected<OfflineRegion, std::exception_ptr> createRegion(const OfflineRegionDefinition&,
const OfflineRegionMetadata&);
expected<OfflineRegions, std::exception_ptr>
mergeDatabase(const std::string& sideDatabasePath);
expected<OfflineRegionMetadata, std::exception_ptr>
updateMetadata(const int64_t regionID, const OfflineRegionMetadata&);