563 lines
22 KiB
C++
563 lines
22 KiB
C++
#include <mbgl/storage/offline_database.hpp>
|
|
#include <mbgl/storage/offline_download.hpp>
|
|
#include <mbgl/storage/resource.hpp>
|
|
#include <mbgl/storage/response.hpp>
|
|
#include <mbgl/storage/http_file_source.hpp>
|
|
#include <mbgl/style/parser.hpp>
|
|
#include <mbgl/style/sources/vector_source.hpp>
|
|
#include <mbgl/style/sources/raster_source.hpp>
|
|
#include <mbgl/style/sources/raster_dem_source.hpp>
|
|
#include <mbgl/style/sources/geojson_source.hpp>
|
|
#include <mbgl/style/sources/image_source.hpp>
|
|
#include <mbgl/style/conversion/json.hpp>
|
|
#include <mbgl/style/conversion/tileset.hpp>
|
|
#include <mbgl/style/sprite.hpp>
|
|
#include <mbgl/text/glyph.hpp>
|
|
#include <mbgl/util/i18n.hpp>
|
|
#include <mbgl/util/mapbox.hpp>
|
|
#include <mbgl/util/run_loop.hpp>
|
|
#include <mbgl/util/tile_cover.hpp>
|
|
#include <mbgl/util/tileset.hpp>
|
|
|
|
#include <set>
|
|
|
|
namespace {
|
|
|
|
const size_t kResourcesBatchSize = 64;
|
|
const size_t kMarkBatchSize = 200;
|
|
|
|
} // namespace
|
|
|
|
namespace mbgl {
|
|
|
|
using namespace style;
|
|
|
|
// Generic functions
|
|
|
|
template <class RegionDefinition>
|
|
Range<uint8_t> coveringZoomRange(const RegionDefinition& definition,
|
|
style::SourceType type,
|
|
uint16_t tileSize,
|
|
const Range<uint8_t>& zoomRange) {
|
|
double minZ = std::max<double>(util::coveringZoomLevel(definition.minZoom, type, tileSize), zoomRange.min);
|
|
double maxZ = std::min<double>(util::coveringZoomLevel(definition.maxZoom, type, tileSize), zoomRange.max);
|
|
|
|
assert(minZ >= 0);
|
|
assert(maxZ >= 0);
|
|
assert(minZ < std::numeric_limits<uint8_t>::max());
|
|
assert(maxZ < std::numeric_limits<uint8_t>::max());
|
|
return {static_cast<uint8_t>(minZ), static_cast<uint8_t>(maxZ)};
|
|
}
|
|
|
|
template <class Geometry, class Fn>
|
|
void tileCover(const Geometry& geometry, uint8_t z, Fn&& fn) {
|
|
util::TileCover cover(geometry, z);
|
|
while (cover.hasNext()) {
|
|
fn(cover.next()->canonical);
|
|
}
|
|
}
|
|
|
|
template <class Fn>
|
|
void tileCover(const OfflineRegionDefinition& definition,
|
|
style::SourceType type,
|
|
uint16_t tileSize,
|
|
const Range<uint8_t>& zoomRange,
|
|
Fn&& fn) {
|
|
const Range<uint8_t> clampedZoomRange = std::visit(
|
|
[&](auto& reg) { return coveringZoomRange(reg, type, tileSize, zoomRange); }, definition);
|
|
|
|
for (uint8_t z = clampedZoomRange.min; z <= clampedZoomRange.max; z++) {
|
|
std::visit(overloaded{[&](const OfflineTilePyramidRegionDefinition& reg) { tileCover(reg.bounds, z, fn); },
|
|
[&](const OfflineGeometryRegionDefinition& reg) {
|
|
tileCover(reg.geometry, z, fn);
|
|
}},
|
|
definition);
|
|
}
|
|
}
|
|
|
|
uint64_t tileCount(const OfflineRegionDefinition& definition,
|
|
style::SourceType type,
|
|
uint16_t tileSize,
|
|
const Range<uint8_t>& zoomRange) {
|
|
const Range<uint8_t> clampedZoomRange = std::visit(
|
|
[&](auto& reg) { return coveringZoomRange(reg, type, tileSize, zoomRange); }, definition);
|
|
|
|
uint64_t result{};
|
|
for (uint8_t z = clampedZoomRange.min; z <= clampedZoomRange.max; z++) {
|
|
result += std::visit(
|
|
overloaded{[&](const OfflineTilePyramidRegionDefinition& reg) { return util::tileCount(reg.bounds, z); },
|
|
[&](const OfflineGeometryRegionDefinition& reg) {
|
|
return util::tileCount(reg.geometry, z);
|
|
}},
|
|
definition);
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
// OfflineDownload
|
|
|
|
OfflineDownload::OfflineDownload(int64_t id_,
|
|
OfflineRegionDefinition definition_,
|
|
OfflineDatabase& offlineDatabase_,
|
|
FileSource& onlineFileSource_)
|
|
: id(id_),
|
|
definition(std::move(definition_)),
|
|
offlineDatabase(offlineDatabase_),
|
|
onlineFileSource(onlineFileSource_) {
|
|
setObserver(nullptr);
|
|
}
|
|
|
|
OfflineDownload::~OfflineDownload() = default;
|
|
|
|
void OfflineDownload::setObserver(std::unique_ptr<OfflineRegionObserver> observer_) {
|
|
observer = observer_ ? std::move(observer_) : std::make_unique<OfflineRegionObserver>();
|
|
}
|
|
|
|
void OfflineDownload::setState(OfflineRegionDownloadState state) {
|
|
if (status.downloadState == state) {
|
|
return;
|
|
}
|
|
|
|
status.downloadState = state;
|
|
|
|
if (status.downloadState == OfflineRegionDownloadState::Active) {
|
|
activateDownload();
|
|
} else {
|
|
deactivateDownload();
|
|
}
|
|
|
|
observer->statusChanged(status);
|
|
}
|
|
|
|
OfflineRegionStatus OfflineDownload::getStatus() const {
|
|
if (status.downloadState == OfflineRegionDownloadState::Active) {
|
|
return status;
|
|
}
|
|
|
|
auto result = offlineDatabase.getRegionCompletedStatus(id);
|
|
if (!result) {
|
|
// We can't find this offline region because the database is
|
|
// unavailable, or the download does not exist.
|
|
return {};
|
|
}
|
|
|
|
result->requiredResourceCount++;
|
|
std::optional<Response> styleResponse = offlineDatabase.get(
|
|
Resource::style(std::visit([](auto& reg) { return reg.styleURL; }, definition)));
|
|
if (!styleResponse) {
|
|
return *result;
|
|
}
|
|
|
|
style::Parser parser;
|
|
parser.parse(*styleResponse->data);
|
|
|
|
result->requiredResourceCountIsPrecise = true;
|
|
|
|
for (const auto& source : parser.sources) {
|
|
SourceType type = source->getType();
|
|
|
|
auto handleTiledSource = [&](const variant<std::string, Tileset>& urlOrTileset, const uint16_t tileSize) {
|
|
if (urlOrTileset.is<Tileset>()) {
|
|
uint64_t tileSourceCount = tileCount(definition, type, tileSize, urlOrTileset.get<Tileset>().zoomRange);
|
|
result->requiredTileCount += tileSourceCount;
|
|
result->requiredResourceCount += tileSourceCount;
|
|
} else {
|
|
result->requiredResourceCount += 1;
|
|
const auto& url = urlOrTileset.get<std::string>();
|
|
std::optional<Response> sourceResponse = offlineDatabase.get(Resource::source(url));
|
|
if (sourceResponse) {
|
|
style::conversion::Error error;
|
|
std::optional<Tileset> tileset = style::conversion::convertJSON<Tileset>(*sourceResponse->data,
|
|
error);
|
|
if (tileset) {
|
|
uint64_t tileSourceCount = tileCount(definition, type, tileSize, (*tileset).zoomRange);
|
|
result->requiredTileCount += tileSourceCount;
|
|
result->requiredResourceCount += tileSourceCount;
|
|
}
|
|
} else {
|
|
result->requiredResourceCountIsPrecise = false;
|
|
}
|
|
}
|
|
};
|
|
|
|
switch (type) {
|
|
case SourceType::Vector: {
|
|
const auto& vectorSource = *source->as<VectorSource>();
|
|
handleTiledSource(vectorSource.getURLOrTileset(), util::tileSize_I);
|
|
break;
|
|
}
|
|
|
|
case SourceType::Raster: {
|
|
const auto& rasterSource = *source->as<RasterSource>();
|
|
handleTiledSource(rasterSource.getURLOrTileset(), rasterSource.getTileSize());
|
|
break;
|
|
}
|
|
|
|
case SourceType::RasterDEM: {
|
|
const auto& rasterDEMSource = *source->as<RasterDEMSource>();
|
|
handleTiledSource(rasterDEMSource.getURLOrTileset(), rasterDEMSource.getTileSize());
|
|
break;
|
|
}
|
|
|
|
case SourceType::GeoJSON: {
|
|
const auto& geojsonSource = *source->as<GeoJSONSource>();
|
|
if (geojsonSource.getURL()) {
|
|
result->requiredResourceCount += 1;
|
|
}
|
|
break;
|
|
}
|
|
|
|
case SourceType::Image: {
|
|
const auto& imageSource = *source->as<ImageSource>();
|
|
if (imageSource.getURL()) {
|
|
result->requiredResourceCount += 1;
|
|
}
|
|
break;
|
|
}
|
|
|
|
case SourceType::Video:
|
|
case SourceType::Annotations:
|
|
case SourceType::CustomVector:
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (!parser.glyphURL.empty()) {
|
|
uint32_t ttfCount = 0;
|
|
if (parser.fontFaces) ttfCount = static_cast<uint32_t>(parser.fontFaces->size()); // custom faces
|
|
result->requiredResourceCount += parser.fontStacks().size() *
|
|
(std::visit([](auto& reg) { return reg.includeIdeographs; }, definition)
|
|
? GLYPH_RANGES_PER_FONT_STACK + ttfCount
|
|
: NON_IDEOGRAPH_GLYPH_RANGES_PER_FONT_STACK + ttfCount);
|
|
}
|
|
|
|
if (!parser.sprites.empty()) {
|
|
result->requiredResourceCount += 4 * parser.sprites.size();
|
|
}
|
|
|
|
return *result;
|
|
}
|
|
|
|
void OfflineDownload::activateDownload() {
|
|
status = OfflineRegionStatus();
|
|
status.downloadState = OfflineRegionDownloadState::Active;
|
|
status.requiredResourceCount++;
|
|
|
|
auto styleResource = Resource::style(std::visit([](auto& reg) { return reg.styleURL; }, definition));
|
|
styleResource.setPriority(Resource::Priority::Low);
|
|
styleResource.setUsage(Resource::Usage::Offline);
|
|
|
|
ensureResource(std::move(styleResource), [&](const Response& styleResponse) {
|
|
status.requiredResourceCountIsPrecise = true;
|
|
|
|
style::Parser parser;
|
|
parser.parse(*styleResponse.data);
|
|
|
|
auto tileServerOptions = onlineFileSource.getResourceOptions().tileServerOptions();
|
|
parser.glyphURL = util::mapbox::canonicalizeGlyphURL(tileServerOptions, parser.glyphURL);
|
|
|
|
for (const auto& source : parser.sources) {
|
|
SourceType type = source->getType();
|
|
|
|
auto handleTiledSource = [&](const variant<std::string, Tileset>& urlOrTileset, const uint16_t tileSize) {
|
|
if (urlOrTileset.is<Tileset>()) {
|
|
queueTiles(type, tileSize, urlOrTileset.get<Tileset>());
|
|
} else {
|
|
const auto& rawUrl = urlOrTileset.get<std::string>();
|
|
const auto& url = util::mapbox::canonicalizeSourceURL(tileServerOptions, rawUrl);
|
|
|
|
status.requiredResourceCountIsPrecise = false;
|
|
status.requiredResourceCount++;
|
|
requiredSourceURLs.insert(url);
|
|
|
|
auto sourceResource = Resource::source(url);
|
|
sourceResource.setPriority(Resource::Priority::Low);
|
|
sourceResource.setUsage(Resource::Usage::Offline);
|
|
|
|
ensureResource(std::move(sourceResource), [=, this](const Response& sourceResponse) {
|
|
style::conversion::Error error;
|
|
std::optional<Tileset> tileset = style::conversion::convertJSON<Tileset>(*sourceResponse.data,
|
|
error);
|
|
if (tileset) {
|
|
auto resourceOptions = onlineFileSource.getResourceOptions();
|
|
util::mapbox::canonicalizeTileset(
|
|
resourceOptions.tileServerOptions(), *tileset, url, type, tileSize);
|
|
queueTiles(type, tileSize, *tileset);
|
|
|
|
requiredSourceURLs.erase(url);
|
|
if (requiredSourceURLs.empty()) {
|
|
status.requiredResourceCountIsPrecise = true;
|
|
}
|
|
}
|
|
});
|
|
}
|
|
};
|
|
|
|
switch (type) {
|
|
case SourceType::Vector: {
|
|
const auto& vectorSource = *source->as<VectorSource>();
|
|
handleTiledSource(vectorSource.getURLOrTileset(), util::tileSize_I);
|
|
break;
|
|
}
|
|
|
|
case SourceType::Raster: {
|
|
const auto& rasterSource = *source->as<RasterSource>();
|
|
handleTiledSource(rasterSource.getURLOrTileset(), rasterSource.getTileSize());
|
|
break;
|
|
}
|
|
|
|
case SourceType::RasterDEM: {
|
|
const auto& rasterDEMSource = *source->as<RasterDEMSource>();
|
|
handleTiledSource(rasterDEMSource.getURLOrTileset(), rasterDEMSource.getTileSize());
|
|
break;
|
|
}
|
|
|
|
case SourceType::GeoJSON: {
|
|
const auto& geojsonSource = *source->as<GeoJSONSource>();
|
|
if (geojsonSource.getURL()) {
|
|
queueResource(Resource::source(*geojsonSource.getURL()));
|
|
}
|
|
break;
|
|
}
|
|
|
|
case SourceType::Image: {
|
|
const auto& imageSource = *source->as<ImageSource>();
|
|
auto imageUrl = imageSource.getURL();
|
|
if (imageUrl && !imageUrl->empty()) {
|
|
queueResource(Resource::image(*imageUrl));
|
|
}
|
|
break;
|
|
}
|
|
|
|
case SourceType::Video:
|
|
case SourceType::Annotations:
|
|
case SourceType::CustomVector:
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (!parser.glyphURL.empty()) {
|
|
const bool includeIdeographs = std::visit([](auto& reg) { return reg.includeIdeographs; }, definition);
|
|
for (const auto& fontStack : parser.fontStacks()) {
|
|
for (char16_t i = 0; i < GLYPH_RANGES_PER_FONT_STACK; i++) {
|
|
// Assumes that if a glyph range starts with fixed width/ideographic
|
|
// characters, the entire range will be fixed width.
|
|
if (includeIdeographs || !util::i18n::allowsFixedWidthGlyphGeneration(i * GLYPHS_PER_GLYPH_RANGE)) {
|
|
auto range = getGlyphRange(i * GLYPHS_PER_GLYPH_RANGE);
|
|
queueResource(Resource::glyphs(
|
|
parser.glyphURL, fontStack, std::pair<uint16_t, uint16_t>{range.first, range.second}));
|
|
}
|
|
if (parser.fontFaces) {
|
|
FontFaces& faces = *parser.fontFaces;
|
|
for (const auto& face : faces) {
|
|
queueResource(Resource::fontFace(face.url));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!parser.sprites.empty()) {
|
|
// Always request 1x and @2x sprite images for portability.
|
|
for (const auto& sprite : parser.sprites) {
|
|
std::string spriteURL = util::mapbox::canonicalizeSpriteURL(tileServerOptions, sprite.spriteURL);
|
|
queueResource(Resource::spriteImage(spriteURL, 1));
|
|
queueResource(Resource::spriteImage(spriteURL, 2));
|
|
queueResource(Resource::spriteJSON(spriteURL, 1));
|
|
queueResource(Resource::spriteJSON(spriteURL, 2));
|
|
}
|
|
}
|
|
|
|
continueDownload();
|
|
});
|
|
}
|
|
|
|
/*
|
|
Fill up our own request queue by requesting the next few resources. This is
|
|
called when activating the download, or when a request completes
|
|
successfully.
|
|
|
|
Note "successfully"; it's not called when a requests receives an error. A
|
|
request that errors will be retried after some delay. So in that sense it's
|
|
still "active" and consuming resources, notably the request object, its
|
|
timer, and network resources when the timer fires.
|
|
|
|
We could try to squeeze in subsequent requests while we wait for the errored
|
|
request to retry. But that risks overloading the upstream request queue --
|
|
defeating our own metering -- if there are a lot of errored requests that all
|
|
come up for retry at the same time. And many times, the cause of a request
|
|
error will apply to many requests of the same type. For instance if a server
|
|
is unreachable, all the requests to that host are going to error. In that
|
|
case, continuing to try subsequent resources after the first few errors is
|
|
fruitless anyway.
|
|
*/
|
|
void OfflineDownload::continueDownload() {
|
|
if (resourcesRemaining.empty()) {
|
|
// Flush pending buffers.
|
|
if (!flushResourcesBuffer()) return;
|
|
if (status.complete()) {
|
|
markPendingUsedResources();
|
|
setState(OfflineRegionDownloadState::Inactive);
|
|
return;
|
|
}
|
|
}
|
|
|
|
if (resourcesToBeMarkedAsUsed.size() >= kMarkBatchSize) markPendingUsedResources();
|
|
|
|
uint32_t maxConcurrentRequests = util::DEFAULT_MAXIMUM_CONCURRENT_REQUESTS;
|
|
auto value = onlineFileSource.getProperty(MAX_CONCURRENT_REQUESTS_KEY);
|
|
if (uint64_t* maxRequests = value.getUint()) {
|
|
maxConcurrentRequests = static_cast<uint32_t>(*maxRequests);
|
|
}
|
|
|
|
while (!resourcesRemaining.empty() && requests.size() < maxConcurrentRequests) {
|
|
ensureResource(std::move(resourcesRemaining.front()));
|
|
resourcesRemaining.pop_front();
|
|
}
|
|
}
|
|
|
|
void OfflineDownload::deactivateDownload() {
|
|
requiredSourceURLs.clear();
|
|
resourcesRemaining.clear();
|
|
requests.clear();
|
|
buffer.clear();
|
|
}
|
|
|
|
bool OfflineDownload::flushResourcesBuffer() {
|
|
if (buffer.empty()) return true;
|
|
try {
|
|
offlineDatabase.putRegionResources(id, buffer, status);
|
|
buffer.clear();
|
|
observer->statusChanged(status);
|
|
return true;
|
|
} catch (const MapboxTileLimitExceededException&) {
|
|
onMapboxTileCountLimitExceeded();
|
|
return false;
|
|
}
|
|
}
|
|
|
|
void OfflineDownload::queueResource(Resource&& resource) {
|
|
resource.setPriority(Resource::Priority::Low);
|
|
resource.setUsage(Resource::Usage::Offline);
|
|
status.requiredResourceCount++;
|
|
if (resource.kind == mbgl::Resource::Kind::Tile) {
|
|
status.requiredTileCount++;
|
|
}
|
|
resourcesRemaining.push_front(std::move(resource));
|
|
}
|
|
|
|
void OfflineDownload::queueTiles(SourceType type, uint16_t tileSize, const Tileset& tileset) {
|
|
tileCover(definition, type, tileSize, tileset.zoomRange, [&](const auto& tile) {
|
|
status.requiredResourceCount++;
|
|
status.requiredTileCount++;
|
|
|
|
auto tileResource = Resource::tile(tileset.tiles[0],
|
|
std::visit([](auto& def) { return def.pixelRatio; }, definition),
|
|
tile.x,
|
|
tile.y,
|
|
tile.z,
|
|
tileset.scheme);
|
|
|
|
tileResource.setPriority(Resource::Priority::Low);
|
|
tileResource.setUsage(Resource::Usage::Offline);
|
|
|
|
resourcesRemaining.push_back(std::move(tileResource));
|
|
});
|
|
}
|
|
|
|
void OfflineDownload::markPendingUsedResources() {
|
|
offlineDatabase.markUsedResources(id, resourcesToBeMarkedAsUsed);
|
|
resourcesToBeMarkedAsUsed.clear();
|
|
}
|
|
|
|
void OfflineDownload::ensureResource(Resource&& resource, std::function<void(Response)> callback) {
|
|
assert(resource.priority == Resource::Priority::Low);
|
|
assert(resource.usage == Resource::Usage::Offline);
|
|
|
|
auto workRequestsIt = requests.insert(requests.begin(), nullptr);
|
|
*workRequestsIt = util::RunLoop::Get()->invokeCancellable([=, this]() {
|
|
requests.erase(workRequestsIt);
|
|
const auto resourceKind = resource.kind;
|
|
auto getResourceSizeInDatabase = [&]() -> std::optional<int64_t> {
|
|
std::optional<int64_t> result;
|
|
if (!callback) {
|
|
result = offlineDatabase.hasRegionResource(resource);
|
|
} else {
|
|
std::optional<std::pair<Response, uint64_t>> response = offlineDatabase.getRegionResource(resource);
|
|
if (response) {
|
|
callback(response->first);
|
|
result = response->second;
|
|
}
|
|
}
|
|
|
|
if (result) resourcesToBeMarkedAsUsed.emplace_back(resource);
|
|
return result;
|
|
};
|
|
|
|
std::optional<int64_t> offlineResponse = getResourceSizeInDatabase();
|
|
if (offlineResponse) {
|
|
assert(!resourcesToBeMarkedAsUsed.empty());
|
|
status.completedResourceCount++;
|
|
status.completedResourceSize += *offlineResponse;
|
|
if (resourceKind == Resource::Kind::Tile) {
|
|
status.completedTileCount += 1;
|
|
status.completedTileSize += *offlineResponse;
|
|
}
|
|
|
|
observer->statusChanged(status);
|
|
continueDownload();
|
|
return;
|
|
}
|
|
|
|
if (offlineDatabase.exceedsOfflineMapboxTileCountLimit(resource)) {
|
|
onMapboxTileCountLimitExceeded();
|
|
return;
|
|
}
|
|
|
|
auto fileRequestsIt = requests.insert(requests.begin(), nullptr);
|
|
*fileRequestsIt = onlineFileSource.request(resource, [=, this](const Response& onlineResponse) {
|
|
if (onlineResponse.error) {
|
|
observer->responseError(*onlineResponse.error);
|
|
if (onlineResponse.error->reason == Response::Error::Reason::NotFound) {
|
|
// On error 404, we skip this request and go further.
|
|
requests.erase(fileRequestsIt);
|
|
assert(status.requiredResourceCount > 0);
|
|
status.requiredResourceCount--;
|
|
continueDownload();
|
|
}
|
|
return;
|
|
}
|
|
|
|
requests.erase(fileRequestsIt);
|
|
|
|
if (callback) {
|
|
callback(onlineResponse);
|
|
}
|
|
|
|
// Queue up for batched insertion
|
|
buffer.emplace_back(resource, onlineResponse);
|
|
|
|
// Flush buffer periodically.
|
|
// Have to keep `resourcesRemaining.empty()` as the following
|
|
// condition would fail otherwise.
|
|
// TODO: Simplify the tile count limit check code path!
|
|
if ((buffer.size() == kResourcesBatchSize || resourcesRemaining.empty()) && !flushResourcesBuffer()) return;
|
|
|
|
if (offlineDatabase.exceedsOfflineMapboxTileCountLimit(resource)) {
|
|
onMapboxTileCountLimitExceeded();
|
|
return;
|
|
}
|
|
|
|
continueDownload();
|
|
});
|
|
});
|
|
}
|
|
|
|
void OfflineDownload::onMapboxTileCountLimitExceeded() {
|
|
observer->mapboxTileCountLimitExceeded(offlineDatabase.getOfflineMapboxTileCountLimit());
|
|
setState(OfflineRegionDownloadState::Inactive);
|
|
}
|
|
|
|
} // namespace mbgl
|