Files
MapLibreParticleLab/platform/default/src/mbgl/storage/offline_download.cpp
Alexander Shalamov 879c44f661 [core] Modularize FileSource codebase (#15768)
* [core] Introduce FileSourceManager and use it for default platform impl

  - Add `FileSourceManager` interface that provides access to `FileSource`
    instances and means of registering / unregistering `FileSource` factories
  - Split `DefaultFileSource` into smaller parts
  - Add `DatabaseFileSource` interface and it's default implementation
  - Remove inter-dependencies between concrete `FileSource` classes

* [build] Add files to next build system

* [core] Add generic property setters / getters

* [core] Remove setOnlineStatus from OnlineFileSource interface

* [core] Hide threading implementation details from DatabaseFileSource interface

* [core] Make DB file source methods virtual

* [core] Add documentation for DatabaseFileSource and rename one method

* [core] Use simple callback instead of ActorRef

* [core] Remove ActorRef from OnlineFileSource public header

* [core] Add callback to FileSource::forward async API

* [core] Pass OfflineRegionDefinition by value

* [core] Update tests to use modular file sources

* [core] Update unit tests

* [core] Update unit tests after rebase

* [core] Backport low prio fix for cached requests

* [core] Backport pack database

* [core] Return removed factory from unRegisterFileSourceFactory

* [core] Rename shadowed args in onlinefilesource

* [core] Remove simple std::function callback aliases

* [core] Expose online file source property keys in public header file

* [test-runner] Add proxy file source test runner

* [cache] Update mbgl-cache utility to use new file source

* [metrics] Rebaseline binary size metrics

* [offline] Update offline utility

* [core] Update changelog
2020-01-13 10:57:23 +02:00

511 lines
19 KiB
C++

#include <mbgl/storage/offline_database.hpp>
#include <mbgl/storage/offline_download.hpp>
#include <mbgl/storage/resource.hpp>
#include <mbgl/storage/response.hpp>
#include <mbgl/storage/http_file_source.hpp>
#include <mbgl/style/parser.hpp>
#include <mbgl/style/sources/vector_source.hpp>
#include <mbgl/style/sources/raster_source.hpp>
#include <mbgl/style/sources/raster_dem_source.hpp>
#include <mbgl/style/sources/geojson_source.hpp>
#include <mbgl/style/sources/image_source.hpp>
#include <mbgl/style/conversion/json.hpp>
#include <mbgl/style/conversion/tileset.hpp>
#include <mbgl/text/glyph.hpp>
#include <mbgl/util/i18n.hpp>
#include <mbgl/util/mapbox.hpp>
#include <mbgl/util/run_loop.hpp>
#include <mbgl/util/tile_cover.hpp>
#include <mbgl/util/tileset.hpp>
#include <set>
namespace {
const size_t kResourcesBatchSize = 64;
const size_t kMarkBatchSize = 200;
} // namespace
namespace mbgl {
using namespace style;
// Generic functions
template <class RegionDefinition>
Range<uint8_t> coveringZoomRange(const RegionDefinition& definition,
style::SourceType type, uint16_t tileSize, const Range<uint8_t>& zoomRange) {
double minZ = std::max<double>(util::coveringZoomLevel(definition.minZoom, type, tileSize), zoomRange.min);
double maxZ = std::min<double>(util::coveringZoomLevel(definition.maxZoom, type, tileSize), zoomRange.max);
assert(minZ >= 0);
assert(maxZ >= 0);
assert(minZ < std::numeric_limits<uint8_t>::max());
assert(maxZ < std::numeric_limits<uint8_t>::max());
return { static_cast<uint8_t>(minZ), static_cast<uint8_t>(maxZ) };
}
template <class Geometry, class Fn>
void tileCover(const Geometry& geometry, uint8_t z, Fn&& fn) {
util::TileCover cover(geometry, z);
while (cover.hasNext()) {
fn(cover.next()->canonical);
}
}
template <class Fn>
void tileCover(const OfflineRegionDefinition& definition, style::SourceType type,
uint16_t tileSize, const Range<uint8_t>& zoomRange, Fn&& fn) {
const Range<uint8_t> clampedZoomRange =
definition.match([&](auto& reg) { return coveringZoomRange(reg, type, tileSize, zoomRange); });
for (uint8_t z = clampedZoomRange.min; z <= clampedZoomRange.max; z++) {
definition.match(
[&](const OfflineTilePyramidRegionDefinition& reg){ tileCover(reg.bounds, z, fn); },
[&](const OfflineGeometryRegionDefinition& reg){ tileCover(reg.geometry, z, fn); }
);
}
}
uint64_t tileCount(const OfflineRegionDefinition& definition, style::SourceType type,
uint16_t tileSize, const Range<uint8_t>& zoomRange) {
const Range<uint8_t> clampedZoomRange =
definition.match([&](auto& reg) { return coveringZoomRange(reg, type, tileSize, zoomRange); });
unsigned long result = 0;;
for (uint8_t z = clampedZoomRange.min; z <= clampedZoomRange.max; z++) {
result += definition.match(
[&](const OfflineTilePyramidRegionDefinition& reg){ return util::tileCount(reg.bounds, z); },
[&](const OfflineGeometryRegionDefinition& reg){ return util::tileCount(reg.geometry, z); }
);
}
return result;
}
// OfflineDownload
OfflineDownload::OfflineDownload(int64_t id_,
OfflineRegionDefinition definition_,
OfflineDatabase& offlineDatabase_,
FileSource& onlineFileSource_)
: id(id_),
definition(std::move(definition_)),
offlineDatabase(offlineDatabase_),
onlineFileSource(onlineFileSource_) {
setObserver(nullptr);
}
OfflineDownload::~OfflineDownload() = default;
void OfflineDownload::setObserver(std::unique_ptr<OfflineRegionObserver> observer_) {
observer = observer_ ? std::move(observer_) : std::make_unique<OfflineRegionObserver>();
}
void OfflineDownload::setState(OfflineRegionDownloadState state) {
if (status.downloadState == state) {
return;
}
status.downloadState = state;
if (status.downloadState == OfflineRegionDownloadState::Active) {
activateDownload();
} else {
deactivateDownload();
}
observer->statusChanged(status);
}
OfflineRegionStatus OfflineDownload::getStatus() const {
if (status.downloadState == OfflineRegionDownloadState::Active) {
return status;
}
auto result = offlineDatabase.getRegionCompletedStatus(id);
if (!result) {
// We can't find this offline region because the database is unavailable, or the download
// does not exist.
return {};
}
result->requiredResourceCount++;
optional<Response> styleResponse =
offlineDatabase.get(Resource::style(definition.match([](auto& reg){ return reg.styleURL; })));
if (!styleResponse) {
return *result;
}
style::Parser parser;
parser.parse(*styleResponse->data);
result->requiredResourceCountIsPrecise = true;
for (const auto& source : parser.sources) {
SourceType type = source->getType();
auto handleTiledSource = [&] (const variant<std::string, Tileset>& urlOrTileset, const uint16_t tileSize) {
if (urlOrTileset.is<Tileset>()) {
uint64_t tileSourceCount = tileCount(definition, type, tileSize, urlOrTileset.get<Tileset>().zoomRange);
result->requiredTileCount += tileSourceCount;
result->requiredResourceCount += tileSourceCount;
} else {
result->requiredResourceCount += 1;
const auto& url = urlOrTileset.get<std::string>();
optional<Response> sourceResponse = offlineDatabase.get(Resource::source(url));
if (sourceResponse) {
style::conversion::Error error;
optional<Tileset> tileset = style::conversion::convertJSON<Tileset>(*sourceResponse->data, error);
if (tileset) {
uint64_t tileSourceCount = tileCount(definition, type, tileSize, (*tileset).zoomRange);
result->requiredTileCount += tileSourceCount;
result->requiredResourceCount += tileSourceCount;
}
} else {
result->requiredResourceCountIsPrecise = false;
}
}
};
switch (type) {
case SourceType::Vector: {
const auto& vectorSource = *source->as<VectorSource>();
handleTiledSource(vectorSource.getURLOrTileset(), util::tileSize);
break;
}
case SourceType::Raster: {
const auto& rasterSource = *source->as<RasterSource>();
handleTiledSource(rasterSource.getURLOrTileset(), rasterSource.getTileSize());
break;
}
case SourceType::RasterDEM: {
const auto& rasterDEMSource = *source->as<RasterDEMSource>();
handleTiledSource(rasterDEMSource.getURLOrTileset(), rasterDEMSource.getTileSize());
break;
}
case SourceType::GeoJSON: {
const auto& geojsonSource = *source->as<GeoJSONSource>();
if (geojsonSource.getURL()) {
result->requiredResourceCount += 1;
}
break;
}
case SourceType::Image: {
const auto& imageSource = *source->as<ImageSource>();
if (imageSource.getURL()) {
result->requiredResourceCount += 1;
}
break;
}
case SourceType::Video:
case SourceType::Annotations:
case SourceType::CustomVector:
break;
}
}
if (!parser.glyphURL.empty()) {
result->requiredResourceCount += parser.fontStacks().size() *
(definition.match([](auto& reg){ return reg.includeIdeographs; }) ?
GLYPH_RANGES_PER_FONT_STACK :
NON_IDEOGRAPH_GLYPH_RANGES_PER_FONT_STACK);
}
if (!parser.spriteURL.empty()) {
result->requiredResourceCount += 4;
}
return *result;
}
void OfflineDownload::activateDownload() {
status = OfflineRegionStatus();
status.downloadState = OfflineRegionDownloadState::Active;
status.requiredResourceCount++;
auto styleResource = Resource::style(definition.match([](auto& reg){ return reg.styleURL; }));
styleResource.setPriority(Resource::Priority::Low);
styleResource.setUsage(Resource::Usage::Offline);
ensureResource(std::move(styleResource), [&](Response styleResponse) {
status.requiredResourceCountIsPrecise = true;
style::Parser parser;
parser.parse(*styleResponse.data);
for (const auto& source : parser.sources) {
SourceType type = source->getType();
auto handleTiledSource = [&] (const variant<std::string, Tileset>& urlOrTileset, const uint16_t tileSize) {
if (urlOrTileset.is<Tileset>()) {
queueTiles(type, tileSize, urlOrTileset.get<Tileset>());
} else {
const auto& url = urlOrTileset.get<std::string>();
status.requiredResourceCountIsPrecise = false;
status.requiredResourceCount++;
requiredSourceURLs.insert(url);
auto sourceResource = Resource::source(url);
sourceResource.setPriority(Resource::Priority::Low);
sourceResource.setUsage(Resource::Usage::Offline);
ensureResource(std::move(sourceResource), [=](Response sourceResponse) {
style::conversion::Error error;
optional<Tileset> tileset = style::conversion::convertJSON<Tileset>(*sourceResponse.data, error);
if (tileset) {
util::mapbox::canonicalizeTileset(*tileset, url, type, tileSize);
queueTiles(type, tileSize, *tileset);
requiredSourceURLs.erase(url);
if (requiredSourceURLs.empty()) {
status.requiredResourceCountIsPrecise = true;
}
}
});
}
};
switch (type) {
case SourceType::Vector: {
const auto& vectorSource = *source->as<VectorSource>();
handleTiledSource(vectorSource.getURLOrTileset(), util::tileSize);
break;
}
case SourceType::Raster: {
const auto& rasterSource = *source->as<RasterSource>();
handleTiledSource(rasterSource.getURLOrTileset(), rasterSource.getTileSize());
break;
}
case SourceType::RasterDEM: {
const auto& rasterDEMSource = *source->as<RasterDEMSource>();
handleTiledSource(rasterDEMSource.getURLOrTileset(), rasterDEMSource.getTileSize());
break;
}
case SourceType::GeoJSON: {
const auto& geojsonSource = *source->as<GeoJSONSource>();
if (geojsonSource.getURL()) {
queueResource(Resource::source(*geojsonSource.getURL()));
}
break;
}
case SourceType::Image: {
const auto& imageSource = *source->as<ImageSource>();
auto imageUrl = imageSource.getURL();
if (imageUrl && !imageUrl->empty()) {
queueResource(Resource::image(*imageUrl));
}
break;
}
case SourceType::Video:
case SourceType::Annotations:
case SourceType::CustomVector:
break;
}
}
if (!parser.glyphURL.empty()) {
const bool includeIdeographs = definition.match([](auto& reg){ return reg.includeIdeographs; });
for (const auto& fontStack : parser.fontStacks()) {
for (char16_t i = 0; i < GLYPH_RANGES_PER_FONT_STACK; i++) {
// Assumes that if a glyph range starts with fixed width/ideographic characters, the entire
// range will be fixed width.
if (includeIdeographs || !util::i18n::allowsFixedWidthGlyphGeneration(i * GLYPHS_PER_GLYPH_RANGE)) {
queueResource(Resource::glyphs(parser.glyphURL, fontStack, getGlyphRange(i * GLYPHS_PER_GLYPH_RANGE)));
}
}
}
}
if (!parser.spriteURL.empty()) {
// Always request 1x and @2x sprite images for portability.
queueResource(Resource::spriteImage(parser.spriteURL, 1));
queueResource(Resource::spriteImage(parser.spriteURL, 2));
queueResource(Resource::spriteJSON(parser.spriteURL, 1));
queueResource(Resource::spriteJSON(parser.spriteURL, 2));
}
continueDownload();
});
}
/*
Fill up our own request queue by requesting the next few resources. This is called
when activating the download, or when a request completes successfully.
Note "successfully"; it's not called when a requests receives an error. A request
that errors will be retried after some delay. So in that sense it's still "active"
and consuming resources, notably the request object, its timer, and network resources
when the timer fires.
We could try to squeeze in subsequent requests while we wait for the errored request
to retry. But that risks overloading the upstream request queue -- defeating our own
metering -- if there are a lot of errored requests that all come up for retry at the
same time. And many times, the cause of a request error will apply to many requests
of the same type. For instance if a server is unreachable, all the requests to that
host are going to error. In that case, continuing to try subsequent resources after
the first few errors is fruitless anyway.
*/
void OfflineDownload::continueDownload() {
if (resourcesRemaining.empty() && status.complete()) {
markPendingUsedResources();
setState(OfflineRegionDownloadState::Inactive);
return;
}
if (resourcesToBeMarkedAsUsed.size() >= kMarkBatchSize) markPendingUsedResources();
uint32_t maxConcurrentRequests = util::DEFAULT_MAXIMUM_CONCURRENT_REQUESTS;
auto value = onlineFileSource.getProperty("max-concurrent-requests");
if (uint64_t* maxRequests = value.getUint()) {
maxConcurrentRequests = static_cast<uint32_t>(*maxRequests);
}
while (!resourcesRemaining.empty() && requests.size() < maxConcurrentRequests) {
ensureResource(std::move(resourcesRemaining.front()));
resourcesRemaining.pop_front();
}
}
void OfflineDownload::deactivateDownload() {
requiredSourceURLs.clear();
resourcesRemaining.clear();
requests.clear();
buffer.clear();
}
void OfflineDownload::queueResource(Resource&& resource) {
resource.setPriority(Resource::Priority::Low);
resource.setUsage(Resource::Usage::Offline);
status.requiredResourceCount++;
if (resource.kind == mbgl::Resource::Kind::Tile) {
status.requiredTileCount++;
}
resourcesRemaining.push_front(std::move(resource));
}
void OfflineDownload::queueTiles(SourceType type, uint16_t tileSize, const Tileset& tileset) {
tileCover(definition, type, tileSize, tileset.zoomRange, [&](const auto& tile) {
status.requiredResourceCount++;
status.requiredTileCount++;
auto tileResource = Resource::tile(
tileset.tiles[0], definition.match([](auto& def) { return def.pixelRatio; }),
tile.x, tile.y, tile.z, tileset.scheme);
tileResource.setPriority(Resource::Priority::Low);
tileResource.setUsage(Resource::Usage::Offline);
resourcesRemaining.push_back(std::move(tileResource));
});
}
void OfflineDownload::markPendingUsedResources() {
offlineDatabase.markUsedResources(id, resourcesToBeMarkedAsUsed);
resourcesToBeMarkedAsUsed.clear();
}
void OfflineDownload::ensureResource(Resource&& resource,
std::function<void(Response)> callback) {
assert(resource.priority == Resource::Priority::Low);
assert(resource.usage == Resource::Usage::Offline);
auto workRequestsIt = requests.insert(requests.begin(), nullptr);
*workRequestsIt = util::RunLoop::Get()->invokeCancellable([=]() {
requests.erase(workRequestsIt);
const auto resourceKind = resource.kind;
auto getResourceSizeInDatabase = [&] () -> optional<int64_t> {
optional<int64_t> result;
if (!callback) {
result = offlineDatabase.hasRegionResource(resource);
} else {
optional<std::pair<Response, uint64_t>> response = offlineDatabase.getRegionResource(resource);
if (response) {
callback(response->first);
result = response->second;
}
}
if (result) resourcesToBeMarkedAsUsed.emplace_back(resource);
return result;
};
optional<int64_t> offlineResponse = getResourceSizeInDatabase();
if (offlineResponse) {
assert(!resourcesToBeMarkedAsUsed.empty());
status.completedResourceCount++;
status.completedResourceSize += *offlineResponse;
if (resourceKind == Resource::Kind::Tile) {
status.completedTileCount += 1;
status.completedTileSize += *offlineResponse;
}
observer->statusChanged(status);
continueDownload();
return;
}
if (offlineDatabase.exceedsOfflineMapboxTileCountLimit(resource)) {
onMapboxTileCountLimitExceeded();
return;
}
auto fileRequestsIt = requests.insert(requests.begin(), nullptr);
*fileRequestsIt = onlineFileSource.request(resource, [=](Response onlineResponse) {
if (onlineResponse.error) {
observer->responseError(*onlineResponse.error);
return;
}
requests.erase(fileRequestsIt);
if (callback) {
callback(onlineResponse);
}
// Queue up for batched insertion
buffer.emplace_back(resource, onlineResponse);
// Flush buffer periodically
if (buffer.size() == kResourcesBatchSize || resourcesRemaining.empty()) {
try {
offlineDatabase.putRegionResources(id, buffer, status);
} catch (const MapboxTileLimitExceededException&) {
onMapboxTileCountLimitExceeded();
return;
}
buffer.clear();
observer->statusChanged(status);
}
if (offlineDatabase.exceedsOfflineMapboxTileCountLimit(resource)) {
onMapboxTileCountLimitExceeded();
return;
}
continueDownload();
});
});
}
void OfflineDownload::onMapboxTileCountLimitExceeded() {
observer->mapboxTileCountLimitExceeded(offlineDatabase.getOfflineMapboxTileCountLimit());
setState(OfflineRegionDownloadState::Inactive);
}
} // namespace mbgl