153acd981b
When creating a offline region, we've previously only requested the sprite image for the specified resolution. This lead to offline packs not being usable on devices that have a different pixel ratio. We're now requesting both 1x and 2x sprites. Some devices use even higher or fractional pixel ratios. However, we only ever use 1x and 2x sprite images in our requests.
451 lines
16 KiB
C++
451 lines
16 KiB
C++
#include <mbgl/storage/file_source.hpp>
|
|
#include <mbgl/storage/offline_database.hpp>
|
|
#include <mbgl/storage/offline_download.hpp>
|
|
#include <mbgl/storage/resource.hpp>
|
|
#include <mbgl/storage/response.hpp>
|
|
#include <mbgl/storage/http_file_source.hpp>
|
|
#include <mbgl/style/parser.hpp>
|
|
#include <mbgl/style/sources/vector_source.hpp>
|
|
#include <mbgl/style/sources/raster_source.hpp>
|
|
#include <mbgl/style/sources/raster_dem_source.hpp>
|
|
#include <mbgl/style/sources/geojson_source.hpp>
|
|
#include <mbgl/style/sources/image_source.hpp>
|
|
#include <mbgl/style/conversion/json.hpp>
|
|
#include <mbgl/style/conversion/tileset.hpp>
|
|
#include <mbgl/text/glyph.hpp>
|
|
#include <mbgl/util/mapbox.hpp>
|
|
#include <mbgl/util/run_loop.hpp>
|
|
#include <mbgl/util/tile_cover.hpp>
|
|
#include <mbgl/util/tileset.hpp>
|
|
|
|
#include <set>
|
|
|
|
namespace mbgl {
|
|
|
|
using namespace style;
|
|
|
|
// Generic functions
|
|
|
|
template <class RegionDefinition>
|
|
Range<uint8_t> coveringZoomRange(const RegionDefinition& definition,
|
|
style::SourceType type, uint16_t tileSize, const Range<uint8_t>& zoomRange) {
|
|
double minZ = std::max<double>(util::coveringZoomLevel(definition.minZoom, type, tileSize), zoomRange.min);
|
|
double maxZ = std::min<double>(util::coveringZoomLevel(definition.maxZoom, type, tileSize), zoomRange.max);
|
|
|
|
assert(minZ >= 0);
|
|
assert(maxZ >= 0);
|
|
assert(minZ < std::numeric_limits<uint8_t>::max());
|
|
assert(maxZ < std::numeric_limits<uint8_t>::max());
|
|
return { static_cast<uint8_t>(minZ), static_cast<uint8_t>(maxZ) };
|
|
}
|
|
|
|
template <class Geometry, class Fn>
|
|
void tileCover(const Geometry& geometry, uint8_t z, Fn&& fn) {
|
|
util::TileCover cover(geometry, z);
|
|
while (cover.hasNext()) {
|
|
fn(cover.next()->canonical);
|
|
}
|
|
}
|
|
|
|
|
|
template <class Fn>
|
|
void tileCover(const OfflineRegionDefinition& definition, style::SourceType type,
|
|
uint16_t tileSize, const Range<uint8_t>& zoomRange, Fn&& fn) {
|
|
const Range<uint8_t> clampedZoomRange =
|
|
definition.match([&](auto& reg) { return coveringZoomRange(reg, type, tileSize, zoomRange); });
|
|
|
|
for (uint8_t z = clampedZoomRange.min; z <= clampedZoomRange.max; z++) {
|
|
definition.match(
|
|
[&](const OfflineTilePyramidRegionDefinition& reg){ tileCover(reg.bounds, z, fn); },
|
|
[&](const OfflineGeometryRegionDefinition& reg){ tileCover(reg.geometry, z, fn); }
|
|
);
|
|
}
|
|
}
|
|
|
|
uint64_t tileCount(const OfflineRegionDefinition& definition, style::SourceType type,
|
|
uint16_t tileSize, const Range<uint8_t>& zoomRange) {
|
|
|
|
const Range<uint8_t> clampedZoomRange =
|
|
definition.match([&](auto& reg) { return coveringZoomRange(reg, type, tileSize, zoomRange); });
|
|
|
|
unsigned long result = 0;;
|
|
for (uint8_t z = clampedZoomRange.min; z <= clampedZoomRange.max; z++) {
|
|
result += definition.match(
|
|
[&](const OfflineTilePyramidRegionDefinition& reg){ return util::tileCount(reg.bounds, z); },
|
|
[&](const OfflineGeometryRegionDefinition& reg){ return util::tileCount(reg.geometry, z); }
|
|
);
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
// OfflineDownload
|
|
|
|
OfflineDownload::OfflineDownload(int64_t id_,
|
|
OfflineRegionDefinition&& definition_,
|
|
OfflineDatabase& offlineDatabase_,
|
|
FileSource& onlineFileSource_)
|
|
: id(id_),
|
|
definition(definition_),
|
|
offlineDatabase(offlineDatabase_),
|
|
onlineFileSource(onlineFileSource_) {
|
|
setObserver(nullptr);
|
|
}
|
|
|
|
OfflineDownload::~OfflineDownload() = default;
|
|
|
|
void OfflineDownload::setObserver(std::unique_ptr<OfflineRegionObserver> observer_) {
|
|
observer = observer_ ? std::move(observer_) : std::make_unique<OfflineRegionObserver>();
|
|
}
|
|
|
|
void OfflineDownload::setState(OfflineRegionDownloadState state) {
|
|
if (status.downloadState == state) {
|
|
return;
|
|
}
|
|
|
|
status.downloadState = state;
|
|
|
|
if (status.downloadState == OfflineRegionDownloadState::Active) {
|
|
activateDownload();
|
|
} else {
|
|
deactivateDownload();
|
|
}
|
|
|
|
observer->statusChanged(status);
|
|
}
|
|
|
|
OfflineRegionStatus OfflineDownload::getStatus() const {
|
|
if (status.downloadState == OfflineRegionDownloadState::Active) {
|
|
return status;
|
|
}
|
|
|
|
auto result = offlineDatabase.getRegionCompletedStatus(id);
|
|
if (!result) {
|
|
// We can't find this offline region because the database is unavailable, or the download
|
|
// does not exist.
|
|
return {};
|
|
}
|
|
|
|
result->requiredResourceCount++;
|
|
optional<Response> styleResponse =
|
|
offlineDatabase.get(Resource::style(definition.match([](auto& reg){ return reg.styleURL; })));
|
|
if (!styleResponse) {
|
|
return *result;
|
|
}
|
|
|
|
style::Parser parser;
|
|
parser.parse(*styleResponse->data);
|
|
|
|
result->requiredResourceCountIsPrecise = true;
|
|
|
|
for (const auto& source : parser.sources) {
|
|
SourceType type = source->getType();
|
|
|
|
auto handleTiledSource = [&] (const variant<std::string, Tileset>& urlOrTileset, const uint16_t tileSize) {
|
|
if (urlOrTileset.is<Tileset>()) {
|
|
result->requiredResourceCount +=
|
|
tileCount(definition, type, tileSize, urlOrTileset.get<Tileset>().zoomRange);
|
|
} else {
|
|
result->requiredResourceCount += 1;
|
|
const auto& url = urlOrTileset.get<std::string>();
|
|
optional<Response> sourceResponse = offlineDatabase.get(Resource::source(url));
|
|
if (sourceResponse) {
|
|
style::conversion::Error error;
|
|
optional<Tileset> tileset = style::conversion::convertJSON<Tileset>(*sourceResponse->data, error);
|
|
if (tileset) {
|
|
result->requiredResourceCount +=
|
|
tileCount(definition, type, tileSize, (*tileset).zoomRange);
|
|
}
|
|
} else {
|
|
result->requiredResourceCountIsPrecise = false;
|
|
}
|
|
}
|
|
};
|
|
|
|
switch (type) {
|
|
case SourceType::Vector: {
|
|
const auto& vectorSource = *source->as<VectorSource>();
|
|
handleTiledSource(vectorSource.getURLOrTileset(), util::tileSize);
|
|
break;
|
|
}
|
|
|
|
case SourceType::Raster: {
|
|
const auto& rasterSource = *source->as<RasterSource>();
|
|
handleTiledSource(rasterSource.getURLOrTileset(), rasterSource.getTileSize());
|
|
break;
|
|
}
|
|
|
|
case SourceType::RasterDEM: {
|
|
const auto& rasterDEMSource = *source->as<RasterDEMSource>();
|
|
handleTiledSource(rasterDEMSource.getURLOrTileset(), rasterDEMSource.getTileSize());
|
|
break;
|
|
}
|
|
|
|
case SourceType::GeoJSON: {
|
|
const auto& geojsonSource = *source->as<GeoJSONSource>();
|
|
if (geojsonSource.getURL()) {
|
|
result->requiredResourceCount += 1;
|
|
}
|
|
break;
|
|
}
|
|
|
|
case SourceType::Image: {
|
|
const auto& imageSource = *source->as<ImageSource>();
|
|
if (imageSource.getURL()) {
|
|
result->requiredResourceCount += 1;
|
|
}
|
|
break;
|
|
}
|
|
|
|
case SourceType::Video:
|
|
case SourceType::Annotations:
|
|
case SourceType::CustomVector:
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (!parser.glyphURL.empty()) {
|
|
result->requiredResourceCount += parser.fontStacks().size() * GLYPH_RANGES_PER_FONT_STACK;
|
|
}
|
|
|
|
if (!parser.spriteURL.empty()) {
|
|
result->requiredResourceCount += 4;
|
|
}
|
|
|
|
return *result;
|
|
}
|
|
|
|
void OfflineDownload::activateDownload() {
|
|
status = OfflineRegionStatus();
|
|
status.downloadState = OfflineRegionDownloadState::Active;
|
|
status.requiredResourceCount++;
|
|
ensureResource(Resource::style(definition.match([](auto& reg){ return reg.styleURL; })),
|
|
[&](Response styleResponse) {
|
|
status.requiredResourceCountIsPrecise = true;
|
|
|
|
style::Parser parser;
|
|
parser.parse(*styleResponse.data);
|
|
|
|
for (const auto& source : parser.sources) {
|
|
SourceType type = source->getType();
|
|
|
|
auto handleTiledSource = [&] (const variant<std::string, Tileset>& urlOrTileset, const uint16_t tileSize) {
|
|
if (urlOrTileset.is<Tileset>()) {
|
|
queueTiles(type, tileSize, urlOrTileset.get<Tileset>());
|
|
} else {
|
|
const auto& url = urlOrTileset.get<std::string>();
|
|
status.requiredResourceCountIsPrecise = false;
|
|
status.requiredResourceCount++;
|
|
requiredSourceURLs.insert(url);
|
|
|
|
ensureResource(Resource::source(url), [=](Response sourceResponse) {
|
|
style::conversion::Error error;
|
|
optional<Tileset> tileset = style::conversion::convertJSON<Tileset>(*sourceResponse.data, error);
|
|
if (tileset) {
|
|
util::mapbox::canonicalizeTileset(*tileset, url, type, tileSize);
|
|
queueTiles(type, tileSize, *tileset);
|
|
|
|
requiredSourceURLs.erase(url);
|
|
if (requiredSourceURLs.empty()) {
|
|
status.requiredResourceCountIsPrecise = true;
|
|
}
|
|
}
|
|
});
|
|
}
|
|
};
|
|
|
|
switch (type) {
|
|
case SourceType::Vector: {
|
|
const auto& vectorSource = *source->as<VectorSource>();
|
|
handleTiledSource(vectorSource.getURLOrTileset(), util::tileSize);
|
|
break;
|
|
}
|
|
|
|
case SourceType::Raster: {
|
|
const auto& rasterSource = *source->as<RasterSource>();
|
|
handleTiledSource(rasterSource.getURLOrTileset(), rasterSource.getTileSize());
|
|
break;
|
|
}
|
|
|
|
case SourceType::RasterDEM: {
|
|
const auto& rasterDEMSource = *source->as<RasterDEMSource>();
|
|
handleTiledSource(rasterDEMSource.getURLOrTileset(), rasterDEMSource.getTileSize());
|
|
break;
|
|
}
|
|
|
|
case SourceType::GeoJSON: {
|
|
const auto& geojsonSource = *source->as<GeoJSONSource>();
|
|
if (geojsonSource.getURL()) {
|
|
queueResource(Resource::source(*geojsonSource.getURL()));
|
|
}
|
|
break;
|
|
}
|
|
|
|
case SourceType::Image: {
|
|
const auto& imageSource = *source->as<ImageSource>();
|
|
auto imageUrl = imageSource.getURL();
|
|
if (imageUrl && !imageUrl->empty()) {
|
|
queueResource(Resource::image(*imageUrl));
|
|
}
|
|
break;
|
|
}
|
|
|
|
case SourceType::Video:
|
|
case SourceType::Annotations:
|
|
case SourceType::CustomVector:
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (!parser.glyphURL.empty()) {
|
|
for (const auto& fontStack : parser.fontStacks()) {
|
|
for (char16_t i = 0; i < GLYPH_RANGES_PER_FONT_STACK; i++) {
|
|
queueResource(Resource::glyphs(parser.glyphURL, fontStack, getGlyphRange(i * GLYPHS_PER_GLYPH_RANGE)));
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!parser.spriteURL.empty()) {
|
|
// Always request 1x and @2x sprite images for portability.
|
|
queueResource(Resource::spriteImage(parser.spriteURL, 1));
|
|
queueResource(Resource::spriteImage(parser.spriteURL, 2));
|
|
queueResource(Resource::spriteJSON(parser.spriteURL, 1));
|
|
queueResource(Resource::spriteJSON(parser.spriteURL, 2));
|
|
}
|
|
|
|
continueDownload();
|
|
});
|
|
}
|
|
|
|
/*
|
|
Fill up our own request queue by requesting the next few resources. This is called
|
|
when activating the download, or when a request completes successfully.
|
|
|
|
Note "successfully"; it's not called when a requests receives an error. A request
|
|
that errors will be retried after some delay. So in that sense it's still "active"
|
|
and consuming resources, notably the request object, its timer, and network resources
|
|
when the timer fires.
|
|
|
|
We could try to squeeze in subsequent requests while we wait for the errored request
|
|
to retry. But that risks overloading the upstream request queue -- defeating our own
|
|
metering -- if there are a lot of errored requests that all come up for retry at the
|
|
same time. And many times, the cause of a request error will apply to many requests
|
|
of the same type. For instance if a server is unreachable, all the requests to that
|
|
host are going to error. In that case, continuing to try subsequent resources after
|
|
the first few errors is fruitless anyway.
|
|
*/
|
|
void OfflineDownload::continueDownload() {
|
|
if (resourcesRemaining.empty() && status.complete()) {
|
|
setState(OfflineRegionDownloadState::Inactive);
|
|
return;
|
|
}
|
|
|
|
while (!resourcesRemaining.empty() && requests.size() < HTTPFileSource::maximumConcurrentRequests()) {
|
|
ensureResource(resourcesRemaining.front());
|
|
resourcesRemaining.pop_front();
|
|
}
|
|
}
|
|
|
|
void OfflineDownload::deactivateDownload() {
|
|
requiredSourceURLs.clear();
|
|
resourcesRemaining.clear();
|
|
requests.clear();
|
|
}
|
|
|
|
void OfflineDownload::queueResource(Resource resource) {
|
|
status.requiredResourceCount++;
|
|
resourcesRemaining.push_front(std::move(resource));
|
|
}
|
|
|
|
void OfflineDownload::queueTiles(SourceType type, uint16_t tileSize, const Tileset& tileset) {
|
|
tileCover(definition, type, tileSize, tileset.zoomRange, [&](const auto& tile) {
|
|
status.requiredResourceCount++;
|
|
resourcesRemaining.push_back(Resource::tile(
|
|
tileset.tiles[0], definition.match([](auto& def) { return def.pixelRatio; }), tile.x,
|
|
tile.y, tile.z, tileset.scheme));
|
|
});
|
|
}
|
|
|
|
void OfflineDownload::ensureResource(const Resource& resource,
|
|
std::function<void(Response)> callback) {
|
|
auto workRequestsIt = requests.insert(requests.begin(), nullptr);
|
|
*workRequestsIt = util::RunLoop::Get()->invokeCancellable([=]() {
|
|
requests.erase(workRequestsIt);
|
|
|
|
auto getResourceSizeInDatabase = [&] () -> optional<int64_t> {
|
|
if (!callback) {
|
|
return offlineDatabase.hasRegionResource(id, resource);
|
|
}
|
|
optional<std::pair<Response, uint64_t>> response = offlineDatabase.getRegionResource(id, resource);
|
|
if (!response) {
|
|
return {};
|
|
}
|
|
callback(response->first);
|
|
return response->second;
|
|
};
|
|
|
|
optional<int64_t> offlineResponse = getResourceSizeInDatabase();
|
|
if (offlineResponse) {
|
|
status.completedResourceCount++;
|
|
status.completedResourceSize += *offlineResponse;
|
|
if (resource.kind == Resource::Kind::Tile) {
|
|
status.completedTileCount += 1;
|
|
status.completedTileSize += *offlineResponse;
|
|
}
|
|
|
|
observer->statusChanged(status);
|
|
continueDownload();
|
|
return;
|
|
}
|
|
|
|
if (offlineDatabase.exceedsOfflineMapboxTileCountLimit(resource)) {
|
|
onMapboxTileCountLimitExceeded();
|
|
return;
|
|
}
|
|
|
|
auto fileRequestsIt = requests.insert(requests.begin(), nullptr);
|
|
*fileRequestsIt = onlineFileSource.request(resource, [=](Response onlineResponse) {
|
|
if (onlineResponse.error) {
|
|
observer->responseError(*onlineResponse.error);
|
|
return;
|
|
}
|
|
|
|
requests.erase(fileRequestsIt);
|
|
|
|
if (callback) {
|
|
callback(onlineResponse);
|
|
}
|
|
|
|
// Queue up for batched insertion
|
|
buffer.emplace_back(resource, onlineResponse);
|
|
|
|
// Flush buffer periodically
|
|
if (buffer.size() == 64 || resourcesRemaining.size() == 0) {
|
|
try {
|
|
offlineDatabase.putRegionResources(id, buffer, status);
|
|
} catch (const MapboxTileLimitExceededException&) {
|
|
onMapboxTileCountLimitExceeded();
|
|
return;
|
|
}
|
|
|
|
buffer.clear();
|
|
observer->statusChanged(status);
|
|
}
|
|
|
|
if (offlineDatabase.exceedsOfflineMapboxTileCountLimit(resource)) {
|
|
onMapboxTileCountLimitExceeded();
|
|
return;
|
|
}
|
|
|
|
continueDownload();
|
|
});
|
|
});
|
|
}
|
|
|
|
void OfflineDownload::onMapboxTileCountLimitExceeded() {
|
|
observer->mapboxTileCountLimitExceeded(offlineDatabase.getOfflineMapboxTileCountLimit());
|
|
setState(OfflineRegionDownloadState::Inactive);
|
|
}
|
|
|
|
} // namespace mbgl
|