tei/MapLibreParticleLab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[core] add priorities to resources

Browse Source 
- priorities can be low or regular
- offline downloads should have low priority to not throttle "regular requests"
This commit is contained in:
Huyen Chau Nguyen
2018-10-17 20:12:28 +02:00
parent 9afe75aa76
commit 1e3a744d22
6 changed files with 109 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
include/mbgl/storage/resource.hpp
View File

@@ -24,6 +24,11 @@ public:
Image
};
enum class Priority : bool {
Regular,
Low
};
struct TileData {
std::string urlTemplate;
uint8_t pixelRatio;
@@ -44,34 +49,41 @@ public:
Resource(Kind kind_,
std::string url_,
Priority priority_ = Resource::Priority::Regular,
optional<TileData> tileData_ = {},
LoadingMethod loadingMethod_ = LoadingMethod::All)
: kind(kind_),
loadingMethod(loadingMethod_),
priority(priority_),
url(std::move(url_)),
tileData(std::move(tileData_)) {
}
void setPriority(Priority p) { priority = p; }
bool hasLoadingMethod(LoadingMethod method);
static Resource style(const std::string& url);
static Resource source(const std::string& url);
static Resource style(const std::string& url, const Priority priority = Resource::Priority::Regular);
static Resource source(const std::string& url, const Priority priority = Resource::Priority::Regular);
static Resource tile(const std::string& urlTemplate,
float pixelRatio,
int32_t x,
int32_t y,
int8_t z,
Tileset::Scheme scheme,
const Priority priority = Resource::Priority::Regular,
LoadingMethod = LoadingMethod::All);
static Resource glyphs(const std::string& urlTemplate,
const FontStack& fontStack,
const std::pair<uint16_t, uint16_t>& glyphRange);
static Resource spriteImage(const std::string& base, float pixelRatio);
static Resource spriteJSON(const std::string& base, float pixelRatio);
static Resource image(const std::string& url);
const std::pair<uint16_t, uint16_t>& glyphRange,
const Priority priority = Resource::Priority::Regular);
static Resource spriteImage(const std::string& base, float pixelRatio, const Priority priority = Resource::Priority::Regular);
static Resource spriteJSON(const std::string& base, float pixelRatio, const Priority priority = Resource::Priority::Regular);
static Resource image(const std::string& url, const Priority priority = Resource::Priority::Regular);
Kind kind;
LoadingMethod loadingMethod;
Priority priority;
std::string url;
// Includes auxiliary data if this is a tile request.

9
platform/default/mbgl/storage/offline_download.cpp
View File

@@ -219,7 +219,7 @@ void OfflineDownload::activateDownload() {
status = OfflineRegionStatus();
status.downloadState = OfflineRegionDownloadState::Active;
status.requiredResourceCount++;
ensureResource(Resource::style(definition.match([](auto& reg){ return reg.styleURL; })),
ensureResource(Resource::style(definition.match([](auto& reg){ return reg.styleURL; }), Resource::Priority::Low),
[&](Response styleResponse) {
status.requiredResourceCountIsPrecise = true;
@@ -238,7 +238,7 @@ void OfflineDownload::activateDownload() {
status.requiredResourceCount++;
requiredSourceURLs.insert(url);
ensureResource(Resource::source(url), [=](Response sourceResponse) {
ensureResource(Resource::source(url, Resource::Priority::Low), [=](Response sourceResponse) {
style::conversion::Error error;
optional<Tileset> tileset = style::conversion::convertJSON<Tileset>(*sourceResponse.data, error);
if (tileset) {
@@ -353,6 +353,7 @@ void OfflineDownload::deactivateDownload() {
}
void OfflineDownload::queueResource(Resource resource) {
resource.setPriority(Resource::Priority::Low);
status.requiredResourceCount++;
resourcesRemaining.push_front(std::move(resource));
}
@@ -362,12 +363,14 @@ void OfflineDownload::queueTiles(SourceType type, uint16_t tileSize, const Tiles
status.requiredResourceCount++;
resourcesRemaining.push_back(Resource::tile(
tileset.tiles[0], definition.match([](auto& def) { return def.pixelRatio; }), tile.x,
tile.y, tile.z, tileset.scheme));
tile.y, tile.z, tileset.scheme, Resource::Priority::Low));
});
}
void OfflineDownload::ensureResource(const Resource& resource,
std::function<void(Response)> callback) {
assert(resource.priority == Resource::Priority::Low);
auto workRequestsIt = requests.insert(requests.begin(), nullptr);
*workRequestsIt = util::RunLoop::Get()->invokeCancellable([=]() {
requests.erase(workRequestsIt);

32
src/mbgl/storage/resource.cpp
View File

@@ -39,44 +39,49 @@ static std::string getTileBBox(int32_t x, int32_t y, int8_t z) {
util::toString(max.x) + "," + util::toString(max.y));
}
Resource Resource::style(const std::string& url) {
Resource Resource::style(const std::string& url, const Priority priority) {
return Resource {
Resource::Kind::Style,
url
url,
priority
};
}
Resource Resource::source(const std::string& url) {
Resource Resource::source(const std::string& url, const Priority priority) {
return Resource {
Resource::Kind::Source,
url
url,
priority
};
}
Resource Resource::image(const std::string& url) {
Resource Resource::image(const std::string& url, const Priority priority) {
return Resource {
Resource::Kind::Image,
url
url,
priority
};
}
Resource Resource::spriteImage(const std::string& base, float pixelRatio) {
Resource Resource::spriteImage(const std::string& base, float pixelRatio, const Priority priority) {
util::URL url(base);
return Resource{ Resource::Kind::SpriteImage,
base.substr(0, url.path.first + url.path.second) +
(pixelRatio > 1 ? "@2x" : "") + ".png" +
base.substr(url.query.first, url.query.second) };
base.substr(url.query.first, url.query.second),
priority };
}
Resource Resource::spriteJSON(const std::string& base, float pixelRatio) {
Resource Resource::spriteJSON(const std::string& base, float pixelRatio, const Priority priority) {
util::URL url(base);
return Resource{ Resource::Kind::SpriteJSON,
base.substr(0, url.path.first + url.path.second) +
(pixelRatio > 1 ? "@2x" : "") + ".json" +
base.substr(url.query.first, url.query.second) };
base.substr(url.query.first, url.query.second),
priority };
}
Resource Resource::glyphs(const std::string& urlTemplate, const FontStack& fontStack, const std::pair<uint16_t, uint16_t>& glyphRange) {
Resource Resource::glyphs(const std::string& urlTemplate, const FontStack& fontStack, const std::pair<uint16_t, uint16_t>& glyphRange, const Priority priority) {
return Resource {
Resource::Kind::Glyphs,
util::replaceTokens(urlTemplate, [&](const std::string& token) -> optional<std::string> {
@@ -87,7 +92,8 @@ Resource Resource::glyphs(const std::string& urlTemplate, const FontStack& fontS
} else {
return {};
}
})
}),
priority
};
}
@@ -97,6 +103,7 @@ Resource Resource::tile(const std::string& urlTemplate,
int32_t y,
int8_t z,
Tileset::Scheme scheme,
const Priority priority,
LoadingMethod loadingMethod) {
bool supportsRatio = urlTemplate.find("{ratio}") != std::string::npos;
if (scheme == Tileset::Scheme::TMS) {
@@ -126,6 +133,7 @@ Resource Resource::tile(const std::string& urlTemplate,
return {};
}
}),
priority,
Resource::TileData {
urlTemplate,
uint8_t(supportsRatio && pixelRatio > 1.0 ? 2 : 1),

1
src/mbgl/tile/tile_loader_impl.hpp
View File

@@ -23,6 +23,7 @@ TileLoader<T>::TileLoader(T& tile_,
id.canonical.y,
id.canonical.z,
tileset.scheme,
Resource::Priority::Regular,
Resource::LoadingMethod::CacheOnly)),
fileSource(parameters.fileSource) {
assert(!request);

6
test/storage/default_file_source.test.cpp
View File

@@ -251,7 +251,7 @@ TEST(DefaultFileSource, OptionalNonExpired) {
util::RunLoop loop;
DefaultFileSource fs(":memory:", ".");
const Resource optionalResource { Resource::Unknown, "http://127.0.0.1:3000/test", {}, Resource::LoadingMethod::CacheOnly };
const Resource optionalResource { Resource::Unknown, "http://127.0.0.1:3000/test", Resource::Priority::Regular, {}, Resource::LoadingMethod::CacheOnly };
using namespace std::chrono_literals;
@@ -281,7 +281,7 @@ TEST(DefaultFileSource, OptionalExpired) {
util::RunLoop loop;
DefaultFileSource fs(":memory:", ".");
const Resource optionalResource { Resource::Unknown, "http://127.0.0.1:3000/test", {}, Resource::LoadingMethod::CacheOnly };
const Resource optionalResource { Resource::Unknown, "http://127.0.0.1:3000/test", Resource::Priority::Regular, {}, Resource::LoadingMethod::CacheOnly };
using namespace std::chrono_literals;
@@ -327,7 +327,7 @@ TEST(DefaultFileSource, OptionalNotFound) {
util::RunLoop loop;
DefaultFileSource fs(":memory:", ".");
const Resource optionalResource { Resource::Unknown, "http://127.0.0.1:3000/test", {}, Resource::LoadingMethod::CacheOnly };
const Resource optionalResource { Resource::Unknown, "http://127.0.0.1:3000/test", Resource::Priority::Regular, {}, Resource::LoadingMethod::CacheOnly };
using namespace std::chrono_literals;

60
test/storage/offline_download.test.cpp
View File

@@ -662,6 +662,66 @@ TEST(OfflineDownload, Deactivate) {
test.loop.run();
}
TEST(OfflineDownload, LowPriorityRequests) {
OfflineTest test;
auto region = test.createRegion();
ASSERT_TRUE(region);
OfflineDownload download(
region->getID(),
OfflineTilePyramidRegionDefinition("http://127.0.0.1:3000/style.json", LatLngBounds::world(), 0.0, 0.0, 1.0),
test.db, test.fileSource);
test.fileSource.styleResponse = [&] (const Resource& resource) {
EXPECT_TRUE(resource.priority == Resource::Priority::Low);
return test.response("style.json");
};
test.fileSource.spriteImageResponse = [&] (const Resource& resource) {
EXPECT_TRUE(resource.priority == Resource::Priority::Low);
return test.response("sprite.png");
};
test.fileSource.imageResponse = [&] (const Resource& resource) {
EXPECT_TRUE(resource.priority == Resource::Priority::Low);
return test.response("radar.gif");
};
test.fileSource.spriteJSONResponse = [&] (const Resource& resource) {
EXPECT_TRUE(resource.priority == Resource::Priority::Low);
return test.response("sprite.json");
};
test.fileSource.glyphsResponse = [&] (const Resource& resource) {
EXPECT_TRUE(resource.priority == Resource::Priority::Low);
return test.response("glyph.pbf");
};
test.fileSource.sourceResponse = [&] (const Resource& resource) {
EXPECT_TRUE(resource.priority == Resource::Priority::Low);
return test.response("streets.json");
};
test.fileSource.tileResponse = [&] (const Resource& resource) {
EXPECT_TRUE(resource.priority == Resource::Priority::Low);
return test.response("0-0-0.vector.pbf");
};
auto observer = std::make_unique<MockObserver>();
observer->statusChangedFn = [&] (OfflineRegionStatus status) {
if (status.complete()) {
test.loop.stop();
}
};
download.setObserver(std::move(observer));
download.setState(OfflineRegionDownloadState::Active);
test.loop.run();
}
#ifndef __QT__ // Qt doesn't expose the ability to register virtual file system handlers.
TEST(OfflineDownload, DiskFull) {
FixtureLog log;