tei/MapLibreParticleLab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[core] Mark offline region resources in batches

Browse Source
This commit is contained in:
Mikhail Pozdnyakov
2019-08-29 14:39:57 +03:00
parent 0c80f3a383
commit f84e46ad26
5 changed files with 48 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
platform/default/include/mbgl/storage/offline_database.hpp
View File

@@ -78,8 +78,8 @@ public:
std::exception_ptr invalidateRegion(int64_t regionID);
// Return value is (response, stored size)
optional<std::pair<Response, uint64_t>> getRegionResource(int64_t regionID, const Resource&);
optional<int64_t> hasRegionResource(int64_t regionID, const Resource&);
optional<std::pair<Response, uint64_t>> getRegionResource(const Resource&);
optional<int64_t> hasRegionResource(const Resource&);
uint64_t putRegionResource(int64_t regionID, const Resource&, const Response&);
void putRegionResources(int64_t regionID, const std::list<std::tuple<Resource, Response>>&, OfflineRegionStatus&);

2
platform/default/include/mbgl/storage/offline_download.hpp
View File

@@ -60,10 +60,12 @@ private:
std::list<std::unique_ptr<AsyncRequest>> requests;
std::unordered_set<std::string> requiredSourceURLs;
std::deque<Resource> resourcesRemaining;
std::list<Resource> resourcesToBeMarkedAsUsed;
std::list<std::tuple<Resource, Response>> buffer;
void queueResource(Resource&&);
void queueTiles(style::SourceType, uint16_t tileSize, const Tileset&);
void markPendingUsedResources();
};
} // namespace mbgl

20
platform/default/src/mbgl/storage/offline_database.cpp
View File

@@ -877,27 +877,15 @@ std::exception_ptr OfflineDatabase::deleteRegion(OfflineRegion&& region) try {
return std::current_exception();
}
optional<std::pair<Response, uint64_t>> OfflineDatabase::getRegionResource(int64_t regionID, const Resource& resource) try {
auto response = getInternal(resource);
if (response) {
markUsed(regionID, resource);
}
return response;
optional<std::pair<Response, uint64_t>> OfflineDatabase::getRegionResource(const Resource& resource) try {
return getInternal(resource);
} catch (const mapbox::sqlite::Exception& ex) {
handleError(ex, "read region resource");
return nullopt;
}
optional<int64_t> OfflineDatabase::hasRegionResource(int64_t regionID, const Resource& resource) try {
auto response = hasInternal(resource);
if (response) {
markUsed(regionID, resource);
}
return response;
optional<int64_t> OfflineDatabase::hasRegionResource(const Resource& resource) try {
return hasInternal(resource);
} catch (const mapbox::sqlite::Exception& ex) {
handleError(ex, "query region resource");
return nullopt;

31
platform/default/src/mbgl/storage/offline_download.cpp
View File

@@ -353,10 +353,13 @@ void OfflineDownload::activateDownload() {
*/
void OfflineDownload::continueDownload() {
if (resourcesRemaining.empty() && status.complete()) {
markPendingUsedResources();
setState(OfflineRegionDownloadState::Inactive);
return;
}
if (resourcesToBeMarkedAsUsed.size() >= 200) markPendingUsedResources();
while (!resourcesRemaining.empty() && requests.size() < onlineFileSource.getMaximumConcurrentRequests()) {
ensureResource(std::move(resourcesRemaining.front()));
resourcesRemaining.pop_front();
@@ -391,6 +394,11 @@ void OfflineDownload::queueTiles(SourceType type, uint16_t tileSize, const Tiles
});
}
void OfflineDownload::markPendingUsedResources() {
offlineDatabase.markUsedResources(id, resourcesToBeMarkedAsUsed);
resourcesToBeMarkedAsUsed.clear();
}
void OfflineDownload::ensureResource(Resource&& resource,
std::function<void(Response)> callback) {
assert(resource.priority == Resource::Priority::Low);
@@ -399,24 +407,29 @@ void OfflineDownload::ensureResource(Resource&& resource,
auto workRequestsIt = requests.insert(requests.begin(), nullptr);
*workRequestsIt = util::RunLoop::Get()->invokeCancellable([=]() {
requests.erase(workRequestsIt);
const auto resourceKind = resource.kind;
auto getResourceSizeInDatabase = [&] () -> optional<int64_t> {
optional<int64_t> result;
if (!callback) {
return offlineDatabase.hasRegionResource(id, resource);
result = offlineDatabase.hasRegionResource(resource);
} else {
optional<std::pair<Response, uint64_t>> response = offlineDatabase.getRegionResource(resource);
if (response) {
callback(response->first);
result = response->second;
}
}
optional<std::pair<Response, uint64_t>> response = offlineDatabase.getRegionResource(id, resource);
if (!response) {
return {};
}
callback(response->first);
return response->second;
if (result) resourcesToBeMarkedAsUsed.emplace_back(std::move(resource));
return result;
};
optional<int64_t> offlineResponse = getResourceSizeInDatabase();
if (offlineResponse) {
assert(!resourcesToBeMarkedAsUsed.empty());
status.completedResourceCount++;
status.completedResourceSize += *offlineResponse;
if (resource.kind == Resource::Kind::Tile) {
if (resourceKind == Resource::Kind::Tile) {
status.completedTileCount += 1;
status.completedTileSize += *offlineResponse;
}

38
test/storage/offline_database.test.cpp
View File

@@ -1093,8 +1093,8 @@ TEST(OfflineDatabase, HasRegionResource) {
auto region = db.createRegion(definition, OfflineRegionMetadata());
ASSERT_TRUE(region);
EXPECT_FALSE(bool(db.hasRegionResource(region->getID(), Resource::style("http://example.com/1"))));
EXPECT_FALSE(bool(db.hasRegionResource(region->getID(), Resource::style("http://example.com/20"))));
EXPECT_FALSE(bool(db.hasRegionResource(Resource::style("http://example.com/1"))));
EXPECT_FALSE(bool(db.hasRegionResource(Resource::style("http://example.com/20"))));
Response response;
response.data = randomString(1024);
@@ -1103,9 +1103,9 @@ TEST(OfflineDatabase, HasRegionResource) {
db.putRegionResource(region->getID(), Resource::style("http://example.com/"s + util::toString(i)), response);
}
EXPECT_TRUE(bool(db.hasRegionResource(region->getID(), Resource::style("http://example.com/1"))));
EXPECT_TRUE(bool(db.hasRegionResource(region->getID(), Resource::style("http://example.com/20"))));
EXPECT_EQ(1024, *(db.hasRegionResource(region->getID(), Resource::style("http://example.com/20"))));
EXPECT_TRUE(bool(db.hasRegionResource(Resource::style("http://example.com/1"))));
EXPECT_TRUE(bool(db.hasRegionResource(Resource::style("http://example.com/20"))));
EXPECT_EQ(1024, *(db.hasRegionResource(Resource::style("http://example.com/20"))));
EXPECT_EQ(0u, log.uncheckedCount());
}
@@ -1131,16 +1131,16 @@ TEST(OfflineDatabase, HasRegionResourceTile) {
response.data = std::make_shared<std::string>("first");
EXPECT_FALSE(bool(db.hasRegionResource(region->getID(), resource)));
EXPECT_FALSE(bool(db.hasRegionResource(resource)));
db.putRegionResource(region->getID(), resource, response);
EXPECT_TRUE(bool(db.hasRegionResource(region->getID(), resource)));
EXPECT_EQ(5, *(db.hasRegionResource(region->getID(), resource)));
EXPECT_TRUE(bool(db.hasRegionResource(resource)));
EXPECT_EQ(5, *(db.hasRegionResource(resource)));
auto anotherRegion = db.createRegion(definition, OfflineRegionMetadata());
ASSERT_TRUE(anotherRegion);
EXPECT_LT(region->getID(), anotherRegion->getID());
EXPECT_TRUE(bool(db.hasRegionResource(anotherRegion->getID(), resource)));
EXPECT_EQ(5, *(db.hasRegionResource(anotherRegion->getID(), resource)));
EXPECT_TRUE(bool(db.hasRegionResource(resource)));
EXPECT_EQ(5, *(db.hasRegionResource(resource)));
EXPECT_EQ(0u, log.uncheckedCount());
@@ -1488,12 +1488,12 @@ TEST(OfflineDatabase, TEST_REQUIRES_WRITE(DisallowedIO)) {
EXPECT_EQ(1u, log.count(warning(ResultCode::Auth, "Can't update region metadata: authorization denied")));
EXPECT_EQ(0u, log.uncheckedCount());
EXPECT_EQ(nullopt, db.getRegionResource(region->getID(), fixture::resource));
EXPECT_EQ(nullopt, db.getRegionResource(fixture::resource));
EXPECT_EQ(1u, log.count(warning(ResultCode::Auth, "Can't update timestamp: authorization denied")));
EXPECT_EQ(1u, log.count(warning(ResultCode::Auth, "Can't read region resource: authorization denied")));
EXPECT_EQ(0u, log.uncheckedCount());
EXPECT_EQ(nullopt, db.hasRegionResource(region->getID(), fixture::resource));
EXPECT_EQ(nullopt, db.hasRegionResource(fixture::resource));
EXPECT_EQ(1u, log.count(warning(ResultCode::Auth, "Can't query region resource: authorization denied")));
EXPECT_EQ(0u, log.uncheckedCount());
@@ -1566,7 +1566,7 @@ TEST(OfflineDatabase, TEST_REQUIRES_WRITE(MergeDatabaseWithSingleRegion_Update))
EXPECT_EQ(1u, status->completedTileCount);
//Verify the modified timestamp matches the tile in the sideloaded db.
auto updatedTile = db.getRegionResource(regionId,
auto updatedTile = db.getRegionResource(
Resource::tile("mapbox://tiles/mapbox.satellite/{z}/{x}/{y}{ratio}.webp",
1, 0, 0, 1, Tileset::Scheme::XYZ));
EXPECT_EQ(Timestamp{ Seconds(1520409600) }, *(updatedTile->first.modified));
@@ -1586,8 +1586,7 @@ TEST(OfflineDatabase, MergeDatabaseWithSingleRegion_NoUpdate) {
EXPECT_EQ(1u, result->size());
EXPECT_EQ(1u, db.listRegions()->size());
auto regionId = result->front().getID();
auto updatedTile = db.getRegionResource(regionId,
auto updatedTile = db.getRegionResource(
Resource::tile("mapbox://tiles/mapbox.satellite/{z}/{x}/{y}{ratio}.webp",
1, 0, 0, 1, Tileset::Scheme::XYZ));
@@ -1601,14 +1600,13 @@ TEST(OfflineDatabase, MergeDatabaseWithSingleRegion_AmbientTiles) {
OfflineDatabase db(":memory:");
auto result = db.mergeDatabase(filename_sideload);
auto regionId = result->front().getID();
EXPECT_TRUE(bool(db.hasRegionResource(regionId, Resource::tile("mapbox://tiles/mapbox.satellite/{z}/{x}/{y}{ratio}.png", 1, 0, 0, 1, Tileset::Scheme::XYZ))));
EXPECT_TRUE(bool(db.hasRegionResource(Resource::tile("mapbox://tiles/mapbox.satellite/{z}/{x}/{y}{ratio}.png", 1, 0, 0, 1, Tileset::Scheme::XYZ))));
//Ambient resources should not be copied
EXPECT_FALSE(bool(db.hasRegionResource(regionId, Resource::style("mapbox://styles/mapbox/streets-v9"))));
EXPECT_FALSE(bool(db.hasRegionResource(regionId, Resource::tile("mapbox://tiles/mapbox.satellite/{z}/{x}/{y}{ratio}.png", 1, 0, 1, 2, Tileset::Scheme::XYZ))));
EXPECT_FALSE(bool(db.hasRegionResource(regionId, Resource::tile("mapbox://tiles/mapbox.satellite/{z}/{x}/{y}{ratio}.png", 1, 1, 1, 2, Tileset::Scheme::XYZ))));
EXPECT_FALSE(bool(db.hasRegionResource(Resource::style("mapbox://styles/mapbox/streets-v9"))));
EXPECT_FALSE(bool(db.hasRegionResource(Resource::tile("mapbox://tiles/mapbox.satellite/{z}/{x}/{y}{ratio}.png", 1, 0, 1, 2, Tileset::Scheme::XYZ))));
EXPECT_FALSE(bool(db.hasRegionResource(Resource::tile("mapbox://tiles/mapbox.satellite/{z}/{x}/{y}{ratio}.png", 1, 1, 1, 2, Tileset::Scheme::XYZ))));
}
TEST(OfflineDatabase, MergeDatabaseWithMultipleRegions_New) {