tei/MapLibreParticleLab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[core] Limit total number of offline Mapbox tiles

Browse Source
This commit is contained in:
John Firebaugh
2016-02-23 18:25:36 -08:00
parent 5cefaeb857
commit 065cda83fa
11 changed files with 261 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
include/mbgl/storage/default_file_source.hpp
View File

@@ -94,6 +94,12 @@ public:
*/
void deleteOfflineRegion(OfflineRegion&&, std::function<void (std::exception_ptr)>);
/*
* Changing or bypassing this limit without permission from Mapbox is prohibited
* by the Mapbox Terms of Service.
*/
void setOfflineMapboxTileCountLimit(uint64_t) const;
// For testing only.
void put(const Resource&, const Response&);
void goOffline();

16
include/mbgl/storage/offline.hpp
View File

@@ -150,6 +150,22 @@ public:
* re-executes the user-provided implementation on the main thread.
*/
virtual void responseError(Response::Error) {}
/*
* Implement this method to be notified when the limit on the number of Mapbox
* tiles stored for offline regions has been reached.
*
* Once the limit has been reached, the SDK will not download further offline
* tiles from Mapbox APIs until existing tiles have been removed. Contact your
* Mapbox sales representative to raise the limit.
*
* This limit does not apply to non-Mapbox tile sources.
*
* Note that this method will be executed on the database thread; it is the
* responsibility of the SDK bindings to wrap this object in an interface that
* re-executes the user-provided implementation on the main thread.
*/
virtual void mapboxTileCountLimitExceeded(uint64_t /* limit */) {}
};
class OfflineRegion {

8
platform/default/default_file_source.cpp
View File

@@ -113,6 +113,10 @@ public:
tasks.erase(req);
}
void setOfflineMapboxTileCountLimit(uint64_t limit) {
offlineDatabase.setOfflineMapboxTileCountLimit(limit);
}
void put(const Resource& resource, const Response& response) {
offlineDatabase.put(resource, response);
}
@@ -205,6 +209,10 @@ void DefaultFileSource::getOfflineRegionStatus(OfflineRegion& region, std::funct
thread->invoke(&Impl::getRegionStatus, region.getID(), callback);
}
void DefaultFileSource::setOfflineMapboxTileCountLimit(uint64_t limit) const {
thread->invokeSync(&Impl::setOfflineMapboxTileCountLimit, limit);
}
// For testing only:
void DefaultFileSource::put(const Resource& resource, const Response& response) {

114
platform/default/mbgl/storage/offline_database.cpp
View File

@@ -447,6 +447,9 @@ void OfflineDatabase::deleteRegion(OfflineRegion&& region) {
stmt->run();
evict(0);
// Ensure that the cached offlineTileCount value is recalculated.
offlineMapboxTileCount = {};
}
optional<Response> OfflineDatabase::getRegionResource(int64_t regionID, const Resource& resource) {
@@ -461,13 +464,21 @@ optional<Response> OfflineDatabase::getRegionResource(int64_t regionID, const Re
uint64_t OfflineDatabase::putRegionResource(int64_t regionID, const Resource& resource, const Response& response) {
uint64_t size = putInternal(resource, response, false).second;
markUsed(regionID, resource);
bool previouslyUnused = markUsed(regionID, resource);
if (offlineMapboxTileCount
&& resource.kind == Resource::Kind::Tile
&& util::mapbox::isMapboxURL(resource.url)
&& previouslyUnused) {
*offlineMapboxTileCount += 1;
}
return size;
}
void OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) {
bool OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) {
if (resource.kind == Resource::Kind::Tile) {
Statement stmt = getStatement(
Statement insert = getStatement(
"INSERT OR IGNORE INTO region_tiles (region_id, tile_id) "
"SELECT ?1, tiles.id "
"FROM tiles "
@@ -478,23 +489,61 @@ void OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) {
" AND z = ?6 ");
const Resource::TileData& tile = *resource.tileData;
stmt->bind(1, regionID);
stmt->bind(2, tile.urlTemplate);
stmt->bind(3, tile.pixelRatio);
stmt->bind(4, tile.x);
stmt->bind(5, tile.y);
stmt->bind(6, tile.z);
stmt->run();
insert->bind(1, regionID);
insert->bind(2, tile.urlTemplate);
insert->bind(3, tile.pixelRatio);
insert->bind(4, tile.x);
insert->bind(5, tile.y);
insert->bind(6, tile.z);
insert->run();
if (db->changes() == 0) {
return false;
}
Statement select = getStatement(
"SELECT region_id "
"FROM region_tiles, tiles "
"WHERE region_id != ?1 "
" AND url_template = ?2 "
" AND pixel_ratio = ?3 "
" AND x = ?4 "
" AND y = ?5 "
" AND z = ?6 "
"LIMIT 1 ");
select->bind(1, regionID);
select->bind(2, tile.urlTemplate);
select->bind(3, tile.pixelRatio);
select->bind(4, tile.x);
select->bind(5, tile.y);
select->bind(6, tile.z);
return !select->run();
} else {
Statement stmt = getStatement(
Statement insert = getStatement(
"INSERT OR IGNORE INTO region_resources (region_id, resource_id) "
"SELECT ?1, resources.id "
"FROM resources "
"WHERE resources.url = ?2 ");
stmt->bind(1, regionID);
stmt->bind(2, resource.url);
stmt->run();
insert->bind(1, regionID);
insert->bind(2, resource.url);
insert->run();
if (db->changes() == 0) {
return false;
}
Statement select = getStatement(
"SELECT region_id "
"FROM region_resources, resources "
"WHERE region_id != ?1 "
" AND resources.url = ?2 "
"LIMIT 1 ");
select->bind(1, regionID);
select->bind(2, resource.url);
return !select->run();
}
}
@@ -591,6 +640,9 @@ bool OfflineDatabase::evict(uint64_t neededFreeSize) {
stmt2->run();
uint64_t changes2 = db->changes();
// The cached value of offlineTileCount does not need to be updated
// here because only non-offline tiles can be removed by eviction.
if (changes1 == 0 && changes2 == 0) {
return false;
}
@@ -599,4 +651,38 @@ bool OfflineDatabase::evict(uint64_t neededFreeSize) {
return true;
}
void OfflineDatabase::setOfflineMapboxTileCountLimit(uint64_t limit) {
offlineMapboxTileCountLimit = limit;
}
uint64_t OfflineDatabase::getOfflineMapboxTileCountLimit() {
return offlineMapboxTileCountLimit;
}
bool OfflineDatabase::offlineMapboxTileCountLimitExceeded() {
return getOfflineMapboxTileCount() >= offlineMapboxTileCountLimit;
}
uint64_t OfflineDatabase::getOfflineMapboxTileCount() {
// Calculating this on every call would be much simpler than caching and
// manually updating the value, but it would make offline downloads an O(n²)
// operation, because the database query below involves an index scan of
// region_tiles.
if (offlineMapboxTileCount) {
return *offlineMapboxTileCount;
}
Statement stmt = getStatement(
"SELECT COUNT(DISTINCT id) "
"FROM region_tiles, tiles "
"WHERE tile_id = tiles.id "
"AND url_template LIKE 'mapbox://%' ");
stmt->run();
offlineMapboxTileCount = stmt->get<int64_t>(0);
return *offlineMapboxTileCount;
}
} // namespace mbgl

13
platform/default/mbgl/storage/offline_database.hpp
View File

@@ -6,6 +6,7 @@
#include <mbgl/util/noncopyable.hpp>
#include <mbgl/util/optional.hpp>
#include <mbgl/util/constants.hpp>
#include <mbgl/util/mapbox.hpp>
#include <unordered_map>
#include <memory>
@@ -49,6 +50,11 @@ public:
OfflineRegionDefinition getRegionDefinition(int64_t regionID);
OfflineRegionStatus getRegionCompletedStatus(int64_t regionID);
void setOfflineMapboxTileCountLimit(uint64_t);
uint64_t getOfflineMapboxTileCountLimit();
bool offlineMapboxTileCountLimitExceeded();
uint64_t getOfflineMapboxTileCount();
private:
void connect(int flags);
void ensureSchema();
@@ -78,7 +84,9 @@ private:
const std::string&, bool compressed);
std::pair<bool, uint64_t> putInternal(const Resource&, const Response&, bool evict);
void markUsed(int64_t regionID, const Resource&);
// Return value is true iff the resource was previously unused by any other regions.
bool markUsed(int64_t regionID, const Resource&);
const std::string path;
std::unique_ptr<::mapbox::sqlite::Database> db;
@@ -89,6 +97,9 @@ private:
uint64_t maximumCacheSize;
uint64_t offlineMapboxTileCountLimit = util::mapbox::DEFAULT_OFFLINE_TILE_COUNT_LIMIT;
optional<uint64_t> offlineMapboxTileCount;
bool evict(uint64_t neededFreeSize);
};

8
platform/default/mbgl/storage/offline_download.cpp
View File

@@ -7,6 +7,7 @@
#include <mbgl/layer/symbol_layer.hpp>
#include <mbgl/text/glyph.hpp>
#include <mbgl/util/tile_cover.hpp>
#include <mbgl/util/mapbox.hpp>
#include <set>
@@ -218,6 +219,13 @@ void OfflineDownload::ensureResource(const Resource& resource, std::function<voi
return;
}
if (resource.kind == Resource::Kind::Tile
&& util::mapbox::isMapboxURL(resource.url)
&& offlineDatabase.offlineMapboxTileCountLimitExceeded()) {
observer->mapboxTileCountLimitExceeded(offlineDatabase.getOfflineMapboxTileCountLimit());
return;
}
auto it = requests.insert(requests.begin(), nullptr);
*it = onlineFileSource.request(resource, [=] (Response onlineResponse) {
if (onlineResponse.error) {

2
src/mbgl/util/mapbox.cpp
View File

@@ -169,6 +169,8 @@ std::string canonicalizeTileURL(const std::string& url, SourceType type, uint16_
return result;
}
const uint64_t DEFAULT_OFFLINE_TILE_COUNT_LIMIT = std::numeric_limits<uint64_t>::max();
} // end namespace mapbox
} // end namespace util
} // end namespace mbgl

2
src/mbgl/util/mapbox.hpp
View File

@@ -19,6 +19,8 @@ std::string normalizeTileURL(const std::string& url, const std::string& accessTo
// Return a "mapbox://tiles/..." URL (suitable for normalizeTileURL) for the given Mapbox tile URL.
std::string canonicalizeTileURL(const std::string& url, SourceType, uint16_t tileSize);
extern const uint64_t DEFAULT_OFFLINE_TILE_COUNT_LIMIT;
} // namespace mapbox
} // namespace util
} // namespace mbgl

17
test/fixtures/offline/mapbox_source.style.json vendored Normal file
View File

@@ -0,0 +1,17 @@
{
"version": 8,
"sources": {
"inline": {
"type": "vector",
"maxzoom": 15,
"minzoom": 0,
"tiles": [ "mapbox://{z}-{x}-{y}.vector.pbf" ]
}
},
"layers": [{
"id": "fill",
"type": "fill",
"source": "inline",
"source-layer": "water"
}]
}

57
test/storage/offline_database.cpp
View File

@@ -603,3 +603,60 @@ TEST(OfflineDatabase, PutFailsWhenEvictionInsuffices) {
auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
EXPECT_EQ(1ul, flo->count({ EventSeverity::Warning, Event::Database, -1, "Unable to make space for entry" }));
}
TEST(OfflineDatabase, OfflineMapboxTileCount) {
using namespace mbgl;
OfflineDatabase db(":memory:");
OfflineRegionDefinition definition { "http://example.com/style", LatLngBounds::hull({1, 2}, {3, 4}), 5, 6, 2.0 };
OfflineRegionMetadata metadata;
OfflineRegion region1 = db.createRegion(definition, metadata);
OfflineRegion region2 = db.createRegion(definition, metadata);
Resource nonMapboxTile = Resource::tile("http://example.com/", 1.0, 0, 0, 0);
Resource mapboxTile1 = Resource::tile("mapbox://tiles/1", 1.0, 0, 0, 0);
Resource mapboxTile2 = Resource::tile("mapbox://tiles/2", 1.0, 0, 0, 1);
Response response;
response.data = std::make_shared<std::string>("data");
// Count is initially zero.
EXPECT_EQ(0, db.getOfflineMapboxTileCount());
// Count stays the same after putting a non-tile resource.
db.putRegionResource(region1.getID(), Resource::style("http://example.com/"), response);
EXPECT_EQ(0, db.getOfflineMapboxTileCount());
// Count stays the same after putting a non-Mapbox tile.
db.putRegionResource(region1.getID(), nonMapboxTile, response);
EXPECT_EQ(0, db.getOfflineMapboxTileCount());
// Count increases after putting a Mapbox tile not used by another region.
db.putRegionResource(region1.getID(), mapboxTile1, response);
EXPECT_EQ(1, db.getOfflineMapboxTileCount());
// Count stays the same after putting a Mapbox tile used by another region.
db.putRegionResource(region2.getID(), mapboxTile1, response);
EXPECT_EQ(1, db.getOfflineMapboxTileCount());
// Count stays the same after putting a Mapbox tile used by the same region.
db.putRegionResource(region2.getID(), mapboxTile1, response);
EXPECT_EQ(1, db.getOfflineMapboxTileCount());
// Count stays the same after deleting a region when the tile is still used by another region.
db.deleteRegion(std::move(region2));
EXPECT_EQ(1, db.getOfflineMapboxTileCount());
// Count stays the same after the putting a non-offline Mapbox tile.
db.put(mapboxTile2, response);
EXPECT_EQ(1, db.getOfflineMapboxTileCount());
// Count increases after putting a pre-existing, but non-offline Mapbox tile.
db.putRegionResource(region1.getID(), mapboxTile2, response);
EXPECT_EQ(2, db.getOfflineMapboxTileCount());
// Count decreases after deleting a region when the tiles are not used by other regions.
db.deleteRegion(std::move(region1));
EXPECT_EQ(0, db.getOfflineMapboxTileCount());
}

33
test/storage/offline_download.cpp
View File

@@ -24,8 +24,13 @@ public:
if (responseErrorFn) responseErrorFn(error);
}
void mapboxTileCountLimitExceeded(uint64_t limit) override {
if (mapboxTileCountLimitExceededFn) mapboxTileCountLimitExceededFn(limit);
}
std::function<void (OfflineRegionStatus)> statusChangedFn;
std::function<void (Response::Error)> responseErrorFn;
std::function<void (uint64_t)> mapboxTileCountLimitExceededFn;
};
class OfflineTest {
@@ -347,3 +352,31 @@ TEST(OfflineDownload, RequestErrorsAreRetried) {
test.loop.run();
}
TEST(OfflineDownload, TileCountLimitExceeded) {
OfflineTest test;
OfflineRegion region = test.createRegion();
OfflineDownload download(
region.getID(),
OfflineTilePyramidRegionDefinition("http://127.0.0.1:3000/offline/style.json", LatLngBounds::world(), 0.0, 0.0, 1.0),
test.db, test.fileSource);
test.db.setOfflineMapboxTileCountLimit(0);
test.fileSource.styleResponse = [&] (const Resource& resource) {
EXPECT_EQ("http://127.0.0.1:3000/offline/style.json", resource.url);
return test.response("offline/mapbox_source.style.json");
};
auto observer = std::make_unique<MockObserver>();
observer->mapboxTileCountLimitExceededFn = [&] (uint64_t limit) {
EXPECT_EQ(0, limit);
test.loop.stop();
};
download.setObserver(std::move(observer));
download.setState(OfflineRegionDownloadState::Active);
test.loop.run();
}