[core] Implement an eviction policy for OfflineDatabase

When inserting an cached resource, or removing a region, remove least-recently used resources and tiles, not used by offline regions, until the used database size, as calculated by multiplying the number of in-use pages by the page size, is less than the maximum cache size minus 5 times the page size.

In addition, OfflineDatabase may be configured to ignore cache puts of individual resources larger than a certain size.

This policy is similar but not identical to the former SQLiteCache policy:

* It accounts for offline, by exempting resources required by offline regions from eviction.
* It must delete from two tables (resources and tiles), rather than one. Currently the strategy is naive: evict 50 rows at a time from each table.
* It makes maximumCacheSize and maximumCacheEntrySize completely independent. The SQLiteCache implementation evicted when `usedSize > maximumCacheSize - 2 * maximumCacheEntrySize`. This evicts when `usedSize > maximumCacheSize - 5 * pageSize`.
* It uses a non-unlimited default value for maximumCacheSize: 50 MB. We should have always had a limit in place; "a cache without an eviction policy is a resource leak".
This commit is contained in:
John Firebaugh
2016-02-09 17:19:27 -08:00
parent 5910124010
commit db3620c58f
10 changed files with 273 additions and 102 deletions

View File

@@ -3,6 +3,7 @@
#include <mbgl/storage/file_source.hpp>
#include <mbgl/storage/offline.hpp>
#include <mbgl/util/constants.hpp>
#include <vector>
@@ -14,15 +15,15 @@ template <typename T> class Thread;
class DefaultFileSource : public FileSource {
public:
DefaultFileSource(const std::string& cachePath, const std::string& assetRoot);
DefaultFileSource(const std::string& cachePath,
const std::string& assetRoot,
uint64_t maximumCacheSize = util::DEFAULT_MAX_CACHE_SIZE,
uint64_t maximumCacheEntrySize = util::DEFAULT_MAX_CACHE_ENTRY_SIZE);
~DefaultFileSource() override;
void setAccessToken(const std::string&);
std::string getAccessToken() const;
void setMaximumCacheSize(uint64_t size);
void setMaximumCacheEntrySize(uint64_t size);
std::unique_ptr<FileRequest> request(const Resource&, Callback) override;
/*
@@ -71,13 +72,11 @@ public:
optional<OfflineRegionStatus>)>) const;
/*
* Initiate the removal of offline region from the database.
* Remove an offline region from the database and perform any resources evictions
* necessary as a result.
*
* All resources required by the region, but not also required by other regions, will
* become eligible for removal for space-optimization. Because the offline database is
* also used for ambient usage-based caching, and offline resources may still be useful
* for ambient usage, they are not immediately removed. To immediately remove resources
* not used by any extant region, call removeUnusedOfflineResources().
* Eviction works by removing the least-recently requested resources not also required
* by other regions, until the database shrinks below a certain size.
*
* Note that this method takes ownership of the input, reflecting the fact that once
* region deletion is initiated, it is not legal to perform further actions with the
@@ -89,16 +88,6 @@ public:
*/
void deleteOfflineRegion(OfflineRegion&&, std::function<void (std::exception_ptr)>);
/*
* Remove all resources in the database that are not required by any region, thus
* optimizing the disk space used by the offline database.
*
* When the operation is complete or encounters an error, the given callback will be
* executed on the database thread; it is the responsibility of the SDK bindings
* to re-execute a user-provided callback on the main thread.
*/
void removeUnusedOfflineResources(std::function<void (std::exception_ptr)>);
// For testing only.
void put(const Resource&, const Response&);
void goOffline();

View File

@@ -21,6 +21,9 @@ extern const double PITCH_MAX;
extern const double MIN_ZOOM;
extern const double MAX_ZOOM;
extern const uint64_t DEFAULT_MAX_CACHE_SIZE;
extern const uint64_t DEFAULT_MAX_CACHE_ENTRY_SIZE;
} // namespace util
namespace debug {

View File

@@ -50,8 +50,8 @@ public:
std::unique_ptr<FileRequest> onlineRequest;
};
Impl(const std::string& cachePath)
: offlineDatabase(cachePath) {
Impl(const std::string& cachePath, uint64_t maximumCacheSize, uint64_t maximumCacheEntrySize)
: offlineDatabase(cachePath, maximumCacheSize, maximumCacheEntrySize) {
}
void setAccessToken(const std::string& accessToken) {
@@ -97,15 +97,6 @@ public:
}
}
void removeUnusedOfflineResources(std::function<void (std::exception_ptr)> callback) {
try {
offlineDatabase.removeUnusedResources();
callback({});
} catch (...) {
callback(std::current_exception());
}
}
void setRegionObserver(int64_t regionID, std::unique_ptr<OfflineRegionObserver> observer) {
getDownload(regionID).setObserver(std::move(observer));
}
@@ -147,8 +138,12 @@ private:
bool offline = false;
};
DefaultFileSource::DefaultFileSource(const std::string& cachePath, const std::string& assetRoot)
: thread(std::make_unique<util::Thread<DefaultFileSource::Impl>>(util::ThreadContext{"DefaultFileSource", util::ThreadType::Unknown, util::ThreadPriority::Low}, cachePath)),
DefaultFileSource::DefaultFileSource(const std::string& cachePath,
const std::string& assetRoot,
uint64_t maximumCacheSize,
uint64_t maximumCacheEntrySize)
: thread(std::make_unique<util::Thread<Impl>>(util::ThreadContext{"DefaultFileSource", util::ThreadType::Unknown, util::ThreadPriority::Low},
cachePath, maximumCacheSize, maximumCacheEntrySize)),
assetFileSource(std::make_unique<AssetFileSource>(assetRoot)) {
}
@@ -162,14 +157,6 @@ std::string DefaultFileSource::getAccessToken() const {
return thread->invokeSync<std::string>(&Impl::getAccessToken);
}
void DefaultFileSource::setMaximumCacheSize(uint64_t) {
// TODO
}
void DefaultFileSource::setMaximumCacheEntrySize(uint64_t) {
// TODO
}
std::unique_ptr<FileRequest> DefaultFileSource::request(const Resource& resource, Callback callback) {
class DefaultFileRequest : public FileRequest {
public:
@@ -219,10 +206,6 @@ void DefaultFileSource::getOfflineRegionStatus(OfflineRegion& region, std::funct
thread->invoke(&Impl::getRegionStatus, region.getID(), callback);
}
void DefaultFileSource::removeUnusedOfflineResources(std::function<void (std::exception_ptr)> callback) {
thread->invoke(&Impl::removeUnusedOfflineResources, callback);
}
// For testing only:
void DefaultFileSource::put(const Resource& resource, const Response& response) {

View File

@@ -22,8 +22,10 @@ OfflineDatabase::Statement::~Statement() {
stmt.clearBindings();
}
OfflineDatabase::OfflineDatabase(const std::string& path_)
: path(path_) {
OfflineDatabase::OfflineDatabase(const std::string& path_, uint64_t maximumCacheSize_, uint64_t maximumCacheEntrySize_)
: path(path_),
maximumCacheSize(maximumCacheSize_),
maximumCacheEntrySize(maximumCacheEntrySize_) {
ensureSchema();
}
@@ -110,6 +112,18 @@ optional<Response> OfflineDatabase::get(const Resource& resource) {
}
uint64_t OfflineDatabase::put(const Resource& resource, const Response& response) {
if (response.data && response.data->size() > maximumCacheEntrySize) {
Log::Warning(Event::Database, "Entry too big for caching");
return 0;
} else if (!evict()) {
Log::Warning(Event::Database, "Unable to make space for new entries");
return 0;
} else {
return putInternal(resource, response);
}
}
uint64_t OfflineDatabase::putInternal(const Resource& resource, const Response& response) {
// Don't store errors in the cache.
if (response.error) {
return 0;
@@ -124,6 +138,13 @@ uint64_t OfflineDatabase::put(const Resource& resource, const Response& response
}
optional<Response> OfflineDatabase::getResource(const Resource& resource) {
Statement accessedStmt = getStatement(
"UPDATE resources SET accessed = ?1 WHERE url = ?2");
accessedStmt->bind(1, SystemClock::now());
accessedStmt->bind(2, resource.url);
accessedStmt->run();
Statement stmt = getStatement(
// 0 1 2 3 4
"SELECT etag, expires, modified, data, compressed "
@@ -203,6 +224,24 @@ uint64_t OfflineDatabase::putResource(const Resource& resource, const Response&
}
optional<Response> OfflineDatabase::getTile(const Resource::TileData& tile) {
Statement accessedStmt = getStatement(
"UPDATE tiles SET accessed = ?1 "
"WHERE tileset_id = ( "
" SELECT id FROM tilesets "
" WHERE url_template = ?2 "
" AND pixel_ratio = ?3) "
"AND tiles.x = ?4 "
"AND tiles.y = ?5 "
"AND tiles.z = ?6 ");
accessedStmt->bind(1, SystemClock::now());
accessedStmt->bind(2, tile.urlTemplate);
accessedStmt->bind(3, tile.pixelRatio);
accessedStmt->bind(4, tile.x);
accessedStmt->bind(5, tile.y);
accessedStmt->bind(6, tile.z);
accessedStmt->run();
Statement stmt = getStatement(
// 0 1 2 3 4
"SELECT etag, expires, modified, data, compressed "
@@ -348,6 +387,8 @@ void OfflineDatabase::deleteRegion(OfflineRegion&& region) {
stmt->bind(1, region.getID());
stmt->run();
evict();
}
optional<Response> OfflineDatabase::getRegionResource(int64_t regionID, const Resource& resource) {
@@ -361,7 +402,7 @@ optional<Response> OfflineDatabase::getRegionResource(int64_t regionID, const Re
}
uint64_t OfflineDatabase::putRegionResource(int64_t regionID, const Resource& resource, const Response& response) {
uint64_t result = put(resource, response);
uint64_t result = putInternal(resource, response);
markUsed(regionID, resource);
return result;
}
@@ -431,27 +472,76 @@ OfflineRegionStatus OfflineDatabase::getRegionCompletedStatus(int64_t regionID)
return result;
}
void OfflineDatabase::removeUnusedResources() {
Statement stmt1 = getStatement(
"DELETE FROM resources "
"WHERE ROWID NOT IN ( "
" SELECT resources.ROWID "
" FROM resources, region_resources "
" WHERE resources.url = region_resources.resource_url "
") ");
stmt1->run();
template <class T>
T OfflineDatabase::getPragma(const char * sql) {
Statement stmt = getStatement(sql);
stmt->run();
return stmt->get<T>(0);
}
Statement stmt2 = getStatement(
"DELETE FROM tiles "
"WHERE ROWID NOT IN ( "
" SELECT tiles.ROWID "
" FROM tiles, region_tiles "
" AND tiles.tileset_id = region_tiles.tileset_id "
" AND tiles.z = region_tiles.z "
" AND tiles.x = region_tiles.x "
" AND tiles.y = region_tiles.y "
") ");
stmt2->run();
// Remove least-recently used resources and tiles until the used database size,
// as calculated by multiplying the number of in-use pages by the page size, is
// less than the maximum cache size. Returns false if this condition cannot be
// satisfied.
//
// SQLite database never shrinks in size unless we call VACCUM. We here
// are monitoring the soft limit (i.e. number of free pages in the file)
// and as it approaches to the hard limit (i.e. the actual file size) we
// delete an arbitrary number of old cache entries.
//
// The free pages approach saves us from calling VACCUM or keeping a
// running total, which can be costly. We need a buffer because pages can
// get fragmented on the database.
bool OfflineDatabase::evict() {
uint64_t pageSize = getPragma<int64_t>("PRAGMA page_size");
uint64_t pageCount = getPragma<int64_t>("PRAGMA page_count");
if (pageSize * pageCount > maximumCacheSize) {
Log::Warning(mbgl::Event::Database, "Current size is larger than the maximum size. Database won't get truncated.");
}
auto usedSize = [&] {
return pageSize * (pageCount - getPragma<int64_t>("PRAGMA freelist_count"));
};
while (usedSize() > maximumCacheSize - 5 * pageSize) {
Statement stmt1 = getStatement(
"DELETE FROM resources "
"WHERE ROWID IN ( "
" SELECT resources.ROWID "
" FROM resources "
" LEFT JOIN region_resources "
" ON resources.url = region_resources.resource_url "
" WHERE region_resources.resource_url IS NULL "
" ORDER BY accessed ASC LIMIT ?1 "
") ");
stmt1->bind(1, 50);
stmt1->run();
uint64_t changes1 = db->changes();
Statement stmt2 = getStatement(
"DELETE FROM tiles "
"WHERE ROWID IN ( "
" SELECT tiles.ROWID "
" FROM tiles "
" LEFT JOIN region_tiles "
" ON tiles.tileset_id = region_tiles.tileset_id "
" AND tiles.z = region_tiles.z "
" AND tiles.x = region_tiles.x "
" AND tiles.y = region_tiles.y "
" WHERE region_tiles.tileset_id IS NULL "
" ORDER BY accessed ASC LIMIT ?1 "
") ");
stmt2->bind(1, 50);
stmt2->run();
uint64_t changes2 = db->changes();
if (changes1 == 0 && changes2 == 0) {
return false;
}
}
return true;
}
} // namespace mbgl

View File

@@ -5,6 +5,7 @@
#include <mbgl/storage/offline.hpp>
#include <mbgl/util/noncopyable.hpp>
#include <mbgl/util/optional.hpp>
#include <mbgl/util/constants.hpp>
#include <unordered_map>
#include <memory>
@@ -24,7 +25,11 @@ class TileID;
class OfflineDatabase : private util::noncopyable {
public:
OfflineDatabase(const std::string& path);
// Limits affect ambient caching (put) only; resources required by offline
// regions are exempt.
OfflineDatabase(const std::string& path,
uint64_t maximumCacheSize = util::DEFAULT_MAX_CACHE_SIZE,
uint64_t maximumCacheEntrySize = util::DEFAULT_MAX_CACHE_ENTRY_SIZE);
~OfflineDatabase();
optional<Response> get(const Resource&);
@@ -36,7 +41,6 @@ public:
const OfflineRegionMetadata&);
void deleteRegion(OfflineRegion&&);
void removeUnusedResources();
optional<Response> getRegionResource(int64_t regionID, const Resource&);
uint64_t putRegionResource(int64_t regionID, const Resource&, const Response&);
@@ -62,6 +66,7 @@ private:
};
Statement getStatement(const char *);
uint64_t putInternal(const Resource&, const Response&);
optional<Response> getTile(const Resource::TileData&);
uint64_t putTile(const Resource::TileData&, const Response&);
@@ -74,6 +79,14 @@ private:
const std::string path;
std::unique_ptr<::mapbox::sqlite::Database> db;
std::unordered_map<const char *, std::unique_ptr<::mapbox::sqlite::Statement>> statements;
template <class T>
T getPragma(const char *);
uint64_t maximumCacheSize;
uint64_t maximumCacheEntrySize;
bool evict();
};
} // namespace mbgl

View File

@@ -91,6 +91,11 @@ int64_t Database::lastInsertRowid() const {
return sqlite3_last_insert_rowid(db);
}
uint64_t Database::changes() const {
assert(db);
return sqlite3_changes(db);
}
Statement::Statement(sqlite3 *db, const char *sql) {
const int err = sqlite3_prepare_v2(db, sql, -1, &stmt, nullptr);
if (err != SQLITE_OK) {

View File

@@ -47,6 +47,7 @@ public:
Statement prepare(const char *query);
int64_t lastInsertRowid() const;
uint64_t changes() const;
private:
sqlite3 *db = nullptr;

View File

@@ -106,8 +106,6 @@ int main(int argc, char *argv[]) {
view = std::make_unique<GLFWView>(fullscreen, benchmark);
mbgl::DefaultFileSource fileSource("/tmp/mbgl-cache.db", ".");
fileSource.setMaximumCacheEntrySize(1 * 1024 * 1024); // 1 MB
fileSource.setMaximumCacheSize(50 * 1024 * 1024); // 50 MB
// Set access token if present
const char *token = getenv("MAPBOX_ACCESS_TOKEN");

View File

@@ -1,6 +1,12 @@
#include <mbgl/util/constants.hpp>
const float mbgl::util::tileSize = 512.0f;
#include <limits>
namespace mbgl {
namespace util {
const float tileSize = 512.0f;
/*
* The maximum extent of a feature that can be safely stored in the buffer.
@@ -12,37 +18,48 @@ const float mbgl::util::tileSize = 512.0f;
* One bit is lost to support features extending past the extent on the right edge of the tile.
* This leaves us with 2^13 = 8192
*/
const int32_t mbgl::util::EXTENT = 8192;
const int32_t EXTENT = 8192;
const double mbgl::util::DEG2RAD = M_PI / 180.0;
const double mbgl::util::RAD2DEG = 180.0 / M_PI;
const double mbgl::util::M2PI = 2 * M_PI;
const double mbgl::util::EARTH_RADIUS_M = 6378137;
const double mbgl::util::LATITUDE_MAX = 85.05112878;
const double mbgl::util::PITCH_MAX = M_PI / 3;
const double mbgl::util::MIN_ZOOM = 0.0;
const double mbgl::util::MAX_ZOOM = 25.5;
const double DEG2RAD = M_PI / 180.0;
const double RAD2DEG = 180.0 / M_PI;
const double M2PI = 2 * M_PI;
const double EARTH_RADIUS_M = 6378137;
const double LATITUDE_MAX = 85.05112878;
const double PITCH_MAX = M_PI / 3;
const double MIN_ZOOM = 0.0;
const double MAX_ZOOM = 25.5;
const uint64_t DEFAULT_MAX_CACHE_SIZE = 50 * 1024 * 1024;
const uint64_t DEFAULT_MAX_CACHE_ENTRY_SIZE = std::numeric_limits<uint64_t>::max();
} // namespace util
namespace debug {
#if defined(DEBUG)
const bool mbgl::debug::tileParseWarnings = false;
const bool mbgl::debug::styleParseWarnings = false;
const bool mbgl::debug::spriteWarnings = false;
const bool mbgl::debug::renderWarnings = false;
const bool mbgl::debug::renderTree = false;
const bool mbgl::debug::labelTextMissingWarning = true;
const bool mbgl::debug::missingFontStackWarning = true;
const bool mbgl::debug::missingFontFaceWarning = true;
const bool mbgl::debug::glyphWarning = true;
const bool mbgl::debug::shapingWarning = true;
const bool tileParseWarnings = false;
const bool styleParseWarnings = false;
const bool spriteWarnings = false;
const bool renderWarnings = false;
const bool renderTree = false;
const bool labelTextMissingWarning = true;
const bool missingFontStackWarning = true;
const bool missingFontFaceWarning = true;
const bool glyphWarning = true;
const bool shapingWarning = true;
#else
const bool mbgl::debug::tileParseWarnings = false;
const bool mbgl::debug::styleParseWarnings = false;
const bool mbgl::debug::spriteWarnings = false;
const bool mbgl::debug::renderWarnings = false;
const bool mbgl::debug::renderTree = false;
const bool mbgl::debug::labelTextMissingWarning = false;
const bool mbgl::debug::missingFontStackWarning = false;
const bool mbgl::debug::missingFontFaceWarning = false;
const bool mbgl::debug::glyphWarning = false;
const bool mbgl::debug::shapingWarning = false;
const bool tileParseWarnings = false;
const bool styleParseWarnings = false;
const bool spriteWarnings = false;
const bool renderWarnings = false;
const bool renderTree = false;
const bool labelTextMissingWarning = false;
const bool missingFontStackWarning = false;
const bool missingFontFaceWarning = false;
const bool glyphWarning = false;
const bool shapingWarning = false;
#endif
} // namespace debug
} // namespace mbgl

View File

@@ -4,11 +4,14 @@
#include <mbgl/storage/resource.hpp>
#include <mbgl/storage/response.hpp>
#include <mbgl/util/io.hpp>
#include <mbgl/util/string.hpp>
#include <gtest/gtest.h>
#include <sqlite3.h>
#include <thread>
using namespace std::literals::string_literals;
namespace {
void createDir(const char* name) {
@@ -480,3 +483,72 @@ TEST(OfflineDatabase, ConcurrentUse) {
thread1.join();
thread2.join();
}
TEST(OfflineDatabase, PutIgnoresOversizedResources) {
using namespace mbgl;
Log::setObserver(std::make_unique<FixtureLogObserver>());
OfflineDatabase db(":memory:", 1000, 1);
Resource resource = Resource::style("http://example.com/");
Response response;
response.data = std::make_shared<std::string>("data");
db.put(resource, response);
EXPECT_FALSE(bool(db.get(resource)));
auto observer = Log::removeObserver();
auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
EXPECT_EQ(1ul, flo->count({ EventSeverity::Warning, Event::Database, -1, "Entry too big for caching" }));
}
TEST(OfflineDatabase, PutRegionResourceDoesNotIgnoreOversizedResources) {
using namespace mbgl;
OfflineDatabase db(":memory:", 1000, 1);
OfflineRegionDefinition definition { "", LatLngBounds::world(), 0, INFINITY, 1.0 };
OfflineRegion region = db.createRegion(definition, OfflineRegionMetadata());
Resource resource = Resource::style("http://example.com/");
Response response;
response.data = std::make_shared<std::string>("data");
db.putRegionResource(region.getID(), resource, response);
EXPECT_TRUE(bool(db.get(resource)));
}
TEST(OfflineDatabase, PutEvictsLeastRecentlyUsedResources) {
using namespace mbgl;
OfflineDatabase db(":memory:", 1024 * 20);
Response response;
response.data = std::make_shared<std::string>(1024, '0');
for (uint32_t i = 1; i <= 20; i++) {
db.put(Resource::style("http://example.com/"s + util::toString(i)), response);
}
EXPECT_FALSE(bool(db.get(Resource::style("http://example.com/1"))));
EXPECT_TRUE(bool(db.get(Resource::style("http://example.com/20"))));
}
TEST(OfflineDatabase, PutRegionResourceDoesNotEvict) {
using namespace mbgl;
OfflineDatabase db(":memory:", 1024 * 20);
OfflineRegionDefinition definition { "", LatLngBounds::world(), 0, INFINITY, 1.0 };
OfflineRegion region = db.createRegion(definition, OfflineRegionMetadata());
Response response;
response.data = std::make_shared<std::string>(1024, '0');
for (uint32_t i = 1; i <= 20; i++) {
db.putRegionResource(region.getID(), Resource::style("http://example.com/"s + util::toString(i)), response);
}
EXPECT_TRUE(bool(db.get(Resource::style("http://example.com/1"))));
EXPECT_TRUE(bool(db.get(Resource::style("http://example.com/20"))));
}