tei/MapLibreParticleLab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[core] Make it possible to limit the size of the SQLite cache

Browse Source
This commit is contained in:
Thiago Marcos P. Santos
2015-12-14 11:16:01 -08:00
parent 225eda4fff
commit 6d0b01c5fb
4 changed files with 269 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
include/mbgl/storage/sqlite_cache.hpp
View File

@@ -16,6 +16,9 @@ public:
SQLiteCache(const std::string &path = ":memory:");
~SQLiteCache() override;
void setMaximumCacheSize(uint64_t size);
void setMaximumCacheEntrySize(uint64_t size);
// FileCache API
std::unique_ptr<WorkRequest> get(const Resource &resource, Callback callback) override;
void put(const Resource &resource, std::shared_ptr<const Response> response, Hint hint) override;

272
platform/default/sqlite_cache.cpp
View File

@@ -11,6 +11,21 @@
#include "sqlite3.hpp"
#include <sqlite3.h>
namespace {
// The cache won't accept entries larger than this arbitrary size
// and will silently discard request for adding them to the cache.
// Large entries can cause the database to grow in disk size and
// never shrink again.
const uint64_t kMaximumCacheEntrySize = 5 * 1024 * 1024; // 5 MB
// Number of records we delete when we are close to the maximum
// database size, if set. The current criteria is to prune
// the least used entries based on `accessed` time.
const int kPrunedEntriesLimit = 100;
} // namespace
namespace mbgl {
using namespace mapbox::sqlite;
@@ -22,7 +37,9 @@ SQLiteCache::SQLiteCache(const std::string& path_)
SQLiteCache::~SQLiteCache() = default;
SQLiteCache::Impl::Impl(const std::string& path_)
: path(path_) {
: maximumCacheSize(0), // Unlimited
maximumCacheEntrySize(kMaximumCacheEntrySize),
path(path_) {
}
SQLiteCache::Impl::~Impl() {
@@ -32,6 +49,10 @@ SQLiteCache::Impl::~Impl() {
getStmt.reset();
putStmt.reset();
refreshStmt.reset();
countStmt.reset();
freeStmt.reset();
pruneStmt.reset();
accessedStmt.reset();
db.reset();
} catch (mapbox::sqlite::Exception& ex) {
Log::Error(Event::Database, ex.code, ex.what());
@@ -51,10 +72,12 @@ void SQLiteCache::Impl::createSchema() {
" `modified` INTEGER," // Timestamp when the file was last modified.
" `etag` TEXT,"
" `expires` INTEGER," // Timestamp when the server says the file expires.
" `accessed` INTEGER," // Timestamp when the database record was last accessed.
" `data` BLOB,"
" `compressed` INTEGER NOT NULL DEFAULT 0" // Whether the data is compressed.
");"
"CREATE INDEX IF NOT EXISTS `http_cache_kind_idx` ON `http_cache` (`kind`);";
"CREATE INDEX IF NOT EXISTS `http_cache_kind_idx` ON `http_cache` (`kind`);"
"CREATE INDEX IF NOT EXISTS `http_cache_accessed_idx` ON `http_cache` (`modified`);";
try {
db->exec(sql);
@@ -80,6 +103,167 @@ void SQLiteCache::Impl::createSchema() {
}
}
void SQLiteCache::setMaximumCacheSize(uint64_t size) {
thread->invoke(&Impl::setMaximumCacheSize, size);
}
void SQLiteCache::Impl::setMaximumCacheSize(uint64_t size) {
maximumCacheSize = size;
// Unlimited.
if (size == 0) {
return;
}
uint64_t lastSoftSize = cacheSoftSize();
// Keep pruning until we fit in the new
// size limit.
while (lastSoftSize > maximumCacheSize) {
pruneEntries();
if (lastSoftSize != cacheSoftSize()) {
lastSoftSize = cacheSoftSize();
} else {
break;
}
}
if (cacheHardSize() > size) {
Log::Warning(mbgl::Event::Database,
"Current cache hard size is bigger than the defined "
"maximum size. Database won't get truncated.");
}
}
void SQLiteCache::setMaximumCacheEntrySize(uint64_t size) {
thread->invoke(&Impl::setMaximumCacheEntrySize, size);
}
void SQLiteCache::Impl::setMaximumCacheEntrySize(uint64_t size) {
maximumCacheEntrySize = size;
}
void SQLiteCache::Impl::initializeDatabase() {
if (!db) {
createDatabase();
}
if (!schema) {
createSchema();
}
}
int SQLiteCache::Impl::cachePageSize() {
try {
if (!pageSize) {
Statement pageSizeStmt(db->prepare("PRAGMA page_size"));
if (pageSizeStmt.run()) {
pageSize = pageSizeStmt.get<int>(0);
}
}
} catch (mapbox::sqlite::Exception& ex) {
Log::Error(Event::Database, ex.code, ex.what());
}
return pageSize;
}
uint64_t SQLiteCache::Impl::cacheHardSize() {
try {
initializeDatabase();
if (!countStmt) {
countStmt = std::make_unique<Statement>(db->prepare("PRAGMA page_count"));
} else {
countStmt->reset();
}
if (countStmt->run()) {
return cachePageSize() * countStmt->get<int>(0);
}
} catch (mapbox::sqlite::Exception& ex) {
Log::Error(Event::Database, ex.code, ex.what());
}
return 0;
}
uint64_t SQLiteCache::Impl::cacheSoftSize() {
if (!softSizeDirty) {
return softSize;
}
try {
initializeDatabase();
if (!freeStmt) {
freeStmt = std::make_unique<Statement>(db->prepare("PRAGMA freelist_count"));
} else {
freeStmt->reset();
}
uint64_t hardSize = cacheHardSize();
if (!hardSize) {
return 0;
}
if (freeStmt->run()) {
return hardSize - cachePageSize() * freeStmt->get<int>(0);
}
softSizeDirty = false;
} catch (mapbox::sqlite::Exception& ex) {
Log::Error(Event::Database, ex.code, ex.what());
}
return 0;
}
bool SQLiteCache::Impl::needsPruning() {
// SQLite database never shrinks in size unless we call VACCUM. We here
// are monitoring the soft limit (i.e. number of free pages in the file)
// and as it approaches to the hard limit (i.e. the actual file size) we
// delete an arbitrary number of old cache entries.
//
// The free pages approach saves us from calling VACCUM or keeping a
// running total, which can be costly. We need a buffer because pages can
// get fragmented on the database.
if (cacheSoftSize() + maximumCacheEntrySize * 2 < maximumCacheSize) {
return false;
} else {
return true;
}
}
void SQLiteCache::Impl::pruneEntries() {
if (!maximumCacheSize) {
return;
}
if (!needsPruning()) {
return;
}
try {
if (!pruneStmt) {
pruneStmt = std::make_unique<Statement>(db->prepare(
"DELETE FROM `http_cache` WHERE `rowid` IN (SELECT `rowid` FROM "
// 1
"`http_cache` ORDER BY `accessed` ASC LIMIT ?)"));
} else {
pruneStmt->reset();
}
pruneStmt->bind(1, kPrunedEntriesLimit);
pruneStmt->run();
softSizeDirty = true;
} catch (mapbox::sqlite::Exception& ex) {
Log::Error(Event::Database, ex.code, ex.what());
}
}
std::unique_ptr<WorkRequest> SQLiteCache::get(const Resource &resource, Callback callback) {
// Can be called from any thread, but most likely from the file source thread.
// Will try to load the URL from the SQLite database and call the callback when done.
@@ -90,14 +274,7 @@ std::unique_ptr<WorkRequest> SQLiteCache::get(const Resource &resource, Callback
void SQLiteCache::Impl::get(const Resource &resource, Callback callback) {
try {
// This is called in the SQLite event loop.
if (!db) {
createDatabase();
}
if (!schema) {
createSchema();
}
initializeDatabase();
if (!getStmt) {
// Initialize the statement 0 1
@@ -130,6 +307,23 @@ void SQLiteCache::Impl::get(const Resource &resource, Callback callback) {
// There is no data.
callback(nullptr);
}
// We do an extra query for refreshing the last time
// the record was accessed that can be costly and is only
// worth doing if we are monitoring the database size.
if (maximumCacheSize) {
if (!accessedStmt) {
accessedStmt = std::make_unique<Statement>(
// 1 2
db->prepare("UPDATE `http_cache` SET `accessed` = ? WHERE `url` = ?"));
} else {
accessedStmt->reset();
}
accessedStmt->bind(1, toSeconds(SystemClock::now()).count());
accessedStmt->bind(2, canonicalURL.c_str());
accessedStmt->run();
}
} catch (mapbox::sqlite::Exception& ex) {
Log::Error(Event::Database, ex.code, ex.what());
callback(nullptr);
@@ -152,19 +346,28 @@ void SQLiteCache::put(const Resource &resource, std::shared_ptr<const Response>
void SQLiteCache::Impl::put(const Resource& resource, std::shared_ptr<const Response> response) {
try {
if (!db) {
createDatabase();
}
initializeDatabase();
pruneEntries();
if (!schema) {
createSchema();
if (response->data) {
auto entrySize = response->data->size();
if (entrySize > maximumCacheEntrySize) {
Log::Warning(Event::Database, "Entry too big for caching.");
return;
}
if (maximumCacheSize && entrySize + cacheSoftSize() > maximumCacheSize) {
Log::Warning(Event::Database, "Unable to make space for new entries.");
return;
}
}
if (!putStmt) {
putStmt = std::make_unique<Statement>(db->prepare("REPLACE INTO `http_cache` ("
// 1 2 3 4 5 6 7 8
"`url`, `status`, `kind`, `modified`, `etag`, `expires`, `data`, `compressed`"
") VALUES(?, ?, ?, ?, ?, ?, ?, ?)"));
// 1 2 3 4 5 6 7 8 9
"`url`, `status`, `kind`, `modified`, `etag`, `expires`, `accessed`, `data`, `compressed`"
") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)"));
} else {
putStmt->reset();
}
@@ -180,6 +383,7 @@ void SQLiteCache::Impl::put(const Resource& resource, std::shared_ptr<const Resp
putStmt->bind(4 /* modified */, response->modified.count());
putStmt->bind(5 /* etag */, response->etag.c_str());
putStmt->bind(6 /* expires */, response->expires.count());
putStmt->bind(7 /* accessed */, toSeconds(SystemClock::now()).count());
std::string data;
if (resource.kind != Resource::SpriteImage && response->data) {
@@ -190,17 +394,18 @@ void SQLiteCache::Impl::put(const Resource& resource, std::shared_ptr<const Resp
if (!data.empty() && data.size() < response->data->size()) {
// Store the compressed data when it is smaller than the original
// uncompressed data.
putStmt->bind(7 /* data */, data, false); // do not retain the string internally.
putStmt->bind(8 /* compressed */, true);
putStmt->bind(8 /* data */, data, false); // do not retain the string internally.
putStmt->bind(9 /* compressed */, true);
} else if (response->data) {
putStmt->bind(7 /* data */, *response->data, false); // do not retain the string internally.
putStmt->bind(8 /* compressed */, false);
putStmt->bind(8 /* data */, *response->data, false); // do not retain the string internally.
putStmt->bind(9 /* compressed */, false);
} else {
putStmt->bind(7 /* data */, "", false);
putStmt->bind(8 /* compressed */, false);
putStmt->bind(8 /* data */, "", false);
putStmt->bind(9 /* compressed */, false);
}
putStmt->run();
softSizeDirty = true;
} catch (mapbox::sqlite::Exception& ex) {
Log::Error(Event::Database, ex.code, ex.what());
} catch (std::runtime_error& ex) {
@@ -210,24 +415,21 @@ void SQLiteCache::Impl::put(const Resource& resource, std::shared_ptr<const Resp
void SQLiteCache::Impl::refresh(const Resource& resource, Seconds expires) {
try {
if (!db) {
createDatabase();
}
if (!schema) {
createSchema();
}
initializeDatabase();
if (!refreshStmt) {
refreshStmt = std::make_unique<Statement>( // 1 2
db->prepare("UPDATE `http_cache` SET `expires` = ? WHERE `url` = ?"));
refreshStmt = std::make_unique<Statement>(
db->prepare("UPDATE `http_cache` SET "
// 1 2 3
"`accessed` = ?, `expires` = ? WHERE `url` = ?"));
} else {
refreshStmt->reset();
}
const auto canonicalURL = util::mapbox::canonicalURL(resource.url);
refreshStmt->bind(1, expires.count());
refreshStmt->bind(2, canonicalURL.c_str());
refreshStmt->bind(1, toSeconds(SystemClock::now()).count());
refreshStmt->bind(2, expires.count());
refreshStmt->bind(3, canonicalURL.c_str());
refreshStmt->run();
} catch (mapbox::sqlite::Exception& ex) {
Log::Error(Event::Database, ex.code, ex.what());

28
platform/default/sqlite_cache_impl.hpp
View File

@@ -18,23 +18,47 @@ public:
explicit Impl(const std::string &path = ":memory:");
~Impl();
void setMaximumCacheSize(uint64_t size);
void setMaximumCacheEntrySize(uint64_t size);
void get(const Resource&, Callback);
void put(const Resource& resource, std::shared_ptr<const Response> response);
void refresh(const Resource& resource, Seconds expires);
private:
void initializeDatabase();
int cachePageSize();
uint64_t cacheHardSize();
uint64_t cacheSoftSize();
uint64_t softSize = 0;
bool softSizeDirty = true;
bool needsPruning();
void pruneEntries();
void createDatabase();
void createSchema();
int pageSize = 0;
uint64_t maximumCacheSize;
uint64_t maximumCacheEntrySize;
const std::string path;
std::unique_ptr<::mapbox::sqlite::Database> db;
std::unique_ptr<::mapbox::sqlite::Statement> getStmt;
std::unique_ptr<::mapbox::sqlite::Statement> putStmt;
std::unique_ptr<::mapbox::sqlite::Statement> refreshStmt;
std::unique_ptr<::mapbox::sqlite::Statement> countStmt;
std::unique_ptr<::mapbox::sqlite::Statement> freeStmt;
std::unique_ptr<::mapbox::sqlite::Statement> pruneStmt;
std::unique_ptr<::mapbox::sqlite::Statement> accessedStmt;
bool schema = false;
};
}
} // namespace mbgl
#endif

3
platform/linux/main.cpp
View File

@@ -107,6 +107,9 @@ int main(int argc, char *argv[]) {
view = std::make_unique<GLFWView>(fullscreen, benchmark);
mbgl::SQLiteCache cache("/tmp/mbgl-cache.db");
cache.setMaximumCacheEntrySize(1 * 1024 * 1024); // 1 MB
cache.setMaximumCacheSize(50 * 1024 * 1024); // 50 MB
mbgl::DefaultFileSource fileSource(&cache);
// Set access token if present