From 34fe7dc627f08d57ac259c383e248ddf2f6fb0d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Fri, 22 Aug 2014 18:50:09 +0200 Subject: [PATCH] add basic request caching with sqlite --- configure | 9 ++ include/mbgl/platform/event.hpp | 2 + include/mbgl/util/filesource.hpp | 15 +++- mapboxgl.gyp | 53 ++++++++---- setup-libraries.sh | 7 ++ src/util/filesource.cpp | 143 ++++++++++++++++++++++++++++++- 6 files changed, 210 insertions(+), 19 deletions(-) diff --git a/configure b/configure index e8f880b54..b92f97758 100755 --- a/configure +++ b/configure @@ -107,6 +107,14 @@ def configure_curl(o): o['variables']['curl_libraries'] = ret[0].split() o['variables']['curl_cflags'] = ret[1].split() +def configure_sqlite3(o): + ret = pkg_config('sqlite3', options.pkgconfig_root) + if not ret: + sys.stderr.write('could not find sqlite3 with pkg-config') + sys.exit(-1) + o['variables']['sqlite3_libraries'] = ret[0].split() + o['variables']['sqlite3_cflags'] = ret[1].split() + def write(filename, data): filename = os.path.join(root_dir, filename) print "creating ", filename @@ -129,6 +137,7 @@ if __name__ == '__main__': configure_uv(output) configure_png(output) configure_curl(output) + configure_sqlite3(output) pprint.pprint(output, indent=2) write('config.gypi', "# Do not edit. Generated by the configure script.\n" + diff --git a/include/mbgl/platform/event.hpp b/include/mbgl/platform/event.hpp index 1676f40d2..d2a06618a 100644 --- a/include/mbgl/platform/event.hpp +++ b/include/mbgl/platform/event.hpp @@ -29,6 +29,7 @@ enum class Event : uint8_t { ParseStyle, ParseTile, Render, + Database, HttpRequest, Sprite, }; @@ -40,6 +41,7 @@ MBGL_DEFINE_ENUM_CLASS(EventClass, Event, { { Event::ParseStyle, "ParseStyle" }, { Event::ParseTile, "ParseTile" }, { Event::Render, "Render" }, + { Event::Database, "Database" }, { Event::HttpRequest, "HttpRequest" }, { Event::Sprite, "Sprite" }, { Event(-1), "Unknown" }, diff --git a/include/mbgl/util/filesource.hpp b/include/mbgl/util/filesource.hpp index 18c63ddfe..bc0a2927d 100644 --- a/include/mbgl/util/filesource.hpp +++ b/include/mbgl/util/filesource.hpp @@ -2,11 +2,15 @@ #define MBGL_UTIL_FILESOURCE #include +#include #include #include #include +typedef struct sqlite3 sqlite3; +typedef struct sqlite3_stmt sqlite3_stmt; + namespace mbgl { namespace platform { @@ -21,21 +25,30 @@ enum class ResourceType : uint8_t { JSON }; -class FileSource { +class FileSource : private util::noncopyable, public std::enable_shared_from_this { public: FileSource(); + ~FileSource(); void setBase(const std::string &value); const std::string &getBase() const; void load(ResourceType type, const std::string &url, std::function callback, const std::shared_ptr loop = nullptr); +private: + void closeDatabase(); + void createSchema(); + bool loadFile(const std::string &url, std::function callback); + void saveFile(const std::string &url, platform::Response *res); + private: // Stores a URL that is used as a base for loading resources with relative path. std::string base; // Stores the absolute path to the cache directory. const std::string cache; + + sqlite3 *db = nullptr; }; } diff --git a/mapboxgl.gyp b/mapboxgl.gyp index dcd89f3a7..49c4150aa 100644 --- a/mapboxgl.gyp +++ b/mapboxgl.gyp @@ -126,23 +126,36 @@ ' #include +#include + +#include #include #include namespace mbgl { -FileSource::FileSource() {} +FileSource::FileSource() { + const int err = sqlite3_open_v2("cache.db", &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, nullptr); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + db = nullptr; + } + createSchema(); +} + +FileSource::~FileSource() { + closeDatabase(); +} + +void FileSource::createSchema() { + if (db) { + // Create schema + const int err = sqlite3_exec(db, "CREATE TABLE IF NOT EXISTS `http_cache` (`url` TEXT PRIMARY KEY, `expires` INTEGER, `data` BLOB)", nullptr, nullptr, nullptr); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + closeDatabase(); + } + } +} + +void FileSource::closeDatabase() { + if (db) { + const int err = sqlite3_close_v2(db); + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + db = nullptr; + } +} void FileSource::setBase(const std::string &value) { base = value; @@ -17,6 +50,104 @@ const std::string &FileSource::getBase() const { return base; } +bool FileSource::loadFile(const std::string &url, std::function callback) { + if (!db) { + return false; + } + + sqlite3_stmt *stmt = nullptr; + int err; + + err = sqlite3_prepare_v2(db, "SELECT `data` FROM `http_cache` WHERE `url` = ?", -1, &stmt, nullptr); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + return false; + } + + err = sqlite3_bind_text(stmt, 1, url.data(), url.size(), nullptr); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + err = sqlite3_finalize(stmt); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } + return false; + } + + bool status = false; + + err = sqlite3_step(stmt); + if (err == SQLITE_ROW) { + // There is data. + platform::Response res(callback); + res.code = 200; + const char *data = reinterpret_cast(sqlite3_column_blob(stmt, 0)); + const size_t size = sqlite3_column_bytes(stmt, 0); + res.body = { data, size }; + callback(&res); + status = true; + } else if (err == SQLITE_DONE) { + // There is no data. + // This is a noop. + } else { + // Another error occured. + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } + + err = sqlite3_finalize(stmt); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } + return status; +} + +void FileSource::saveFile(const std::string &url, platform::Response *const res) { + if (!db) { + return; + } + + sqlite3_stmt *stmt = nullptr; + int err; + + err = sqlite3_prepare_v2(db, "REPLACE INTO `http_cache` (`url`, `data`) VALUES(?, ?)", -1, &stmt, nullptr); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + return; + } + + err = sqlite3_bind_text(stmt, 1, url.data(), url.size(), nullptr); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + err = sqlite3_finalize(stmt); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } + return; + } + + err = sqlite3_bind_blob(stmt, 2, res->body.data(), res->body.size(), nullptr); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + err = sqlite3_finalize(stmt); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } + return; + } + + err = sqlite3_step(stmt); + if (err != SQLITE_DONE) { + // Another error occured. + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } + + err = sqlite3_finalize(stmt); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } +} + + void FileSource::load(ResourceType /*type*/, const std::string &url, std::function callback, const std::shared_ptr loop) { // convert relative URLs to absolute URLs @@ -51,7 +182,15 @@ void FileSource::load(ResourceType /*type*/, const std::string &url, std::functi callback(&response); } else { // load from the internet - platform::request_http(absoluteURL, callback, loop); + if (!loadFile(absoluteURL, callback)) { + const std::shared_ptr source = shared_from_this(); + platform::request_http(absoluteURL, [=](platform::Response *res) { + if (res->code == 200) { + source->saveFile(absoluteURL, res); + } + callback(res); + }, loop); + } } }