diff --git a/include/mbgl/util/compression.hpp b/include/mbgl/util/compression.hpp new file mode 100644 index 000000000..a33b2476a --- /dev/null +++ b/include/mbgl/util/compression.hpp @@ -0,0 +1,15 @@ +#ifndef MBGL_UTIL_COMPRESSION +#define MBGL_UTIL_COMPRESSION + +#include + +namespace mbgl { +namespace util { + +std::string compress(const std::string &raw); +std::string decompress(const std::string &raw); + +} +} + +#endif diff --git a/include/mbgl/util/filesource.hpp b/include/mbgl/util/filesource.hpp index bc0a2927d..ccff4b512 100644 --- a/include/mbgl/util/filesource.hpp +++ b/include/mbgl/util/filesource.hpp @@ -18,11 +18,11 @@ struct Response; } enum class ResourceType : uint8_t { - Unknown, - Tile, - Glyphs, - Image, - JSON + Unknown = 0, + Tile = 1, + Glyphs = 2, + Image = 3, + JSON = 4 }; class FileSource : private util::noncopyable, public std::enable_shared_from_this { @@ -38,8 +38,8 @@ public: private: void closeDatabase(); void createSchema(); - bool loadFile(const std::string &url, std::function callback); - void saveFile(const std::string &url, platform::Response *res); + bool loadFile(ResourceType type, const std::string &url, std::function callback); + void saveFile(ResourceType type, const std::string &url, platform::Response *res); private: // Stores a URL that is used as a base for loading resources with relative path. diff --git a/src/util/compression.cpp b/src/util/compression.cpp new file mode 100644 index 000000000..392250186 --- /dev/null +++ b/src/util/compression.cpp @@ -0,0 +1,78 @@ +#include + +#include + +namespace mbgl { +namespace util { + +std::string compress(const std::string &raw) { + z_stream deflate_stream; + memset(&deflate_stream, 0, sizeof(deflate_stream)); + + // TODO: reuse z_streams + if (deflateInit(&deflate_stream, Z_DEFAULT_COMPRESSION) != Z_OK) { + throw std::runtime_error("failed to initialize deflate"); + } + + deflate_stream.next_in = (Bytef *)raw.data(); + deflate_stream.avail_in = raw.size(); + + std::string result; + char out[16384]; + + int code; + do { + deflate_stream.next_out = reinterpret_cast(out); + deflate_stream.avail_out = sizeof(out); + code = deflate(&deflate_stream, Z_FINISH); + if (result.size() < deflate_stream.total_out) { + // append the block to the output string + result.append(out, deflate_stream.total_out - result.size()); + } + } while (code == Z_OK); + + deflateEnd(&deflate_stream); + + if (code != Z_STREAM_END) { + throw std::runtime_error(deflate_stream.msg); + } + + return result; +} + +std::string decompress(const std::string &raw) { + z_stream inflate_stream; + memset(&inflate_stream, 0, sizeof(inflate_stream)); + + // TODO: reuse z_streams + if (inflateInit(&inflate_stream) != Z_OK) { + throw std::runtime_error("failed to initialize inflate"); + } + + inflate_stream.next_in = (Bytef *)raw.data(); + inflate_stream.avail_in = raw.size(); + + std::string result; + char out[15384]; + + int code; + do { + inflate_stream.next_out = reinterpret_cast(out); + inflate_stream.avail_out = sizeof(out); + code = inflate(&inflate_stream, 0); + // result.append(out, sizeof(out) - inflate_stream.avail_out); + if (result.size() < inflate_stream.total_out) { + result.append(out, inflate_stream.total_out - result.size()); + } + } while (code == Z_OK); + + inflateEnd(&inflate_stream); + + if (code != Z_STREAM_END) { + throw std::runtime_error(inflate_stream.msg); + } + + return result; +} +} +} diff --git a/src/util/filesource.cpp b/src/util/filesource.cpp index 018465990..d626092bc 100644 --- a/src/util/filesource.cpp +++ b/src/util/filesource.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include @@ -30,9 +31,12 @@ void FileSource::createSchema() { "CREATE TABLE IF NOT EXISTS `http_cache` (" "`url` TEXT PRIMARY KEY NOT NULL," "`code` INTEGER NOT NULL," + "`type` INTEGER NOT NULL," "`expires` INTEGER," - "`data` BLOB" - ")", nullptr, nullptr, nullptr); + "`data` BLOB," + "`compressed` INTEGER NOT NULL DEFAULT 0" + ");" + "CREATE INDEX IF NOT EXISTS `http_cache_type_idx` ON `http_cache` (`type`);", nullptr, nullptr, nullptr); if (err != SQLITE_OK) { Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); closeDatabase(); @@ -73,7 +77,7 @@ std::string removeAccessTokenFromURL(const std::string &url) { } } -bool FileSource::loadFile(const std::string &url, std::function callback) { +bool FileSource::loadFile(ResourceType /*type*/, const std::string &url, std::function callback) { if (!db) { return false; } @@ -81,7 +85,7 @@ bool FileSource::loadFile(const std::string &url, std::function(sqlite3_column_blob(stmt, 1)); const size_t size = sqlite3_column_bytes(stmt, 1); - res.body = { data, size }; + const bool compressed = sqlite3_column_int(stmt, 2); + + if (compressed) { + res.body = util::decompress({ data, size }); + } else { + res.body = { data, size }; + } + callback(&res); status = true; } else if (err == SQLITE_DONE) { @@ -124,7 +135,7 @@ bool FileSource::loadFile(const std::string &url, std::functionbody.data(), res->body.size(), nullptr); + err = sqlite3_bind_int(stmt, 3, int(type)); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + err = sqlite3_finalize(stmt); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } + return; + } + + bool compressed = false; + switch (type) { + case ResourceType::Image: + err = sqlite3_bind_blob(stmt, 4, res->body.data(), res->body.size(), SQLITE_STATIC); + break; + default: + const std::string *data = new std::string(std::move(util::compress(res->body))); + compressed = true; + err = sqlite3_bind_blob(stmt, 4, data->data(), data->size(), [](void *p) { delete (std::string *)p; }); + break; + } + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + err = sqlite3_finalize(stmt); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } + return; + } + + err = sqlite3_bind_int(stmt, 5, compressed); if (err != SQLITE_OK) { Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); err = sqlite3_finalize(stmt); @@ -181,7 +222,7 @@ void FileSource::saveFile(const std::string &url, platform::Response *const res) } -void FileSource::load(ResourceType /*type*/, const std::string &url, std::function callback, const std::shared_ptr loop) { +void FileSource::load(ResourceType type, const std::string &url, std::function callback, const std::shared_ptr loop) { // convert relative URLs to absolute URLs const std::string absoluteURL = [&]() -> std::string { @@ -218,10 +259,10 @@ void FileSource::load(ResourceType /*type*/, const std::string &url, std::functi const std::string cleanURL = removeAccessTokenFromURL(url); // load from the internet - if (!loadFile(cleanURL, callback)) { + if (!loadFile(type, cleanURL, callback)) { const std::shared_ptr source = shared_from_this(); platform::request_http(absoluteURL, [=](platform::Response *res) { - source->saveFile(cleanURL, res); + source->saveFile(type, cleanURL, res); callback(res); }, loop); }