make storage lib separate so we can build without storage libs

This commit is contained in:
Konstantin Käfer
2015-01-23 16:31:48 +01:00
parent fbe30e04c4
commit 272fa8935e
26 changed files with 451 additions and 247 deletions

View File

@@ -54,38 +54,6 @@ const static bool uvVersionCheck = []() {
return true;
}();
#include <zlib.h>
// Check zlib library version.
const static bool zlibVersionCheck = []() {
const char *const version = zlibVersion();
if (version[0] != ZLIB_VERSION[0]) {
throw std::runtime_error(mbgl::util::sprintf<96>(
"zlib version mismatch: headers report %s, but library reports %s", ZLIB_VERSION, version));
}
return true;
}();
#include <sqlite3.h>
// Check sqlite3 library version.
const static bool sqliteVersionCheck = []() {
if (sqlite3_libversion_number() != SQLITE_VERSION_NUMBER) {
throw std::runtime_error(mbgl::util::sprintf<96>(
"sqlite3 libversion mismatch: headers report %d, but library reports %d",
SQLITE_VERSION_NUMBER, sqlite3_libversion_number()));
}
if (strcmp(sqlite3_sourceid(), SQLITE_SOURCE_ID) != 0) {
throw std::runtime_error(mbgl::util::sprintf<256>(
"sqlite3 sourceid mismatch: headers report \"%s\", but library reports \"%s\"",
SQLITE_SOURCE_ID, sqlite3_sourceid()));
}
return true;
}();
using namespace mbgl;
Map::Map(View& view_, FileSource& fileSource_)

View File

@@ -1,239 +0,0 @@
#include <mbgl/storage/default/default_file_source.hpp>
#include <mbgl/storage/default/request.hpp>
#include <mbgl/storage/default/asset_request.hpp>
#include <mbgl/storage/default/http_request.hpp>
#include <mbgl/storage/response.hpp>
#include <mbgl/util/async_queue.hpp>
#include <mbgl/util/util.hpp>
#include <mbgl/util/variant.hpp>
#include <boost/algorithm/string.hpp>
#include <thread>
#include <algorithm>
#include <cassert>
namespace algo = boost::algorithm;
namespace mbgl {
struct DefaultFileSource::ActionDispatcher {
DefaultFileSource &fileSource;
template <typename T> void operator()(T &t) { fileSource.process(t); }
};
struct DefaultFileSource::AddRequestAction {
Request *const request;
};
struct DefaultFileSource::RemoveRequestAction {
Request *const request;
};
struct DefaultFileSource::ResultAction {
const Resource resource;
std::unique_ptr<Response> response;
};
struct DefaultFileSource::StopAction {
};
DefaultFileSource::DefaultFileSource(FileCache *cache_)
: loop(uv_loop_new()),
cache(cache_),
queue(new Queue(loop, [this](Action &action) {
mapbox::util::apply_visitor(ActionDispatcher{*this}, action);
})),
thread([this]() {
#ifdef __APPLE__
pthread_setname_np("FileSource");
#endif
uv_run(loop, UV_RUN_DEFAULT);
}) {
}
DefaultFileSource::DefaultFileSource(FileCache *cache_, uv_loop_t *loop_)
: loop(loop_),
cache(cache_),
queue(new Queue(loop, [this](Action &action) {
mapbox::util::apply_visitor(ActionDispatcher{*this}, action);
})) {
// Make sure that the queue doesn't block the loop from exiting.
queue->unref();
}
DefaultFileSource::~DefaultFileSource() {
MBGL_VERIFY_THREAD(tid);
if (thread.joinable()) {
if (queue) {
queue->send(StopAction{ });
}
thread.join();
uv_loop_delete(loop);
} else {
// Assume that the loop we received is running in the current thread.
StopAction action {};
process(action);
}
}
SharedRequestBase *DefaultFileSource::find(const Resource &resource) {
// We're using a set of pointers here instead of a map between url and SharedRequestBase because
// we need to find the requests both by pointer and by URL. Given that the number of requests
// is generally very small (typically < 10 at a time), hashing by URL incurs too much overhead
// anyway.
const auto it = pending.find(resource);
if (it != pending.end()) {
return it->second;
}
return nullptr;
}
Request *DefaultFileSource::request(const Resource &resource, uv_loop_t *l, Callback callback) {
auto req = new Request(resource, l, std::move(callback));
// This function can be called from any thread. Make sure we're executing the actual call in the
// file source loop by sending it over the queue. It will be processed in processAction().
queue->send(AddRequestAction{ req });
return req;
}
void DefaultFileSource::request(const Resource &resource, Callback callback) {
auto req = new Request(resource, nullptr, std::move(callback));
// This function can be called from any thread. Make sure we're executing the actual call in the
// file source loop by sending it over the queue. It will be processed in processAction().
queue->send(AddRequestAction{ req });
}
void DefaultFileSource::cancel(Request *req) {
req->cancel();
// This function can be called from any thread. Make sure we're executing the actual call in the
// file source loop by sending it over the queue. It will be processed in processAction().
queue->send(RemoveRequestAction{ req });
}
void DefaultFileSource::process(AddRequestAction &action) {
const Resource &resource = action.request->resource;
// We're adding a new Request.
SharedRequestBase *sharedRequest = find(resource);
if (!sharedRequest) {
// There is no request for this URL yet. Create a new one and start it.
if (algo::starts_with(resource.url, "asset://")) {
sharedRequest = new AssetRequest(this, resource);
} else {
sharedRequest = new HTTPRequest(this, resource);
}
// Make sure the loop stays alive when we're not running the file source in it's own thread.
if (!thread.joinable() && pending.empty()) {
queue->ref();
}
const bool inserted = pending.emplace(resource, sharedRequest).second;
assert(inserted);
(void (inserted)); // silence unused variable warning on Release builds.
// But first, we're going to start querying the database if it exists.
if (!cache) {
sharedRequest->start(loop);
} else {
// Otherwise, first check the cache for existing data so that we can potentially
// revalidate the information without having to redownload everything.
cache->get(resource, [this, resource](std::unique_ptr<Response> response) {
queue->send(ResultAction { resource, std::move(response) });
});
}
}
sharedRequest->subscribe(action.request);
}
void DefaultFileSource::process(RemoveRequestAction &action) {
SharedRequestBase *sharedRequest = find(action.request->resource);
if (sharedRequest) {
// If the number of dependent requests of the SharedRequestBase drops to zero, the
// unsubscribe callback triggers the removal of the SharedRequestBase pointer from the list
// of pending requests and initiates cancelation.
sharedRequest->unsubscribe(action.request);
} else {
// There is no request for this URL anymore. Likely, the request already completed
// before we got around to process the cancelation request.
}
// Send a message back to the requesting thread and notify it that this request has been
// canceled and is now safe to be deleted.
action.request->destruct();
}
void DefaultFileSource::process(ResultAction &action) {
SharedRequestBase *sharedRequest = find(action.resource);
if (sharedRequest) {
if (action.response) {
// This entry was stored in the cache. Now determine if we need to revalidate.
const int64_t now = std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now().time_since_epoch()).count();
if (action.response->expires > now) {
// The response is fresh. We're good to notify the caller.
sharedRequest->notify(std::move(action.response), FileCache::Hint::No);
sharedRequest->cancel();
return;
} else {
// The cached response is stale. Now run the real request.
sharedRequest->start(loop, std::move(action.response));
}
} else {
// There is no response. Now run the real request.
sharedRequest->start(loop);
}
} else {
// There is no request for this URL anymore. Likely, the request was canceled
// before we got around to process the cache result.
}
}
void DefaultFileSource::process(StopAction &) {
// Cancel all remaining requests.
for (auto it : pending) {
it.second->unsubscribeAll();
}
pending.clear();
assert(queue);
queue->stop();
queue = nullptr;
}
void DefaultFileSource::notify(SharedRequestBase *sharedRequest,
const std::set<Request *> &observers,
std::shared_ptr<const Response> response, FileCache::Hint hint) {
// First, remove the request, since it might be destructed at any point now.
assert(find(sharedRequest->resource) == sharedRequest);
pending.erase(sharedRequest->resource);
if (response) {
if (cache) {
// Store response in database
cache->put(sharedRequest->resource, response, hint);
}
// Notify all observers.
for (auto it : observers) {
it->notify(response);
}
}
if (!thread.joinable() && pending.empty()) {
// When there are no pending requests, we're going to allow the queue to stop.
queue->unref();
}
}
}

View File

@@ -1,271 +0,0 @@
#include <mbgl/storage/default/sqlite_cache.hpp>
#include <mbgl/storage/default/request.hpp>
#include <mbgl/storage/response.hpp>
#include <mbgl/util/util.hpp>
#include <mbgl/util/async_queue.hpp>
#include <mbgl/util/variant.hpp>
#include <mbgl/platform/log.hpp>
#include "../util/sqlite3.hpp"
#include "../util/compression.hpp"
#include <uv.h>
#include <cassert>
namespace mbgl {
std::string removeAccessTokenFromURL(const std::string &url) {
const size_t token_start = url.find("access_token=");
// Ensure that token exists, isn't at the front and is preceded by either & or ?.
if (token_start == std::string::npos || token_start == 0 || !(url[token_start - 1] == '&' || url[token_start - 1] == '?')) {
return url;
}
const size_t token_end = url.find_first_of('&', token_start);
if (token_end == std::string::npos) {
// The token is the last query argument. We slice away the "&access_token=..." part
return url.substr(0, token_start - 1);
} else {
// We slice away the "access_token=...&" part.
return url.substr(0, token_start) + url.substr(token_end + 1);
}
}
std::string convertMapboxDomainsToProtocol(const std::string &url) {
const size_t protocol_separator = url.find("://");
if (protocol_separator == std::string::npos) {
return url;
}
const std::string protocol = url.substr(0, protocol_separator);
if (!(protocol == "http" || protocol == "https")) {
return url;
}
const size_t domain_begin = protocol_separator + 3;
const size_t path_separator = url.find("/", domain_begin);
if (path_separator == std::string::npos) {
return url;
}
const std::string domain = url.substr(domain_begin, path_separator - domain_begin);
if (domain.find(".tiles.mapbox.com") != std::string::npos) {
return "mapbox://" + url.substr(path_separator + 1);
} else {
return url;
}
}
std::string unifyMapboxURLs(const std::string &url) {
return removeAccessTokenFromURL(convertMapboxDomainsToProtocol(url));
}
using namespace mapbox::sqlite;
struct SQLiteCache::GetAction {
const Resource resource;
const std::function<void(std::unique_ptr<Response>)> callback;
};
struct SQLiteCache::PutAction {
const Resource resource;
const std::shared_ptr<const Response> response;
};
struct SQLiteCache::RefreshAction {
const Resource resource;
const int64_t expires;
};
struct SQLiteCache::StopAction {
};
struct SQLiteCache::ActionDispatcher {
SQLiteCache &cache;
template <typename T> void operator()(T &t) { cache.process(t); }
};
SQLiteCache::SQLiteCache(const std::string &path_)
: path(path_),
loop(uv_loop_new()),
queue(new Queue(loop, [this](Action &action) {
mapbox::util::apply_visitor(ActionDispatcher{ *this }, action);
})),
thread([this]() {
#ifdef __APPLE__
pthread_setname_np("SQLite Cache");
#endif
uv_run(loop, UV_RUN_DEFAULT);
})
{
}
SQLiteCache::~SQLiteCache() {
if (thread.joinable()) {
if (queue) {
queue->send(StopAction{ });
}
thread.join();
uv_loop_delete(loop);
}
}
void SQLiteCache::get(const Resource &resource, std::function<void(std::unique_ptr<Response>)> callback) {
// Can be called from any thread, but most likely from the file source thread.
// Will try to load the URL from the SQLite database and call the callback when done.
// Note that the callback is probably going to invoked from another thread, so the caller
// must make sure that it can run in that thread.
assert(queue);
queue->send(GetAction{ resource, callback });
}
void SQLiteCache::put(const Resource &resource, std::shared_ptr<const Response> response, Hint hint) {
// Can be called from any thread, but most likely from the file source thread. We are either
// storing a new response or updating the currently stored response, potentially setting a new
// expiry date.
assert(queue);
assert(response);
if (hint == Hint::Full) {
queue->send(PutAction{ resource, response });
} else if (hint == Hint::Refresh) {
queue->send(RefreshAction{ resource, response->expires });
}
}
void SQLiteCache::createDatabase() {
db = util::make_unique<Database>(path.c_str(), ReadWrite | Create);
constexpr const char *const sql = ""
"CREATE TABLE IF NOT EXISTS `http_cache` ("
" `url` TEXT PRIMARY KEY NOT NULL,"
" `status` INTEGER NOT NULL," // The response status (Successful or Error).
" `kind` INTEGER NOT NULL," // The kind of file.
" `modified` INTEGER," // Timestamp when the file was last modified.
" `etag` TEXT,"
" `expires` INTEGER," // Timestamp when the server says the file expires.
" `data` BLOB,"
" `compressed` INTEGER NOT NULL DEFAULT 0" // Whether the data is compressed.
");"
"CREATE INDEX IF NOT EXISTS `http_cache_kind_idx` ON `http_cache` (`kind`);";
try {
db->exec(sql);
} catch(mapbox::sqlite::Exception &) {
// Creating the database table + index failed. That means there may already be one, likely
// with different columsn. Drop it and try to create a new one.
try {
db->exec("DROP TABLE IF EXISTS `http_cache`");
db->exec(sql);
} catch (mapbox::sqlite::Exception &ex) {
Log::Error(Event::Database, "Failed to create database: %s", ex.what());
db.release();
}
}
}
void SQLiteCache::process(GetAction &action) {
// This is called in the SQLite event loop.
if (!db) {
createDatabase();
}
if (!getStmt) {
// Initialize the statement 0 1
getStmt = util::make_unique<Statement>(db->prepare("SELECT `status`, `modified`, "
// 2 3 4 5 1
"`etag`, `expires`, `data`, `compressed` FROM `http_cache` WHERE `url` = ?"));
} else {
getStmt->reset();
}
const std::string unifiedURL = unifyMapboxURLs(action.resource.url);
getStmt->bind(1, unifiedURL.c_str());
if (getStmt->run()) {
// There is data.
auto response = util::make_unique<Response>();
response->status = Response::Status(getStmt->get<int>(0));
response->modified = getStmt->get<int64_t>(1);
response->etag = getStmt->get<std::string>(2);
response->expires = getStmt->get<int64_t>(3);
response->data = getStmt->get<std::string>(4);
if (getStmt->get<int>(5)) { // == compressed
response->data = util::decompress(response->data);
}
action.callback(std::move(response));
} else {
// There is no data.
action.callback(nullptr);
}
}
void SQLiteCache::process(PutAction &action) {
if (!db) {
createDatabase();
}
if (!putStmt) {
putStmt = util::make_unique<Statement>(db->prepare("REPLACE INTO `http_cache` ("
// 1 2 3 4 5 6 7 8
"`url`, `status`, `kind`, `modified`, `etag`, `expires`, `data`, `compressed`"
") VALUES(?, ?, ?, ?, ?, ?, ?, ?)"));
} else {
putStmt->reset();
}
const std::string unifiedURL = unifyMapboxURLs(action.resource.url);
putStmt->bind(1 /* url */, unifiedURL.c_str());
putStmt->bind(2 /* status */, int(action.response->status));
putStmt->bind(3 /* kind */, int(action.resource.kind));
putStmt->bind(4 /* modified */, action.response->modified);
putStmt->bind(5 /* etag */, action.response->etag.c_str());
putStmt->bind(6 /* expires */, action.response->expires);
std::string data;
if (action.resource.kind != Resource::Image) {
// Do not compress images, since they are typically compressed already.
data = util::compress(action.response->data);
}
if (!data.empty() && data.size() < action.response->data.size()) {
// Store the compressed data when it is smaller than the original
// uncompressed data.
putStmt->bind(7 /* data */, data, false); // do not retain the string internally.
putStmt->bind(8 /* compressed */, true);
} else {
putStmt->bind(7 /* data */, action.response->data, false); // do not retain the string internally.
putStmt->bind(8 /* compressed */, false);
}
putStmt->run();
}
void SQLiteCache::process(RefreshAction &action) {
if (!db) {
createDatabase();
}
if (!refreshStmt) {
refreshStmt = util::make_unique<Statement>( // 1 2
db->prepare("UPDATE `http_cache` SET `expires` = ? WHERE `url` = ?"));
} else {
refreshStmt->reset();
}
const std::string unifiedURL = unifyMapboxURLs(action.resource.url);
refreshStmt->bind(1, int64_t(action.expires));
refreshStmt->bind(2, unifiedURL.c_str());
refreshStmt->run();
}
void SQLiteCache::process(StopAction &) {
assert(queue);
queue->stop();
queue = nullptr;
}
}

View File

@@ -1,81 +0,0 @@
#include <mbgl/util/compression.hpp>
#include <zlib.h>
#include <cstring>
#include <stdexcept>
namespace mbgl {
namespace util {
std::string compress(const std::string &raw) {
z_stream deflate_stream;
memset(&deflate_stream, 0, sizeof(deflate_stream));
// TODO: reuse z_streams
if (deflateInit(&deflate_stream, Z_DEFAULT_COMPRESSION) != Z_OK) {
throw std::runtime_error("failed to initialize deflate");
}
deflate_stream.next_in = (Bytef *)raw.data();
deflate_stream.avail_in = uInt(raw.size());
std::string result;
char out[16384];
int code;
do {
deflate_stream.next_out = reinterpret_cast<Bytef *>(out);
deflate_stream.avail_out = sizeof(out);
code = deflate(&deflate_stream, Z_FINISH);
if (result.size() < deflate_stream.total_out) {
// append the block to the output string
result.append(out, deflate_stream.total_out - result.size());
}
} while (code == Z_OK);
deflateEnd(&deflate_stream);
if (code != Z_STREAM_END) {
throw std::runtime_error(deflate_stream.msg);
}
return result;
}
std::string decompress(const std::string &raw) {
z_stream inflate_stream;
memset(&inflate_stream, 0, sizeof(inflate_stream));
// TODO: reuse z_streams
if (inflateInit(&inflate_stream) != Z_OK) {
throw std::runtime_error("failed to initialize inflate");
}
inflate_stream.next_in = (Bytef *)raw.data();
inflate_stream.avail_in = uInt(raw.size());
std::string result;
char out[15384];
int code;
do {
inflate_stream.next_out = reinterpret_cast<Bytef *>(out);
inflate_stream.avail_out = sizeof(out);
code = inflate(&inflate_stream, 0);
// result.append(out, sizeof(out) - inflate_stream.avail_out);
if (result.size() < inflate_stream.total_out) {
result.append(out, inflate_stream.total_out - result.size());
}
} while (code == Z_OK);
inflateEnd(&inflate_stream);
if (code != Z_STREAM_END) {
throw std::runtime_error(inflate_stream.msg);
}
return result;
}
}
}

View File

@@ -1,15 +0,0 @@
#ifndef MBGL_UTIL_COMPRESSION
#define MBGL_UTIL_COMPRESSION
#include <string>
namespace mbgl {
namespace util {
std::string compress(const std::string &raw);
std::string decompress(const std::string &raw);
}
}
#endif

View File

@@ -1,165 +0,0 @@
#include <mbgl/util/sqlite3.hpp>
#include <sqlite3.h>
#include <cassert>
namespace mapbox {
namespace sqlite {
Database::Database(const std::string &filename, int flags) {
const int err = sqlite3_open_v2(filename.c_str(), &db, flags, nullptr);
if (err != SQLITE_OK) {
Exception ex { err, sqlite3_errmsg(db) };
db = nullptr;
throw ex;
}
}
Database::Database(Database &&other)
: db(std::move(other.db)) {}
Database &Database::operator=(Database &&other) {
std::swap(db, other.db);
return *this;
}
Database::~Database() {
if (db) {
const int err = sqlite3_close(db);
if (err != SQLITE_OK) {
throw Exception { err, sqlite3_errmsg(db) };
}
}
}
Database::operator bool() const {
return db != nullptr;
}
void Database::exec(const std::string &sql) {
assert(db);
char *msg = nullptr;
const int err = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &msg);
if (msg) {
Exception ex { err, msg };
sqlite3_free(msg);
throw ex;
} else if (err != SQLITE_OK) {
throw Exception { err, sqlite3_errmsg(db) };
}
}
Statement Database::prepare(const char *query) {
assert(db);
return std::move(Statement(db, query));
}
Statement::Statement(sqlite3 *db, const char *sql) {
const int err = sqlite3_prepare_v2(db, sql, -1, &stmt, nullptr);
if (err != SQLITE_OK) {
stmt = nullptr;
throw Exception { err, sqlite3_errmsg(db) };
}
}
#define CHECK_SQLITE_OK(err) \
if (err != SQLITE_OK) { \
throw Exception { err, sqlite3_errmsg(sqlite3_db_handle(stmt)) }; \
}
Statement::Statement(Statement &&other) {
*this = std::move(other);
}
Statement &Statement::operator=(Statement &&other) {
std::swap(stmt, other.stmt);
return *this;
}
Statement::~Statement() {
if (stmt) {
const int err = sqlite3_finalize(stmt);
CHECK_SQLITE_OK(err)
}
}
Statement::operator bool() const {
return stmt != nullptr;
}
#define BIND_3(type, value) \
assert(stmt); \
const int err = sqlite3_bind_##type(stmt, offset, value); \
CHECK_SQLITE_OK(err)
#define BIND_5(type, value, length, param) \
assert(stmt); \
const int err = sqlite3_bind_##type(stmt, offset, value, length, param); \
CHECK_SQLITE_OK(err)
template <> void Statement::bind(int offset, int value) {
BIND_3(int, value)
}
template <> void Statement::bind(int offset, int64_t value) {
BIND_3(int64, value)
}
template <> void Statement::bind(int offset, double value) {
BIND_3(double, value)
}
template <> void Statement::bind(int offset, bool value) {
BIND_3(int, value)
}
template <> void Statement::bind(int offset, const char *value) {
BIND_5(text, value, -1, nullptr)
}
void Statement::bind(int offset, const std::string &value, bool retain) {
BIND_5(blob, value.data(), int(value.size()), retain ? SQLITE_TRANSIENT : SQLITE_STATIC)
}
bool Statement::run() {
assert(stmt);
const int err = sqlite3_step(stmt);
if (err == SQLITE_DONE) {
return false;
} else if (err == SQLITE_ROW) {
return true;
} else {
throw std::runtime_error("failed to run statement");
}
}
template <> int Statement::get(int offset) {
assert(stmt);
return sqlite3_column_int(stmt, offset);
}
template <> int64_t Statement::get(int offset) {
assert(stmt);
return sqlite3_column_int64(stmt, offset);
}
template <> double Statement::get(int offset) {
assert(stmt);
return sqlite3_column_double(stmt, offset);
}
template <> std::string Statement::get(int offset) {
assert(stmt);
return {
reinterpret_cast<const char *>(sqlite3_column_blob(stmt, offset)),
size_t(sqlite3_column_bytes(stmt, offset))
};
}
void Statement::reset() {
assert(stmt);
sqlite3_reset(stmt);
}
}
}

View File

@@ -1,74 +0,0 @@
#pragma once
#include <string>
#include <stdexcept>
typedef struct sqlite3 sqlite3;
typedef struct sqlite3_stmt sqlite3_stmt;
namespace mapbox {
namespace sqlite {
enum OpenFlag : int {
ReadOnly = 0x00000001,
ReadWrite = 0x00000002,
Create = 0x00000004,
NoMutex = 0x00008000,
FullMutex = 0x00010000,
SharedCache = 0x00020000,
PrivateCache = 0x00040000,
};
struct Exception : std::runtime_error {
inline Exception(int err, const char *msg) : std::runtime_error(msg), code(err) {}
const int code = 0;
};
class Statement;
class Database {
private:
Database(const Database &) = delete;
Database &operator=(const Database &) = delete;
public:
Database(const std::string &filename, int flags = 0);
Database(Database &&);
~Database();
Database &operator=(Database &&);
operator bool() const;
void exec(const std::string &sql);
Statement prepare(const char *query);
private:
sqlite3 *db = nullptr;
};
class Statement {
private:
Statement(const Statement &) = delete;
Statement &operator=(const Statement &) = delete;
public:
Statement(sqlite3 *db, const char *sql);
Statement(Statement &&);
~Statement();
Statement &operator=(Statement &&);
operator bool() const;
template <typename T> void bind(int offset, T value);
void bind(int offset, const std::string &value, bool retain = true);
template <typename T> T get(int offset);
bool run();
void reset();
private:
sqlite3_stmt *stmt = nullptr;
};
}
}