[core] Reimplement existing caching within an offline-capable database schema
4
.gitignore
vendored
@@ -23,8 +23,8 @@
|
||||
/test/fixtures/api/1.png
|
||||
/test/fixtures/api/2.png
|
||||
/test/fixtures/database/*.db
|
||||
/test/fixtures/*/*/actual.png
|
||||
/test/fixtures/*/*/diff.png
|
||||
/test/fixtures/**/actual.png
|
||||
/test/fixtures/**/diff.png
|
||||
/test/output
|
||||
/include/mbgl/shader/shaders.hpp
|
||||
/src/shader/shaders_gl.cpp
|
||||
|
||||
@@ -21,7 +21,8 @@
|
||||
'../platform/default/timer.cpp',
|
||||
'../platform/default/default_file_source.cpp',
|
||||
'../platform/default/online_file_source.cpp',
|
||||
'../platform/default/sqlite_cache.cpp',
|
||||
'../platform/default/mbgl/storage/offline_database.hpp',
|
||||
'../platform/default/mbgl/storage/offline_database.cpp',
|
||||
'../platform/default/sqlite3.hpp',
|
||||
'../platform/default/sqlite3.cpp',
|
||||
],
|
||||
@@ -56,6 +57,7 @@
|
||||
'include_dirs': [
|
||||
'../include',
|
||||
'../src',
|
||||
'../platform/default',
|
||||
],
|
||||
|
||||
'conditions': [
|
||||
|
||||
@@ -16,7 +16,8 @@
|
||||
'../platform/default/timer.cpp',
|
||||
'../platform/default/default_file_source.cpp',
|
||||
'../platform/default/online_file_source.cpp',
|
||||
'../platform/default/sqlite_cache.cpp',
|
||||
'../platform/default/mbgl/storage/offline_database.hpp',
|
||||
'../platform/default/mbgl/storage/offline_database.cpp',
|
||||
'../platform/default/sqlite3.hpp',
|
||||
'../platform/default/sqlite3.cpp',
|
||||
'../platform/darwin/log_nslog.mm',
|
||||
@@ -98,6 +99,7 @@
|
||||
'../include/mbgl/darwin',
|
||||
'../include',
|
||||
'../src',
|
||||
'../platform/default',
|
||||
],
|
||||
|
||||
'xcode_settings': {
|
||||
|
||||
@@ -23,7 +23,8 @@
|
||||
'../platform/default/timer.cpp',
|
||||
'../platform/default/default_file_source.cpp',
|
||||
'../platform/default/online_file_source.cpp',
|
||||
'../platform/default/sqlite_cache.cpp',
|
||||
'../platform/default/mbgl/storage/offline_database.hpp',
|
||||
'../platform/default/mbgl/storage/offline_database.cpp',
|
||||
'../platform/default/sqlite3.hpp',
|
||||
'../platform/default/sqlite3.cpp',
|
||||
],
|
||||
@@ -61,6 +62,7 @@
|
||||
'include_dirs': [
|
||||
'../include',
|
||||
'../src',
|
||||
'../platform/default',
|
||||
],
|
||||
|
||||
'conditions': [
|
||||
|
||||
@@ -15,7 +15,8 @@
|
||||
'../platform/default/timer.cpp',
|
||||
'../platform/default/default_file_source.cpp',
|
||||
'../platform/default/online_file_source.cpp',
|
||||
'../platform/default/sqlite_cache.cpp',
|
||||
'../platform/default/mbgl/storage/offline_database.hpp',
|
||||
'../platform/default/mbgl/storage/offline_database.cpp',
|
||||
'../platform/default/sqlite3.hpp',
|
||||
'../platform/default/sqlite3.cpp',
|
||||
'../platform/darwin/log_nslog.mm',
|
||||
@@ -83,6 +84,7 @@
|
||||
'../include/mbgl/darwin',
|
||||
'../include',
|
||||
'../src',
|
||||
'../platform/default',
|
||||
],
|
||||
|
||||
'xcode_settings': {
|
||||
|
||||
@@ -5,7 +5,9 @@
|
||||
|
||||
namespace mbgl {
|
||||
|
||||
class SQLiteCache;
|
||||
namespace util {
|
||||
template <typename T> class Thread;
|
||||
} // namespace util
|
||||
|
||||
class DefaultFileSource : public FileSource {
|
||||
public:
|
||||
@@ -20,14 +22,15 @@ public:
|
||||
|
||||
std::unique_ptr<FileRequest> request(const Resource&, Callback) override;
|
||||
|
||||
// For testing purposes only.
|
||||
SQLiteCache& getCache();
|
||||
// For testing only.
|
||||
void put(const Resource&, const Response&);
|
||||
void goOffline();
|
||||
|
||||
class Impl;
|
||||
|
||||
private:
|
||||
class Impl;
|
||||
friend class DefaultFileRequest;
|
||||
|
||||
const std::unique_ptr<Impl> impl;
|
||||
const std::unique_ptr<util::Thread<Impl>> thread;
|
||||
const std::unique_ptr<FileSource> assetFileSource;
|
||||
};
|
||||
|
||||
} // namespace mbgl
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
#include <mbgl/storage/default_file_source.hpp>
|
||||
#include <mbgl/storage/asset_file_source.hpp>
|
||||
#include <mbgl/storage/online_file_source.hpp>
|
||||
#include <mbgl/storage/sqlite_cache.hpp>
|
||||
#include <mbgl/storage/offline_database.hpp>
|
||||
|
||||
#include <mbgl/platform/platform.hpp>
|
||||
#include <mbgl/util/url.hpp>
|
||||
#include <mbgl/util/thread.hpp>
|
||||
#include <mbgl/util/work_request.hpp>
|
||||
|
||||
#include <cassert>
|
||||
@@ -23,76 +24,117 @@ namespace mbgl {
|
||||
|
||||
class DefaultFileSource::Impl {
|
||||
public:
|
||||
Impl(const std::string& cachePath, const std::string& assetRoot)
|
||||
: assetFileSource(assetRoot),
|
||||
cache(SQLiteCache::getShared(cachePath)) {
|
||||
class Task {
|
||||
public:
|
||||
Task(Resource resource, FileSource::Callback callback, DefaultFileSource::Impl* impl) {
|
||||
auto offlineResponse = impl->offlineDatabase.get(resource);
|
||||
|
||||
Resource revalidation = resource;
|
||||
|
||||
if (offlineResponse) {
|
||||
revalidation.priorModified = offlineResponse->modified;
|
||||
revalidation.priorExpires = offlineResponse->expires;
|
||||
revalidation.priorEtag = offlineResponse->etag;
|
||||
callback(*offlineResponse);
|
||||
}
|
||||
|
||||
if (!impl->offline) {
|
||||
onlineRequest = impl->onlineFileSource.request(revalidation, [=] (Response onlineResponse) {
|
||||
impl->offlineDatabase.put(revalidation, onlineResponse);
|
||||
callback(onlineResponse);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
std::unique_ptr<FileRequest> onlineRequest;
|
||||
};
|
||||
|
||||
Impl(const std::string& cachePath)
|
||||
: offlineDatabase(cachePath) {
|
||||
}
|
||||
|
||||
AssetFileSource assetFileSource;
|
||||
std::shared_ptr<SQLiteCache> cache;
|
||||
void setAccessToken(const std::string& accessToken) {
|
||||
onlineFileSource.setAccessToken(accessToken);
|
||||
}
|
||||
|
||||
std::string getAccessToken() const {
|
||||
return onlineFileSource.getAccessToken();
|
||||
}
|
||||
|
||||
void add(FileRequest* req, Resource resource, Callback callback) {
|
||||
tasks[req] = std::make_unique<Task>(resource, callback, this);
|
||||
}
|
||||
|
||||
void cancel(FileRequest* req) {
|
||||
tasks.erase(req);
|
||||
}
|
||||
|
||||
void put(const Resource& resource, const Response& response) {
|
||||
offlineDatabase.put(resource, response);
|
||||
}
|
||||
|
||||
void goOffline() {
|
||||
offline = true;
|
||||
}
|
||||
|
||||
OfflineDatabase offlineDatabase;
|
||||
OnlineFileSource onlineFileSource;
|
||||
std::unordered_map<FileRequest*, std::unique_ptr<Task>> tasks;
|
||||
bool offline = false;
|
||||
};
|
||||
|
||||
class DefaultFileRequest : public FileRequest {
|
||||
public:
|
||||
DefaultFileRequest(Resource resource, FileSource::Callback callback, util::Thread<DefaultFileSource::Impl>& thread_)
|
||||
: thread(thread_),
|
||||
workRequest(thread.invokeWithCallback(&DefaultFileSource::Impl::add, callback, this, resource)) {
|
||||
}
|
||||
|
||||
~DefaultFileRequest() {
|
||||
thread.invoke(&DefaultFileSource::Impl::cancel, this);
|
||||
}
|
||||
|
||||
util::Thread<DefaultFileSource::Impl>& thread;
|
||||
std::unique_ptr<WorkRequest> workRequest;
|
||||
};
|
||||
|
||||
DefaultFileSource::DefaultFileSource(const std::string& cachePath, const std::string& assetRoot)
|
||||
: impl(std::make_unique<DefaultFileSource::Impl>(cachePath, assetRoot)) {
|
||||
: thread(std::make_unique<util::Thread<DefaultFileSource::Impl>>(util::ThreadContext{"DefaultFileSource", util::ThreadType::Unknown, util::ThreadPriority::Low}, cachePath)),
|
||||
assetFileSource(std::make_unique<AssetFileSource>(assetRoot)) {
|
||||
}
|
||||
|
||||
DefaultFileSource::~DefaultFileSource() = default;
|
||||
|
||||
void DefaultFileSource::setAccessToken(const std::string& accessToken) {
|
||||
impl->onlineFileSource.setAccessToken(accessToken);
|
||||
thread->invokeSync(&Impl::setAccessToken, accessToken);
|
||||
}
|
||||
|
||||
std::string DefaultFileSource::getAccessToken() const {
|
||||
return impl->onlineFileSource.getAccessToken();
|
||||
return thread->invokeSync<std::string>(&Impl::getAccessToken);
|
||||
}
|
||||
|
||||
void DefaultFileSource::setMaximumCacheSize(uint64_t size) {
|
||||
impl->cache->setMaximumCacheSize(size);
|
||||
void DefaultFileSource::setMaximumCacheSize(uint64_t) {
|
||||
// TODO
|
||||
}
|
||||
|
||||
void DefaultFileSource::setMaximumCacheEntrySize(uint64_t size) {
|
||||
impl->cache->setMaximumCacheEntrySize(size);
|
||||
void DefaultFileSource::setMaximumCacheEntrySize(uint64_t) {
|
||||
// TODO
|
||||
}
|
||||
|
||||
SQLiteCache& DefaultFileSource::getCache() {
|
||||
return *impl->cache;
|
||||
}
|
||||
|
||||
class DefaultFileRequest : public FileRequest {
|
||||
public:
|
||||
DefaultFileRequest(Resource resource, FileSource::Callback callback, DefaultFileSource::Impl* impl) {
|
||||
cacheRequest = impl->cache->get(resource, [=](std::shared_ptr<Response> cacheResponse) mutable {
|
||||
cacheRequest.reset();
|
||||
|
||||
if (cacheResponse) {
|
||||
resource.priorModified = cacheResponse->modified;
|
||||
resource.priorExpires = cacheResponse->expires;
|
||||
resource.priorEtag = cacheResponse->etag;
|
||||
}
|
||||
|
||||
onlineRequest = impl->onlineFileSource.request(resource, [=] (Response onlineResponse) {
|
||||
impl->cache->put(resource, onlineResponse);
|
||||
callback(onlineResponse);
|
||||
});
|
||||
|
||||
// Do this last because it may result in deleting this DefaultFileRequest.
|
||||
if (cacheResponse) {
|
||||
callback(*cacheResponse);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
std::unique_ptr<WorkRequest> cacheRequest;
|
||||
std::unique_ptr<FileRequest> onlineRequest;
|
||||
};
|
||||
|
||||
std::unique_ptr<FileRequest> DefaultFileSource::request(const Resource& resource, Callback callback) {
|
||||
if (isAssetURL(resource.url)) {
|
||||
return impl->assetFileSource.request(resource, callback);
|
||||
return assetFileSource->request(resource, callback);
|
||||
} else {
|
||||
return std::make_unique<DefaultFileRequest>(resource, callback, impl.get());
|
||||
return std::make_unique<DefaultFileRequest>(resource, callback, *thread);
|
||||
}
|
||||
}
|
||||
|
||||
void DefaultFileSource::put(const Resource& resource, const Response& response) {
|
||||
thread->invokeSync(&Impl::put, resource, response);
|
||||
}
|
||||
|
||||
void DefaultFileSource::goOffline() {
|
||||
thread->invokeSync(&Impl::goOffline);
|
||||
}
|
||||
|
||||
} // namespace mbgl
|
||||
|
||||
297
platform/default/mbgl/storage/offline_database.cpp
Normal file
@@ -0,0 +1,297 @@
|
||||
#include <mbgl/storage/offline_database.hpp>
|
||||
#include <mbgl/storage/response.hpp>
|
||||
#include <mbgl/util/compression.hpp>
|
||||
#include <mbgl/util/io.hpp>
|
||||
#include <mbgl/util/string.hpp>
|
||||
#include <mbgl/util/chrono.hpp>
|
||||
#include <mbgl/map/tile_id.hpp>
|
||||
#include <mbgl/platform/log.hpp>
|
||||
|
||||
#include "sqlite3.hpp"
|
||||
#include <sqlite3.h>
|
||||
|
||||
namespace mbgl {
|
||||
|
||||
using namespace mapbox::sqlite;
|
||||
|
||||
// If you change the schema you must write a migration from the previous version.
|
||||
static const uint32_t schemaVersion = 2;
|
||||
|
||||
OfflineDatabase::OfflineDatabase(const std::string& path_)
|
||||
: path(path_) {
|
||||
ensureSchema();
|
||||
}
|
||||
|
||||
OfflineDatabase::~OfflineDatabase() {
|
||||
// Deleting these SQLite objects may result in exceptions, but we're in a destructor, so we
|
||||
// can't throw anything.
|
||||
try {
|
||||
statements.clear();
|
||||
db.reset();
|
||||
} catch (mapbox::sqlite::Exception& ex) {
|
||||
Log::Error(Event::Database, ex.code, ex.what());
|
||||
}
|
||||
}
|
||||
|
||||
void OfflineDatabase::ensureSchema() {
|
||||
if (path != ":memory:") {
|
||||
try {
|
||||
db = std::make_unique<Database>(path.c_str(), ReadWrite);
|
||||
|
||||
{
|
||||
Statement userVersionStmt(db->prepare("PRAGMA user_version"));
|
||||
userVersionStmt.run();
|
||||
switch (userVersionStmt.get<int>(0)) {
|
||||
case 0: break; // cache-only database; ok to delete
|
||||
case 1: break; // cache-only database; ok to delete
|
||||
case 2: return;
|
||||
default: throw std::runtime_error("unknown schema version");
|
||||
}
|
||||
}
|
||||
|
||||
removeExisting();
|
||||
db = std::make_unique<Database>(path.c_str(), ReadWrite | Create);
|
||||
} catch (mapbox::sqlite::Exception& ex) {
|
||||
if (ex.code == SQLITE_CANTOPEN) {
|
||||
db = std::make_unique<Database>(path.c_str(), ReadWrite | Create);
|
||||
} else if (ex.code == SQLITE_NOTADB) {
|
||||
removeExisting();
|
||||
db = std::make_unique<Database>(path.c_str(), ReadWrite | Create);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#include "offline_schema.cpp.include"
|
||||
|
||||
db = std::make_unique<Database>(path.c_str(), ReadWrite | Create);
|
||||
db->exec(schema);
|
||||
db->exec("PRAGMA user_version = " + util::toString(schemaVersion));
|
||||
}
|
||||
|
||||
void OfflineDatabase::removeExisting() {
|
||||
Log::Warning(Event::Database, "Removing existing incompatible offline database");
|
||||
|
||||
db.reset();
|
||||
|
||||
try {
|
||||
util::deleteFile(path);
|
||||
} catch (util::IOException& ex) {
|
||||
Log::Error(Event::Database, ex.code, ex.what());
|
||||
}
|
||||
}
|
||||
|
||||
mapbox::sqlite::Statement& OfflineDatabase::getStatement(const char * sql) {
|
||||
auto it = statements.find(sql);
|
||||
|
||||
if (it != statements.end()) {
|
||||
it->second->reset();
|
||||
return *it->second;
|
||||
}
|
||||
|
||||
return *(statements[sql] = std::make_unique<Statement>(db->prepare(sql)));
|
||||
}
|
||||
|
||||
optional<Response> OfflineDatabase::get(const Resource& resource) {
|
||||
if (resource.kind == Resource::Kind::Tile) {
|
||||
assert(resource.tileData);
|
||||
return getTile(*resource.tileData);
|
||||
} else {
|
||||
return getResource(resource);
|
||||
}
|
||||
}
|
||||
|
||||
void OfflineDatabase::put(const Resource& resource, const Response& response) {
|
||||
// Except for 404s, don't store errors in the cache.
|
||||
if (response.error && response.error->reason != Response::Error::Reason::NotFound) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (resource.kind == Resource::Kind::Tile) {
|
||||
assert(resource.tileData);
|
||||
putTile(*resource.tileData, response);
|
||||
} else {
|
||||
putResource(resource, response);
|
||||
}
|
||||
}
|
||||
|
||||
optional<Response> OfflineDatabase::getResource(const Resource& resource) {
|
||||
mapbox::sqlite::Statement& stmt = getStatement(
|
||||
// 0 1 2 3 4
|
||||
"SELECT etag, expires, modified, data, compressed "
|
||||
"FROM resources "
|
||||
"WHERE url = ?");
|
||||
|
||||
stmt.bind(1, resource.url);
|
||||
|
||||
if (!stmt.run()) {
|
||||
return {};
|
||||
}
|
||||
|
||||
Response response;
|
||||
|
||||
response.etag = stmt.get<optional<std::string>>(0);
|
||||
response.expires = stmt.get<optional<SystemTimePoint>>(1);
|
||||
response.modified = stmt.get<optional<SystemTimePoint>>(2);
|
||||
|
||||
optional<std::string> data = stmt.get<optional<std::string>>(3);
|
||||
if (!data) {
|
||||
response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound);
|
||||
} else if (stmt.get<int>(4)) {
|
||||
response.data = std::make_shared<std::string>(util::decompress(*data));
|
||||
} else {
|
||||
response.data = std::make_shared<std::string>(*data);
|
||||
}
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
void OfflineDatabase::putResource(const Resource& resource, const Response& response) {
|
||||
if (response.notModified) {
|
||||
mapbox::sqlite::Statement& stmt = getStatement(
|
||||
// 1 2 3
|
||||
"UPDATE resources SET accessed = ?, expires = ? WHERE url = ?");
|
||||
|
||||
stmt.bind(1, SystemClock::now());
|
||||
stmt.bind(2, response.expires);
|
||||
stmt.bind(3, resource.url);
|
||||
stmt.run();
|
||||
} else {
|
||||
mapbox::sqlite::Statement& stmt = getStatement(
|
||||
// 1 2 3 4 5 6 7 8
|
||||
"REPLACE INTO resources (url, kind, etag, expires, modified, accessed, data, compressed) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?)");
|
||||
|
||||
stmt.bind(1 /* url */, resource.url);
|
||||
stmt.bind(2 /* kind */, int(resource.kind));
|
||||
stmt.bind(3 /* etag */, response.etag);
|
||||
stmt.bind(4 /* expires */, response.expires);
|
||||
stmt.bind(5 /* modified */, response.modified);
|
||||
stmt.bind(6 /* accessed */, SystemClock::now());
|
||||
|
||||
std::string data;
|
||||
|
||||
if (response.error) { // Can only be NotFound
|
||||
stmt.bind(7 /* data */, nullptr);
|
||||
stmt.bind(8 /* compressed */, false);
|
||||
} else {
|
||||
data = util::compress(*response.data);
|
||||
if (data.size() < response.data->size()) {
|
||||
stmt.bind(7 /* data */, data, false); // do not retain the string internally.
|
||||
stmt.bind(8 /* compressed */, true);
|
||||
} else {
|
||||
stmt.bind(7 /* data */, *response.data, false); // do not retain the string internally.
|
||||
stmt.bind(8 /* compressed */, false);
|
||||
}
|
||||
}
|
||||
|
||||
stmt.run();
|
||||
}
|
||||
}
|
||||
|
||||
optional<Response> OfflineDatabase::getTile(const Resource::TileData& tile) {
|
||||
mapbox::sqlite::Statement& stmt = getStatement(
|
||||
// 0 1 2 3 4
|
||||
"SELECT etag, expires, modified, data, compressed "
|
||||
"FROM tilesets, tiles "
|
||||
"WHERE tilesets.url_template = ? " // 1
|
||||
"AND tilesets.pixel_ratio = ? " // 2
|
||||
"AND tiles.x = ? " // 3
|
||||
"AND tiles.y = ? " // 4
|
||||
"AND tiles.z = ? " // 5
|
||||
"AND tilesets.id = tiles.tileset_id ");
|
||||
|
||||
stmt.bind(1, tile.urlTemplate);
|
||||
stmt.bind(2, tile.pixelRatio);
|
||||
stmt.bind(3, tile.x);
|
||||
stmt.bind(4, tile.y);
|
||||
stmt.bind(5, tile.z);
|
||||
|
||||
if (!stmt.run()) {
|
||||
return {};
|
||||
}
|
||||
|
||||
Response response;
|
||||
|
||||
response.etag = stmt.get<optional<std::string>>(0);
|
||||
response.expires = stmt.get<optional<SystemTimePoint>>(1);
|
||||
response.modified = stmt.get<optional<SystemTimePoint>>(2);
|
||||
|
||||
optional<std::string> data = stmt.get<optional<std::string>>(3);
|
||||
if (!data) {
|
||||
response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound);
|
||||
} else if (stmt.get<int>(4)) {
|
||||
response.data = std::make_shared<std::string>(util::decompress(*data));
|
||||
} else {
|
||||
response.data = std::make_shared<std::string>(*data);
|
||||
}
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
void OfflineDatabase::putTile(const Resource::TileData& tile, const Response& response) {
|
||||
if (response.notModified) {
|
||||
mapbox::sqlite::Statement& stmt = getStatement(
|
||||
"UPDATE tiles SET accessed = ?1, expires = ?2 "
|
||||
"WHERE tileset_id = ( "
|
||||
" SELECT id FROM tilesets "
|
||||
" WHERE url_template = ?3 "
|
||||
" AND pixel_ratio = ?4) "
|
||||
"AND tiles.x = ?5 "
|
||||
"AND tiles.y = ?6 "
|
||||
"AND tiles.z = ?7 ");
|
||||
|
||||
stmt.bind(1, SystemClock::now());
|
||||
stmt.bind(2, response.expires);
|
||||
stmt.bind(3, tile.urlTemplate);
|
||||
stmt.bind(4, tile.pixelRatio);
|
||||
stmt.bind(5, tile.x);
|
||||
stmt.bind(6, tile.y);
|
||||
stmt.bind(7, tile.z);
|
||||
stmt.run();
|
||||
} else {
|
||||
mapbox::sqlite::Statement& stmt1 = getStatement(
|
||||
"REPLACE INTO tilesets (url_template, pixel_ratio) "
|
||||
"VALUES (?1, ?2) ");
|
||||
|
||||
stmt1.bind(1 /* url_template */, tile.urlTemplate);
|
||||
stmt1.bind(2 /* pixel_ratio */, tile.pixelRatio);
|
||||
stmt1.run();
|
||||
|
||||
mapbox::sqlite::Statement& stmt2 = getStatement(
|
||||
"REPLACE INTO tiles (tileset_id, x, y, z, modified, etag, expires, accessed, data, compressed) "
|
||||
"SELECT tilesets.id, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11 "
|
||||
"FROM tilesets "
|
||||
"WHERE url_template = ?1 "
|
||||
"AND pixel_ratio = ?2 ");
|
||||
|
||||
stmt2.bind(1 /* url_template */, tile.urlTemplate);
|
||||
stmt2.bind(2 /* pixel_ratio */, tile.pixelRatio);
|
||||
stmt2.bind(3 /* x */, tile.x);
|
||||
stmt2.bind(4 /* y */, tile.y);
|
||||
stmt2.bind(5 /* z */, tile.z);
|
||||
stmt2.bind(6 /* modified */, response.modified);
|
||||
stmt2.bind(7 /* etag */, response.etag);
|
||||
stmt2.bind(8 /* expires */, response.expires);
|
||||
stmt2.bind(9 /* accessed */, SystemClock::now());
|
||||
|
||||
std::string data;
|
||||
|
||||
if (response.error) { // Can only be NotFound
|
||||
stmt2.bind(10 /* data */, nullptr);
|
||||
stmt2.bind(11 /* compressed */, false);
|
||||
} else {
|
||||
data = util::compress(*response.data);
|
||||
if (data.size() < response.data->size()) {
|
||||
stmt2.bind(10 /* data */, data, false); // do not retain the string internally.
|
||||
stmt2.bind(11 /* compressed */, true);
|
||||
} else {
|
||||
stmt2.bind(10 /* data */, *response.data, false); // do not retain the string internally.
|
||||
stmt2.bind(11 /* compressed */, false);
|
||||
}
|
||||
}
|
||||
|
||||
stmt2.run();
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace mbgl
|
||||
50
platform/default/mbgl/storage/offline_database.hpp
Normal file
@@ -0,0 +1,50 @@
|
||||
#ifndef MBGL_OFFLINE_DATABASE
|
||||
#define MBGL_OFFLINE_DATABASE
|
||||
|
||||
#include <mbgl/storage/resource.hpp>
|
||||
#include <mbgl/util/noncopyable.hpp>
|
||||
#include <mbgl/util/optional.hpp>
|
||||
|
||||
#include <unordered_map>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
namespace mapbox {
|
||||
namespace sqlite {
|
||||
class Database;
|
||||
class Statement;
|
||||
}
|
||||
}
|
||||
|
||||
namespace mbgl {
|
||||
|
||||
class Response;
|
||||
class TileID;
|
||||
|
||||
class OfflineDatabase : private util::noncopyable {
|
||||
public:
|
||||
OfflineDatabase(const std::string& path);
|
||||
~OfflineDatabase();
|
||||
|
||||
optional<Response> get(const Resource&);
|
||||
void put(const Resource&, const Response&);
|
||||
|
||||
private:
|
||||
void ensureSchema();
|
||||
void removeExisting();
|
||||
mapbox::sqlite::Statement& getStatement(const char *);
|
||||
|
||||
optional<Response> getTile(const Resource::TileData&);
|
||||
void putTile(const Resource::TileData&, const Response&);
|
||||
|
||||
optional<Response> getResource(const Resource&);
|
||||
void putResource(const Resource&, const Response&);
|
||||
|
||||
const std::string path;
|
||||
std::unique_ptr<::mapbox::sqlite::Database> db;
|
||||
std::unordered_map<const char *, std::unique_ptr<::mapbox::sqlite::Statement>> statements;
|
||||
};
|
||||
|
||||
} // namespace mbgl
|
||||
|
||||
#endif
|
||||
51
platform/default/mbgl/storage/offline_schema.cpp.include
Normal file
@@ -0,0 +1,51 @@
|
||||
/* THIS IS A GENERATED FILE; EDIT offline_schema.sql INSTEAD */
|
||||
static const char * schema =
|
||||
"CREATE TABLE resources (\n"
|
||||
" url TEXT NOT NULL PRIMARY KEY,\n"
|
||||
" kind INTEGER NOT NULL,\n"
|
||||
" expires INTEGER,\n"
|
||||
" modified INTEGER,\n"
|
||||
" accessed INTEGER,\n"
|
||||
" etag TEXT,\n"
|
||||
" data BLOB,\n"
|
||||
" compressed INTEGER NOT NULL DEFAULT 0\n"
|
||||
");\n"
|
||||
"CREATE TABLE tilesets (\n"
|
||||
" id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,\n"
|
||||
" url_template TEXT NOT NULL,\n"
|
||||
" pixel_ratio INTEGER,\n"
|
||||
" UNIQUE (url_template, pixel_ratio)\n"
|
||||
");\n"
|
||||
"CREATE TABLE tiles (\n"
|
||||
" tileset_id INTEGER NOT NULL REFERENCES tilesets(id),\n"
|
||||
" z INTEGER NOT NULL,\n"
|
||||
" x INTEGER NOT NULL,\n"
|
||||
" y INTEGER NOT NULL,\n"
|
||||
" expires INTEGER,\n"
|
||||
" modified INTEGER,\n"
|
||||
" accessed INTEGER,\n"
|
||||
" etag TEXT,\n"
|
||||
" data BLOB,\n"
|
||||
" compressed INTEGER NOT NULL DEFAULT 0,\n"
|
||||
" PRIMARY KEY (tileset_id, z, x, y)\n"
|
||||
");\n"
|
||||
"CREATE TABLE regions (\n"
|
||||
" id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,\n"
|
||||
" definition TEXT NOT NULL,\n"
|
||||
" description BLOB\n"
|
||||
");\n"
|
||||
"CREATE TABLE region_resources (\n"
|
||||
" region_id INTEGER NOT NULL REFERENCES regions(id),\n"
|
||||
" resource_url TEXT NOT NULL REFERENCES resources(url),\n"
|
||||
" PRIMARY KEY (region_id, resource_url)\n"
|
||||
");\n"
|
||||
"CREATE TABLE region_tiles (\n"
|
||||
" region_id INTEGER NOT NULL REFERENCES regions(id),\n"
|
||||
" tileset_id INTEGER NOT NULL REFERENCES tilesets(id),\n"
|
||||
" z INTEGER NOT NULL,\n"
|
||||
" x INTEGER NOT NULL,\n"
|
||||
" y INTEGER NOT NULL,\n"
|
||||
" PRIMARY KEY (region_id, tileset_id, z, x, y),\n"
|
||||
" FOREIGN KEY (tileset_id, z, x, y) REFERENCES tiles (tileset_id, z, x, y)\n"
|
||||
");\n"
|
||||
;
|
||||
24
platform/default/mbgl/storage/offline_schema.js
Normal file
@@ -0,0 +1,24 @@
|
||||
// To regenerate:
|
||||
// (cd platform/default/mbgl/storage && node offline_schema.js)
|
||||
|
||||
var fs = require('fs');
|
||||
var readline = require('readline');
|
||||
|
||||
var lineReader = readline.createInterface({
|
||||
input: fs.createReadStream('offline_schema.sql')
|
||||
});
|
||||
|
||||
var lines = [
|
||||
"/* THIS IS A GENERATED FILE; EDIT offline_schema.sql INSTEAD */",
|
||||
"static const char * schema = ",
|
||||
];
|
||||
|
||||
lineReader
|
||||
.on('line', function (line) {
|
||||
line = line.replace(/ *--.*/, '');
|
||||
if (line) lines.push('"' + line + '\\n"');
|
||||
})
|
||||
.on('close', function () {
|
||||
lines.push(';\n');
|
||||
fs.writeFileSync('offline_schema.cpp.include', lines.join('\n'));
|
||||
});
|
||||
62
platform/default/mbgl/storage/offline_schema.sql
Normal file
@@ -0,0 +1,62 @@
|
||||
CREATE TABLE resources ( -- Generic table for style, source, sprite, and glyph resources.
|
||||
url TEXT NOT NULL PRIMARY KEY, -- Same schema as http_cache table.
|
||||
kind INTEGER NOT NULL,
|
||||
expires INTEGER,
|
||||
modified INTEGER,
|
||||
accessed INTEGER,
|
||||
etag TEXT,
|
||||
data BLOB, -- NULL if the response was a 404
|
||||
compressed INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE TABLE tilesets (
|
||||
id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
|
||||
url_template TEXT NOT NULL, -- As it would appear in TileJSON (but no support for host sharding).
|
||||
pixel_ratio INTEGER, -- If NULL, 1 is assumed for raster sources.
|
||||
UNIQUE (url_template, pixel_ratio) -- Capable of caching the same tileset at multiple resolutions.
|
||||
);
|
||||
|
||||
CREATE TABLE tiles (
|
||||
tileset_id INTEGER NOT NULL REFERENCES tilesets(id),
|
||||
z INTEGER NOT NULL, -- Fully abandon TMS coordinates in favor of ZXY.
|
||||
x INTEGER NOT NULL,
|
||||
y INTEGER NOT NULL,
|
||||
expires INTEGER,
|
||||
modified INTEGER,
|
||||
accessed INTEGER,
|
||||
etag TEXT,
|
||||
data BLOB, -- NULL if the response was a 404
|
||||
compressed INTEGER NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (tileset_id, z, x, y)
|
||||
);
|
||||
|
||||
CREATE TABLE regions (
|
||||
id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
|
||||
definition TEXT NOT NULL, -- JSON formatted definition of region. Regions may be of variant types:
|
||||
-- e.g. bbox and zoom range, route path, flyTo parameters, etc. Note that
|
||||
-- the set of tiles required for a region may span multiple sources.
|
||||
description BLOB -- User provided data in user-defined format
|
||||
);
|
||||
|
||||
CREATE TABLE region_resources (
|
||||
region_id INTEGER NOT NULL REFERENCES regions(id),
|
||||
resource_url TEXT NOT NULL REFERENCES resources(url),
|
||||
PRIMARY KEY (region_id, resource_url)
|
||||
);
|
||||
|
||||
CREATE TABLE region_tiles (
|
||||
region_id INTEGER NOT NULL REFERENCES regions(id),
|
||||
tileset_id INTEGER NOT NULL REFERENCES tilesets(id),
|
||||
z INTEGER NOT NULL,
|
||||
x INTEGER NOT NULL,
|
||||
y INTEGER NOT NULL,
|
||||
PRIMARY KEY (region_id, tileset_id, z, x, y),
|
||||
FOREIGN KEY (tileset_id, z, x, y) REFERENCES tiles (tileset_id, z, x, y)
|
||||
);
|
||||
|
||||
-- `region_resources` and `region_tiles` are used for deduplication and deletion logic.
|
||||
-- A row in `tiles` exists IFF one or more corresponding rows exist in `region_tiles`. If
|
||||
-- more than one corresponding row exists, it indicates multiple regions contain the tile, and
|
||||
-- storage for the tile is being deduplicated. When a region is deleted, corresponding rows in
|
||||
-- `region_tiles` must also be deleted, and then rows in `tiles` and `tilesets` without a
|
||||
-- corresponding `region_tiles` row must be deleted. Similarly for `resources` / `region_resources`.
|
||||
@@ -115,9 +115,19 @@ template <> void Statement::bind(int offset, std::nullptr_t) {
|
||||
check(sqlite3_bind_null(stmt, offset));
|
||||
}
|
||||
|
||||
template <> void Statement::bind(int offset, int value) {
|
||||
template <> void Statement::bind(int offset, int8_t value) {
|
||||
assert(stmt);
|
||||
check(sqlite3_bind_int(stmt, offset, value));
|
||||
check(sqlite3_bind_int64(stmt, offset, value));
|
||||
}
|
||||
|
||||
template <> void Statement::bind(int offset, int16_t value) {
|
||||
assert(stmt);
|
||||
check(sqlite3_bind_int64(stmt, offset, value));
|
||||
}
|
||||
|
||||
template <> void Statement::bind(int offset, int32_t value) {
|
||||
assert(stmt);
|
||||
check(sqlite3_bind_int64(stmt, offset, value));
|
||||
}
|
||||
|
||||
template <> void Statement::bind(int offset, int64_t value) {
|
||||
@@ -125,6 +135,26 @@ template <> void Statement::bind(int offset, int64_t value) {
|
||||
check(sqlite3_bind_int64(stmt, offset, value));
|
||||
}
|
||||
|
||||
template <> void Statement::bind(int offset, uint8_t value) {
|
||||
assert(stmt);
|
||||
check(sqlite3_bind_int64(stmt, offset, value));
|
||||
}
|
||||
|
||||
template <> void Statement::bind(int offset, uint16_t value) {
|
||||
assert(stmt);
|
||||
check(sqlite3_bind_int64(stmt, offset, value));
|
||||
}
|
||||
|
||||
template <> void Statement::bind(int offset, uint32_t value) {
|
||||
assert(stmt);
|
||||
check(sqlite3_bind_int64(stmt, offset, value));
|
||||
}
|
||||
|
||||
template <> void Statement::bind(int offset, float value) {
|
||||
assert(stmt);
|
||||
check(sqlite3_bind_double(stmt, offset, value));
|
||||
}
|
||||
|
||||
template <> void Statement::bind(int offset, double value) {
|
||||
assert(stmt);
|
||||
check(sqlite3_bind_double(stmt, offset, value));
|
||||
|
||||
@@ -1,502 +0,0 @@
|
||||
#include "sqlite_cache_impl.hpp"
|
||||
#include <mbgl/storage/resource.hpp>
|
||||
#include <mbgl/storage/response.hpp>
|
||||
|
||||
#include <mbgl/util/compression.hpp>
|
||||
#include <mbgl/util/io.hpp>
|
||||
#include <mbgl/util/string.hpp>
|
||||
#include <mbgl/util/thread.hpp>
|
||||
#include <mbgl/util/mapbox.hpp>
|
||||
#include <mbgl/platform/log.hpp>
|
||||
|
||||
#include "sqlite3.hpp"
|
||||
#include <sqlite3.h>
|
||||
|
||||
#include <unordered_map>
|
||||
#include <mutex>
|
||||
|
||||
namespace {
|
||||
|
||||
// The cache won't accept entries larger than this arbitrary size
|
||||
// and will silently discard request for adding them to the cache.
|
||||
// Large entries can cause the database to grow in disk size and
|
||||
// never shrink again.
|
||||
const uint64_t kMaximumCacheEntrySize = 5 * 1024 * 1024; // 5 MB
|
||||
|
||||
// Number of records we delete when we are close to the maximum
|
||||
// database size, if set. The current criteria is to prune
|
||||
// the least used entries based on `accessed` time.
|
||||
const int kPrunedEntriesLimit = 100;
|
||||
|
||||
} // namespace
|
||||
|
||||
namespace mbgl {
|
||||
|
||||
using namespace mapbox::sqlite;
|
||||
|
||||
SQLiteCache::SQLiteCache(const std::string& path_)
|
||||
: thread(std::make_unique<util::Thread<Impl>>(util::ThreadContext{"SQLiteCache", util::ThreadType::Unknown, util::ThreadPriority::Low}, path_)) {
|
||||
}
|
||||
|
||||
SQLiteCache::~SQLiteCache() = default;
|
||||
|
||||
SQLiteCache::Impl::Impl(const std::string& path_)
|
||||
: maximumCacheSize(0), // Unlimited
|
||||
maximumCacheEntrySize(kMaximumCacheEntrySize),
|
||||
path(path_) {
|
||||
}
|
||||
|
||||
SQLiteCache::Impl::~Impl() {
|
||||
// Deleting these SQLite objects may result in exceptions, but we're in a destructor, so we
|
||||
// can't throw anything.
|
||||
try {
|
||||
getStmt.reset();
|
||||
putStmt.reset();
|
||||
refreshStmt.reset();
|
||||
countStmt.reset();
|
||||
freeStmt.reset();
|
||||
pruneStmt.reset();
|
||||
accessedStmt.reset();
|
||||
db.reset();
|
||||
} catch (mapbox::sqlite::Exception& ex) {
|
||||
Log::Error(Event::Database, ex.code, ex.what());
|
||||
}
|
||||
}
|
||||
|
||||
void SQLiteCache::Impl::createDatabase() {
|
||||
db = std::make_unique<Database>(path.c_str(), ReadWrite | Create);
|
||||
}
|
||||
|
||||
int SQLiteCache::Impl::schemaVersion() const {
|
||||
// WARNING: Bump the version when changing the cache
|
||||
// scheme to force the table to be recreated.
|
||||
return 1;
|
||||
}
|
||||
|
||||
void SQLiteCache::Impl::createSchema() {
|
||||
constexpr const char *const sql = ""
|
||||
"CREATE TABLE IF NOT EXISTS `http_cache` ("
|
||||
" `url` TEXT PRIMARY KEY NOT NULL,"
|
||||
" `status` INTEGER NOT NULL," // The response status (Successful or Error).
|
||||
" `kind` INTEGER NOT NULL," // The kind of file.
|
||||
" `modified` INTEGER," // Timestamp when the file was last modified.
|
||||
" `etag` TEXT,"
|
||||
" `expires` INTEGER," // Timestamp when the server says the file expires.
|
||||
" `accessed` INTEGER," // Timestamp when the database record was last accessed.
|
||||
" `data` BLOB,"
|
||||
" `compressed` INTEGER NOT NULL DEFAULT 0" // Whether the data is compressed.
|
||||
");"
|
||||
"CREATE INDEX IF NOT EXISTS `http_cache_kind_idx` ON `http_cache` (`kind`);"
|
||||
"CREATE INDEX IF NOT EXISTS `http_cache_accessed_idx` ON `http_cache` (`accessed`);";
|
||||
|
||||
ensureSchemaVersion();
|
||||
|
||||
try {
|
||||
db->exec(sql);
|
||||
db->exec("PRAGMA user_version = " + util::toString(schemaVersion()));
|
||||
schema = true;
|
||||
} catch (mapbox::sqlite::Exception &ex) {
|
||||
if (ex.code == SQLITE_NOTADB) {
|
||||
Log::Warning(Event::Database, "Trashing invalid database");
|
||||
db.reset();
|
||||
try {
|
||||
util::deleteFile(path);
|
||||
} catch (util::IOException& ioEx) {
|
||||
Log::Error(Event::Database, ex.code, ex.what());
|
||||
}
|
||||
db = std::make_unique<Database>(path.c_str(), ReadWrite | Create);
|
||||
} else {
|
||||
Log::Error(Event::Database, ex.code, ex.what());
|
||||
}
|
||||
|
||||
// Creating the database table + index failed. That means there may already be one, likely
|
||||
// with different columns. Drop it and try to create a new one.
|
||||
db->exec("DROP TABLE IF EXISTS `http_cache`");
|
||||
db->exec(sql);
|
||||
db->exec("PRAGMA user_version = " + util::toString(schemaVersion()));
|
||||
}
|
||||
}
|
||||
|
||||
void SQLiteCache::Impl::ensureSchemaVersion() {
|
||||
try {
|
||||
Statement userVersionStmt(db->prepare("PRAGMA user_version"));
|
||||
if (userVersionStmt.run() && userVersionStmt.get<int>(0) == schemaVersion()) {
|
||||
return;
|
||||
}
|
||||
} catch (mapbox::sqlite::Exception& ex) {
|
||||
if (ex.code == SQLITE_NOTADB) {
|
||||
return;
|
||||
}
|
||||
|
||||
Log::Error(Event::Database, ex.code, ex.what());
|
||||
}
|
||||
|
||||
// Version mismatch, drop the table so it will
|
||||
// get recreated by `createSchema()`.
|
||||
try {
|
||||
db->exec("DROP TABLE IF EXISTS `http_cache`");
|
||||
} catch (mapbox::sqlite::Exception& ex) {
|
||||
Log::Error(Event::Database, ex.code, ex.what());
|
||||
}
|
||||
}
|
||||
|
||||
void SQLiteCache::setMaximumCacheSize(uint64_t size) {
|
||||
thread->invoke(&Impl::setMaximumCacheSize, size);
|
||||
}
|
||||
|
||||
void SQLiteCache::Impl::setMaximumCacheSize(uint64_t size) {
|
||||
maximumCacheSize = size;
|
||||
|
||||
// Unlimited.
|
||||
if (size == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
uint64_t lastSoftSize = cacheSoftSize();
|
||||
|
||||
// Keep pruning until we fit in the new
|
||||
// size limit.
|
||||
while (lastSoftSize > maximumCacheSize) {
|
||||
pruneEntries();
|
||||
|
||||
if (lastSoftSize != cacheSoftSize()) {
|
||||
lastSoftSize = cacheSoftSize();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (cacheHardSize() > size) {
|
||||
Log::Warning(mbgl::Event::Database,
|
||||
"Current cache hard size is bigger than the defined "
|
||||
"maximum size. Database won't get truncated.");
|
||||
}
|
||||
}
|
||||
|
||||
void SQLiteCache::setMaximumCacheEntrySize(uint64_t size) {
|
||||
thread->invoke(&Impl::setMaximumCacheEntrySize, size);
|
||||
}
|
||||
|
||||
void SQLiteCache::Impl::setMaximumCacheEntrySize(uint64_t size) {
|
||||
maximumCacheEntrySize = size;
|
||||
}
|
||||
|
||||
void SQLiteCache::Impl::initializeDatabase() {
|
||||
if (!db) {
|
||||
createDatabase();
|
||||
}
|
||||
|
||||
if (!schema) {
|
||||
createSchema();
|
||||
}
|
||||
}
|
||||
|
||||
int SQLiteCache::Impl::cachePageSize() {
|
||||
try {
|
||||
if (!pageSize) {
|
||||
Statement pageSizeStmt(db->prepare("PRAGMA page_size"));
|
||||
if (pageSizeStmt.run()) {
|
||||
pageSize = pageSizeStmt.get<int>(0);
|
||||
}
|
||||
}
|
||||
} catch (mapbox::sqlite::Exception& ex) {
|
||||
Log::Error(Event::Database, ex.code, ex.what());
|
||||
}
|
||||
|
||||
return pageSize;
|
||||
}
|
||||
|
||||
uint64_t SQLiteCache::Impl::cacheHardSize() {
|
||||
try {
|
||||
initializeDatabase();
|
||||
|
||||
if (!countStmt) {
|
||||
countStmt = std::make_unique<Statement>(db->prepare("PRAGMA page_count"));
|
||||
} else {
|
||||
countStmt->reset();
|
||||
}
|
||||
|
||||
if (countStmt->run()) {
|
||||
return cachePageSize() * countStmt->get<int>(0);
|
||||
}
|
||||
} catch (mapbox::sqlite::Exception& ex) {
|
||||
Log::Error(Event::Database, ex.code, ex.what());
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
uint64_t SQLiteCache::Impl::cacheSoftSize() {
|
||||
if (!softSizeDirty) {
|
||||
return softSize;
|
||||
}
|
||||
|
||||
try {
|
||||
initializeDatabase();
|
||||
|
||||
if (!freeStmt) {
|
||||
freeStmt = std::make_unique<Statement>(db->prepare("PRAGMA freelist_count"));
|
||||
} else {
|
||||
freeStmt->reset();
|
||||
}
|
||||
|
||||
uint64_t hardSize = cacheHardSize();
|
||||
if (!hardSize) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (freeStmt->run()) {
|
||||
return hardSize - cachePageSize() * freeStmt->get<int>(0);
|
||||
}
|
||||
|
||||
softSizeDirty = false;
|
||||
} catch (mapbox::sqlite::Exception& ex) {
|
||||
Log::Error(Event::Database, ex.code, ex.what());
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool SQLiteCache::Impl::needsPruning() {
|
||||
// SQLite database never shrinks in size unless we call VACCUM. We here
|
||||
// are monitoring the soft limit (i.e. number of free pages in the file)
|
||||
// and as it approaches to the hard limit (i.e. the actual file size) we
|
||||
// delete an arbitrary number of old cache entries.
|
||||
//
|
||||
// The free pages approach saves us from calling VACCUM or keeping a
|
||||
// running total, which can be costly. We need a buffer because pages can
|
||||
// get fragmented on the database.
|
||||
if (cacheSoftSize() + maximumCacheEntrySize * 2 < maximumCacheSize) {
|
||||
return false;
|
||||
} else {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
void SQLiteCache::Impl::pruneEntries() {
|
||||
if (!maximumCacheSize) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!needsPruning()) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
if (!pruneStmt) {
|
||||
pruneStmt = std::make_unique<Statement>(db->prepare(
|
||||
"DELETE FROM `http_cache` WHERE `rowid` IN (SELECT `rowid` FROM "
|
||||
// 1
|
||||
"`http_cache` ORDER BY `accessed` ASC LIMIT ?)"));
|
||||
} else {
|
||||
pruneStmt->reset();
|
||||
}
|
||||
|
||||
pruneStmt->bind(1, kPrunedEntriesLimit);
|
||||
|
||||
pruneStmt->run();
|
||||
softSizeDirty = true;
|
||||
} catch (mapbox::sqlite::Exception& ex) {
|
||||
Log::Error(Event::Database, ex.code, ex.what());
|
||||
}
|
||||
}
|
||||
|
||||
std::unique_ptr<WorkRequest> SQLiteCache::get(const Resource &resource, Callback callback) {
|
||||
// Can be called from any thread, but most likely from the file source thread.
|
||||
// Will try to load the URL from the SQLite database and call the callback when done.
|
||||
// Note that the callback is probably going to invoked from another thread, so the caller
|
||||
// must make sure that it can run in that thread.
|
||||
return thread->invokeWithCallback(&Impl::get, callback, resource);
|
||||
}
|
||||
|
||||
void SQLiteCache::Impl::get(const Resource &resource, Callback callback) {
|
||||
try {
|
||||
initializeDatabase();
|
||||
|
||||
if (!getStmt) {
|
||||
// Initialize the statement 0 1
|
||||
getStmt = std::make_unique<Statement>(db->prepare("SELECT `status`, `modified`, "
|
||||
// 2 3 4 5 1
|
||||
"`etag`, `expires`, `data`, `compressed` FROM `http_cache` WHERE `url` = ?"));
|
||||
} else {
|
||||
getStmt->reset();
|
||||
}
|
||||
|
||||
getStmt->bind(1, resource.url);
|
||||
if (getStmt->run()) {
|
||||
// There is data.
|
||||
auto response = std::make_unique<Response>();
|
||||
const auto status = getStmt->get<int>(0);
|
||||
if (status > 1) {
|
||||
// Status codes > 1 indicate an error
|
||||
response->error = std::make_unique<Response::Error>(Response::Error::Reason(status));
|
||||
}
|
||||
response->modified = getStmt->get<optional<SystemTimePoint>>(1);
|
||||
response->etag = getStmt->get<optional<std::string>>(2);
|
||||
response->expires = getStmt->get<optional<SystemTimePoint>>(3);
|
||||
response->data = std::make_shared<std::string>(getStmt->get<std::string>(4));
|
||||
if (getStmt->get<int>(5)) { // == compressed
|
||||
response->data = std::make_shared<std::string>(util::decompress(*response->data));
|
||||
}
|
||||
callback(std::move(response));
|
||||
} else {
|
||||
// There is no data.
|
||||
callback(nullptr);
|
||||
}
|
||||
|
||||
// We do an extra query for refreshing the last time
|
||||
// the record was accessed that can be costly and is only
|
||||
// worth doing if we are monitoring the database size.
|
||||
if (maximumCacheSize) {
|
||||
if (!accessedStmt) {
|
||||
accessedStmt = std::make_unique<Statement>(
|
||||
// 1 2
|
||||
db->prepare("UPDATE `http_cache` SET `accessed` = ? WHERE `url` = ?"));
|
||||
} else {
|
||||
accessedStmt->reset();
|
||||
}
|
||||
|
||||
accessedStmt->bind(1, SystemClock::now());
|
||||
accessedStmt->bind(2, resource.url);
|
||||
accessedStmt->run();
|
||||
}
|
||||
} catch (mapbox::sqlite::Exception& ex) {
|
||||
Log::Error(Event::Database, ex.code, ex.what());
|
||||
callback(nullptr);
|
||||
} catch (std::runtime_error& ex) {
|
||||
Log::Error(Event::Database, "%s", ex.what());
|
||||
callback(nullptr);
|
||||
}
|
||||
}
|
||||
|
||||
void SQLiteCache::put(const Resource& resource, const Response& response) {
|
||||
// Except for 404s, don't store errors in the cache.
|
||||
if (response.error && response.error->reason != Response::Error::Reason::NotFound) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (response.notModified) {
|
||||
thread->invoke(&Impl::refresh, resource, response.expires);
|
||||
} else {
|
||||
thread->invoke(&Impl::put, resource, response);
|
||||
}
|
||||
}
|
||||
|
||||
void SQLiteCache::Impl::put(const Resource& resource, const Response& response) {
|
||||
try {
|
||||
initializeDatabase();
|
||||
pruneEntries();
|
||||
|
||||
if (response.data) {
|
||||
auto entrySize = response.data->size();
|
||||
|
||||
if (entrySize > maximumCacheEntrySize) {
|
||||
Log::Warning(Event::Database, "Entry too big for caching.");
|
||||
return;
|
||||
}
|
||||
|
||||
if (maximumCacheSize && entrySize + cacheSoftSize() > maximumCacheSize) {
|
||||
Log::Warning(Event::Database, "Unable to make space for new entries.");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (!putStmt) {
|
||||
putStmt = std::make_unique<Statement>(db->prepare("REPLACE INTO `http_cache` ("
|
||||
// 1 2 3 4 5 6 7 8 9
|
||||
"`url`, `status`, `kind`, `modified`, `etag`, `expires`, `accessed`, `data`, `compressed`"
|
||||
") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)"));
|
||||
} else {
|
||||
putStmt->reset();
|
||||
}
|
||||
|
||||
putStmt->bind(1 /* url */, resource.url);
|
||||
if (response.error) {
|
||||
putStmt->bind(2 /* status */, int(response.error->reason));
|
||||
} else {
|
||||
putStmt->bind(2 /* status */, 1 /* success */);
|
||||
}
|
||||
putStmt->bind(3 /* kind */, int(resource.kind));
|
||||
putStmt->bind(4 /* modified */, response.modified);
|
||||
putStmt->bind(5 /* etag */, response.etag);
|
||||
putStmt->bind(6 /* expires */, response.expires);
|
||||
putStmt->bind(7 /* accessed */, SystemClock::now());
|
||||
|
||||
std::string data;
|
||||
if (resource.kind != Resource::SpriteImage && response.data) {
|
||||
// Do not compress images, since they are typically compressed already.
|
||||
data = util::compress(*response.data);
|
||||
}
|
||||
|
||||
if (!data.empty() && data.size() < response.data->size()) {
|
||||
// Store the compressed data when it is smaller than the original
|
||||
// uncompressed data.
|
||||
putStmt->bind(8 /* data */, data, false); // do not retain the string internally.
|
||||
putStmt->bind(9 /* compressed */, true);
|
||||
} else if (response.data) {
|
||||
putStmt->bind(8 /* data */, *response.data, false); // do not retain the string internally.
|
||||
putStmt->bind(9 /* compressed */, false);
|
||||
} else {
|
||||
putStmt->bind(8 /* data */, "", false);
|
||||
putStmt->bind(9 /* compressed */, false);
|
||||
}
|
||||
|
||||
putStmt->run();
|
||||
softSizeDirty = true;
|
||||
} catch (mapbox::sqlite::Exception& ex) {
|
||||
Log::Error(Event::Database, ex.code, ex.what());
|
||||
} catch (std::runtime_error& ex) {
|
||||
Log::Error(Event::Database, "%s", ex.what());
|
||||
}
|
||||
}
|
||||
|
||||
void SQLiteCache::Impl::refresh(const Resource& resource, optional<SystemTimePoint> expires) {
|
||||
try {
|
||||
initializeDatabase();
|
||||
|
||||
if (!refreshStmt) {
|
||||
refreshStmt = std::make_unique<Statement>(
|
||||
db->prepare("UPDATE `http_cache` SET "
|
||||
// 1 2 3
|
||||
"`accessed` = ?, `expires` = ? WHERE `url` = ?"));
|
||||
} else {
|
||||
refreshStmt->reset();
|
||||
}
|
||||
|
||||
refreshStmt->bind(1, SystemClock::now());
|
||||
refreshStmt->bind(2, expires);
|
||||
refreshStmt->bind(3, resource.url);
|
||||
refreshStmt->run();
|
||||
} catch (mapbox::sqlite::Exception& ex) {
|
||||
Log::Error(Event::Database, ex.code, ex.what());
|
||||
}
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
static std::mutex sharedMutex;
|
||||
static std::unordered_map<std::string, std::weak_ptr<SQLiteCache>> shared;
|
||||
|
||||
} // namespace
|
||||
|
||||
std::shared_ptr<SQLiteCache> SQLiteCache::getShared(const std::string &path) {
|
||||
std::lock_guard<std::mutex> lock(sharedMutex);
|
||||
|
||||
std::shared_ptr<SQLiteCache> cache;
|
||||
|
||||
auto it = shared.find(path);
|
||||
if (it != shared.end()) {
|
||||
cache = it->second.lock();
|
||||
if (!cache) {
|
||||
cache = std::make_shared<SQLiteCache>(path);
|
||||
it->second = cache;
|
||||
}
|
||||
} else {
|
||||
cache = std::make_shared<SQLiteCache>(path);
|
||||
shared.emplace(path, cache);
|
||||
}
|
||||
|
||||
return cache;
|
||||
}
|
||||
|
||||
} // namespace mbgl
|
||||
@@ -1,68 +0,0 @@
|
||||
#ifndef MBGL_STORAGE_DEFAULT_SQLITE_CACHE_IMPL
|
||||
#define MBGL_STORAGE_DEFAULT_SQLITE_CACHE_IMPL
|
||||
|
||||
#include <mbgl/storage/sqlite_cache.hpp>
|
||||
#include <mbgl/util/chrono.hpp>
|
||||
#include <mbgl/util/optional.hpp>
|
||||
|
||||
namespace mapbox {
|
||||
namespace sqlite {
|
||||
class Database;
|
||||
class Statement;
|
||||
}
|
||||
}
|
||||
|
||||
namespace mbgl {
|
||||
|
||||
class SQLiteCache::Impl {
|
||||
public:
|
||||
explicit Impl(const std::string &path = ":memory:");
|
||||
~Impl();
|
||||
|
||||
void setMaximumCacheSize(uint64_t size);
|
||||
void setMaximumCacheEntrySize(uint64_t size);
|
||||
|
||||
void get(const Resource&, Callback);
|
||||
void put(const Resource&, const Response&);
|
||||
void refresh(const Resource&, optional<SystemTimePoint> expires);
|
||||
|
||||
private:
|
||||
void initializeDatabase();
|
||||
|
||||
int cachePageSize();
|
||||
|
||||
uint64_t cacheHardSize();
|
||||
uint64_t cacheSoftSize();
|
||||
|
||||
uint64_t softSize = 0;
|
||||
bool softSizeDirty = true;
|
||||
|
||||
bool needsPruning();
|
||||
void pruneEntries();
|
||||
|
||||
void createDatabase();
|
||||
void createSchema();
|
||||
|
||||
int schemaVersion() const;
|
||||
void ensureSchemaVersion();
|
||||
|
||||
int pageSize = 0;
|
||||
|
||||
uint64_t maximumCacheSize;
|
||||
uint64_t maximumCacheEntrySize;
|
||||
|
||||
const std::string path;
|
||||
std::unique_ptr<::mapbox::sqlite::Database> db;
|
||||
std::unique_ptr<::mapbox::sqlite::Statement> getStmt;
|
||||
std::unique_ptr<::mapbox::sqlite::Statement> putStmt;
|
||||
std::unique_ptr<::mapbox::sqlite::Statement> refreshStmt;
|
||||
std::unique_ptr<::mapbox::sqlite::Statement> countStmt;
|
||||
std::unique_ptr<::mapbox::sqlite::Statement> freeStmt;
|
||||
std::unique_ptr<::mapbox::sqlite::Statement> pruneStmt;
|
||||
std::unique_ptr<::mapbox::sqlite::Statement> accessedStmt;
|
||||
bool schema = false;
|
||||
};
|
||||
|
||||
} // namespace mbgl
|
||||
|
||||
#endif
|
||||
@@ -1,44 +0,0 @@
|
||||
#ifndef MBGL_STORAGE_DEFAULT_SQLITE_CACHE
|
||||
#define MBGL_STORAGE_DEFAULT_SQLITE_CACHE
|
||||
|
||||
#include <mbgl/util/noncopyable.hpp>
|
||||
#include <mbgl/util/chrono.hpp>
|
||||
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
namespace mbgl {
|
||||
|
||||
class Resource;
|
||||
class Response;
|
||||
class WorkRequest;
|
||||
|
||||
namespace util {
|
||||
template <typename T> class Thread;
|
||||
} // namespace util
|
||||
|
||||
class SQLiteCache : private util::noncopyable {
|
||||
public:
|
||||
static std::shared_ptr<SQLiteCache> getShared(const std::string &path = ":memory:");
|
||||
|
||||
SQLiteCache(const std::string &path = ":memory:");
|
||||
~SQLiteCache();
|
||||
|
||||
void setMaximumCacheSize(uint64_t size);
|
||||
void setMaximumCacheEntrySize(uint64_t size);
|
||||
|
||||
using Callback = std::function<void(std::unique_ptr<Response>)>;
|
||||
|
||||
std::unique_ptr<WorkRequest> get(const Resource&, Callback);
|
||||
void put(const Resource&, const Response&);
|
||||
|
||||
class Impl;
|
||||
|
||||
private:
|
||||
const std::unique_ptr<util::Thread<Impl>> thread;
|
||||
};
|
||||
|
||||
} // namespace mbgl
|
||||
|
||||
#endif
|
||||
50
test/api/offline.cpp
Normal file
@@ -0,0 +1,50 @@
|
||||
#include "../fixtures/util.hpp"
|
||||
|
||||
#include <mbgl/platform/default/headless_display.hpp>
|
||||
#include <mbgl/platform/default/headless_view.hpp>
|
||||
#include <mbgl/storage/offline_database.hpp>
|
||||
#include <mbgl/storage/default_file_source.hpp>
|
||||
|
||||
#include <mbgl/platform/log.hpp>
|
||||
#include <mbgl/util/work_request.hpp>
|
||||
#include <mbgl/util/io.hpp>
|
||||
|
||||
using namespace mbgl;
|
||||
using namespace std::literals::chrono_literals;
|
||||
using namespace std::literals::string_literals;
|
||||
|
||||
namespace {
|
||||
|
||||
Response expiredItem(const std::string& path) {
|
||||
Response response;
|
||||
response.data = std::make_shared<std::string>(util::read_file("test/fixtures/"s + path));
|
||||
response.expires = SystemClock::from_time_t(0);
|
||||
return response;
|
||||
}
|
||||
|
||||
const std::string prefix = "http://127.0.0.1:3000";
|
||||
|
||||
}
|
||||
|
||||
auto display = std::make_shared<mbgl::HeadlessDisplay>();
|
||||
|
||||
TEST(API, Offline) {
|
||||
HeadlessView view(display, 1);
|
||||
DefaultFileSource fileSource(":memory:", ".");
|
||||
|
||||
fileSource.put(Resource::style(prefix + "/offline/style.json"), expiredItem("offline/style.json"));
|
||||
fileSource.put(Resource::source(prefix + "/offline/streets.json"), expiredItem("offline/streets.json"));
|
||||
fileSource.put(Resource::spriteJSON(prefix + "/offline/sprite", 1.0), expiredItem("offline/sprite.json"));
|
||||
fileSource.put(Resource::spriteImage(prefix + "/offline/sprite", 1.0), expiredItem("offline/sprite.png"));
|
||||
fileSource.put(Resource::tile(prefix + "/offline/{z}-{x}-{y}.vector.pbf", 1.0, 0, 0, 0), expiredItem("offline/0-0-0.vector.pbf"));
|
||||
fileSource.put(Resource::glyphs(prefix + "/offline/{fontstack}/{range}.pbf", "Helvetica", {0, 255}), expiredItem("offline/glyph.pbf"));
|
||||
fileSource.goOffline();
|
||||
|
||||
Map map(view, fileSource, MapMode::Still);
|
||||
map.setStyleURL(prefix + "/offline/style.json");
|
||||
|
||||
test::checkImage("test/fixtures/offline"s,
|
||||
test::render(map),
|
||||
0.0015,
|
||||
0.1);
|
||||
}
|
||||
BIN
test/fixtures/offline/expected.png
vendored
Normal file
|
After Width: | Height: | Size: 47 KiB |
|
Before Width: | Height: | Size: 3.3 KiB After Width: | Height: | Size: 3.3 KiB |
@@ -10,5 +10,5 @@
|
||||
"name": "Streets",
|
||||
"scheme": "xyz",
|
||||
"tilejson": "2.0.0",
|
||||
"tiles": [ "asset://test/fixtures/stale/{z}-{x}-{y}.vector.pbf" ]
|
||||
"tiles": [ "http://127.0.0.1:3000/offline/{z}-{x}-{y}.vector.pbf" ]
|
||||
}
|
||||
@@ -4,10 +4,11 @@
|
||||
"sources": {
|
||||
"mapbox": {
|
||||
"type": "vector",
|
||||
"url": "asset://test/fixtures/stale/streets.json"
|
||||
"url": "http://127.0.0.1:3000/offline/streets.json"
|
||||
}
|
||||
},
|
||||
"glyphs": "http://127.0.0.1:3000/stale/{fontstack}/{range}.pbf",
|
||||
"glyphs": "http://127.0.0.1:3000/offline/{fontstack}/{range}.pbf",
|
||||
"sprite": "http://127.0.0.1:3000/offline/sprite",
|
||||
"layers": [{
|
||||
"id": "background",
|
||||
"type": "background",
|
||||
@@ -20,7 +21,7 @@
|
||||
"source": "mapbox",
|
||||
"source-layer": "water",
|
||||
"paint": {
|
||||
"fill-color": "blue"
|
||||
"fill-pattern": "noise"
|
||||
}
|
||||
}, {
|
||||
"id": "admin",
|
||||
@@ -28,7 +29,8 @@
|
||||
"source": "mapbox",
|
||||
"source-layer": "admin",
|
||||
"layout": {
|
||||
"text-field": "Text"
|
||||
"text-font": ["Helvetica"],
|
||||
"text-field": "Text"
|
||||
},
|
||||
"paint": {
|
||||
"text-color": "black",
|
||||
BIN
test/fixtures/stale/stale_style/expected.png
vendored
|
Before Width: | Height: | Size: 13 KiB |
|
Before Width: | Height: | Size: 35 KiB |
|
Before Width: | Height: | Size: 26 KiB |
|
Before Width: | Height: | Size: 13 KiB |
25
test/fixtures/stale/style.json
vendored
@@ -1,25 +0,0 @@
|
||||
{
|
||||
"version": 8,
|
||||
"name": "Water",
|
||||
"sources": {
|
||||
"mapbox": {
|
||||
"type": "vector",
|
||||
"url": "asset://test/fixtures/stale/streets.json"
|
||||
}
|
||||
},
|
||||
"layers": [{
|
||||
"id": "background",
|
||||
"type": "background",
|
||||
"paint": {
|
||||
"background-color": "red"
|
||||
}
|
||||
}, {
|
||||
"id": "water",
|
||||
"type": "fill",
|
||||
"source": "mapbox",
|
||||
"source-layer": "water",
|
||||
"paint": {
|
||||
"fill-color": "blue"
|
||||
}
|
||||
}]
|
||||
}
|
||||
26
test/fixtures/stale/style_and_sprite.json
vendored
@@ -1,26 +0,0 @@
|
||||
{
|
||||
"version": 8,
|
||||
"name": "Water",
|
||||
"sources": {
|
||||
"mapbox": {
|
||||
"type": "vector",
|
||||
"url": "asset://test/fixtures/stale/streets.json"
|
||||
}
|
||||
},
|
||||
"sprite": "http://127.0.0.1:3000/stale/sprite",
|
||||
"layers": [{
|
||||
"id": "background",
|
||||
"type": "background",
|
||||
"paint": {
|
||||
"background-color": "red"
|
||||
}
|
||||
}, {
|
||||
"id": "water",
|
||||
"type": "fill",
|
||||
"source": "mapbox",
|
||||
"source-layer": "water",
|
||||
"paint": {
|
||||
"fill-pattern": "noise"
|
||||
}
|
||||
}]
|
||||
}
|
||||
25
test/fixtures/stale/style_and_tilejson.json
vendored
@@ -1,25 +0,0 @@
|
||||
{
|
||||
"version": 8,
|
||||
"name": "Water",
|
||||
"sources": {
|
||||
"mapbox": {
|
||||
"type": "vector",
|
||||
"url": "http://127.0.0.1:3000/stale/streets.json"
|
||||
}
|
||||
},
|
||||
"layers": [{
|
||||
"id": "background",
|
||||
"type": "background",
|
||||
"paint": {
|
||||
"background-color": "red"
|
||||
}
|
||||
}, {
|
||||
"id": "water",
|
||||
"type": "fill",
|
||||
"source": "mapbox",
|
||||
"source-layer": "water",
|
||||
"paint": {
|
||||
"fill-color": "blue"
|
||||
}
|
||||
}]
|
||||
}
|
||||
12
test/fixtures/util.cpp
vendored
@@ -106,20 +106,12 @@ uint64_t crc64(const PremultipliedImage &image) {
|
||||
return crc64(reinterpret_cast<const char*>(image.data.get()), image.size());
|
||||
}
|
||||
|
||||
PremultipliedImage render(Map& map, Milliseconds timeout) {
|
||||
PremultipliedImage render(Map& map) {
|
||||
std::promise<PremultipliedImage> promise;
|
||||
map.renderStill([&](std::exception_ptr, PremultipliedImage&& image) {
|
||||
promise.set_value(std::move(image));
|
||||
});
|
||||
|
||||
// Limit maximum wait time.
|
||||
auto future = promise.get_future();
|
||||
if (future.wait_for(timeout) != std::future_status::ready) {
|
||||
// Alas, we didn't get the promised future :(
|
||||
Log::Error(Event::Image, "Failed to generate image within %dms", timeout.count());
|
||||
return { 0, 0 };
|
||||
}
|
||||
return future.get();
|
||||
return promise.get_future().get();
|
||||
}
|
||||
|
||||
void checkImage(const std::string& base,
|
||||
|
||||
4
test/fixtures/util.hpp
vendored
@@ -4,7 +4,6 @@
|
||||
#include <mbgl/util/image.hpp>
|
||||
#include <mbgl/util/chrono.hpp>
|
||||
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
@@ -36,12 +35,13 @@ uint64_t crc64(const char*, size_t);
|
||||
uint64_t crc64(const std::string&);
|
||||
uint64_t crc64(const PremultipliedImage&);
|
||||
|
||||
PremultipliedImage render(Map&, Milliseconds timeout = Milliseconds(1000));
|
||||
PremultipliedImage render(Map&);
|
||||
|
||||
void checkImage(const std::string& base,
|
||||
const PremultipliedImage& actual,
|
||||
double imageThreshold = 0,
|
||||
double pixelThreshold = 0);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
#include "storage.hpp"
|
||||
|
||||
#include <mbgl/storage/asset_file_source.hpp>
|
||||
#include <mbgl/storage/sqlite_cache.hpp>
|
||||
#include <mbgl/platform/platform.hpp>
|
||||
#include <mbgl/util/chrono.hpp>
|
||||
#include <mbgl/util/run_loop.hpp>
|
||||
|
||||
@@ -1,215 +0,0 @@
|
||||
#include "storage.hpp"
|
||||
|
||||
#include <mbgl/storage/default_file_source.hpp>
|
||||
#include <mbgl/storage/sqlite_cache.hpp>
|
||||
#include <mbgl/util/chrono.hpp>
|
||||
#include <mbgl/util/run_loop.hpp>
|
||||
|
||||
TEST_F(Storage, CacheResponse) {
|
||||
SCOPED_TEST(CacheResponse);
|
||||
|
||||
using namespace mbgl;
|
||||
|
||||
util::RunLoop loop;
|
||||
DefaultFileSource fs(":memory:", ".");
|
||||
|
||||
const Resource resource { Resource::Unknown, "http://127.0.0.1:3000/cache" };
|
||||
Response response;
|
||||
|
||||
std::unique_ptr<FileRequest> req1;
|
||||
std::unique_ptr<FileRequest> req2;
|
||||
|
||||
req1 = fs.request(resource, [&](Response res) {
|
||||
req1.reset();
|
||||
EXPECT_EQ(nullptr, res.error);
|
||||
ASSERT_TRUE(res.data.get());
|
||||
EXPECT_EQ("Response 1", *res.data);
|
||||
EXPECT_TRUE(bool(res.expires));
|
||||
EXPECT_FALSE(bool(res.modified));
|
||||
EXPECT_FALSE(bool(res.etag));
|
||||
response = res;
|
||||
|
||||
// Now test that we get the same values as in the previous request. If we'd go to the server
|
||||
// again, we'd get different values.
|
||||
req2 = fs.request(resource, [&](Response res2) {
|
||||
req2.reset();
|
||||
EXPECT_EQ(response.error, res2.error);
|
||||
ASSERT_TRUE(res2.data.get());
|
||||
EXPECT_EQ(*response.data, *res2.data);
|
||||
EXPECT_EQ(response.expires, res2.expires);
|
||||
EXPECT_EQ(response.modified, res2.modified);
|
||||
EXPECT_EQ(response.etag, res2.etag);
|
||||
|
||||
loop.stop();
|
||||
CacheResponse.finish();
|
||||
});
|
||||
});
|
||||
|
||||
loop.run();
|
||||
}
|
||||
|
||||
// Make sure we /do/ store 404 Not Found responses into the cache
|
||||
TEST_F(Storage, CacheNotFound) {
|
||||
SCOPED_TEST(CacheNotFound);
|
||||
|
||||
using namespace mbgl;
|
||||
|
||||
util::RunLoop loop;
|
||||
DefaultFileSource fs(":memory:", ".");
|
||||
|
||||
const Resource resource{ Resource::Unknown, "http://127.0.0.1:3000/not-found" };
|
||||
|
||||
// Insert existing data into the cache that will be marked as stale.
|
||||
Response response;
|
||||
response.data = std::make_shared<const std::string>("existing data");
|
||||
fs.getCache().put(resource, response);
|
||||
|
||||
std::unique_ptr<FileRequest> req1;
|
||||
std::unique_ptr<WorkRequest> req2;
|
||||
|
||||
int counter = 0;
|
||||
|
||||
// Then, request the actual URL and check that we're getting the rigged cache response first,
|
||||
// then the connection error message.
|
||||
req1 = fs.request(resource, [&](Response res) {
|
||||
if (counter == 0) {
|
||||
EXPECT_EQ(nullptr, res.error);
|
||||
ASSERT_TRUE(res.data.get());
|
||||
EXPECT_EQ("existing data", *res.data);
|
||||
EXPECT_FALSE(bool(res.expires));
|
||||
EXPECT_FALSE(bool(res.modified));
|
||||
EXPECT_FALSE(bool(res.etag));
|
||||
} else if (counter == 1) {
|
||||
EXPECT_NE(nullptr, res.error);
|
||||
EXPECT_EQ(Response::Error::Reason::NotFound, res.error->reason);
|
||||
ASSERT_TRUE(res.data.get());
|
||||
EXPECT_EQ("Not Found!", *res.data);
|
||||
req1.reset();
|
||||
|
||||
// Finally, check the cache to make sure we cached the 404 response.
|
||||
req2 = fs.getCache().get(resource, [&](std::unique_ptr<Response> res2) {
|
||||
EXPECT_NE(nullptr, res2->error);
|
||||
EXPECT_EQ(Response::Error::Reason::NotFound, res2->error->reason);
|
||||
ASSERT_TRUE(res2->data.get());
|
||||
EXPECT_EQ("Not Found!", *res2->data);
|
||||
CacheNotFound.finish();
|
||||
loop.stop();
|
||||
});
|
||||
} else {
|
||||
FAIL() << "Got too many responses";
|
||||
}
|
||||
counter++;
|
||||
});
|
||||
|
||||
loop.run();
|
||||
}
|
||||
|
||||
// Make sure we don't store a connection error into the cache
|
||||
TEST_F(Storage, DontCacheConnectionErrors) {
|
||||
SCOPED_TEST(DontCacheConnectionErrors);
|
||||
|
||||
using namespace mbgl;
|
||||
|
||||
util::RunLoop loop;
|
||||
DefaultFileSource fs(":memory:", ".");
|
||||
|
||||
const Resource resource{ Resource::Unknown, "http://127.0.0.1:3001" };
|
||||
|
||||
// Insert existing data into the cache that will be marked as stale.
|
||||
Response response;
|
||||
response.data = std::make_shared<const std::string>("existing data");
|
||||
fs.getCache().put(resource, response);
|
||||
|
||||
std::unique_ptr<FileRequest> req1;
|
||||
std::unique_ptr<WorkRequest> req2;
|
||||
|
||||
int counter = 0;
|
||||
|
||||
// Then, request the actual URL and check that we're getting the rigged cache response first,
|
||||
// then the connection error message.
|
||||
req1 = fs.request(resource, [&](Response res) {
|
||||
if (counter == 0) {
|
||||
EXPECT_EQ(nullptr, res.error);
|
||||
ASSERT_TRUE(res.data.get());
|
||||
EXPECT_EQ("existing data", *res.data);
|
||||
EXPECT_FALSE(bool(res.expires));
|
||||
EXPECT_FALSE(bool(res.modified));
|
||||
EXPECT_FALSE(bool(res.etag));
|
||||
} else if (counter == 1) {
|
||||
EXPECT_NE(nullptr, res.error);
|
||||
EXPECT_EQ(Response::Error::Reason::Connection, res.error->reason);
|
||||
req1.reset();
|
||||
|
||||
// Finally, check the cache to make sure we still have our original data in there rather
|
||||
// than the failed connection attempt.
|
||||
req2 = fs.getCache().get(resource, [&](std::unique_ptr<Response> res2) {
|
||||
EXPECT_EQ(nullptr, res2->error);
|
||||
ASSERT_TRUE(res2->data.get());
|
||||
EXPECT_EQ("existing data", *res2->data);
|
||||
DontCacheConnectionErrors.finish();
|
||||
loop.stop();
|
||||
});
|
||||
} else {
|
||||
FAIL() << "Got too many responses";
|
||||
}
|
||||
counter++;
|
||||
});
|
||||
|
||||
loop.run();
|
||||
}
|
||||
|
||||
// Make sure we don't store a bad server response into the cache
|
||||
TEST_F(Storage, DontCacheServerErrors) {
|
||||
SCOPED_TEST(DontCacheServerErrors);
|
||||
|
||||
using namespace mbgl;
|
||||
|
||||
util::RunLoop loop;
|
||||
DefaultFileSource fs(":memory:", ".");
|
||||
|
||||
const Resource resource{ Resource::Unknown, "http://127.0.0.1:3000/permanent-error" };
|
||||
|
||||
// Insert existing data into the cache that will be marked as stale.
|
||||
Response response;
|
||||
response.data = std::make_shared<const std::string>("existing data");
|
||||
fs.getCache().put(resource, response);
|
||||
|
||||
std::unique_ptr<FileRequest> req1;
|
||||
std::unique_ptr<WorkRequest> req2;
|
||||
|
||||
int counter = 0;
|
||||
|
||||
// Then, request the actual URL and check that we're getting the rigged cache response first,
|
||||
// then the server error message.
|
||||
req1 = fs.request(resource, [&](Response res) {
|
||||
if (counter == 0) {
|
||||
EXPECT_EQ(nullptr, res.error);
|
||||
ASSERT_TRUE(res.data.get());
|
||||
EXPECT_EQ("existing data", *res.data);
|
||||
EXPECT_FALSE(bool(res.expires));
|
||||
EXPECT_FALSE(bool(res.modified));
|
||||
EXPECT_FALSE(bool(res.etag));
|
||||
} else if (counter == 1) {
|
||||
EXPECT_NE(nullptr, res.error);
|
||||
EXPECT_EQ(Response::Error::Reason::Server, res.error->reason);
|
||||
ASSERT_TRUE(res.data.get());
|
||||
EXPECT_EQ("Server Error!", *res.data);
|
||||
req1.reset();
|
||||
|
||||
// Finally, check the cache to make sure we still have our original data in there rather
|
||||
// than the failed connection attempt.
|
||||
req2 = fs.getCache().get(resource, [&](std::unique_ptr<Response> res2) {
|
||||
EXPECT_EQ(nullptr, res2->error);
|
||||
ASSERT_TRUE(res2->data.get());
|
||||
EXPECT_EQ("existing data", *res2->data);
|
||||
DontCacheServerErrors.finish();
|
||||
loop.stop();
|
||||
});
|
||||
} else {
|
||||
FAIL() << "Got too many responses";
|
||||
}
|
||||
counter++;
|
||||
});
|
||||
|
||||
loop.run();
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
#include "storage.hpp"
|
||||
|
||||
#include <mbgl/storage/sqlite_cache.hpp>
|
||||
#include <mbgl/storage/resource.hpp>
|
||||
#include <mbgl/util/run_loop.hpp>
|
||||
|
||||
TEST_F(Storage, CacheShared) {
|
||||
SCOPED_TEST(CacheShared)
|
||||
using namespace mbgl;
|
||||
|
||||
util::RunLoop loop;
|
||||
|
||||
// Check that we're getting two different caches when we request different paths.
|
||||
auto memory = SQLiteCache::getShared();
|
||||
auto file = SQLiteCache::getShared("test/fixtures/database/cache.db");
|
||||
EXPECT_NE(memory.get(), file.get());
|
||||
EXPECT_EQ(memory.get(), SQLiteCache::getShared().get());
|
||||
|
||||
// Store a response into the memory file cache, then delete the last reference.
|
||||
const Resource resource { Resource::Kind::Unknown, "http://example.com" };
|
||||
memory->put(resource, Response());
|
||||
memory.reset();
|
||||
|
||||
// Now check that the original memory file cache has been destructed and that it doesn't contain
|
||||
// the information we put into it.
|
||||
memory = SQLiteCache::getShared();
|
||||
auto req = memory->get(resource, [&](std::unique_ptr<Response> res) {
|
||||
EXPECT_FALSE(res.get());
|
||||
CacheShared.finish();
|
||||
loop.stop();
|
||||
});
|
||||
|
||||
loop.run();
|
||||
}
|
||||
@@ -1,241 +0,0 @@
|
||||
#include "storage.hpp"
|
||||
|
||||
#include <mbgl/storage/resource.hpp>
|
||||
#include <mbgl/storage/response.hpp>
|
||||
#include <mbgl/storage/sqlite_cache.hpp>
|
||||
#include <mbgl/util/run_loop.hpp>
|
||||
#include <mbgl/util/string.hpp>
|
||||
#include <mbgl/util/timer.hpp>
|
||||
#include <mbgl/util/chrono.hpp>
|
||||
|
||||
#include <memory>
|
||||
#include <random>
|
||||
|
||||
bool tileIsCached(mbgl::SQLiteCache* cache, unsigned id) {
|
||||
using namespace mbgl;
|
||||
|
||||
auto url = std::string("http://tile") + mbgl::util::toString(id);
|
||||
bool replied = false;
|
||||
|
||||
std::unique_ptr<Response> response;
|
||||
auto callback = [&] (std::unique_ptr<Response> res) {
|
||||
replied = true;
|
||||
response = std::move(res);
|
||||
};
|
||||
|
||||
Resource resource{ Resource::Kind::Tile, url };
|
||||
auto req = cache->get(resource, callback);
|
||||
|
||||
while (!replied) {
|
||||
util::RunLoop::Get()->runOnce();
|
||||
}
|
||||
|
||||
return response != nullptr;
|
||||
}
|
||||
|
||||
void insertTile(mbgl::SQLiteCache* cache, unsigned id, uint64_t size) {
|
||||
using namespace mbgl;
|
||||
|
||||
auto url = std::string("http://tile") + mbgl::util::toString(id);
|
||||
|
||||
Response response;
|
||||
response.modified = SystemClock::now();
|
||||
response.expires = SystemClock::now() + Seconds(5000);
|
||||
response.etag = url;
|
||||
|
||||
auto data = std::make_shared<std::string>(size, 0);
|
||||
|
||||
// Fill data with garbage so SQLite won't try to
|
||||
// optimize allocation by reusing pages.
|
||||
static std::mt19937 generator;
|
||||
std::generate_n(data->begin(), size, generator);
|
||||
|
||||
response.data = data;
|
||||
|
||||
Resource resource{ Resource::Kind::Tile, url };
|
||||
cache->put(resource, response);
|
||||
}
|
||||
|
||||
void refreshTile(mbgl::SQLiteCache* cache, unsigned id) {
|
||||
using namespace mbgl;
|
||||
|
||||
auto url = std::string("http://tile") + mbgl::util::toString(id);
|
||||
|
||||
Response response;
|
||||
response.modified = SystemClock::now();
|
||||
response.expires = SystemClock::now() + Seconds(5000);
|
||||
response.notModified = true;
|
||||
|
||||
Resource resource{ Resource::Kind::Tile, url };
|
||||
cache->put(resource, response);
|
||||
}
|
||||
|
||||
uint64_t cacheSize(mbgl::SQLiteCache* cache, unsigned entryCount, uint64_t entrySize) {
|
||||
uint64_t total = 0;
|
||||
|
||||
for (unsigned i = 0; i < entryCount; ++i) {
|
||||
if (tileIsCached(cache, i)) {
|
||||
total += entrySize;
|
||||
}
|
||||
}
|
||||
|
||||
return total;
|
||||
}
|
||||
|
||||
TEST_F(Storage, CacheEntrySizeLimit) {
|
||||
using namespace mbgl;
|
||||
|
||||
util::RunLoop loop;
|
||||
SQLiteCache cache(":memory:");
|
||||
|
||||
const uint64_t entrySize = 5 * 1024 * 1024; // 5 MB
|
||||
|
||||
insertTile(&cache, 0, entrySize);
|
||||
EXPECT_TRUE(tileIsCached(&cache, 0));
|
||||
|
||||
insertTile(&cache, 1, entrySize + 1);
|
||||
EXPECT_FALSE(tileIsCached(&cache, 1));
|
||||
|
||||
insertTile(&cache, 2, entrySize - 1);
|
||||
EXPECT_TRUE(tileIsCached(&cache, 2));
|
||||
|
||||
// Setting a new size should not delete existing entries.
|
||||
cache.setMaximumCacheEntrySize(entrySize / 2);
|
||||
EXPECT_TRUE(tileIsCached(&cache, 2));
|
||||
|
||||
insertTile(&cache, 3, entrySize / 2 - 1);
|
||||
EXPECT_TRUE(tileIsCached(&cache, 3));
|
||||
|
||||
insertTile(&cache, 4, entrySize);
|
||||
EXPECT_FALSE(tileIsCached(&cache, 4));
|
||||
|
||||
cache.setMaximumCacheEntrySize(entrySize * 2);
|
||||
insertTile(&cache, 5, entrySize);
|
||||
EXPECT_TRUE(tileIsCached(&cache, 5));
|
||||
}
|
||||
|
||||
TEST_F(Storage, CacheSizeSetNewLimit) {
|
||||
using namespace mbgl;
|
||||
|
||||
util::RunLoop loop;
|
||||
SQLiteCache cache(":memory:");
|
||||
|
||||
const unsigned entryCount = 800;
|
||||
const uint64_t entrySize = 10 * 1024; // 10 KB
|
||||
|
||||
cache.setMaximumCacheEntrySize(entrySize + 1);
|
||||
|
||||
// Cache size defaults to unlimited, all these
|
||||
// inserts should work.
|
||||
for (unsigned i = 0; i < entryCount; ++i) {
|
||||
insertTile(&cache, i, entrySize);
|
||||
}
|
||||
|
||||
for (unsigned i = 0; i < entryCount; ++i) {
|
||||
EXPECT_TRUE(tileIsCached(&cache, i));
|
||||
}
|
||||
|
||||
uint64_t expectedCacheSize = entryCount * entrySize;
|
||||
EXPECT_EQ(cacheSize(&cache, entryCount, entrySize), expectedCacheSize);
|
||||
|
||||
// Setting a new size should remove records until the new
|
||||
// size limit is satisfied.
|
||||
cache.setMaximumCacheSize(expectedCacheSize / 2);
|
||||
EXPECT_LT(cacheSize(&cache, entryCount, entrySize), expectedCacheSize / 2);
|
||||
|
||||
// Cache size 1 should practically clean the cache and
|
||||
// prevent adding any record, although it makes no sense
|
||||
// to use such size limit IRL.
|
||||
cache.setMaximumCacheSize(1);
|
||||
EXPECT_EQ(cacheSize(&cache, entryCount, entrySize), 0);
|
||||
|
||||
insertTile(&cache, 1000, entrySize);
|
||||
EXPECT_FALSE(tileIsCached(&cache, 1000));
|
||||
|
||||
// Zero should be treated as unlimited.
|
||||
cache.setMaximumCacheSize(0);
|
||||
|
||||
for (unsigned i = 0; i < entryCount; ++i) {
|
||||
insertTile(&cache, i, entrySize);
|
||||
}
|
||||
|
||||
EXPECT_EQ(cacheSize(&cache, entryCount, entrySize), expectedCacheSize);
|
||||
}
|
||||
|
||||
TEST_F(Storage, CacheSizePruneLeastAccessed) {
|
||||
using namespace mbgl;
|
||||
|
||||
util::RunLoop loop;
|
||||
SQLiteCache cache(":memory:");
|
||||
|
||||
const unsigned entryCount = 400;
|
||||
const uint64_t entrySize = 10 * 1024; // 10 KB
|
||||
|
||||
cache.setMaximumCacheEntrySize(entrySize + 1);
|
||||
cache.setMaximumCacheSize(entrySize * 350);
|
||||
|
||||
for (unsigned i = 0; i < entryCount; ++i) {
|
||||
insertTile(&cache, i, entrySize);
|
||||
|
||||
if (i == entryCount / 2) {
|
||||
// We need to sleep for 1s here because
|
||||
// that is the time resolution for the
|
||||
// `accessed` time. Then we 'ping' the
|
||||
// entry, that should update the
|
||||
// `accessed` time, so it won't get
|
||||
// pruned when we need more space.
|
||||
bool done = false;
|
||||
|
||||
util::Timer timer;
|
||||
timer.start(Milliseconds(1300),
|
||||
Duration::zero(),
|
||||
[&done] { done = true; });
|
||||
|
||||
while (!done) {
|
||||
loop.runOnce();
|
||||
}
|
||||
|
||||
EXPECT_TRUE(tileIsCached(&cache, 7));
|
||||
|
||||
// Refresh should also update the `accessed`
|
||||
// time of a tile.
|
||||
refreshTile(&cache, 9);
|
||||
}
|
||||
}
|
||||
|
||||
EXPECT_FALSE(tileIsCached(&cache, 6));
|
||||
EXPECT_FALSE(tileIsCached(&cache, 8));
|
||||
EXPECT_FALSE(tileIsCached(&cache, 10));
|
||||
|
||||
EXPECT_TRUE(tileIsCached(&cache, 7));
|
||||
EXPECT_TRUE(tileIsCached(&cache, 9));
|
||||
}
|
||||
|
||||
TEST_F(Storage, CacheSizeStress) {
|
||||
using namespace mbgl;
|
||||
|
||||
util::RunLoop loop;
|
||||
SQLiteCache cache(":memory:");
|
||||
|
||||
const unsigned entryCount = 2000;
|
||||
const uint64_t entrySize = 10 * 1024; // 10 KB
|
||||
|
||||
cache.setMaximumCacheEntrySize(entrySize + 1);
|
||||
cache.setMaximumCacheSize(entrySize * 300);
|
||||
|
||||
for (unsigned i = 0; i < entryCount; ++i) {
|
||||
insertTile(&cache, i, entrySize);
|
||||
}
|
||||
|
||||
// Should not be in the cache as they were
|
||||
// first inserted.
|
||||
EXPECT_FALSE(tileIsCached(&cache, 0));
|
||||
EXPECT_FALSE(tileIsCached(&cache, 99));
|
||||
EXPECT_FALSE(tileIsCached(&cache, 199));
|
||||
EXPECT_FALSE(tileIsCached(&cache, 299));
|
||||
EXPECT_FALSE(tileIsCached(&cache, 399));
|
||||
|
||||
EXPECT_TRUE(tileIsCached(&cache, entryCount - 1));
|
||||
|
||||
EXPECT_LT(cacheSize(&cache, entryCount, entrySize), entrySize * 300);
|
||||
}
|
||||
@@ -1,131 +0,0 @@
|
||||
#include "storage.hpp"
|
||||
|
||||
#include <mbgl/platform/default/headless_display.hpp>
|
||||
#include <mbgl/platform/default/headless_view.hpp>
|
||||
#include <mbgl/storage/sqlite_cache.hpp>
|
||||
#include <mbgl/storage/default_file_source.hpp>
|
||||
|
||||
#include <mbgl/platform/log.hpp>
|
||||
#include <mbgl/util/work_request.hpp>
|
||||
#include <mbgl/util/io.hpp>
|
||||
|
||||
using namespace mbgl;
|
||||
using namespace std::literals::chrono_literals;
|
||||
using namespace std::literals::string_literals;
|
||||
|
||||
namespace {
|
||||
|
||||
void checkRendering(Map& map,
|
||||
const char* name,
|
||||
std::chrono::milliseconds timeout,
|
||||
double imageThreshold = 0.001,
|
||||
double pixelThreshold = 0.1) {
|
||||
test::checkImage("test/fixtures/stale/"s + name, test::render(map, timeout), imageThreshold,
|
||||
pixelThreshold);
|
||||
}
|
||||
|
||||
Response expiredItem(const std::string& path) {
|
||||
Response response;
|
||||
response.data = std::make_shared<std::string>(util::read_file("test/fixtures/"s + path));
|
||||
response.expires = SystemClock::from_time_t(0);
|
||||
return response;
|
||||
}
|
||||
|
||||
const std::string prefix = "http://127.0.0.1:3000";
|
||||
|
||||
}
|
||||
|
||||
auto display = std::make_shared<mbgl::HeadlessDisplay>();
|
||||
|
||||
TEST_F(Storage, CacheStaleStyle) {
|
||||
HeadlessView view(display, 1);
|
||||
|
||||
auto cache = SQLiteCache::getShared(":memory:");
|
||||
|
||||
// Rig the cache with an expired stylesheet.
|
||||
const std::string stylePath = "stale/style.json";
|
||||
const Resource styleResource{ Resource::Kind::Style, prefix + "/" + stylePath };
|
||||
cache->put(styleResource, expiredItem(stylePath));
|
||||
|
||||
DefaultFileSource fileSource(":memory:", ".");
|
||||
|
||||
Map map(view, fileSource, MapMode::Still);
|
||||
map.setStyleURL(styleResource.url);
|
||||
|
||||
checkRendering(map, "stale_style", 1000ms);
|
||||
}
|
||||
|
||||
TEST_F(Storage, CacheStaleStyleAndTileJSON) {
|
||||
HeadlessView view(display, 1);
|
||||
|
||||
auto cache = SQLiteCache::getShared(":memory:");
|
||||
|
||||
// Rig the cache with an expired stylesheet.
|
||||
const std::string stylePath = "stale/style_and_tilejson.json";
|
||||
const Resource styleResource{ Resource::Kind::Style, prefix + "/" + stylePath };
|
||||
cache->put(styleResource, expiredItem(stylePath));
|
||||
|
||||
// Rig the cache with an expired TileJSON.
|
||||
const std::string tilejsonPath = "stale/streets.json";
|
||||
const Resource tilejsonResource{ Resource::Kind::Source, prefix + "/" + tilejsonPath };
|
||||
cache->put(tilejsonResource, expiredItem(tilejsonPath));
|
||||
|
||||
DefaultFileSource fileSource(":memory:", ".");
|
||||
|
||||
Map map(view, fileSource, MapMode::Still);
|
||||
map.setStyleURL(styleResource.url);
|
||||
|
||||
checkRendering(map, "stale_style_and_tilejson", 1000ms);
|
||||
}
|
||||
|
||||
TEST_F(Storage, CacheStaleStyleAndSprite) {
|
||||
HeadlessView view(display, 1);
|
||||
|
||||
auto cache = SQLiteCache::getShared(":memory:");
|
||||
|
||||
// Rig the cache with an expired stylesheet.
|
||||
const std::string stylePath = "stale/style_and_sprite.json";
|
||||
const Resource styleResource{ Resource::Kind::Style, prefix + "/" + stylePath };
|
||||
cache->put(styleResource, expiredItem(stylePath));
|
||||
|
||||
// Rig the cache with an expired sprite JSON.
|
||||
const std::string spritejsonPath = "stale/sprite.json";
|
||||
const Resource spritejsonResource{ Resource::Kind::SpriteJSON, prefix + "/" + spritejsonPath };
|
||||
cache->put(spritejsonResource, expiredItem(spritejsonPath));
|
||||
|
||||
// Rig the cache with an expired sprite JSON.
|
||||
const std::string spriteimagePath = "stale/sprite.png";
|
||||
const Resource spriteimageResource{ Resource::Kind::SpriteImage, prefix + "/" + spriteimagePath };
|
||||
cache->put(spriteimageResource, expiredItem(spriteimagePath));
|
||||
|
||||
DefaultFileSource fileSource(":memory:", ".");
|
||||
|
||||
Map map(view, fileSource, MapMode::Still);
|
||||
map.setStyleURL(styleResource.url);
|
||||
|
||||
checkRendering(map, "stale_style_and_sprite", 1000ms);
|
||||
}
|
||||
|
||||
TEST_F(Storage, CacheStaleStyleAndGlyphs) {
|
||||
HeadlessView view(display, 1);
|
||||
|
||||
auto cache = SQLiteCache::getShared(":memory:");
|
||||
|
||||
// Rig the cache with an expired stylesheet.
|
||||
const std::string stylePath = "stale/style_and_glyphs.json";
|
||||
const Resource styleResource{ Resource::Kind::Style, prefix + "/" + stylePath };
|
||||
cache->put(styleResource, expiredItem(stylePath));
|
||||
|
||||
// Rig the cache with an expired glyph PBF.
|
||||
const std::string glyphPath = "stale/glyph.pbf";
|
||||
const Resource glyphResource{ Resource::Kind::Glyphs, prefix + "/stale/Open%20Sans%20Regular%2c%20Arial%20Unicode%20MS%20Regular/0-255.pbf" };
|
||||
cache->put(glyphResource, expiredItem(glyphPath));
|
||||
|
||||
DefaultFileSource fileSource(":memory:", ".");
|
||||
|
||||
Map map(view, fileSource, MapMode::Still);
|
||||
map.setStyleURL(styleResource.url);
|
||||
|
||||
// TODO: this shouldn't take > 1 second
|
||||
checkRendering(map, "stale_style_and_glyphs", 2000ms, 0.0015);
|
||||
}
|
||||
@@ -1,349 +0,0 @@
|
||||
#include "storage.hpp"
|
||||
#include "../fixtures/fixture_log_observer.hpp"
|
||||
|
||||
#include "sqlite_cache_impl.hpp"
|
||||
#include <mbgl/storage/resource.hpp>
|
||||
#include <mbgl/storage/response.hpp>
|
||||
#include <mbgl/util/io.hpp>
|
||||
|
||||
#include <sqlite3.h>
|
||||
|
||||
TEST_F(Storage, DatabaseDoesNotExist) {
|
||||
using namespace mbgl;
|
||||
|
||||
Log::setObserver(std::make_unique<FixtureLogObserver>());
|
||||
|
||||
SQLiteCache::Impl cache("test/fixtures/404/cache.db");
|
||||
|
||||
cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
|
||||
EXPECT_EQ(nullptr, res.get());
|
||||
});
|
||||
|
||||
auto observer = Log::removeObserver();
|
||||
EXPECT_EQ(1ul, dynamic_cast<FixtureLogObserver*>(observer.get())->count({ EventSeverity::Error, Event::Database, 14, "unable to open database file" }));
|
||||
}
|
||||
|
||||
void createDir(const char* name) {
|
||||
const int ret = mkdir(name, 0755);
|
||||
if (ret == -1) {
|
||||
ASSERT_EQ(EEXIST, errno);
|
||||
} else {
|
||||
ASSERT_EQ(0, ret);
|
||||
}
|
||||
}
|
||||
|
||||
void deleteFile(const char* name) {
|
||||
const int ret = unlink(name);
|
||||
if (ret == -1) {
|
||||
ASSERT_EQ(ENOENT, errno);
|
||||
} else {
|
||||
ASSERT_EQ(0, ret);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void writeFile(const char* name, const std::string& data) {
|
||||
mbgl::util::write_file(name, data);
|
||||
}
|
||||
|
||||
|
||||
TEST_F(Storage, DatabaseCreate) {
|
||||
using namespace mbgl;
|
||||
|
||||
createDir("test/fixtures/database");
|
||||
deleteFile("test/fixtures/database/cache.db");
|
||||
|
||||
Log::setObserver(std::make_unique<FixtureLogObserver>());
|
||||
|
||||
SQLiteCache::Impl cache("test/fixtures/database/cache.db");
|
||||
|
||||
cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
|
||||
EXPECT_EQ(nullptr, res.get());
|
||||
});
|
||||
|
||||
Log::removeObserver();
|
||||
}
|
||||
|
||||
TEST_F(Storage, DatabaseVersion) {
|
||||
using namespace mbgl;
|
||||
|
||||
createDir("test/fixtures/database");
|
||||
deleteFile("test/fixtures/database/cache.db");
|
||||
std::string path("test/fixtures/database/cache.db");
|
||||
|
||||
Log::setObserver(std::make_unique<FixtureLogObserver>());
|
||||
|
||||
{
|
||||
SQLiteCache::Impl cache(path);
|
||||
cache.put({ Resource::Unknown, "mapbox://test" }, Response());
|
||||
}
|
||||
|
||||
sqlite3* db;
|
||||
sqlite3_open_v2(path.c_str(), &db, SQLITE_OPEN_READWRITE, nullptr);
|
||||
sqlite3_exec(db, "PRAGMA user_version = 999999", nullptr, nullptr, nullptr);
|
||||
sqlite3_close_v2(db);
|
||||
|
||||
// Changing the version will force the database to get recreated
|
||||
// thus removing every pre-existing cache entry.
|
||||
{
|
||||
SQLiteCache::Impl cache(path);
|
||||
|
||||
cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
|
||||
EXPECT_EQ(nullptr, res.get());
|
||||
});
|
||||
}
|
||||
|
||||
Log::removeObserver();
|
||||
}
|
||||
|
||||
class FileLock {
|
||||
public:
|
||||
FileLock(const std::string& path) {
|
||||
const int err = sqlite3_open_v2(path.c_str(), &db, SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE, nullptr);
|
||||
if (err != SQLITE_OK) {
|
||||
throw std::runtime_error("Could not open db");
|
||||
}
|
||||
lock();
|
||||
}
|
||||
|
||||
void lock() {
|
||||
assert(!locked);
|
||||
const int err = sqlite3_exec(db, "begin exclusive transaction", nullptr, nullptr, nullptr);
|
||||
if (err != SQLITE_OK) {
|
||||
throw std::runtime_error("Could not lock db");
|
||||
}
|
||||
locked = true;
|
||||
}
|
||||
|
||||
void unlock() {
|
||||
assert(locked);
|
||||
const int err = sqlite3_exec(db, "commit", nullptr, nullptr, nullptr);
|
||||
if (err != SQLITE_OK) {
|
||||
throw std::runtime_error("Could not unlock db");
|
||||
}
|
||||
locked = false;
|
||||
}
|
||||
|
||||
~FileLock() {
|
||||
if (locked) {
|
||||
unlock();
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
sqlite3* db;
|
||||
bool locked = false;
|
||||
};
|
||||
|
||||
TEST_F(Storage, DatabaseLockedRead) {
|
||||
using namespace mbgl;
|
||||
|
||||
// Create a locked file.
|
||||
createDir("test/fixtures/database");
|
||||
deleteFile("test/fixtures/database/locked.db");
|
||||
FileLock guard("test/fixtures/database/locked.db");
|
||||
|
||||
SQLiteCache::Impl cache("test/fixtures/database/locked.db");
|
||||
|
||||
{
|
||||
// First request should fail.
|
||||
Log::setObserver(std::make_unique<FixtureLogObserver>());
|
||||
|
||||
cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
|
||||
EXPECT_EQ(nullptr, res.get());
|
||||
});
|
||||
|
||||
// Make sure that we got a few "database locked" errors
|
||||
auto observer = Log::removeObserver();
|
||||
auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
|
||||
EXPECT_EQ(4ul, flo->count({ EventSeverity::Error, Event::Database, 5, "database is locked" }));
|
||||
}
|
||||
|
||||
// Then, unlock the file and try again.
|
||||
guard.unlock();
|
||||
|
||||
{
|
||||
// First, try getting a file (the cache value should not exist).
|
||||
Log::setObserver(std::make_unique<FixtureLogObserver>());
|
||||
|
||||
cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
|
||||
EXPECT_EQ(nullptr, res.get());
|
||||
});
|
||||
|
||||
// Make sure that we got a no errors
|
||||
Log::removeObserver();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
TEST_F(Storage, DatabaseLockedWrite) {
|
||||
using namespace mbgl;
|
||||
|
||||
// Create a locked file.
|
||||
createDir("test/fixtures/database");
|
||||
deleteFile("test/fixtures/database/locked.db");
|
||||
FileLock guard("test/fixtures/database/locked.db");
|
||||
|
||||
SQLiteCache::Impl cache("test/fixtures/database/locked.db");
|
||||
|
||||
{
|
||||
// Adds a file (which should fail).
|
||||
Log::setObserver(std::make_unique<FixtureLogObserver>());
|
||||
|
||||
cache.put({ Resource::Unknown, "mapbox://test" }, Response());
|
||||
cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
|
||||
EXPECT_EQ(nullptr, res.get());
|
||||
});
|
||||
|
||||
auto observer = Log::removeObserver();
|
||||
auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
|
||||
EXPECT_EQ(8ul, flo->count({ EventSeverity::Error, Event::Database, 5, "database is locked" }));
|
||||
}
|
||||
|
||||
// Then, unlock the file and try again.
|
||||
guard.unlock();
|
||||
|
||||
{
|
||||
// Then, set a file and obtain it again.
|
||||
Log::setObserver(std::make_unique<FixtureLogObserver>());
|
||||
|
||||
Response response;
|
||||
response.data = std::make_shared<std::string>("Demo");
|
||||
cache.put({ Resource::Unknown, "mapbox://test" }, response);
|
||||
cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
|
||||
ASSERT_NE(nullptr, res.get());
|
||||
ASSERT_TRUE(res->data.get());
|
||||
EXPECT_EQ("Demo", *res->data);
|
||||
});
|
||||
|
||||
// Make sure that we got a no errors
|
||||
Log::removeObserver();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
TEST_F(Storage, DatabaseLockedRefresh) {
|
||||
using namespace mbgl;
|
||||
|
||||
// Create a locked file.
|
||||
createDir("test/fixtures/database");
|
||||
deleteFile("test/fixtures/database/locked.db");
|
||||
|
||||
SQLiteCache::Impl cache("test/fixtures/database/locked.db");
|
||||
|
||||
// Then, lock the file and try again.
|
||||
FileLock guard("test/fixtures/database/locked.db");
|
||||
|
||||
{
|
||||
// Adds a file.
|
||||
Log::setObserver(std::make_unique<FixtureLogObserver>());
|
||||
|
||||
Response response;
|
||||
response.data = std::make_shared<std::string>("Demo");
|
||||
cache.put({ Resource::Unknown, "mapbox://test" }, response);
|
||||
cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
|
||||
EXPECT_EQ(nullptr, res.get());
|
||||
});
|
||||
|
||||
auto observer = Log::removeObserver();
|
||||
auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
|
||||
EXPECT_EQ(8ul, flo->count({ EventSeverity::Error, Event::Database, 5, "database is locked" }));
|
||||
}
|
||||
|
||||
{
|
||||
// Then, try to refresh it.
|
||||
Log::setObserver(std::make_unique<FixtureLogObserver>());
|
||||
|
||||
cache.refresh({ Resource::Unknown, "mapbox://test" }, {});
|
||||
cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
|
||||
EXPECT_EQ(nullptr, res.get());
|
||||
});
|
||||
|
||||
// Make sure that we got the right errors.
|
||||
auto observer = Log::removeObserver();
|
||||
auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
|
||||
EXPECT_EQ(8ul, flo->count({ EventSeverity::Error, Event::Database, 5, "database is locked" }));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
TEST_F(Storage, DatabaseDeleted) {
|
||||
using namespace mbgl;
|
||||
|
||||
// Create a locked file.
|
||||
createDir("test/fixtures/database");
|
||||
deleteFile("test/fixtures/database/locked.db");
|
||||
|
||||
SQLiteCache::Impl cache("test/fixtures/database/locked.db");
|
||||
|
||||
{
|
||||
// Adds a file.
|
||||
Log::setObserver(std::make_unique<FixtureLogObserver>());
|
||||
|
||||
Response response;
|
||||
response.data = std::make_shared<std::string>("Demo");
|
||||
cache.put({ Resource::Unknown, "mapbox://test" }, response);
|
||||
cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
|
||||
ASSERT_NE(nullptr, res.get());
|
||||
ASSERT_TRUE(res->data.get());
|
||||
EXPECT_EQ("Demo", *res->data);
|
||||
});
|
||||
|
||||
Log::removeObserver();
|
||||
}
|
||||
|
||||
deleteFile("test/fixtures/database/locked.db");
|
||||
|
||||
{
|
||||
// Adds a file.
|
||||
Log::setObserver(std::make_unique<FixtureLogObserver>());
|
||||
|
||||
Response response;
|
||||
response.data = std::make_shared<std::string>("Demo");
|
||||
cache.put({ Resource::Unknown, "mapbox://test" }, response);
|
||||
cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
|
||||
ASSERT_NE(nullptr, res.get());
|
||||
ASSERT_TRUE(res->data.get());
|
||||
EXPECT_EQ("Demo", *res->data);
|
||||
});
|
||||
|
||||
auto observer = Log::removeObserver();
|
||||
auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
|
||||
EXPECT_EQ(1ul, flo->count({ EventSeverity::Error, Event::Database, 8, "attempt to write a readonly database" }));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
TEST_F(Storage, DatabaseInvalid) {
|
||||
using namespace mbgl;
|
||||
|
||||
// Create a locked file.
|
||||
createDir("test/fixtures/database");
|
||||
deleteFile("test/fixtures/database/invalid.db");
|
||||
writeFile("test/fixtures/database/invalid.db", "this is an invalid file");
|
||||
|
||||
SQLiteCache::Impl cache("test/fixtures/database/invalid.db");
|
||||
|
||||
{
|
||||
// Adds a file.
|
||||
Log::setObserver(std::make_unique<FixtureLogObserver>());
|
||||
|
||||
Response response;
|
||||
response.data = std::make_shared<std::string>("Demo");
|
||||
cache.put({ Resource::Unknown, "mapbox://test" }, response);
|
||||
cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
|
||||
ASSERT_NE(nullptr, res.get());
|
||||
ASSERT_TRUE(res->data.get());
|
||||
EXPECT_EQ("Demo", *res->data);
|
||||
});
|
||||
|
||||
auto observer = Log::removeObserver();
|
||||
auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
|
||||
EXPECT_EQ(1ul, flo->count({ EventSeverity::Warning, Event::Database, -1, "Trashing invalid database" }));
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,54 @@
|
||||
#include "storage.hpp"
|
||||
|
||||
#include <mbgl/storage/default_file_source.hpp>
|
||||
#include <mbgl/util/chrono.hpp>
|
||||
#include <mbgl/util/run_loop.hpp>
|
||||
|
||||
TEST_F(Storage, CacheRevalidateSame) {
|
||||
class DefaultFileSourceTest : public Storage {};
|
||||
|
||||
TEST_F(DefaultFileSourceTest, CacheResponse) {
|
||||
SCOPED_TEST(CacheResponse);
|
||||
|
||||
using namespace mbgl;
|
||||
|
||||
util::RunLoop loop;
|
||||
DefaultFileSource fs(":memory:", ".");
|
||||
|
||||
const Resource resource { Resource::Unknown, "http://127.0.0.1:3000/cache" };
|
||||
Response response;
|
||||
|
||||
std::unique_ptr<FileRequest> req1;
|
||||
std::unique_ptr<FileRequest> req2;
|
||||
|
||||
req1 = fs.request(resource, [&](Response res) {
|
||||
req1.reset();
|
||||
EXPECT_EQ(nullptr, res.error);
|
||||
ASSERT_TRUE(res.data.get());
|
||||
EXPECT_EQ("Response 1", *res.data);
|
||||
EXPECT_TRUE(bool(res.expires));
|
||||
EXPECT_FALSE(bool(res.modified));
|
||||
EXPECT_FALSE(bool(res.etag));
|
||||
response = res;
|
||||
|
||||
// Now test that we get the same values as in the previous request. If we'd go to the server
|
||||
// again, we'd get different values.
|
||||
req2 = fs.request(resource, [&](Response res2) {
|
||||
req2.reset();
|
||||
EXPECT_EQ(response.error, res2.error);
|
||||
ASSERT_TRUE(res2.data.get());
|
||||
EXPECT_EQ(*response.data, *res2.data);
|
||||
EXPECT_EQ(response.expires, res2.expires);
|
||||
EXPECT_EQ(response.modified, res2.modified);
|
||||
EXPECT_EQ(response.etag, res2.etag);
|
||||
|
||||
loop.stop();
|
||||
CacheResponse.finish();
|
||||
});
|
||||
});
|
||||
|
||||
loop.run();
|
||||
}
|
||||
|
||||
TEST_F(DefaultFileSourceTest, CacheRevalidateSame) {
|
||||
SCOPED_TEST(CacheRevalidateSame)
|
||||
|
||||
using namespace mbgl;
|
||||
@@ -53,7 +97,7 @@ TEST_F(Storage, CacheRevalidateSame) {
|
||||
loop.run();
|
||||
}
|
||||
|
||||
TEST_F(Storage, CacheRevalidateModified) {
|
||||
TEST_F(DefaultFileSourceTest, CacheRevalidateModified) {
|
||||
SCOPED_TEST(CacheRevalidateModified)
|
||||
|
||||
using namespace mbgl;
|
||||
@@ -102,7 +146,7 @@ TEST_F(Storage, CacheRevalidateModified) {
|
||||
loop.run();
|
||||
}
|
||||
|
||||
TEST_F(Storage, CacheRevalidateEtag) {
|
||||
TEST_F(DefaultFileSourceTest, CacheRevalidateEtag) {
|
||||
SCOPED_TEST(CacheRevalidateEtag)
|
||||
|
||||
using namespace mbgl;
|
||||
374
test/storage/offline_database.cpp
Normal file
@@ -0,0 +1,374 @@
|
||||
#include "../fixtures/fixture_log_observer.hpp"
|
||||
|
||||
#include <mbgl/storage/offline_database.hpp>
|
||||
#include <mbgl/storage/resource.hpp>
|
||||
#include <mbgl/storage/response.hpp>
|
||||
#include <mbgl/util/io.hpp>
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
#include <sqlite3.h>
|
||||
|
||||
namespace {
|
||||
|
||||
void createDir(const char* name) {
|
||||
const int ret = mkdir(name, 0755);
|
||||
if (ret == -1) {
|
||||
ASSERT_EQ(EEXIST, errno);
|
||||
} else {
|
||||
ASSERT_EQ(0, ret);
|
||||
}
|
||||
}
|
||||
|
||||
void deleteFile(const char* name) {
|
||||
const int ret = unlink(name);
|
||||
if (ret == -1) {
|
||||
ASSERT_EQ(ENOENT, errno);
|
||||
} else {
|
||||
ASSERT_EQ(0, ret);
|
||||
}
|
||||
}
|
||||
|
||||
void writeFile(const char* name, const std::string& data) {
|
||||
mbgl::util::write_file(name, data);
|
||||
}
|
||||
|
||||
class FileLock {
|
||||
public:
|
||||
FileLock(const std::string& path) {
|
||||
const int err = sqlite3_open_v2(path.c_str(), &db, SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE, nullptr);
|
||||
if (err != SQLITE_OK) {
|
||||
throw std::runtime_error("Could not open db");
|
||||
}
|
||||
lock();
|
||||
}
|
||||
|
||||
void lock() {
|
||||
assert(!locked);
|
||||
const int err = sqlite3_exec(db, "begin exclusive transaction", nullptr, nullptr, nullptr);
|
||||
if (err != SQLITE_OK) {
|
||||
throw std::runtime_error("Could not lock db");
|
||||
}
|
||||
locked = true;
|
||||
}
|
||||
|
||||
void unlock() {
|
||||
assert(locked);
|
||||
const int err = sqlite3_exec(db, "commit", nullptr, nullptr, nullptr);
|
||||
if (err != SQLITE_OK) {
|
||||
throw std::runtime_error("Could not unlock db");
|
||||
}
|
||||
locked = false;
|
||||
}
|
||||
|
||||
~FileLock() {
|
||||
if (locked) {
|
||||
unlock();
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
sqlite3* db;
|
||||
bool locked = false;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
//TEST(OfflineDatabase, NonexistentDirectory) {
|
||||
// using namespace mbgl;
|
||||
//
|
||||
// Log::setObserver(std::make_unique<FixtureLogObserver>());
|
||||
//
|
||||
// OfflineDatabase db("test/fixtures/404/offline.db");
|
||||
//
|
||||
// db.get({ Resource::Unknown, "mapbox://test" }, [] (optional<Response> res) {
|
||||
// EXPECT_FALSE(bool(res));
|
||||
// });
|
||||
//
|
||||
// auto observer = Log::removeObserver();
|
||||
// EXPECT_EQ(1ul, dynamic_cast<FixtureLogObserver*>(observer.get())->count({ EventSeverity::Error, Event::Database, 14, "unable to open database file" }));
|
||||
//}
|
||||
|
||||
TEST(OfflineDatabase, Create) {
|
||||
using namespace mbgl;
|
||||
|
||||
createDir("test/fixtures/database");
|
||||
deleteFile("test/fixtures/database/offline.db");
|
||||
|
||||
Log::setObserver(std::make_unique<FixtureLogObserver>());
|
||||
|
||||
OfflineDatabase db("test/fixtures/database/offline.db");
|
||||
EXPECT_FALSE(bool(db.get({ Resource::Unknown, "mapbox://test" })));
|
||||
|
||||
Log::removeObserver();
|
||||
}
|
||||
|
||||
TEST(OfflineDatabase, SchemaVersion) {
|
||||
using namespace mbgl;
|
||||
|
||||
createDir("test/fixtures/database");
|
||||
deleteFile("test/fixtures/database/offline.db");
|
||||
std::string path("test/fixtures/database/offline.db");
|
||||
|
||||
{
|
||||
sqlite3* db;
|
||||
sqlite3_open_v2(path.c_str(), &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, nullptr);
|
||||
sqlite3_exec(db, "PRAGMA user_version = 1", nullptr, nullptr, nullptr);
|
||||
sqlite3_close_v2(db);
|
||||
}
|
||||
|
||||
Log::setObserver(std::make_unique<FixtureLogObserver>());
|
||||
OfflineDatabase db(path);
|
||||
|
||||
auto observer = Log::removeObserver();
|
||||
auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
|
||||
EXPECT_EQ(1ul, flo->count({ EventSeverity::Warning, Event::Database, -1, "Removing existing incompatible offline database" }));
|
||||
}
|
||||
|
||||
TEST(OfflineDatabase, Invalid) {
|
||||
using namespace mbgl;
|
||||
|
||||
createDir("test/fixtures/database");
|
||||
deleteFile("test/fixtures/database/invalid.db");
|
||||
writeFile("test/fixtures/database/invalid.db", "this is an invalid file");
|
||||
|
||||
Log::setObserver(std::make_unique<FixtureLogObserver>());
|
||||
|
||||
OfflineDatabase db("test/fixtures/database/invalid.db");
|
||||
|
||||
auto observer = Log::removeObserver();
|
||||
auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
|
||||
EXPECT_EQ(1ul, flo->count({ EventSeverity::Warning, Event::Database, -1, "Removing existing incompatible offline database" }));
|
||||
}
|
||||
|
||||
//TEST(OfflineDatabase, DatabaseLockedRead) {
|
||||
// using namespace mbgl;
|
||||
//
|
||||
// // Create a locked file.
|
||||
// createDir("test/fixtures/database");
|
||||
// deleteFile("test/fixtures/database/locked.db");
|
||||
// FileLock guard("test/fixtures/database/locked.db");
|
||||
//
|
||||
// OfflineDatabase db("test/fixtures/database/locked.db");
|
||||
//
|
||||
// {
|
||||
// // First request should fail.
|
||||
// Log::setObserver(std::make_unique<FixtureLogObserver>());
|
||||
//
|
||||
// db.get({ Resource::Unknown, "mapbox://test" }, [] (optional<Response> res) {
|
||||
// EXPECT_FALSE(bool(res));
|
||||
// });
|
||||
//
|
||||
// // Make sure that we got a few "database locked" errors
|
||||
// auto observer = Log::removeObserver();
|
||||
// auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
|
||||
// EXPECT_EQ(4ul, flo->count({ EventSeverity::Error, Event::Database, 5, "database is locked" }));
|
||||
// }
|
||||
//
|
||||
// // Then, unlock the file and try again.
|
||||
// guard.unlock();
|
||||
//
|
||||
// {
|
||||
// // First, try getting a file (the cache value should not exist).
|
||||
// Log::setObserver(std::make_unique<FixtureLogObserver>());
|
||||
//
|
||||
// db.get({ Resource::Unknown, "mapbox://test" }, [] (optional<Response> res) {
|
||||
// EXPECT_FALSE(bool(res));
|
||||
// });
|
||||
//
|
||||
// // Make sure that we got a no errors
|
||||
// Log::removeObserver();
|
||||
// }
|
||||
//}
|
||||
//
|
||||
//TEST(OfflineDatabase, DatabaseLockedWrite) {
|
||||
// using namespace mbgl;
|
||||
//
|
||||
// // Create a locked file.
|
||||
// createDir("test/fixtures/database");
|
||||
// deleteFile("test/fixtures/database/locked.db");
|
||||
// FileLock guard("test/fixtures/database/locked.db");
|
||||
//
|
||||
// OfflineDatabase db("test/fixtures/database/locked.db");
|
||||
//
|
||||
// {
|
||||
// // Adds a file (which should fail).
|
||||
// Log::setObserver(std::make_unique<FixtureLogObserver>());
|
||||
//
|
||||
// db.put({ Resource::Unknown, "mapbox://test" }, Response());
|
||||
// db.get({ Resource::Unknown, "mapbox://test" }, [] (optional<Response> res) {
|
||||
// EXPECT_FALSE(bool(res));
|
||||
// });
|
||||
//
|
||||
// auto observer = Log::removeObserver();
|
||||
// auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
|
||||
// EXPECT_EQ(8ul, flo->count({ EventSeverity::Error, Event::Database, 5, "database is locked" }));
|
||||
// }
|
||||
//
|
||||
// // Then, unlock the file and try again.
|
||||
// guard.unlock();
|
||||
//
|
||||
// {
|
||||
// // Then, set a file and obtain it again.
|
||||
// Log::setObserver(std::make_unique<FixtureLogObserver>());
|
||||
//
|
||||
// Response response;
|
||||
// response.data = std::make_shared<std::string>("Demo");
|
||||
// db.put({ Resource::Unknown, "mapbox://test" }, response);
|
||||
// db.get({ Resource::Unknown, "mapbox://test" }, [] (optional<Response> res) {
|
||||
// ASSERT_TRUE(bool(res));
|
||||
// ASSERT_TRUE(res->data.get());
|
||||
// EXPECT_EQ("Demo", *res->data);
|
||||
// });
|
||||
//
|
||||
// // Make sure that we got a no errors
|
||||
// Log::removeObserver();
|
||||
// }
|
||||
//}
|
||||
//
|
||||
//TEST(OfflineDatabase, DatabaseDeleted) {
|
||||
// using namespace mbgl;
|
||||
//
|
||||
// // Create a locked file.
|
||||
// createDir("test/fixtures/database");
|
||||
// deleteFile("test/fixtures/database/locked.db");
|
||||
//
|
||||
// OfflineDatabase db("test/fixtures/database/locked.db");
|
||||
//
|
||||
// {
|
||||
// // Adds a file.
|
||||
// Log::setObserver(std::make_unique<FixtureLogObserver>());
|
||||
//
|
||||
// Response response;
|
||||
// response.data = std::make_shared<std::string>("Demo");
|
||||
// db.put({ Resource::Unknown, "mapbox://test" }, response);
|
||||
// db.get({ Resource::Unknown, "mapbox://test" }, [] (optional<Response> res) {
|
||||
// ASSERT_TRUE(bool(res));
|
||||
// ASSERT_TRUE(res->data.get());
|
||||
// EXPECT_EQ("Demo", *res->data);
|
||||
// });
|
||||
//
|
||||
// Log::removeObserver();
|
||||
// }
|
||||
//
|
||||
// deleteFile("test/fixtures/database/locked.db");
|
||||
//
|
||||
// {
|
||||
// // Adds a file.
|
||||
// Log::setObserver(std::make_unique<FixtureLogObserver>());
|
||||
//
|
||||
// Response response;
|
||||
// response.data = std::make_shared<std::string>("Demo");
|
||||
// db.put({ Resource::Unknown, "mapbox://test" }, response);
|
||||
// db.get({ Resource::Unknown, "mapbox://test" }, [] (optional<Response> res) {
|
||||
// ASSERT_TRUE(bool(res));
|
||||
// ASSERT_TRUE(res->data.get());
|
||||
// EXPECT_EQ("Demo", *res->data);
|
||||
// });
|
||||
//
|
||||
// auto observer = Log::removeObserver();
|
||||
// auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
|
||||
// EXPECT_EQ(1ul, flo->count({ EventSeverity::Error, Event::Database, 8, "attempt to write a readonly database" }));
|
||||
// }
|
||||
//}
|
||||
|
||||
TEST(OfflineDatabase, PutDoesNotStoreConnectionErrors) {
|
||||
using namespace mbgl;
|
||||
|
||||
OfflineDatabase db(":memory:");
|
||||
|
||||
Resource resource { Resource::Unknown, "http://example.com/" };
|
||||
Response response;
|
||||
response.error = std::make_unique<Response::Error>(Response::Error::Reason::Connection);
|
||||
|
||||
db.put(resource, response);
|
||||
EXPECT_FALSE(bool(db.get(resource)));
|
||||
}
|
||||
|
||||
TEST(OfflineDatabase, PutDoesNotStoreServerErrors) {
|
||||
using namespace mbgl;
|
||||
|
||||
OfflineDatabase db(":memory:");
|
||||
|
||||
Resource resource { Resource::Unknown, "http://example.com/" };
|
||||
Response response;
|
||||
response.error = std::make_unique<Response::Error>(Response::Error::Reason::Server);
|
||||
|
||||
db.put(resource, response);
|
||||
EXPECT_FALSE(bool(db.get(resource)));
|
||||
}
|
||||
|
||||
TEST(OfflineDatabase, PutResource) {
|
||||
using namespace mbgl;
|
||||
|
||||
OfflineDatabase db(":memory:");
|
||||
|
||||
Resource resource { Resource::Style, "http://example.com/" };
|
||||
Response response;
|
||||
response.data = std::make_shared<std::string>("data");
|
||||
|
||||
db.put(resource, response);
|
||||
auto res = db.get(resource);
|
||||
EXPECT_EQ(nullptr, res->error.get());
|
||||
EXPECT_EQ("data", *res->data);
|
||||
}
|
||||
|
||||
TEST(OfflineDatabase, PutTile) {
|
||||
using namespace mbgl;
|
||||
|
||||
OfflineDatabase db(":memory:");
|
||||
|
||||
Resource resource { Resource::Tile, "http://example.com/" };
|
||||
resource.tileData = Resource::TileData {
|
||||
"http://example.com/",
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
0
|
||||
};
|
||||
Response response;
|
||||
response.data = std::make_shared<std::string>("data");
|
||||
|
||||
db.put(resource, response);
|
||||
auto res = db.get(resource);
|
||||
EXPECT_EQ(nullptr, res->error.get());
|
||||
EXPECT_EQ("data", *res->data);
|
||||
}
|
||||
|
||||
TEST(OfflineDatabase, PutResourceNotFound) {
|
||||
using namespace mbgl;
|
||||
|
||||
OfflineDatabase db(":memory:");
|
||||
|
||||
Resource resource { Resource::Style, "http://example.com/" };
|
||||
Response response;
|
||||
response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound);
|
||||
|
||||
db.put(resource, response);
|
||||
auto res = db.get(resource);
|
||||
EXPECT_NE(nullptr, res->error);
|
||||
EXPECT_EQ(Response::Error::Reason::NotFound, res->error->reason);
|
||||
EXPECT_FALSE(res->data.get());
|
||||
}
|
||||
|
||||
TEST(OfflineDatabase, PutTileNotFound) {
|
||||
using namespace mbgl;
|
||||
|
||||
OfflineDatabase db(":memory:");
|
||||
|
||||
Resource resource { Resource::Tile, "http://example.com/" };
|
||||
resource.tileData = Resource::TileData {
|
||||
"http://example.com/",
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
0
|
||||
};
|
||||
Response response;
|
||||
response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound);
|
||||
|
||||
db.put(resource, response);
|
||||
auto res = db.get(resource);
|
||||
EXPECT_NE(nullptr, res->error);
|
||||
EXPECT_EQ(Response::Error::Reason::NotFound, res->error->reason);
|
||||
EXPECT_FALSE(res->data.get());
|
||||
}
|
||||
@@ -58,6 +58,7 @@
|
||||
'api/render_missing.cpp',
|
||||
'api/set_style.cpp',
|
||||
'api/custom_layer.cpp',
|
||||
'api/offline.cpp',
|
||||
|
||||
'geometry/binpack.cpp',
|
||||
|
||||
@@ -68,12 +69,8 @@
|
||||
|
||||
'storage/storage.hpp',
|
||||
'storage/storage.cpp',
|
||||
'storage/cache_response.cpp',
|
||||
'storage/cache_revalidate.cpp',
|
||||
'storage/cache_shared.cpp',
|
||||
'storage/cache_size.cpp',
|
||||
'storage/cache_stale.cpp',
|
||||
'storage/database.cpp',
|
||||
'storage/default_file_source.cpp',
|
||||
'storage/offline_database.cpp',
|
||||
'storage/asset_file_source.cpp',
|
||||
'storage/headers.cpp',
|
||||
'storage/http_cancel.cpp',
|
||||
|
||||