[core] Reimplement existing caching within an offline-capable database schema

This commit is contained in:
John Firebaugh
2016-01-26 12:19:04 -08:00
parent 025375ad0b
commit cdedb66387
42 changed files with 1115 additions and 1750 deletions

4
.gitignore vendored
View File

@@ -23,8 +23,8 @@
/test/fixtures/api/1.png
/test/fixtures/api/2.png
/test/fixtures/database/*.db
/test/fixtures/*/*/actual.png
/test/fixtures/*/*/diff.png
/test/fixtures/**/actual.png
/test/fixtures/**/diff.png
/test/output
/include/mbgl/shader/shaders.hpp
/src/shader/shaders_gl.cpp

View File

@@ -21,7 +21,8 @@
'../platform/default/timer.cpp',
'../platform/default/default_file_source.cpp',
'../platform/default/online_file_source.cpp',
'../platform/default/sqlite_cache.cpp',
'../platform/default/mbgl/storage/offline_database.hpp',
'../platform/default/mbgl/storage/offline_database.cpp',
'../platform/default/sqlite3.hpp',
'../platform/default/sqlite3.cpp',
],
@@ -56,6 +57,7 @@
'include_dirs': [
'../include',
'../src',
'../platform/default',
],
'conditions': [

View File

@@ -16,7 +16,8 @@
'../platform/default/timer.cpp',
'../platform/default/default_file_source.cpp',
'../platform/default/online_file_source.cpp',
'../platform/default/sqlite_cache.cpp',
'../platform/default/mbgl/storage/offline_database.hpp',
'../platform/default/mbgl/storage/offline_database.cpp',
'../platform/default/sqlite3.hpp',
'../platform/default/sqlite3.cpp',
'../platform/darwin/log_nslog.mm',
@@ -98,6 +99,7 @@
'../include/mbgl/darwin',
'../include',
'../src',
'../platform/default',
],
'xcode_settings': {

View File

@@ -23,7 +23,8 @@
'../platform/default/timer.cpp',
'../platform/default/default_file_source.cpp',
'../platform/default/online_file_source.cpp',
'../platform/default/sqlite_cache.cpp',
'../platform/default/mbgl/storage/offline_database.hpp',
'../platform/default/mbgl/storage/offline_database.cpp',
'../platform/default/sqlite3.hpp',
'../platform/default/sqlite3.cpp',
],
@@ -61,6 +62,7 @@
'include_dirs': [
'../include',
'../src',
'../platform/default',
],
'conditions': [

View File

@@ -15,7 +15,8 @@
'../platform/default/timer.cpp',
'../platform/default/default_file_source.cpp',
'../platform/default/online_file_source.cpp',
'../platform/default/sqlite_cache.cpp',
'../platform/default/mbgl/storage/offline_database.hpp',
'../platform/default/mbgl/storage/offline_database.cpp',
'../platform/default/sqlite3.hpp',
'../platform/default/sqlite3.cpp',
'../platform/darwin/log_nslog.mm',
@@ -83,6 +84,7 @@
'../include/mbgl/darwin',
'../include',
'../src',
'../platform/default',
],
'xcode_settings': {

View File

@@ -5,7 +5,9 @@
namespace mbgl {
class SQLiteCache;
namespace util {
template <typename T> class Thread;
} // namespace util
class DefaultFileSource : public FileSource {
public:
@@ -20,14 +22,15 @@ public:
std::unique_ptr<FileRequest> request(const Resource&, Callback) override;
// For testing purposes only.
SQLiteCache& getCache();
// For testing only.
void put(const Resource&, const Response&);
void goOffline();
class Impl;
private:
class Impl;
friend class DefaultFileRequest;
const std::unique_ptr<Impl> impl;
const std::unique_ptr<util::Thread<Impl>> thread;
const std::unique_ptr<FileSource> assetFileSource;
};
} // namespace mbgl

View File

@@ -1,10 +1,11 @@
#include <mbgl/storage/default_file_source.hpp>
#include <mbgl/storage/asset_file_source.hpp>
#include <mbgl/storage/online_file_source.hpp>
#include <mbgl/storage/sqlite_cache.hpp>
#include <mbgl/storage/offline_database.hpp>
#include <mbgl/platform/platform.hpp>
#include <mbgl/util/url.hpp>
#include <mbgl/util/thread.hpp>
#include <mbgl/util/work_request.hpp>
#include <cassert>
@@ -23,76 +24,117 @@ namespace mbgl {
class DefaultFileSource::Impl {
public:
Impl(const std::string& cachePath, const std::string& assetRoot)
: assetFileSource(assetRoot),
cache(SQLiteCache::getShared(cachePath)) {
class Task {
public:
Task(Resource resource, FileSource::Callback callback, DefaultFileSource::Impl* impl) {
auto offlineResponse = impl->offlineDatabase.get(resource);
Resource revalidation = resource;
if (offlineResponse) {
revalidation.priorModified = offlineResponse->modified;
revalidation.priorExpires = offlineResponse->expires;
revalidation.priorEtag = offlineResponse->etag;
callback(*offlineResponse);
}
if (!impl->offline) {
onlineRequest = impl->onlineFileSource.request(revalidation, [=] (Response onlineResponse) {
impl->offlineDatabase.put(revalidation, onlineResponse);
callback(onlineResponse);
});
}
}
std::unique_ptr<FileRequest> onlineRequest;
};
Impl(const std::string& cachePath)
: offlineDatabase(cachePath) {
}
AssetFileSource assetFileSource;
std::shared_ptr<SQLiteCache> cache;
void setAccessToken(const std::string& accessToken) {
onlineFileSource.setAccessToken(accessToken);
}
std::string getAccessToken() const {
return onlineFileSource.getAccessToken();
}
void add(FileRequest* req, Resource resource, Callback callback) {
tasks[req] = std::make_unique<Task>(resource, callback, this);
}
void cancel(FileRequest* req) {
tasks.erase(req);
}
void put(const Resource& resource, const Response& response) {
offlineDatabase.put(resource, response);
}
void goOffline() {
offline = true;
}
OfflineDatabase offlineDatabase;
OnlineFileSource onlineFileSource;
std::unordered_map<FileRequest*, std::unique_ptr<Task>> tasks;
bool offline = false;
};
class DefaultFileRequest : public FileRequest {
public:
DefaultFileRequest(Resource resource, FileSource::Callback callback, util::Thread<DefaultFileSource::Impl>& thread_)
: thread(thread_),
workRequest(thread.invokeWithCallback(&DefaultFileSource::Impl::add, callback, this, resource)) {
}
~DefaultFileRequest() {
thread.invoke(&DefaultFileSource::Impl::cancel, this);
}
util::Thread<DefaultFileSource::Impl>& thread;
std::unique_ptr<WorkRequest> workRequest;
};
DefaultFileSource::DefaultFileSource(const std::string& cachePath, const std::string& assetRoot)
: impl(std::make_unique<DefaultFileSource::Impl>(cachePath, assetRoot)) {
: thread(std::make_unique<util::Thread<DefaultFileSource::Impl>>(util::ThreadContext{"DefaultFileSource", util::ThreadType::Unknown, util::ThreadPriority::Low}, cachePath)),
assetFileSource(std::make_unique<AssetFileSource>(assetRoot)) {
}
DefaultFileSource::~DefaultFileSource() = default;
void DefaultFileSource::setAccessToken(const std::string& accessToken) {
impl->onlineFileSource.setAccessToken(accessToken);
thread->invokeSync(&Impl::setAccessToken, accessToken);
}
std::string DefaultFileSource::getAccessToken() const {
return impl->onlineFileSource.getAccessToken();
return thread->invokeSync<std::string>(&Impl::getAccessToken);
}
void DefaultFileSource::setMaximumCacheSize(uint64_t size) {
impl->cache->setMaximumCacheSize(size);
void DefaultFileSource::setMaximumCacheSize(uint64_t) {
// TODO
}
void DefaultFileSource::setMaximumCacheEntrySize(uint64_t size) {
impl->cache->setMaximumCacheEntrySize(size);
void DefaultFileSource::setMaximumCacheEntrySize(uint64_t) {
// TODO
}
SQLiteCache& DefaultFileSource::getCache() {
return *impl->cache;
}
class DefaultFileRequest : public FileRequest {
public:
DefaultFileRequest(Resource resource, FileSource::Callback callback, DefaultFileSource::Impl* impl) {
cacheRequest = impl->cache->get(resource, [=](std::shared_ptr<Response> cacheResponse) mutable {
cacheRequest.reset();
if (cacheResponse) {
resource.priorModified = cacheResponse->modified;
resource.priorExpires = cacheResponse->expires;
resource.priorEtag = cacheResponse->etag;
}
onlineRequest = impl->onlineFileSource.request(resource, [=] (Response onlineResponse) {
impl->cache->put(resource, onlineResponse);
callback(onlineResponse);
});
// Do this last because it may result in deleting this DefaultFileRequest.
if (cacheResponse) {
callback(*cacheResponse);
}
});
}
std::unique_ptr<WorkRequest> cacheRequest;
std::unique_ptr<FileRequest> onlineRequest;
};
std::unique_ptr<FileRequest> DefaultFileSource::request(const Resource& resource, Callback callback) {
if (isAssetURL(resource.url)) {
return impl->assetFileSource.request(resource, callback);
return assetFileSource->request(resource, callback);
} else {
return std::make_unique<DefaultFileRequest>(resource, callback, impl.get());
return std::make_unique<DefaultFileRequest>(resource, callback, *thread);
}
}
void DefaultFileSource::put(const Resource& resource, const Response& response) {
thread->invokeSync(&Impl::put, resource, response);
}
void DefaultFileSource::goOffline() {
thread->invokeSync(&Impl::goOffline);
}
} // namespace mbgl

View File

@@ -0,0 +1,297 @@
#include <mbgl/storage/offline_database.hpp>
#include <mbgl/storage/response.hpp>
#include <mbgl/util/compression.hpp>
#include <mbgl/util/io.hpp>
#include <mbgl/util/string.hpp>
#include <mbgl/util/chrono.hpp>
#include <mbgl/map/tile_id.hpp>
#include <mbgl/platform/log.hpp>
#include "sqlite3.hpp"
#include <sqlite3.h>
namespace mbgl {
using namespace mapbox::sqlite;
// If you change the schema you must write a migration from the previous version.
static const uint32_t schemaVersion = 2;
OfflineDatabase::OfflineDatabase(const std::string& path_)
: path(path_) {
ensureSchema();
}
OfflineDatabase::~OfflineDatabase() {
// Deleting these SQLite objects may result in exceptions, but we're in a destructor, so we
// can't throw anything.
try {
statements.clear();
db.reset();
} catch (mapbox::sqlite::Exception& ex) {
Log::Error(Event::Database, ex.code, ex.what());
}
}
void OfflineDatabase::ensureSchema() {
if (path != ":memory:") {
try {
db = std::make_unique<Database>(path.c_str(), ReadWrite);
{
Statement userVersionStmt(db->prepare("PRAGMA user_version"));
userVersionStmt.run();
switch (userVersionStmt.get<int>(0)) {
case 0: break; // cache-only database; ok to delete
case 1: break; // cache-only database; ok to delete
case 2: return;
default: throw std::runtime_error("unknown schema version");
}
}
removeExisting();
db = std::make_unique<Database>(path.c_str(), ReadWrite | Create);
} catch (mapbox::sqlite::Exception& ex) {
if (ex.code == SQLITE_CANTOPEN) {
db = std::make_unique<Database>(path.c_str(), ReadWrite | Create);
} else if (ex.code == SQLITE_NOTADB) {
removeExisting();
db = std::make_unique<Database>(path.c_str(), ReadWrite | Create);
}
}
}
#include "offline_schema.cpp.include"
db = std::make_unique<Database>(path.c_str(), ReadWrite | Create);
db->exec(schema);
db->exec("PRAGMA user_version = " + util::toString(schemaVersion));
}
void OfflineDatabase::removeExisting() {
Log::Warning(Event::Database, "Removing existing incompatible offline database");
db.reset();
try {
util::deleteFile(path);
} catch (util::IOException& ex) {
Log::Error(Event::Database, ex.code, ex.what());
}
}
mapbox::sqlite::Statement& OfflineDatabase::getStatement(const char * sql) {
auto it = statements.find(sql);
if (it != statements.end()) {
it->second->reset();
return *it->second;
}
return *(statements[sql] = std::make_unique<Statement>(db->prepare(sql)));
}
optional<Response> OfflineDatabase::get(const Resource& resource) {
if (resource.kind == Resource::Kind::Tile) {
assert(resource.tileData);
return getTile(*resource.tileData);
} else {
return getResource(resource);
}
}
void OfflineDatabase::put(const Resource& resource, const Response& response) {
// Except for 404s, don't store errors in the cache.
if (response.error && response.error->reason != Response::Error::Reason::NotFound) {
return;
}
if (resource.kind == Resource::Kind::Tile) {
assert(resource.tileData);
putTile(*resource.tileData, response);
} else {
putResource(resource, response);
}
}
optional<Response> OfflineDatabase::getResource(const Resource& resource) {
mapbox::sqlite::Statement& stmt = getStatement(
// 0 1 2 3 4
"SELECT etag, expires, modified, data, compressed "
"FROM resources "
"WHERE url = ?");
stmt.bind(1, resource.url);
if (!stmt.run()) {
return {};
}
Response response;
response.etag = stmt.get<optional<std::string>>(0);
response.expires = stmt.get<optional<SystemTimePoint>>(1);
response.modified = stmt.get<optional<SystemTimePoint>>(2);
optional<std::string> data = stmt.get<optional<std::string>>(3);
if (!data) {
response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound);
} else if (stmt.get<int>(4)) {
response.data = std::make_shared<std::string>(util::decompress(*data));
} else {
response.data = std::make_shared<std::string>(*data);
}
return response;
}
void OfflineDatabase::putResource(const Resource& resource, const Response& response) {
if (response.notModified) {
mapbox::sqlite::Statement& stmt = getStatement(
// 1 2 3
"UPDATE resources SET accessed = ?, expires = ? WHERE url = ?");
stmt.bind(1, SystemClock::now());
stmt.bind(2, response.expires);
stmt.bind(3, resource.url);
stmt.run();
} else {
mapbox::sqlite::Statement& stmt = getStatement(
// 1 2 3 4 5 6 7 8
"REPLACE INTO resources (url, kind, etag, expires, modified, accessed, data, compressed) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?)");
stmt.bind(1 /* url */, resource.url);
stmt.bind(2 /* kind */, int(resource.kind));
stmt.bind(3 /* etag */, response.etag);
stmt.bind(4 /* expires */, response.expires);
stmt.bind(5 /* modified */, response.modified);
stmt.bind(6 /* accessed */, SystemClock::now());
std::string data;
if (response.error) { // Can only be NotFound
stmt.bind(7 /* data */, nullptr);
stmt.bind(8 /* compressed */, false);
} else {
data = util::compress(*response.data);
if (data.size() < response.data->size()) {
stmt.bind(7 /* data */, data, false); // do not retain the string internally.
stmt.bind(8 /* compressed */, true);
} else {
stmt.bind(7 /* data */, *response.data, false); // do not retain the string internally.
stmt.bind(8 /* compressed */, false);
}
}
stmt.run();
}
}
optional<Response> OfflineDatabase::getTile(const Resource::TileData& tile) {
mapbox::sqlite::Statement& stmt = getStatement(
// 0 1 2 3 4
"SELECT etag, expires, modified, data, compressed "
"FROM tilesets, tiles "
"WHERE tilesets.url_template = ? " // 1
"AND tilesets.pixel_ratio = ? " // 2
"AND tiles.x = ? " // 3
"AND tiles.y = ? " // 4
"AND tiles.z = ? " // 5
"AND tilesets.id = tiles.tileset_id ");
stmt.bind(1, tile.urlTemplate);
stmt.bind(2, tile.pixelRatio);
stmt.bind(3, tile.x);
stmt.bind(4, tile.y);
stmt.bind(5, tile.z);
if (!stmt.run()) {
return {};
}
Response response;
response.etag = stmt.get<optional<std::string>>(0);
response.expires = stmt.get<optional<SystemTimePoint>>(1);
response.modified = stmt.get<optional<SystemTimePoint>>(2);
optional<std::string> data = stmt.get<optional<std::string>>(3);
if (!data) {
response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound);
} else if (stmt.get<int>(4)) {
response.data = std::make_shared<std::string>(util::decompress(*data));
} else {
response.data = std::make_shared<std::string>(*data);
}
return response;
}
void OfflineDatabase::putTile(const Resource::TileData& tile, const Response& response) {
if (response.notModified) {
mapbox::sqlite::Statement& stmt = getStatement(
"UPDATE tiles SET accessed = ?1, expires = ?2 "
"WHERE tileset_id = ( "
" SELECT id FROM tilesets "
" WHERE url_template = ?3 "
" AND pixel_ratio = ?4) "
"AND tiles.x = ?5 "
"AND tiles.y = ?6 "
"AND tiles.z = ?7 ");
stmt.bind(1, SystemClock::now());
stmt.bind(2, response.expires);
stmt.bind(3, tile.urlTemplate);
stmt.bind(4, tile.pixelRatio);
stmt.bind(5, tile.x);
stmt.bind(6, tile.y);
stmt.bind(7, tile.z);
stmt.run();
} else {
mapbox::sqlite::Statement& stmt1 = getStatement(
"REPLACE INTO tilesets (url_template, pixel_ratio) "
"VALUES (?1, ?2) ");
stmt1.bind(1 /* url_template */, tile.urlTemplate);
stmt1.bind(2 /* pixel_ratio */, tile.pixelRatio);
stmt1.run();
mapbox::sqlite::Statement& stmt2 = getStatement(
"REPLACE INTO tiles (tileset_id, x, y, z, modified, etag, expires, accessed, data, compressed) "
"SELECT tilesets.id, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11 "
"FROM tilesets "
"WHERE url_template = ?1 "
"AND pixel_ratio = ?2 ");
stmt2.bind(1 /* url_template */, tile.urlTemplate);
stmt2.bind(2 /* pixel_ratio */, tile.pixelRatio);
stmt2.bind(3 /* x */, tile.x);
stmt2.bind(4 /* y */, tile.y);
stmt2.bind(5 /* z */, tile.z);
stmt2.bind(6 /* modified */, response.modified);
stmt2.bind(7 /* etag */, response.etag);
stmt2.bind(8 /* expires */, response.expires);
stmt2.bind(9 /* accessed */, SystemClock::now());
std::string data;
if (response.error) { // Can only be NotFound
stmt2.bind(10 /* data */, nullptr);
stmt2.bind(11 /* compressed */, false);
} else {
data = util::compress(*response.data);
if (data.size() < response.data->size()) {
stmt2.bind(10 /* data */, data, false); // do not retain the string internally.
stmt2.bind(11 /* compressed */, true);
} else {
stmt2.bind(10 /* data */, *response.data, false); // do not retain the string internally.
stmt2.bind(11 /* compressed */, false);
}
}
stmt2.run();
}
}
} // namespace mbgl

View File

@@ -0,0 +1,50 @@
#ifndef MBGL_OFFLINE_DATABASE
#define MBGL_OFFLINE_DATABASE
#include <mbgl/storage/resource.hpp>
#include <mbgl/util/noncopyable.hpp>
#include <mbgl/util/optional.hpp>
#include <unordered_map>
#include <memory>
#include <string>
namespace mapbox {
namespace sqlite {
class Database;
class Statement;
}
}
namespace mbgl {
class Response;
class TileID;
class OfflineDatabase : private util::noncopyable {
public:
OfflineDatabase(const std::string& path);
~OfflineDatabase();
optional<Response> get(const Resource&);
void put(const Resource&, const Response&);
private:
void ensureSchema();
void removeExisting();
mapbox::sqlite::Statement& getStatement(const char *);
optional<Response> getTile(const Resource::TileData&);
void putTile(const Resource::TileData&, const Response&);
optional<Response> getResource(const Resource&);
void putResource(const Resource&, const Response&);
const std::string path;
std::unique_ptr<::mapbox::sqlite::Database> db;
std::unordered_map<const char *, std::unique_ptr<::mapbox::sqlite::Statement>> statements;
};
} // namespace mbgl
#endif

View File

@@ -0,0 +1,51 @@
/* THIS IS A GENERATED FILE; EDIT offline_schema.sql INSTEAD */
static const char * schema =
"CREATE TABLE resources (\n"
" url TEXT NOT NULL PRIMARY KEY,\n"
" kind INTEGER NOT NULL,\n"
" expires INTEGER,\n"
" modified INTEGER,\n"
" accessed INTEGER,\n"
" etag TEXT,\n"
" data BLOB,\n"
" compressed INTEGER NOT NULL DEFAULT 0\n"
");\n"
"CREATE TABLE tilesets (\n"
" id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,\n"
" url_template TEXT NOT NULL,\n"
" pixel_ratio INTEGER,\n"
" UNIQUE (url_template, pixel_ratio)\n"
");\n"
"CREATE TABLE tiles (\n"
" tileset_id INTEGER NOT NULL REFERENCES tilesets(id),\n"
" z INTEGER NOT NULL,\n"
" x INTEGER NOT NULL,\n"
" y INTEGER NOT NULL,\n"
" expires INTEGER,\n"
" modified INTEGER,\n"
" accessed INTEGER,\n"
" etag TEXT,\n"
" data BLOB,\n"
" compressed INTEGER NOT NULL DEFAULT 0,\n"
" PRIMARY KEY (tileset_id, z, x, y)\n"
");\n"
"CREATE TABLE regions (\n"
" id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,\n"
" definition TEXT NOT NULL,\n"
" description BLOB\n"
");\n"
"CREATE TABLE region_resources (\n"
" region_id INTEGER NOT NULL REFERENCES regions(id),\n"
" resource_url TEXT NOT NULL REFERENCES resources(url),\n"
" PRIMARY KEY (region_id, resource_url)\n"
");\n"
"CREATE TABLE region_tiles (\n"
" region_id INTEGER NOT NULL REFERENCES regions(id),\n"
" tileset_id INTEGER NOT NULL REFERENCES tilesets(id),\n"
" z INTEGER NOT NULL,\n"
" x INTEGER NOT NULL,\n"
" y INTEGER NOT NULL,\n"
" PRIMARY KEY (region_id, tileset_id, z, x, y),\n"
" FOREIGN KEY (tileset_id, z, x, y) REFERENCES tiles (tileset_id, z, x, y)\n"
");\n"
;

View File

@@ -0,0 +1,24 @@
// To regenerate:
// (cd platform/default/mbgl/storage && node offline_schema.js)
var fs = require('fs');
var readline = require('readline');
var lineReader = readline.createInterface({
input: fs.createReadStream('offline_schema.sql')
});
var lines = [
"/* THIS IS A GENERATED FILE; EDIT offline_schema.sql INSTEAD */",
"static const char * schema = ",
];
lineReader
.on('line', function (line) {
line = line.replace(/ *--.*/, '');
if (line) lines.push('"' + line + '\\n"');
})
.on('close', function () {
lines.push(';\n');
fs.writeFileSync('offline_schema.cpp.include', lines.join('\n'));
});

View File

@@ -0,0 +1,62 @@
CREATE TABLE resources ( -- Generic table for style, source, sprite, and glyph resources.
url TEXT NOT NULL PRIMARY KEY, -- Same schema as http_cache table.
kind INTEGER NOT NULL,
expires INTEGER,
modified INTEGER,
accessed INTEGER,
etag TEXT,
data BLOB, -- NULL if the response was a 404
compressed INTEGER NOT NULL DEFAULT 0
);
CREATE TABLE tilesets (
id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
url_template TEXT NOT NULL, -- As it would appear in TileJSON (but no support for host sharding).
pixel_ratio INTEGER, -- If NULL, 1 is assumed for raster sources.
UNIQUE (url_template, pixel_ratio) -- Capable of caching the same tileset at multiple resolutions.
);
CREATE TABLE tiles (
tileset_id INTEGER NOT NULL REFERENCES tilesets(id),
z INTEGER NOT NULL, -- Fully abandon TMS coordinates in favor of ZXY.
x INTEGER NOT NULL,
y INTEGER NOT NULL,
expires INTEGER,
modified INTEGER,
accessed INTEGER,
etag TEXT,
data BLOB, -- NULL if the response was a 404
compressed INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (tileset_id, z, x, y)
);
CREATE TABLE regions (
id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
definition TEXT NOT NULL, -- JSON formatted definition of region. Regions may be of variant types:
-- e.g. bbox and zoom range, route path, flyTo parameters, etc. Note that
-- the set of tiles required for a region may span multiple sources.
description BLOB -- User provided data in user-defined format
);
CREATE TABLE region_resources (
region_id INTEGER NOT NULL REFERENCES regions(id),
resource_url TEXT NOT NULL REFERENCES resources(url),
PRIMARY KEY (region_id, resource_url)
);
CREATE TABLE region_tiles (
region_id INTEGER NOT NULL REFERENCES regions(id),
tileset_id INTEGER NOT NULL REFERENCES tilesets(id),
z INTEGER NOT NULL,
x INTEGER NOT NULL,
y INTEGER NOT NULL,
PRIMARY KEY (region_id, tileset_id, z, x, y),
FOREIGN KEY (tileset_id, z, x, y) REFERENCES tiles (tileset_id, z, x, y)
);
-- `region_resources` and `region_tiles` are used for deduplication and deletion logic.
-- A row in `tiles` exists IFF one or more corresponding rows exist in `region_tiles`. If
-- more than one corresponding row exists, it indicates multiple regions contain the tile, and
-- storage for the tile is being deduplicated. When a region is deleted, corresponding rows in
-- `region_tiles` must also be deleted, and then rows in `tiles` and `tilesets` without a
-- corresponding `region_tiles` row must be deleted. Similarly for `resources` / `region_resources`.

View File

@@ -115,9 +115,19 @@ template <> void Statement::bind(int offset, std::nullptr_t) {
check(sqlite3_bind_null(stmt, offset));
}
template <> void Statement::bind(int offset, int value) {
template <> void Statement::bind(int offset, int8_t value) {
assert(stmt);
check(sqlite3_bind_int(stmt, offset, value));
check(sqlite3_bind_int64(stmt, offset, value));
}
template <> void Statement::bind(int offset, int16_t value) {
assert(stmt);
check(sqlite3_bind_int64(stmt, offset, value));
}
template <> void Statement::bind(int offset, int32_t value) {
assert(stmt);
check(sqlite3_bind_int64(stmt, offset, value));
}
template <> void Statement::bind(int offset, int64_t value) {
@@ -125,6 +135,26 @@ template <> void Statement::bind(int offset, int64_t value) {
check(sqlite3_bind_int64(stmt, offset, value));
}
template <> void Statement::bind(int offset, uint8_t value) {
assert(stmt);
check(sqlite3_bind_int64(stmt, offset, value));
}
template <> void Statement::bind(int offset, uint16_t value) {
assert(stmt);
check(sqlite3_bind_int64(stmt, offset, value));
}
template <> void Statement::bind(int offset, uint32_t value) {
assert(stmt);
check(sqlite3_bind_int64(stmt, offset, value));
}
template <> void Statement::bind(int offset, float value) {
assert(stmt);
check(sqlite3_bind_double(stmt, offset, value));
}
template <> void Statement::bind(int offset, double value) {
assert(stmt);
check(sqlite3_bind_double(stmt, offset, value));

View File

@@ -1,502 +0,0 @@
#include "sqlite_cache_impl.hpp"
#include <mbgl/storage/resource.hpp>
#include <mbgl/storage/response.hpp>
#include <mbgl/util/compression.hpp>
#include <mbgl/util/io.hpp>
#include <mbgl/util/string.hpp>
#include <mbgl/util/thread.hpp>
#include <mbgl/util/mapbox.hpp>
#include <mbgl/platform/log.hpp>
#include "sqlite3.hpp"
#include <sqlite3.h>
#include <unordered_map>
#include <mutex>
namespace {
// The cache won't accept entries larger than this arbitrary size
// and will silently discard request for adding them to the cache.
// Large entries can cause the database to grow in disk size and
// never shrink again.
const uint64_t kMaximumCacheEntrySize = 5 * 1024 * 1024; // 5 MB
// Number of records we delete when we are close to the maximum
// database size, if set. The current criteria is to prune
// the least used entries based on `accessed` time.
const int kPrunedEntriesLimit = 100;
} // namespace
namespace mbgl {
using namespace mapbox::sqlite;
SQLiteCache::SQLiteCache(const std::string& path_)
: thread(std::make_unique<util::Thread<Impl>>(util::ThreadContext{"SQLiteCache", util::ThreadType::Unknown, util::ThreadPriority::Low}, path_)) {
}
SQLiteCache::~SQLiteCache() = default;
SQLiteCache::Impl::Impl(const std::string& path_)
: maximumCacheSize(0), // Unlimited
maximumCacheEntrySize(kMaximumCacheEntrySize),
path(path_) {
}
SQLiteCache::Impl::~Impl() {
// Deleting these SQLite objects may result in exceptions, but we're in a destructor, so we
// can't throw anything.
try {
getStmt.reset();
putStmt.reset();
refreshStmt.reset();
countStmt.reset();
freeStmt.reset();
pruneStmt.reset();
accessedStmt.reset();
db.reset();
} catch (mapbox::sqlite::Exception& ex) {
Log::Error(Event::Database, ex.code, ex.what());
}
}
void SQLiteCache::Impl::createDatabase() {
db = std::make_unique<Database>(path.c_str(), ReadWrite | Create);
}
int SQLiteCache::Impl::schemaVersion() const {
// WARNING: Bump the version when changing the cache
// scheme to force the table to be recreated.
return 1;
}
void SQLiteCache::Impl::createSchema() {
constexpr const char *const sql = ""
"CREATE TABLE IF NOT EXISTS `http_cache` ("
" `url` TEXT PRIMARY KEY NOT NULL,"
" `status` INTEGER NOT NULL," // The response status (Successful or Error).
" `kind` INTEGER NOT NULL," // The kind of file.
" `modified` INTEGER," // Timestamp when the file was last modified.
" `etag` TEXT,"
" `expires` INTEGER," // Timestamp when the server says the file expires.
" `accessed` INTEGER," // Timestamp when the database record was last accessed.
" `data` BLOB,"
" `compressed` INTEGER NOT NULL DEFAULT 0" // Whether the data is compressed.
");"
"CREATE INDEX IF NOT EXISTS `http_cache_kind_idx` ON `http_cache` (`kind`);"
"CREATE INDEX IF NOT EXISTS `http_cache_accessed_idx` ON `http_cache` (`accessed`);";
ensureSchemaVersion();
try {
db->exec(sql);
db->exec("PRAGMA user_version = " + util::toString(schemaVersion()));
schema = true;
} catch (mapbox::sqlite::Exception &ex) {
if (ex.code == SQLITE_NOTADB) {
Log::Warning(Event::Database, "Trashing invalid database");
db.reset();
try {
util::deleteFile(path);
} catch (util::IOException& ioEx) {
Log::Error(Event::Database, ex.code, ex.what());
}
db = std::make_unique<Database>(path.c_str(), ReadWrite | Create);
} else {
Log::Error(Event::Database, ex.code, ex.what());
}
// Creating the database table + index failed. That means there may already be one, likely
// with different columns. Drop it and try to create a new one.
db->exec("DROP TABLE IF EXISTS `http_cache`");
db->exec(sql);
db->exec("PRAGMA user_version = " + util::toString(schemaVersion()));
}
}
void SQLiteCache::Impl::ensureSchemaVersion() {
try {
Statement userVersionStmt(db->prepare("PRAGMA user_version"));
if (userVersionStmt.run() && userVersionStmt.get<int>(0) == schemaVersion()) {
return;
}
} catch (mapbox::sqlite::Exception& ex) {
if (ex.code == SQLITE_NOTADB) {
return;
}
Log::Error(Event::Database, ex.code, ex.what());
}
// Version mismatch, drop the table so it will
// get recreated by `createSchema()`.
try {
db->exec("DROP TABLE IF EXISTS `http_cache`");
} catch (mapbox::sqlite::Exception& ex) {
Log::Error(Event::Database, ex.code, ex.what());
}
}
void SQLiteCache::setMaximumCacheSize(uint64_t size) {
thread->invoke(&Impl::setMaximumCacheSize, size);
}
void SQLiteCache::Impl::setMaximumCacheSize(uint64_t size) {
maximumCacheSize = size;
// Unlimited.
if (size == 0) {
return;
}
uint64_t lastSoftSize = cacheSoftSize();
// Keep pruning until we fit in the new
// size limit.
while (lastSoftSize > maximumCacheSize) {
pruneEntries();
if (lastSoftSize != cacheSoftSize()) {
lastSoftSize = cacheSoftSize();
} else {
break;
}
}
if (cacheHardSize() > size) {
Log::Warning(mbgl::Event::Database,
"Current cache hard size is bigger than the defined "
"maximum size. Database won't get truncated.");
}
}
void SQLiteCache::setMaximumCacheEntrySize(uint64_t size) {
thread->invoke(&Impl::setMaximumCacheEntrySize, size);
}
void SQLiteCache::Impl::setMaximumCacheEntrySize(uint64_t size) {
maximumCacheEntrySize = size;
}
void SQLiteCache::Impl::initializeDatabase() {
if (!db) {
createDatabase();
}
if (!schema) {
createSchema();
}
}
int SQLiteCache::Impl::cachePageSize() {
try {
if (!pageSize) {
Statement pageSizeStmt(db->prepare("PRAGMA page_size"));
if (pageSizeStmt.run()) {
pageSize = pageSizeStmt.get<int>(0);
}
}
} catch (mapbox::sqlite::Exception& ex) {
Log::Error(Event::Database, ex.code, ex.what());
}
return pageSize;
}
uint64_t SQLiteCache::Impl::cacheHardSize() {
try {
initializeDatabase();
if (!countStmt) {
countStmt = std::make_unique<Statement>(db->prepare("PRAGMA page_count"));
} else {
countStmt->reset();
}
if (countStmt->run()) {
return cachePageSize() * countStmt->get<int>(0);
}
} catch (mapbox::sqlite::Exception& ex) {
Log::Error(Event::Database, ex.code, ex.what());
}
return 0;
}
uint64_t SQLiteCache::Impl::cacheSoftSize() {
if (!softSizeDirty) {
return softSize;
}
try {
initializeDatabase();
if (!freeStmt) {
freeStmt = std::make_unique<Statement>(db->prepare("PRAGMA freelist_count"));
} else {
freeStmt->reset();
}
uint64_t hardSize = cacheHardSize();
if (!hardSize) {
return 0;
}
if (freeStmt->run()) {
return hardSize - cachePageSize() * freeStmt->get<int>(0);
}
softSizeDirty = false;
} catch (mapbox::sqlite::Exception& ex) {
Log::Error(Event::Database, ex.code, ex.what());
}
return 0;
}
bool SQLiteCache::Impl::needsPruning() {
// SQLite database never shrinks in size unless we call VACCUM. We here
// are monitoring the soft limit (i.e. number of free pages in the file)
// and as it approaches to the hard limit (i.e. the actual file size) we
// delete an arbitrary number of old cache entries.
//
// The free pages approach saves us from calling VACCUM or keeping a
// running total, which can be costly. We need a buffer because pages can
// get fragmented on the database.
if (cacheSoftSize() + maximumCacheEntrySize * 2 < maximumCacheSize) {
return false;
} else {
return true;
}
}
void SQLiteCache::Impl::pruneEntries() {
if (!maximumCacheSize) {
return;
}
if (!needsPruning()) {
return;
}
try {
if (!pruneStmt) {
pruneStmt = std::make_unique<Statement>(db->prepare(
"DELETE FROM `http_cache` WHERE `rowid` IN (SELECT `rowid` FROM "
// 1
"`http_cache` ORDER BY `accessed` ASC LIMIT ?)"));
} else {
pruneStmt->reset();
}
pruneStmt->bind(1, kPrunedEntriesLimit);
pruneStmt->run();
softSizeDirty = true;
} catch (mapbox::sqlite::Exception& ex) {
Log::Error(Event::Database, ex.code, ex.what());
}
}
std::unique_ptr<WorkRequest> SQLiteCache::get(const Resource &resource, Callback callback) {
// Can be called from any thread, but most likely from the file source thread.
// Will try to load the URL from the SQLite database and call the callback when done.
// Note that the callback is probably going to invoked from another thread, so the caller
// must make sure that it can run in that thread.
return thread->invokeWithCallback(&Impl::get, callback, resource);
}
void SQLiteCache::Impl::get(const Resource &resource, Callback callback) {
try {
initializeDatabase();
if (!getStmt) {
// Initialize the statement 0 1
getStmt = std::make_unique<Statement>(db->prepare("SELECT `status`, `modified`, "
// 2 3 4 5 1
"`etag`, `expires`, `data`, `compressed` FROM `http_cache` WHERE `url` = ?"));
} else {
getStmt->reset();
}
getStmt->bind(1, resource.url);
if (getStmt->run()) {
// There is data.
auto response = std::make_unique<Response>();
const auto status = getStmt->get<int>(0);
if (status > 1) {
// Status codes > 1 indicate an error
response->error = std::make_unique<Response::Error>(Response::Error::Reason(status));
}
response->modified = getStmt->get<optional<SystemTimePoint>>(1);
response->etag = getStmt->get<optional<std::string>>(2);
response->expires = getStmt->get<optional<SystemTimePoint>>(3);
response->data = std::make_shared<std::string>(getStmt->get<std::string>(4));
if (getStmt->get<int>(5)) { // == compressed
response->data = std::make_shared<std::string>(util::decompress(*response->data));
}
callback(std::move(response));
} else {
// There is no data.
callback(nullptr);
}
// We do an extra query for refreshing the last time
// the record was accessed that can be costly and is only
// worth doing if we are monitoring the database size.
if (maximumCacheSize) {
if (!accessedStmt) {
accessedStmt = std::make_unique<Statement>(
// 1 2
db->prepare("UPDATE `http_cache` SET `accessed` = ? WHERE `url` = ?"));
} else {
accessedStmt->reset();
}
accessedStmt->bind(1, SystemClock::now());
accessedStmt->bind(2, resource.url);
accessedStmt->run();
}
} catch (mapbox::sqlite::Exception& ex) {
Log::Error(Event::Database, ex.code, ex.what());
callback(nullptr);
} catch (std::runtime_error& ex) {
Log::Error(Event::Database, "%s", ex.what());
callback(nullptr);
}
}
void SQLiteCache::put(const Resource& resource, const Response& response) {
// Except for 404s, don't store errors in the cache.
if (response.error && response.error->reason != Response::Error::Reason::NotFound) {
return;
}
if (response.notModified) {
thread->invoke(&Impl::refresh, resource, response.expires);
} else {
thread->invoke(&Impl::put, resource, response);
}
}
void SQLiteCache::Impl::put(const Resource& resource, const Response& response) {
try {
initializeDatabase();
pruneEntries();
if (response.data) {
auto entrySize = response.data->size();
if (entrySize > maximumCacheEntrySize) {
Log::Warning(Event::Database, "Entry too big for caching.");
return;
}
if (maximumCacheSize && entrySize + cacheSoftSize() > maximumCacheSize) {
Log::Warning(Event::Database, "Unable to make space for new entries.");
return;
}
}
if (!putStmt) {
putStmt = std::make_unique<Statement>(db->prepare("REPLACE INTO `http_cache` ("
// 1 2 3 4 5 6 7 8 9
"`url`, `status`, `kind`, `modified`, `etag`, `expires`, `accessed`, `data`, `compressed`"
") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)"));
} else {
putStmt->reset();
}
putStmt->bind(1 /* url */, resource.url);
if (response.error) {
putStmt->bind(2 /* status */, int(response.error->reason));
} else {
putStmt->bind(2 /* status */, 1 /* success */);
}
putStmt->bind(3 /* kind */, int(resource.kind));
putStmt->bind(4 /* modified */, response.modified);
putStmt->bind(5 /* etag */, response.etag);
putStmt->bind(6 /* expires */, response.expires);
putStmt->bind(7 /* accessed */, SystemClock::now());
std::string data;
if (resource.kind != Resource::SpriteImage && response.data) {
// Do not compress images, since they are typically compressed already.
data = util::compress(*response.data);
}
if (!data.empty() && data.size() < response.data->size()) {
// Store the compressed data when it is smaller than the original
// uncompressed data.
putStmt->bind(8 /* data */, data, false); // do not retain the string internally.
putStmt->bind(9 /* compressed */, true);
} else if (response.data) {
putStmt->bind(8 /* data */, *response.data, false); // do not retain the string internally.
putStmt->bind(9 /* compressed */, false);
} else {
putStmt->bind(8 /* data */, "", false);
putStmt->bind(9 /* compressed */, false);
}
putStmt->run();
softSizeDirty = true;
} catch (mapbox::sqlite::Exception& ex) {
Log::Error(Event::Database, ex.code, ex.what());
} catch (std::runtime_error& ex) {
Log::Error(Event::Database, "%s", ex.what());
}
}
void SQLiteCache::Impl::refresh(const Resource& resource, optional<SystemTimePoint> expires) {
try {
initializeDatabase();
if (!refreshStmt) {
refreshStmt = std::make_unique<Statement>(
db->prepare("UPDATE `http_cache` SET "
// 1 2 3
"`accessed` = ?, `expires` = ? WHERE `url` = ?"));
} else {
refreshStmt->reset();
}
refreshStmt->bind(1, SystemClock::now());
refreshStmt->bind(2, expires);
refreshStmt->bind(3, resource.url);
refreshStmt->run();
} catch (mapbox::sqlite::Exception& ex) {
Log::Error(Event::Database, ex.code, ex.what());
}
}
namespace {
static std::mutex sharedMutex;
static std::unordered_map<std::string, std::weak_ptr<SQLiteCache>> shared;
} // namespace
std::shared_ptr<SQLiteCache> SQLiteCache::getShared(const std::string &path) {
std::lock_guard<std::mutex> lock(sharedMutex);
std::shared_ptr<SQLiteCache> cache;
auto it = shared.find(path);
if (it != shared.end()) {
cache = it->second.lock();
if (!cache) {
cache = std::make_shared<SQLiteCache>(path);
it->second = cache;
}
} else {
cache = std::make_shared<SQLiteCache>(path);
shared.emplace(path, cache);
}
return cache;
}
} // namespace mbgl

View File

@@ -1,68 +0,0 @@
#ifndef MBGL_STORAGE_DEFAULT_SQLITE_CACHE_IMPL
#define MBGL_STORAGE_DEFAULT_SQLITE_CACHE_IMPL
#include <mbgl/storage/sqlite_cache.hpp>
#include <mbgl/util/chrono.hpp>
#include <mbgl/util/optional.hpp>
namespace mapbox {
namespace sqlite {
class Database;
class Statement;
}
}
namespace mbgl {
class SQLiteCache::Impl {
public:
explicit Impl(const std::string &path = ":memory:");
~Impl();
void setMaximumCacheSize(uint64_t size);
void setMaximumCacheEntrySize(uint64_t size);
void get(const Resource&, Callback);
void put(const Resource&, const Response&);
void refresh(const Resource&, optional<SystemTimePoint> expires);
private:
void initializeDatabase();
int cachePageSize();
uint64_t cacheHardSize();
uint64_t cacheSoftSize();
uint64_t softSize = 0;
bool softSizeDirty = true;
bool needsPruning();
void pruneEntries();
void createDatabase();
void createSchema();
int schemaVersion() const;
void ensureSchemaVersion();
int pageSize = 0;
uint64_t maximumCacheSize;
uint64_t maximumCacheEntrySize;
const std::string path;
std::unique_ptr<::mapbox::sqlite::Database> db;
std::unique_ptr<::mapbox::sqlite::Statement> getStmt;
std::unique_ptr<::mapbox::sqlite::Statement> putStmt;
std::unique_ptr<::mapbox::sqlite::Statement> refreshStmt;
std::unique_ptr<::mapbox::sqlite::Statement> countStmt;
std::unique_ptr<::mapbox::sqlite::Statement> freeStmt;
std::unique_ptr<::mapbox::sqlite::Statement> pruneStmt;
std::unique_ptr<::mapbox::sqlite::Statement> accessedStmt;
bool schema = false;
};
} // namespace mbgl
#endif

View File

@@ -1,44 +0,0 @@
#ifndef MBGL_STORAGE_DEFAULT_SQLITE_CACHE
#define MBGL_STORAGE_DEFAULT_SQLITE_CACHE
#include <mbgl/util/noncopyable.hpp>
#include <mbgl/util/chrono.hpp>
#include <functional>
#include <memory>
#include <string>
namespace mbgl {
class Resource;
class Response;
class WorkRequest;
namespace util {
template <typename T> class Thread;
} // namespace util
class SQLiteCache : private util::noncopyable {
public:
static std::shared_ptr<SQLiteCache> getShared(const std::string &path = ":memory:");
SQLiteCache(const std::string &path = ":memory:");
~SQLiteCache();
void setMaximumCacheSize(uint64_t size);
void setMaximumCacheEntrySize(uint64_t size);
using Callback = std::function<void(std::unique_ptr<Response>)>;
std::unique_ptr<WorkRequest> get(const Resource&, Callback);
void put(const Resource&, const Response&);
class Impl;
private:
const std::unique_ptr<util::Thread<Impl>> thread;
};
} // namespace mbgl
#endif

50
test/api/offline.cpp Normal file
View File

@@ -0,0 +1,50 @@
#include "../fixtures/util.hpp"
#include <mbgl/platform/default/headless_display.hpp>
#include <mbgl/platform/default/headless_view.hpp>
#include <mbgl/storage/offline_database.hpp>
#include <mbgl/storage/default_file_source.hpp>
#include <mbgl/platform/log.hpp>
#include <mbgl/util/work_request.hpp>
#include <mbgl/util/io.hpp>
using namespace mbgl;
using namespace std::literals::chrono_literals;
using namespace std::literals::string_literals;
namespace {
Response expiredItem(const std::string& path) {
Response response;
response.data = std::make_shared<std::string>(util::read_file("test/fixtures/"s + path));
response.expires = SystemClock::from_time_t(0);
return response;
}
const std::string prefix = "http://127.0.0.1:3000";
}
auto display = std::make_shared<mbgl::HeadlessDisplay>();
TEST(API, Offline) {
HeadlessView view(display, 1);
DefaultFileSource fileSource(":memory:", ".");
fileSource.put(Resource::style(prefix + "/offline/style.json"), expiredItem("offline/style.json"));
fileSource.put(Resource::source(prefix + "/offline/streets.json"), expiredItem("offline/streets.json"));
fileSource.put(Resource::spriteJSON(prefix + "/offline/sprite", 1.0), expiredItem("offline/sprite.json"));
fileSource.put(Resource::spriteImage(prefix + "/offline/sprite", 1.0), expiredItem("offline/sprite.png"));
fileSource.put(Resource::tile(prefix + "/offline/{z}-{x}-{y}.vector.pbf", 1.0, 0, 0, 0), expiredItem("offline/0-0-0.vector.pbf"));
fileSource.put(Resource::glyphs(prefix + "/offline/{fontstack}/{range}.pbf", "Helvetica", {0, 255}), expiredItem("offline/glyph.pbf"));
fileSource.goOffline();
Map map(view, fileSource, MapMode::Still);
map.setStyleURL(prefix + "/offline/style.json");
test::checkImage("test/fixtures/offline"s,
test::render(map),
0.0015,
0.1);
}

BIN
test/fixtures/offline/expected.png vendored Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 47 KiB

View File

Before

Width:  |  Height:  |  Size: 3.3 KiB

After

Width:  |  Height:  |  Size: 3.3 KiB

View File

@@ -10,5 +10,5 @@
"name": "Streets",
"scheme": "xyz",
"tilejson": "2.0.0",
"tiles": [ "asset://test/fixtures/stale/{z}-{x}-{y}.vector.pbf" ]
"tiles": [ "http://127.0.0.1:3000/offline/{z}-{x}-{y}.vector.pbf" ]
}

View File

@@ -4,10 +4,11 @@
"sources": {
"mapbox": {
"type": "vector",
"url": "asset://test/fixtures/stale/streets.json"
"url": "http://127.0.0.1:3000/offline/streets.json"
}
},
"glyphs": "http://127.0.0.1:3000/stale/{fontstack}/{range}.pbf",
"glyphs": "http://127.0.0.1:3000/offline/{fontstack}/{range}.pbf",
"sprite": "http://127.0.0.1:3000/offline/sprite",
"layers": [{
"id": "background",
"type": "background",
@@ -20,7 +21,7 @@
"source": "mapbox",
"source-layer": "water",
"paint": {
"fill-color": "blue"
"fill-pattern": "noise"
}
}, {
"id": "admin",
@@ -28,7 +29,8 @@
"source": "mapbox",
"source-layer": "admin",
"layout": {
"text-field": "Text"
"text-font": ["Helvetica"],
"text-field": "Text"
},
"paint": {
"text-color": "black",

Binary file not shown.

Before

Width:  |  Height:  |  Size: 13 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 35 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 26 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 13 KiB

View File

@@ -1,25 +0,0 @@
{
"version": 8,
"name": "Water",
"sources": {
"mapbox": {
"type": "vector",
"url": "asset://test/fixtures/stale/streets.json"
}
},
"layers": [{
"id": "background",
"type": "background",
"paint": {
"background-color": "red"
}
}, {
"id": "water",
"type": "fill",
"source": "mapbox",
"source-layer": "water",
"paint": {
"fill-color": "blue"
}
}]
}

View File

@@ -1,26 +0,0 @@
{
"version": 8,
"name": "Water",
"sources": {
"mapbox": {
"type": "vector",
"url": "asset://test/fixtures/stale/streets.json"
}
},
"sprite": "http://127.0.0.1:3000/stale/sprite",
"layers": [{
"id": "background",
"type": "background",
"paint": {
"background-color": "red"
}
}, {
"id": "water",
"type": "fill",
"source": "mapbox",
"source-layer": "water",
"paint": {
"fill-pattern": "noise"
}
}]
}

View File

@@ -1,25 +0,0 @@
{
"version": 8,
"name": "Water",
"sources": {
"mapbox": {
"type": "vector",
"url": "http://127.0.0.1:3000/stale/streets.json"
}
},
"layers": [{
"id": "background",
"type": "background",
"paint": {
"background-color": "red"
}
}, {
"id": "water",
"type": "fill",
"source": "mapbox",
"source-layer": "water",
"paint": {
"fill-color": "blue"
}
}]
}

View File

@@ -106,20 +106,12 @@ uint64_t crc64(const PremultipliedImage &image) {
return crc64(reinterpret_cast<const char*>(image.data.get()), image.size());
}
PremultipliedImage render(Map& map, Milliseconds timeout) {
PremultipliedImage render(Map& map) {
std::promise<PremultipliedImage> promise;
map.renderStill([&](std::exception_ptr, PremultipliedImage&& image) {
promise.set_value(std::move(image));
});
// Limit maximum wait time.
auto future = promise.get_future();
if (future.wait_for(timeout) != std::future_status::ready) {
// Alas, we didn't get the promised future :(
Log::Error(Event::Image, "Failed to generate image within %dms", timeout.count());
return { 0, 0 };
}
return future.get();
return promise.get_future().get();
}
void checkImage(const std::string& base,

View File

@@ -4,7 +4,6 @@
#include <mbgl/util/image.hpp>
#include <mbgl/util/chrono.hpp>
#include <chrono>
#include <cstdint>
#include <gtest/gtest.h>
@@ -36,12 +35,13 @@ uint64_t crc64(const char*, size_t);
uint64_t crc64(const std::string&);
uint64_t crc64(const PremultipliedImage&);
PremultipliedImage render(Map&, Milliseconds timeout = Milliseconds(1000));
PremultipliedImage render(Map&);
void checkImage(const std::string& base,
const PremultipliedImage& actual,
double imageThreshold = 0,
double pixelThreshold = 0);
}
}

View File

@@ -1,7 +1,6 @@
#include "storage.hpp"
#include <mbgl/storage/asset_file_source.hpp>
#include <mbgl/storage/sqlite_cache.hpp>
#include <mbgl/platform/platform.hpp>
#include <mbgl/util/chrono.hpp>
#include <mbgl/util/run_loop.hpp>

View File

@@ -1,215 +0,0 @@
#include "storage.hpp"
#include <mbgl/storage/default_file_source.hpp>
#include <mbgl/storage/sqlite_cache.hpp>
#include <mbgl/util/chrono.hpp>
#include <mbgl/util/run_loop.hpp>
TEST_F(Storage, CacheResponse) {
SCOPED_TEST(CacheResponse);
using namespace mbgl;
util::RunLoop loop;
DefaultFileSource fs(":memory:", ".");
const Resource resource { Resource::Unknown, "http://127.0.0.1:3000/cache" };
Response response;
std::unique_ptr<FileRequest> req1;
std::unique_ptr<FileRequest> req2;
req1 = fs.request(resource, [&](Response res) {
req1.reset();
EXPECT_EQ(nullptr, res.error);
ASSERT_TRUE(res.data.get());
EXPECT_EQ("Response 1", *res.data);
EXPECT_TRUE(bool(res.expires));
EXPECT_FALSE(bool(res.modified));
EXPECT_FALSE(bool(res.etag));
response = res;
// Now test that we get the same values as in the previous request. If we'd go to the server
// again, we'd get different values.
req2 = fs.request(resource, [&](Response res2) {
req2.reset();
EXPECT_EQ(response.error, res2.error);
ASSERT_TRUE(res2.data.get());
EXPECT_EQ(*response.data, *res2.data);
EXPECT_EQ(response.expires, res2.expires);
EXPECT_EQ(response.modified, res2.modified);
EXPECT_EQ(response.etag, res2.etag);
loop.stop();
CacheResponse.finish();
});
});
loop.run();
}
// Make sure we /do/ store 404 Not Found responses into the cache
TEST_F(Storage, CacheNotFound) {
SCOPED_TEST(CacheNotFound);
using namespace mbgl;
util::RunLoop loop;
DefaultFileSource fs(":memory:", ".");
const Resource resource{ Resource::Unknown, "http://127.0.0.1:3000/not-found" };
// Insert existing data into the cache that will be marked as stale.
Response response;
response.data = std::make_shared<const std::string>("existing data");
fs.getCache().put(resource, response);
std::unique_ptr<FileRequest> req1;
std::unique_ptr<WorkRequest> req2;
int counter = 0;
// Then, request the actual URL and check that we're getting the rigged cache response first,
// then the connection error message.
req1 = fs.request(resource, [&](Response res) {
if (counter == 0) {
EXPECT_EQ(nullptr, res.error);
ASSERT_TRUE(res.data.get());
EXPECT_EQ("existing data", *res.data);
EXPECT_FALSE(bool(res.expires));
EXPECT_FALSE(bool(res.modified));
EXPECT_FALSE(bool(res.etag));
} else if (counter == 1) {
EXPECT_NE(nullptr, res.error);
EXPECT_EQ(Response::Error::Reason::NotFound, res.error->reason);
ASSERT_TRUE(res.data.get());
EXPECT_EQ("Not Found!", *res.data);
req1.reset();
// Finally, check the cache to make sure we cached the 404 response.
req2 = fs.getCache().get(resource, [&](std::unique_ptr<Response> res2) {
EXPECT_NE(nullptr, res2->error);
EXPECT_EQ(Response::Error::Reason::NotFound, res2->error->reason);
ASSERT_TRUE(res2->data.get());
EXPECT_EQ("Not Found!", *res2->data);
CacheNotFound.finish();
loop.stop();
});
} else {
FAIL() << "Got too many responses";
}
counter++;
});
loop.run();
}
// Make sure we don't store a connection error into the cache
TEST_F(Storage, DontCacheConnectionErrors) {
SCOPED_TEST(DontCacheConnectionErrors);
using namespace mbgl;
util::RunLoop loop;
DefaultFileSource fs(":memory:", ".");
const Resource resource{ Resource::Unknown, "http://127.0.0.1:3001" };
// Insert existing data into the cache that will be marked as stale.
Response response;
response.data = std::make_shared<const std::string>("existing data");
fs.getCache().put(resource, response);
std::unique_ptr<FileRequest> req1;
std::unique_ptr<WorkRequest> req2;
int counter = 0;
// Then, request the actual URL and check that we're getting the rigged cache response first,
// then the connection error message.
req1 = fs.request(resource, [&](Response res) {
if (counter == 0) {
EXPECT_EQ(nullptr, res.error);
ASSERT_TRUE(res.data.get());
EXPECT_EQ("existing data", *res.data);
EXPECT_FALSE(bool(res.expires));
EXPECT_FALSE(bool(res.modified));
EXPECT_FALSE(bool(res.etag));
} else if (counter == 1) {
EXPECT_NE(nullptr, res.error);
EXPECT_EQ(Response::Error::Reason::Connection, res.error->reason);
req1.reset();
// Finally, check the cache to make sure we still have our original data in there rather
// than the failed connection attempt.
req2 = fs.getCache().get(resource, [&](std::unique_ptr<Response> res2) {
EXPECT_EQ(nullptr, res2->error);
ASSERT_TRUE(res2->data.get());
EXPECT_EQ("existing data", *res2->data);
DontCacheConnectionErrors.finish();
loop.stop();
});
} else {
FAIL() << "Got too many responses";
}
counter++;
});
loop.run();
}
// Make sure we don't store a bad server response into the cache
TEST_F(Storage, DontCacheServerErrors) {
SCOPED_TEST(DontCacheServerErrors);
using namespace mbgl;
util::RunLoop loop;
DefaultFileSource fs(":memory:", ".");
const Resource resource{ Resource::Unknown, "http://127.0.0.1:3000/permanent-error" };
// Insert existing data into the cache that will be marked as stale.
Response response;
response.data = std::make_shared<const std::string>("existing data");
fs.getCache().put(resource, response);
std::unique_ptr<FileRequest> req1;
std::unique_ptr<WorkRequest> req2;
int counter = 0;
// Then, request the actual URL and check that we're getting the rigged cache response first,
// then the server error message.
req1 = fs.request(resource, [&](Response res) {
if (counter == 0) {
EXPECT_EQ(nullptr, res.error);
ASSERT_TRUE(res.data.get());
EXPECT_EQ("existing data", *res.data);
EXPECT_FALSE(bool(res.expires));
EXPECT_FALSE(bool(res.modified));
EXPECT_FALSE(bool(res.etag));
} else if (counter == 1) {
EXPECT_NE(nullptr, res.error);
EXPECT_EQ(Response::Error::Reason::Server, res.error->reason);
ASSERT_TRUE(res.data.get());
EXPECT_EQ("Server Error!", *res.data);
req1.reset();
// Finally, check the cache to make sure we still have our original data in there rather
// than the failed connection attempt.
req2 = fs.getCache().get(resource, [&](std::unique_ptr<Response> res2) {
EXPECT_EQ(nullptr, res2->error);
ASSERT_TRUE(res2->data.get());
EXPECT_EQ("existing data", *res2->data);
DontCacheServerErrors.finish();
loop.stop();
});
} else {
FAIL() << "Got too many responses";
}
counter++;
});
loop.run();
}

View File

@@ -1,34 +0,0 @@
#include "storage.hpp"
#include <mbgl/storage/sqlite_cache.hpp>
#include <mbgl/storage/resource.hpp>
#include <mbgl/util/run_loop.hpp>
TEST_F(Storage, CacheShared) {
SCOPED_TEST(CacheShared)
using namespace mbgl;
util::RunLoop loop;
// Check that we're getting two different caches when we request different paths.
auto memory = SQLiteCache::getShared();
auto file = SQLiteCache::getShared("test/fixtures/database/cache.db");
EXPECT_NE(memory.get(), file.get());
EXPECT_EQ(memory.get(), SQLiteCache::getShared().get());
// Store a response into the memory file cache, then delete the last reference.
const Resource resource { Resource::Kind::Unknown, "http://example.com" };
memory->put(resource, Response());
memory.reset();
// Now check that the original memory file cache has been destructed and that it doesn't contain
// the information we put into it.
memory = SQLiteCache::getShared();
auto req = memory->get(resource, [&](std::unique_ptr<Response> res) {
EXPECT_FALSE(res.get());
CacheShared.finish();
loop.stop();
});
loop.run();
}

View File

@@ -1,241 +0,0 @@
#include "storage.hpp"
#include <mbgl/storage/resource.hpp>
#include <mbgl/storage/response.hpp>
#include <mbgl/storage/sqlite_cache.hpp>
#include <mbgl/util/run_loop.hpp>
#include <mbgl/util/string.hpp>
#include <mbgl/util/timer.hpp>
#include <mbgl/util/chrono.hpp>
#include <memory>
#include <random>
bool tileIsCached(mbgl::SQLiteCache* cache, unsigned id) {
using namespace mbgl;
auto url = std::string("http://tile") + mbgl::util::toString(id);
bool replied = false;
std::unique_ptr<Response> response;
auto callback = [&] (std::unique_ptr<Response> res) {
replied = true;
response = std::move(res);
};
Resource resource{ Resource::Kind::Tile, url };
auto req = cache->get(resource, callback);
while (!replied) {
util::RunLoop::Get()->runOnce();
}
return response != nullptr;
}
void insertTile(mbgl::SQLiteCache* cache, unsigned id, uint64_t size) {
using namespace mbgl;
auto url = std::string("http://tile") + mbgl::util::toString(id);
Response response;
response.modified = SystemClock::now();
response.expires = SystemClock::now() + Seconds(5000);
response.etag = url;
auto data = std::make_shared<std::string>(size, 0);
// Fill data with garbage so SQLite won't try to
// optimize allocation by reusing pages.
static std::mt19937 generator;
std::generate_n(data->begin(), size, generator);
response.data = data;
Resource resource{ Resource::Kind::Tile, url };
cache->put(resource, response);
}
void refreshTile(mbgl::SQLiteCache* cache, unsigned id) {
using namespace mbgl;
auto url = std::string("http://tile") + mbgl::util::toString(id);
Response response;
response.modified = SystemClock::now();
response.expires = SystemClock::now() + Seconds(5000);
response.notModified = true;
Resource resource{ Resource::Kind::Tile, url };
cache->put(resource, response);
}
uint64_t cacheSize(mbgl::SQLiteCache* cache, unsigned entryCount, uint64_t entrySize) {
uint64_t total = 0;
for (unsigned i = 0; i < entryCount; ++i) {
if (tileIsCached(cache, i)) {
total += entrySize;
}
}
return total;
}
TEST_F(Storage, CacheEntrySizeLimit) {
using namespace mbgl;
util::RunLoop loop;
SQLiteCache cache(":memory:");
const uint64_t entrySize = 5 * 1024 * 1024; // 5 MB
insertTile(&cache, 0, entrySize);
EXPECT_TRUE(tileIsCached(&cache, 0));
insertTile(&cache, 1, entrySize + 1);
EXPECT_FALSE(tileIsCached(&cache, 1));
insertTile(&cache, 2, entrySize - 1);
EXPECT_TRUE(tileIsCached(&cache, 2));
// Setting a new size should not delete existing entries.
cache.setMaximumCacheEntrySize(entrySize / 2);
EXPECT_TRUE(tileIsCached(&cache, 2));
insertTile(&cache, 3, entrySize / 2 - 1);
EXPECT_TRUE(tileIsCached(&cache, 3));
insertTile(&cache, 4, entrySize);
EXPECT_FALSE(tileIsCached(&cache, 4));
cache.setMaximumCacheEntrySize(entrySize * 2);
insertTile(&cache, 5, entrySize);
EXPECT_TRUE(tileIsCached(&cache, 5));
}
TEST_F(Storage, CacheSizeSetNewLimit) {
using namespace mbgl;
util::RunLoop loop;
SQLiteCache cache(":memory:");
const unsigned entryCount = 800;
const uint64_t entrySize = 10 * 1024; // 10 KB
cache.setMaximumCacheEntrySize(entrySize + 1);
// Cache size defaults to unlimited, all these
// inserts should work.
for (unsigned i = 0; i < entryCount; ++i) {
insertTile(&cache, i, entrySize);
}
for (unsigned i = 0; i < entryCount; ++i) {
EXPECT_TRUE(tileIsCached(&cache, i));
}
uint64_t expectedCacheSize = entryCount * entrySize;
EXPECT_EQ(cacheSize(&cache, entryCount, entrySize), expectedCacheSize);
// Setting a new size should remove records until the new
// size limit is satisfied.
cache.setMaximumCacheSize(expectedCacheSize / 2);
EXPECT_LT(cacheSize(&cache, entryCount, entrySize), expectedCacheSize / 2);
// Cache size 1 should practically clean the cache and
// prevent adding any record, although it makes no sense
// to use such size limit IRL.
cache.setMaximumCacheSize(1);
EXPECT_EQ(cacheSize(&cache, entryCount, entrySize), 0);
insertTile(&cache, 1000, entrySize);
EXPECT_FALSE(tileIsCached(&cache, 1000));
// Zero should be treated as unlimited.
cache.setMaximumCacheSize(0);
for (unsigned i = 0; i < entryCount; ++i) {
insertTile(&cache, i, entrySize);
}
EXPECT_EQ(cacheSize(&cache, entryCount, entrySize), expectedCacheSize);
}
TEST_F(Storage, CacheSizePruneLeastAccessed) {
using namespace mbgl;
util::RunLoop loop;
SQLiteCache cache(":memory:");
const unsigned entryCount = 400;
const uint64_t entrySize = 10 * 1024; // 10 KB
cache.setMaximumCacheEntrySize(entrySize + 1);
cache.setMaximumCacheSize(entrySize * 350);
for (unsigned i = 0; i < entryCount; ++i) {
insertTile(&cache, i, entrySize);
if (i == entryCount / 2) {
// We need to sleep for 1s here because
// that is the time resolution for the
// `accessed` time. Then we 'ping' the
// entry, that should update the
// `accessed` time, so it won't get
// pruned when we need more space.
bool done = false;
util::Timer timer;
timer.start(Milliseconds(1300),
Duration::zero(),
[&done] { done = true; });
while (!done) {
loop.runOnce();
}
EXPECT_TRUE(tileIsCached(&cache, 7));
// Refresh should also update the `accessed`
// time of a tile.
refreshTile(&cache, 9);
}
}
EXPECT_FALSE(tileIsCached(&cache, 6));
EXPECT_FALSE(tileIsCached(&cache, 8));
EXPECT_FALSE(tileIsCached(&cache, 10));
EXPECT_TRUE(tileIsCached(&cache, 7));
EXPECT_TRUE(tileIsCached(&cache, 9));
}
TEST_F(Storage, CacheSizeStress) {
using namespace mbgl;
util::RunLoop loop;
SQLiteCache cache(":memory:");
const unsigned entryCount = 2000;
const uint64_t entrySize = 10 * 1024; // 10 KB
cache.setMaximumCacheEntrySize(entrySize + 1);
cache.setMaximumCacheSize(entrySize * 300);
for (unsigned i = 0; i < entryCount; ++i) {
insertTile(&cache, i, entrySize);
}
// Should not be in the cache as they were
// first inserted.
EXPECT_FALSE(tileIsCached(&cache, 0));
EXPECT_FALSE(tileIsCached(&cache, 99));
EXPECT_FALSE(tileIsCached(&cache, 199));
EXPECT_FALSE(tileIsCached(&cache, 299));
EXPECT_FALSE(tileIsCached(&cache, 399));
EXPECT_TRUE(tileIsCached(&cache, entryCount - 1));
EXPECT_LT(cacheSize(&cache, entryCount, entrySize), entrySize * 300);
}

View File

@@ -1,131 +0,0 @@
#include "storage.hpp"
#include <mbgl/platform/default/headless_display.hpp>
#include <mbgl/platform/default/headless_view.hpp>
#include <mbgl/storage/sqlite_cache.hpp>
#include <mbgl/storage/default_file_source.hpp>
#include <mbgl/platform/log.hpp>
#include <mbgl/util/work_request.hpp>
#include <mbgl/util/io.hpp>
using namespace mbgl;
using namespace std::literals::chrono_literals;
using namespace std::literals::string_literals;
namespace {
void checkRendering(Map& map,
const char* name,
std::chrono::milliseconds timeout,
double imageThreshold = 0.001,
double pixelThreshold = 0.1) {
test::checkImage("test/fixtures/stale/"s + name, test::render(map, timeout), imageThreshold,
pixelThreshold);
}
Response expiredItem(const std::string& path) {
Response response;
response.data = std::make_shared<std::string>(util::read_file("test/fixtures/"s + path));
response.expires = SystemClock::from_time_t(0);
return response;
}
const std::string prefix = "http://127.0.0.1:3000";
}
auto display = std::make_shared<mbgl::HeadlessDisplay>();
TEST_F(Storage, CacheStaleStyle) {
HeadlessView view(display, 1);
auto cache = SQLiteCache::getShared(":memory:");
// Rig the cache with an expired stylesheet.
const std::string stylePath = "stale/style.json";
const Resource styleResource{ Resource::Kind::Style, prefix + "/" + stylePath };
cache->put(styleResource, expiredItem(stylePath));
DefaultFileSource fileSource(":memory:", ".");
Map map(view, fileSource, MapMode::Still);
map.setStyleURL(styleResource.url);
checkRendering(map, "stale_style", 1000ms);
}
TEST_F(Storage, CacheStaleStyleAndTileJSON) {
HeadlessView view(display, 1);
auto cache = SQLiteCache::getShared(":memory:");
// Rig the cache with an expired stylesheet.
const std::string stylePath = "stale/style_and_tilejson.json";
const Resource styleResource{ Resource::Kind::Style, prefix + "/" + stylePath };
cache->put(styleResource, expiredItem(stylePath));
// Rig the cache with an expired TileJSON.
const std::string tilejsonPath = "stale/streets.json";
const Resource tilejsonResource{ Resource::Kind::Source, prefix + "/" + tilejsonPath };
cache->put(tilejsonResource, expiredItem(tilejsonPath));
DefaultFileSource fileSource(":memory:", ".");
Map map(view, fileSource, MapMode::Still);
map.setStyleURL(styleResource.url);
checkRendering(map, "stale_style_and_tilejson", 1000ms);
}
TEST_F(Storage, CacheStaleStyleAndSprite) {
HeadlessView view(display, 1);
auto cache = SQLiteCache::getShared(":memory:");
// Rig the cache with an expired stylesheet.
const std::string stylePath = "stale/style_and_sprite.json";
const Resource styleResource{ Resource::Kind::Style, prefix + "/" + stylePath };
cache->put(styleResource, expiredItem(stylePath));
// Rig the cache with an expired sprite JSON.
const std::string spritejsonPath = "stale/sprite.json";
const Resource spritejsonResource{ Resource::Kind::SpriteJSON, prefix + "/" + spritejsonPath };
cache->put(spritejsonResource, expiredItem(spritejsonPath));
// Rig the cache with an expired sprite JSON.
const std::string spriteimagePath = "stale/sprite.png";
const Resource spriteimageResource{ Resource::Kind::SpriteImage, prefix + "/" + spriteimagePath };
cache->put(spriteimageResource, expiredItem(spriteimagePath));
DefaultFileSource fileSource(":memory:", ".");
Map map(view, fileSource, MapMode::Still);
map.setStyleURL(styleResource.url);
checkRendering(map, "stale_style_and_sprite", 1000ms);
}
TEST_F(Storage, CacheStaleStyleAndGlyphs) {
HeadlessView view(display, 1);
auto cache = SQLiteCache::getShared(":memory:");
// Rig the cache with an expired stylesheet.
const std::string stylePath = "stale/style_and_glyphs.json";
const Resource styleResource{ Resource::Kind::Style, prefix + "/" + stylePath };
cache->put(styleResource, expiredItem(stylePath));
// Rig the cache with an expired glyph PBF.
const std::string glyphPath = "stale/glyph.pbf";
const Resource glyphResource{ Resource::Kind::Glyphs, prefix + "/stale/Open%20Sans%20Regular%2c%20Arial%20Unicode%20MS%20Regular/0-255.pbf" };
cache->put(glyphResource, expiredItem(glyphPath));
DefaultFileSource fileSource(":memory:", ".");
Map map(view, fileSource, MapMode::Still);
map.setStyleURL(styleResource.url);
// TODO: this shouldn't take > 1 second
checkRendering(map, "stale_style_and_glyphs", 2000ms, 0.0015);
}

View File

@@ -1,349 +0,0 @@
#include "storage.hpp"
#include "../fixtures/fixture_log_observer.hpp"
#include "sqlite_cache_impl.hpp"
#include <mbgl/storage/resource.hpp>
#include <mbgl/storage/response.hpp>
#include <mbgl/util/io.hpp>
#include <sqlite3.h>
TEST_F(Storage, DatabaseDoesNotExist) {
using namespace mbgl;
Log::setObserver(std::make_unique<FixtureLogObserver>());
SQLiteCache::Impl cache("test/fixtures/404/cache.db");
cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
EXPECT_EQ(nullptr, res.get());
});
auto observer = Log::removeObserver();
EXPECT_EQ(1ul, dynamic_cast<FixtureLogObserver*>(observer.get())->count({ EventSeverity::Error, Event::Database, 14, "unable to open database file" }));
}
void createDir(const char* name) {
const int ret = mkdir(name, 0755);
if (ret == -1) {
ASSERT_EQ(EEXIST, errno);
} else {
ASSERT_EQ(0, ret);
}
}
void deleteFile(const char* name) {
const int ret = unlink(name);
if (ret == -1) {
ASSERT_EQ(ENOENT, errno);
} else {
ASSERT_EQ(0, ret);
}
}
void writeFile(const char* name, const std::string& data) {
mbgl::util::write_file(name, data);
}
TEST_F(Storage, DatabaseCreate) {
using namespace mbgl;
createDir("test/fixtures/database");
deleteFile("test/fixtures/database/cache.db");
Log::setObserver(std::make_unique<FixtureLogObserver>());
SQLiteCache::Impl cache("test/fixtures/database/cache.db");
cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
EXPECT_EQ(nullptr, res.get());
});
Log::removeObserver();
}
TEST_F(Storage, DatabaseVersion) {
using namespace mbgl;
createDir("test/fixtures/database");
deleteFile("test/fixtures/database/cache.db");
std::string path("test/fixtures/database/cache.db");
Log::setObserver(std::make_unique<FixtureLogObserver>());
{
SQLiteCache::Impl cache(path);
cache.put({ Resource::Unknown, "mapbox://test" }, Response());
}
sqlite3* db;
sqlite3_open_v2(path.c_str(), &db, SQLITE_OPEN_READWRITE, nullptr);
sqlite3_exec(db, "PRAGMA user_version = 999999", nullptr, nullptr, nullptr);
sqlite3_close_v2(db);
// Changing the version will force the database to get recreated
// thus removing every pre-existing cache entry.
{
SQLiteCache::Impl cache(path);
cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
EXPECT_EQ(nullptr, res.get());
});
}
Log::removeObserver();
}
class FileLock {
public:
FileLock(const std::string& path) {
const int err = sqlite3_open_v2(path.c_str(), &db, SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE, nullptr);
if (err != SQLITE_OK) {
throw std::runtime_error("Could not open db");
}
lock();
}
void lock() {
assert(!locked);
const int err = sqlite3_exec(db, "begin exclusive transaction", nullptr, nullptr, nullptr);
if (err != SQLITE_OK) {
throw std::runtime_error("Could not lock db");
}
locked = true;
}
void unlock() {
assert(locked);
const int err = sqlite3_exec(db, "commit", nullptr, nullptr, nullptr);
if (err != SQLITE_OK) {
throw std::runtime_error("Could not unlock db");
}
locked = false;
}
~FileLock() {
if (locked) {
unlock();
}
}
private:
sqlite3* db;
bool locked = false;
};
TEST_F(Storage, DatabaseLockedRead) {
using namespace mbgl;
// Create a locked file.
createDir("test/fixtures/database");
deleteFile("test/fixtures/database/locked.db");
FileLock guard("test/fixtures/database/locked.db");
SQLiteCache::Impl cache("test/fixtures/database/locked.db");
{
// First request should fail.
Log::setObserver(std::make_unique<FixtureLogObserver>());
cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
EXPECT_EQ(nullptr, res.get());
});
// Make sure that we got a few "database locked" errors
auto observer = Log::removeObserver();
auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
EXPECT_EQ(4ul, flo->count({ EventSeverity::Error, Event::Database, 5, "database is locked" }));
}
// Then, unlock the file and try again.
guard.unlock();
{
// First, try getting a file (the cache value should not exist).
Log::setObserver(std::make_unique<FixtureLogObserver>());
cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
EXPECT_EQ(nullptr, res.get());
});
// Make sure that we got a no errors
Log::removeObserver();
}
}
TEST_F(Storage, DatabaseLockedWrite) {
using namespace mbgl;
// Create a locked file.
createDir("test/fixtures/database");
deleteFile("test/fixtures/database/locked.db");
FileLock guard("test/fixtures/database/locked.db");
SQLiteCache::Impl cache("test/fixtures/database/locked.db");
{
// Adds a file (which should fail).
Log::setObserver(std::make_unique<FixtureLogObserver>());
cache.put({ Resource::Unknown, "mapbox://test" }, Response());
cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
EXPECT_EQ(nullptr, res.get());
});
auto observer = Log::removeObserver();
auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
EXPECT_EQ(8ul, flo->count({ EventSeverity::Error, Event::Database, 5, "database is locked" }));
}
// Then, unlock the file and try again.
guard.unlock();
{
// Then, set a file and obtain it again.
Log::setObserver(std::make_unique<FixtureLogObserver>());
Response response;
response.data = std::make_shared<std::string>("Demo");
cache.put({ Resource::Unknown, "mapbox://test" }, response);
cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
ASSERT_NE(nullptr, res.get());
ASSERT_TRUE(res->data.get());
EXPECT_EQ("Demo", *res->data);
});
// Make sure that we got a no errors
Log::removeObserver();
}
}
TEST_F(Storage, DatabaseLockedRefresh) {
using namespace mbgl;
// Create a locked file.
createDir("test/fixtures/database");
deleteFile("test/fixtures/database/locked.db");
SQLiteCache::Impl cache("test/fixtures/database/locked.db");
// Then, lock the file and try again.
FileLock guard("test/fixtures/database/locked.db");
{
// Adds a file.
Log::setObserver(std::make_unique<FixtureLogObserver>());
Response response;
response.data = std::make_shared<std::string>("Demo");
cache.put({ Resource::Unknown, "mapbox://test" }, response);
cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
EXPECT_EQ(nullptr, res.get());
});
auto observer = Log::removeObserver();
auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
EXPECT_EQ(8ul, flo->count({ EventSeverity::Error, Event::Database, 5, "database is locked" }));
}
{
// Then, try to refresh it.
Log::setObserver(std::make_unique<FixtureLogObserver>());
cache.refresh({ Resource::Unknown, "mapbox://test" }, {});
cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
EXPECT_EQ(nullptr, res.get());
});
// Make sure that we got the right errors.
auto observer = Log::removeObserver();
auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
EXPECT_EQ(8ul, flo->count({ EventSeverity::Error, Event::Database, 5, "database is locked" }));
}
}
TEST_F(Storage, DatabaseDeleted) {
using namespace mbgl;
// Create a locked file.
createDir("test/fixtures/database");
deleteFile("test/fixtures/database/locked.db");
SQLiteCache::Impl cache("test/fixtures/database/locked.db");
{
// Adds a file.
Log::setObserver(std::make_unique<FixtureLogObserver>());
Response response;
response.data = std::make_shared<std::string>("Demo");
cache.put({ Resource::Unknown, "mapbox://test" }, response);
cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
ASSERT_NE(nullptr, res.get());
ASSERT_TRUE(res->data.get());
EXPECT_EQ("Demo", *res->data);
});
Log::removeObserver();
}
deleteFile("test/fixtures/database/locked.db");
{
// Adds a file.
Log::setObserver(std::make_unique<FixtureLogObserver>());
Response response;
response.data = std::make_shared<std::string>("Demo");
cache.put({ Resource::Unknown, "mapbox://test" }, response);
cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
ASSERT_NE(nullptr, res.get());
ASSERT_TRUE(res->data.get());
EXPECT_EQ("Demo", *res->data);
});
auto observer = Log::removeObserver();
auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
EXPECT_EQ(1ul, flo->count({ EventSeverity::Error, Event::Database, 8, "attempt to write a readonly database" }));
}
}
TEST_F(Storage, DatabaseInvalid) {
using namespace mbgl;
// Create a locked file.
createDir("test/fixtures/database");
deleteFile("test/fixtures/database/invalid.db");
writeFile("test/fixtures/database/invalid.db", "this is an invalid file");
SQLiteCache::Impl cache("test/fixtures/database/invalid.db");
{
// Adds a file.
Log::setObserver(std::make_unique<FixtureLogObserver>());
Response response;
response.data = std::make_shared<std::string>("Demo");
cache.put({ Resource::Unknown, "mapbox://test" }, response);
cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
ASSERT_NE(nullptr, res.get());
ASSERT_TRUE(res->data.get());
EXPECT_EQ("Demo", *res->data);
});
auto observer = Log::removeObserver();
auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
EXPECT_EQ(1ul, flo->count({ EventSeverity::Warning, Event::Database, -1, "Trashing invalid database" }));
}
}

View File

@@ -1,10 +1,54 @@
#include "storage.hpp"
#include <mbgl/storage/default_file_source.hpp>
#include <mbgl/util/chrono.hpp>
#include <mbgl/util/run_loop.hpp>
TEST_F(Storage, CacheRevalidateSame) {
class DefaultFileSourceTest : public Storage {};
TEST_F(DefaultFileSourceTest, CacheResponse) {
SCOPED_TEST(CacheResponse);
using namespace mbgl;
util::RunLoop loop;
DefaultFileSource fs(":memory:", ".");
const Resource resource { Resource::Unknown, "http://127.0.0.1:3000/cache" };
Response response;
std::unique_ptr<FileRequest> req1;
std::unique_ptr<FileRequest> req2;
req1 = fs.request(resource, [&](Response res) {
req1.reset();
EXPECT_EQ(nullptr, res.error);
ASSERT_TRUE(res.data.get());
EXPECT_EQ("Response 1", *res.data);
EXPECT_TRUE(bool(res.expires));
EXPECT_FALSE(bool(res.modified));
EXPECT_FALSE(bool(res.etag));
response = res;
// Now test that we get the same values as in the previous request. If we'd go to the server
// again, we'd get different values.
req2 = fs.request(resource, [&](Response res2) {
req2.reset();
EXPECT_EQ(response.error, res2.error);
ASSERT_TRUE(res2.data.get());
EXPECT_EQ(*response.data, *res2.data);
EXPECT_EQ(response.expires, res2.expires);
EXPECT_EQ(response.modified, res2.modified);
EXPECT_EQ(response.etag, res2.etag);
loop.stop();
CacheResponse.finish();
});
});
loop.run();
}
TEST_F(DefaultFileSourceTest, CacheRevalidateSame) {
SCOPED_TEST(CacheRevalidateSame)
using namespace mbgl;
@@ -53,7 +97,7 @@ TEST_F(Storage, CacheRevalidateSame) {
loop.run();
}
TEST_F(Storage, CacheRevalidateModified) {
TEST_F(DefaultFileSourceTest, CacheRevalidateModified) {
SCOPED_TEST(CacheRevalidateModified)
using namespace mbgl;
@@ -102,7 +146,7 @@ TEST_F(Storage, CacheRevalidateModified) {
loop.run();
}
TEST_F(Storage, CacheRevalidateEtag) {
TEST_F(DefaultFileSourceTest, CacheRevalidateEtag) {
SCOPED_TEST(CacheRevalidateEtag)
using namespace mbgl;

View File

@@ -0,0 +1,374 @@
#include "../fixtures/fixture_log_observer.hpp"
#include <mbgl/storage/offline_database.hpp>
#include <mbgl/storage/resource.hpp>
#include <mbgl/storage/response.hpp>
#include <mbgl/util/io.hpp>
#include <gtest/gtest.h>
#include <sqlite3.h>
namespace {
void createDir(const char* name) {
const int ret = mkdir(name, 0755);
if (ret == -1) {
ASSERT_EQ(EEXIST, errno);
} else {
ASSERT_EQ(0, ret);
}
}
void deleteFile(const char* name) {
const int ret = unlink(name);
if (ret == -1) {
ASSERT_EQ(ENOENT, errno);
} else {
ASSERT_EQ(0, ret);
}
}
void writeFile(const char* name, const std::string& data) {
mbgl::util::write_file(name, data);
}
class FileLock {
public:
FileLock(const std::string& path) {
const int err = sqlite3_open_v2(path.c_str(), &db, SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE, nullptr);
if (err != SQLITE_OK) {
throw std::runtime_error("Could not open db");
}
lock();
}
void lock() {
assert(!locked);
const int err = sqlite3_exec(db, "begin exclusive transaction", nullptr, nullptr, nullptr);
if (err != SQLITE_OK) {
throw std::runtime_error("Could not lock db");
}
locked = true;
}
void unlock() {
assert(locked);
const int err = sqlite3_exec(db, "commit", nullptr, nullptr, nullptr);
if (err != SQLITE_OK) {
throw std::runtime_error("Could not unlock db");
}
locked = false;
}
~FileLock() {
if (locked) {
unlock();
}
}
private:
sqlite3* db;
bool locked = false;
};
}
//TEST(OfflineDatabase, NonexistentDirectory) {
// using namespace mbgl;
//
// Log::setObserver(std::make_unique<FixtureLogObserver>());
//
// OfflineDatabase db("test/fixtures/404/offline.db");
//
// db.get({ Resource::Unknown, "mapbox://test" }, [] (optional<Response> res) {
// EXPECT_FALSE(bool(res));
// });
//
// auto observer = Log::removeObserver();
// EXPECT_EQ(1ul, dynamic_cast<FixtureLogObserver*>(observer.get())->count({ EventSeverity::Error, Event::Database, 14, "unable to open database file" }));
//}
TEST(OfflineDatabase, Create) {
using namespace mbgl;
createDir("test/fixtures/database");
deleteFile("test/fixtures/database/offline.db");
Log::setObserver(std::make_unique<FixtureLogObserver>());
OfflineDatabase db("test/fixtures/database/offline.db");
EXPECT_FALSE(bool(db.get({ Resource::Unknown, "mapbox://test" })));
Log::removeObserver();
}
TEST(OfflineDatabase, SchemaVersion) {
using namespace mbgl;
createDir("test/fixtures/database");
deleteFile("test/fixtures/database/offline.db");
std::string path("test/fixtures/database/offline.db");
{
sqlite3* db;
sqlite3_open_v2(path.c_str(), &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, nullptr);
sqlite3_exec(db, "PRAGMA user_version = 1", nullptr, nullptr, nullptr);
sqlite3_close_v2(db);
}
Log::setObserver(std::make_unique<FixtureLogObserver>());
OfflineDatabase db(path);
auto observer = Log::removeObserver();
auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
EXPECT_EQ(1ul, flo->count({ EventSeverity::Warning, Event::Database, -1, "Removing existing incompatible offline database" }));
}
TEST(OfflineDatabase, Invalid) {
using namespace mbgl;
createDir("test/fixtures/database");
deleteFile("test/fixtures/database/invalid.db");
writeFile("test/fixtures/database/invalid.db", "this is an invalid file");
Log::setObserver(std::make_unique<FixtureLogObserver>());
OfflineDatabase db("test/fixtures/database/invalid.db");
auto observer = Log::removeObserver();
auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
EXPECT_EQ(1ul, flo->count({ EventSeverity::Warning, Event::Database, -1, "Removing existing incompatible offline database" }));
}
//TEST(OfflineDatabase, DatabaseLockedRead) {
// using namespace mbgl;
//
// // Create a locked file.
// createDir("test/fixtures/database");
// deleteFile("test/fixtures/database/locked.db");
// FileLock guard("test/fixtures/database/locked.db");
//
// OfflineDatabase db("test/fixtures/database/locked.db");
//
// {
// // First request should fail.
// Log::setObserver(std::make_unique<FixtureLogObserver>());
//
// db.get({ Resource::Unknown, "mapbox://test" }, [] (optional<Response> res) {
// EXPECT_FALSE(bool(res));
// });
//
// // Make sure that we got a few "database locked" errors
// auto observer = Log::removeObserver();
// auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
// EXPECT_EQ(4ul, flo->count({ EventSeverity::Error, Event::Database, 5, "database is locked" }));
// }
//
// // Then, unlock the file and try again.
// guard.unlock();
//
// {
// // First, try getting a file (the cache value should not exist).
// Log::setObserver(std::make_unique<FixtureLogObserver>());
//
// db.get({ Resource::Unknown, "mapbox://test" }, [] (optional<Response> res) {
// EXPECT_FALSE(bool(res));
// });
//
// // Make sure that we got a no errors
// Log::removeObserver();
// }
//}
//
//TEST(OfflineDatabase, DatabaseLockedWrite) {
// using namespace mbgl;
//
// // Create a locked file.
// createDir("test/fixtures/database");
// deleteFile("test/fixtures/database/locked.db");
// FileLock guard("test/fixtures/database/locked.db");
//
// OfflineDatabase db("test/fixtures/database/locked.db");
//
// {
// // Adds a file (which should fail).
// Log::setObserver(std::make_unique<FixtureLogObserver>());
//
// db.put({ Resource::Unknown, "mapbox://test" }, Response());
// db.get({ Resource::Unknown, "mapbox://test" }, [] (optional<Response> res) {
// EXPECT_FALSE(bool(res));
// });
//
// auto observer = Log::removeObserver();
// auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
// EXPECT_EQ(8ul, flo->count({ EventSeverity::Error, Event::Database, 5, "database is locked" }));
// }
//
// // Then, unlock the file and try again.
// guard.unlock();
//
// {
// // Then, set a file and obtain it again.
// Log::setObserver(std::make_unique<FixtureLogObserver>());
//
// Response response;
// response.data = std::make_shared<std::string>("Demo");
// db.put({ Resource::Unknown, "mapbox://test" }, response);
// db.get({ Resource::Unknown, "mapbox://test" }, [] (optional<Response> res) {
// ASSERT_TRUE(bool(res));
// ASSERT_TRUE(res->data.get());
// EXPECT_EQ("Demo", *res->data);
// });
//
// // Make sure that we got a no errors
// Log::removeObserver();
// }
//}
//
//TEST(OfflineDatabase, DatabaseDeleted) {
// using namespace mbgl;
//
// // Create a locked file.
// createDir("test/fixtures/database");
// deleteFile("test/fixtures/database/locked.db");
//
// OfflineDatabase db("test/fixtures/database/locked.db");
//
// {
// // Adds a file.
// Log::setObserver(std::make_unique<FixtureLogObserver>());
//
// Response response;
// response.data = std::make_shared<std::string>("Demo");
// db.put({ Resource::Unknown, "mapbox://test" }, response);
// db.get({ Resource::Unknown, "mapbox://test" }, [] (optional<Response> res) {
// ASSERT_TRUE(bool(res));
// ASSERT_TRUE(res->data.get());
// EXPECT_EQ("Demo", *res->data);
// });
//
// Log::removeObserver();
// }
//
// deleteFile("test/fixtures/database/locked.db");
//
// {
// // Adds a file.
// Log::setObserver(std::make_unique<FixtureLogObserver>());
//
// Response response;
// response.data = std::make_shared<std::string>("Demo");
// db.put({ Resource::Unknown, "mapbox://test" }, response);
// db.get({ Resource::Unknown, "mapbox://test" }, [] (optional<Response> res) {
// ASSERT_TRUE(bool(res));
// ASSERT_TRUE(res->data.get());
// EXPECT_EQ("Demo", *res->data);
// });
//
// auto observer = Log::removeObserver();
// auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
// EXPECT_EQ(1ul, flo->count({ EventSeverity::Error, Event::Database, 8, "attempt to write a readonly database" }));
// }
//}
TEST(OfflineDatabase, PutDoesNotStoreConnectionErrors) {
using namespace mbgl;
OfflineDatabase db(":memory:");
Resource resource { Resource::Unknown, "http://example.com/" };
Response response;
response.error = std::make_unique<Response::Error>(Response::Error::Reason::Connection);
db.put(resource, response);
EXPECT_FALSE(bool(db.get(resource)));
}
TEST(OfflineDatabase, PutDoesNotStoreServerErrors) {
using namespace mbgl;
OfflineDatabase db(":memory:");
Resource resource { Resource::Unknown, "http://example.com/" };
Response response;
response.error = std::make_unique<Response::Error>(Response::Error::Reason::Server);
db.put(resource, response);
EXPECT_FALSE(bool(db.get(resource)));
}
TEST(OfflineDatabase, PutResource) {
using namespace mbgl;
OfflineDatabase db(":memory:");
Resource resource { Resource::Style, "http://example.com/" };
Response response;
response.data = std::make_shared<std::string>("data");
db.put(resource, response);
auto res = db.get(resource);
EXPECT_EQ(nullptr, res->error.get());
EXPECT_EQ("data", *res->data);
}
TEST(OfflineDatabase, PutTile) {
using namespace mbgl;
OfflineDatabase db(":memory:");
Resource resource { Resource::Tile, "http://example.com/" };
resource.tileData = Resource::TileData {
"http://example.com/",
1,
0,
0,
0
};
Response response;
response.data = std::make_shared<std::string>("data");
db.put(resource, response);
auto res = db.get(resource);
EXPECT_EQ(nullptr, res->error.get());
EXPECT_EQ("data", *res->data);
}
TEST(OfflineDatabase, PutResourceNotFound) {
using namespace mbgl;
OfflineDatabase db(":memory:");
Resource resource { Resource::Style, "http://example.com/" };
Response response;
response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound);
db.put(resource, response);
auto res = db.get(resource);
EXPECT_NE(nullptr, res->error);
EXPECT_EQ(Response::Error::Reason::NotFound, res->error->reason);
EXPECT_FALSE(res->data.get());
}
TEST(OfflineDatabase, PutTileNotFound) {
using namespace mbgl;
OfflineDatabase db(":memory:");
Resource resource { Resource::Tile, "http://example.com/" };
resource.tileData = Resource::TileData {
"http://example.com/",
1,
0,
0,
0
};
Response response;
response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound);
db.put(resource, response);
auto res = db.get(resource);
EXPECT_NE(nullptr, res->error);
EXPECT_EQ(Response::Error::Reason::NotFound, res->error->reason);
EXPECT_FALSE(res->data.get());
}

View File

@@ -58,6 +58,7 @@
'api/render_missing.cpp',
'api/set_style.cpp',
'api/custom_layer.cpp',
'api/offline.cpp',
'geometry/binpack.cpp',
@@ -68,12 +69,8 @@
'storage/storage.hpp',
'storage/storage.cpp',
'storage/cache_response.cpp',
'storage/cache_revalidate.cpp',
'storage/cache_shared.cpp',
'storage/cache_size.cpp',
'storage/cache_stale.cpp',
'storage/database.cpp',
'storage/default_file_source.cpp',
'storage/offline_database.cpp',
'storage/asset_file_source.cpp',
'storage/headers.cpp',
'storage/http_cancel.cpp',