[core] Reset SQLite statements after use in order to release locks

This commit is contained in:
John Firebaugh
2016-02-09 11:16:55 -08:00
parent ff15bd51b9
commit 5910124010
5 changed files with 168 additions and 112 deletions

View File

@@ -17,6 +17,11 @@ using namespace mapbox::sqlite;
// If you change the schema you must write a migration from the previous version.
static const uint32_t schemaVersion = 2;
OfflineDatabase::Statement::~Statement() {
stmt.reset();
stmt.clearBindings();
}
OfflineDatabase::OfflineDatabase(const std::string& path_)
: path(path_) {
ensureSchema();
@@ -40,7 +45,7 @@ void OfflineDatabase::ensureSchema() {
db->setBusyTimeout(Milliseconds::max());
{
Statement userVersionStmt(db->prepare("PRAGMA user_version"));
auto userVersionStmt = db->prepare("PRAGMA user_version");
userVersionStmt.run();
switch (userVersionStmt.get<int>(0)) {
case 0: break; // cache-only database; ok to delete
@@ -85,15 +90,14 @@ void OfflineDatabase::removeExisting() {
}
}
mapbox::sqlite::Statement& OfflineDatabase::getStatement(const char * sql) {
OfflineDatabase::Statement OfflineDatabase::getStatement(const char * sql) {
auto it = statements.find(sql);
if (it != statements.end()) {
it->second->reset();
return *it->second;
return Statement(*it->second);
}
return *(statements[sql] = std::make_unique<Statement>(db->prepare(sql)));
return Statement(*statements.emplace(sql, std::make_unique<mapbox::sqlite::Statement>(db->prepare(sql))).first->second);
}
optional<Response> OfflineDatabase::get(const Resource& resource) {
@@ -120,28 +124,28 @@ uint64_t OfflineDatabase::put(const Resource& resource, const Response& response
}
optional<Response> OfflineDatabase::getResource(const Resource& resource) {
mapbox::sqlite::Statement& stmt = getStatement(
Statement stmt = getStatement(
// 0 1 2 3 4
"SELECT etag, expires, modified, data, compressed "
"FROM resources "
"WHERE url = ?");
stmt.bind(1, resource.url);
stmt->bind(1, resource.url);
if (!stmt.run()) {
if (!stmt->run()) {
return {};
}
Response response;
response.etag = stmt.get<optional<std::string>>(0);
response.expires = stmt.get<optional<SystemTimePoint>>(1);
response.modified = stmt.get<optional<SystemTimePoint>>(2);
response.etag = stmt->get<optional<std::string>>(0);
response.expires = stmt->get<optional<SystemTimePoint>>(1);
response.modified = stmt->get<optional<SystemTimePoint>>(2);
optional<std::string> data = stmt.get<optional<std::string>>(3);
optional<std::string> data = stmt->get<optional<std::string>>(3);
if (!data) {
response.noContent = true;
} else if (stmt.get<int>(4)) {
} else if (stmt->get<int>(4)) {
response.data = std::make_shared<std::string>(util::decompress(*data));
} else {
response.data = std::make_shared<std::string>(*data);
@@ -152,54 +156,54 @@ optional<Response> OfflineDatabase::getResource(const Resource& resource) {
uint64_t OfflineDatabase::putResource(const Resource& resource, const Response& response) {
if (response.notModified) {
mapbox::sqlite::Statement& stmt = getStatement(
Statement stmt = getStatement(
// 1 2 3
"UPDATE resources SET accessed = ?, expires = ? WHERE url = ?");
stmt.bind(1, SystemClock::now());
stmt.bind(2, response.expires);
stmt.bind(3, resource.url);
stmt.run();
stmt->bind(1, SystemClock::now());
stmt->bind(2, response.expires);
stmt->bind(3, resource.url);
stmt->run();
return 0;
} else {
mapbox::sqlite::Statement& stmt = getStatement(
Statement stmt = getStatement(
// 1 2 3 4 5 6 7 8
"REPLACE INTO resources (url, kind, etag, expires, modified, accessed, data, compressed) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?)");
stmt.bind(1 /* url */, resource.url);
stmt.bind(2 /* kind */, int(resource.kind));
stmt.bind(3 /* etag */, response.etag);
stmt.bind(4 /* expires */, response.expires);
stmt.bind(5 /* modified */, response.modified);
stmt.bind(6 /* accessed */, SystemClock::now());
stmt->bind(1 /* url */, resource.url);
stmt->bind(2 /* kind */, int(resource.kind));
stmt->bind(3 /* etag */, response.etag);
stmt->bind(4 /* expires */, response.expires);
stmt->bind(5 /* modified */, response.modified);
stmt->bind(6 /* accessed */, SystemClock::now());
std::string data;
uint64_t size = 0;
if (response.noContent) {
stmt.bind(7 /* data */, nullptr);
stmt.bind(8 /* compressed */, false);
stmt->bind(7 /* data */, nullptr);
stmt->bind(8 /* compressed */, false);
} else {
data = util::compress(*response.data);
if (data.size() < response.data->size()) {
size = data.size();
stmt.bindBlob(7 /* data */, data.data(), size, false);
stmt.bind(8 /* compressed */, true);
stmt->bindBlob(7 /* data */, data.data(), size, false);
stmt->bind(8 /* compressed */, true);
} else {
size = response.data->size();
stmt.bindBlob(7 /* data */, response.data->data(), size, false);
stmt.bind(8 /* compressed */, false);
stmt->bindBlob(7 /* data */, response.data->data(), size, false);
stmt->bind(8 /* compressed */, false);
}
}
stmt.run();
stmt->run();
return size;
}
}
optional<Response> OfflineDatabase::getTile(const Resource::TileData& tile) {
mapbox::sqlite::Statement& stmt = getStatement(
Statement stmt = getStatement(
// 0 1 2 3 4
"SELECT etag, expires, modified, data, compressed "
"FROM tilesets, tiles "
@@ -210,26 +214,26 @@ optional<Response> OfflineDatabase::getTile(const Resource::TileData& tile) {
"AND tiles.z = ? " // 5
"AND tilesets.id = tiles.tileset_id ");
stmt.bind(1, tile.urlTemplate);
stmt.bind(2, tile.pixelRatio);
stmt.bind(3, tile.x);
stmt.bind(4, tile.y);
stmt.bind(5, tile.z);
stmt->bind(1, tile.urlTemplate);
stmt->bind(2, tile.pixelRatio);
stmt->bind(3, tile.x);
stmt->bind(4, tile.y);
stmt->bind(5, tile.z);
if (!stmt.run()) {
if (!stmt->run()) {
return {};
}
Response response;
response.etag = stmt.get<optional<std::string>>(0);
response.expires = stmt.get<optional<SystemTimePoint>>(1);
response.modified = stmt.get<optional<SystemTimePoint>>(2);
response.etag = stmt->get<optional<std::string>>(0);
response.expires = stmt->get<optional<SystemTimePoint>>(1);
response.modified = stmt->get<optional<SystemTimePoint>>(2);
optional<std::string> data = stmt.get<optional<std::string>>(3);
optional<std::string> data = stmt->get<optional<std::string>>(3);
if (!data) {
response.noContent = true;
} else if (stmt.get<int>(4)) {
} else if (stmt->get<int>(4)) {
response.data = std::make_shared<std::string>(util::decompress(*data));
} else {
response.data = std::make_shared<std::string>(*data);
@@ -240,7 +244,7 @@ optional<Response> OfflineDatabase::getTile(const Resource::TileData& tile) {
uint64_t OfflineDatabase::putTile(const Resource::TileData& tile, const Response& response) {
if (response.notModified) {
mapbox::sqlite::Statement& stmt = getStatement(
Statement stmt = getStatement(
"UPDATE tiles SET accessed = ?1, expires = ?2 "
"WHERE tileset_id = ( "
" SELECT id FROM tilesets "
@@ -250,76 +254,76 @@ uint64_t OfflineDatabase::putTile(const Resource::TileData& tile, const Response
"AND tiles.y = ?6 "
"AND tiles.z = ?7 ");
stmt.bind(1, SystemClock::now());
stmt.bind(2, response.expires);
stmt.bind(3, tile.urlTemplate);
stmt.bind(4, tile.pixelRatio);
stmt.bind(5, tile.x);
stmt.bind(6, tile.y);
stmt.bind(7, tile.z);
stmt.run();
stmt->bind(1, SystemClock::now());
stmt->bind(2, response.expires);
stmt->bind(3, tile.urlTemplate);
stmt->bind(4, tile.pixelRatio);
stmt->bind(5, tile.x);
stmt->bind(6, tile.y);
stmt->bind(7, tile.z);
stmt->run();
return 0;
} else {
mapbox::sqlite::Statement& stmt1 = getStatement(
Statement stmt1 = getStatement(
"REPLACE INTO tilesets (url_template, pixel_ratio) "
"VALUES (?1, ?2) ");
stmt1.bind(1 /* url_template */, tile.urlTemplate);
stmt1.bind(2 /* pixel_ratio */, tile.pixelRatio);
stmt1.run();
stmt1->bind(1 /* url_template */, tile.urlTemplate);
stmt1->bind(2 /* pixel_ratio */, tile.pixelRatio);
stmt1->run();
mapbox::sqlite::Statement& stmt2 = getStatement(
Statement stmt2 = getStatement(
"REPLACE INTO tiles (tileset_id, x, y, z, modified, etag, expires, accessed, data, compressed) "
"SELECT tilesets.id, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11 "
"FROM tilesets "
"WHERE url_template = ?1 "
"AND pixel_ratio = ?2 ");
stmt2.bind(1 /* url_template */, tile.urlTemplate);
stmt2.bind(2 /* pixel_ratio */, tile.pixelRatio);
stmt2.bind(3 /* x */, tile.x);
stmt2.bind(4 /* y */, tile.y);
stmt2.bind(5 /* z */, tile.z);
stmt2.bind(6 /* modified */, response.modified);
stmt2.bind(7 /* etag */, response.etag);
stmt2.bind(8 /* expires */, response.expires);
stmt2.bind(9 /* accessed */, SystemClock::now());
stmt2->bind(1 /* url_template */, tile.urlTemplate);
stmt2->bind(2 /* pixel_ratio */, tile.pixelRatio);
stmt2->bind(3 /* x */, tile.x);
stmt2->bind(4 /* y */, tile.y);
stmt2->bind(5 /* z */, tile.z);
stmt2->bind(6 /* modified */, response.modified);
stmt2->bind(7 /* etag */, response.etag);
stmt2->bind(8 /* expires */, response.expires);
stmt2->bind(9 /* accessed */, SystemClock::now());
std::string data;
uint64_t size = 0;
if (response.noContent) {
stmt2.bind(10 /* data */, nullptr);
stmt2.bind(11 /* compressed */, false);
stmt2->bind(10 /* data */, nullptr);
stmt2->bind(11 /* compressed */, false);
} else {
data = util::compress(*response.data);
if (data.size() < response.data->size()) {
size = data.size();
stmt2.bindBlob(10 /* data */, data.data(), size, false);
stmt2.bind(11 /* compressed */, true);
stmt2->bindBlob(10 /* data */, data.data(), size, false);
stmt2->bind(11 /* compressed */, true);
} else {
size = response.data->size();
stmt2.bindBlob(10 /* data */, response.data->data(), size, false);
stmt2.bind(11 /* compressed */, false);
stmt2->bindBlob(10 /* data */, response.data->data(), size, false);
stmt2->bind(11 /* compressed */, false);
}
}
stmt2.run();
stmt2->run();
return size;
}
}
std::vector<OfflineRegion> OfflineDatabase::listRegions() {
mapbox::sqlite::Statement& stmt = getStatement(
Statement stmt = getStatement(
"SELECT id, definition, description FROM regions");
std::vector<OfflineRegion> result;
while (stmt.run()) {
while (stmt->run()) {
result.push_back(OfflineRegion(
stmt.get<int64_t>(0),
decodeOfflineRegionDefinition(stmt.get<std::string>(1)),
stmt.get<std::vector<uint8_t>>(2)));
stmt->get<int64_t>(0),
decodeOfflineRegionDefinition(stmt->get<std::string>(1)),
stmt->get<std::vector<uint8_t>>(2)));
}
return std::move(result);
@@ -327,23 +331,23 @@ std::vector<OfflineRegion> OfflineDatabase::listRegions() {
OfflineRegion OfflineDatabase::createRegion(const OfflineRegionDefinition& definition,
const OfflineRegionMetadata& metadata) {
mapbox::sqlite::Statement& stmt = getStatement(
Statement stmt = getStatement(
"INSERT INTO regions (definition, description) "
"VALUES (?1, ?2) ");
stmt.bind(1, encodeOfflineRegionDefinition(definition));
stmt.bindBlob(2, metadata);
stmt.run();
stmt->bind(1, encodeOfflineRegionDefinition(definition));
stmt->bindBlob(2, metadata);
stmt->run();
return OfflineRegion(db->lastInsertRowid(), definition, metadata);
}
void OfflineDatabase::deleteRegion(OfflineRegion&& region) {
mapbox::sqlite::Statement& stmt = getStatement(
Statement stmt = getStatement(
"DELETE FROM regions WHERE id = ?");
stmt.bind(1, region.getID());
stmt.run();
stmt->bind(1, region.getID());
stmt->run();
}
optional<Response> OfflineDatabase::getRegionResource(int64_t regionID, const Resource& resource) {
@@ -364,45 +368,45 @@ uint64_t OfflineDatabase::putRegionResource(int64_t regionID, const Resource& re
void OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) {
if (resource.kind == Resource::Kind::Tile) {
mapbox::sqlite::Statement& stmt1 = getStatement(
Statement stmt1 = getStatement(
"REPLACE INTO region_tiles (region_id, tileset_id, x, y, z) "
"SELECT ?1, tilesets.id, ?4, ?5, ?6 "
"FROM tilesets "
"WHERE url_template = ?2 "
"AND pixel_ratio = ?3 ");
stmt1.bind(1, regionID);
stmt1.bind(2, (*resource.tileData).urlTemplate);
stmt1.bind(3, (*resource.tileData).pixelRatio);
stmt1.bind(4, (*resource.tileData).x);
stmt1.bind(5, (*resource.tileData).y);
stmt1.bind(6, (*resource.tileData).z);
stmt1.run();
stmt1->bind(1, regionID);
stmt1->bind(2, (*resource.tileData).urlTemplate);
stmt1->bind(3, (*resource.tileData).pixelRatio);
stmt1->bind(4, (*resource.tileData).x);
stmt1->bind(5, (*resource.tileData).y);
stmt1->bind(6, (*resource.tileData).z);
stmt1->run();
} else {
mapbox::sqlite::Statement& stmt1 = getStatement(
Statement stmt1 = getStatement(
"REPLACE INTO region_resources (region_id, resource_url) "
"VALUES (?1, ?2) ");
stmt1.bind(1, regionID);
stmt1.bind(2, resource.url);
stmt1.run();
stmt1->bind(1, regionID);
stmt1->bind(2, resource.url);
stmt1->run();
}
}
OfflineRegionDefinition OfflineDatabase::getRegionDefinition(int64_t regionID) {
mapbox::sqlite::Statement& stmt = getStatement(
Statement stmt = getStatement(
"SELECT definition FROM regions WHERE id = ?1");
stmt.bind(1, regionID);
stmt.run();
stmt->bind(1, regionID);
stmt->run();
return decodeOfflineRegionDefinition(stmt.get<std::string>(0));
return decodeOfflineRegionDefinition(stmt->get<std::string>(0));
}
OfflineRegionStatus OfflineDatabase::getRegionCompletedStatus(int64_t regionID) {
OfflineRegionStatus result;
mapbox::sqlite::Statement& stmt = getStatement(
Statement stmt = getStatement(
"SELECT COUNT(*), SUM(size) FROM ( "
" SELECT LENGTH(data) as size "
" FROM region_resources, resources "
@@ -418,26 +422,26 @@ OfflineRegionStatus OfflineDatabase::getRegionCompletedStatus(int64_t regionID)
" AND tiles.y = region_tiles.y "
") ");
stmt.bind(1, regionID);
stmt.run();
stmt->bind(1, regionID);
stmt->run();
result.completedResourceCount = stmt.get<int64_t>(0);
result.completedResourceSize = stmt.get<int64_t>(1);
result.completedResourceCount = stmt->get<int64_t>(0);
result.completedResourceSize = stmt->get<int64_t>(1);
return result;
}
void OfflineDatabase::removeUnusedResources() {
mapbox::sqlite::Statement& stmt1 = getStatement(
Statement stmt1 = getStatement(
"DELETE FROM resources "
"WHERE ROWID NOT IN ( "
" SELECT resources.ROWID "
" FROM resources, region_resources "
" WHERE resources.url = region_resources.resource_url "
") ");
stmt1.run();
stmt1->run();
mapbox::sqlite::Statement& stmt2 = getStatement(
Statement stmt2 = getStatement(
"DELETE FROM tiles "
"WHERE ROWID NOT IN ( "
" SELECT tiles.ROWID "
@@ -447,7 +451,7 @@ void OfflineDatabase::removeUnusedResources() {
" AND tiles.x = region_tiles.x "
" AND tiles.y = region_tiles.y "
") ");
stmt2.run();
stmt2->run();
}
} // namespace mbgl

View File

@@ -47,7 +47,21 @@ public:
private:
void ensureSchema();
void removeExisting();
mapbox::sqlite::Statement& getStatement(const char *);
class Statement {
public:
explicit Statement(mapbox::sqlite::Statement& stmt_) : stmt(stmt_) {}
Statement(Statement&&) = default;
Statement(const Statement&) = delete;
~Statement();
mapbox::sqlite::Statement* operator->() { return &stmt; };
private:
mapbox::sqlite::Statement& stmt;
};
Statement getStatement(const char *);
optional<Response> getTile(const Resource::TileData&);
uint64_t putTile(const Resource::TileData&, const Response&);

View File

@@ -310,5 +310,10 @@ void Statement::reset() {
sqlite3_reset(stmt);
}
void Statement::clearBindings() {
assert(stmt);
sqlite3_clear_bindings(stmt);
}
} // namespace sqlite
} // namespace mapbox

View File

@@ -81,6 +81,7 @@ public:
bool run();
void reset();
void clearBindings();
private:
sqlite3_stmt *stmt = nullptr;

View File

@@ -7,6 +7,7 @@
#include <gtest/gtest.h>
#include <sqlite3.h>
#include <thread>
namespace {
@@ -448,3 +449,34 @@ TEST(OfflineDatabase, CreateRegionInfiniteMaxZoom) {
EXPECT_EQ(0, region.getDefinition().minZoom);
EXPECT_EQ(INFINITY, region.getDefinition().maxZoom);
}
TEST(OfflineDatabase, ConcurrentUse) {
using namespace mbgl;
createDir("test/fixtures/database");
deleteFile("test/fixtures/database/offline.db");
OfflineDatabase db1("test/fixtures/database/offline.db");
OfflineDatabase db2("test/fixtures/database/offline.db");
Resource resource { Resource::Style, "http://example.com/" };
Response response;
response.noContent = true;
std::thread thread1([&] {
for (auto i = 0; i < 100; i++) {
db1.put(resource, response);
EXPECT_TRUE(bool(db1.get(resource)));
}
});
std::thread thread2([&] {
for (auto i = 0; i < 100; i++) {
db2.put(resource, response);
EXPECT_TRUE(bool(db2.get(resource)));
}
});
thread1.join();
thread2.join();
}