tei/MapLibreParticleLab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add basic request caching with sqlite

Browse Source
This commit is contained in:
Konstantin Käfer
2014-08-22 18:50:09 +02:00
parent ee3e0827a2
commit 34fe7dc627
6 changed files with 210 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
configure vendored
View File

@@ -107,6 +107,14 @@ def configure_curl(o):
o['variables']['curl_libraries'] = ret[0].split()
o['variables']['curl_cflags'] = ret[1].split()
def configure_sqlite3(o):
ret = pkg_config('sqlite3', options.pkgconfig_root)
if not ret:
sys.stderr.write('could not find sqlite3 with pkg-config')
sys.exit(-1)
o['variables']['sqlite3_libraries'] = ret[0].split()
o['variables']['sqlite3_cflags'] = ret[1].split()
def write(filename, data):
filename = os.path.join(root_dir, filename)
print "creating ", filename
@@ -129,6 +137,7 @@ if __name__ == '__main__':
configure_uv(output)
configure_png(output)
configure_curl(output)
configure_sqlite3(output)
pprint.pprint(output, indent=2)
write('config.gypi', "# Do not edit. Generated by the configure script.\n" +

2
include/mbgl/platform/event.hpp
View File

@@ -29,6 +29,7 @@ enum class Event : uint8_t {
ParseStyle,
ParseTile,
Render,
Database,
HttpRequest,
Sprite,
};
@@ -40,6 +41,7 @@ MBGL_DEFINE_ENUM_CLASS(EventClass, Event, {
{ Event::ParseStyle, "ParseStyle" },
{ Event::ParseTile, "ParseTile" },
{ Event::Render, "Render" },
{ Event::Database, "Database" },
{ Event::HttpRequest, "HttpRequest" },
{ Event::Sprite, "Sprite" },
{ Event(-1), "Unknown" },

15
include/mbgl/util/filesource.hpp
View File

@@ -2,11 +2,15 @@
#define MBGL_UTIL_FILESOURCE
#include <mbgl/util/uv.hpp>
#include <mbgl/util/noncopyable.hpp>
#include <string>
#include <memory>
#include <functional>
typedef struct sqlite3 sqlite3;
typedef struct sqlite3_stmt sqlite3_stmt;
namespace mbgl {
namespace platform {
@@ -21,21 +25,30 @@ enum class ResourceType : uint8_t {
JSON
};
class FileSource {
class FileSource : private util::noncopyable, public std::enable_shared_from_this<FileSource> {
public:
FileSource();
~FileSource();
void setBase(const std::string &value);
const std::string &getBase() const;
void load(ResourceType type, const std::string &url, std::function<void(platform::Response *)> callback, const std::shared_ptr<uv::loop> loop = nullptr);
private:
void closeDatabase();
void createSchema();
bool loadFile(const std::string &url, std::function<void(platform::Response *)> callback);
void saveFile(const std::string &url, platform::Response *res);
private:
// Stores a URL that is used as a base for loading resources with relative path.
std::string base;
// Stores the absolute path to the cache directory.
const std::string cache;
sqlite3 *db = nullptr;
};
}

53
mapboxgl.gyp
View File

@@ -126,23 +126,36 @@
'<!@(find src -name "*.glsl")',
'bin/style.json'
],
'xcode_settings': {
'SDKROOT': 'macosx',
'SUPPORTED_PLATFORMS':['macosx'],
'MACOSX_DEPLOYMENT_TARGET':'10.9',
'PUBLIC_HEADERS_FOLDER_PATH': 'include',
'OTHER_CPLUSPLUSFLAGS':[
'<@(png_cflags)',
'<@(uv_cflags)',
'-I<(boost_root)/include',
]
},
'include_dirs':[
'./include'
'./include'
],
'cflags': [
'<@(png_cflags)',
'-I<(boost_root)/include',
'conditions': [
['OS == "mac"', {
'xcode_settings': {
'SDKROOT': 'macosx',
'SUPPORTED_PLATFORMS':['macosx'],
'MACOSX_DEPLOYMENT_TARGET':'10.9',
'PUBLIC_HEADERS_FOLDER_PATH': 'include',
'OTHER_CPLUSPLUSFLAGS':[
'<@(png_cflags)',
'<@(uv_cflags)',
'<@(sqlite3_cflags)',
'-I<(boost_root)/include',
],
'OTHER_LDFLAGS': [
'<(sqlite3_libraries)',
],
},
}, {
'cflags': [
'<@(png_cflags)',
'<@(sqlite3_cflags)',
'-I<(boost_root)/include',
],
'libraries': [
'<@(sqlite3_libraries)',
],
}]
],
'direct_dependent_settings': {
'include_dirs':[
@@ -205,8 +218,12 @@
'OTHER_CPLUSPLUSFLAGS':[
'<@(png_cflags)',
'<@(uv_cflags)',
'<@(sqlite3_cflags)',
'-I<(boost_root)/include',
]
],
'OTHER_LDFLAGS': [
'<@(sqlite3_libraries)',
],
},
'include_dirs':[
'./include'
@@ -214,8 +231,12 @@
'cflags': [
'<@(png_cflags)',
'<@(uv_cflags)',
'<@(sqlite3_cflags)',
'-I<(boost_root)/include',
],
'libraries': [
'<@(sqlite3_libraries)',
],
'direct_dependent_settings': {
'include_dirs':[
'./include'

7
setup-libraries.sh
View File

@@ -82,32 +82,38 @@ if test -z "${TRAVIS:-}" || ! test -d out/build-cpp11-libcpp-universal; then
source iPhoneOS.sh
if [ ! -f out/build-cpp11-libcpp-armv7-iphoneos/lib/libpng.a ] ; then ./scripts/build_png.sh ; fi
if [ ! -f out/build-cpp11-libcpp-armv7-iphoneos/lib/libuv.a ] ; then ./scripts/build_libuv.sh ; fi
if [ ! -f out/build-cpp11-libcpp-armv7-iphoneos/lib/libsqlite3.a ] ; then ./scripts/build_sqlite.sh ; fi
echo ' ...done'
source iPhoneOSs.sh
if [ ! -f out/build-cpp11-libcpp-armv7s-iphoneoss/lib/libpng.a ] ; then ./scripts/build_png.sh ; fi
if [ ! -f out/build-cpp11-libcpp-armv7s-iphoneoss/lib/libuv.a ] ; then ./scripts/build_libuv.sh ; fi
if [ ! -f out/build-cpp11-libcpp-armv7s-iphoneoss/lib/libsqlite3.a ] ; then ./scripts/build_libsqlite.sh ; fi
echo ' ...done'
source iPhoneOS64.sh
if [ ! -f out/build-cpp11-libcpp-arm64-iphoneos64/lib/libpng.a ] ; then ./scripts/build_png.sh ; fi
if [ ! -f out/build-cpp11-libcpp-arm64-iphoneos64/lib/libuv.a ] ; then ./scripts/build_libuv.sh ; fi
if [ ! -f out/build-cpp11-libcpp-arm64-iphoneos64/lib/libsqlite3.a ] ; then ./scripts/build_libsqlite.sh ; fi
echo ' ...done'
source iPhoneSimulator.sh
if [ ! -f out/build-cpp11-libcpp-i386-iphonesimulator/lib/libpng.a ] ; then ./scripts/build_png.sh ; fi
if [ ! -f out/build-cpp11-libcpp-i386-iphonesimulator/lib/libuv.a ] ; then ./scripts/build_libuv.sh ; fi
if [ ! -f out/build-cpp11-libcpp-i386-iphonesimulator/lib/libsqlite3.a ] ; then ./scripts/build_sqlite.sh ; fi
echo ' ...done'
source iPhoneSimulator64.sh
if [ ! -f out/build-cpp11-libcpp-x86_64-iphonesimulator/lib/libpng.a ] ; then ./scripts/build_png.sh ; fi
if [ ! -f out/build-cpp11-libcpp-x86_64-iphonesimulator/lib/libuv.a ] ; then ./scripts/build_libuv.sh ; fi
if [ ! -f out/build-cpp11-libcpp-x86_64-iphonesimulator/lib/libsqlite3.a ] ; then ./scripts/build_sqlite.sh ; fi
echo ' ...done'
source MacOSX.sh
if [ ! -f out/build-cpp11-libcpp-x86_64-macosx/lib/libpng.a ] ; then ./scripts/build_png.sh ; fi
if [ ! -f out/build-cpp11-libcpp-x86_64-macosx/lib/libglfw3.a ] ; then ./scripts/build_glfw.sh ; fi
if [ ! -f out/build-cpp11-libcpp-x86_64-macosx/lib/libuv.a ] ; then ./scripts/build_libuv.sh ; fi
if [ ! -f out/build-cpp11-libcpp-x86_64-macosx/lib/libsqlite3.a ] ; then ./scripts/build_sqlite.sh ; fi
if [ ! -f out/build-cpp11-libcpp-x86_64-macosx/lib/libssl.a ] ; then ./scripts/build_openssl.sh ; fi
if [ ! -f out/build-cpp11-libcpp-x86_64-macosx/lib/libcurl.a ] ; then ./scripts/build_curl.sh ; fi
if [ ! -d out/build-cpp11-libcpp-x86_64-macosx/include/boost ] ; then ./scripts/build_boost.sh `pwd`/../../src/ `pwd`/../../include/ `pwd`/../../linux/ `pwd`/../../common/ ; fi
@@ -143,6 +149,7 @@ source Linux.sh
if [ ! -f out/build-cpp11-libstdcpp-gcc-x86_64-linux/lib/libglfw3.a ] ; then ./scripts/build_glfw.sh ; fi
if [ ! -f out/build-cpp11-libstdcpp-gcc-x86_64-linux/lib/libpng.a ] ; then ./scripts/build_png.sh ; fi
if [ ! -f out/build-cpp11-libstdcpp-gcc-x86_64-linux/lib/libuv.a ] ; then ./scripts/build_libuv.sh ; fi
if [ ! -f out/build-cpp11-libstdcpp-gcc-x86_64-linux/lib/libsqlite3.a ] ; then ./scripts/build_sqlite.sh ; fi
if [ ! -f out/build-cpp11-libstdcpp-gcc-x86_64-linux/lib/libssl.a ] ; then ./scripts/build_openssl.sh ; fi
if [ ! -f out/build-cpp11-libstdcpp-gcc-x86_64-linux/lib/libcurl.a ] ; then ./scripts/build_curl.sh ; fi
if [ ! -f out/build-cpp11-libstdcpp-gcc-x86_64-linux/lib/libboost_regex.a ] ; then ./scripts/build_boost.sh --with-regex ; fi

143
src/util/filesource.cpp
View File

@@ -1,13 +1,46 @@
#include <mbgl/util/filesource.hpp>
#include <mbgl/platform/platform.hpp>
#include <mbgl/platform/log.hpp>
#include <sqlite3.h>
#include <fstream>
#include <sstream>
namespace mbgl {
FileSource::FileSource() {}
FileSource::FileSource() {
const int err = sqlite3_open_v2("cache.db", &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, nullptr);
if (err != SQLITE_OK) {
Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db));
db = nullptr;
}
createSchema();
}
FileSource::~FileSource() {
closeDatabase();
}
void FileSource::createSchema() {
if (db) {
// Create schema
const int err = sqlite3_exec(db, "CREATE TABLE IF NOT EXISTS `http_cache` (`url` TEXT PRIMARY KEY, `expires` INTEGER, `data` BLOB)", nullptr, nullptr, nullptr);
if (err != SQLITE_OK) {
Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db));
closeDatabase();
}
}
}
void FileSource::closeDatabase() {
if (db) {
const int err = sqlite3_close_v2(db);
Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db));
db = nullptr;
}
}
void FileSource::setBase(const std::string &value) {
base = value;
@@ -17,6 +50,104 @@ const std::string &FileSource::getBase() const {
return base;
}
bool FileSource::loadFile(const std::string &url, std::function<void(platform::Response *)> callback) {
if (!db) {
return false;
}
sqlite3_stmt *stmt = nullptr;
int err;
err = sqlite3_prepare_v2(db, "SELECT `data` FROM `http_cache` WHERE `url` = ?", -1, &stmt, nullptr);
if (err != SQLITE_OK) {
Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db));
return false;
}
err = sqlite3_bind_text(stmt, 1, url.data(), url.size(), nullptr);
if (err != SQLITE_OK) {
Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db));
err = sqlite3_finalize(stmt);
if (err != SQLITE_OK) {
Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db));
}
return false;
}
bool status = false;
err = sqlite3_step(stmt);
if (err == SQLITE_ROW) {
// There is data.
platform::Response res(callback);
res.code = 200;
const char *data = reinterpret_cast<const char *>(sqlite3_column_blob(stmt, 0));
const size_t size = sqlite3_column_bytes(stmt, 0);
res.body = { data, size };
callback(&res);
status = true;
} else if (err == SQLITE_DONE) {
// There is no data.
// This is a noop.
} else {
// Another error occured.
Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db));
}
err = sqlite3_finalize(stmt);
if (err != SQLITE_OK) {
Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db));
}
return status;
}
void FileSource::saveFile(const std::string &url, platform::Response *const res) {
if (!db) {
return;
}
sqlite3_stmt *stmt = nullptr;
int err;
err = sqlite3_prepare_v2(db, "REPLACE INTO `http_cache` (`url`, `data`) VALUES(?, ?)", -1, &stmt, nullptr);
if (err != SQLITE_OK) {
Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db));
return;
}
err = sqlite3_bind_text(stmt, 1, url.data(), url.size(), nullptr);
if (err != SQLITE_OK) {
Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db));
err = sqlite3_finalize(stmt);
if (err != SQLITE_OK) {
Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db));
}
return;
}
err = sqlite3_bind_blob(stmt, 2, res->body.data(), res->body.size(), nullptr);
if (err != SQLITE_OK) {
Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db));
err = sqlite3_finalize(stmt);
if (err != SQLITE_OK) {
Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db));
}
return;
}
err = sqlite3_step(stmt);
if (err != SQLITE_DONE) {
// Another error occured.
Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db));
}
err = sqlite3_finalize(stmt);
if (err != SQLITE_OK) {
Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db));
}
}
void FileSource::load(ResourceType /*type*/, const std::string &url, std::function<void(platform::Response *)> callback, const std::shared_ptr<uv::loop> loop) {
// convert relative URLs to absolute URLs
@@ -51,7 +182,15 @@ void FileSource::load(ResourceType /*type*/, const std::string &url, std::functi
callback(&response);
} else {
// load from the internet
platform::request_http(absoluteURL, callback, loop);
if (!loadFile(absoluteURL, callback)) {
const std::shared_ptr<FileSource> source = shared_from_this();
platform::request_http(absoluteURL, [=](platform::Response *res) {
if (res->code == 200) {
source->saveFile(absoluteURL, res);
}
callback(res);
}, loop);
}
}
}