tei/MapLibreParticleLab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[core] Simplify asset:// implementation

Browse Source 
* Move asset:// URL handling to DefaultFileSource.
* AssetFileSource implements FileSource interface and follows familiar implementation patterns.
* Move default implementation to platform/default, zip implementation to platform/android.
* Don't bother with modified / expires / etag -- assets are not cached so it doesn't matter.
* Don't bother with interleaving individual IO calls on the implementation thread. That adds a lot of complexity for very little benefit.
This commit is contained in:
John Firebaugh
2016-01-11 12:52:41 -08:00
parent 2126e34e52
commit 86c8446d3a
22 changed files with 287 additions and 731 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
gyp/asset-fs.gypi
View File

@@ -7,7 +7,7 @@
'hard_dependency': 1,
'sources': [
'../platform/default/asset_request_fs.cpp',
'../platform/default/asset_file_source.cpp',
],
'include_dirs': [

2
gyp/asset-zip.gypi
View File

@@ -7,7 +7,7 @@
'hard_dependency': 1,
'sources': [
'../platform/default/asset_request_zip.cpp',
'../platform/android/src/asset_file_source.cpp',
],
'include_dirs': [

2
include/mbgl/storage/online_file_source.hpp
View File

@@ -12,7 +12,7 @@ template <typename T> class Thread;
class OnlineFileSource : public FileSource {
public:
OnlineFileSource(FileCache *cache, const std::string &root = "");
OnlineFileSource(FileCache*);
~OnlineFileSource() override;
void setAccessToken(const std::string& t) { accessToken = t; }

111
platform/android/src/asset_file_source.cpp Normal file
View File

@@ -0,0 +1,111 @@
#include <mbgl/storage/asset_file_source.hpp>
#include <mbgl/storage/response.hpp>
#include <mbgl/util/util.hpp>
#include <mbgl/util/thread.hpp>
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wshadow"
#include <zip.h>
#pragma GCC diagnostic pop
namespace {
struct ZipHolder {
ZipHolder(struct zip* archive_) : archive(archive_) {}
~ZipHolder() {
if (archive) ::zip_close(archive);
}
struct zip* archive;
};
struct ZipFileHolder {
ZipFileHolder(struct zip_file* file_) : file(file_) {}
~ZipFileHolder() {
if (file) ::zip_fclose(file);
}
struct zip_file* file;
};
}
namespace mbgl {
class AssetFileRequest : public FileRequest {
public:
AssetFileRequest(std::unique_ptr<WorkRequest> workRequest_)
: workRequest(std::move(workRequest_)) {
}
std::unique_ptr<WorkRequest> workRequest;
};
class AssetFileSource::Impl {
public:
Impl(const std::string& root_)
: root(root_) {
}
void request(const std::string& url, FileSource::Callback callback) {
ZipHolder archive = ::zip_open(root.c_str(), 0, nullptr);
if (!archive.archive) {
reportError(Response::Error::Reason::Other, "Could not open zip archive", callback);
return;
}
struct zip_stat stat;
::zip_stat_init(&stat);
std::string path = std::string("assets/") + url.substr(8);
int ret = ::zip_stat(archive.archive, path.c_str(), 0, &stat);
if (ret < 0 || !(stat.valid & ZIP_STAT_SIZE)) {
reportError(Response::Error::Reason::NotFound, "Could not stat file in zip archive", callback);
return;
}
ZipFileHolder file = ::zip_fopen(archive.archive, path.c_str(), 0);
if (!file.file) {
reportError(Response::Error::Reason::NotFound, "Could not open file in zip archive", callback);
return;
}
std::shared_ptr<std::string> buf = std::make_shared<std::string>();
buf->resize(stat.size);
ret = ::zip_fread(file.file, &buf->front(), stat.size);
if (ret < 0) {
reportError(Response::Error::Reason::Other, "Could not read file in zip archive", callback);
return;
}
Response response;
response.data = buf;
callback(response);
}
void reportError(Response::Error::Reason reason, const char * message, FileSource::Callback callback) {
Response response;
response.error = std::make_unique<Response::Error>(reason, message);
callback(response);
}
private:
std::string root;
};
AssetFileSource::AssetFileSource(const std::string& root)
: thread(std::make_unique<util::Thread<Impl>>(
util::ThreadContext{"AssetFileSource", util::ThreadType::Worker, util::ThreadPriority::Regular},
root.empty() ? platform::assetRoot() : root)) {
}
AssetFileSource::~AssetFileSource() = default;
std::unique_ptr<FileRequest> AssetFileSource::request(const Resource& resource, Callback callback) {
return std::make_unique<AssetFileRequest>(thread->invokeWithCallback(&Impl::request, callback, resource.url));
}
}

79
platform/default/asset_file_source.cpp Normal file
View File

@@ -0,0 +1,79 @@
#include <mbgl/storage/asset_file_source.hpp>
#include <mbgl/storage/response.hpp>
#include <mbgl/util/string.hpp>
#include <mbgl/util/thread.hpp>
#include <mbgl/util/url.hpp>
#include <mbgl/util/util.hpp>
#include <mbgl/util/io.hpp>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
namespace mbgl {
class AssetFileRequest : public FileRequest {
public:
AssetFileRequest(std::unique_ptr<WorkRequest> workRequest_)
: workRequest(std::move(workRequest_)) {
}
std::unique_ptr<WorkRequest> workRequest;
};
class AssetFileSource::Impl {
public:
Impl(const std::string& root_)
: root(root_) {
}
void request(const std::string& url, FileSource::Callback callback) {
std::string path;
if (url.size() <= 8 || url[8] == '/') {
// This is an empty or absolute path.
path = mbgl::util::percentDecode(url.substr(8));
} else {
// This is a relative path. Prefix with the application root.
path = root + "/" + mbgl::util::percentDecode(url.substr(8));
}
Response response;
struct stat buf;
int result = stat(path.c_str(), &buf);
if (result == 0 && S_ISDIR(buf.st_mode)) {
response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound);
} else if (result == -1 && errno == ENOENT) {
response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound);
} else {
try {
response.data = std::make_shared<std::string>(util::read_file(path));
} catch (...) {
response.error = std::make_unique<Response::Error>(
Response::Error::Reason::Other,
util::toString(std::current_exception()));
}
}
callback(response);
}
private:
std::string root;
};
AssetFileSource::AssetFileSource(const std::string& root)
: thread(std::make_unique<util::Thread<Impl>>(
util::ThreadContext{"AssetFileSource", util::ThreadType::Worker, util::ThreadPriority::Regular},
root.empty() ? platform::assetRoot() : root)) {
}
AssetFileSource::~AssetFileSource() = default;
std::unique_ptr<FileRequest> AssetFileSource::request(const Resource& resource, Callback callback) {
return std::make_unique<AssetFileRequest>(thread->invokeWithCallback(&Impl::request, callback, resource.url));
}
} // namespace mbgl

234
platform/default/asset_request_fs.cpp
View File

@@ -1,234 +0,0 @@
#include <mbgl/storage/asset_context_base.hpp>
#include <mbgl/storage/response.hpp>
#include <mbgl/util/string.hpp>
#include <mbgl/util/thread.hpp>
#include <mbgl/util/url.hpp>
#include <mbgl/util/util.hpp>
#include <errno.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
namespace {
struct FDHolder {
FDHolder(int fd_) : fd(fd_) {}
~FDHolder() {
if (fd >= 0) ::close(fd);
}
int fd;
};
class FileIOWorker {
public:
FileIOWorker() = default;
void open(const std::string& path, int flags, mode_t mode,
std::function<void(std::unique_ptr<FDHolder>)> cb) {
// FDHolder is needed because if the request gets canceled
// we would otherwise end up with a fd leaking.
int ret = ::open(path.c_str(), flags, mode);
cb(std::make_unique<FDHolder>(ret < 0 ? -errno : ret));
}
void fstat(int fd, std::function<void (int, std::unique_ptr<struct stat>)> cb) {
std::unique_ptr<struct stat> buf = std::make_unique<struct stat>();
int ret = ::fstat(fd, buf.get());
cb(ret < 0 ? -errno : ret, std::move(buf));
}
void read(int fd, size_t size,
std::function<void (ssize_t, std::shared_ptr<std::string>)> cb) {
std::shared_ptr<std::string> buf = std::make_shared<std::string>();
buf->resize(size);
ssize_t ret = ::read(fd, &buf->front(), size);
cb(ret < 0 ? -errno : ret, std::move(buf));
}
void close(int fd, std::function<void (void)> cb) {
::close(fd);
cb();
}
};
} // namespace
namespace mbgl {
using namespace std::placeholders;
class AssetRequest : public RequestBase {
MBGL_STORE_THREAD(tid)
public:
AssetRequest(const std::string& url, Callback, const std::string& assetRoot,
util::Thread<FileIOWorker> *worker);
~AssetRequest();
// RequestBase implementation.
void cancel() final;
private:
void fileOpened(std::unique_ptr<FDHolder>);
void fileStated(int result, std::unique_ptr<struct stat>);
void fileRead(int result, std::shared_ptr<std::string>);
void fileClosed();
void notifyError(int result);
void close();
void cleanup();
std::string assetRoot;
std::unique_ptr<Response> response;
int fd = -1;
util::Thread<FileIOWorker> *worker;
std::unique_ptr<WorkRequest> request;
};
class AssetFSContext : public AssetContextBase {
public:
AssetFSContext()
: worker({"FileIOWorker", util::ThreadType::Worker, util::ThreadPriority::Regular}) {}
private:
RequestBase* createRequest(const std::string& url,
RequestBase::Callback callback,
const std::string& assetRoot) final {
return new AssetRequest(url, callback, assetRoot, &worker);
}
util::Thread<FileIOWorker> worker;
};
AssetRequest::~AssetRequest() {
MBGL_VERIFY_THREAD(tid);
}
AssetRequest::AssetRequest(const std::string& url_, Callback callback_,
const std::string& assetRoot_, util::Thread<FileIOWorker> *worker_)
: RequestBase(url_, callback_),
assetRoot(assetRoot_),
worker(worker_) {
std::string path;
if (url.size() <= 8 || url[8] == '/') {
// This is an empty or absolute path.
path = mbgl::util::percentDecode(url.substr(8));
} else {
// This is a relative path. Prefix with the application root.
path = assetRoot + "/" + mbgl::util::percentDecode(url.substr(8));
}
request = worker->invokeWithCallback(&FileIOWorker::open,
std::bind(&AssetRequest::fileOpened, this, _1), path, O_RDONLY, S_IRUSR);
}
void AssetRequest::cancel() {
MBGL_VERIFY_THREAD(tid);
request.reset();
close();
}
void AssetRequest::fileOpened(std::unique_ptr<FDHolder> holder) {
MBGL_VERIFY_THREAD(tid);
std::swap(fd, holder->fd);
if (fd < 0) {
// Opening failed. There isn't much left we can do.
notifyError(fd);
cleanup();
return;
}
request = worker->invokeWithCallback(&FileIOWorker::fstat,
std::bind(&AssetRequest::fileStated, this, _1, _2), fd);
}
void AssetRequest::fileStated(int result, std::unique_ptr<struct stat> stat) {
MBGL_VERIFY_THREAD(tid);
if (result != 0) {
// Stating failed. We already have an open file handle
// though, which we'll have to close.
notifyError(result);
close();
} else {
response = std::make_unique<Response>();
response->modified = Seconds(stat->st_mtime);
response->etag = util::toString(stat->st_ino);
request = worker->invokeWithCallback(&FileIOWorker::read,
std::bind(&AssetRequest::fileRead, this, _1, _2), fd, stat->st_size);
}
}
void AssetRequest::fileRead(int result, std::shared_ptr<std::string> data) {
MBGL_VERIFY_THREAD(tid);
if (result < 0) {
// Reading failed. We already have an open file handle
// though, which we'll have to close.
notifyError(result);
} else {
response->data = data;
notify(std::move(response));
}
close();
}
void AssetRequest::fileClosed() {
MBGL_VERIFY_THREAD(tid);
cleanup();
}
void AssetRequest::close() {
MBGL_VERIFY_THREAD(tid);
if (fd >= 0) {
request = worker->invokeWithCallback(&FileIOWorker::close,
std::bind(&AssetRequest::fileClosed, this), fd);
fd = -1;
} else {
cleanup();
}
}
void AssetRequest::cleanup() {
MBGL_VERIFY_THREAD(tid);
delete this;
}
void AssetRequest::notifyError(int result) {
MBGL_VERIFY_THREAD(tid);
result = std::abs(result);
Response::Error::Reason reason = Response::Error::Reason::Other;
if (result == ENOENT || result == EISDIR) {
reason = Response::Error::Reason::NotFound;
}
response = std::make_unique<Response>();
response->error = std::make_unique<Response::Error>(reason, ::strerror(result));
notify(std::move(response));
}
std::unique_ptr<AssetContextBase> AssetContextBase::createContext() {
return std::make_unique<AssetFSContext>();
}
} // namespace mbgl

319
platform/default/asset_request_zip.cpp
View File

@@ -1,319 +0,0 @@
#include <mbgl/storage/asset_context_base.hpp>
#include <mbgl/storage/response.hpp>
#include <mbgl/util/chrono.hpp>
#include <mbgl/util/util.hpp>
#include <mbgl/util/thread.hpp>
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wshadow"
#include <zip.h>
#pragma GCC diagnostic pop
#include <cassert>
#include <forward_list>
#include <map>
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/types.h>
namespace {
struct ZipHolder {
ZipHolder(struct zip* archive_) : archive(archive_) {}
~ZipHolder() {
if (archive) ::zip_discard(archive);
}
struct zip* archive;
};
struct ZipFileHolder {
ZipFileHolder(struct zip_file* file_) : file(file_) {}
~ZipFileHolder() {
if (file) ::zip_fclose(file);
}
struct zip_file* file;
};
class ZipIOWorker {
public:
ZipIOWorker() = default;
void zip_fdopen(const std::string& path,
std::function<void(std::unique_ptr<ZipHolder>)> cb) {
int fd = ::open(path.c_str(), O_RDONLY, S_IRUSR);
if (fd < 0) {
cb(std::make_unique<ZipHolder>(nullptr));
} else {
int errorp;
struct zip* archive = ::zip_fdopen(fd, 0, &errorp);
cb(std::make_unique<ZipHolder>(archive));
}
}
void zip_stat(struct zip* archive, const std::string& path,
std::function<void (int, std::unique_ptr<struct zip_stat>)> cb) {
std::unique_ptr<struct zip_stat> buf = std::make_unique<struct zip_stat>();
::zip_stat_init(buf.get());
int ret = ::zip_stat(archive, path.c_str(), 0, buf.get());
cb(ret, std::move(buf));
}
void zip_fopen(struct zip* archive, const std::string& path,
std::function<void (std::unique_ptr<ZipFileHolder>)> cb) {
struct zip_file* file = ::zip_fopen(archive, path.c_str(), 0);
cb(std::make_unique<ZipFileHolder>(file));
}
void zip_fread(struct zip_file* file, size_t size,
std::function<void (int, std::shared_ptr<std::string>)> cb) {
std::shared_ptr<std::string> buf = std::make_shared<std::string>();
buf->resize(size);
int ret = ::zip_fread(file, &buf->front(), size);
cb(ret, std::move(buf));
}
void zip_fclose(struct zip_file* file, std::function<void (int)> cb) {
cb(::zip_fclose(file));
}
};
}
namespace mbgl {
using namespace std::placeholders;
class AssetZipContext;
class AssetRequest : public RequestBase {
MBGL_STORE_THREAD(tid)
public:
AssetRequest(const std::string& url, Callback, const std::string& assetRoot,
AssetZipContext* context, util::Thread<ZipIOWorker>* worker);
~AssetRequest();
// RequestBase implementation.
void cancel() final;
private:
void openArchive();
void archiveOpened(std::unique_ptr<ZipHolder>);
void fileStated(int result, std::unique_ptr<struct zip_stat>);
void fileOpened(std::unique_ptr<ZipFileHolder>);
void fileRead(int result, std::shared_ptr<std::string>);
void fileClosed();
void close();
void cleanup();
void notifyError(Response::Error::Reason, const char *message);
const std::string root;
const std::string path;
std::unique_ptr<Response> response;
struct zip* archive = nullptr;
struct zip_file* file = nullptr;
size_t size = 0;
AssetZipContext *context;
util::Thread<ZipIOWorker> *worker;
std::unique_ptr<WorkRequest> request;
};
class AssetZipContext : public AssetContextBase {
public:
AssetZipContext();
~AssetZipContext();
RequestBase* createRequest(const std::string& url,
RequestBase::Callback callback,
const std::string& assetRoot) final {
return new AssetRequest(url, callback, assetRoot, this, &worker);
}
struct zip *getHandle(const std::string &path);
void returnHandle(const std::string &path, struct zip *zip);
// A list of resuable zip handles to avoid creating
// and destroying them all the time.
std::map<std::string, std::forward_list<struct zip*>> handles;
util::Thread<ZipIOWorker> worker;
std::unique_ptr<WorkRequest> request;
};
AssetZipContext::AssetZipContext()
: worker({"ZipIOWorker", util::ThreadType::Worker, util::ThreadPriority::Regular}) {}
AssetZipContext::~AssetZipContext() {
// Close all zip handles
for (auto &list : handles) {
for (auto archive : list.second) {
zip_discard(archive);
}
}
handles.clear();
}
struct zip* AssetZipContext::getHandle(const std::string &path) {
auto &list = handles[path];
if (!list.empty()) {
auto archive = list.front();
list.pop_front();
return archive;
} else {
return nullptr;
}
}
void AssetZipContext::returnHandle(const std::string &path, struct zip* archive) {
handles[path].push_front(archive);
}
AssetRequest::AssetRequest(const std::string& url_, Callback callback_,
const std::string& assetRoot, AssetZipContext* context_, util::Thread<ZipIOWorker>* worker_)
: RequestBase(url_, callback_),
root(assetRoot),
path(std::string("assets/") + url.substr(8)),
context(context_),
worker(worker_) {
archive = context->getHandle(root);
if (archive) {
request = worker->invokeWithCallback(&ZipIOWorker::zip_stat,
std::bind(&AssetRequest::fileStated, this, _1, _2), archive, path);
} else {
request = worker->invokeWithCallback(&ZipIOWorker::zip_fdopen,
std::bind(&AssetRequest::archiveOpened, this, _1), root);
}
}
AssetRequest::~AssetRequest() {
MBGL_VERIFY_THREAD(tid);
}
void AssetRequest::cancel() {
MBGL_VERIFY_THREAD(tid);
request.reset();
close();
}
void AssetRequest::archiveOpened(std::unique_ptr<ZipHolder> holder) {
MBGL_VERIFY_THREAD(tid);
std::swap(archive, holder->archive);
if (!archive) {
notifyError(Response::Error::Reason::Other, "Could not open zip archive");
cleanup();
} else {
request = worker->invokeWithCallback(&ZipIOWorker::zip_stat,
std::bind(&AssetRequest::fileStated, this, _1, _2), archive, path);
}
}
void AssetRequest::fileStated(int result, std::unique_ptr<struct zip_stat> stat) {
MBGL_VERIFY_THREAD(tid);
if (result < 0 || !(stat->valid & ZIP_STAT_SIZE)) {
notifyError(Response::Error::Reason::NotFound, "Could not stat file in zip archive");
cleanup();
} else {
response = std::make_unique<Response>();
if (stat->valid & ZIP_STAT_MTIME) {
response->modified = Seconds(stat->mtime);
}
if (stat->valid & ZIP_STAT_INDEX) {
response->etag = std::to_string(stat->index);
}
size = stat->size;
request = worker->invokeWithCallback(&ZipIOWorker::zip_fopen,
std::bind(&AssetRequest::fileOpened, this, _1), archive, path);
}
}
void AssetRequest::fileOpened(std::unique_ptr<ZipFileHolder> holder) {
MBGL_VERIFY_THREAD(tid);
std::swap(file, holder->file);
if (!file) {
notifyError(Response::Error::Reason::NotFound, "Could not open file in zip archive"),
cleanup();
} else {
request = worker->invokeWithCallback(&ZipIOWorker::zip_fread,
std::bind(&AssetRequest::fileRead, this, _1, _2), file, size);
}
}
void AssetRequest::fileRead(int result, std::shared_ptr<std::string> data) {
MBGL_VERIFY_THREAD(tid);
if (result < 0) {
notifyError(Response::Error::Reason::Other, "Could not read file in zip archive");
} else {
response->data = data;
notify(std::move(response));
}
close();
}
void AssetRequest::fileClosed() {
MBGL_VERIFY_THREAD(tid);
cleanup();
}
void AssetRequest::close() {
MBGL_VERIFY_THREAD(tid);
if (file) {
request = worker->invokeWithCallback(&ZipIOWorker::zip_fclose,
std::bind(&AssetRequest::fileClosed, this), file);
file = nullptr;
} else {
cleanup();
}
}
void AssetRequest::cleanup() {
MBGL_VERIFY_THREAD(tid);
if (archive) {
context->returnHandle(root, archive);
}
delete this;
}
void AssetRequest::notifyError(Response::Error::Reason reason, const char *message) {
MBGL_VERIFY_THREAD(tid);
response = std::make_unique<Response>();
response->error = std::make_unique<Response::Error>(reason, message);
notify(std::move(response));
}
std::unique_ptr<AssetContextBase> AssetContextBase::createContext() {
return std::make_unique<AssetZipContext>();
}
}

28
platform/default/default_file_source.cpp
View File

@@ -1,16 +1,32 @@
#include <mbgl/storage/default_file_source.hpp>
#include <mbgl/storage/asset_file_source.hpp>
#include <mbgl/storage/online_file_source.hpp>
#include <mbgl/storage/sqlite_cache.hpp>
#include <mbgl/platform/platform.hpp>
#include <mbgl/util/url.hpp>
namespace {
const std::string assetProtocol = "asset://";
bool isAssetURL(const std::string& url) {
return std::equal(assetProtocol.begin(), assetProtocol.end(), url.begin());
}
} // namespace
namespace mbgl {
class DefaultFileSource::Impl {
public:
Impl(const std::string& cachePath, const std::string& root)
: cache(SharedSQLiteCache::get(cachePath)),
onlineFileSource(cache.get(), root) {
Impl(const std::string& cachePath, const std::string& assetRoot)
: assetFileSource(assetRoot),
cache(SharedSQLiteCache::get(cachePath)),
onlineFileSource(cache.get()) {
}
AssetFileSource assetFileSource;
std::shared_ptr<SQLiteCache> cache;
OnlineFileSource onlineFileSource;
};
@@ -38,7 +54,11 @@ void DefaultFileSource::setMaximumCacheEntrySize(uint64_t size) {
}
std::unique_ptr<FileRequest> DefaultFileSource::request(const Resource& resource, Callback callback) {
return impl->onlineFileSource.request(resource, callback);
if (isAssetURL(resource.url)) {
return impl->assetFileSource.request(resource, callback);
} else {
return impl->onlineFileSource.request(resource, callback);
}
}
} // namespace mbgl

26
platform/default/online_file_source.cpp
View File

@@ -1,10 +1,8 @@
#include <mbgl/storage/online_file_source.hpp>
#include <mbgl/storage/asset_context_base.hpp>
#include <mbgl/storage/http_context_base.hpp>
#include <mbgl/storage/network_status.hpp>
#include <mbgl/storage/response.hpp>
#include <mbgl/platform/platform.hpp>
#include <mbgl/platform/log.hpp>
#include <mbgl/util/thread.hpp>
@@ -14,7 +12,6 @@
#include <mbgl/util/async_task.hpp>
#include <mbgl/util/noncopyable.hpp>
#include <mbgl/util/timer.hpp>
#include <mbgl/util/url.hpp>
#include <algorithm>
#include <cassert>
@@ -83,7 +80,7 @@ class OnlineFileSource::Impl {
public:
using Callback = std::function<void (Response)>;
Impl(FileCache*, const std::string& = "");
Impl(FileCache*);
~Impl();
void networkIsReachableAgain();
@@ -96,17 +93,14 @@ private:
std::unordered_map<Resource, std::unique_ptr<OnlineFileRequestImpl>, Resource::Hash> pending;
FileCache* const cache;
const std::string assetRoot;
const std::unique_ptr<AssetContextBase> assetContext;
const std::unique_ptr<HTTPContextBase> httpContext;
util::AsyncTask reachability;
};
OnlineFileSource::OnlineFileSource(FileCache* cache, const std::string& root)
OnlineFileSource::OnlineFileSource(FileCache* cache)
: thread(std::make_unique<util::Thread<Impl>>(
util::ThreadContext{ "OnlineFileSource", util::ThreadType::Unknown, util::ThreadPriority::Low },
cache,
root)) {
cache)) {
}
OnlineFileSource::~OnlineFileSource() = default;
@@ -152,10 +146,8 @@ void OnlineFileSource::cancel(const Resource& res, FileRequest* req) {
// ----- Impl -----
OnlineFileSource::Impl::Impl(FileCache* cache_, const std::string& root)
OnlineFileSource::Impl::Impl(FileCache* cache_)
: cache(cache_),
assetRoot(root.empty() ? platform::assetRoot() : root),
assetContext(AssetContextBase::createContext()),
httpContext(HTTPContextBase::createContext()),
reachability(std::bind(&Impl::networkIsReachableAgain, this)) {
// Subscribe to network status changes, but make sure that this async handle doesn't keep the
@@ -242,7 +234,7 @@ void OnlineFileRequestImpl::addObserver(FileRequest* req, Callback callback, Onl
} else if (!cacheRequest && !realRequest) {
// There is no request in progress, and we don't have a response yet. This means we'll have
// to start the request ourselves.
if (impl.cache && !util::isAssetURL(resource.url)) {
if (impl.cache) {
scheduleCacheRequest(impl);
} else {
scheduleRealRequest(impl);
@@ -330,7 +322,7 @@ void OnlineFileRequestImpl::scheduleRealRequest(OnlineFileSource::Impl& impl, bo
// In particular, we don't want to write a Canceled request, or one that failed due to
// connection errors to the cache. Server errors are hopefully also temporary, so we're not
// caching them either.
if (impl.cache && !util::isAssetURL(resource.url) &&
if (impl.cache &&
(!response_->error || (response_->error->reason == Response::Error::Reason::NotFound))) {
// Store response in database. Make sure we only refresh the expires column if the data
// didn't change.
@@ -357,11 +349,7 @@ void OnlineFileRequestImpl::scheduleRealRequest(OnlineFileSource::Impl& impl, bo
scheduleRealRequest(impl);
};
if (util::isAssetURL(resource.url)) {
realRequest = impl.assetContext->createRequest(resource.url, callback, impl.assetRoot);
} else {
realRequest = impl.httpContext->createRequest(resource.url, callback, response);
}
realRequest = impl.httpContext->createRequest(resource.url, callback, response);
});
}

20
src/mbgl/storage/asset_context_base.hpp
View File

@@ -1,20 +0,0 @@
#ifndef MBGL_STORAGE_ASSET_CONTEXT_BASE
#define MBGL_STORAGE_ASSET_CONTEXT_BASE
#include <mbgl/storage/request_base.hpp>
namespace mbgl {
class AssetContextBase {
public:
static std::unique_ptr<AssetContextBase> createContext();
virtual ~AssetContextBase() = default;
virtual RequestBase* createRequest(const std::string& url,
RequestBase::Callback,
const std::string& assetRoot) = 0;
};
} // namespace mbgl
#endif // MBGL_STORAGE_ASSET_CONTEXT_BASE

26
src/mbgl/storage/asset_file_source.hpp Normal file
View File

@@ -0,0 +1,26 @@
#ifndef MBGL_STORAGE_ASSET_FILE_SOURCE
#define MBGL_STORAGE_ASSET_FILE_SOURCE
#include <mbgl/storage/file_source.hpp>
namespace mbgl {
namespace util {
template <typename T> class Thread;
} // namespace util
class AssetFileSource : public FileSource {
public:
AssetFileSource(const std::string& assetRoot);
~AssetFileSource() override;
std::unique_ptr<FileRequest> request(const Resource&, Callback) override;
private:
class Impl;
std::unique_ptr<util::Thread<Impl>> thread;
};
} // namespace mbgl
#endif // MBGL_STORAGE_ASSET_FILE_SOURCE

8
src/mbgl/util/url.cpp
View File

@@ -71,13 +71,5 @@ std::string templateTileURL(const std::string& url, const TileID& id, float pixe
});
}
namespace {
const std::string assetProtocol = "asset://";
}
bool isAssetURL(const std::string& url) {
return std::equal(assetProtocol.begin(), assetProtocol.end(), url.begin());
}
} // namespace util
} // namespace mbgl

2
src/mbgl/util/url.hpp
View File

@@ -14,8 +14,6 @@ std::string percentDecode(const std::string&);
std::string templateTileURL(const std::string& url, const TileID& id, float pixelRatio = 1);
bool isAssetURL(const std::string&);
} // namespace util
} // namespace mbgl

22
test/api/annotations.cpp
View File

@@ -31,7 +31,7 @@ void checkRendering(Map& map, const char * name) {
TEST(Annotations, PointAnnotation) {
auto display = std::make_shared<mbgl::HeadlessDisplay>();
HeadlessView view(display, 1);
OnlineFileSource fileSource(nullptr, test::getFileSourceRoot());
OnlineFileSource fileSource(nullptr);
Map map(view, fileSource, MapMode::Still);
map.setStyleJSON(util::read_file("test/fixtures/api/empty.json"), "");
@@ -44,7 +44,7 @@ TEST(Annotations, PointAnnotation) {
TEST(Annotations, LineAnnotation) {
auto display = std::make_shared<mbgl::HeadlessDisplay>();
HeadlessView view(display, 1);
OnlineFileSource fileSource(nullptr, test::getFileSourceRoot());
OnlineFileSource fileSource(nullptr);
Map map(view, fileSource, MapMode::Still);
map.setStyleJSON(util::read_file("test/fixtures/api/empty.json"), "");
@@ -63,7 +63,7 @@ TEST(Annotations, LineAnnotation) {
TEST(Annotations, FillAnnotation) {
auto display = std::make_shared<mbgl::HeadlessDisplay>();
HeadlessView view(display, 1);
OnlineFileSource fileSource(nullptr, test::getFileSourceRoot());
OnlineFileSource fileSource(nullptr);
Map map(view, fileSource, MapMode::Still);
map.setStyleJSON(util::read_file("test/fixtures/api/empty.json"), "");
@@ -81,7 +81,7 @@ TEST(Annotations, FillAnnotation) {
TEST(Annotations, StyleSourcedShapeAnnotation) {
auto display = std::make_shared<mbgl::HeadlessDisplay>();
HeadlessView view(display, 1);
OnlineFileSource fileSource(nullptr, test::getFileSourceRoot());
OnlineFileSource fileSource(nullptr);
Map map(view, fileSource, MapMode::Still);
map.setStyleJSON(util::read_file("test/fixtures/api/annotation.json"), "");
@@ -96,7 +96,7 @@ TEST(Annotations, StyleSourcedShapeAnnotation) {
TEST(Annotations, AddMultiple) {
auto display = std::make_shared<mbgl::HeadlessDisplay>();
HeadlessView view(display, 1);
OnlineFileSource fileSource(nullptr, test::getFileSourceRoot());
OnlineFileSource fileSource(nullptr);
Map map(view, fileSource, MapMode::Still);
map.setStyleJSON(util::read_file("test/fixtures/api/empty.json"), "");
@@ -113,7 +113,7 @@ TEST(Annotations, AddMultiple) {
TEST(Annotations, NonImmediateAdd) {
auto display = std::make_shared<mbgl::HeadlessDisplay>();
HeadlessView view(display, 1);
OnlineFileSource fileSource(nullptr, test::getFileSourceRoot());
OnlineFileSource fileSource(nullptr);
Map map(view, fileSource, MapMode::Still);
map.setStyleJSON(util::read_file("test/fixtures/api/empty.json"), "");
@@ -133,7 +133,7 @@ TEST(Annotations, NonImmediateAdd) {
TEST(Annotations, UpdatePoint) {
auto display = std::make_shared<mbgl::HeadlessDisplay>();
HeadlessView view(display, 1);
OnlineFileSource fileSource(nullptr, test::getFileSourceRoot());
OnlineFileSource fileSource(nullptr);
Map map(view, fileSource, MapMode::Still);
map.setStyleJSON(util::read_file("test/fixtures/api/empty.json"), "");
@@ -152,7 +152,7 @@ TEST(Annotations, UpdatePoint) {
TEST(Annotations, RemovePoint) {
auto display = std::make_shared<mbgl::HeadlessDisplay>();
HeadlessView view(display, 1);
OnlineFileSource fileSource(nullptr, test::getFileSourceRoot());
OnlineFileSource fileSource(nullptr);
Map map(view, fileSource, MapMode::Still);
map.setStyleJSON(util::read_file("test/fixtures/api/empty.json"), "");
@@ -169,7 +169,7 @@ TEST(Annotations, RemovePoint) {
TEST(Annotations, RemoveShape) {
auto display = std::make_shared<mbgl::HeadlessDisplay>();
HeadlessView view(display, 1);
OnlineFileSource fileSource(nullptr, test::getFileSourceRoot());
OnlineFileSource fileSource(nullptr);
AnnotationSegments segments = {{ {{ { 0, 0 }, { 45, 45 } }} }};
@@ -191,7 +191,7 @@ TEST(Annotations, RemoveShape) {
TEST(Annotations, ImmediateRemoveShape) {
auto display = std::make_shared<mbgl::HeadlessDisplay>();
HeadlessView view(display, 1);
OnlineFileSource fileSource(nullptr, test::getFileSourceRoot());
OnlineFileSource fileSource(nullptr);
Map map(view, fileSource, MapMode::Still);
map.removeAnnotation(map.addShapeAnnotation(ShapeAnnotation({}, {})));
@@ -203,7 +203,7 @@ TEST(Annotations, ImmediateRemoveShape) {
TEST(Annotations, SwitchStyle) {
auto display = std::make_shared<mbgl::HeadlessDisplay>();
HeadlessView view(display, 1);
OnlineFileSource fileSource(nullptr, test::getFileSourceRoot());
OnlineFileSource fileSource(nullptr);
Map map(view, fileSource, MapMode::Still);
map.setStyleJSON(util::read_file("test/fixtures/api/empty.json"), "");

2
test/api/custom_layer.cpp
View File

@@ -66,7 +66,7 @@ public:
TEST(CustomLayer, Basic) {
auto display = std::make_shared<mbgl::HeadlessDisplay>();
HeadlessView view(display, 1);
OnlineFileSource fileSource(nullptr, test::getFileSourceRoot());
OnlineFileSource fileSource(nullptr);
Map map(view, fileSource, MapMode::Still);
map.setStyleJSON(util::read_file("test/fixtures/api/empty.json"), "");

6
test/api/repeated_render.cpp
View File

@@ -4,7 +4,7 @@
#include <mbgl/map/map.hpp>
#include <mbgl/platform/default/headless_view.hpp>
#include <mbgl/platform/default/headless_display.hpp>
#include <mbgl/storage/online_file_source.hpp>
#include <mbgl/storage/default_file_source.hpp>
#include <mbgl/util/image.hpp>
#include <mbgl/util/io.hpp>
@@ -18,9 +18,9 @@ TEST(API, RepeatedRender) {
auto display = std::make_shared<mbgl::HeadlessDisplay>();
HeadlessView view(display, 1, 256, 512);
#ifdef MBGL_ASSET_ZIP
OnlineFileSource fileSource(nullptr, "test/fixtures/api/assets.zip");
DefaultFileSource fileSource(":memory", "test/fixtures/api/assets.zip");
#else
OnlineFileSource fileSource(nullptr);
DefaultFileSource fileSource;
#endif
Log::setObserver(std::make_unique<FixtureLogObserver>());

5
test/fixtures/api/annotation.json vendored
View File

@@ -3,7 +3,7 @@
"sources": {
"fake": {
"type": "vector",
"url": "asset://TEST_DATA/fixtures/tiles/streets.json"
"url": "fake"
}
},
"layers": [{
@@ -17,6 +17,5 @@
"paint": {
"fill-color": "rgba(255,0,0,1)"
}
}],
"sprite": "asset://TEST_DATA/fixtures/resources/sprite"
}]
}

8
test/fixtures/util.cpp vendored
View File

@@ -20,14 +20,6 @@
namespace mbgl {
namespace test {
std::string getFileSourceRoot() {
#ifdef MBGL_ASSET_ZIP
return "test/fixtures/annotations/assets.zip";
#else
return "";
#endif
}
Server::Server(const char* executable) {
int input[2];
int output[2];

2
test/fixtures/util.hpp vendored
View File

@@ -22,8 +22,6 @@ class Map;
namespace test {
std::string getFileSourceRoot();
class Server {
public:
Server(const char* executable);

71
test/storage/file_reading.cpp → test/storage/asset_file_source.cpp
View File

@@ -1,6 +1,6 @@
#include "storage.hpp"
#include <mbgl/storage/online_file_source.hpp>
#include <mbgl/storage/asset_file_source.hpp>
#include <mbgl/storage/sqlite_cache.hpp>
#include <mbgl/platform/platform.hpp>
#include <mbgl/util/chrono.hpp>
@@ -19,7 +19,7 @@ std::string getFileSourceRoot() {
class TestWorker {
public:
TestWorker(mbgl::OnlineFileSource* fs_) : fs(fs_) {}
TestWorker(mbgl::AssetFileSource* fs_) : fs(fs_) {}
void run(std::function<void()> endCallback) {
const std::string asset("asset://TEST_DATA/fixtures/storage/nonempty");
@@ -47,7 +47,7 @@ public:
private:
unsigned numRequests = 1000;
mbgl::OnlineFileSource* fs;
mbgl::AssetFileSource* fs;
std::unique_ptr<mbgl::FileRequest> request;
std::function<void(mbgl::Response)> requestCallback;
@@ -62,7 +62,7 @@ TEST_F(Storage, AssetStress) {
util::RunLoop loop;
mbgl::OnlineFileSource fs(nullptr, getFileSourceRoot());
AssetFileSource fs(getFileSourceRoot());
unsigned numThreads = 50;
@@ -98,7 +98,7 @@ TEST_F(Storage, AssetEmptyFile) {
util::RunLoop loop;
OnlineFileSource fs(nullptr, getFileSourceRoot());
AssetFileSource fs(getFileSourceRoot());
std::unique_ptr<FileRequest> req = fs.request({ Resource::Unknown, "asset://TEST_DATA/fixtures/storage/empty" }, [&](Response res) {
req.reset();
@@ -106,9 +106,6 @@ TEST_F(Storage, AssetEmptyFile) {
EXPECT_EQ(false, res.stale);
ASSERT_TRUE(res.data.get());
EXPECT_EQ("", *res.data);
EXPECT_EQ(Seconds::zero(), res.expires);
EXPECT_LT(1420000000, res.modified.count());
EXPECT_NE("", res.etag);
loop.stop();
EmptyFile.finish();
});
@@ -123,7 +120,7 @@ TEST_F(Storage, AssetNonEmptyFile) {
util::RunLoop loop;
OnlineFileSource fs(nullptr, getFileSourceRoot());
AssetFileSource fs(getFileSourceRoot());
std::unique_ptr<FileRequest> req = fs.request({ Resource::Unknown, "asset://TEST_DATA/fixtures/storage/nonempty" }, [&](Response res) {
req.reset();
@@ -131,11 +128,6 @@ TEST_F(Storage, AssetNonEmptyFile) {
EXPECT_EQ(false, res.stale);
ASSERT_TRUE(res.data.get());
EXPECT_EQ("content is here\n", *res.data);
EXPECT_EQ(Seconds::zero(), res.expires);
EXPECT_LT(1420000000, res.modified.count());
EXPECT_NE("", res.etag);
ASSERT_TRUE(res.data.get());
EXPECT_EQ("content is here\n", *res.data);
loop.stop();
NonEmptyFile.finish();
});
@@ -150,7 +142,7 @@ TEST_F(Storage, AssetNonExistentFile) {
util::RunLoop loop;
OnlineFileSource fs(nullptr, getFileSourceRoot());
AssetFileSource fs(getFileSourceRoot());
std::unique_ptr<FileRequest> req = fs.request({ Resource::Unknown, "asset://TEST_DATA/fixtures/storage/does_not_exist" }, [&](Response res) {
req.reset();
@@ -158,14 +150,7 @@ TEST_F(Storage, AssetNonExistentFile) {
EXPECT_EQ(Response::Error::Reason::NotFound, res.error->reason);
EXPECT_EQ(false, res.stale);
ASSERT_FALSE(res.data.get());
EXPECT_EQ(Seconds::zero(), res.expires);
EXPECT_EQ(Seconds::zero(), res.modified);
EXPECT_EQ("", res.etag);
#ifdef MBGL_ASSET_ZIP
EXPECT_EQ("Could not stat file in zip archive", res.error->message);
#elif MBGL_ASSET_FS
EXPECT_EQ("No such file or directory", res.error->message);
#endif
// Do not assert on platform-specific error message.
loop.stop();
NonExistentFile.finish();
});
@@ -173,42 +158,24 @@ TEST_F(Storage, AssetNonExistentFile) {
loop.run();
}
TEST_F(Storage, AssetNotCached) {
SCOPED_TEST(NotCached)
TEST_F(Storage, AssetReadDirectory) {
SCOPED_TEST(ReadDirectory)
using namespace mbgl;
const Resource resource { Resource::Unknown, "asset://TEST_DATA/fixtures/storage/nonempty" };
util::RunLoop loop;
SQLiteCache cache(":memory:");
AssetFileSource fs(getFileSourceRoot());
// Add a fake response to the cache to verify that we don't retrieve it.
{
auto response = std::make_shared<Response>();
response->data = std::make_shared<const std::string>("cached data");
cache.put(resource, response, FileCache::Hint::Full);
}
OnlineFileSource fs(&cache, getFileSourceRoot());
std::unique_ptr<WorkRequest> workReq;
std::unique_ptr<FileRequest> req = fs.request(resource, [&](Response res) {
std::unique_ptr<FileRequest> req = fs.request({ Resource::Unknown, "asset://TEST_DATA/fixtures/storage" }, [&](Response res) {
req.reset();
EXPECT_EQ(nullptr, res.error);
ASSERT_TRUE(res.data.get());
EXPECT_EQ("content is here\n", *res.data);
workReq = cache.get(resource, [&](std::shared_ptr<Response> response) {
// Check that we didn't put the file into the cache
ASSERT_TRUE(response->data.get());
EXPECT_EQ(*response->data, "cached data");
loop.stop();
NotCached.finish();
});
ASSERT_NE(nullptr, res.error);
EXPECT_EQ(Response::Error::Reason::NotFound, res.error->reason);
EXPECT_EQ(false, res.stale);
ASSERT_FALSE(res.data.get());
// Do not assert on platform-specific error message.
loop.stop();
ReadDirectory.finish();
});
loop.run();

40
test/storage/directory_reading.cpp
View File

@@ -1,40 +0,0 @@
#include "storage.hpp"
#include <mbgl/storage/online_file_source.hpp>
#include <mbgl/util/chrono.hpp>
#include <mbgl/util/run_loop.hpp>
TEST_F(Storage, AssetReadDirectory) {
SCOPED_TEST(ReadDirectory)
using namespace mbgl;
util::RunLoop loop;
#ifdef MBGL_ASSET_ZIP
OnlineFileSource fs(nullptr, "test/fixtures/storage/assets.zip");
#else
OnlineFileSource fs(nullptr);
#endif
std::unique_ptr<FileRequest> req = fs.request({ Resource::Unknown, "asset://TEST_DATA/fixtures/storage" }, [&](Response res) {
req.reset();
ASSERT_NE(nullptr, res.error);
EXPECT_EQ(Response::Error::Reason::NotFound, res.error->reason);
EXPECT_EQ(false, res.stale);
ASSERT_FALSE(res.data.get());
EXPECT_EQ(Seconds::zero(), res.expires);
EXPECT_EQ(Seconds::zero(), res.modified);
EXPECT_EQ("", res.etag);
#ifdef MBGL_ASSET_ZIP
EXPECT_EQ("Could not stat file in zip archive", res.error->message);
#elif MBGL_ASSET_FS
EXPECT_EQ("Is a directory", res.error->message);
#endif
loop.stop();
ReadDirectory.finish();
});
loop.run();
}

3
test/test.gypi
View File

@@ -73,8 +73,7 @@
'storage/cache_revalidate.cpp',
'storage/cache_size.cpp',
'storage/database.cpp',
'storage/directory_reading.cpp',
'storage/file_reading.cpp',
'storage/asset_file_source.cpp',
'storage/headers.cpp',
'storage/http_cancel.cpp',
'storage/http_coalescing.cpp',