Merge remote-tracking branch 'origin/file-source'

This commit is contained in:
John Firebaugh
2015-04-28 09:39:47 -04:00
33 changed files with 273 additions and 527 deletions

View File

@@ -54,11 +54,6 @@ public:
// Only call this while the OpenGL context is exclusive to this thread.
void performCleanup();
// #############################################################################################
// Request to terminate the environment.
void terminate();
private:
unsigned id;
FileSource& fileSource;

View File

@@ -16,12 +16,9 @@ public:
~DefaultFileSource() override;
// FileSource API
Request *request(const Resource &resource, uv_loop_t *loop, const Environment &env,
Callback callback) override;
void cancel(Request *request) override;
void request(const Resource &resource, const Environment &env, Callback callback) override;
void abort(const Environment &env) override;
Request* request(const Resource&, uv_loop_t*, Callback) override;
void cancel(Request*) override;
void request(const Resource&, Callback) override;
public:
class Impl;

View File

@@ -15,7 +15,6 @@ typedef struct uv_loop_s uv_loop_t;
namespace mbgl {
class Request;
class Environment;
class FileSource : private util::noncopyable {
protected:
@@ -28,19 +27,12 @@ public:
// These can be called from any thread. The callback will be invoked in the loop.
// You can only cancel a request from the same thread it was created in.
virtual Request *request(const Resource &resource, uv_loop_t *loop, const Environment &env,
Callback callback) = 0;
virtual void cancel(Request *request) = 0;
virtual Request* request(const Resource&, uv_loop_t*, Callback) = 0;
virtual void cancel(Request*) = 0;
// These can be called from any thread. The callback will be invoked in an arbitrary other thread.
// You cannot cancel these requests.
virtual void request(const Resource &resource, const Environment &env, Callback callback) = 0;
// This can be called from any thread. All requests with the environment pointer env should be
// notified as errored. Note that this is /different/ from canceling requests; a canceled
// request's callback is never called, while an aborted request's callback is called with
// a error message.
virtual void abort(const Environment &env) = 0;
virtual void request(const Resource&, Callback) = 0;
};
}

View File

@@ -17,14 +17,13 @@ typedef struct uv_loop_s uv_loop_t;
namespace mbgl {
class Response;
class Environment;
class Request : private util::noncopyable {
MBGL_STORE_THREAD(tid)
public:
using Callback = std::function<void(const Response &)>;
Request(const Resource &resource, uv_loop_t *loop, const Environment &env, Callback callback);
Request(const Resource &resource, uv_loop_t *loop, Callback callback);
public:
// May be called from any thread.
@@ -48,11 +47,6 @@ private:
public:
const Resource resource;
// The environment ref is used to associate requests with a particular environment. This allows
// us to only terminate requests associated with that environment, e.g. when the map the env
// belongs to is discarded.
const Environment &env;
};
}

View File

@@ -1,5 +1,6 @@
#include <mbgl/storage/http_request.hpp>
#include <mbgl/storage/http_context.hpp>
#include <mbgl/storage/resource.hpp>
#include <mbgl/storage/response.hpp>
#include <mbgl/util/uv.hpp>
@@ -199,7 +200,7 @@ void HTTPRequestImpl::handleResponse() {
}
context->removeRequest(request);
request->ptr = nullptr;
request->impl = nullptr;
delete request;
request = nullptr;
}
@@ -242,7 +243,7 @@ HTTPRequestImpl::~HTTPRequestImpl() {
if (request) {
context->removeRequest(request);
request->ptr = nullptr;
request->impl = nullptr;
}
}
@@ -397,36 +398,25 @@ void HTTPRequestImpl::restart(uv_timer_t *timer, int) {
// -------------------------------------------------------------------------------------------------
HTTPRequest::HTTPRequest(DefaultFileSource::Impl *source, const Resource &resource)
: SharedRequestBase(source, resource) {
HTTPRequest::HTTPRequest(const Resource& resource, Callback callback,
uv_loop_t* loop, std::shared_ptr<const Response> response)
: RequestBase(resource, callback)
, impl(new HTTPRequestImpl(this, loop, response)) {
}
HTTPRequest::~HTTPRequest() {
MBGL_VERIFY_THREAD(tid);
if (ptr) {
reinterpret_cast<HTTPRequestImpl *>(ptr)->cancel();
if (impl) {
impl->cancel();
}
}
void HTTPRequest::start(uv_loop_t *loop, std::shared_ptr<const Response> response) {
MBGL_VERIFY_THREAD(tid);
assert(!ptr);
ptr = new HTTPRequestImpl(this, loop, response);
}
void HTTPRequest::retryImmediately() {
MBGL_VERIFY_THREAD(tid);
if (ptr) {
reinterpret_cast<HTTPRequestImpl *>(ptr)->retryImmediately();
if (impl) {
impl->retryImmediately();
}
}
void HTTPRequest::cancel() {
MBGL_VERIFY_THREAD(tid);
delete this;
}

View File

@@ -1,4 +1,5 @@
#include <mbgl/storage/asset_request.hpp>
#include <mbgl/storage/resource.hpp>
#include <mbgl/storage/response.hpp>
#include <mbgl/util/std.hpp>
#include <mbgl/util/util.hpp>
@@ -25,7 +26,7 @@ class AssetRequestImpl {
MBGL_STORE_THREAD(tid)
public:
AssetRequestImpl(AssetRequest *request, uv_loop_t *loop);
AssetRequestImpl(AssetRequest*, uv_loop_t*, const std::string& assetRoot);
~AssetRequestImpl();
static void fileOpened(uv_fs_t *req);
@@ -35,8 +36,8 @@ public:
static void notifyError(uv_fs_t *req);
static void cleanup(uv_fs_t *req);
AssetRequest *request = nullptr;
std::string assetRoot;
bool canceled = false;
uv_fs_t req;
uv_file fd = -1;
@@ -48,11 +49,13 @@ AssetRequestImpl::~AssetRequestImpl() {
MBGL_VERIFY_THREAD(tid);
if (request) {
request->ptr = nullptr;
request->impl = nullptr;
}
}
AssetRequestImpl::AssetRequestImpl(AssetRequest *request_, uv_loop_t *loop) : request(request_) {
AssetRequestImpl::AssetRequestImpl(AssetRequest* request_, uv_loop_t* loop, const std::string& assetRoot_)
: request(request_)
, assetRoot(assetRoot_) {
req.data = this;
const auto &url = request->resource.url;
@@ -62,7 +65,7 @@ AssetRequestImpl::AssetRequestImpl(AssetRequest *request_, uv_loop_t *loop) : re
path = url.substr(8);
} else {
// This is a relative path. Prefix with the application root.
path = request->source->assetRoot + "/" + url.substr(8);
path = assetRoot + "/" + url.substr(8);
}
uv_fs_open(loop, &req, path.c_str(), O_RDONLY, S_IRUSR, fileOpened);
@@ -207,45 +210,26 @@ void AssetRequestImpl::cleanup(uv_fs_t *req) {
// -------------------------------------------------------------------------------------------------
AssetRequest::AssetRequest(DefaultFileSource::Impl *source_, const Resource &resource_)
: SharedRequestBase(source_, resource_) {
AssetRequest::AssetRequest(const Resource& resource_, Callback callback_,
uv_loop_t* loop, const std::string& assetRoot)
: RequestBase(resource_, callback_)
, impl(new AssetRequestImpl(this, loop, assetRoot)) {
assert(algo::starts_with(resource.url, "asset://"));
}
AssetRequest::~AssetRequest() {
MBGL_VERIFY_THREAD(tid);
if (ptr) {
reinterpret_cast<AssetRequestImpl *>(ptr)->request = nullptr;
}
}
void AssetRequest::start(uv_loop_t *loop, std::shared_ptr<const Response> response) {
MBGL_VERIFY_THREAD(tid);
// We're ignoring the existing response if any.
(void(response));
assert(!ptr);
ptr = new AssetRequestImpl(this, loop);
// Note: the AssetRequestImpl deletes itself.
}
void AssetRequest::cancel() {
MBGL_VERIFY_THREAD(tid);
if (ptr) {
reinterpret_cast<AssetRequestImpl *>(ptr)->canceled = true;
// uv_cancel fails frequently when the request has already been started.
// In that case, we have to let it complete and check the canceled bool
// instead. The cancelation callback will delete the AssetRequest object.
uv_cancel((uv_req_t *)&reinterpret_cast<AssetRequestImpl *>(ptr)->req);
} else {
// This request is canceled before we called start. We're safe to delete
// ourselves now.
delete this;
AssetRequest::~AssetRequest() {
if (impl) {
impl->request = nullptr;
}
}
void AssetRequest::cancel() {
impl->canceled = true;
// uv_cancel fails frequently when the request has already been started.
// In that case, we have to let it complete and check the canceled bool
// instead. The cancelation callback will delete the AssetRequest object.
uv_cancel((uv_req_t *)&impl->req);
}
}

View File

@@ -1,6 +1,7 @@
#include <mbgl/storage/asset_request.hpp>
#include <mbgl/storage/thread_context.hpp>
#include <mbgl/android/jni.hpp>
#include <mbgl/storage/resource.hpp>
#include <mbgl/storage/response.hpp>
#include <mbgl/platform/log.hpp>
#include <mbgl/util/std.hpp>
@@ -76,7 +77,7 @@ class AssetRequestImpl {
MBGL_STORE_THREAD(tid)
public:
AssetRequestImpl(AssetRequest *request, uv_loop_t *loop);
AssetRequestImpl(AssetRequest*, uv_loop_t*, const std::string& assetRoot);
~AssetRequestImpl();
void cancel();
@@ -107,14 +108,14 @@ AssetRequestImpl::~AssetRequestImpl() {
MBGL_VERIFY_THREAD(tid);
if (request) {
request->ptr = nullptr;
request->impl = nullptr;
}
}
AssetRequestImpl::AssetRequestImpl(AssetRequest *request_, uv_loop_t *loop)
AssetRequestImpl::AssetRequestImpl(AssetRequest* request_, uv_loop_t* loop, const std::string& assetRoot_)
: context(*AssetZipContext::Get(loop)),
request(request_),
root(request->source->assetRoot),
root(assetRoot_),
path(std::string { "assets/" } + request->resource.url.substr(8)) {
auto zip = context.getHandle(root);
if (zip) {
@@ -129,7 +130,6 @@ void AssetRequestImpl::openZipArchive() {
req->data = this;
assert(request);
assert(request->source);
// We're using uv_fs_open first to obtain a file descriptor. Then, uv_zip_fdopen will operate
// on a read-only file.
@@ -275,35 +275,22 @@ void AssetRequestImpl::cancel() {
// -------------------------------------------------------------------------------------------------
AssetRequest::AssetRequest(DefaultFileSource::Impl *source_, const Resource &resource_)
: SharedRequestBase(source_, resource_) {
AssetRequest::AssetRequest(const Resource& resource_, Callback callback_,
uv_loop_t* loop, const std::string& assetRoot)
: RequestBase(resource_, callback_)
, impl(new AssetRequestImpl(this, loop, assetRoot)) {
assert(algo::starts_with(resource.url, "asset://"));
}
AssetRequest::~AssetRequest() {
MBGL_VERIFY_THREAD(tid);
if (ptr) {
reinterpret_cast<AssetRequestImpl *>(ptr)->cancel();
if (impl) {
impl->cancel();
}
}
void AssetRequest::start(uv_loop_t *loop, std::shared_ptr<const Response> response) {
MBGL_VERIFY_THREAD(tid);
// We're ignoring the existing response if any.
(void(response));
assert(!ptr);
ptr = new AssetRequestImpl(this, loop);
// Note: the AssetRequestImpl deletes itself.
}
void AssetRequest::cancel() {
MBGL_VERIFY_THREAD(tid);
if (ptr) {
reinterpret_cast<AssetRequestImpl *>(ptr)->cancel();
if (impl) {
impl->cancel();
}
delete this;

View File

@@ -1,5 +1,6 @@
#include <mbgl/storage/http_request.hpp>
#include <mbgl/storage/http_context.hpp>
#include <mbgl/storage/resource.hpp>
#include <mbgl/storage/response.hpp>
#include <mbgl/util/chrono.hpp>
#include <mbgl/platform/log.hpp>
@@ -494,7 +495,7 @@ HTTPRequestImpl::~HTTPRequestImpl() {
if (request) {
context->removeRequest(request);
request->ptr = nullptr;
request->impl = nullptr;
}
handleError(curl_multi_remove_handle(context->multi, handle));
@@ -722,41 +723,27 @@ void HTTPRequestImpl::handleResult(CURLcode code) {
// -------------------------------------------------------------------------------------------------
HTTPRequest::HTTPRequest(DefaultFileSource::Impl *source_, const Resource &resource_)
: SharedRequestBase(source_, resource_) {
HTTPRequest::HTTPRequest(const Resource& resource_, Callback callback_,
uv_loop_t* loop, std::shared_ptr<const Response> response)
: RequestBase(resource_, callback_)
, impl(new HTTPRequestImpl(this, loop, response)) {
}
HTTPRequest::~HTTPRequest() {
MBGL_VERIFY_THREAD(tid);
if (ptr) {
reinterpret_cast<HTTPRequestImpl *>(ptr)->abandon();
if (impl) {
impl->abandon();
}
}
void HTTPRequest::start(uv_loop_t *loop, std::shared_ptr<const Response> response) {
MBGL_VERIFY_THREAD(tid);
assert(!ptr);
ptr = new HTTPRequestImpl(this, loop, response);
}
void HTTPRequest::retryImmediately() {
MBGL_VERIFY_THREAD(tid);
if (ptr) {
reinterpret_cast<HTTPRequestImpl *>(ptr)->retryImmediately();
if (impl) {
impl->retryImmediately();
}
}
void HTTPRequest::cancel() {
MBGL_VERIFY_THREAD(tid);
if (ptr) {
delete reinterpret_cast<HTTPRequestImpl *>(ptr);
ptr = nullptr;
}
delete impl;
impl = nullptr;
delete this;
}

View File

@@ -183,9 +183,9 @@ void SQLiteCache::put(const Resource &resource, std::shared_ptr<const Response>
// storing a new response or updating the currently stored response, potentially setting a new
// expiry date.
if (hint == Hint::Full) {
thread->invoke(&Impl::put, resource, std::move(response));
thread->invoke(&Impl::put, resource, response);
} else if (hint == Hint::Refresh) {
thread->invoke(&Impl::refresh, resource, int64_t(response->expires));
thread->invoke(&Impl::refresh, resource, response->expires);
}
}

View File

@@ -122,13 +122,13 @@ unsigned Environment::getID() const {
void Environment::requestAsync(const Resource& resource,
std::function<void(const Response&)> callback) {
fileSource.request(resource, *this, std::move(callback));
fileSource.request(resource, std::move(callback));
}
Request* Environment::request(const Resource& resource,
std::function<void(const Response&)> callback) {
assert(currentlyOn(ThreadType::Map));
return fileSource.request(resource, loop, *this, std::move(callback));
return fileSource.request(resource, loop, std::move(callback));
}
void Environment::cancelRequest(Request* req) {
@@ -178,10 +178,4 @@ void Environment::performCleanup() {
}
}
// #############################################################################################
void Environment::terminate() {
fileSource.abort(*this);
}
}

View File

@@ -1,21 +1,24 @@
#ifndef MBGL_STORAGE_DEFAULT_ASSET_REQUEST
#define MBGL_STORAGE_DEFAULT_ASSET_REQUEST
#include "shared_request_base.hpp"
#include "request_base.hpp"
typedef struct uv_loop_s uv_loop_t;
namespace mbgl {
class AssetRequest : public SharedRequestBase {
public:
AssetRequest(DefaultFileSource::Impl *source, const Resource &resource);
class AssetRequestImpl;
void start(uv_loop_t *loop, std::shared_ptr<const Response> response = nullptr);
void cancel();
class AssetRequest : public RequestBase {
public:
AssetRequest(const Resource&, Callback, uv_loop_t*, const std::string& assetRoot);
void cancel() override;
private:
~AssetRequest();
void *ptr = nullptr;
AssetRequestImpl* impl;
friend class AssetRequestImpl;
};

View File

@@ -10,7 +10,6 @@
#include <mbgl/util/chrono.hpp>
#include <mbgl/util/thread.hpp>
#include <mbgl/platform/log.hpp>
#include <mbgl/map/environment.hpp>
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wshadow"
@@ -28,10 +27,6 @@ namespace algo = boost::algorithm;
namespace mbgl {
DefaultFileSource::Impl::Impl(FileCache* cache_, const std::string& root)
: assetRoot(root.empty() ? platform::assetRoot() : root), cache(cache_) {
}
DefaultFileSource::DefaultFileSource(FileCache* cache, const std::string& root)
: thread(util::make_unique<util::Thread<Impl>>("FileSource", cache, root)) {
}
@@ -40,33 +35,20 @@ DefaultFileSource::~DefaultFileSource() {
MBGL_VERIFY_THREAD(tid);
}
SharedRequestBase *DefaultFileSource::Impl::find(const Resource &resource) {
// We're using a set of pointers here instead of a map between url and SharedRequestBase because
// we need to find the requests both by pointer and by URL. Given that the number of requests
// is generally very small (typically < 10 at a time), hashing by URL incurs too much overhead
// anyway.
const auto it = pending.find(resource);
if (it != pending.end()) {
return it->second;
}
return nullptr;
}
Request* DefaultFileSource::request(const Resource& resource,
uv_loop_t* l,
const Environment& env,
Callback callback) {
auto req = new Request(resource, l, env, std::move(callback));
auto req = new Request(resource, l, std::move(callback));
// This function can be called from any thread. Make sure we're executing the actual call in the
// file source loop by sending it over the queue.
thread->invoke(&Impl::add, std::move(req), thread->get());
thread->invoke(&Impl::add, req, thread->get());
return req;
}
void DefaultFileSource::request(const Resource& resource, const Environment& env, Callback callback) {
request(resource, nullptr, env, std::move(callback));
void DefaultFileSource::request(const Resource& resource, Callback callback) {
request(resource, nullptr, std::move(callback));
}
void DefaultFileSource::cancel(Request *req) {
@@ -74,51 +56,97 @@ void DefaultFileSource::cancel(Request *req) {
// This function can be called from any thread. Make sure we're executing the actual call in the
// file source loop by sending it over the queue.
thread->invoke(&Impl::cancel, std::move(req));
thread->invoke(&Impl::cancel, req);
}
void DefaultFileSource::abort(const Environment &env) {
thread->invoke(&Impl::abort, std::ref(env));
// ----- Impl -----
DefaultFileSource::Impl::Impl(FileCache* cache_, const std::string& root)
: cache(cache_), assetRoot(root.empty() ? platform::assetRoot() : root) {
}
DefaultFileRequest* DefaultFileSource::Impl::find(const Resource& resource) {
const auto it = pending.find(resource);
if (it != pending.end()) {
return &it->second;
}
return nullptr;
}
void DefaultFileSource::Impl::add(Request* req, uv_loop_t* loop) {
const Resource &resource = req->resource;
const Resource& resource = req->resource;
DefaultFileRequest* request = find(resource);
// We're adding a new Request.
SharedRequestBase *sharedRequest = find(resource);
if (!sharedRequest) {
// There is no request for this URL yet. Create a new one and start it.
if (algo::starts_with(resource.url, "asset://")) {
sharedRequest = new AssetRequest(this, resource);
} else {
sharedRequest = new HTTPRequest(this, resource);
}
const bool inserted = pending.emplace(resource, sharedRequest).second;
assert(inserted);
(void (inserted)); // silence unused variable warning on Release builds.
// But first, we're going to start querying the database if it exists.
if (!cache) {
sharedRequest->start(loop);
} else {
// Otherwise, first check the cache for existing data so that we can potentially
// revalidate the information without having to redownload everything.
cache->get(resource, [this, resource, loop](std::unique_ptr<Response> response) {
processResult(resource, std::move(response), loop);
});
}
if (request) {
request->observers.insert(req);
return;
}
request = &pending.emplace(resource, DefaultFileRequest(resource, loop)).first->second;
request->observers.insert(req);
if (cache) {
startCacheRequest(resource);
} else {
startRealRequest(resource);
}
}
void DefaultFileSource::Impl::startCacheRequest(const Resource& resource) {
// Check the cache for existing data so that we can potentially
// revalidate the information without having to redownload everything.
cache->get(resource, [this, resource](std::unique_ptr<Response> response) {
DefaultFileRequest* request = find(resource);
if (!request) {
// There is no request for this URL anymore. Likely, the request was canceled
// before we got around to process the cache result.
return;
}
auto expired = [&response] {
const int64_t now = std::chrono::duration_cast<std::chrono::seconds>(
SystemClock::now().time_since_epoch()).count();
return response->expires <= now;
};
if (!response || expired()) {
// No response or stale cache. Run the real request.
startRealRequest(resource, std::move(response));
} else {
// The response is fresh. We're good to notify the caller.
notify(request, std::move(response), FileCache::Hint::No);
}
});
}
void DefaultFileSource::Impl::startRealRequest(const Resource& resource, std::shared_ptr<const Response> response) {
DefaultFileRequest* request = find(resource);
auto callback = [request, this] (std::shared_ptr<const Response> res, FileCache::Hint hint) {
notify(request, res, hint);
};
if (algo::starts_with(resource.url, "asset://")) {
request->request = new AssetRequest(resource, callback, request->loop, assetRoot);
} else {
request->request = new HTTPRequest(resource, callback, request->loop, response);
}
sharedRequest->subscribe(req);
}
void DefaultFileSource::Impl::cancel(Request* req) {
SharedRequestBase *sharedRequest = find(req->resource);
if (sharedRequest) {
// If the number of dependent requests of the SharedRequestBase drops to zero, the
// unsubscribe callback triggers the removal of the SharedRequestBase pointer from the list
// of pending requests and initiates cancelation.
sharedRequest->unsubscribe(req);
DefaultFileRequest* request = find(req->resource);
if (request) {
// If the number of dependent requests of the DefaultFileRequest drops to zero,
// cancel the request and remove it from the pending list.
request->observers.erase(req);
if (request->observers.empty()) {
if (request->request) {
request->request->cancel();
}
pending.erase(request->resource);
}
} else {
// There is no request for this URL anymore. Likely, the request already completed
// before we got around to process the cancelation request.
@@ -129,78 +157,22 @@ void DefaultFileSource::Impl::cancel(Request* req) {
req->destruct();
}
void DefaultFileSource::Impl::processResult(const Resource& resource, std::shared_ptr<const Response> response, uv_loop_t* loop) {
SharedRequestBase *sharedRequest = find(resource);
if (sharedRequest) {
if (response) {
// This entry was stored in the cache. Now determine if we need to revalidate.
const int64_t now = std::chrono::duration_cast<std::chrono::seconds>(
SystemClock::now().time_since_epoch()).count();
if (response->expires > now) {
// The response is fresh. We're good to notify the caller.
sharedRequest->notify(response, FileCache::Hint::No);
sharedRequest->cancel();
return;
} else {
// The cached response is stale. Now run the real request.
sharedRequest->start(loop, response);
}
} else {
// There is no response. Now run the real request.
sharedRequest->start(loop);
}
} else {
// There is no request for this URL anymore. Likely, the request was canceled
// before we got around to process the cache result.
}
}
// Aborts all requests that are part of the current environment.
void DefaultFileSource::Impl::abort(const Environment& env) {
// Construct a cancellation response.
auto res = util::make_unique<Response>();
res->status = Response::Error;
res->message = "Environment is terminating";
std::shared_ptr<const Response> response = std::move(res);
// Iterate through all pending requests and remove them in case they're abandoned.
util::erase_if(pending, [&](const std::pair<Resource, SharedRequestBase *> &it) -> bool {
// Obtain all pending requests that are in the current environment.
const auto aborted = it.second->removeAllInEnvironment(env);
// Notify all observers.
for (auto req : aborted) {
req->notify(response);
}
// Finally, remove all requests that are now abandoned.
if (it.second->abandoned()) {
it.second->cancel();
return true;
} else {
return false;
}
});
}
void DefaultFileSource::Impl::notify(SharedRequestBase *sharedRequest,
const std::set<Request *> &observers,
std::shared_ptr<const Response> response, FileCache::Hint hint) {
void DefaultFileSource::Impl::notify(DefaultFileRequest* request, std::shared_ptr<const Response> response, FileCache::Hint hint) {
// First, remove the request, since it might be destructed at any point now.
assert(find(sharedRequest->resource) == sharedRequest);
pending.erase(sharedRequest->resource);
assert(find(request->resource) == request);
assert(response);
if (response) {
if (cache) {
// Store response in database
cache->put(sharedRequest->resource, response, hint);
}
// Notify all observers.
for (auto req : observers) {
req->notify(response);
}
// Notify all observers.
for (auto req : request->observers) {
req->notify(response);
}
if (cache) {
// Store response in database
cache->put(request->resource, response, hint);
}
pending.erase(request->resource);
}
}

View File

@@ -8,27 +8,35 @@
namespace mbgl {
class SharedRequestBase;
class RequestBase;
struct DefaultFileRequest {
const Resource resource;
uv_loop_t* loop;
std::set<Request*> observers;
RequestBase* request = nullptr;
DefaultFileRequest(const Resource& resource_, uv_loop_t* loop_)
: resource(resource_), loop(loop_) {}
};
class DefaultFileSource::Impl {
public:
Impl(FileCache *cache, const std::string &root = "");
Impl(FileCache*, const std::string& = "");
void notify(SharedRequestBase *sharedRequest, const std::set<Request *> &observers,
std::shared_ptr<const Response> response, FileCache::Hint hint);
SharedRequestBase *find(const Resource &resource);
void add(Request* request, uv_loop_t* loop);
void cancel(Request* request);
void abort(const Environment& env);
const std::string assetRoot;
void add(Request*, uv_loop_t*);
void cancel(Request*);
private:
void processResult(const Resource& resource, std::shared_ptr<const Response> response, uv_loop_t* loop);
DefaultFileRequest* find(const Resource&);
std::unordered_map<Resource, SharedRequestBase *, Resource::Hash> pending;
FileCache *cache = nullptr;
void startCacheRequest(const Resource&);
void startRealRequest(const Resource&, std::shared_ptr<const Response> = nullptr);
void notify(DefaultFileRequest*, std::shared_ptr<const Response>, FileCache::Hint);
std::unordered_map<Resource, DefaultFileRequest, Resource::Hash> pending;
FileCache* cache = nullptr;
const std::string assetRoot;
};
}

View File

@@ -1,23 +1,25 @@
#ifndef MBGL_STORAGE_DEFAULT_HTTP_REQUEST
#define MBGL_STORAGE_DEFAULT_HTTP_REQUEST
#include "shared_request_base.hpp"
#include "request_base.hpp"
typedef struct uv_loop_s uv_loop_t;
namespace mbgl {
class HTTPRequest : public SharedRequestBase {
class HTTPRequestImpl;
class HTTPRequest : public RequestBase {
public:
HTTPRequest(DefaultFileSource::Impl *source, const Resource &resource);
void start(uv_loop_t *loop, std::shared_ptr<const Response> response = nullptr);
void cancel();
HTTPRequest(const Resource&, Callback, uv_loop_t*, std::shared_ptr<const Response> = nullptr);
void cancel() override;
void retryImmediately();
private:
~HTTPRequest();
void *ptr = nullptr;
HTTPRequestImpl* impl;
friend class HTTPRequestImpl;
};

View File

@@ -16,8 +16,8 @@ namespace mbgl {
struct Request::Canceled { std::mutex mutex; bool confirmed = false; };
// Note: This requires that loop is running in the current thread (or not yet running).
Request::Request(const Resource &resource_, uv_loop_t *loop, const Environment &env_, Callback callback_)
: callback(callback_), resource(resource_), env(env_) {
Request::Request(const Resource &resource_, uv_loop_t *loop, Callback callback_)
: callback(callback_), resource(resource_) {
// When there is no loop supplied (== nullptr), the callback will be fired in an arbitrary
// thread (the thread notify() is called from) rather than kicking back to the calling thread.
if (loop) {

View File

@@ -0,0 +1,34 @@
#ifndef MBGL_STORAGE_REQUEST
#define MBGL_STORAGE_REQUEST
#include <mbgl/util/noncopyable.hpp>
#include <mbgl/storage/file_cache.hpp>
#include <memory>
#include <functional>
namespace mbgl {
struct Resource;
class Response;
class RequestBase : private util::noncopyable {
public:
using Callback = std::function<void (std::shared_ptr<const Response> response, FileCache::Hint hint)>;
RequestBase(const Resource& resource_, Callback notify_)
: resource(resource_)
, notify(notify_) {
}
virtual ~RequestBase() = default;
virtual void cancel() = 0;
protected:
const Resource& resource;
Callback notify;
};
}
#endif

View File

@@ -1,106 +0,0 @@
#ifndef MBGL_STORAGE_DEFAULT_SHARED_REQUEST_BASE
#define MBGL_STORAGE_DEFAULT_SHARED_REQUEST_BASE
#include <mbgl/storage/resource.hpp>
#include <mbgl/storage/file_cache.hpp>
#include <mbgl/storage/default_file_source_impl.hpp>
#include <mbgl/storage/request.hpp>
#include <mbgl/util/util.hpp>
#include <mbgl/util/noncopyable.hpp>
#include <string>
#include <set>
#include <vector>
#include <cassert>
typedef struct uv_loop_s uv_loop_t;
namespace mbgl {
class Request;
class Response;
class DefaultFileSource;
class SharedRequestBase : private util::noncopyable {
protected:
MBGL_STORE_THREAD(tid)
public:
SharedRequestBase(DefaultFileSource::Impl *source_, const Resource &resource_)
: resource(resource_), source(source_) {}
virtual void start(uv_loop_t *loop, std::shared_ptr<const Response> response = nullptr) = 0;
virtual void cancel() = 0;
void notify(std::shared_ptr<const Response> response, FileCache::Hint hint) {
MBGL_VERIFY_THREAD(tid);
if (source) {
source->notify(this, observers, response, hint);
}
}
void subscribe(Request *request) {
MBGL_VERIFY_THREAD(tid);
observers.insert(request);
}
void unsubscribe(Request *request) {
MBGL_VERIFY_THREAD(tid);
observers.erase(request);
if (abandoned()) {
// There are no observers anymore. We are initiating cancelation.
if (source) {
// First, remove this SharedRequestBase from the source.
source->notify(this, observers, nullptr, FileCache::Hint::No);
}
// Then, initiate cancelation of this request
cancel();
}
}
bool abandoned() const {
return observers.empty();
}
std::vector<Request *> removeAllInEnvironment(const Environment &env) {
MBGL_VERIFY_THREAD(tid);
std::vector<Request *> result;
// Removes all Requests in the supplied environment and returns a list
// of them.
util::erase_if(observers, [&](Request *req) -> bool {
if (&req->env == &env) {
result.push_back(req);
return true;
} else {
return false;
}
});
return result;
}
protected:
virtual ~SharedRequestBase() {
MBGL_VERIFY_THREAD(tid);
}
public:
const Resource resource;
protected:
DefaultFileSource::Impl *source = nullptr;
private:
std::set<Request *> observers;
};
}
#endif

BIN
test/fixtures/database/cache.db vendored Normal file

Binary file not shown.

BIN
test/fixtures/database/invalid.db vendored Normal file

Binary file not shown.

View File

@@ -14,9 +14,8 @@ TEST_F(Storage, CacheResponse) {
DefaultFileSource fs(&cache);
const Resource resource { Resource::Unknown, "http://127.0.0.1:3000/cache" };
auto &env = *static_cast<const Environment *>(nullptr);
fs.request(resource, uv_default_loop(), env, [&](const Response &res) {
fs.request(resource, uv_default_loop(), [&](const Response &res) {
EXPECT_EQ(Response::Successful, res.status);
EXPECT_EQ("Response 1", res.data);
EXPECT_LT(0, res.expires);
@@ -24,7 +23,7 @@ TEST_F(Storage, CacheResponse) {
EXPECT_EQ("", res.etag);
EXPECT_EQ("", res.message);
fs.request(resource, uv_default_loop(), env, [&, res](const Response &res2) {
fs.request(resource, uv_default_loop(), [&, res](const Response &res2) {
EXPECT_EQ(res.status, res2.status);
EXPECT_EQ(res.data, res2.data);
EXPECT_EQ(res.expires, res2.expires);

View File

@@ -15,10 +15,8 @@ TEST_F(Storage, CacheRevalidate) {
SQLiteCache cache(":memory:");
DefaultFileSource fs(&cache);
auto &env = *static_cast<const Environment *>(nullptr);
const Resource revalidateSame { Resource::Unknown, "http://127.0.0.1:3000/revalidate-same" };
fs.request(revalidateSame, uv_default_loop(), env, [&](const Response &res) {
fs.request(revalidateSame, uv_default_loop(), [&](const Response &res) {
EXPECT_EQ(Response::Successful, res.status);
EXPECT_EQ("Response", res.data);
EXPECT_EQ(0, res.expires);
@@ -26,7 +24,7 @@ TEST_F(Storage, CacheRevalidate) {
EXPECT_EQ("snowfall", res.etag);
EXPECT_EQ("", res.message);
fs.request(revalidateSame, uv_default_loop(), env, [&, res](const Response &res2) {
fs.request(revalidateSame, uv_default_loop(), [&, res](const Response &res2) {
EXPECT_EQ(Response::Successful, res2.status);
EXPECT_EQ("Response", res2.data);
// We use this to indicate that a 304 reply came back.
@@ -42,7 +40,7 @@ TEST_F(Storage, CacheRevalidate) {
const Resource revalidateModified{ Resource::Unknown,
"http://127.0.0.1:3000/revalidate-modified" };
fs.request(revalidateModified, uv_default_loop(), env, [&](const Response &res) {
fs.request(revalidateModified, uv_default_loop(), [&](const Response &res) {
EXPECT_EQ(Response::Successful, res.status);
EXPECT_EQ("Response", res.data);
EXPECT_EQ(0, res.expires);
@@ -50,7 +48,7 @@ TEST_F(Storage, CacheRevalidate) {
EXPECT_EQ("", res.etag);
EXPECT_EQ("", res.message);
fs.request(revalidateModified, uv_default_loop(), env, [&, res](const Response &res2) {
fs.request(revalidateModified, uv_default_loop(), [&, res](const Response &res2) {
EXPECT_EQ(Response::Successful, res2.status);
EXPECT_EQ("Response", res2.data);
// We use this to indicate that a 304 reply came back.
@@ -64,7 +62,7 @@ TEST_F(Storage, CacheRevalidate) {
});
const Resource revalidateEtag { Resource::Unknown, "http://127.0.0.1:3000/revalidate-etag" };
fs.request(revalidateEtag, uv_default_loop(), env, [&](const Response &res) {
fs.request(revalidateEtag, uv_default_loop(), [&](const Response &res) {
EXPECT_EQ(Response::Successful, res.status);
EXPECT_EQ("Response 1", res.data);
EXPECT_EQ(0, res.expires);
@@ -72,7 +70,7 @@ TEST_F(Storage, CacheRevalidate) {
EXPECT_EQ("response-1", res.etag);
EXPECT_EQ("", res.message);
fs.request(revalidateEtag, uv_default_loop(), env, [&, res](const Response &res2) {
fs.request(revalidateEtag, uv_default_loop(), [&, res](const Response &res2) {
EXPECT_EQ(Response::Successful, res2.status);
EXPECT_EQ("Response 2", res2.data);
EXPECT_EQ(0, res2.expires);

View File

@@ -15,10 +15,8 @@ TEST_F(Storage, AssetReadDirectory) {
DefaultFileSource fs(nullptr);
#endif
auto &env = *static_cast<const Environment *>(nullptr);
fs.request({ Resource::Unknown, "asset://TEST_DATA/fixtures/storage" }, uv_default_loop(),
env, [&](const Response &res) {
[&](const Response &res) {
EXPECT_EQ(Response::Error, res.status);
EXPECT_EQ(0ul, res.data.size());
EXPECT_EQ(0, res.expires);
@@ -27,7 +25,7 @@ TEST_F(Storage, AssetReadDirectory) {
#ifdef MBGL_ASSET_ZIP
EXPECT_EQ("No such file", res.message);
#elif MBGL_ASSET_FS
EXPECT_EQ("illegal operation on a directory", res.message);
EXPECT_EQ("illegal operation on a directory", res.message);
#endif
ReadDirectory.finish();
});

View File

@@ -16,10 +16,8 @@ TEST_F(Storage, AssetEmptyFile) {
DefaultFileSource fs(nullptr);
#endif
auto &env = *static_cast<const Environment *>(nullptr);
fs.request({ Resource::Unknown, "asset://TEST_DATA/fixtures/storage/empty" }, uv_default_loop(),
env, [&](const Response &res) {
[&](const Response &res) {
EXPECT_EQ(Response::Successful, res.status);
EXPECT_EQ(0ul, res.data.size());
EXPECT_EQ(0, res.expires);
@@ -43,10 +41,8 @@ TEST_F(Storage, AssetNonEmptyFile) {
DefaultFileSource fs(nullptr);
#endif
auto &env = *static_cast<const Environment *>(nullptr);
fs.request({ Resource::Unknown, "asset://TEST_DATA/fixtures/storage/nonempty" },
uv_default_loop(), env, [&](const Response &res) {
uv_default_loop(), [&](const Response &res) {
EXPECT_EQ(Response::Successful, res.status);
EXPECT_EQ(16ul, res.data.size());
EXPECT_EQ(0, res.expires);
@@ -71,10 +67,8 @@ TEST_F(Storage, AssetNonExistentFile) {
DefaultFileSource fs(nullptr);
#endif
auto &env = *static_cast<const Environment *>(nullptr);
fs.request({ Resource::Unknown, "asset://TEST_DATA/fixtures/storage/does_not_exist" },
uv_default_loop(), env, [&](const Response &res) {
uv_default_loop(), [&](const Response &res) {
EXPECT_EQ(Response::Error, res.status);
EXPECT_EQ(0ul, res.data.size());
EXPECT_EQ(0, res.expires);

View File

@@ -14,10 +14,8 @@ TEST_F(Storage, HTTPCancel) {
DefaultFileSource fs(nullptr);
auto &env = *static_cast<const Environment *>(nullptr);
auto req =
fs.request({ Resource::Unknown, "http://127.0.0.1:3000/test" }, uv_default_loop(), env,
fs.request({ Resource::Unknown, "http://127.0.0.1:3000/test" }, uv_default_loop(),
[&](const Response &) { ADD_FAILURE() << "Callback should not be called"; });
fs.cancel(req);
@@ -33,13 +31,12 @@ TEST_F(Storage, HTTPCancelMultiple) {
DefaultFileSource fs(nullptr);
auto &env = *static_cast<const Environment *>(nullptr);
const Resource resource { Resource::Unknown, "http://127.0.0.1:3000/test" };
auto req2 = fs.request(resource, uv_default_loop(), env, [&](const Response &) {
auto req2 = fs.request(resource, uv_default_loop(), [&](const Response &) {
ADD_FAILURE() << "Callback should not be called";
});
fs.request(resource, uv_default_loop(), env, [&](const Response &res) {
fs.request(resource, uv_default_loop(), [&](const Response &res) {
EXPECT_EQ(Response::Successful, res.status);
EXPECT_EQ("Hello World!", res.data);
EXPECT_EQ(0, res.expires);

View File

@@ -14,8 +14,6 @@ TEST_F(Storage, HTTPCoalescing) {
DefaultFileSource fs(nullptr);
auto &env = *static_cast<const Environment *>(nullptr);
static const Response *reference = nullptr;
const auto complete = [&](const Response &res) {
@@ -43,7 +41,7 @@ TEST_F(Storage, HTTPCoalescing) {
const Resource resource { Resource::Unknown, "http://127.0.0.1:3000/test" };
for (int i = 0; i < total; i++) {
fs.request(resource, uv_default_loop(), env, complete);
fs.request(resource, uv_default_loop(), complete);
}
uv_run(uv_default_loop(), UV_RUN_DEFAULT);

View File

@@ -1,54 +0,0 @@
#include "storage.hpp"
#include <uv.h>
#include <mbgl/storage/default_file_source.hpp>
#include <mbgl/storage/network_status.hpp>
#include <cmath>
TEST_F(Storage, HTTPCancelEnvironment) {
SCOPED_TEST(HTTPRetainedEnvironment)
SCOPED_TEST(HTTPCanceledEnvironment)
using namespace mbgl;
DefaultFileSource fs(nullptr);
// Create two fake environment pointers. The FileSource implementation treats these as opaque
// pointers and doesn't reach into them.
auto &env1 = *reinterpret_cast<const Environment *>(1);
auto &env2 = *reinterpret_cast<const Environment *>(2);
const Resource resource { Resource::Unknown, "http://127.0.0.1:3000/delayed" };
// Environment 1
fs.request(resource, uv_default_loop(), env1, [&](const Response &res) {
// This environment gets aborted below. This means the request is marked as failing and
// will return an error here.
EXPECT_EQ(Response::Error, res.status);
EXPECT_EQ("", res.data);
EXPECT_EQ(0, res.expires);
EXPECT_EQ(0, res.modified);
EXPECT_EQ("", res.etag);
EXPECT_EQ("Environment is terminating", res.message);
HTTPCanceledEnvironment.finish();
});
// Environment 2
fs.request(resource, uv_default_loop(), env2, [&](const Response &res) {
// The same request as above, but in a different environment which doesn't get aborted. This
// means the request should succeed.
EXPECT_EQ(Response::Successful, res.status);
EXPECT_EQ("Response", res.data);
EXPECT_EQ(0, res.expires);
EXPECT_EQ(0, res.modified);
EXPECT_EQ("", res.etag);
EXPECT_EQ("", res.message);
HTTPRetainedEnvironment.finish();
});
fs.abort(env1);
uv_run(uv_default_loop(), UV_RUN_DEFAULT);
}

View File

@@ -22,12 +22,10 @@ TEST_F(Storage, HTTPError) {
DefaultFileSource fs(nullptr);
auto &env = *static_cast<const Environment *>(nullptr);
auto start = uv_hrtime();
fs.request({ Resource::Unknown, "http://127.0.0.1:3000/temporary-error" }, uv_default_loop(),
env, [&](const Response &res) {
[&](const Response &res) {
const auto duration = double(uv_hrtime() - start) / 1e9;
EXPECT_LT(1, duration) << "Backoff timer didn't wait 1 second";
EXPECT_GT(1.2, duration) << "Backoff timer fired too late";
@@ -41,7 +39,7 @@ TEST_F(Storage, HTTPError) {
HTTPTemporaryError.finish();
});
fs.request({ Resource::Unknown, "http://127.0.0.1:3001/" }, uv_default_loop(), env,
fs.request({ Resource::Unknown, "http://127.0.0.1:3001/" }, uv_default_loop(),
[&](const Response &res) {
const auto duration = double(uv_hrtime() - start) / 1e9;
// 1.5 seconds == 4 retries, with a 500ms timeout (see above).

View File

@@ -15,11 +15,9 @@ TEST_F(Storage, HTTPHeaderParsing) {
DefaultFileSource fs(nullptr);
auto &env = *static_cast<const Environment *>(nullptr);
fs.request({ Resource::Unknown,
"http://127.0.0.1:3000/test?modified=1420794326&expires=1420797926&etag=foo" },
uv_default_loop(), env, [&](const Response &res) {
uv_default_loop(), [&](const Response &res) {
EXPECT_EQ(Response::Successful, res.status);
EXPECT_EQ("Hello World!", res.data);
EXPECT_EQ(1420797926, res.expires);
@@ -33,7 +31,7 @@ TEST_F(Storage, HTTPHeaderParsing) {
SystemClock::now().time_since_epoch()).count();
fs.request({ Resource::Unknown, "http://127.0.0.1:3000/test?cachecontrol=max-age=120" },
uv_default_loop(), env, [&](const Response &res) {
uv_default_loop(), [&](const Response &res) {
EXPECT_EQ(Response::Successful, res.status);
EXPECT_EQ("Hello World!", res.data);
EXPECT_GT(2, std::abs(res.expires - now - 120)) << "Expiration date isn't about 120 seconds in the future";

View File

@@ -11,8 +11,6 @@ TEST_F(Storage, HTTPLoad) {
DefaultFileSource fs(nullptr);
auto &env = *static_cast<const Environment *>(nullptr);
const int concurrency = 50;
const int max = 10000;
int number = 1;
@@ -21,7 +19,7 @@ TEST_F(Storage, HTTPLoad) {
const auto current = number++;
fs.request({ Resource::Unknown,
std::string("http://127.0.0.1:3000/load/") + std::to_string(current) },
uv_default_loop(), env, [&, current](const Response &res) {
uv_default_loop(), [&, current](const Response &res) {
EXPECT_EQ(Response::Successful, res.status);
EXPECT_EQ(std::string("Request ") + std::to_string(current), res.data);
EXPECT_EQ(0, res.expires);

View File

@@ -12,8 +12,6 @@ TEST_F(Storage, HTTPNoLoop) {
DefaultFileSource fs(nullptr);
auto &env = *static_cast<const Environment *>(nullptr);
const auto mainThread = uv_thread_self();
// Async handle that keeps the main loop alive until the thread finished
@@ -22,7 +20,7 @@ TEST_F(Storage, HTTPNoLoop) {
uv::close(as);
});
fs.request({ Resource::Unknown, "http://127.0.0.1:3000/temporary-error" }, nullptr, env,
fs.request({ Resource::Unknown, "http://127.0.0.1:3000/temporary-error" }, nullptr,
[&](const Response &res) {
EXPECT_NE(uv_thread_self(), mainThread) << "Response was called in the same thread";
EXPECT_EQ(Response::Successful, res.status);

View File

@@ -12,9 +12,7 @@ TEST_F(Storage, HTTPOtherLoop) {
// This file source launches a separate thread to do the processing.
DefaultFileSource fs(nullptr);
auto &env = *static_cast<const Environment *>(nullptr);
fs.request({ Resource::Unknown, "http://127.0.0.1:3000/test" }, uv_default_loop(), env,
fs.request({ Resource::Unknown, "http://127.0.0.1:3000/test" }, uv_default_loop(),
[&](const Response &res) {
EXPECT_EQ(Response::Successful, res.status);
EXPECT_EQ("Hello World!", res.data);

View File

@@ -14,11 +14,9 @@ TEST_F(Storage, HTTPReading) {
DefaultFileSource fs(nullptr);
auto &env = *static_cast<const Environment *>(nullptr);
const auto mainThread = uv_thread_self();
fs.request({ Resource::Unknown, "http://127.0.0.1:3000/test" }, uv_default_loop(), env,
fs.request({ Resource::Unknown, "http://127.0.0.1:3000/test" }, uv_default_loop(),
[&](const Response &res) {
EXPECT_EQ(uv_thread_self(), mainThread);
EXPECT_EQ(Response::Successful, res.status);
@@ -31,7 +29,7 @@ TEST_F(Storage, HTTPReading) {
});
fs.request({ Resource::Unknown, "http://127.0.0.1:3000/doesnotexist" }, uv_default_loop(),
env, [&](const Response &res) {
[&](const Response &res) {
EXPECT_EQ(uv_thread_self(), mainThread);
EXPECT_EQ(Response::Error, res.status);
EXPECT_EQ("HTTP status code 404", res.message);
@@ -51,9 +49,7 @@ TEST_F(Storage, HTTPNoCallback) {
DefaultFileSource fs(nullptr);
auto &env = *static_cast<const Environment *>(nullptr);
fs.request({ Resource::Unknown, "http://127.0.0.1:3000/test" }, uv_default_loop(), env,
fs.request({ Resource::Unknown, "http://127.0.0.1:3000/test" }, uv_default_loop(),
nullptr);
uv_run(uv_default_loop(), UV_RUN_DEFAULT);
@@ -68,10 +64,8 @@ TEST_F(Storage, HTTPCallbackNotOnLoop) {
SCOPED_TEST(HTTPTest)
auto &env = *static_cast<const Environment *>(nullptr);
std::promise<void> promise;
fs.request({ Resource::Unknown, "http://127.0.0.1:3000/test" }, env, [&] (const Response &) {
fs.request({ Resource::Unknown, "http://127.0.0.1:3000/test" }, [&] (const Response &) {
promise.set_value();
});

View File

@@ -62,7 +62,6 @@
'storage/file_reading.cpp',
'storage/http_cancel.cpp',
'storage/http_coalescing.cpp',
'storage/http_environment.cpp',
'storage/http_error.cpp',
'storage/http_header_parsing.cpp',
'storage/http_load.cpp',