From 6c58101b12ee8e599aff3541f10c1fbfe5a027fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Sat, 1 Mar 2014 13:05:13 +0100 Subject: [PATCH] multithreaded curl for linux --- Makefile | 6 ++ include/llmr/map/tile.hpp | 2 +- include/llmr/platform/platform.hpp | 9 +-- linux/llmr-app.gyp | 56 ++++++++++++++--- linux/main.cpp | 98 ++++++++++++++++++++---------- macosx/main.mm | 17 ++++-- src/map/tile.cpp | 2 +- 7 files changed, 135 insertions(+), 55 deletions(-) diff --git a/Makefile b/Makefile index 854b5fb81..c6f4eb0b3 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,12 @@ linux: config.gypi src linux/llmr-app.gyp make -C build/linux-make V=$(V) ./build/linux-make/out/Release/llmr.app + +# build OS X app with Xcode +lproj: config.gypi src linux/llmr-app.gyp + deps/run_gyp linux/llmr-app.gyp -Goutput_dir=./out/ --depth=. --generator-output=./ -f xcode + open ./linux/llmr-app.xcodeproj + # build just xcode project for libllmr xcode: config.gypi llmr.gyp deps/run_gyp llmr.gyp -Goutput_dir=./out/ --depth=. --generator-output=./ -f xcode diff --git a/include/llmr/map/tile.hpp b/include/llmr/map/tile.hpp index d738c99bb..aba9e8a28 100644 --- a/include/llmr/map/tile.hpp +++ b/include/llmr/map/tile.hpp @@ -95,7 +95,7 @@ private: // Source data std::string data; const Style& style; - platform::Request req; + platform::Request *req = nullptr; }; } diff --git a/include/llmr/platform/platform.hpp b/include/llmr/platform/platform.hpp index ef137c206..fba922d1e 100644 --- a/include/llmr/platform/platform.hpp +++ b/include/llmr/platform/platform.hpp @@ -15,10 +15,7 @@ namespace platform { // Restarts painting. This could for example trigger the event loop of the controlling application. void restart(); -struct Request { - int16_t identifier = -1; - std::string original_url; -}; +struct Request; struct Response { int16_t code = -1; @@ -26,10 +23,10 @@ struct Response { }; // Makes an HTTP request of a URL on a background thread, calls a function with the results on the same thread, and finally calls a callback function on the main thread. Returns a cancellable request. -Request request_http(std::string url, std::function background_function, std::function foreground_callback); +Request *request_http(std::string url, std::function background_function, std::function foreground_callback); // Cancels an HTTP request. -void cancel_request_http(Request request); +void cancel_request_http(Request *request); // Returns a relative timestamp in seconds. This value must be monotonic. double time(); diff --git a/linux/llmr-app.gyp b/linux/llmr-app.gyp index 06d7ed822..79c1d4560 100644 --- a/linux/llmr-app.gyp +++ b/linux/llmr-app.gyp @@ -13,16 +13,52 @@ "./settings.cpp", "./settings.hpp" ], - 'product_extension': 'app', - 'link_settings': { - 'libraries': [ - '<@(glfw3_libraries)', - '-lcurl' - ], - }, - 'cflags': [ - '<@(png_cflags)', - '<@(glfw3_cflags)' + 'conditions': [ + ['OS == "mac"', { + 'product_extension': 'app', + 'mac_bundle_resources': [ + 'Icon.icns', + ], + 'link_settings': { + 'libraries': [ + '$(SDKROOT)/System/Library/Frameworks/Cocoa.framework', + '$(SDKROOT)/System/Library/Frameworks/IOKit.framework', + '$(SDKROOT)/System/Library/Frameworks/OpenGL.framework', + '$(SDKROOT)/System/Library/Frameworks/CoreVideo.framework', + ], + }, + 'xcode_settings': { + 'ARCHS': [ "x86_64" ], + 'SDKROOT': 'macosx', + 'SUPPORTED_PLATFORMS':'macosx', + 'OTHER_CPLUSPLUSFLAGS':[ + '<@(glfw3_cflags)' + ], + 'OTHER_LDFLAGS': [ + '-stdlib=libc++', + '<@(glfw3_libraries)', + '-lcurl', + ], + 'SDKROOT': 'macosx', + 'INFOPLIST_FILE': 'Info.plist', + 'CLANG_CXX_LIBRARY': 'libc++', + 'CLANG_CXX_LANGUAGE_STANDARD':'c++11', + 'MACOSX_DEPLOYMENT_TARGET':'10.9', + 'GCC_VERSION': 'com.apple.compilers.llvm.clang.1_0', + 'CLANG_ENABLE_OBJC_ARC': 'YES' + }, + }, { + 'link_settings': { + 'libraries': [ + '<@(glfw3_libraries)', + '-lcurl' + ], + }, + 'cflags': [ + '<@(png_cflags)', + '<@(glfw3_cflags)' + ], + }] ], "dependencies": [ "../llmr.gyp:llmr-x86" diff --git a/linux/main.cpp b/linux/main.cpp index 1e6c409a0..328027414 100644 --- a/linux/main.cpp +++ b/linux/main.cpp @@ -5,15 +5,28 @@ #include "settings.hpp" #include #include +#include #include -#include +#include + +namespace llmr { +namespace platform { + +struct Request { + pthread_t thread; + pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; + std::string url; + bool done = false; + std::function background_function; + std::function foreground_callback; +}; + +} +} + +std::forward_list requests; -std::list> futures; -std::list>::iterator f_it; -std::list> callbacks; -std::list>::iterator c_it; -std::chrono::milliseconds zero (0); class MapView { public: @@ -159,21 +172,22 @@ public: int run() { while (!glfwWindowShouldClose(window)) { - - f_it = futures.begin(); - c_it = callbacks.begin(); - - while (f_it != futures.end()) { - if (f_it->wait_for(zero) == std::future_status::ready) { - std::function cb = *c_it; - futures.erase(f_it++); - callbacks.erase(c_it++); - cb(); + bool& dirty = this->dirty; + requests.remove_if([&dirty](llmr::platform::Request *req) { + pthread_mutex_lock(&req->mutex); + if (req->done) { + pthread_mutex_unlock(&req->mutex); + req->foreground_callback(); + pthread_join(req->thread, nullptr); + pthread_mutex_destroy(&req->mutex); + delete req; + dirty = true; + return true; } else { - f_it++; - c_it++; + pthread_mutex_unlock(&req->mutex); + return false; } - } + }); if (dirty) { try { @@ -185,7 +199,7 @@ public: fps(); } - if (dirty || futures.size()) { + if (!requests.empty() || dirty) { glfwPollEvents(); } else { glfwWaitEvents(); @@ -237,29 +251,29 @@ MapView *mapView = nullptr; namespace llmr { namespace platform { -void restart(void *) { +void restart() { if (mapView) { mapView->dirty = true; } } - - static size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp) { - ((std::string*)userp)->append((char*)contents, size *nmemb); + ((std::string *)userp)->append((char *)contents, size * nmemb); return size * nmemb; } -void request_http_async(std::string url, std::function func) { - +void *request_http(void *ptr) { + Request *req = static_cast(ptr); Response res; + // TODO: use curl multi to be able to cancel + CURL *curl; CURLcode code; curl = curl_easy_init(); if (curl) { - curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_URL, req->url.c_str()); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &res.body); curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, "deflate"); @@ -268,14 +282,33 @@ void request_http_async(std::string url, std::function func) { curl_easy_cleanup(curl); } - func(res); + req->background_function(res); + + pthread_mutex_lock(&req->mutex); + req->done = true; + pthread_mutex_unlock(&req->mutex); + + pthread_exit(nullptr); } -void request_http(std::string url, std::function func, std::function cb) { - futures.emplace_back(std::async(std::launch::async, request_http_async, url, func)); - callbacks.push_back(cb); +Request *request_http(std::string url, std::function background_function, std::function foreground_callback) { + Request *req = new Request(); + req->url = url; + req->background_function = background_function; + req->foreground_callback = foreground_callback; + requests.push_front(req); + int rc = pthread_create(&req->thread, nullptr, request_http, (void *)req); + if (rc) { + fprintf(stderr, "http request failed\n"); + return nullptr; + } + + return req; } +void cancel_request_http(Request *request) { + fprintf(stderr, "cancel %p\n", request); +} double time() { return glfwGetTime(); @@ -285,10 +318,13 @@ double time() { } int main() { + curl_global_init(CURL_GLOBAL_ALL); + mapView = new MapView(); mapView->init(); int ret = mapView->run(); mapView->settings.sync(); delete mapView; + pthread_exit(NULL); return ret; } diff --git a/macosx/main.mm b/macosx/main.mm index 973db9a86..bed18013c 100644 --- a/macosx/main.mm +++ b/macosx/main.mm @@ -240,7 +240,12 @@ void restart() { [[NSApplication sharedApplication] postEvent: [NSEvent eventWithCGEvent:event] atStart:NO]; } -Request request_http(std::string url, std::function background_function, std::function foreground_callback) +struct Request { + int16_t identifier; + std::string original_url; +}; + +Request *request_http(std::string url, std::function background_function, std::function foreground_callback) { NSURLSessionDataTask *task = [[NSURLSession sharedSession] dataTaskWithURL:[NSURL URLWithString:@(url.c_str())] completionHandler:^(NSData *data, NSURLResponse *response, NSError *error) { @@ -262,20 +267,20 @@ Request request_http(std::string url, std::function background_ [task resume]; - Request req; + Request *req = new Request(); - req.identifier = task.taskIdentifier; - req.original_url = url; + req->identifier = task.taskIdentifier; + req->original_url = url; return req; } -void cancel_request_http(Request request) +void cancel_request_http(Request *request) { [[NSURLSession sharedSession] getTasksWithCompletionHandler:^(NSArray *dataTasks, NSArray *uploadTasks, NSArray *downloadTasks) { for (NSURLSessionDownloadTask *task in downloadTasks) - if (task.taskIdentifier == request.identifier) + if (task.taskIdentifier == request->identifier) return [task cancel]; }]; } diff --git a/src/map/tile.cpp b/src/map/tile.cpp index 6d88050a7..ff0824621 100644 --- a/src/map/tile.cpp +++ b/src/map/tile.cpp @@ -74,7 +74,7 @@ void Tile::request() { // Note: Somehow this feels slower than the change to request_http() std::shared_ptr tile = shared_from_this(); - platform::Request request = platform::request_http(url, [=](platform::Response& res) { + platform::Request *request = platform::request_http(url, [=](platform::Response& res) { if (res.code == 200 && tile->state != obsolete) { tile->state = Tile::loaded; tile->data.swap(res.body);