MapLibreParticleLab/platform/default/http_file_source.cpp

#include <mbgl/storage/http_file_source.hpp>
#include <mbgl/storage/resource.hpp>
#include <mbgl/storage/response.hpp>
#include <mbgl/util/logging.hpp>

#include <mbgl/util/util.hpp>
#include <mbgl/util/optional.hpp>
#include <mbgl/util/run_loop.hpp>
#include <mbgl/util/string.hpp>
#include <mbgl/util/timer.hpp>
#include <mbgl/util/chrono.hpp>
#include <mbgl/util/http_header.hpp>

#include <curl/curl.h>

// Dynamically load all cURL functions. Debian-derived systems upgraded the OpenSSL version linked
// to in https://salsa.debian.org/debian/curl/commit/95c94957bb7e89e36e78b995fed468c42f64d18d
// They state:
//     Rename libcurl3 to libcurl4, because libcurl exposes an SSL_CTX via
//     CURLOPT_SSL_CTX_FUNCTION, and this object changes incompatibly between
//     openssl 1.0 and openssl 1.1.
// Since we are not accessing the underlying OpenSSL context, we don't care whether we're linking
// against libcurl3 or libcurl4; both use the ABI version 4 which hasn't changed since 2006
// (see https://curl.haxx.se/libcurl/abi.html). In fact, cURL's ABI compatibility is very good as
// shown on https://abi-laboratory.pro/tracker/timeline/curl/
// Therefore, we're dynamically loading the cURL symbols we need to avoid linking against versioned
// symbols.
#include <dlfcn.h>

namespace curl {

#define CURL_FUNCTIONS \
    X(global_init) \
    X(getdate) \
    X(easy_strerror) \
    X(easy_init) \
    X(easy_setopt) \
    X(easy_cleanup) \
    X(easy_getinfo) \
    X(easy_reset) \
    X(multi_init) \
    X(multi_add_handle) \
    X(multi_remove_handle) \
    X(multi_cleanup) \
    X(multi_info_read) \
    X(multi_strerror) \
    X(multi_socket_action) \
    X(multi_setopt) \
    X(share_init) \
    X(share_cleanup) \
    X(slist_append) \
    X(slist_free_all)

#define X(name) static decltype(&curl_ ## name) name = nullptr;
CURL_FUNCTIONS
#undef X

static void* handle = nullptr;

static void* load(const char* name) {
    void* symbol = dlsym(handle, name);
    if (const char* error = dlerror()) {
        fprintf(stderr, "Cannot load symbol '%s': %s\n", name, error);
        dlclose(handle);
        handle = nullptr;
        abort();
    }
    return symbol;
}

__attribute__((constructor))
static void load() {
    assert(!handle);
    handle = dlopen("libcurl.so.4", RTLD_LAZY | RTLD_LOCAL);
    if (!handle) {
        fprintf(stderr, "Could not open shared library '%s'\n", "libcurl.so.4");
        abort();
    }

    #define X(name) name = (decltype(&curl_ ## name))load("curl_" #name);
    CURL_FUNCTIONS
    #undef X
}

__attribute__((constructor))
static void unload() {
    if (handle) {
        dlclose(handle);
    }
}

} // namespace curl


#include <queue>
#include <map>
#include <cassert>
#include <cstring>
#include <cstdio>

static void handleError(CURLMcode code) {
    if (code != CURLM_OK) {
        throw std::runtime_error(std::string("CURL multi error: ") + curl::multi_strerror(code));
    }
}

static void handleError(CURLcode code) {
    if (code != CURLE_OK) {
        throw std::runtime_error(std::string("CURL easy error: ") + curl::easy_strerror(code));
    }
}

namespace mbgl {

class HTTPFileSource::Impl {
public:
    Impl();
    ~Impl();

    static int handleSocket(CURL *handle, curl_socket_t s, int action, void *userp, void *socketp);
    static int startTimeout(CURLM *multi, long timeout_ms, void *userp);
    static void onTimeout(HTTPFileSource::Impl *context);

    void perform(curl_socket_t s, util::RunLoop::Event event);
    CURL *getHandle();
    void returnHandle(CURL *handle);
    void checkMultiInfo();

    // Used as the CURL timer function to periodically check for socket updates.
    util::Timer timeout;

    // CURL multi handle that we use to request multiple URLs at the same time, without having to
    // block and spawn threads.
    CURLM *multi = nullptr;

    // CURL share handles are used for sharing session state (e.g.)
    CURLSH *share = nullptr;

    // A queue that we use for storing resuable CURL easy handles to avoid creating and destroying
    // them all the time.
    std::queue<CURL *> handles;
};

class HTTPRequest : public AsyncRequest {
public:
    HTTPRequest(HTTPFileSource::Impl*, Resource, FileSource::Callback);
    ~HTTPRequest() override;

    void handleResult(CURLcode code);

private:
    static size_t headerCallback(char *const buffer, const size_t size, const size_t nmemb, void *userp);
    static size_t writeCallback(void *const contents, const size_t size, const size_t nmemb, void *userp);

    HTTPFileSource::Impl* context = nullptr;
    Resource resource;
    FileSource::Callback callback;

    // Will store the current response.
    std::shared_ptr<std::string> data;
    std::unique_ptr<Response> response;

    optional<std::string> retryAfter;
    optional<std::string> xRateLimitReset;

    CURL *handle = nullptr;
    curl_slist *headers = nullptr;

    char error[CURL_ERROR_SIZE] = { 0 };
};

HTTPFileSource::Impl::Impl() {
    if (curl::global_init(CURL_GLOBAL_ALL)) {
        throw std::runtime_error("Could not init cURL");
    }

    share = curl::share_init();

    multi = curl::multi_init();
    handleError(curl::multi_setopt(multi, CURLMOPT_SOCKETFUNCTION, handleSocket));
    handleError(curl::multi_setopt(multi, CURLMOPT_SOCKETDATA, this));
    handleError(curl::multi_setopt(multi, CURLMOPT_TIMERFUNCTION, startTimeout));
    handleError(curl::multi_setopt(multi, CURLMOPT_TIMERDATA, this));
}

HTTPFileSource::Impl::~Impl() {
    while (!handles.empty()) {
        curl::easy_cleanup(handles.front());
        handles.pop();
    }

    curl::multi_cleanup(multi);
    multi = nullptr;

    curl::share_cleanup(share);
    share = nullptr;

    timeout.stop();
}

CURL *HTTPFileSource::Impl::getHandle() {
    if (!handles.empty()) {
        auto handle = handles.front();
        handles.pop();
        return handle;
    } else {
        return curl::easy_init();
    }
}

void HTTPFileSource::Impl::returnHandle(CURL *handle) {
    curl::easy_reset(handle);
    handles.push(handle);
}

void HTTPFileSource::Impl::checkMultiInfo() {
    CURLMsg *message = nullptr;
    int pending = 0;

    while ((message = curl::multi_info_read(multi, &pending))) {
        switch (message->msg) {
        case CURLMSG_DONE: {
            HTTPRequest *baton = nullptr;
            curl::easy_getinfo(message->easy_handle, CURLINFO_PRIVATE, (char *)&baton);
            assert(baton);
            baton->handleResult(message->data.result);
        } break;

        default:
            // This should never happen, because there are no other message types.
            throw std::runtime_error("CURLMsg returned unknown message type");
        }
    }
}

void HTTPFileSource::Impl::perform(curl_socket_t s, util::RunLoop::Event events) {
    int flags = 0;

    if (events == util::RunLoop::Event::Read) {
        flags |= CURL_CSELECT_IN;
    }
    if (events == util::RunLoop::Event::Write) {
        flags |= CURL_CSELECT_OUT;
    }


    int running_handles = 0;
    curl::multi_socket_action(multi, s, flags, &running_handles);
    checkMultiInfo();
}

int HTTPFileSource::Impl::handleSocket(CURL * /* handle */, curl_socket_t s, int action, void *userp,
                              void * /* socketp */) {
    assert(userp);
    auto context = reinterpret_cast<Impl *>(userp);

    switch (action) {
    case CURL_POLL_IN: {
        using namespace std::placeholders;
        util::RunLoop::Get()->addWatch(s, util::RunLoop::Event::Read,
                std::bind(&Impl::perform, context, _1, _2));
        break;
    }
    case CURL_POLL_OUT: {
        using namespace std::placeholders;
        util::RunLoop::Get()->addWatch(s, util::RunLoop::Event::Write,
                std::bind(&Impl::perform, context, _1, _2));
        break;
    }
    case CURL_POLL_REMOVE:
        util::RunLoop::Get()->removeWatch(s);
        break;
    default:
        throw std::runtime_error("Unhandled CURL socket action");
    }

    return 0;
}

void HTTPFileSource::Impl::onTimeout(Impl *context) {
    int running_handles;
    CURLMcode error = curl::multi_socket_action(context->multi, CURL_SOCKET_TIMEOUT, 0, &running_handles);
    if (error != CURLM_OK) {
        throw std::runtime_error(std::string("CURL multi error: ") + curl::multi_strerror(error));
    }
    context->checkMultiInfo();
}

int HTTPFileSource::Impl::startTimeout(CURLM * /* multi */, long timeout_ms, void *userp) {
    assert(userp);
    auto context = reinterpret_cast<Impl *>(userp);

    if (timeout_ms < 0) {
        // A timeout of 0 ms means that the timer will invoked in the next loop iteration.
        timeout_ms = 0;
    }

    context->timeout.stop();
    context->timeout.start(mbgl::Milliseconds(timeout_ms), Duration::zero(),
        std::bind(&Impl::onTimeout, context));

    return 0;
}

HTTPRequest::HTTPRequest(HTTPFileSource::Impl* context_, Resource resource_, FileSource::Callback callback_)
    : context(context_),
      resource(std::move(resource_)),
      callback(std::move(callback_)),
      handle(context->getHandle()) {

    // If there's already a response, set the correct etags/modified headers to make sure we are
    // getting a 304 response if possible. This avoids redownloading unchanged data.
    if (resource.priorEtag) {
        const std::string header = std::string("If-None-Match: ") + *resource.priorEtag;
        headers = curl::slist_append(headers, header.c_str());
    } else if (resource.priorModified) {
        const std::string time =
            std::string("If-Modified-Since: ") + util::rfc1123(*resource.priorModified);
        headers = curl::slist_append(headers, time.c_str());
    }

    if (headers) {
        curl::easy_setopt(handle, CURLOPT_HTTPHEADER, headers);
    }

    handleError(curl::easy_setopt(handle, CURLOPT_PRIVATE, this));
    handleError(curl::easy_setopt(handle, CURLOPT_ERRORBUFFER, error));
    handleError(curl::easy_setopt(handle, CURLOPT_CAINFO, "ca-bundle.crt"));
    handleError(curl::easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1));
    handleError(curl::easy_setopt(handle, CURLOPT_URL, resource.url.c_str()));
    handleError(curl::easy_setopt(handle, CURLOPT_WRITEFUNCTION, writeCallback));
    handleError(curl::easy_setopt(handle, CURLOPT_WRITEDATA, this));
    handleError(curl::easy_setopt(handle, CURLOPT_HEADERFUNCTION, headerCallback));
    handleError(curl::easy_setopt(handle, CURLOPT_HEADERDATA, this));
#if LIBCURL_VERSION_NUM >= ((7) << 16 | (21) << 8 | 6) // Renamed in 7.21.6
    handleError(curl::easy_setopt(handle, CURLOPT_ACCEPT_ENCODING, "gzip, deflate"));
#else
    handleError(curl::easy_setopt(handle, CURLOPT_ENCODING, "gzip, deflate"));
#endif
    handleError(curl::easy_setopt(handle, CURLOPT_USERAGENT, "MapboxGL/1.0"));
    handleError(curl::easy_setopt(handle, CURLOPT_SHARE, context->share));

    // Start requesting the information.
    handleError(curl::multi_add_handle(context->multi, handle));
}

HTTPRequest::~HTTPRequest() {
    handleError(curl::multi_remove_handle(context->multi, handle));
    context->returnHandle(handle);
    handle = nullptr;

    if (headers) {
        curl::slist_free_all(headers);
        headers = nullptr;
    }
}

// This function is called when we have new data for a request. We just append it to the string
// containing the previous data.
size_t HTTPRequest::writeCallback(void *const contents, const size_t size, const size_t nmemb, void *userp) {
    assert(userp);
    auto impl = reinterpret_cast<HTTPRequest *>(userp);

    if (!impl->data) {
        impl->data = std::make_shared<std::string>();
    }

    impl->data->append((char *)contents, size * nmemb);
    return size * nmemb;
}

// Compares the beginning of the (non-zero-terminated!) data buffer with the (zero-terminated!)
// header string. If the data buffer contains the header string at the beginning, it returns
// the length of the header string == begin of the value, otherwise it returns npos.
// The comparison of the header is ASCII-case-insensitive.
size_t headerMatches(const char *const header, const char *const buffer, const size_t length) {
    const size_t headerLength = strlen(header);
    if (length < headerLength) {
        return std::string::npos;
    }
    size_t i = 0;
    while (i < length && i < headerLength && std::tolower(buffer[i]) == std::tolower(header[i])) {
        i++;
    }
    return i == headerLength ? i : std::string::npos;
}

size_t HTTPRequest::headerCallback(char *const buffer, const size_t size, const size_t nmemb, void *userp) {
    assert(userp);
    auto baton = reinterpret_cast<HTTPRequest *>(userp);

    if (!baton->response) {
        baton->response = std::make_unique<Response>();
    }

    const size_t length = size * nmemb;
    size_t begin = std::string::npos;
    if ((begin = headerMatches("last-modified: ", buffer, length)) != std::string::npos) {
        // Always overwrite the modification date; We might already have a value here from the
        // Date header, but this one is more accurate.
        const std::string value { buffer + begin, length - begin - 2 }; // remove \r\n
        baton->response->modified = Timestamp{ Seconds(curl::getdate(value.c_str(), nullptr)) };
    } else if ((begin = headerMatches("etag: ", buffer, length)) != std::string::npos) {
        baton->response->etag = std::string(buffer + begin, length - begin - 2); // remove \r\n
    } else if ((begin = headerMatches("cache-control: ", buffer, length)) != std::string::npos) {
        const std::string value { buffer + begin, length - begin - 2 }; // remove \r\n
        const auto cc = http::CacheControl::parse(value.c_str());
        baton->response->expires = cc.toTimePoint();
        baton->response->mustRevalidate = cc.mustRevalidate;
    } else if ((begin = headerMatches("expires: ", buffer, length)) != std::string::npos) {
        const std::string value { buffer + begin, length - begin - 2 }; // remove \r\n
        baton->response->expires = Timestamp{ Seconds(curl::getdate(value.c_str(), nullptr)) };
    } else if ((begin = headerMatches("retry-after: ", buffer, length)) != std::string::npos) {
        baton->retryAfter = std::string(buffer + begin, length - begin - 2); // remove \r\n
    } else if ((begin = headerMatches("x-rate-limit-reset: ", buffer, length)) != std::string::npos) {
        baton->xRateLimitReset = std::string(buffer + begin, length - begin - 2); // remove \r\n
    }

    return length;
}

void HTTPRequest::handleResult(CURLcode code) {
    // Make sure a response object exists in case we haven't got any headers or content.
    if (!response) {
        response = std::make_unique<Response>();
    }

    using Error = Response::Error;

    // Add human-readable error code
    if (code != CURLE_OK) {
        switch (code) {
        case CURLE_COULDNT_RESOLVE_PROXY:
        case CURLE_COULDNT_RESOLVE_HOST:
        case CURLE_COULDNT_CONNECT:
        case CURLE_OPERATION_TIMEDOUT:

            response->error = std::make_unique<Error>(
                Error::Reason::Connection, std::string{ curl::easy_strerror(code) } + ": " + error);
            break;

        default:
            response->error = std::make_unique<Error>(
                Error::Reason::Other, std::string{ curl::easy_strerror(code) } + ": " + error);
            break;
        }
    } else {
        long responseCode = 0;
        curl::easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &responseCode);

        if (responseCode == 200) {
            if (data) {
                response->data = std::move(data);
            } else {
                response->data = std::make_shared<std::string>();
            }
        } else if (responseCode == 204 || (responseCode == 404 && resource.kind == Resource::Kind::Tile)) {
            response->noContent = true;
        } else if (responseCode == 304) {
            response->notModified = true;
        } else if (responseCode == 404) {
            response->error =
                std::make_unique<Error>(Error::Reason::NotFound, "HTTP status code 404");
        } else if (responseCode == 429) {
            response->error =
                std::make_unique<Error>(Error::Reason::RateLimit, "HTTP status code 429",
                                        http::parseRetryHeaders(retryAfter, xRateLimitReset));
        } else if (responseCode >= 500 && responseCode < 600) {
            response->error =
                std::make_unique<Error>(Error::Reason::Server, std::string{ "HTTP status code " } +
                                                                   util::toString(responseCode));
        } else {
            response->error =
                std::make_unique<Error>(Error::Reason::Other, std::string{ "HTTP status code " } +
                                                                  util::toString(responseCode));
        }
    }

    // Calling `callback` may result in deleting `this`. Copy data to temporaries first.
    auto callback_ = callback;
    auto response_ = *response;
    callback_(response_);
}

HTTPFileSource::HTTPFileSource()
    : impl(std::make_unique<Impl>()) {
}

HTTPFileSource::~HTTPFileSource() = default;

std::unique_ptr<AsyncRequest> HTTPFileSource::request(const Resource& resource, Callback callback) {
    return std::make_unique<HTTPRequest>(impl.get(), resource, callback);
}

} // namespace mbgl