tei/MapLibreParticleLab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[core] OnlineFileSource - rate limit

Browse Source
This commit is contained in:
Ivo van Dongen
2016-08-31 16:43:33 +02:00
parent fdaf26b2c0
commit 290bd07a92
15 changed files with 213 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cmake/core-files.cmake
View File

@@ -421,6 +421,8 @@ set(MBGL_CORE_FILES
src/mbgl/util/grid_index.hpp
src/mbgl/util/http_header.cpp
src/mbgl/util/http_header.hpp
src/mbgl/util/http_timeout.cpp
src/mbgl/util/http_timeout.hpp
src/mbgl/util/interpolate.hpp
src/mbgl/util/intersection_tests.cpp
src/mbgl/util/intersection_tests.hpp

1
cmake/test-files.cmake
View File

@@ -87,6 +87,7 @@ set(MBGL_TEST_FILES
# util
test/util/async_task.cpp
test/util/geo.cpp
test/util/http_timeout.cpp
test/util/image.cpp
test/util/mapbox.cpp
test/util/memory.cpp

6
include/mbgl/storage/response.hpp
View File

@@ -2,6 +2,7 @@
#include <mbgl/util/chrono.hpp>
#include <mbgl/util/optional.hpp>
#include <mbgl/util/variant.hpp>
#include <string>
#include <memory>
@@ -45,15 +46,18 @@ public:
NotFound = 2,
Server = 3,
Connection = 4,
RateLimit = 5,
Other = 6,
} reason = Reason::Other;
// An error message from the request handler, e.g. a server message or a system message
// informing the user about the reason for the failure.
std::string message;
optional<Timestamp> retryAfter;
public:
Error(Reason, std::string = "");
Error(Reason, std::string = "", optional<Timestamp> = {});
};
std::ostream& operator<<(std::ostream&, Response::Error::Reason);

2
include/mbgl/util/chrono.hpp
View File

@@ -29,6 +29,8 @@ std::string rfc1123(Timestamp);
std::string iso8601(Timestamp);
Timestamp parseTimestamp(const char *);
Timestamp parseTimestamp(const int32_t timestamp);
// C++17 polyfill
template <class Rep, class Period, class = std::enable_if_t<

2
include/mbgl/util/constants.hpp
View File

@@ -42,6 +42,8 @@ constexpr Duration DEFAULT_FADE_DURATION = Milliseconds(300);
constexpr Seconds CLOCK_SKEW_RETRY_TIMEOUT { 30 };
constexpr UnitBezier DEFAULT_TRANSITION_EASE = { 0, 0, 0.25, 1 };
constexpr int DEFAULT_RATE_LIMIT_TIMEOUT = 5;
} // namespace util

32
platform/default/online_file_source.cpp
View File

@@ -13,6 +13,7 @@
#include <mbgl/util/async_task.hpp>
#include <mbgl/util/noncopyable.hpp>
#include <mbgl/util/timer.hpp>
#include <mbgl/util/http_timeout.hpp>
#include <algorithm>
#include <cassert>
@@ -48,6 +49,7 @@ public:
// backoff when retrying requests.
uint32_t failedRequests = 0;
Response::Error::Reason failedRequestReason = Response::Error::Reason::Success;
optional<Timestamp> retryAfter;
};
class OnlineFileSource::Impl {
@@ -200,30 +202,6 @@ OnlineFileRequest::~OnlineFileRequest() {
impl.remove(this);
}
static Duration errorRetryTimeout(Response::Error::Reason failedRequestReason, uint32_t failedRequests) {
if (failedRequestReason == Response::Error::Reason::Server) {
// Retry after one second three times, then start exponential backoff.
return Seconds(failedRequests <= 3 ? 1 : 1 << std::min(failedRequests - 3, 31u));
} else if (failedRequestReason == Response::Error::Reason::Connection) {
// Immediate exponential backoff.
assert(failedRequests > 0);
return Seconds(1 << std::min(failedRequests - 1, 31u));
} else {
// No error, or not an error that triggers retries.
return Duration::max();
}
}
static Duration expirationTimeout(optional<Timestamp> expires, uint32_t expiredRequests) {
if (expiredRequests) {
return Seconds(1 << std::min(expiredRequests - 1, 31u));
} else if (expires) {
return std::max(Seconds::zero(), *expires - util::now());
} else {
return Duration::max();
}
}
Timestamp interpolateExpiration(const Timestamp& current,
optional<Timestamp> prior,
bool& expired) {
@@ -267,8 +245,9 @@ void OnlineFileRequest::schedule(optional<Timestamp> expires) {
// If we're not being asked for a forced refresh, calculate a timeout that depends on how many
// consecutive errors we've encountered, and on the expiration time, if present.
Duration timeout = std::min(errorRetryTimeout(failedRequestReason, failedRequests),
expirationTimeout(expires, expiredRequests));
Duration timeout = std::min(
http::errorRetryTimeout(failedRequestReason, failedRequests, retryAfter),
http::expirationTimeout(expires, expiredRequests));
if (timeout == Duration::max()) {
return;
@@ -321,6 +300,7 @@ void OnlineFileRequest::completed(Response response) {
if (response.error) {
failedRequests++;
failedRequestReason = response.error->reason;
retryAfter = response.error->retryAfter;
} else {
failedRequests = 0;
failedRequestReason = Response::Error::Reason::Success;

6
src/mbgl/storage/response.cpp
View File

@@ -21,8 +21,8 @@ Response& Response::operator=(const Response& res) {
return *this;
}
Response::Error::Error(Reason reason_, std::string message_)
: reason(reason_), message(std::move(message_)) {
Response::Error::Error(Reason reason_, std::string message_, optional<Timestamp> retryAfter_)
: reason(reason_), message(std::move(message_)), retryAfter(std::move(retryAfter_)) {
}
std::ostream& operator<<(std::ostream& os, Response::Error::Reason r) {
@@ -35,6 +35,8 @@ std::ostream& operator<<(std::ostream& os, Response::Error::Reason r) {
return os << "Response::Error::Reason::Server";
case Response::Error::Reason::Connection:
return os << "Response::Error::Reason::Connection";
case Response::Error::Reason::RateLimit:
return os << "Response::Error::Reason::RateLimit";
case Response::Error::Reason::Other:
return os << "Response::Error::Reason::Other";
}

4
src/mbgl/util/chrono.cpp
View File

@@ -33,6 +33,10 @@ std::string iso8601(Timestamp timestamp) {
Timestamp parseTimestamp(const char* timestamp) {
return std::chrono::time_point_cast<Seconds>(std::chrono::system_clock::from_time_t(parse_date(timestamp)));
}
Timestamp parseTimestamp(const int32_t timestamp) {
return std::chrono::time_point_cast<Seconds>(std::chrono::system_clock::from_time_t(timestamp));
}
} // namespace util

20
src/mbgl/util/http_header.cpp
View File

@@ -28,6 +28,26 @@ CacheControl CacheControl::parse(const std::string& value) {
optional<Timestamp> CacheControl::toTimePoint() const {
return maxAge ? util::now() + Seconds(*maxAge) : optional<Timestamp>{};
}
optional<Timestamp> parseRetryHeaders(const optional<std::string>& retryAfter,
const optional<std::string>& xRateLimitReset) {
if (retryAfter) {
try {
auto secs = std::chrono::seconds(std::stoi(*retryAfter));
return std::chrono::time_point_cast<Seconds>(std::chrono::system_clock::now() + secs);
} catch (...) {
return util::parseTimestamp((*retryAfter).c_str());
}
} else if (xRateLimitReset) {
try {
return util::parseTimestamp(std::stoi(*xRateLimitReset));
} catch (...) {
return {};
}
}
return {};
}
} // namespace http
} // namespace mbgl

3
src/mbgl/util/http_header.hpp
View File

@@ -17,6 +17,9 @@ public:
optional<Timestamp> toTimePoint() const;
};
optional<Timestamp> parseRetryHeaders(const optional<std::string>& retryAfter,
const optional<std::string>& xRateLimitReset);
} // namespace http
} // namespace mbgl

40
src/mbgl/util/http_timeout.cpp Normal file
View File

@@ -0,0 +1,40 @@
#include <mbgl/util/http_timeout.hpp>
#include <mbgl/util/constants.hpp>
namespace mbgl {
namespace http {
Duration errorRetryTimeout(Response::Error::Reason failedRequestReason, uint32_t failedRequests, optional<Timestamp> retryAfter) {
if (failedRequestReason == Response::Error::Reason::Server) {
// Retry after one second three times, then start exponential backoff.
return Seconds(failedRequests <= 3 ? 1 : 1u << std::min(failedRequests - 3, 31u));
} else if (failedRequestReason == Response::Error::Reason::Connection) {
// Immediate exponential backoff.
assert(failedRequests > 0);
return Seconds(1u << std::min(failedRequests - 1, 31u));
} else if (failedRequestReason == Response::Error::Reason::RateLimit) {
if (retryAfter) {
return *retryAfter - util::now();
} else {
//Default
return Seconds(util::DEFAULT_RATE_LIMIT_TIMEOUT);
}
} else {
// No error, or not an error that triggers retries.
return Duration::max();
}
}
Duration expirationTimeout(optional<Timestamp> expires, uint32_t expiredRequests) {
if (expiredRequests) {
return Seconds(1u << std::min(expiredRequests - 1, 31u));
} else if (expires) {
return std::max(Seconds::zero(), *expires - util::now());
} else {
return Duration::max();
}
}
} // namespace http
} // namespace mbgl

15
src/mbgl/util/http_timeout.hpp Normal file
View File

@@ -0,0 +1,15 @@
#pragma once
#include <mbgl/storage/response.hpp>
#include <mbgl/util/optional.hpp>
#include <mbgl/util/chrono.hpp>
namespace mbgl {
namespace http {
Duration errorRetryTimeout(Response::Error::Reason failedRequestReason, uint32_t failedRequests, optional<Timestamp> retryAfter = {});
Duration expirationTimeout(optional<Timestamp> expires, uint32_t expiredRequests);
} // namespace http
} // namespace mbgl

44
test/storage/online_file_source.cpp
View File

@@ -359,3 +359,47 @@ TEST(OnlineFileSource, TEST_REQUIRES_SERVER(NetworkStatusOnlineOffline)) {
loop.run();
}
TEST(OnlineFileSource, TEST_REQUIRES_SERVER(RateLimitStandard)) {
util::RunLoop loop;
OnlineFileSource fs;
auto req = fs.request({ Resource::Unknown, "http://127.0.0.1:3000/rate-limit?std=true" }, [&](Response res) {
ASSERT_NE(nullptr, res.error);
EXPECT_EQ(Response::Error::Reason::RateLimit, res.error->reason);
ASSERT_EQ(true, bool(res.error->retryAfter));
ASSERT_LT(util::now(), res.error->retryAfter);
loop.stop();
});
loop.run();
}
TEST(OnlineFileSource, TEST_REQUIRES_SERVER(RateLimitMBX)) {
util::RunLoop loop;
OnlineFileSource fs;
auto req = fs.request({ Resource::Unknown, "http://127.0.0.1:3000/rate-limit?mbx=true" }, [&](Response res) {
ASSERT_NE(nullptr, res.error);
EXPECT_EQ(Response::Error::Reason::RateLimit, res.error->reason);
ASSERT_EQ(true, bool(res.error->retryAfter));
ASSERT_LT(util::now(), res.error->retryAfter);
loop.stop();
});
loop.run();
}
TEST(OnlineFileSource, TEST_REQUIRES_SERVER(RateLimitDefault)) {
util::RunLoop loop;
OnlineFileSource fs;
auto req = fs.request({ Resource::Unknown, "http://127.0.0.1:3000/rate-limit" }, [&](Response res) {
ASSERT_NE(nullptr, res.error);
EXPECT_EQ(Response::Error::Reason::RateLimit, res.error->reason);
ASSERT_EQ(false, bool(res.error->retryAfter));
loop.stop();
});
loop.run();
}

11
test/storage/server.js
View File

@@ -116,6 +116,17 @@ app.get('/temporary-error', function(req, res) {
temporaryErrorCounter++;
});
app.get('/rate-limit', function(req, res) {
if (req.query.std) {
res.setHeader('Retry-After', 1);
} else if (req.query.mbx) {
res.setHeader('x-rate-limit-reset', Math.round(Date.now() / 1000) + 1);
}
res.status(429).end();
});
app.get('/delayed', function(req, res) {
setTimeout(function() {
res.status(200).send('Response');

54
test/util/http_timeout.cpp Normal file
View File

@@ -0,0 +1,54 @@
#include <mbgl/test/util.hpp>
#include <mbgl/platform/log.hpp>
#include <mbgl/util/http_timeout.hpp>
#include <regex>
#include <iostream>
using namespace mbgl;
using namespace mbgl::http;
TEST(HttpRetry, OtherError) {
//Non-retryable
ASSERT_EQ(Duration::max(), errorRetryTimeout(Response::Error::Reason::Other, 1));
}
TEST(HttpRetry, ServerError) {
// 1-3 failures -> 1 sec
ASSERT_EQ(Seconds(1), errorRetryTimeout(Response::Error::Reason::Server, 1));
ASSERT_EQ(Seconds(1), errorRetryTimeout(Response::Error::Reason::Server, 3));
// After 3, exponential backoff
ASSERT_EQ(Seconds(2), errorRetryTimeout(Response::Error::Reason::Server, 4));
ASSERT_EQ(Seconds(1u << 31), errorRetryTimeout(Response::Error::Reason::Server, 50));
}
TEST(HttpRetry, ConnectionError) {
// Exponential backoff
ASSERT_EQ(Seconds(1), errorRetryTimeout(Response::Error::Reason::Server, 1));
ASSERT_EQ(Seconds(1u << 31), errorRetryTimeout(Response::Error::Reason::Connection, 50));
}
TEST(HttpRetry, RateLimit) {
// Pre-set value from header
ASSERT_EQ(Seconds(1), errorRetryTimeout(Response::Error::Reason::Server, 1, { util::now() + Seconds(1) }));
//Default
ASSERT_EQ(Seconds(5), errorRetryTimeout(Response::Error::Reason::RateLimit, 1, {}));
}
TEST(HttpRetry, ExpiredInitial) {
// 1 sec timeout
ASSERT_EQ(Seconds(1), expirationTimeout({ util::now() + Seconds(1) }, 0));
}
TEST(HttpRetry, ExpiredSubsequent) {
// exponential backoff
ASSERT_EQ(Seconds(1), expirationTimeout({}, 1));
ASSERT_EQ(Seconds(1u << 31), expirationTimeout({}, 50));
}
TEST(HttpRetry, ExpiredNotSet) {
// No expires header set
ASSERT_EQ(Duration::max(), expirationTimeout({}, 0));
}