[core] Cache with mapbox:// tile URLs

This commit is contained in:
John Firebaugh
2016-01-29 17:49:22 -08:00
parent dbb8e58aea
commit 025375ad0b
9 changed files with 124 additions and 185 deletions

View File

@@ -92,6 +92,9 @@ std::unique_ptr<FileRequest> OnlineFileSource::request(const Resource& resource,
Resource res = resource;
switch (resource.kind) {
case Resource::Kind::Unknown:
break;
case Resource::Kind::Style:
res.url = mbgl::util::mapbox::normalizeStyleURL(resource.url, accessToken);
break;
@@ -109,7 +112,8 @@ std::unique_ptr<FileRequest> OnlineFileSource::request(const Resource& resource,
res.url = util::mapbox::normalizeSpriteURL(resource.url, accessToken);
break;
default:
case Resource::Kind::Tile:
res.url = util::mapbox::normalizeTileURL(resource.url, accessToken);
break;
}

View File

@@ -322,8 +322,7 @@ void SQLiteCache::Impl::get(const Resource &resource, Callback callback) {
getStmt->reset();
}
const auto canonicalURL = util::mapbox::canonicalURL(resource.url);
getStmt->bind(1, canonicalURL.c_str());
getStmt->bind(1, resource.url);
if (getStmt->run()) {
// There is data.
auto response = std::make_unique<Response>();
@@ -358,7 +357,7 @@ void SQLiteCache::Impl::get(const Resource &resource, Callback callback) {
}
accessedStmt->bind(1, SystemClock::now());
accessedStmt->bind(2, canonicalURL.c_str());
accessedStmt->bind(2, resource.url);
accessedStmt->run();
}
} catch (mapbox::sqlite::Exception& ex) {
@@ -411,8 +410,7 @@ void SQLiteCache::Impl::put(const Resource& resource, const Response& response)
putStmt->reset();
}
const auto canonicalURL = util::mapbox::canonicalURL(resource.url);
putStmt->bind(1 /* url */, canonicalURL.c_str());
putStmt->bind(1 /* url */, resource.url);
if (response.error) {
putStmt->bind(2 /* status */, int(response.error->reason));
} else {
@@ -465,10 +463,9 @@ void SQLiteCache::Impl::refresh(const Resource& resource, optional<SystemTimePoi
refreshStmt->reset();
}
const auto canonicalURL = util::mapbox::canonicalURL(resource.url);
refreshStmt->bind(1, SystemClock::now());
refreshStmt->bind(2, expires);
refreshStmt->bind(3, canonicalURL.c_str());
refreshStmt->bind(3, resource.url);
refreshStmt->run();
} catch (mapbox::sqlite::Exception& ex) {
Log::Error(Event::Database, ex.code, ex.what());

View File

@@ -269,11 +269,11 @@ std::unique_ptr<SourceInfo> StyleParser::parseTileJSON(const std::string& json,
std::unique_ptr<SourceInfo> result = StyleParser::parseTileJSON(document);
// TODO: Remove this hack by delivering proper URLs in the TileJSON to begin with.
if (type == SourceType::Raster && util::mapbox::isMapboxURL(sourceURL)) {
if (util::mapbox::isMapboxURL(sourceURL)) {
std::transform(result->tiles.begin(),
result->tiles.end(),
result->tiles.begin(),
util::mapbox::normalizeRasterTileURL);
std::bind(util::mapbox::canonicalizeTileURL, std::placeholders::_1, type));
}
return result;

View File

@@ -111,83 +111,56 @@ std::string normalizeGlyphsURL(const std::string& url, const std::string& access
return baseURL + "fonts/v1/" + user + "/" + fontstack + "/" + range + "?access_token=" + accessToken;
}
std::string normalizeRasterTileURL(const std::string& url) {
std::string::size_type queryIdx = url.rfind("?");
// Trim off the right end but never touch anything before the extension dot.
std::string urlSansParams((queryIdx == std::string::npos) ? url : url.substr(0, queryIdx));
while (!urlSansParams.empty() && isdigit(urlSansParams.back())) {
urlSansParams.pop_back();
}
std::string::size_type basenameIdx = url.rfind("/", queryIdx);
std::string::size_type extensionIdx = url.rfind(".", queryIdx);
if (basenameIdx == std::string::npos || extensionIdx == std::string::npos ||
basenameIdx > extensionIdx) {
// No file extension: probably not a file name we can tack a ratio onto.
std::string normalizeTileURL(const std::string& url, const std::string& accessToken) {
if (!isMapboxURL(url)) {
return url;
}
std::string normalizedURL(url);
return baseURL + "v4/" + url.substr(sizeof("mapbox://tiles/") - 1) + "?access_token=" + accessToken;
}
std::string canonicalizeTileURL(const std::string& url, SourceType type) {
auto tilesetStartIdx = url.find("/v4/");
if (tilesetStartIdx == std::string::npos) {
return url;
}
tilesetStartIdx += sizeof("/v4/") - 1;
auto tilesetEndIdx = url.find("/", tilesetStartIdx);
if (tilesetEndIdx == std::string::npos) {
return url;
}
auto queryIdx = url.rfind("?");
if (queryIdx == std::string::npos) {
queryIdx = url.length();
}
auto basenameIdx = url.rfind("/", queryIdx);
if (basenameIdx == std::string::npos || basenameIdx == queryIdx - 1) {
basenameIdx = url.length();
} else {
basenameIdx += 1;
}
auto extensionIdx = url.find(".", basenameIdx);
if (extensionIdx == std::string::npos || extensionIdx == queryIdx - 1) {
return url;
}
auto tileset = url.substr(tilesetStartIdx, tilesetEndIdx - tilesetStartIdx);
auto extension = url.substr(extensionIdx + 1, queryIdx - extensionIdx - 1);
#if !defined(__ANDROID__) && !defined(__APPLE__)
// Replace PNG with WebP.
if (normalizedURL.compare(extensionIdx + 1, 3, "png") == 0) {
normalizedURL.replace(extensionIdx + 1, 3, "webp");
if (extension == "png") {
extension = "webp";
}
#endif // !defined(__ANDROID__) && !defined(__APPLE__)
normalizedURL.insert(extensionIdx, "{ratio}");
return normalizedURL;
}
std::string removeAccessTokenFromURL(const std::string &url) {
const size_t token_start = url.find("access_token=");
// Ensure that token exists, isn't at the front and is preceded by either & or ?.
if (token_start == std::string::npos || token_start == 0 || !(url[token_start - 1] == '&' || url[token_start - 1] == '?')) {
return url;
}
const size_t token_end = url.find_first_of('&', token_start);
if (token_end == std::string::npos) {
// The token is the last query argument. We slice away the "&access_token=..." part
return url.substr(0, token_start - 1);
} else {
// We slice away the "access_token=...&" part.
return url.substr(0, token_start) + url.substr(token_end + 1);
}
}
namespace {
std::string convertMapboxDomainsToProtocol(const std::string &url) {
const size_t protocol_separator = url.find("://");
if (protocol_separator == std::string::npos) {
return url;
}
const std::string protocol = url.substr(0, protocol_separator);
if (!(protocol == "http" || protocol == "https")) {
return url;
}
const size_t domain_begin = protocol_separator + 3;
const size_t path_separator = url.find("/", domain_begin);
if (path_separator == std::string::npos) {
return url;
}
const std::string domain = url.substr(domain_begin, path_separator - domain_begin);
if (domain == "api.mapbox.com" || domain.find(".tiles.mapbox.com") != std::string::npos) {
return std::string{ "mapbox://" } + url.substr(path_separator + 1);
} else {
return url;
}
}
} // end namespace
std::string canonicalURL(const std::string &url) {
return removeAccessTokenFromURL(convertMapboxDomainsToProtocol(url));
return "mapbox://tiles/" + tileset + "/{z}/{x}/{y}" +
(type == SourceType::Raster ? "{ratio}" : "") + "." + extension;
}
} // end namespace mapbox

View File

@@ -14,13 +14,10 @@ std::string normalizeSourceURL(const std::string& url, const std::string& access
std::string normalizeStyleURL(const std::string& url, const std::string& accessToken);
std::string normalizeSpriteURL(const std::string& url, const std::string& accessToken);
std::string normalizeGlyphsURL(const std::string& url, const std::string& accessToken);
std::string normalizeRasterTileURL(const std::string& url);
std::string normalizeTileURL(const std::string& url, const std::string& accessToken);
// Canonicalizes Mapbox URLs by removing [a-d] subdomain prefixes, access tokens, and protocol.
// Note that this is close, but not exactly the reverse operation as above, as this retains certain
// information, such as the API version. It is used to cache resources retrieved from the URL, that
// sometimes have multiple valid URLs.
std::string canonicalURL(const std::string &url);
// Return a "mapbox://tiles/..." URL (suitable for normalizeTileURL) for the given Mapbox tile URL.
std::string canonicalizeTileURL(const std::string& url, SourceType);
} // namespace mapbox
} // namespace util

View File

@@ -4,5 +4,5 @@
"center": [ 1, 2, 3 ],
"bounds": [ 4, 5, 6, 7 ],
"attribution": "attribution",
"tiles": [ "http://a.tiles.mapbox.com/mapbox.satellite/{z}-{x}-{y}.png?access_token=key" ]
"tiles": [ "http://a.tiles.mapbox.com/v4/mapbox.satellite/{z}/{x}/{y}.png?access_token=key" ]
}

View File

@@ -4,5 +4,5 @@
"center": [ 1, 2, 3 ],
"bounds": [ 4, 5, 6, 7 ],
"attribution": "attribution",
"tiles": [ "http://a.tiles.mapbox.com/mapbox.streets/{z}-{x}-{y}.vector.pbf?access_token=key" ]
"tiles": [ "http://a.tiles.mapbox.com/v4/mapbox.streets/{z}/{x}/{y}.vector.pbf?access_token=key" ]
}

View File

@@ -96,9 +96,9 @@ TEST(StyleParser, ParseTileJSONRaster) {
EXPECT_EQ(15, result->maxZoom);
EXPECT_EQ("attribution", result->attribution);
#if !defined(__ANDROID__) && !defined(__APPLE__)
EXPECT_EQ("http://a.tiles.mapbox.com/mapbox.satellite/{z}-{x}-{y}{ratio}.webp?access_token=key", result->tiles[0]);
EXPECT_EQ("mapbox://tiles/mapbox.satellite/{z}/{x}/{y}{ratio}.webp", result->tiles[0]);
#else
EXPECT_EQ("http://a.tiles.mapbox.com/mapbox.satellite/{z}-{x}-{y}{ratio}.png?access_token=key", result->tiles[0]);
EXPECT_EQ("mapbox://tiles/mapbox.satellite/{z}/{x}/{y}{ratio}.png", result->tiles[0]);
#endif
}
@@ -111,7 +111,7 @@ TEST(StyleParser, ParseTileJSONVector) {
EXPECT_EQ(0, result->minZoom);
EXPECT_EQ(15, result->maxZoom);
EXPECT_EQ("attribution", result->attribution);
EXPECT_EQ("http://a.tiles.mapbox.com/mapbox.streets/{z}-{x}-{y}.vector.pbf?access_token=key", result->tiles[0]);
EXPECT_EQ("mapbox://tiles/mapbox.streets/{z}/{x}/{y}.vector.pbf", result->tiles[0]);
}
TEST(StyleParser, FontStacks) {

View File

@@ -7,6 +7,8 @@
using namespace mbgl;
// TODO: correct all EXPECT_EQ(actual, expected) to EXPECT_EQ(expected, actual)
TEST(Mapbox, SourceURL) {
EXPECT_EQ(mbgl::util::mapbox::normalizeSourceURL("mapbox://user.map", "key"), "https://api.mapbox.com/v4/user.map.json?access_token=key&secure");
EXPECT_EQ(mbgl::util::mapbox::normalizeSourceURL("mapbox://user.map", "token"), "https://api.mapbox.com/v4/user.map.json?access_token=token&secure");
@@ -36,110 +38,76 @@ TEST(Mapbox, SpriteURL) {
}
TEST(Mapbox, TileURL) {
try {
#if defined(__ANDROID__) || defined(__APPLE__)
EXPECT_EQ("http://path.png/tile{ratio}.png", mbgl::util::mapbox::normalizeRasterTileURL("http://path.png/tile.png"));
EXPECT_EQ("http://path.png/tile{ratio}.png32", mbgl::util::mapbox::normalizeRasterTileURL("http://path.png/tile.png32"));
EXPECT_EQ("http://path.png/tile{ratio}.png70", mbgl::util::mapbox::normalizeRasterTileURL("http://path.png/tile.png70"));
EXPECT_EQ("http://path.png/tile{ratio}.png?access_token=foo", mbgl::util::mapbox::normalizeRasterTileURL("http://path.png/tile.png?access_token=foo"));
#else
EXPECT_EQ("http://path.png/tile{ratio}.webp", mbgl::util::mapbox::normalizeRasterTileURL("http://path.png/tile.png"));
EXPECT_EQ("http://path.png/tile{ratio}.webp32", mbgl::util::mapbox::normalizeRasterTileURL("http://path.png/tile.png32"));
EXPECT_EQ("http://path.png/tile{ratio}.webp70", mbgl::util::mapbox::normalizeRasterTileURL("http://path.png/tile.png70"));
EXPECT_EQ("http://path.png/tile{ratio}.webp?access_token=foo", mbgl::util::mapbox::normalizeRasterTileURL("http://path.png/tile.png?access_token=foo"));
#endif // defined(__ANDROID__) || defined(__APPLE__)
EXPECT_EQ("http://path.png/tile{ratio}.pbf", mbgl::util::mapbox::normalizeRasterTileURL("http://path.png/tile.pbf"));
EXPECT_EQ("http://path.png/tile{ratio}.pbf?access_token=foo", mbgl::util::mapbox::normalizeRasterTileURL("http://path.png/tile.pbf?access_token=foo"));
EXPECT_EQ("http://path.png/tile{ratio}.pbf?access_token=foo.png", mbgl::util::mapbox::normalizeRasterTileURL("http://path.png/tile.pbf?access_token=foo.png"));
EXPECT_EQ("http://path.png/tile{ratio}.pbf?access_token=foo.png/bar", mbgl::util::mapbox::normalizeRasterTileURL("http://path.png/tile.pbf?access_token=foo.png/bar"));
EXPECT_EQ("http://path.png/tile{ratio}.pbf?access_token=foo.png/bar.png", mbgl::util::mapbox::normalizeRasterTileURL("http://path.png/tile.pbf?access_token=foo.png/bar.png"));
} catch (const std::regex_error& e) {
const char *error = "unknown";
switch (e.code()) {
case std::regex_constants::error_collate:
error = "error_collate"; break;
case std::regex_constants::error_ctype:
error = "error_ctype"; break;
case std::regex_constants::error_escape:
error = "error_escape"; break;
case std::regex_constants::error_backref:
error = "error_backref"; break;
case std::regex_constants::error_paren:
error = "error_paren"; break;
case std::regex_constants::error_brace:
error = "error_brace"; break;
case std::regex_constants::error_badbrace:
error = "error_badbrace"; break;
case std::regex_constants::error_range:
error = "error_range"; break;
case std::regex_constants::error_space:
error = "error_space"; break;
case std::regex_constants::error_badrepeat:
error = "error_badrepeat"; break;
case std::regex_constants::error_complexity:
error = "error_complexity"; break;
case std::regex_constants::error_stack:
error = "error_stack"; break;
default:
break;
}
mbgl::Log::Error(mbgl::Event::General, "regex_error caught: %s - %s (%d)", e.what(), error, e.code());
throw e;
}
EXPECT_EQ(
"https://api.mapbox.com/v4/a.b/0/0/0.pbf?access_token=key",
mbgl::util::mapbox::normalizeTileURL("mapbox://tiles/a.b/0/0/0.pbf", "key"));
EXPECT_EQ(
"https://api.mapbox.com/v4/a.b/0/0/0.png?access_token=key",
mbgl::util::mapbox::normalizeTileURL("mapbox://tiles/a.b/0/0/0.png", "key"));
EXPECT_EQ(
"https://api.mapbox.com/v4/a.b/0/0/0@2x.webp?access_token=key",
mbgl::util::mapbox::normalizeTileURL("mapbox://tiles/a.b/0/0/0@2x.webp", "key"));
EXPECT_EQ(
"https://api.mapbox.com/v4/a.b,c.d/0/0/0.pbf?access_token=key",
mbgl::util::mapbox::normalizeTileURL("mapbox://tiles/a.b,c.d/0/0/0.pbf", "key"));
EXPECT_EQ(
"http://path",
mbgl::util::mapbox::normalizeSpriteURL("http://path", "key"));
}
TEST(Mapbox, CanonicalURL) {
using mbgl::util::mapbox::canonicalURL;
EXPECT_EQ(
canonicalURL("https://a.tiles.mapbox.com/v4/"
"mapbox.mapbox-terrain-v2,mapbox.mapbox-streets-v6/15/17599/"
"10744.vector.pbf?access_token=pk.kAeslEm93Sjf3mXk."
"vbiF02XnvkPkzlFhGSn2iIm6De3Cxsk5tmips2tvkG8sF"),
"mapbox://v4/mapbox.mapbox-terrain-v2,mapbox.mapbox-streets-v6/15/17599/10744.vector.pbf");
"mapbox://tiles/a.b/{z}/{x}/{y}.vector.pbf",
mbgl::util::mapbox::canonicalizeTileURL("http://a.tiles.mapbox.com/v4/a.b/{z}/{x}/{y}.vector.pbf", SourceType::Vector));
EXPECT_EQ(
canonicalURL("http://a.tiles.mapbox.com/v4/"
"mapbox.mapbox-terrain-v2,mapbox.mapbox-streets-v6/15/17599/"
"10744.vector.pbf?access_token=pk.kAeslEm93Sjf3mXk."
"vbiF02XnvkPkzlFhGSn2iIm6De3Cxsk5tmips2tvkG8sF"),
"mapbox://v4/mapbox.mapbox-terrain-v2,mapbox.mapbox-streets-v6/15/17599/10744.vector.pbf");
"mapbox://tiles/a.b/{z}/{x}/{y}.vector.pbf",
mbgl::util::mapbox::canonicalizeTileURL("http://b.tiles.mapbox.com/v4/a.b/{z}/{x}/{y}.vector.pbf", SourceType::Vector));
EXPECT_EQ(
canonicalURL("https://b.tiles.mapbox.com/v4/"
"mapbox.mapbox-terrain-v2,mapbox.mapbox-streets-v6/15/17599/"
"10744.vector.pbf?access_token=pk.kAeslEm93Sjf3mXk."
"vbiF02XnvkPkzlFhGSn2iIm6De3Cxsk5tmips2tvkG8sF"),
"mapbox://v4/mapbox.mapbox-terrain-v2,mapbox.mapbox-streets-v6/15/17599/10744.vector.pbf");
"mapbox://tiles/a.b/{z}/{x}/{y}.vector.pbf",
mbgl::util::mapbox::canonicalizeTileURL("http://api.mapbox.com/v4/a.b/{z}/{x}/{y}.vector.pbf", SourceType::Vector));
EXPECT_EQ(
canonicalURL("http://c.tiles.mapbox.com/v4/"
"mapbox.mapbox-terrain-v2,mapbox.mapbox-streets-v6/15/17599/"
"10744.vector.pbf?access_token=pk.kAeslEm93Sjf3mXk."
"vbiF02XnvkPkzlFhGSn2iIm6De3Cxsk5tmips2tvkG8sF"),
"mapbox://v4/mapbox.mapbox-terrain-v2,mapbox.mapbox-streets-v6/15/17599/10744.vector.pbf");
"mapbox://tiles/a.b/{z}/{x}/{y}.vector.pbf",
mbgl::util::mapbox::canonicalizeTileURL("http://api.mapbox.com/v4/a.b/{z}/{x}/{y}.vector.pbf?access_token=key", SourceType::Vector));
EXPECT_EQ(
canonicalURL("https://api.mapbox.com/v4/"
"mapbox.mapbox-terrain-v2,mapbox.mapbox-streets-v6/15/17599/"
"10744.vector.pbf?access_token=pk.kAeslEm93Sjf3mXk."
"vbiF02XnvkPkzlFhGSn2iIm6De3Cxsk5tmips2tvkG8sF"),
"mapbox://v4/mapbox.mapbox-terrain-v2,mapbox.mapbox-streets-v6/15/17599/10744.vector.pbf");
"mapbox://tiles/a.b,c.d/{z}/{x}/{y}.vector.pbf",
mbgl::util::mapbox::canonicalizeTileURL("http://api.mapbox.com/v4/a.b,c.d/{z}/{x}/{y}.vector.pbf?access_token=key", SourceType::Vector));
EXPECT_EQ(
canonicalURL("http://api.mapbox.com/v4/"
"mapbox.mapbox-terrain-v2,mapbox.mapbox-streets-v6/15/17599/"
"10744.vector.pbf"),
"mapbox://v4/mapbox.mapbox-terrain-v2,mapbox.mapbox-streets-v6/15/17599/10744.vector.pbf");
"mapbox://tiles/a.b/{z}/{x}/{y}{ratio}.jpg",
mbgl::util::mapbox::canonicalizeTileURL("http://api.mapbox.com/v4/a.b/{z}/{x}/{y}.jpg?access_token=key", SourceType::Raster));
EXPECT_EQ(
"mapbox://tiles/a.b/{z}/{x}/{y}{ratio}.jpg70",
mbgl::util::mapbox::canonicalizeTileURL("http://api.mapbox.com/v4/a.b/{z}/{x}/{y}.jpg70?access_token=key", SourceType::Raster));
EXPECT_EQ(canonicalURL("https://api.mapbox.com/fonts/v1/mapbox/"
"DIN%20Offc%20Pro%20Italic%2cArial%20Unicode%20MS%20Regular/"
"0-255.pbf?access_token=pk.kAeslEm93Sjf3mXk."
"vbiF02XnvkPkzlFhGSn2iIm6De3Cxsk5tmips2tvkG8sF"),
"mapbox://fonts/v1/mapbox/DIN%20Offc%20Pro%20Italic%2cArial%20Unicode%20MS%20Regular/"
"0-255.pbf");
#if defined(__ANDROID__) || defined(__APPLE__)
EXPECT_EQ(
"mapbox://tiles/a.b/{z}/{x}/{y}{ratio}.png",
mbgl::util::mapbox::canonicalizeTileURL("http://api.mapbox.com/v4/a.b/{z}/{x}/{y}.png", SourceType::Raster));
EXPECT_EQ(
"mapbox://tiles/a.b/{z}/{x}/{y}{ratio}.png",
mbgl::util::mapbox::canonicalizeTileURL("http://api.mapbox.com/v4/a.b/{z}/{x}/{y}.png?access_token=key", SourceType::Raster));
#else
EXPECT_EQ(
"mapbox://tiles/a.b/{z}/{x}/{y}{ratio}.webp",
mbgl::util::mapbox::canonicalizeTileURL("http://api.mapbox.com/v4/a.b/{z}/{x}/{y}.png", SourceType::Raster));
EXPECT_EQ(
"mapbox://tiles/a.b/{z}/{x}/{y}{ratio}.webp",
mbgl::util::mapbox::canonicalizeTileURL("http://api.mapbox.com/v4/a.b/{z}/{x}/{y}.png?access_token=key", SourceType::Raster));
#endif // defined(__ANDROID__) || defined(__APPLE__)
EXPECT_EQ(canonicalURL("https://api.mapbox.com/styles/v1/mapbox/streets-v8/"
"sprite.json?access_token=pk.kAeslEm93Sjf3mXk."
"vbiF02XnvkPkzlFhGSn2iIm6De3Cxsk5tmips2tvkG8sF"),
"mapbox://styles/v1/mapbox/streets-v8/sprite.json");
// We don't ever expect to see these inputs, but be safe anyway.
EXPECT_EQ(
"",
mbgl::util::mapbox::canonicalizeTileURL("", SourceType::Raster));
EXPECT_EQ(
"http://path",
mbgl::util::mapbox::canonicalizeTileURL("http://path", SourceType::Raster));
EXPECT_EQ(
"http://api.mapbox.com/v4/",
mbgl::util::mapbox::canonicalizeTileURL("http://api.mapbox.com/v4/", SourceType::Raster));
EXPECT_EQ(
"http://api.mapbox.com/v4/a.b/{z}/{x}/{y}.",
mbgl::util::mapbox::canonicalizeTileURL("http://api.mapbox.com/v4/a.b/{z}/{x}/{y}.", SourceType::Raster));
EXPECT_EQ(
"http://api.mapbox.com/v4/a.b/{z}/{x}/{y}/.",
mbgl::util::mapbox::canonicalizeTileURL("http://api.mapbox.com/v4/a.b/{z}/{x}/{y}/.", SourceType::Raster));
}