From c82b02c508bb4eb1d6ee25320cf83995a807e603 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Tue, 3 May 2016 13:57:17 -0700 Subject: [PATCH] [core] port to protozero (#4906) --- configure | 1 + mbgl.gypi | 1 + platform/android/scripts/configure.sh | 1 + platform/ios/scripts/configure.sh | 1 + platform/linux/scripts/configure.sh | 1 + platform/osx/scripts/configure.sh | 1 + platform/qt/scripts/configure.sh | 1 + src/mbgl/text/glyph_pbf.cpp | 70 +++++----- src/mbgl/tile/vector_tile.cpp | 118 +++++++++-------- src/mbgl/tile/vector_tile.hpp | 15 ++- src/mbgl/util/pbf.hpp | 184 -------------------------- test/style/glyph_store.cpp | 2 +- test/style/source.cpp | 2 +- test/test.gypi | 1 + 14 files changed, 119 insertions(+), 280 deletions(-) delete mode 100644 src/mbgl/util/pbf.hpp diff --git a/configure b/configure index 0747197ff..329089949 100755 --- a/configure +++ b/configure @@ -97,6 +97,7 @@ CONFIG+=" 'variables': {"$LN print_opengl_flags print_qt_flags +print_flags protozero cflags print_flags boost cflags print_flags boost_libprogram_options static_libs print_flags openssl static_libs cflags ldflags diff --git a/mbgl.gypi b/mbgl.gypi index 965f2dce9..6619e0011 100644 --- a/mbgl.gypi +++ b/mbgl.gypi @@ -172,6 +172,7 @@ 'variables': { 'cflags_cc': [ '<@(opengl_cflags)', + '<@(protozero_cflags)', '<@(boost_cflags)', '<@(geojsonvt_cflags)', '<@(rapidjson_cflags)', diff --git a/platform/android/scripts/configure.sh b/platform/android/scripts/configure.sh index c53805a9c..cf20f4a99 100644 --- a/platform/android/scripts/configure.sh +++ b/platform/android/scripts/configure.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash +PROTOZERO_VERSION=1.3.0 BOOST_VERSION=1.60.0 LIBPNG_VERSION=1.6.20 LIBJPEG_TURBO_VERSION=1.4.2 diff --git a/platform/ios/scripts/configure.sh b/platform/ios/scripts/configure.sh index 31ef374e8..769407445 100644 --- a/platform/ios/scripts/configure.sh +++ b/platform/ios/scripts/configure.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash +PROTOZERO_VERSION=1.3.0 BOOST_VERSION=1.60.0 SQLITE_VERSION=system ZLIB_VERSION=system diff --git a/platform/linux/scripts/configure.sh b/platform/linux/scripts/configure.sh index d58618d4b..4d1b26fc9 100644 --- a/platform/linux/scripts/configure.sh +++ b/platform/linux/scripts/configure.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash +PROTOZERO_VERSION=1.3.0 BOOST_VERSION=1.60.0 BOOST_LIBPROGRAM_OPTIONS_VERSION=1.60.0 LIBCURL_VERSION=system diff --git a/platform/osx/scripts/configure.sh b/platform/osx/scripts/configure.sh index 0b4c8eaf3..6d2eef03c 100644 --- a/platform/osx/scripts/configure.sh +++ b/platform/osx/scripts/configure.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash +PROTOZERO_VERSION=1.3.0 BOOST_VERSION=1.60.0 BOOST_LIBPROGRAM_OPTIONS_VERSION=1.60.0 GLFW_VERSION=3.1.2 diff --git a/platform/qt/scripts/configure.sh b/platform/qt/scripts/configure.sh index aa5c7615d..d9a8f15cc 100644 --- a/platform/qt/scripts/configure.sh +++ b/platform/qt/scripts/configure.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash +PROTOZERO_VERSION=1.3.0 BOOST_VERSION=1.60.0 GEOJSONVT_VERSION=4.1.2 GTEST_VERSION=1.7.0 diff --git a/src/mbgl/text/glyph_pbf.cpp b/src/mbgl/text/glyph_pbf.cpp index 2588afaef..eab972507 100644 --- a/src/mbgl/text/glyph_pbf.cpp +++ b/src/mbgl/text/glyph_pbf.cpp @@ -6,52 +6,54 @@ #include #include #include -#include #include #include #include +#include + namespace { void parseGlyphPBF(mbgl::GlyphSet& glyphSet, const std::string& data) { - mbgl::pbf glyphs_pbf(reinterpret_cast(data.data()), data.size()); + protozero::pbf_reader glyphs_pbf(data); - while (glyphs_pbf.next()) { - if (glyphs_pbf.tag == 1) { // stacks - mbgl::pbf fontstack_pbf = glyphs_pbf.message(); - while (fontstack_pbf.next()) { - if (fontstack_pbf.tag == 3) { // glyphs - mbgl::pbf glyph_pbf = fontstack_pbf.message(); + while (glyphs_pbf.next(1)) { + auto fontstack_pbf = glyphs_pbf.get_message(); + while (fontstack_pbf.next(3)) { + auto glyph_pbf = fontstack_pbf.get_message(); - mbgl::SDFGlyph glyph; + mbgl::SDFGlyph glyph; - while (glyph_pbf.next()) { - if (glyph_pbf.tag == 1) { // id - glyph.id = glyph_pbf.varint(); - } else if (glyph_pbf.tag == 2) { // bitmap - glyph.bitmap = glyph_pbf.string(); - } else if (glyph_pbf.tag == 3) { // width - glyph.metrics.width = glyph_pbf.varint(); - } else if (glyph_pbf.tag == 4) { // height - glyph.metrics.height = glyph_pbf.varint(); - } else if (glyph_pbf.tag == 5) { // left - glyph.metrics.left = glyph_pbf.svarint(); - } else if (glyph_pbf.tag == 6) { // top - glyph.metrics.top = glyph_pbf.svarint(); - } else if (glyph_pbf.tag == 7) { // advance - glyph.metrics.advance = glyph_pbf.varint(); - } else { - glyph_pbf.skip(); - } - } - - glyphSet.insert(glyph.id, glyph); - } else { - fontstack_pbf.skip(); + while (glyph_pbf.next()) { + switch (glyph_pbf.tag()) { + case 1: // id + glyph.id = glyph_pbf.get_uint32(); + break; + case 2: // bitmap + glyph.bitmap = glyph_pbf.get_string(); + break; + case 3: // width + glyph.metrics.width = glyph_pbf.get_uint32(); + break; + case 4: // height + glyph.metrics.height = glyph_pbf.get_uint32(); + break; + case 5: // left + glyph.metrics.left = glyph_pbf.get_sint32(); + break; + case 6: // top + glyph.metrics.top = glyph_pbf.get_sint32(); + break; + case 7: // advance + glyph.metrics.advance = glyph_pbf.get_uint32(); + break; + default: + glyph_pbf.skip(); + break; } } - } else { - glyphs_pbf.skip(); + + glyphSet.insert(glyph.id, glyph); } } } diff --git a/src/mbgl/tile/vector_tile.cpp b/src/mbgl/tile/vector_tile.cpp index f3a02c5d7..af30fa998 100644 --- a/src/mbgl/tile/vector_tile.cpp +++ b/src/mbgl/tile/vector_tile.cpp @@ -9,25 +9,24 @@ namespace mbgl { -Value parseValue(pbf data) { +Value parseValue(protozero::pbf_reader data) { while (data.next()) { - switch (data.tag) - { + switch (data.tag()) { case 1: // string_value - return data.string(); + return data.get_string(); case 2: // float_value - return static_cast(data.float32()); + return static_cast(data.get_float()); case 3: // double_value - return data.float64(); + return data.get_double(); case 4: // int_value - return data.varint(); + return data.get_int64(); case 5: // uint_value - return data.varint(); + return data.get_uint64(); case 6: // sint_value - return data.svarint(); + return data.get_sint64(); case 7: // bool_value - return data.boolean(); + return data.get_bool(); default: data.skip(); break; @@ -36,19 +35,25 @@ Value parseValue(pbf data) { return false; } -VectorTileFeature::VectorTileFeature(pbf feature_pbf, const VectorTileLayer& layer_) +VectorTileFeature::VectorTileFeature(protozero::pbf_reader feature_pbf, const VectorTileLayer& layer_) : layer(layer_) { while (feature_pbf.next()) { - if (feature_pbf.tag == 1) { // id - id = feature_pbf.varint(); - } else if (feature_pbf.tag == 2) { // tags - tags_pbf = feature_pbf.message(); - } else if (feature_pbf.tag == 3) { // type - type = (FeatureType)feature_pbf.varint(); - } else if (feature_pbf.tag == 4) { // geometry - geometry_pbf = feature_pbf.message(); - } else { + switch (feature_pbf.tag()) { + case 1: // id + id = feature_pbf.get_uint64(); + break; + case 2: // tags + tags_iter = feature_pbf.get_packed_uint32(); + break; + case 3: // type + type = static_cast(feature_pbf.get_enum()); + break; + case 4: // geometry + geometry_iter = feature_pbf.get_packed_uint32(); + break; + default: feature_pbf.skip(); + break; } } } @@ -59,19 +64,20 @@ optional VectorTileFeature::getValue(const std::string& key) const { return optional(); } - pbf tags = tags_pbf; - while (tags) { - uint32_t tag_key = tags.varint(); + auto start_itr = tags_iter.first; + const auto & end_itr = tags_iter.second; + while (start_itr != end_itr) { + uint32_t tag_key = static_cast(*start_itr++); if (layer.keysMap.size() <= tag_key) { throw std::runtime_error("feature referenced out of range key"); } - if (!tags) { + if (start_itr == end_itr) { throw std::runtime_error("uneven number of feature tag ids"); } - uint32_t tag_val = tags.varint(); + uint32_t tag_val = static_cast(*start_itr++);; if (layer.values.size() <= tag_val) { throw std::runtime_error("feature referenced out of range value"); } @@ -86,10 +92,11 @@ optional VectorTileFeature::getValue(const std::string& key) const { std::unordered_map VectorTileFeature::getProperties() const { std::unordered_map properties; - pbf tags = tags_pbf; - while (tags) { - uint32_t tag_key = tags.varint(); - uint32_t tag_val = tags.varint(); + auto start_itr = tags_iter.first; + const auto & end_itr = tags_iter.second; + while (start_itr != end_itr) { + uint32_t tag_key = static_cast(*start_itr++); + uint32_t tag_val = static_cast(*start_itr++); properties[layer.keys.at(tag_key)] = layer.values.at(tag_val); } return properties; @@ -100,7 +107,6 @@ uint64_t VectorTileFeature::getID() const { } GeometryCollection VectorTileFeature::getGeometries() const { - pbf data(geometry_pbf); uint8_t cmd = 1; uint32_t length = 0; int32_t x = 0; @@ -111,9 +117,10 @@ GeometryCollection VectorTileFeature::getGeometries() const { lines.emplace_back(); GeometryCoordinates* line = &lines.back(); - while (data.data < data.end) { + auto g_itr = geometry_iter; + while (g_itr.first != g_itr.second) { if (length == 0) { - uint32_t cmd_length = data.varint(); + uint32_t cmd_length = static_cast(*g_itr.first++); cmd = cmd_length & 0x7; length = cmd_length >> 3; } @@ -121,8 +128,8 @@ GeometryCollection VectorTileFeature::getGeometries() const { --length; if (cmd == 1 || cmd == 2) { - x += data.svarint(); - y += data.svarint(); + x += protozero::decode_zigzag32(static_cast(*g_itr.first++)); + y += protozero::decode_zigzag32(static_cast(*g_itr.first++)); if (cmd == 1 && !line->empty()) { // moveTo lines.emplace_back(); @@ -155,14 +162,10 @@ VectorTile::VectorTile(std::shared_ptr data_) util::ptr VectorTile::getLayer(const std::string& name) const { if (!parsed) { parsed = true; - pbf tile_pbf(reinterpret_cast(data->c_str()), data->size()); - while (tile_pbf.next()) { - if (tile_pbf.tag == 3) { // layer - util::ptr layer = std::make_shared(tile_pbf.message()); - layers.emplace(layer->name, layer); - } else { - tile_pbf.skip(); - } + protozero::pbf_reader tile_pbf(*data); + while (tile_pbf.next(3)) { + util::ptr layer = std::make_shared(tile_pbf.get_message()); + layers.emplace(layer->name, layer); } } @@ -174,20 +177,27 @@ util::ptr VectorTile::getLayer(const std::string& name) const return nullptr; } -VectorTileLayer::VectorTileLayer(pbf layer_pbf) { +VectorTileLayer::VectorTileLayer(protozero::pbf_reader layer_pbf) { while (layer_pbf.next()) { - if (layer_pbf.tag == 1) { // name - name = layer_pbf.string(); - } else if (layer_pbf.tag == 2) { // feature - features.push_back(layer_pbf.message()); - } else if (layer_pbf.tag == 3) { // keys - keysMap.emplace(layer_pbf.string(), keysMap.size()); - } else if (layer_pbf.tag == 4) { // values - values.emplace_back(parseValue(layer_pbf.message())); - } else if (layer_pbf.tag == 5) { // extent - extent = layer_pbf.varint(); - } else { + switch (layer_pbf.tag()) { + case 1: // name + name = layer_pbf.get_string(); + break; + case 2: // feature + features.push_back(layer_pbf.get_message()); + break; + case 3: // keys + keysMap.emplace(layer_pbf.get_string(), keysMap.size()); + break; + case 4: // values + values.emplace_back(parseValue(layer_pbf.get_message())); + break; + case 5: // extent + extent = layer_pbf.get_uint32(); + break; + default: layer_pbf.skip(); + break; } } diff --git a/src/mbgl/tile/vector_tile.hpp b/src/mbgl/tile/vector_tile.hpp index 0e583ab33..441d6827a 100644 --- a/src/mbgl/tile/vector_tile.hpp +++ b/src/mbgl/tile/vector_tile.hpp @@ -3,7 +3,7 @@ #include #include -#include +#include #include #include @@ -13,9 +13,12 @@ namespace mbgl { class VectorTileLayer; +using pbf_iter_type = protozero::pbf_reader::const_uint32_iterator; +using packed_iter_type = std::pair; + class VectorTileFeature : public GeometryTileFeature { public: - VectorTileFeature(pbf, const VectorTileLayer&); + VectorTileFeature(protozero::pbf_reader, const VectorTileLayer&); FeatureType getType() const override { return type; } optional getValue(const std::string&) const override; @@ -28,13 +31,13 @@ private: const VectorTileLayer& layer; uint64_t id = 0; FeatureType type = FeatureType::Unknown; - pbf tags_pbf; - pbf geometry_pbf; + packed_iter_type tags_iter; + packed_iter_type geometry_iter; }; class VectorTileLayer : public GeometryTileLayer { public: - VectorTileLayer(pbf); + VectorTileLayer(protozero::pbf_reader); std::size_t featureCount() const override { return features.size(); } util::ptr getFeature(std::size_t) const override; @@ -49,7 +52,7 @@ private: std::map keysMap; std::vector> keys; std::vector values; - std::vector features; + std::vector features; }; class VectorTile : public GeometryTile { diff --git a/src/mbgl/util/pbf.hpp b/src/mbgl/util/pbf.hpp deleted file mode 100644 index d017219a5..000000000 --- a/src/mbgl/util/pbf.hpp +++ /dev/null @@ -1,184 +0,0 @@ -#ifndef MBGL_UTIL_PBF -#define MBGL_UTIL_PBF - -/* - * Some parts are from upb - a minimalist implementation of protocol buffers. - * - * Copyright (c) 2008-2011 Google Inc. See LICENSE for details. - * Author: Josh Haberman - */ - -#include -#include - -namespace mbgl { - -struct pbf { - struct exception : std::exception { const char *what() const noexcept { return "pbf exception"; } }; - struct unterminated_varint_exception : exception { const char *what() const noexcept { return "pbf unterminated varint exception"; } }; - struct varint_too_long_exception : exception { const char *what() const noexcept { return "pbf varint too long exception"; } }; - struct unknown_field_type_exception : exception { const char *what() const noexcept { return "pbf unknown field type exception"; } }; - struct end_of_buffer_exception : exception { const char *what() const noexcept { return "pbf end of buffer exception"; } }; - - inline pbf(const unsigned char *data, size_t length); - inline pbf(); - - inline operator bool() const; - - inline bool next(); - inline bool next(uint32_t tag); - template inline T varint(); - template inline T svarint(); - - template inline T fixed(); - inline float float32(); - inline double float64(); - - inline std::string string(); - inline bool boolean(); - - inline pbf message(); - - inline void skip(); - inline void skipValue(uint32_t val); - inline void skipBytes(uint32_t bytes); - - const uint8_t *data = nullptr; - const uint8_t *end = nullptr; - uint32_t value = 0; - uint32_t tag = 0; -}; - -pbf::pbf(const unsigned char *data_, size_t length) - : data(data_), - end(data_ + length), - value(0), - tag(0) { -} - -pbf::pbf() - : data(nullptr), - end(nullptr), - value(0), - tag(0) { -} - - -pbf::operator bool() const { - return data < end; -} - -bool pbf::next() { - if (data < end) { - value = static_cast(varint()); - tag = value >> 3; - return true; - } - return false; -} - -bool pbf::next(uint32_t requested_tag) { - while (next()) { - if (tag == requested_tag) { - return true; - } else { - skip(); - } - } - return false; -} - -template -T pbf::varint() { - uint8_t byte = 0x80; - T result = 0; - int bitpos; - for (bitpos = 0; bitpos < 70 && (byte & 0x80); bitpos += 7) { - if (data >= end) { - throw unterminated_varint_exception(); - } - result |= ((T)(byte = *data) & 0x7F) << bitpos; - - data++; - } - if (bitpos == 70 && (byte & 0x80)) { - throw varint_too_long_exception(); - } - - return result; -} - -template -T pbf::svarint() { - T n = varint(); - return (n >> 1) ^ -(T)(n & 1); -} - -template -T pbf::fixed() { - skipBytes(bytes); - T result; - memcpy(&result, data - bytes, bytes); - return result; -} - -float pbf::float32() { - return fixed(); -} - -double pbf::float64() { - return fixed(); -} - -std::string pbf::string() { - uint32_t bytes = static_cast(varint()); - const char *string_data = reinterpret_cast(data); - skipBytes(bytes); - return std::string(string_data, bytes); -} - -bool pbf::boolean() { - skipBytes(1); - return *(bool *)(data - 1); -} - -pbf pbf::message() { - uint32_t bytes = static_cast(varint()); - const uint8_t *pos = data; - skipBytes(bytes); - return pbf(pos, bytes); -} - -void pbf::skip() { - skipValue(value); -} - -void pbf::skipValue(uint32_t val) { - switch (val & 0x7) { - case 0: // varint - varint(); - break; - case 1: // 64 bit - skipBytes(8); - break; - case 2: // string/message - skipBytes(static_cast(varint())); - break; - case 5: // 32 bit - skipBytes(4); - break; - default: - throw unknown_field_type_exception(); - } -} - -void pbf::skipBytes(uint32_t bytes) { - if (data + bytes > end) { - throw end_of_buffer_exception(); - } - data += bytes; -} - -} // end namespace mbgl - -#endif diff --git a/test/style/glyph_store.cpp b/test/style/glyph_store.cpp index 2a874a05a..08ec6902b 100644 --- a/test/style/glyph_store.cpp +++ b/test/style/glyph_store.cpp @@ -110,7 +110,7 @@ TEST(GlyphStore, LoadingCorrupted) { EXPECT_EQ(glyphRange, GlyphRange(0, 255)); EXPECT_TRUE(error != nullptr); - EXPECT_EQ(util::toString(error), "pbf unknown field type exception"); + EXPECT_EQ(util::toString(error), "unknown pbf field type exception"); auto glyphSet = test.glyphStore.getGlyphSet({{"Test Stack"}}); ASSERT_TRUE(glyphSet->getSDFs().empty()); diff --git a/test/style/source.cpp b/test/style/source.cpp index 5ec2714b4..7964a0581 100644 --- a/test/style/source.cpp +++ b/test/style/source.cpp @@ -269,7 +269,7 @@ TEST(Source, VectorTileCorrupt) { test.observer.tileError = [&] (Source& source, const TileID& tileID, std::exception_ptr error) { EXPECT_EQ(source.type, SourceType::Vector); EXPECT_EQ(std::string(tileID), "0/0/0"); - EXPECT_EQ(util::toString(error), "pbf unknown field type exception"); + EXPECT_EQ(util::toString(error), "unknown pbf field type exception"); test.end(); }; diff --git a/test/test.gypi b/test/test.gypi index a9e53c88f..2326fd7cf 100644 --- a/test/test.gypi +++ b/test/test.gypi @@ -85,6 +85,7 @@ 'cflags_cc': [ '<@(gtest_cflags)', '<@(opengl_cflags)', + '<@(protozero_cflags)', '<@(boost_cflags)', '<@(sqlite_cflags)', '<@(geojsonvt_cflags)',