tei/MapLibreParticleLab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[core] port to protozero (#4906)

Browse Source
This commit is contained in:
Dane Springmeyer
2016-05-03 13:57:17 -07:00
parent 7d1a4c5ec1
commit c82b02c508
14 changed files with 119 additions and 280 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
configure vendored
View File

@@ -97,6 +97,7 @@ CONFIG+=" 'variables': {"$LN
print_opengl_flags
print_qt_flags
print_flags protozero cflags
print_flags boost cflags
print_flags boost_libprogram_options static_libs
print_flags openssl static_libs cflags ldflags

1
mbgl.gypi
View File

@@ -172,6 +172,7 @@
'variables': {
'cflags_cc': [
'<@(opengl_cflags)',
'<@(protozero_cflags)',
'<@(boost_cflags)',
'<@(geojsonvt_cflags)',
'<@(rapidjson_cflags)',

1
platform/android/scripts/configure.sh
View File

@@ -1,5 +1,6 @@
#!/usr/bin/env bash
PROTOZERO_VERSION=1.3.0
BOOST_VERSION=1.60.0
LIBPNG_VERSION=1.6.20
LIBJPEG_TURBO_VERSION=1.4.2

1
platform/ios/scripts/configure.sh
View File

@@ -1,5 +1,6 @@
#!/usr/bin/env bash
PROTOZERO_VERSION=1.3.0
BOOST_VERSION=1.60.0
SQLITE_VERSION=system
ZLIB_VERSION=system

1
platform/linux/scripts/configure.sh
View File

@@ -1,5 +1,6 @@
#!/usr/bin/env bash
PROTOZERO_VERSION=1.3.0
BOOST_VERSION=1.60.0
BOOST_LIBPROGRAM_OPTIONS_VERSION=1.60.0
LIBCURL_VERSION=system

1
platform/osx/scripts/configure.sh
View File

@@ -1,5 +1,6 @@
#!/usr/bin/env bash
PROTOZERO_VERSION=1.3.0
BOOST_VERSION=1.60.0
BOOST_LIBPROGRAM_OPTIONS_VERSION=1.60.0
GLFW_VERSION=3.1.2

1
platform/qt/scripts/configure.sh
View File

@@ -1,5 +1,6 @@
#!/usr/bin/env bash
PROTOZERO_VERSION=1.3.0
BOOST_VERSION=1.60.0
GEOJSONVT_VERSION=4.1.2
GTEST_VERSION=1.7.0

70
src/mbgl/text/glyph_pbf.cpp
View File

@@ -6,52 +6,54 @@
#include <mbgl/text/glyph_store.hpp>
#include <mbgl/text/glyph_set.hpp>
#include <mbgl/util/exception.hpp>
#include <mbgl/util/pbf.hpp>
#include <mbgl/util/string.hpp>
#include <mbgl/util/token.hpp>
#include <mbgl/util/url.hpp>
#include <protozero/pbf_reader.hpp>
namespace {
void parseGlyphPBF(mbgl::GlyphSet& glyphSet, const std::string& data) {
mbgl::pbf glyphs_pbf(reinterpret_cast<const uint8_t *>(data.data()), data.size());
protozero::pbf_reader glyphs_pbf(data);
while (glyphs_pbf.next()) {
if (glyphs_pbf.tag == 1) { // stacks
mbgl::pbf fontstack_pbf = glyphs_pbf.message();
while (fontstack_pbf.next()) {
if (fontstack_pbf.tag == 3) { // glyphs
mbgl::pbf glyph_pbf = fontstack_pbf.message();
while (glyphs_pbf.next(1)) {
auto fontstack_pbf = glyphs_pbf.get_message();
while (fontstack_pbf.next(3)) {
auto glyph_pbf = fontstack_pbf.get_message();
mbgl::SDFGlyph glyph;
mbgl::SDFGlyph glyph;
while (glyph_pbf.next()) {
if (glyph_pbf.tag == 1) { // id
glyph.id = glyph_pbf.varint();
} else if (glyph_pbf.tag == 2) { // bitmap
glyph.bitmap = glyph_pbf.string();
} else if (glyph_pbf.tag == 3) { // width
glyph.metrics.width = glyph_pbf.varint();
} else if (glyph_pbf.tag == 4) { // height
glyph.metrics.height = glyph_pbf.varint();
} else if (glyph_pbf.tag == 5) { // left
glyph.metrics.left = glyph_pbf.svarint();
} else if (glyph_pbf.tag == 6) { // top
glyph.metrics.top = glyph_pbf.svarint();
} else if (glyph_pbf.tag == 7) { // advance
glyph.metrics.advance = glyph_pbf.varint();
} else {
glyph_pbf.skip();
}
}
glyphSet.insert(glyph.id, glyph);
} else {
fontstack_pbf.skip();
while (glyph_pbf.next()) {
switch (glyph_pbf.tag()) {
case 1: // id
glyph.id = glyph_pbf.get_uint32();
break;
case 2: // bitmap
glyph.bitmap = glyph_pbf.get_string();
break;
case 3: // width
glyph.metrics.width = glyph_pbf.get_uint32();
break;
case 4: // height
glyph.metrics.height = glyph_pbf.get_uint32();
break;
case 5: // left
glyph.metrics.left = glyph_pbf.get_sint32();
break;
case 6: // top
glyph.metrics.top = glyph_pbf.get_sint32();
break;
case 7: // advance
glyph.metrics.advance = glyph_pbf.get_uint32();
break;
default:
glyph_pbf.skip();
break;
}
}
} else {
glyphs_pbf.skip();
glyphSet.insert(glyph.id, glyph);
}
}
}

118
src/mbgl/tile/vector_tile.cpp
View File

@@ -9,25 +9,24 @@
namespace mbgl {
Value parseValue(pbf data) {
Value parseValue(protozero::pbf_reader data) {
while (data.next())
{
switch (data.tag)
{
switch (data.tag()) {
case 1: // string_value
return data.string();
return data.get_string();
case 2: // float_value
return static_cast<double>(data.float32());
return static_cast<double>(data.get_float());
case 3: // double_value
return data.float64();
return data.get_double();
case 4: // int_value
return data.varint<int64_t>();
return data.get_int64();
case 5: // uint_value
return data.varint<uint64_t>();
return data.get_uint64();
case 6: // sint_value
return data.svarint<int64_t>();
return data.get_sint64();
case 7: // bool_value
return data.boolean();
return data.get_bool();
default:
data.skip();
break;
@@ -36,19 +35,25 @@ Value parseValue(pbf data) {
return false;
}
VectorTileFeature::VectorTileFeature(pbf feature_pbf, const VectorTileLayer& layer_)
VectorTileFeature::VectorTileFeature(protozero::pbf_reader feature_pbf, const VectorTileLayer& layer_)
: layer(layer_) {
while (feature_pbf.next()) {
if (feature_pbf.tag == 1) { // id
id = feature_pbf.varint<uint64_t>();
} else if (feature_pbf.tag == 2) { // tags
tags_pbf = feature_pbf.message();
} else if (feature_pbf.tag == 3) { // type
type = (FeatureType)feature_pbf.varint();
} else if (feature_pbf.tag == 4) { // geometry
geometry_pbf = feature_pbf.message();
} else {
switch (feature_pbf.tag()) {
case 1: // id
id = feature_pbf.get_uint64();
break;
case 2: // tags
tags_iter = feature_pbf.get_packed_uint32();
break;
case 3: // type
type = static_cast<FeatureType>(feature_pbf.get_enum());
break;
case 4: // geometry
geometry_iter = feature_pbf.get_packed_uint32();
break;
default:
feature_pbf.skip();
break;
}
}
}
@@ -59,19 +64,20 @@ optional<Value> VectorTileFeature::getValue(const std::string& key) const {
return optional<Value>();
}
pbf tags = tags_pbf;
while (tags) {
uint32_t tag_key = tags.varint();
auto start_itr = tags_iter.first;
const auto & end_itr = tags_iter.second;
while (start_itr != end_itr) {
uint32_t tag_key = static_cast<uint32_t>(*start_itr++);
if (layer.keysMap.size() <= tag_key) {
throw std::runtime_error("feature referenced out of range key");
}
if (!tags) {
if (start_itr == end_itr) {
throw std::runtime_error("uneven number of feature tag ids");
}
uint32_t tag_val = tags.varint();
uint32_t tag_val = static_cast<uint32_t>(*start_itr++);;
if (layer.values.size() <= tag_val) {
throw std::runtime_error("feature referenced out of range value");
}
@@ -86,10 +92,11 @@ optional<Value> VectorTileFeature::getValue(const std::string& key) const {
std::unordered_map<std::string,Value> VectorTileFeature::getProperties() const {
std::unordered_map<std::string,Value> properties;
pbf tags = tags_pbf;
while (tags) {
uint32_t tag_key = tags.varint();
uint32_t tag_val = tags.varint();
auto start_itr = tags_iter.first;
const auto & end_itr = tags_iter.second;
while (start_itr != end_itr) {
uint32_t tag_key = static_cast<uint32_t>(*start_itr++);
uint32_t tag_val = static_cast<uint32_t>(*start_itr++);
properties[layer.keys.at(tag_key)] = layer.values.at(tag_val);
}
return properties;
@@ -100,7 +107,6 @@ uint64_t VectorTileFeature::getID() const {
}
GeometryCollection VectorTileFeature::getGeometries() const {
pbf data(geometry_pbf);
uint8_t cmd = 1;
uint32_t length = 0;
int32_t x = 0;
@@ -111,9 +117,10 @@ GeometryCollection VectorTileFeature::getGeometries() const {
lines.emplace_back();
GeometryCoordinates* line = &lines.back();
while (data.data < data.end) {
auto g_itr = geometry_iter;
while (g_itr.first != g_itr.second) {
if (length == 0) {
uint32_t cmd_length = data.varint();
uint32_t cmd_length = static_cast<uint32_t>(*g_itr.first++);
cmd = cmd_length & 0x7;
length = cmd_length >> 3;
}
@@ -121,8 +128,8 @@ GeometryCollection VectorTileFeature::getGeometries() const {
--length;
if (cmd == 1 || cmd == 2) {
x += data.svarint();
y += data.svarint();
x += protozero::decode_zigzag32(static_cast<uint32_t>(*g_itr.first++));
y += protozero::decode_zigzag32(static_cast<uint32_t>(*g_itr.first++));
if (cmd == 1 && !line->empty()) { // moveTo
lines.emplace_back();
@@ -155,14 +162,10 @@ VectorTile::VectorTile(std::shared_ptr<const std::string> data_)
util::ptr<GeometryTileLayer> VectorTile::getLayer(const std::string& name) const {
if (!parsed) {
parsed = true;
pbf tile_pbf(reinterpret_cast<const unsigned char *>(data->c_str()), data->size());
while (tile_pbf.next()) {
if (tile_pbf.tag == 3) { // layer
util::ptr<VectorTileLayer> layer = std::make_shared<VectorTileLayer>(tile_pbf.message());
layers.emplace(layer->name, layer);
} else {
tile_pbf.skip();
}
protozero::pbf_reader tile_pbf(*data);
while (tile_pbf.next(3)) {
util::ptr<VectorTileLayer> layer = std::make_shared<VectorTileLayer>(tile_pbf.get_message());
layers.emplace(layer->name, layer);
}
}
@@ -174,20 +177,27 @@ util::ptr<GeometryTileLayer> VectorTile::getLayer(const std::string& name) const
return nullptr;
}
VectorTileLayer::VectorTileLayer(pbf layer_pbf) {
VectorTileLayer::VectorTileLayer(protozero::pbf_reader layer_pbf) {
while (layer_pbf.next()) {
if (layer_pbf.tag == 1) { // name
name = layer_pbf.string();
} else if (layer_pbf.tag == 2) { // feature
features.push_back(layer_pbf.message());
} else if (layer_pbf.tag == 3) { // keys
keysMap.emplace(layer_pbf.string(), keysMap.size());
} else if (layer_pbf.tag == 4) { // values
values.emplace_back(parseValue(layer_pbf.message()));
} else if (layer_pbf.tag == 5) { // extent
extent = layer_pbf.varint();
} else {
switch (layer_pbf.tag()) {
case 1: // name
name = layer_pbf.get_string();
break;
case 2: // feature
features.push_back(layer_pbf.get_message());
break;
case 3: // keys
keysMap.emplace(layer_pbf.get_string(), keysMap.size());
break;
case 4: // values
values.emplace_back(parseValue(layer_pbf.get_message()));
break;
case 5: // extent
extent = layer_pbf.get_uint32();
break;
default:
layer_pbf.skip();
break;
}
}

15
src/mbgl/tile/vector_tile.hpp
View File

@@ -3,7 +3,7 @@
#include <mbgl/tile/geometry_tile.hpp>
#include <mbgl/map/tile_id.hpp>
#include <mbgl/util/pbf.hpp>
#include <protozero/pbf_reader.hpp>
#include <map>
#include <unordered_map>
@@ -13,9 +13,12 @@ namespace mbgl {
class VectorTileLayer;
using pbf_iter_type = protozero::pbf_reader::const_uint32_iterator;
using packed_iter_type = std::pair<pbf_iter_type,pbf_iter_type>;
class VectorTileFeature : public GeometryTileFeature {
public:
VectorTileFeature(pbf, const VectorTileLayer&);
VectorTileFeature(protozero::pbf_reader, const VectorTileLayer&);
FeatureType getType() const override { return type; }
optional<Value> getValue(const std::string&) const override;
@@ -28,13 +31,13 @@ private:
const VectorTileLayer& layer;
uint64_t id = 0;
FeatureType type = FeatureType::Unknown;
pbf tags_pbf;
pbf geometry_pbf;
packed_iter_type tags_iter;
packed_iter_type geometry_iter;
};
class VectorTileLayer : public GeometryTileLayer {
public:
VectorTileLayer(pbf);
VectorTileLayer(protozero::pbf_reader);
std::size_t featureCount() const override { return features.size(); }
util::ptr<const GeometryTileFeature> getFeature(std::size_t) const override;
@@ -49,7 +52,7 @@ private:
std::map<std::string, uint32_t> keysMap;
std::vector<std::reference_wrapper<const std::string>> keys;
std::vector<Value> values;
std::vector<pbf> features;
std::vector<protozero::pbf_reader> features;
};
class VectorTile : public GeometryTile {

184
src/mbgl/util/pbf.hpp
View File

@@ -1,184 +0,0 @@
#ifndef MBGL_UTIL_PBF
#define MBGL_UTIL_PBF
/*
* Some parts are from upb - a minimalist implementation of protocol buffers.
*
* Copyright (c) 2008-2011 Google Inc. See LICENSE for details.
* Author: Josh Haberman <jhaberman@gmail.com>
*/
#include <string>
#include <cstring>
namespace mbgl {
struct pbf {
struct exception : std::exception { const char *what() const noexcept { return "pbf exception"; } };
struct unterminated_varint_exception : exception { const char *what() const noexcept { return "pbf unterminated varint exception"; } };
struct varint_too_long_exception : exception { const char *what() const noexcept { return "pbf varint too long exception"; } };
struct unknown_field_type_exception : exception { const char *what() const noexcept { return "pbf unknown field type exception"; } };
struct end_of_buffer_exception : exception { const char *what() const noexcept { return "pbf end of buffer exception"; } };
inline pbf(const unsigned char *data, size_t length);
inline pbf();
inline operator bool() const;
inline bool next();
inline bool next(uint32_t tag);
template <typename T = uint32_t> inline T varint();
template <typename T = uint32_t> inline T svarint();
template <typename T = uint32_t, int bytes = 4> inline T fixed();
inline float float32();
inline double float64();
inline std::string string();
inline bool boolean();
inline pbf message();
inline void skip();
inline void skipValue(uint32_t val);
inline void skipBytes(uint32_t bytes);
const uint8_t *data = nullptr;
const uint8_t *end = nullptr;
uint32_t value = 0;
uint32_t tag = 0;
};
pbf::pbf(const unsigned char *data_, size_t length)
: data(data_),
end(data_ + length),
value(0),
tag(0) {
}
pbf::pbf()
: data(nullptr),
end(nullptr),
value(0),
tag(0) {
}
pbf::operator bool() const {
return data < end;
}
bool pbf::next() {
if (data < end) {
value = static_cast<uint32_t>(varint());
tag = value >> 3;
return true;
}
return false;
}
bool pbf::next(uint32_t requested_tag) {
while (next()) {
if (tag == requested_tag) {
return true;
} else {
skip();
}
}
return false;
}
template <typename T>
T pbf::varint() {
uint8_t byte = 0x80;
T result = 0;
int bitpos;
for (bitpos = 0; bitpos < 70 && (byte & 0x80); bitpos += 7) {
if (data >= end) {
throw unterminated_varint_exception();
}
result |= ((T)(byte = *data) & 0x7F) << bitpos;
data++;
}
if (bitpos == 70 && (byte & 0x80)) {
throw varint_too_long_exception();
}
return result;
}
template <typename T>
T pbf::svarint() {
T n = varint<T>();
return (n >> 1) ^ -(T)(n & 1);
}
template <typename T, int bytes>
T pbf::fixed() {
skipBytes(bytes);
T result;
memcpy(&result, data - bytes, bytes);
return result;
}
float pbf::float32() {
return fixed<float, 4>();
}
double pbf::float64() {
return fixed<double, 8>();
}
std::string pbf::string() {
uint32_t bytes = static_cast<uint32_t>(varint());
const char *string_data = reinterpret_cast<const char*>(data);
skipBytes(bytes);
return std::string(string_data, bytes);
}
bool pbf::boolean() {
skipBytes(1);
return *(bool *)(data - 1);
}
pbf pbf::message() {
uint32_t bytes = static_cast<uint32_t>(varint());
const uint8_t *pos = data;
skipBytes(bytes);
return pbf(pos, bytes);
}
void pbf::skip() {
skipValue(value);
}
void pbf::skipValue(uint32_t val) {
switch (val & 0x7) {
case 0: // varint
varint();
break;
case 1: // 64 bit
skipBytes(8);
break;
case 2: // string/message
skipBytes(static_cast<uint32_t>(varint()));
break;
case 5: // 32 bit
skipBytes(4);
break;
default:
throw unknown_field_type_exception();
}
}
void pbf::skipBytes(uint32_t bytes) {
if (data + bytes > end) {
throw end_of_buffer_exception();
}
data += bytes;
}
} // end namespace mbgl
#endif

2
test/style/glyph_store.cpp
View File

@@ -110,7 +110,7 @@ TEST(GlyphStore, LoadingCorrupted) {
EXPECT_EQ(glyphRange, GlyphRange(0, 255));
EXPECT_TRUE(error != nullptr);
EXPECT_EQ(util::toString(error), "pbf unknown field type exception");
EXPECT_EQ(util::toString(error), "unknown pbf field type exception");
auto glyphSet = test.glyphStore.getGlyphSet({{"Test Stack"}});
ASSERT_TRUE(glyphSet->getSDFs().empty());

2
test/style/source.cpp
View File

@@ -269,7 +269,7 @@ TEST(Source, VectorTileCorrupt) {
test.observer.tileError = [&] (Source& source, const TileID& tileID, std::exception_ptr error) {
EXPECT_EQ(source.type, SourceType::Vector);
EXPECT_EQ(std::string(tileID), "0/0/0");
EXPECT_EQ(util::toString(error), "pbf unknown field type exception");
EXPECT_EQ(util::toString(error), "unknown pbf field type exception");
test.end();
};

1
test/test.gypi
View File

@@ -85,6 +85,7 @@
'cflags_cc': [
'<@(gtest_cflags)',
'<@(opengl_cflags)',
'<@(protozero_cflags)',
'<@(boost_cflags)',
'<@(sqlite_cflags)',
'<@(geojsonvt_cflags)',