tei/MapLibreParticleLab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[core] add benchmark for vector tile parsing

Browse Source
This commit is contained in:
Konstantin Käfer
2017-06-21 11:03:00 -07:00
parent 6ec406b6af
commit 4bfc559223
7 changed files with 366 additions and 298 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
benchmark/parse/vector_tile.benchmark.cpp Normal file
View File

@@ -0,0 +1,28 @@
#include <benchmark/benchmark.h>
#include <mbgl/tile/vector_tile_data.hpp>
#include <mbgl/util/io.hpp>
using namespace mbgl;
static void Parse_VectorTile(benchmark::State& state) {
auto data = std::make_shared<std::string>(util::read_file("test/fixtures/api/assets/streets/10-163-395.vector.pbf"));
while (state.KeepRunning()) {
std::size_t length = 0;
VectorTileData tile(data);
for (const auto& name : tile.layerNames()) {
if (auto layer = tile.getLayer(name)) {
const std::size_t count = layer->featureCount();
for (std::size_t i = 0; i < count; i++) {
if (auto feature = layer->getFeature(i)) {
length += feature->getGeometries().size();
length += feature->getProperties().size();
}
}
}
}
}
}
BENCHMARK(Parse_VectorTile);

1
cmake/benchmark-files.cmake
View File

@@ -9,6 +9,7 @@ set(MBGL_BENCHMARK_FILES
# parse
benchmark/parse/filter.benchmark.cpp
benchmark/parse/vector_tile.benchmark.cpp
# src
benchmark/src/main.cpp

1
cmake/benchmark.cmake
View File

@@ -19,6 +19,7 @@ target_link_libraries(mbgl-benchmark
target_add_mason_package(mbgl-benchmark PRIVATE benchmark)
target_add_mason_package(mbgl-benchmark PRIVATE rapidjson)
target_add_mason_package(mbgl-benchmark PRIVATE protozero)
mbgl_platform_benchmark()

2
cmake/core-files.cmake
View File

@@ -513,6 +513,8 @@ set(MBGL_CORE_FILES
src/mbgl/tile/tile_observer.hpp
src/mbgl/tile/vector_tile.cpp
src/mbgl/tile/vector_tile.hpp
src/mbgl/tile/vector_tile_data.cpp
src/mbgl/tile/vector_tile_data.hpp
# util
include/mbgl/util/any.hpp

300
src/mbgl/tile/vector_tile.cpp
View File

@@ -1,85 +1,12 @@
#include <mbgl/tile/vector_tile.hpp>
#include <mbgl/tile/vector_tile_data.hpp>
#include <mbgl/tile/tile_loader_impl.hpp>
#include <mbgl/tile/geometry_tile_data.hpp>
#include <mbgl/renderer/tile_parameters.hpp>
#include <protozero/pbf_reader.hpp>
#include <unordered_map>
#include <functional>
#include <utility>
#include <memory>
namespace mbgl {
class VectorTileLayer;
using packed_iter_type = protozero::iterator_range<protozero::pbf_reader::const_uint32_iterator>;
struct VectorTileLayerData {
VectorTileLayerData(std::shared_ptr<const std::string>);
// Hold a reference to the underlying pbf data that backs the lazily-built
// components of the owning VectorTileLayer and VectorTileFeature objects
std::shared_ptr<const std::string> data;
uint32_t version = 1;
uint32_t extent = 4096;
std::unordered_map<std::string, uint32_t> keysMap;
std::vector<std::reference_wrapper<const std::string>> keys;
std::vector<Value> values;
};
class VectorTileFeature : public GeometryTileFeature {
public:
VectorTileFeature(protozero::pbf_reader, std::shared_ptr<VectorTileLayerData> layerData);
FeatureType getType() const override { return type; }
optional<Value> getValue(const std::string&) const override;
std::unordered_map<std::string,Value> getProperties() const override;
optional<FeatureIdentifier> getID() const override;
GeometryCollection getGeometries() const override;
private:
std::shared_ptr<VectorTileLayerData> layerData;
optional<FeatureIdentifier> id;
FeatureType type = FeatureType::Unknown;
packed_iter_type tags_iter;
packed_iter_type geometry_iter;
};
class VectorTileLayer : public GeometryTileLayer {
public:
VectorTileLayer(protozero::pbf_reader, std::shared_ptr<const std::string>);
std::size_t featureCount() const override { return features.size(); }
std::unique_ptr<GeometryTileFeature> getFeature(std::size_t) const override;
std::string getName() const override;
private:
friend class VectorTileData;
friend class VectorTileFeature;
std::string name;
std::vector<protozero::pbf_reader> features;
std::shared_ptr<VectorTileLayerData> data;
};
class VectorTileData : public GeometryTileData {
public:
VectorTileData(std::shared_ptr<const std::string> data);
std::unique_ptr<GeometryTileData> clone() const override {
return std::make_unique<VectorTileData>(*this);
}
const GeometryTileLayer* getLayer(const std::string&) const override;
private:
std::shared_ptr<const std::string> data;
mutable bool parsed = false;
mutable std::unordered_map<std::string, VectorTileLayer> layers;
};
VectorTile::VectorTile(const OverscaledTileID& id_,
std::string sourceID_,
const TileParameters& parameters,
@@ -101,227 +28,4 @@ void VectorTile::setData(std::shared_ptr<const std::string> data_,
GeometryTile::setData(data_ ? std::make_unique<VectorTileData>(data_) : nullptr);
}
Value parseValue(protozero::pbf_reader data) {
Value value;
while (data.next()) {
switch (data.tag()) {
case 1: // string_value
value = data.get_string();
break;
case 2: // float_value
value = static_cast<double>(data.get_float());
break;
case 3: // double_value
value = data.get_double();
break;
case 4: // int_value
value = data.get_int64();
break;
case 5: // uint_value
value = data.get_uint64();
break;
case 6: // sint_value
value = data.get_sint64();
break;
case 7: // bool_value
value = data.get_bool();
break;
default:
data.skip();
break;
}
}
return value;
}
VectorTileFeature::VectorTileFeature(protozero::pbf_reader feature_pbf, std::shared_ptr<VectorTileLayerData> layerData_)
: layerData(std::move(layerData_)) {
while (feature_pbf.next()) {
switch (feature_pbf.tag()) {
case 1: // id
id = { feature_pbf.get_uint64() };
break;
case 2: // tags
tags_iter = feature_pbf.get_packed_uint32();
break;
case 3: // type
type = static_cast<FeatureType>(feature_pbf.get_enum());
break;
case 4: // geometry
geometry_iter = feature_pbf.get_packed_uint32();
break;
default:
feature_pbf.skip();
break;
}
}
}
optional<Value> VectorTileFeature::getValue(const std::string& key) const {
auto keyIter = layerData->keysMap.find(key);
if (keyIter == layerData->keysMap.end()) {
return optional<Value>();
}
auto start_itr = tags_iter.begin();
const auto & end_itr = tags_iter.end();
while (start_itr != end_itr) {
auto tag_key = static_cast<uint32_t>(*start_itr++);
if (layerData->keysMap.size() <= tag_key) {
throw std::runtime_error("feature referenced out of range key");
}
if (start_itr == end_itr) {
throw std::runtime_error("uneven number of feature tag ids");
}
auto tag_val = static_cast<uint32_t>(*start_itr++);;
if (layerData->values.size() <= tag_val) {
throw std::runtime_error("feature referenced out of range value");
}
if (tag_key == keyIter->second) {
return layerData->values[tag_val];
}
}
return optional<Value>();
}
std::unordered_map<std::string,Value> VectorTileFeature::getProperties() const {
std::unordered_map<std::string,Value> properties;
auto start_itr = tags_iter.begin();
const auto & end_itr = tags_iter.end();
while (start_itr != end_itr) {
auto tag_key = static_cast<uint32_t>(*start_itr++);
if (start_itr == end_itr) {
throw std::runtime_error("uneven number of feature tag ids");
}
auto tag_val = static_cast<uint32_t>(*start_itr++);
properties[layerData->keys.at(tag_key)] = layerData->values.at(tag_val);
}
return properties;
}
optional<FeatureIdentifier> VectorTileFeature::getID() const {
return id;
}
GeometryCollection VectorTileFeature::getGeometries() const {
uint8_t cmd = 1;
uint32_t length = 0;
int32_t x = 0;
int32_t y = 0;
const float scale = float(util::EXTENT) / layerData->extent;
GeometryCollection lines;
lines.emplace_back();
GeometryCoordinates* line = &lines.back();
auto g_itr = geometry_iter.begin();
while (g_itr != geometry_iter.end()) {
if (length == 0) {
auto cmd_length = static_cast<uint32_t>(*g_itr++);
cmd = cmd_length & 0x7;
length = cmd_length >> 3;
}
--length;
if (cmd == 1 || cmd == 2) {
x += protozero::decode_zigzag32(static_cast<uint32_t>(*g_itr++));
y += protozero::decode_zigzag32(static_cast<uint32_t>(*g_itr++));
if (cmd == 1 && !line->empty()) { // moveTo
lines.emplace_back();
line = &lines.back();
}
line->emplace_back(::round(x * scale), ::round(y * scale));
} else if (cmd == 7) { // closePolygon
if (!line->empty()) {
line->push_back((*line)[0]);
}
} else {
throw std::runtime_error("unknown command");
}
}
if (layerData->version >= 2 || type != FeatureType::Polygon) {
return lines;
}
return fixupPolygons(lines);
}
VectorTileData::VectorTileData(std::shared_ptr<const std::string> data_)
: data(std::move(data_)) {
}
const GeometryTileLayer* VectorTileData::getLayer(const std::string& name) const {
if (!parsed) {
parsed = true;
protozero::pbf_reader tile_pbf(*data);
while (tile_pbf.next(3)) {
VectorTileLayer layer(tile_pbf.get_message(), data);
layers.emplace(layer.name, std::move(layer));
}
}
auto it = layers.find(name);
if (it != layers.end()) {
return &it->second;
}
return nullptr;
}
VectorTileLayerData::VectorTileLayerData(std::shared_ptr<const std::string> pbfData) :
data(std::move(pbfData))
{}
VectorTileLayer::VectorTileLayer(protozero::pbf_reader layer_pbf, std::shared_ptr<const std::string> pbfData)
: data(std::make_shared<VectorTileLayerData>(std::move(pbfData)))
{
while (layer_pbf.next()) {
switch (layer_pbf.tag()) {
case 1: // name
name = layer_pbf.get_string();
break;
case 2: // feature
features.push_back(layer_pbf.get_message());
break;
case 3: // keys
{
auto iter = data->keysMap.emplace(layer_pbf.get_string(), data->keysMap.size());
data->keys.emplace_back(std::reference_wrapper<const std::string>(iter.first->first));
}
break;
case 4: // values
data->values.emplace_back(parseValue(layer_pbf.get_message()));
break;
case 5: // extent
data->extent = layer_pbf.get_uint32();
break;
case 15: // version
data->version = layer_pbf.get_uint32();
break;
default:
layer_pbf.skip();
break;
}
}
}
std::unique_ptr<GeometryTileFeature> VectorTileLayer::getFeature(std::size_t i) const {
return std::make_unique<VectorTileFeature>(features.at(i), data);
}
std::string VectorTileLayer::getName() const {
return name;
}
} // namespace mbgl

247
src/mbgl/tile/vector_tile_data.cpp Normal file
View File

@@ -0,0 +1,247 @@
#include <mbgl/tile/vector_tile_data.hpp>
#include <mbgl/util/constants.hpp>
namespace mbgl {
Value parseValue(protozero::pbf_reader data) {
Value value;
while (data.next()) {
switch (data.tag()) {
case 1: // string_value
value = data.get_string();
break;
case 2: // float_value
value = static_cast<double>(data.get_float());
break;
case 3: // double_value
value = data.get_double();
break;
case 4: // int_value
value = data.get_int64();
break;
case 5: // uint_value
value = data.get_uint64();
break;
case 6: // sint_value
value = data.get_sint64();
break;
case 7: // bool_value
value = data.get_bool();
break;
default:
data.skip();
break;
}
}
return value;
}
VectorTileFeature::VectorTileFeature(protozero::pbf_reader feature_pbf, std::shared_ptr<VectorTileLayerData> layerData_)
: layerData(std::move(layerData_)) {
while (feature_pbf.next()) {
switch (feature_pbf.tag()) {
case 1: // id
id = { feature_pbf.get_uint64() };
break;
case 2: // tags
tags_iter = feature_pbf.get_packed_uint32();
break;
case 3: // type
type = static_cast<FeatureType>(feature_pbf.get_enum());
break;
case 4: // geometry
geometry_iter = feature_pbf.get_packed_uint32();
break;
default:
feature_pbf.skip();
break;
}
}
}
optional<Value> VectorTileFeature::getValue(const std::string& key) const {
auto keyIter = layerData->keysMap.find(key);
if (keyIter == layerData->keysMap.end()) {
return optional<Value>();
}
auto start_itr = tags_iter.begin();
const auto & end_itr = tags_iter.end();
while (start_itr != end_itr) {
auto tag_key = static_cast<uint32_t>(*start_itr++);
if (layerData->keysMap.size() <= tag_key) {
throw std::runtime_error("feature referenced out of range key");
}
if (start_itr == end_itr) {
throw std::runtime_error("uneven number of feature tag ids");
}
auto tag_val = static_cast<uint32_t>(*start_itr++);;
if (layerData->values.size() <= tag_val) {
throw std::runtime_error("feature referenced out of range value");
}
if (tag_key == keyIter->second) {
return layerData->values[tag_val];
}
}
return optional<Value>();
}
std::unordered_map<std::string,Value> VectorTileFeature::getProperties() const {
std::unordered_map<std::string,Value> properties;
auto start_itr = tags_iter.begin();
const auto & end_itr = tags_iter.end();
while (start_itr != end_itr) {
auto tag_key = static_cast<uint32_t>(*start_itr++);
if (start_itr == end_itr) {
throw std::runtime_error("uneven number of feature tag ids");
}
auto tag_val = static_cast<uint32_t>(*start_itr++);
properties[layerData->keys.at(tag_key)] = layerData->values.at(tag_val);
}
return properties;
}
optional<FeatureIdentifier> VectorTileFeature::getID() const {
return id;
}
GeometryCollection VectorTileFeature::getGeometries() const {
uint8_t cmd = 1;
uint32_t length = 0;
int32_t x = 0;
int32_t y = 0;
const float scale = float(util::EXTENT) / layerData->extent;
GeometryCollection lines;
lines.emplace_back();
GeometryCoordinates* line = &lines.back();
auto g_itr = geometry_iter.begin();
while (g_itr != geometry_iter.end()) {
if (length == 0) {
auto cmd_length = static_cast<uint32_t>(*g_itr++);
cmd = cmd_length & 0x7;
length = cmd_length >> 3;
}
--length;
if (cmd == 1 || cmd == 2) {
x += protozero::decode_zigzag32(static_cast<uint32_t>(*g_itr++));
y += protozero::decode_zigzag32(static_cast<uint32_t>(*g_itr++));
if (cmd == 1 && !line->empty()) { // moveTo
lines.emplace_back();
line = &lines.back();
}
line->emplace_back(::round(x * scale), ::round(y * scale));
} else if (cmd == 7) { // closePolygon
if (!line->empty()) {
line->push_back((*line)[0]);
}
} else {
throw std::runtime_error("unknown command");
}
}
if (layerData->version >= 2 || type != FeatureType::Polygon) {
return lines;
}
return fixupPolygons(lines);
}
VectorTileData::VectorTileData(std::shared_ptr<const std::string> data_)
: data(std::move(data_)) {
}
const GeometryTileLayer* VectorTileData::getLayer(const std::string& name) const {
if (!parsed) {
parse();
}
auto it = layers.find(name);
if (it != layers.end()) {
return &it->second;
}
return nullptr;
}
std::vector<std::string> VectorTileData::layerNames() const {
if (!parsed) {
parse();
}
std::vector<std::string> names;
names.reserve(layers.size());
for (auto const& layer : layers) {
names.emplace_back(layer.first);
}
return names;
}
void VectorTileData::parse() const {
parsed = true;
layers.clear();
protozero::pbf_reader tile_pbf(*data);
while (tile_pbf.next(3)) {
VectorTileLayer layer(tile_pbf.get_message(), data);
layers.emplace(layer.name, std::move(layer));
}
}
VectorTileLayerData::VectorTileLayerData(std::shared_ptr<const std::string> pbfData) :
data(std::move(pbfData))
{}
VectorTileLayer::VectorTileLayer(protozero::pbf_reader layer_pbf, std::shared_ptr<const std::string> pbfData)
: data(std::make_shared<VectorTileLayerData>(std::move(pbfData)))
{
while (layer_pbf.next()) {
switch (layer_pbf.tag()) {
case 1: // name
name = layer_pbf.get_string();
break;
case 2: // feature
features.push_back(layer_pbf.get_message());
break;
case 3: // keys
{
auto iter = data->keysMap.emplace(layer_pbf.get_string(), data->keysMap.size());
data->keys.emplace_back(std::reference_wrapper<const std::string>(iter.first->first));
}
break;
case 4: // values
data->values.emplace_back(parseValue(layer_pbf.get_message()));
break;
case 5: // extent
data->extent = layer_pbf.get_uint32();
break;
case 15: // version
data->version = layer_pbf.get_uint32();
break;
default:
layer_pbf.skip();
break;
}
}
}
std::unique_ptr<GeometryTileFeature> VectorTileLayer::getFeature(std::size_t i) const {
return std::make_unique<VectorTileFeature>(features.at(i), data);
}
std::string VectorTileLayer::getName() const {
return name;
}
} // namespace mbgl

85
src/mbgl/tile/vector_tile_data.hpp Normal file
View File

@@ -0,0 +1,85 @@
#include <mbgl/tile/geometry_tile_data.hpp>
#include <protozero/pbf_reader.hpp>
#include <unordered_map>
#include <functional>
#include <utility>
namespace mbgl {
class VectorTileLayer;
using packed_iter_type = protozero::iterator_range<protozero::pbf_reader::const_uint32_iterator>;
struct VectorTileLayerData {
VectorTileLayerData(std::shared_ptr<const std::string>);
// Hold a reference to the underlying pbf data that backs the lazily-built
// components of the owning VectorTileLayer and VectorTileFeature objects
std::shared_ptr<const std::string> data;
uint32_t version = 1;
uint32_t extent = 4096;
std::unordered_map<std::string, uint32_t> keysMap;
std::vector<std::reference_wrapper<const std::string>> keys;
std::vector<Value> values;
};
class VectorTileFeature : public GeometryTileFeature {
public:
VectorTileFeature(protozero::pbf_reader, std::shared_ptr<VectorTileLayerData> layerData);
FeatureType getType() const override { return type; }
optional<Value> getValue(const std::string&) const override;
std::unordered_map<std::string,Value> getProperties() const override;
optional<FeatureIdentifier> getID() const override;
GeometryCollection getGeometries() const override;
private:
std::shared_ptr<VectorTileLayerData> layerData;
optional<FeatureIdentifier> id;
FeatureType type = FeatureType::Unknown;
packed_iter_type tags_iter;
packed_iter_type geometry_iter;
};
class VectorTileLayer : public GeometryTileLayer {
public:
VectorTileLayer(protozero::pbf_reader, std::shared_ptr<const std::string>);
std::size_t featureCount() const override { return features.size(); }
std::unique_ptr<GeometryTileFeature> getFeature(std::size_t) const override;
std::string getName() const override;
private:
friend class VectorTileData;
friend class VectorTileFeature;
std::string name;
std::vector<protozero::pbf_reader> features;
std::shared_ptr<VectorTileLayerData> data;
};
class VectorTileData : public GeometryTileData {
public:
VectorTileData(std::shared_ptr<const std::string> data);
std::unique_ptr<GeometryTileData> clone() const override {
return std::make_unique<VectorTileData>(*this);
}
const GeometryTileLayer* getLayer(const std::string&) const override;
std::vector<std::string> layerNames() const;
private:
void parse() const;
private:
std::shared_ptr<const std::string> data;
mutable bool parsed = false;
mutable std::unordered_map<std::string, VectorTileLayer> layers;
};
} // namespace mbgl