From 31fca20c16e70b77b0a2cfaba19c5e8e9e1eb438 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Tue, 28 Jan 2014 13:53:59 +0100 Subject: [PATCH] move vector tile parsing to separate class + iterators --- include/llmr/map/tile.hpp | 14 +-- include/llmr/map/vector_tile.hpp | 68 +++++++++++++ include/llmr/util/pbf.hpp | 11 ++ src/CMakeLists.txt | 2 + src/map/tile.cpp | 141 ++++---------------------- src/map/vector_tile.cpp | 169 +++++++++++++++++++++++++++++++ 6 files changed, 277 insertions(+), 128 deletions(-) create mode 100644 include/llmr/map/vector_tile.hpp create mode 100644 src/map/vector_tile.cpp diff --git a/include/llmr/map/tile.hpp b/include/llmr/map/tile.hpp index c4cd39116..05e872a0a 100644 --- a/include/llmr/map/tile.hpp +++ b/include/llmr/map/tile.hpp @@ -6,6 +6,7 @@ #include "../geometry/line_buffer.hpp" #include "layer.hpp" +#include "vector_tile.hpp" #include #include @@ -22,6 +23,8 @@ class Bucket; class LayerDescription; class BucketDescription; + + class Tile { public: struct exception : std::exception {}; @@ -52,13 +55,9 @@ public: // Other functions void setData(uint8_t *data, uint32_t bytes); bool parse(); - void parseLayers(const pbf& tile, const std::vector& layers); - std::shared_ptr createBucket(const pbf& tile, const BucketDescription& bucket_desc); - // void parseLayer(const pbf layer); - - // void (const std::vector& child_layers); - - std::shared_ptr createFillBucket(const pbf data, const BucketDescription& bucket_desc); + void parseStyleLayers(const std::vector& layers); + std::shared_ptr createBucket(const BucketDescription& bucket_desc); + std::shared_ptr createFillBucket(const VectorTileLayer& layer, const BucketDescription& bucket_desc); void cancel(); @@ -85,6 +84,7 @@ private: // Source data uint8_t *data; uint32_t bytes; + VectorTile tile; std::mutex mtx; diff --git a/include/llmr/map/vector_tile.hpp b/include/llmr/map/vector_tile.hpp new file mode 100644 index 000000000..c0cfc9bfb --- /dev/null +++ b/include/llmr/map/vector_tile.hpp @@ -0,0 +1,68 @@ +#ifndef LLMR_MAP_VECTOR_TILE +#define LLMR_MAP_VECTOR_TILE + +#include +#include +#include +#include +#include + +namespace llmr { + +class BucketDescription; +class VectorTileLayer; + +struct pbf; + +class FilteredVectorTileLayer { +public: + class iterator { + public: + iterator(const FilteredVectorTileLayer& filter, const pbf& data); + void operator++(); + bool operator!=(const iterator& other) const; + const pbf& operator*() const; + + private: + const FilteredVectorTileLayer& filter; + bool valid = false; + pbf feature; + pbf data; + }; + +public: + FilteredVectorTileLayer(const VectorTileLayer& layer, const BucketDescription& bucket_desc); + + iterator begin() const; + iterator end() const; + +private: + const VectorTileLayer& layer; + const BucketDescription& bucket_desc; + int32_t key = -1; + std::set values; +}; + +class VectorTileLayer { +public: + VectorTileLayer(pbf data); + + const pbf data; + std::string name; + uint32_t extent = 4096; + std::vector keys; + std::vector values; +}; + +class VectorTile { +public: + VectorTile(); + VectorTile(pbf data); + VectorTile& operator=(VectorTile&& other); + + std::map layers; +}; + +} + +#endif diff --git a/include/llmr/util/pbf.hpp b/include/llmr/util/pbf.hpp index fbf45e6a1..1813b254f 100644 --- a/include/llmr/util/pbf.hpp +++ b/include/llmr/util/pbf.hpp @@ -29,6 +29,7 @@ struct pbf { inline operator bool() const; inline bool next(); + inline bool next(uint32_t tag); template inline T varint(); template inline T svarint(); @@ -75,6 +76,16 @@ bool pbf::next() { return false; } +bool pbf::next(uint32_t requested_tag) { + while (next()) { + if (tag == requested_tag) { + return true; + } else { + skip(); + } + } + return false; +} template T pbf::varint() { diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 45b861ebb..81d365733 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -5,6 +5,7 @@ SET(llmr_SOURCES map/map.cpp map/tile.cpp map/transform.cpp + map/vector_tile.cpp renderer/painter.cpp renderer/shader-fill.cpp renderer/shader-line.cpp @@ -30,6 +31,7 @@ SET(llmr_HEADERS ../include/llmr/map/map.hpp ../include/llmr/map/tile.hpp ../include/llmr/map/transform.hpp + ../include/llmr/map/vector_tile.hpp ../include/llmr/platform/gl.hpp ../include/llmr/platform/platform.hpp ../include/llmr/renderer/painter.hpp diff --git a/src/map/tile.cpp b/src/map/tile.cpp index 57823d993..4bfd0b527 100644 --- a/src/map/tile.cpp +++ b/src/map/tile.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include using namespace llmr; @@ -95,10 +94,8 @@ bool Tile::parse() { } try { - pbf tile(data, bytes); - // fprintf(stderr, "[%p] parsing tile [%d/%d/%d]...\n", this, z, x, y); - - parseLayers(tile, style.layers); + tile = VectorTile(pbf(data, bytes)); + parseStyleLayers(style.layers); } catch (const std::exception& ex) { fprintf(stderr, "[%p] exception [%d/%d/%d]... failed: %s\n", this, id.z, id.x, id.y, ex.what()); cancel(); @@ -114,12 +111,12 @@ bool Tile::parse() { return true; } -void Tile::parseLayers(const pbf& data, const std::vector& layers) { +void Tile::parseStyleLayers(const std::vector& layers) { for (const LayerDescription& layer_desc : layers) { if (layer_desc.child_layer.size()) { // This is a layer group. // TODO: create framebuffer - parseLayers(data, layer_desc.child_layer); + parseStyleLayers(layer_desc.child_layer); // TODO: render framebuffer on previous framebuffer } else { // This is a singular layer. Check if this bucket already exists. If not, @@ -130,7 +127,7 @@ void Tile::parseLayers(const pbf& data, const std::vector& lay if (bucket_it != style.buckets.end()) { // Only create the new bucket if we have an actual specification // for it. - std::shared_ptr bucket = createBucket(data, bucket_it->second); + std::shared_ptr bucket = createBucket(bucket_it->second); if (bucket) { // Bucket creation might fail because the data tile may not // contain any data that falls into this bucket. @@ -146,30 +143,17 @@ void Tile::parseLayers(const pbf& data, const std::vector& lay } } -std::shared_ptr Tile::createBucket(const pbf& data, const BucketDescription& bucket_desc) { - pbf tile(data); - // TODO: remember data locations in tiles for faster parsing so that we don't - // have to go through the entire vector tile all the time. - while (tile.next()) { - if (tile.tag == 3) { // layer - pbf layer = tile.message(); - while (layer.next()) { - if (layer.tag == 1) { - std::string name = layer.string(); - if (name == bucket_desc.source_layer) { - if (bucket_desc.type == BucketType::Fill) { - return createFillBucket(layer, bucket_desc); - } else { - // TODO: create other bucket types. - } - } - } else { - layer.skip(); - } - } +std::shared_ptr Tile::createBucket(const BucketDescription& bucket_desc) { + auto layer_it = tile.layers.find(bucket_desc.source_layer); + if (layer_it != tile.layers.end()) { + const VectorTileLayer& layer = layer_it->second; + if (bucket_desc.type == BucketType::Fill) { + return createFillBucket(layer, bucket_desc); } else { - tile.skip(); + // TODO: create other bucket types. } + } else { + // The layer specified in the bucket does not exist. Do nothing. } return NULL; @@ -182,99 +166,14 @@ enum geom_type { Polygon = 3 }; -std::shared_ptr Tile::createFillBucket(const pbf data, const BucketDescription& bucket_desc) { - pbf layer; +std::shared_ptr Tile::createFillBucket(const VectorTileLayer& layer, const BucketDescription& bucket_desc) { std::shared_ptr bucket = std::make_shared(fillBuffer); - int32_t key = -1; - std::set values; - - // If we filter further by field/value, parse the key/value indices first - // because protobuf doesn't mandate a particular key order. - if (bucket_desc.source_field.size()) { - uint32_t key_id = 0; - uint32_t value_id = 0; - - // Find out what key/value IDs we need. - layer = data; - while (layer.next()) { - if (layer.tag == 3) { // keys - if (layer.string() == bucket_desc.source_field) { - // We found the key - key = key_id; - } - key_id++; - } else if (layer.tag == 4) { // values - Value value = parseValue(layer.message()); - if (std::find(bucket_desc.source_value.begin(), bucket_desc.source_value.end(), value) != bucket_desc.source_value.end()) { - values.insert(value_id); - } - value_id++; - } else { - layer.skip(); - } - } - - if (key < 0 || values.empty()) { - // There are no valid values that we could possibly find. Abort early. - return bucket; - } - } - - // Now parse the features and optionally filter by key/value IDs. - layer = data; - while (layer.next()) { - if (layer.tag == 2) { // feature - pbf feature = layer.message(); - pbf geometry; - geom_type type = Unknown; - bool skip = false; - - while (!skip && feature.next()) { - if (feature.tag == 2) { // tags - if (key < 0) { - // We want to parse the entire layer anyway - feature.skip(); - } else { - // We only want to parse some features. - skip = true; - // tags are packed varints. They should have an even length. - pbf tags = feature.message(); - while (tags) { - uint32_t tag_key = tags.varint(); - if (tags) { - uint32_t tag_val = tags.varint(); - // Now check if the tag_key/tag_val pair is included - // in the set of key/values that we are looking for. - if (key == tag_key && values.find(tag_val) != values.end()) { - skip = false; - break; - } - } else { - // This should not happen; otherwise the vector tile - // is invalid. - throw std::runtime_error("uneven number of feature tag ids"); - } - } - } - } else if (feature.tag == 3) { // geometry type - type = (geom_type)feature.varint(); - if (type != Polygon) { - skip = true; - break; - } - } else if (feature.tag == 4) { // geometry - geometry = feature.message(); - } else { - feature.skip(); - } - } - - if (!skip && geometry) { - bucket->addGeometry(geometry); - } - } else { - layer.skip(); + FilteredVectorTileLayer filtered_layer(layer, bucket_desc); + for (pbf feature : filtered_layer) { + while (feature.next(4)) { // geometry + pbf geometry_pbf = feature.message(); + bucket->addGeometry(geometry_pbf); } } diff --git a/src/map/vector_tile.cpp b/src/map/vector_tile.cpp new file mode 100644 index 000000000..e8ef7f05b --- /dev/null +++ b/src/map/vector_tile.cpp @@ -0,0 +1,169 @@ +#include +#include + + +using namespace llmr; + +VectorTile::VectorTile() {} + +VectorTile::VectorTile(pbf tile) { + while (tile.next()) { + if (tile.tag == 3) { // layer + VectorTileLayer layer(tile.message()); + layers.emplace(layer.name, layer); + } else { + tile.skip(); + } + } +} + +VectorTile& VectorTile::operator=(VectorTile && other) { + if (this != &other) { + layers.swap(other.layers); + } + return *this; +} + + +VectorTileLayer::VectorTileLayer(pbf layer) : data(layer) { + while (layer.next()) { + if (layer.tag == 1) { // name + name = layer.string(); + } else if (layer.tag == 3) { // keys + keys.emplace_back(layer.string()); + } else if (layer.tag == 4) { // values + values.emplace_back(parseValue(layer.message())); + } else if (layer.tag == 5) { // extent + extent = layer.varint(); + } else { + layer.skip(); + } + } +} + +FilteredVectorTileLayer::FilteredVectorTileLayer(const VectorTileLayer& layer, const BucketDescription& bucket_desc) + : layer(layer), + bucket_desc(bucket_desc) { + // If we filter further by field/value, parse the key/value indices first + // because protobuf doesn't mandate a particular key order. + if (bucket_desc.source_field.size()) { + // Find out what key/value IDs we need. + auto key_it = std::find(layer.keys.begin(), layer.keys.end(), bucket_desc.source_field); + if (key_it != layer.keys.end()) { + key = key_it - layer.keys.begin(); + } + + for (const Value& source_value : bucket_desc.source_value) { + auto value_it = std::find(layer.values.begin(), layer.values.end(), source_value); + if (value_it != layer.values.end()) { + values.insert(value_it - layer.values.begin()); + } + } + } +} + +FilteredVectorTileLayer::iterator FilteredVectorTileLayer::begin() const { + if (bucket_desc.source_field.size() && (key < 0 || values.empty())) { + // There are no valid values that we could possibly find. Abort early. + return end(); + } else { + return iterator(*this, layer.data); + } +} + +FilteredVectorTileLayer::iterator FilteredVectorTileLayer::end() const { + return iterator(*this, pbf(layer.data.end, 0)); +} + +FilteredVectorTileLayer::iterator::iterator(const FilteredVectorTileLayer& filter, const pbf& data) + : filter(filter), + feature(pbf()), + data(data) { + operator++(); +} + +void FilteredVectorTileLayer::iterator::operator++() { + valid = false; + + while (data.next(2)) { // feature + feature = data.message(); + pbf feature_pbf = feature; + BucketType type = BucketType::None; + + bool skip = false; + while (!skip && feature_pbf.next()) { + if (feature_pbf.tag == 2) { // tags + if (filter.key < 0) { + // We want to parse the entire layer anyway + feature_pbf.skip(); + } else { + // We only want to parse some features. + skip = true; + + // tags are packed varints. They should have an even length. + pbf tags_pbf = feature_pbf.message(); + while (tags_pbf) { + uint32_t tag_key = tags_pbf.varint(); + if (tags_pbf) { + uint32_t tag_val = tags_pbf.varint(); + // Now check if the tag_key/tag_val pair is included + // in the set of key/values that we are looking for. + if (filter.key == tag_key && filter.values.find(tag_val) != filter.values.end()) { + skip = false; + break; // feature_pbf loop + } + } else { + // This should not happen; otherwise the vector tile + // is invalid. + throw std::runtime_error("uneven number of feature tag ids"); + } + } + } + } else if (feature_pbf.tag == 3) { // geometry type + switch (feature_pbf.varint()) { + case 1 /* Point */: type = BucketType::Point; break; + case 2 /* LineString */: type = BucketType::Line; break; + case 3 /* Polygon */: type = BucketType::Fill; break; + default: type = BucketType::None; break; + } + + if (type != filter.bucket_desc.type) { + break; // feature_pbf loop early + } + } else { + feature_pbf.skip(); + } + } + + if (!skip && type == filter.bucket_desc.type) { + valid = true; + return; // data loop + } else { + valid = false; + } + } +} + +bool FilteredVectorTileLayer::iterator::operator!=(const iterator& other) const { + return !(data.data == other.data.data && data.end == other.data.end && valid == other.valid); +} + +const pbf& FilteredVectorTileLayer::iterator:: operator*() const { + return feature; +} + +// class FilteredVectorTileLayer { +// public: +// class iterator { +// public: +// void operator++(); +// bool operator!=(const iterator& other) const; +// const VectorTileFeature& operator*() const; + +// private: +// iterator(const pbf& data); +// VectorTileFeature feature; +// pbf data; +// }; + +// };