From eb377aced7fc6df31cddd78d0bd74085b64880c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Wed, 28 May 2014 15:50:54 +0200 Subject: [PATCH 01/29] add font names --- bin/style.js | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/bin/style.js b/bin/style.js index bc548ac29..1a6aca8b7 100644 --- a/bin/style.js +++ b/bin/style.js @@ -267,6 +267,7 @@ module.exports = { "value": 10, "path": "curve", "text_field": "ele", + "font": "Open Sans Regular, Arial Unicode MS Regular", "fontSize": 10, "feature_type": "line", "type": "text" @@ -648,6 +649,7 @@ module.exports = { "layer": "country_label", "text_field": "name", "path": "horizontal", + "font": "Open Sans Regular, Arial Unicode MS Regular", "fontSize": 24, "feature_type": "point", "type": "text" @@ -664,6 +666,7 @@ module.exports = { "type": "text", "text_field": "name", "path": "curve", + "font": "Open Sans Regular, Arial Unicode MS Regular", "fontSize": 16 }, "marine_label_point_1": { @@ -675,6 +678,7 @@ module.exports = { "path": "horizontal", "field": "labelrank", "value": 1, + "font": "Open Sans Regular, Arial Unicode MS Regular", "fontSize": 30 }, "marine_label_point_2": { @@ -686,6 +690,7 @@ module.exports = { "path": "horizontal", "field": "labelrank", "value": 2, + "font": "Open Sans Regular, Arial Unicode MS Regular", "fontSize": 24 }, "marine_label_point_other": { @@ -697,6 +702,7 @@ module.exports = { "path": "horizontal", "field": "labelrank", "value": [3,4,5,6], + "font": "Open Sans Regular, Arial Unicode MS Regular", "fontSize": 18 }, "state_label": { @@ -704,6 +710,7 @@ module.exports = { "layer": "state_label", "text_field": "name", "path": "horizontal", + "font": "Open Sans Regular, Arial Unicode MS Regular", "fontSize": 16, "feature_type": "point", "type": "text", @@ -724,6 +731,7 @@ module.exports = { "value": "city", "text_field": "name", "path": "horizontal", + "font": "Open Sans Regular, Arial Unicode MS Regular", "fontSize": 20, "feature_type": "point", "type": "text" @@ -735,6 +743,7 @@ module.exports = { "value": "town", "text_field": "name", "path": "horizontal", + "font": "Open Sans Regular, Arial Unicode MS Regular", "fontSize": 24, "feature_type": "point", "type": "text" @@ -746,6 +755,7 @@ module.exports = { "value": "village", "text_field": "name", "path": "horizontal", + "font": "Open Sans Regular, Arial Unicode MS Regular", "fontSize": 22, "feature_type": "point", "type": "text" @@ -761,6 +771,7 @@ module.exports = { ], "text_field": "name", "path": "horizontal", + "font": "Open Sans Regular, Arial Unicode MS Regular", "fontSize": 14, "feature_type": "point", "type": "text" @@ -771,6 +782,7 @@ module.exports = { "text_field": "name", "path": "curve", "padding": 2, + "font": "Open Sans Regular, Arial Unicode MS Regular", "fontSize": 13, "feature_type": "line", "type": "text", @@ -781,6 +793,7 @@ module.exports = { "layer": "water_label", "text_field": "name", "path": "horizontal", + "font": "Open Sans Regular, Arial Unicode MS Regular", "fontSize": 12, "feature_type": "point", "type": "text" @@ -790,6 +803,7 @@ module.exports = { "layer": "waterway_label", "text_field": "name", "path": "curve", + "font": "Open Sans Regular, Arial Unicode MS Regular", "fontSize": 12, "textMinDistance": 15, "feature_type": "line", @@ -824,6 +838,7 @@ module.exports = { "text_field": "name", "path": "horizontal", "padding": 2, + "font": "Open Sans Regular, Arial Unicode MS Regular", "fontSize": 12, "feature_type": "point", "type": "text", From cca9d81ee75bc045e8fa790822b7a1ec83b3dbaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Wed, 28 May 2014 15:51:09 +0200 Subject: [PATCH 02/29] add debug warnings --- include/llmr/util/constants.hpp | 5 +++++ src/map/tile_parser.cpp | 7 ++++++- src/renderer/text_bucket.cpp | 7 +++++++ src/util/constants.cpp | 5 +++++ 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/include/llmr/util/constants.hpp b/include/llmr/util/constants.hpp index 70befcb37..f315e5171 100644 --- a/include/llmr/util/constants.hpp +++ b/include/llmr/util/constants.hpp @@ -19,6 +19,11 @@ extern const bool styleParseWarnings; extern const bool spriteWarnings; extern const bool renderWarnings; extern const bool renderTree; +extern const bool labelTextMissingWarning; +extern const bool missingFontStackWarning; +extern const bool missingFontFaceWarning; +extern const bool glyphWarning; +extern const bool shapingWarning; } diff --git a/src/map/tile_parser.cpp b/src/map/tile_parser.cpp index 49444428a..434372944 100644 --- a/src/map/tile_parser.cpp +++ b/src/map/tile_parser.cpp @@ -163,9 +163,11 @@ std::unique_ptr TileParser::createIconBucket(const VectorTileLayer& laye } std::unique_ptr TileParser::createTextBucket(const VectorTileLayer& layer, const BucketDescription& bucket_desc) { - // Determine the correct text stack. if (!layer.shaping.size()) { + if (debug::missingFontStackWarning) { + fprintf(stderr, "[WARNING] missing font stack '%s'\n", bucket_desc.geometry.font.c_str()); + } return nullptr; } @@ -179,6 +181,9 @@ std::unique_ptr TileParser::createTextBucket(const VectorTileLayer& laye auto it = const_faces.find(face); if (it == const_faces.end()) { // This layer references an unknown face. + if (debug::missingFontFaceWarning) { + fprintf(stderr, "[WARNING] missing font face '%s'\n", face.c_str()); + } return nullptr; } faces.push_back(&it->second); diff --git a/src/renderer/text_bucket.cpp b/src/renderer/text_bucket.cpp index 0fd351343..310baf374 100644 --- a/src/renderer/text_bucket.cpp +++ b/src/renderer/text_bucket.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -103,12 +104,18 @@ void TextBucket::addFeature(const VectorTileFeature &feature, auto it_prop = feature.properties.find(geom_desc.field); if (it_prop == feature.properties.end()) { // feature does not have the correct property + if (debug::labelTextMissingWarning) { + fprintf(stderr, "[WARNING] feature doesn't have property '%s' required for labelling\n", geom_desc.field.c_str()); + } return; } const Value &value = it_prop->second; auto it_shaping = shapings.find(value); if (it_shaping == shapings.end()) { + if (debug::shapingWarning) { + fprintf(stderr, "[WARNING] missing shaping for '%s'\n", toString(value).c_str()); + } // we lack shaping information for this label return; } diff --git a/src/util/constants.cpp b/src/util/constants.cpp index 2a4ecd374..f67464926 100644 --- a/src/util/constants.cpp +++ b/src/util/constants.cpp @@ -7,3 +7,8 @@ const bool llmr::debug::styleParseWarnings = false; const bool llmr::debug::spriteWarnings = false; const bool llmr::debug::renderWarnings = false; const bool llmr::debug::renderTree = false; +const bool llmr::debug::labelTextMissingWarning = true; +const bool llmr::debug::missingFontStackWarning = true; +const bool llmr::debug::missingFontFaceWarning = true; +const bool llmr::debug::glyphWarning = true; +const bool llmr::debug::shapingWarning = true; From 1749aa3de2744c830becf05608e008a78b92c49d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Wed, 28 May 2014 15:51:29 +0200 Subject: [PATCH 03/29] switch to glyph/shaping-less endpoint --- src/map/map.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/map/map.cpp b/src/map/map.cpp index 835edac0b..e72c7e30b 100644 --- a/src/map/map.cpp +++ b/src/map/map.cpp @@ -163,7 +163,7 @@ void Map::setup() { sources.emplace("outdoors", std::unique_ptr(new Source(*this, painter, - "http://api-maps-gl.tilestream.net/v3/mapbox.mapbox-terrain-v1,mapbox.mapbox-streets-v5/%d/%d/%d.gl.pbf", + "http://api-maps-gl.tilestream.net/v3/mapbox.mapbox-terrain-v1,mapbox.mapbox-streets-v5/%d/%d/%d.vector.pbf", Source::Type::vector, {{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 }}, 512, From ed7b86a1349c0d943e95910d439ac52fd157b28d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Wed, 28 May 2014 20:51:13 +0200 Subject: [PATCH 04/29] add glyph store and load glyph pbfs --- include/llmr/map/map.hpp | 3 + include/llmr/map/tile_parser.hpp | 4 +- include/llmr/platform/platform.hpp | 4 +- include/llmr/text/glyph.hpp | 5 ++ include/llmr/text/glyph_store.hpp | 66 +++++++++++++++++++ src/map/tile_parser.cpp | 67 +++++++++++-------- src/map/vector_tile_data.cpp | 2 +- src/platform/request.cpp | 34 ++++++---- src/text/glyph.cpp | 101 +++++++++++++++++++++++++++++ src/text/glyph_store.cpp | 92 ++++++++++++++++++++++++++ 10 files changed, 334 insertions(+), 44 deletions(-) create mode 100644 include/llmr/text/glyph_store.hpp create mode 100644 src/text/glyph.cpp create mode 100644 src/text/glyph_store.cpp diff --git a/include/llmr/map/map.hpp b/include/llmr/map/map.hpp index ab1738f96..49050540c 100644 --- a/include/llmr/map/map.hpp +++ b/include/llmr/map/map.hpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -98,6 +99,7 @@ public: inline const TransformState &getState() const { return state; } inline const Style &getStyle() const { return style; } inline GlyphAtlas &getGlyphAtlas() { return glyphAtlas; } + inline GlyphStore &getGlyphStore() { return glyphStore; } inline SpriteAtlas &getSpriteAtlas() { return spriteAtlas; } inline uv_loop_t *getLoop() { return loop; } inline time getAnimationTime() const { return animationTime; } @@ -152,6 +154,7 @@ private: Texturepool texturepool; Style style; GlyphAtlas glyphAtlas; + GlyphStore glyphStore; SpriteAtlas spriteAtlas; Painter painter; diff --git a/include/llmr/map/tile_parser.hpp b/include/llmr/map/tile_parser.hpp index 86545e4f7..644cc45b1 100644 --- a/include/llmr/map/tile_parser.hpp +++ b/include/llmr/map/tile_parser.hpp @@ -9,6 +9,7 @@ namespace llmr { class Style; class GlyphAtlas; +class GlyphStore; class SpriteAtlas; class LayerDescription; @@ -16,7 +17,7 @@ class Bucket; class TileParser { public: - TileParser(const std::string& data, VectorTileData& tile, const Style& style, GlyphAtlas& glyphAtlas, SpriteAtlas &spriteAtlas); + TileParser(const std::string& data, VectorTileData& tile, const Style& style, GlyphAtlas& glyphAtlas, GlyphStore &glyphStore, SpriteAtlas &spriteAtlas); private: bool obsolete() const; @@ -35,6 +36,7 @@ private: VectorTileData& tile; const Style& style; GlyphAtlas& glyphAtlas; + GlyphStore &glyphStore; SpriteAtlas &spriteAtlas; Faces faces; Placement placement; diff --git a/include/llmr/platform/platform.hpp b/include/llmr/platform/platform.hpp index 972e7db6b..d516378c2 100644 --- a/include/llmr/platform/platform.hpp +++ b/include/llmr/platform/platform.hpp @@ -25,9 +25,11 @@ struct Response { // Makes an HTTP request of a URL, preferrably on a background thread, and calls a function with the // results in the original thread (which runs the libuv loop). +// If the loop pointer is NULL, the callback function will be called on an arbitrary thread. // Returns a cancellable request. std::shared_ptr request_http(const std::string &url, - std::function callback, uv_loop_t *loop); + std::function callback, + uv_loop_t *loop = nullptr); // Cancels an HTTP request. void cancel_request_http(const std::shared_ptr &req); diff --git a/include/llmr/text/glyph.hpp b/include/llmr/text/glyph.hpp index 91531e24f..e6138e712 100644 --- a/include/llmr/text/glyph.hpp +++ b/include/llmr/text/glyph.hpp @@ -9,6 +9,11 @@ namespace llmr { +typedef std::pair GlyphRange; + +// Note: this only works for the BMP +GlyphRange getGlyphRange(uint32_t glyph); + struct GlyphMetrics { operator bool() const { return width == 0 && height == 0 && advance == 0; diff --git a/include/llmr/text/glyph_store.hpp b/include/llmr/text/glyph_store.hpp new file mode 100644 index 000000000..1cd2fd539 --- /dev/null +++ b/include/llmr/text/glyph_store.hpp @@ -0,0 +1,66 @@ +#ifndef LLMR_TEXT_GLYPH_STORE +#define LLMR_TEXT_GLYPH_STORE + +#include +#include + + +#include +#include +#include +#include +#include +#include + +namespace llmr { + + +class SDFGlyph { +public: + SDFGlyph(); + SDFGlyph(pbf data); + + // A signed distance field of the glyph with a border of 3 pixels. + std::string bitmap; + + // Glyph metrics + GlyphMetrics metrics; +}; + + +class GlyphSet { +public: + GlyphSet(const std::string &fontStack, GlyphRange glyphRange); + + void parse(); + + std::shared_future getFuture(); + +private: + std::map glyphs; + + std::string data; + std::promise promise; + std::shared_future future; + std::mutex mtx; +}; + +// Manages Glyphrange PBF loading. +class GlyphStore { +public: + // Block until all specified GlyphRanges of the specified font stack are loaded. + void waitForGlyphRanges(const std::string &fontStack, const std::set &glyphRanges); + +private: + // Loads an individual glyph range from the font stack and adds it to rangeSets + std::shared_future loadGlyphRange(const std::string &fontStack, std::map> &rangeSets, GlyphRange range); + +private: + std::unordered_map>> stacks; + std::mutex mtx; +}; + + +} + +#endif diff --git a/src/map/tile_parser.cpp b/src/map/tile_parser.cpp index 434372944..c610ca139 100644 --- a/src/map/tile_parser.cpp +++ b/src/map/tile_parser.cpp @@ -9,17 +9,21 @@ #include #include #include +#include #include +#include + using namespace llmr; -TileParser::TileParser(const std::string& data, VectorTileData& tile, const Style& style, GlyphAtlas& glyphAtlas, SpriteAtlas &spriteAtlas) +TileParser::TileParser(const std::string& data, VectorTileData& tile, const Style& style, GlyphAtlas& glyphAtlas, GlyphStore &glyphStore, SpriteAtlas &spriteAtlas) : vector_data(pbf((const uint8_t *)data.data(), data.size())), tile(tile), style(style), glyphAtlas(glyphAtlas), + glyphStore(glyphStore), spriteAtlas(spriteAtlas), placement(tile.id.z) { parseGlyphs(); @@ -162,35 +166,42 @@ std::unique_ptr TileParser::createIconBucket(const VectorTileLayer& laye return obsolete() ? nullptr : std::move(bucket); } +typedef std::pair GlyphRange; + std::unique_ptr TileParser::createTextBucket(const VectorTileLayer& layer, const BucketDescription& bucket_desc) { - // Determine the correct text stack. - if (!layer.shaping.size()) { - if (debug::missingFontStackWarning) { - fprintf(stderr, "[WARNING] missing font stack '%s'\n", bucket_desc.geometry.font.c_str()); - } - return nullptr; - } - - // TODO: currently hardcoded to use the first font stack. - const std::map& shaping = layer.shaping.begin()->second; - - const Faces& const_faces = faces; - - IndexedFaces faces; - for (const std::string& face : layer.faces) { - auto it = const_faces.find(face); - if (it == const_faces.end()) { - // This layer references an unknown face. - if (debug::missingFontFaceWarning) { - fprintf(stderr, "[WARNING] missing font face '%s'\n", face.c_str()); - } - return nullptr; - } - faces.push_back(&it->second); - } - std::unique_ptr bucket = std::make_unique( tile.textVertexBuffer, tile.triangleElementsBuffer, bucket_desc, placement); - addBucketFeatures(bucket, layer, bucket_desc, faces, shaping); + + + std::set ranges; + std::wstring_convert, char32_t> ucs4conv; + + FilteredVectorTileLayer filtered_layer(layer, bucket_desc); + for (const pbf& feature_pbf : filtered_layer) { + if (obsolete()) return nullptr; + VectorTileFeature feature { feature_pbf, layer }; + + auto it_prop = feature.properties.find(bucket_desc.geometry.field); + if (it_prop == feature.properties.end()) { + // feature does not have the correct property + if (debug::labelTextMissingWarning) { + fprintf(stderr, "[WARNING] feature doesn't have property '%s' required for labelling\n", bucket_desc.geometry.field.c_str()); + } + continue; + } + + const std::u32string string = ucs4conv.from_bytes(toString(it_prop->second)); + + // Loop through all characters of this text and collect unique codepoints. + for (uint32_t chr : string) { + ranges.insert(getGlyphRange(chr)); + } + } + + glyphStore.waitForGlyphRanges(bucket_desc.geometry.font, ranges); + + // TODO: Now that we have all the glyph ranges, actually perform the shaping and add features to + // the bucket, and add the required glyphs from the glyphStore to the glyphAtlas. + return std::move(bucket); } diff --git a/src/map/vector_tile_data.cpp b/src/map/vector_tile_data.cpp index 337e2fcc3..3c972aa31 100644 --- a/src/map/vector_tile_data.cpp +++ b/src/map/vector_tile_data.cpp @@ -22,7 +22,7 @@ void VectorTileData::parse() { // Parsing creates state that is encapsulated in TileParser. While parsing, // the TileParser object writes results into this objects. All other state // is going to be discarded afterwards. - TileParser parser(data, *this, map.getStyle(), map.getGlyphAtlas(), map.getSpriteAtlas()); + TileParser parser(data, *this, map.getStyle(), map.getGlyphAtlas(), map.getGlyphStore(), map.getSpriteAtlas()); } catch (const std::exception& ex) { fprintf(stderr, "[%p] exception [%d/%d/%d]... failed: %s\n", this, id.z, id.x, id.y, ex.what()); cancel(); diff --git a/src/platform/request.cpp b/src/platform/request.cpp index aadf83dea..f689517ab 100644 --- a/src/platform/request.cpp +++ b/src/platform/request.cpp @@ -12,25 +12,33 @@ Request::Request(const std::string &url, res(std::make_unique(callback)), cancelled(false), loop(loop) { - // Add a check handle without a callback to keep the default loop running. - // We don't have a real handler attached to the default loop right from the - // beginning, because we're using asynchronous messaging to perform the actual - // request in the request thread. Only after the request is complete, we - // create an actual work request that is attached to the default loop. - async = new uv_async_t(); - async->data = new std::unique_ptr(); - uv_async_init(loop, async, complete); + if (loop) { + // Add a check handle without a callback to keep the default loop running. + // We don't have a real handler attached to the default loop right from the + // beginning, because we're using asynchronous messaging to perform the actual + // request in the request thread. Only after the request is complete, we + // create an actual work request that is attached to the default loop. + async = new uv_async_t(); + async->data = new std::unique_ptr(); + uv_async_init(loop, async, complete); + } } Request::~Request() { } void Request::complete() { - // We're scheduling the response callback to be invoked in the event loop. - // Since the Request object will be deleted before the callback is invoked, - // we move over the Response object to be owned by the async handle. - ((std::unique_ptr *)async->data)->swap(res); - uv_async_send(async); + if (loop) { + // We're scheduling the response callback to be invoked in the event loop. + // Since the Request object will be deleted before the callback is invoked, + // we move over the Response object to be owned by the async handle. + ((std::unique_ptr *)async->data)->swap(res); + uv_async_send(async); + } else { + // We're calling the response callback immediately. We're currently on an + // arbitrary thread, but that's okay. + res->callback(res.get()); + } } void Request::complete(uv_async_t *async) { diff --git a/src/text/glyph.cpp b/src/text/glyph.cpp new file mode 100644 index 000000000..039e3bc36 --- /dev/null +++ b/src/text/glyph.cpp @@ -0,0 +1,101 @@ +#include + +namespace llmr { + +// Note: this only works for the BMP +// Note: we could use a binary lookup table to get averaged constant time lookups, however, +// most of our lookups are going to be within the first 3 ranges listed here, so this is +// likely faster. +GlyphRange getGlyphRange(uint32_t glyph) { + if (glyph <= 127) return { 0, 127 }; // Basic Latin + if (glyph <= 255) return { 128, 255 }; // Latin-1 Supplement + if (glyph >= 8192 && glyph <= 8303) return { 8192, 8303 }; // General Punctuation + if (glyph <= 383) return { 256, 383 }; // Latin Extended-A + if (glyph <= 591) return { 384, 591 }; // Latin Extended-B + if (glyph <= 687) return { 592, 687 }; // IPA Extensions + if (glyph <= 767) return { 688, 767 }; // Spacing Modifier Letters + if (glyph <= 879) return { 768, 879 }; // Combining Diacritical Marks + if (glyph <= 1023) return { 880, 1023 }; // Greek + if (glyph <= 1279) return { 1024, 1279 }; // Cyrillic + if (glyph <= 1423) return { 1280, 1423 }; // Armenian + if (glyph <= 1535) return { 1424, 1535 }; // Hebrew + if (glyph <= 1791) return { 1536, 1791 }; // Arabic + if (glyph <= 1871) return { 1792, 1871 }; // Syriac + if (glyph <= 1983) return { 1872, 1983 }; // Thaana + if (glyph <= 2431) return { 1984, 2431 }; // Devanagari + if (glyph <= 2559) return { 2432, 2559 }; // Bengali + if (glyph <= 2687) return { 2560, 2687 }; // Gurmukhi + if (glyph <= 2815) return { 2688, 2815 }; // Gujarati + if (glyph <= 2943) return { 2816, 2943 }; // Oriya + if (glyph <= 3071) return { 2944, 3071 }; // Tamil + if (glyph <= 3199) return { 3072, 3199 }; // Telugu + if (glyph <= 3327) return { 3200, 3327 }; // Kannada + if (glyph <= 3455) return { 3328, 3455 }; // Malayalam + if (glyph <= 3583) return { 3456, 3583 }; // Sinhala + if (glyph <= 3711) return { 3584, 3711 }; // Thai + if (glyph <= 3839) return { 3712, 3839 }; // Lao + if (glyph <= 4095) return { 3840, 4095 }; // Tibetan + if (glyph <= 4255) return { 4096, 4255 }; // Myanmar + if (glyph <= 4351) return { 4256, 4351 }; // Georgian + if (glyph <= 4607) return { 4352, 4607 }; // Hangul Jamo + if (glyph <= 4991) return { 4608, 4991 }; // Ethiopic + if (glyph <= 5119) return { 4992, 5119 }; // Cherokee + if (glyph <= 5759) return { 5120, 5759 }; // Unified Canadian Aboriginal Syllabics + if (glyph <= 5791) return { 5760, 5791 }; // Ogham + if (glyph <= 5887) return { 5792, 5887 }; // Runic + if (glyph <= 6143) return { 5888, 6143 }; // Khmer + if (glyph <= 6319) return { 6144, 6319 }; // Mongolian + if (glyph <= 7935) return { 6320, 7935 }; // Latin Extended Additional + if (glyph <= 8191) return { 7936, 8191 }; // Greek Extended + if (glyph <= 8351) return { 8304, 8351 }; // Superscripts and Subscripts + if (glyph <= 8399) return { 8352, 8399 }; // Currency Symbols + if (glyph <= 8447) return { 8400, 8447 }; // Combining Marks for Symbols + if (glyph <= 8527) return { 8448, 8527 }; // Letterlike Symbols + if (glyph <= 8591) return { 8528, 8591 }; // Number Forms + if (glyph <= 8703) return { 8592, 8703 }; // Arrows + if (glyph <= 8959) return { 8704, 8959 }; // Mathematical Operators + if (glyph <= 9215) return { 8960, 9215 }; // Miscellaneous Technical + if (glyph <= 9279) return { 9216, 9279 }; // Control Pictures + if (glyph <= 9311) return { 9280, 9311 }; // Optical Character Recognition + if (glyph <= 9471) return { 9312, 9471 }; // Enclosed Alphanumerics + if (glyph <= 9599) return { 9472, 9599 }; // Box Drawing + if (glyph <= 9631) return { 9600, 9631 }; // Block Elements + if (glyph <= 9727) return { 9632, 9727 }; // Geometric Shapes + if (glyph <= 9983) return { 9728, 9983 }; // Miscellaneous Symbols + if (glyph <= 10175) return { 9984, 10175 }; // Dingbats + if (glyph <= 10495) return { 10176, 10495 }; // Braille Patterns + if (glyph <= 12031) return { 10496, 12031 }; // CJK Radicals Supplement + if (glyph <= 12255) return { 12032, 12255 }; // Kangxi Radicals + if (glyph <= 12287) return { 12256, 12287 }; // Ideographic Description Characters + if (glyph <= 12351) return { 12288, 12351 }; // CJK Symbols and Punctuation + if (glyph <= 12447) return { 12352, 12447 }; // Hiragana + if (glyph <= 12543) return { 12448, 12543 }; // Katakana + if (glyph <= 12591) return { 12544, 12591 }; // Bopomofo + if (glyph <= 12687) return { 12592, 12687 }; // Hangul Compatibility Jamo + if (glyph <= 12703) return { 12688, 12703 }; // Kanbun + if (glyph <= 12735) return { 12704, 12735 }; // Bopomofo Extended + if (glyph <= 13055) return { 12736, 13055 }; // Enclosed CJK Letters and Months + if (glyph <= 13311) return { 13056, 13311 }; // CJK Compatibility + if (glyph <= 19893) return { 13312, 19893 }; // CJK Unified Ideographs Extension A + if (glyph <= 40959) return { 19894, 40959 }; // CJK Unified Ideographs + if (glyph <= 42127) return { 40960, 42127 }; // Yi Syllables + if (glyph <= 42191) return { 42128, 42191 }; // Yi Radicals + if (glyph <= 55203) return { 42192, 55203 }; // Hangul Syllables + if (glyph <= 56191) return { 55204, 56191 }; // High Surrogates + if (glyph <= 56319) return { 56192, 56319 }; // High Private Use Surrogates + if (glyph <= 57343) return { 56320, 57343 }; // Low Surrogates + if (glyph <= 63743) return { 57344, 63743 }; // Private Use + if (glyph <= 64255) return { 63744, 64255 }; // CJK Compatibility Ideographs + if (glyph <= 64335) return { 64256, 64335 }; // Alphabetic Presentation Forms + if (glyph <= 65023) return { 64336, 65023 }; // Arabic Presentation Forms-A + if (glyph <= 65071) return { 65024, 65071 }; // Combining Half Marks + if (glyph <= 65103) return { 65072, 65103 }; // CJK Compatibility Forms + if (glyph <= 65135) return { 65104, 65135 }; // Small Form Variants + if (glyph <= 65278) return { 65136, 65278 }; // Arabic Presentation Forms-B + if (glyph <= 65279) return { 65279, 65279 }; // Specials + if (glyph <= 65519) return { 65280, 65519 }; // Halfwidth and Fullwidth Forms + if (glyph <= 65533) return { 65520, 65533 }; // Specials + return { 0, 0 }; +} + +} diff --git a/src/text/glyph_store.cpp b/src/text/glyph_store.cpp new file mode 100644 index 000000000..fae9d1326 --- /dev/null +++ b/src/text/glyph_store.cpp @@ -0,0 +1,92 @@ +#include + +#include +#include +#include +#include + +namespace llmr { + + +GlyphSet::GlyphSet(const std::string &fontStack, GlyphRange glyphRange) + : future(promise.get_future().share()) +{ + // Load the glyph set URL + std::string url = util::sprintf<255>("http://mapbox.s3.amazonaws.com/gl-glyphs/%s/%d-%d.pbf", fontStack.c_str(), glyphRange.first, glyphRange.second); + + // TODO: Find more reliable URL normalization function + std::replace(url.begin(), url.end(), ' ', '+'); + + platform::request_http(url, [&](platform::Response *res) { + if (res->code != 200) { + // Something went wrong with loading the glyph pbf. Pass on the error to the future listeners. + const std::string msg = util::sprintf<255>("[ERROR] failed to load glyphs (%d): %s\n", res->code, res->error_message.c_str()); + promise.set_exception(std::make_exception_ptr(std::runtime_error(msg))); + } else { + // Transfer the data to the GlyphSet and signal its availability. + // Once it is available, the caller will need to call parse() to actually + // parse the data we received. We are not doing this here since this callback is being + // called from another (unknown) thread. + data.swap(res->body); + promise.set_value(*this); + } + }); +} + +std::shared_future GlyphSet::getFuture() { + return future; +} + +void GlyphSet::parse() { + std::lock_guard lock(mtx); + + if (!data.size()) { + // If there is no data, this means we either haven't received any data, or + // we have already parsed the data. + return; + } + + // TODO: parse the glyphs + + + data.clear(); +} + + +void GlyphStore::waitForGlyphRanges(const std::string &fontStack, const std::set &glyphRanges) { + // We are implementing a blocking wait with futures: Every GlyphSet has a future that we are + // waiting for until it is loaded. + + std::vector> futures; + futures.reserve(glyphRanges.size()); + { + std::lock_guard lock(mtx); + auto &rangeSets = stacks[fontStack]; + + // Attempt to load the glyph range. If the GlyphSet already exists, we are getting back + // the same shared_future. + for (GlyphRange range : glyphRanges) { + futures.emplace_back(loadGlyphRange(fontStack, rangeSets, range)); + } + } + + // Now that we potentially created all GlyphSets, we are waiting for the results, one by one. + // When we get a result (or the GlyphSet is aready loaded), we are attempting to parse the + // GlyphSet. + for (std::shared_future &future : futures) { + future.get().parse(); + } +} + +std::shared_future GlyphStore::loadGlyphRange(const std::string &name, std::map> &rangeSets, const GlyphRange range) { + auto range_it = rangeSets.find(range); + if (range_it == rangeSets.end()) { + // We don't have this glyph set yet for this font stack. + range_it = rangeSets.emplace(range, std::make_unique(name, range)).first; + } + + return range_it->second->getFuture(); +} + + +} From d2b44cd61cd47a335ed1f3e6735390ea50d9d81c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Wed, 28 May 2014 21:17:28 +0200 Subject: [PATCH 05/29] add glyphs proto --- proto/glyphs.proto | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 proto/glyphs.proto diff --git a/proto/glyphs.proto b/proto/glyphs.proto new file mode 100644 index 000000000..606050f7a --- /dev/null +++ b/proto/glyphs.proto @@ -0,0 +1,33 @@ +// Protocol Version 1 + +package llmr.glyphs; + +option optimize_for = LITE_RUNTIME; + +// Stores a glyph with metrics and optional SDF bitmap information. +message glyph { + required uint32 id = 1; + + // A signed distance field of the glyph with a border of 3 pixels. + optional bytes bitmap = 2; + + // Glyph metrics. + required uint32 width = 3; + required uint32 height = 4; + required sint32 left = 5; + required sint32 top = 6; + required uint32 advance = 7; +} + +// Stores fontstack information and a list of faces. +message fontstack { + required string name = 1; + required string range = 2; + repeated glyph glyphs = 3; +} + +message glyphs { + repeated fontstack stacks = 1; + + extensions 16 to 8191; +} From 906b4839c614535b5202631222e7b5e1c35e47da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Wed, 28 May 2014 21:55:14 +0200 Subject: [PATCH 06/29] parse glyph pbf --- include/llmr/text/glyph_store.hpp | 32 +++++++------ src/text/glyph_store.cpp | 75 ++++++++++++++++++++++++++----- 2 files changed, 83 insertions(+), 24 deletions(-) diff --git a/include/llmr/text/glyph_store.hpp b/include/llmr/text/glyph_store.hpp index 1cd2fd539..c2d467e78 100644 --- a/include/llmr/text/glyph_store.hpp +++ b/include/llmr/text/glyph_store.hpp @@ -17,9 +17,6 @@ namespace llmr { class SDFGlyph { public: - SDFGlyph(); - SDFGlyph(pbf data); - // A signed distance field of the glyph with a border of 3 pixels. std::string bitmap; @@ -27,21 +24,27 @@ public: GlyphMetrics metrics; }; - -class GlyphSet { +class FontStack { public: - GlyphSet(const std::string &fontStack, GlyphRange glyphRange); - - void parse(); - - std::shared_future getFuture(); + void insert(uint32_t id, const SDFGlyph &glyph); private: std::map glyphs; + std::mutex mtx; +}; +class GlyphPBF { +public: + GlyphPBF(const std::string &fontStack, GlyphRange glyphRange); + + void parse(FontStack &stack); + + std::shared_future getFuture(); + +private: std::string data; - std::promise promise; - std::shared_future future; + std::promise promise; + std::shared_future future; std::mutex mtx; }; @@ -53,10 +56,11 @@ public: private: // Loads an individual glyph range from the font stack and adds it to rangeSets - std::shared_future loadGlyphRange(const std::string &fontStack, std::map> &rangeSets, GlyphRange range); + std::shared_future loadGlyphRange(const std::string &fontStack, std::map> &rangeSets, GlyphRange range); private: - std::unordered_map>> stacks; + std::unordered_map>> ranges; + std::unordered_map> stacks; std::mutex mtx; }; diff --git a/src/text/glyph_store.cpp b/src/text/glyph_store.cpp index fae9d1326..65fe38bc4 100644 --- a/src/text/glyph_store.cpp +++ b/src/text/glyph_store.cpp @@ -2,13 +2,20 @@ #include #include +#include #include #include namespace llmr { -GlyphSet::GlyphSet(const std::string &fontStack, GlyphRange glyphRange) +void FontStack::insert(uint32_t id, const SDFGlyph &glyph) { + std::lock_guard lock(mtx); + glyphs.emplace(id, glyph); +} + + +GlyphPBF::GlyphPBF(const std::string &fontStack, GlyphRange glyphRange) : future(promise.get_future().share()) { // Load the glyph set URL @@ -33,11 +40,11 @@ GlyphSet::GlyphSet(const std::string &fontStack, GlyphRange glyphRange) }); } -std::shared_future GlyphSet::getFuture() { +std::shared_future GlyphPBF::getFuture() { return future; } -void GlyphSet::parse() { +void GlyphPBF::parse(FontStack &stack) { std::lock_guard lock(mtx); if (!data.size()) { @@ -46,8 +53,48 @@ void GlyphSet::parse() { return; } - // TODO: parse the glyphs + // Parse the glyph PBF + pbf glyphs_pbf(reinterpret_cast(data.data()), data.size()); + while (glyphs_pbf.next()) { + if (glyphs_pbf.tag == 1) { // stacks + pbf fontstack_pbf = glyphs_pbf.message(); + while (fontstack_pbf.next()) { + if (fontstack_pbf.tag == 3) { // glyphs + pbf glyph_pbf = fontstack_pbf.message(); + + SDFGlyph glyph; + uint32_t id = 0; + + while (glyph_pbf.next()) { + if (glyph_pbf.tag == 1) { // id + id = glyph_pbf.varint(); + } else if (glyph_pbf.tag == 2) { // bitmap + glyph.bitmap = glyph_pbf.string(); + } else if (glyph_pbf.tag == 3) { // width + glyph.metrics.width = glyph_pbf.varint(); + } else if (glyph_pbf.tag == 4) { // height + glyph.metrics.height = glyph_pbf.varint(); + } else if (glyph_pbf.tag == 5) { // left + glyph.metrics.left = glyph_pbf.svarint(); + } else if (glyph_pbf.tag == 6) { // top + glyph.metrics.top = glyph_pbf.svarint(); + } else if (glyph_pbf.tag == 7) { // advance + glyph.metrics.advance = glyph_pbf.varint(); + } else { + glyph_pbf.skip(); + } + } + + stack.insert(id, glyph); + } else { + fontstack_pbf.skip(); + } + } + } else { + glyphs_pbf.skip(); + } + } data.clear(); } @@ -57,11 +104,19 @@ void GlyphStore::waitForGlyphRanges(const std::string &fontStack, const std::set // We are implementing a blocking wait with futures: Every GlyphSet has a future that we are // waiting for until it is loaded. - std::vector> futures; + FontStack *stack = nullptr; + + std::vector> futures; futures.reserve(glyphRanges.size()); { std::lock_guard lock(mtx); - auto &rangeSets = stacks[fontStack]; + auto &rangeSets = ranges[fontStack]; + + auto stack_it = stacks.find(fontStack); + if (stack_it == stacks.end()) { + stack_it = stacks.emplace(fontStack, std::make_unique()).first; + } + stack = stack_it->second.get(); // Attempt to load the glyph range. If the GlyphSet already exists, we are getting back // the same shared_future. @@ -73,16 +128,16 @@ void GlyphStore::waitForGlyphRanges(const std::string &fontStack, const std::set // Now that we potentially created all GlyphSets, we are waiting for the results, one by one. // When we get a result (or the GlyphSet is aready loaded), we are attempting to parse the // GlyphSet. - for (std::shared_future &future : futures) { - future.get().parse(); + for (std::shared_future &future : futures) { + future.get().parse(*stack); } } -std::shared_future GlyphStore::loadGlyphRange(const std::string &name, std::map> &rangeSets, const GlyphRange range) { +std::shared_future GlyphStore::loadGlyphRange(const std::string &name, std::map> &rangeSets, const GlyphRange range) { auto range_it = rangeSets.find(range); if (range_it == rangeSets.end()) { // We don't have this glyph set yet for this font stack. - range_it = rangeSets.emplace(range, std::make_unique(name, range)).first; + range_it = rangeSets.emplace(range, std::make_unique(name, range)).first; } return range_it->second->getFuture(); From 193b2aacb78872084564e0f3398fb9cb0ed7983b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Wed, 28 May 2014 22:00:00 +0200 Subject: [PATCH 07/29] add second labelling pass --- src/map/tile_parser.cpp | 66 ++++++++++++++++++++++++++++------------- 1 file changed, 46 insertions(+), 20 deletions(-) diff --git a/src/map/tile_parser.cpp b/src/map/tile_parser.cpp index c610ca139..f44d464f1 100644 --- a/src/map/tile_parser.cpp +++ b/src/map/tile_parser.cpp @@ -173,35 +173,61 @@ std::unique_ptr TileParser::createTextBucket(const VectorTileLayer& laye tile.textVertexBuffer, tile.triangleElementsBuffer, bucket_desc, placement); - std::set ranges; std::wstring_convert, char32_t> ucs4conv; - FilteredVectorTileLayer filtered_layer(layer, bucket_desc); - for (const pbf& feature_pbf : filtered_layer) { - if (obsolete()) return nullptr; - VectorTileFeature feature { feature_pbf, layer }; - auto it_prop = feature.properties.find(bucket_desc.geometry.field); - if (it_prop == feature.properties.end()) { - // feature does not have the correct property - if (debug::labelTextMissingWarning) { - fprintf(stderr, "[WARNING] feature doesn't have property '%s' required for labelling\n", bucket_desc.geometry.field.c_str()); + // Determine and load glyph ranges + { + std::set ranges; + + FilteredVectorTileLayer filtered_layer(layer, bucket_desc); + for (const pbf& feature_pbf : filtered_layer) { + if (obsolete()) return nullptr; + VectorTileFeature feature { feature_pbf, layer }; + + auto it_prop = feature.properties.find(bucket_desc.geometry.field); + if (it_prop == feature.properties.end()) { + // feature does not have the correct property + if (debug::labelTextMissingWarning) { + fprintf(stderr, "[WARNING] feature doesn't have property '%s' required for labelling\n", bucket_desc.geometry.field.c_str()); + } + continue; + } + + const std::u32string string = ucs4conv.from_bytes(toString(it_prop->second)); + + // Loop through all characters of this text and collect unique codepoints. + for (uint32_t chr : string) { + ranges.insert(getGlyphRange(chr)); } - continue; } - const std::u32string string = ucs4conv.from_bytes(toString(it_prop->second)); - - // Loop through all characters of this text and collect unique codepoints. - for (uint32_t chr : string) { - ranges.insert(getGlyphRange(chr)); - } + glyphStore.waitForGlyphRanges(bucket_desc.geometry.font, ranges); } - glyphStore.waitForGlyphRanges(bucket_desc.geometry.font, ranges); + // Shape and place all labels. + { + FilteredVectorTileLayer filtered_layer(layer, bucket_desc); + for (const pbf& feature_pbf : filtered_layer) { + if (obsolete()) return nullptr; + VectorTileFeature feature { feature_pbf, layer }; - // TODO: Now that we have all the glyph ranges, actually perform the shaping and add features to - // the bucket, and add the required glyphs from the glyphStore to the glyphAtlas. + auto it_prop = feature.properties.find(bucket_desc.geometry.field); + if (it_prop == feature.properties.end()) { + // feature does not have the correct property + if (debug::labelTextMissingWarning) { + fprintf(stderr, "[WARNING] feature doesn't have property '%s' required for labelling\n", bucket_desc.geometry.field.c_str()); + } + continue; + } + + const std::u32string string = ucs4conv.from_bytes(toString(it_prop->second)); + + // TODO: Shape label + // TODO: Place label + } + + } return std::move(bucket); } From 6f68cd3676fd61e74e4d10675d960a1b9fed58d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Wed, 28 May 2014 22:12:31 +0200 Subject: [PATCH 08/29] return font stack --- include/llmr/text/glyph_store.hpp | 13 ++++++++++--- src/map/tile_parser.cpp | 2 ++ src/text/glyph_store.cpp | 28 +++++++++++++++++++++------- 3 files changed, 33 insertions(+), 10 deletions(-) diff --git a/include/llmr/text/glyph_store.hpp b/include/llmr/text/glyph_store.hpp index c2d467e78..90fa6d908 100644 --- a/include/llmr/text/glyph_store.hpp +++ b/include/llmr/text/glyph_store.hpp @@ -26,11 +26,14 @@ public: class FontStack { public: - void insert(uint32_t id, const SDFGlyph &glyph); + void insert(uint32_t id, const GlyphMetrics &glyphMetrics, const std::string &bitmap); + + const std::map &getMetrics() const; private: - std::map glyphs; - std::mutex mtx; + std::map bitmaps; + std::map metrics; + mutable std::mutex mtx; }; class GlyphPBF { @@ -54,10 +57,14 @@ public: // Block until all specified GlyphRanges of the specified font stack are loaded. void waitForGlyphRanges(const std::string &fontStack, const std::set &glyphRanges); + FontStack &getFontStack(const std::string &fontStack); + private: // Loads an individual glyph range from the font stack and adds it to rangeSets std::shared_future loadGlyphRange(const std::string &fontStack, std::map> &rangeSets, GlyphRange range); + FontStack &createFontStack(const std::string &fontStack); + private: std::unordered_map>> ranges; std::unordered_map> stacks; diff --git a/src/map/tile_parser.cpp b/src/map/tile_parser.cpp index f44d464f1..744f6fe21 100644 --- a/src/map/tile_parser.cpp +++ b/src/map/tile_parser.cpp @@ -205,6 +205,8 @@ std::unique_ptr TileParser::createTextBucket(const VectorTileLayer& laye glyphStore.waitForGlyphRanges(bucket_desc.geometry.font, ranges); } + const FontStack &fontStack = glyphStore.getFontStack(bucket_desc.geometry.font); + // Shape and place all labels. { FilteredVectorTileLayer filtered_layer(layer, bucket_desc); diff --git a/src/text/glyph_store.cpp b/src/text/glyph_store.cpp index 65fe38bc4..a355b28f2 100644 --- a/src/text/glyph_store.cpp +++ b/src/text/glyph_store.cpp @@ -9,11 +9,16 @@ namespace llmr { -void FontStack::insert(uint32_t id, const SDFGlyph &glyph) { +void FontStack::insert(uint32_t id, const GlyphMetrics &glyphMetrics, const std::string &bitmap) { std::lock_guard lock(mtx); - glyphs.emplace(id, glyph); + metrics.emplace(id, glyphMetrics); + bitmaps.emplace(id, bitmap); } +const std::map &FontStack::getMetrics() const { + std::lock_guard lock(mtx); + return metrics; +} GlyphPBF::GlyphPBF(const std::string &fontStack, GlyphRange glyphRange) : future(promise.get_future().share()) @@ -112,11 +117,7 @@ void GlyphStore::waitForGlyphRanges(const std::string &fontStack, const std::set std::lock_guard lock(mtx); auto &rangeSets = ranges[fontStack]; - auto stack_it = stacks.find(fontStack); - if (stack_it == stacks.end()) { - stack_it = stacks.emplace(fontStack, std::make_unique()).first; - } - stack = stack_it->second.get(); + stack = &createFontStack(fontStack); // Attempt to load the glyph range. If the GlyphSet already exists, we are getting back // the same shared_future. @@ -143,5 +144,18 @@ std::shared_future GlyphStore::loadGlyphRange(const std::string &nam return range_it->second->getFuture(); } +FontStack &GlyphStore::createFontStack(const std::string &fontStack) { + auto stack_it = stacks.find(fontStack); + if (stack_it == stacks.end()) { + stack_it = stacks.emplace(fontStack, std::make_unique()).first; + } + return *stack_it->second.get(); +} + +FontStack &GlyphStore::getFontStack(const std::string &fontStack) { + std::lock_guard lock(mtx); + return createFontStack(fontStack); +} + } From 83b3b0bb6a2cc557deebf1cd59486c3fe0e1e063 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Wed, 28 May 2014 22:18:18 +0200 Subject: [PATCH 09/29] create a copy --- src/map/tile_parser.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/map/tile_parser.cpp b/src/map/tile_parser.cpp index 744f6fe21..f097913cc 100644 --- a/src/map/tile_parser.cpp +++ b/src/map/tile_parser.cpp @@ -205,7 +205,8 @@ std::unique_ptr TileParser::createTextBucket(const VectorTileLayer& laye glyphStore.waitForGlyphRanges(bucket_desc.geometry.font, ranges); } - const FontStack &fontStack = glyphStore.getFontStack(bucket_desc.geometry.font); + // Create a copy! + const FontStack fontStack = glyphStore.getFontStack(bucket_desc.geometry.font); // Shape and place all labels. { From a66db39a06023f5848af3725bc7c007790eeae1d Mon Sep 17 00:00:00 2001 From: Young Hahn Date: Wed, 28 May 2014 18:45:47 -0400 Subject: [PATCH 10/29] Monkey patching to get linux compile working. --- src/map/tile_parser.cpp | 11 +++-------- src/text/glyph_store.cpp | 4 ++-- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/map/tile_parser.cpp b/src/map/tile_parser.cpp index f097913cc..673600ecc 100644 --- a/src/map/tile_parser.cpp +++ b/src/map/tile_parser.cpp @@ -13,8 +13,6 @@ #include -#include - using namespace llmr; @@ -173,9 +171,6 @@ std::unique_ptr TileParser::createTextBucket(const VectorTileLayer& laye tile.textVertexBuffer, tile.triangleElementsBuffer, bucket_desc, placement); - std::wstring_convert, char32_t> ucs4conv; - - // Determine and load glyph ranges { std::set ranges; @@ -194,7 +189,7 @@ std::unique_ptr TileParser::createTextBucket(const VectorTileLayer& laye continue; } - const std::u32string string = ucs4conv.from_bytes(toString(it_prop->second)); + const std::string string = toString(it_prop->second); // Loop through all characters of this text and collect unique codepoints. for (uint32_t chr : string) { @@ -206,7 +201,7 @@ std::unique_ptr TileParser::createTextBucket(const VectorTileLayer& laye } // Create a copy! - const FontStack fontStack = glyphStore.getFontStack(bucket_desc.geometry.font); + const FontStack &fontStack = glyphStore.getFontStack(bucket_desc.geometry.font); // Shape and place all labels. { @@ -224,7 +219,7 @@ std::unique_ptr TileParser::createTextBucket(const VectorTileLayer& laye continue; } - const std::u32string string = ucs4conv.from_bytes(toString(it_prop->second)); + const std::string string = toString(it_prop->second); // TODO: Shape label // TODO: Place label diff --git a/src/text/glyph_store.cpp b/src/text/glyph_store.cpp index a355b28f2..bafd6ce82 100644 --- a/src/text/glyph_store.cpp +++ b/src/text/glyph_store.cpp @@ -27,7 +27,7 @@ GlyphPBF::GlyphPBF(const std::string &fontStack, GlyphRange glyphRange) std::string url = util::sprintf<255>("http://mapbox.s3.amazonaws.com/gl-glyphs/%s/%d-%d.pbf", fontStack.c_str(), glyphRange.first, glyphRange.second); // TODO: Find more reliable URL normalization function - std::replace(url.begin(), url.end(), ' ', '+'); + // std::replace(url.begin(), url.end(), ' ', '+'); platform::request_http(url, [&](platform::Response *res) { if (res->code != 200) { @@ -91,7 +91,7 @@ void GlyphPBF::parse(FontStack &stack) { } } - stack.insert(id, glyph); + stack.insert(id, glyph.metrics, glyph.bitmap); } else { fontstack_pbf.skip(); } From 944167296a8921c6184cfbcba05f473bfa60c82e Mon Sep 17 00:00:00 2001 From: Young Hahn Date: Wed, 28 May 2014 18:53:54 -0400 Subject: [PATCH 11/29] replace comes from regex --- src/text/glyph_store.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/text/glyph_store.cpp b/src/text/glyph_store.cpp index bafd6ce82..e2ebb97bd 100644 --- a/src/text/glyph_store.cpp +++ b/src/text/glyph_store.cpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace llmr { @@ -27,7 +28,7 @@ GlyphPBF::GlyphPBF(const std::string &fontStack, GlyphRange glyphRange) std::string url = util::sprintf<255>("http://mapbox.s3.amazonaws.com/gl-glyphs/%s/%d-%d.pbf", fontStack.c_str(), glyphRange.first, glyphRange.second); // TODO: Find more reliable URL normalization function - // std::replace(url.begin(), url.end(), ' ', '+'); + std::replace(url.begin(), url.end(), ' ', '+'); platform::request_http(url, [&](platform::Response *res) { if (res->code != 200) { From 56c55d0b0288a3ab57a95d2b3851a2a39c734e71 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Thu, 29 May 2014 02:36:47 +0000 Subject: [PATCH 12/29] add include needed for std::replace on linux --- src/text/glyph_store.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/text/glyph_store.cpp b/src/text/glyph_store.cpp index a355b28f2..95b295a79 100644 --- a/src/text/glyph_store.cpp +++ b/src/text/glyph_store.cpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace llmr { From c85a0c3da2f296e8c78aa58a01e2bf83259a421d Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Wed, 28 May 2014 19:44:11 -0700 Subject: [PATCH 13/29] comment unfinished FontStack code --- src/map/tile_parser.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/map/tile_parser.cpp b/src/map/tile_parser.cpp index f097913cc..7d8323960 100644 --- a/src/map/tile_parser.cpp +++ b/src/map/tile_parser.cpp @@ -206,7 +206,7 @@ std::unique_ptr TileParser::createTextBucket(const VectorTileLayer& laye } // Create a copy! - const FontStack fontStack = glyphStore.getFontStack(bucket_desc.geometry.font); + //const FontStack fontStack = glyphStore.getFontStack(bucket_desc.geometry.font); // Shape and place all labels. { From 5c46bef7176b6d8c784516eff6cc1915cf9b28a1 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Wed, 28 May 2014 19:57:01 -0700 Subject: [PATCH 14/29] fallback to boost::locale on linux --- include/llmr/util/utf.hpp | 44 +++++++++++++++++++++++++++++++++++++++ src/map/tile_parser.cpp | 11 ++++------ 2 files changed, 48 insertions(+), 7 deletions(-) create mode 100644 include/llmr/util/utf.hpp diff --git a/include/llmr/util/utf.hpp b/include/llmr/util/utf.hpp new file mode 100644 index 000000000..fb9ab2048 --- /dev/null +++ b/include/llmr/util/utf.hpp @@ -0,0 +1,44 @@ +#ifndef LLMR_UTIL_UTF +#define LLMR_UTIL_UTF + +#include + +// g++/libstdc++ is missing c++11 codecvt support +#ifdef __linux__ +#include +#else +#include +#endif + +namespace llmr { + +namespace util { + +#ifdef __linux__ + +class utf8_to_utf32 { + public: + explicit utf8_to_utf32() + : utf32conv_() {} + std::u32string convert(std::string const& utf8) { + return boost::locale::conv::utf_to_utf(utf8) + } +}; +#else + +class utf8_to_utf32 { + public: + explicit utf8_to_utf32() + : utf32conv_() {} + std::u32string convert(std::string const& utf8) { + return utf32conv_.from_bytes(utf8); + } + private: + std::wstring_convert, char32_t> utf32conv_; +}; + +#endif + +}} + +#endif diff --git a/src/map/tile_parser.cpp b/src/map/tile_parser.cpp index 7d8323960..70085b309 100644 --- a/src/map/tile_parser.cpp +++ b/src/map/tile_parser.cpp @@ -12,8 +12,7 @@ #include #include - -#include +#include using namespace llmr; @@ -172,9 +171,7 @@ std::unique_ptr TileParser::createTextBucket(const VectorTileLayer& laye std::unique_ptr bucket = std::make_unique( tile.textVertexBuffer, tile.triangleElementsBuffer, bucket_desc, placement); - - std::wstring_convert, char32_t> ucs4conv; - + util::utf8_to_utf32 ucs4conv; // Determine and load glyph ranges { @@ -194,7 +191,7 @@ std::unique_ptr TileParser::createTextBucket(const VectorTileLayer& laye continue; } - const std::u32string string = ucs4conv.from_bytes(toString(it_prop->second)); + const std::u32string string = ucs4conv.convert(toString(it_prop->second)); // Loop through all characters of this text and collect unique codepoints. for (uint32_t chr : string) { @@ -224,7 +221,7 @@ std::unique_ptr TileParser::createTextBucket(const VectorTileLayer& laye continue; } - const std::u32string string = ucs4conv.from_bytes(toString(it_prop->second)); + const std::u32string string = ucs4conv.convert(toString(it_prop->second)); // TODO: Shape label // TODO: Place label From 97683460c18fe714a6a0f29f81b66739de893f91 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Wed, 28 May 2014 19:57:26 -0700 Subject: [PATCH 15/29] fix compile of glyph_store.cpp --- src/text/glyph_store.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/text/glyph_store.cpp b/src/text/glyph_store.cpp index 95b295a79..459c2d971 100644 --- a/src/text/glyph_store.cpp +++ b/src/text/glyph_store.cpp @@ -92,7 +92,7 @@ void GlyphPBF::parse(FontStack &stack) { } } - stack.insert(id, glyph); + stack.insert(id, glyph.metrics, glyph.bitmap); } else { fontstack_pbf.skip(); } From 25803b4b98192675cb15cb73738d1c554253dd1f Mon Sep 17 00:00:00 2001 From: Young Hahn Date: Wed, 28 May 2014 22:57:32 -0400 Subject: [PATCH 16/29] Guesses. --- src/map/tile_parser.cpp | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/map/tile_parser.cpp b/src/map/tile_parser.cpp index 673600ecc..32344edcc 100644 --- a/src/map/tile_parser.cpp +++ b/src/map/tile_parser.cpp @@ -10,12 +10,11 @@ #include #include #include - +#include #include using namespace llmr; - TileParser::TileParser(const std::string& data, VectorTileData& tile, const Style& style, GlyphAtlas& glyphAtlas, GlyphStore &glyphStore, SpriteAtlas &spriteAtlas) : vector_data(pbf((const uint8_t *)data.data(), data.size())), tile(tile), @@ -219,10 +218,29 @@ std::unique_ptr TileParser::createTextBucket(const VectorTileLayer& laye continue; } + uint32_t i = 0; + uint32_t x = 0; const std::string string = toString(it_prop->second); - + std::map shaping; + std::map metrics = fontStack.getMetrics(); + // fprintf(stderr, "%s\n", string.c_str()); + // Loop through all characters of this label and shape. + for (uint32_t chr : string) { + // Can we reuse GlyphPlacement here? First arg is a faces index ... + GlyphPlacement shaped = GlyphPlacement(0, chr, x, 0); + // No idea how to properly put together a shaping std::map ... + shaping.emplace(i, shaped); + i++; + x += metrics[chr].advance; + } // TODO: Shape label // TODO: Place label + // Can faces here be a std::map of fontstacks? + // It looks like nearly the same interface through the rest + // of the stack. + std::unique_ptr bucket = std::make_unique( + tile.textVertexBuffer, tile.triangleElementsBuffer, bucket_desc, placement); + addBucketFeatures(bucket, layer, bucket_desc, faces, shaping); } } From 6b0465448d324888e7c484ae66a40aca1f71d5d6 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Wed, 28 May 2014 20:04:40 -0700 Subject: [PATCH 17/29] fix utf.hpp linux compile --- include/llmr/util/utf.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/llmr/util/utf.hpp b/include/llmr/util/utf.hpp index fb9ab2048..f23f616f8 100644 --- a/include/llmr/util/utf.hpp +++ b/include/llmr/util/utf.hpp @@ -18,12 +18,12 @@ namespace util { class utf8_to_utf32 { public: - explicit utf8_to_utf32() - : utf32conv_() {} + explicit utf8_to_utf32() {} std::u32string convert(std::string const& utf8) { return boost::locale::conv::utf_to_utf(utf8) } }; + #else class utf8_to_utf32 { From 36796490de07279d8f36fdf0e944309aa79f2ba3 Mon Sep 17 00:00:00 2001 From: Young Hahn Date: Wed, 28 May 2014 23:12:26 -0400 Subject: [PATCH 18/29] More educated guess. --- src/map/tile_parser.cpp | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/map/tile_parser.cpp b/src/map/tile_parser.cpp index 32344edcc..553c8c79a 100644 --- a/src/map/tile_parser.cpp +++ b/src/map/tile_parser.cpp @@ -201,6 +201,7 @@ std::unique_ptr TileParser::createTextBucket(const VectorTileLayer& laye // Create a copy! const FontStack &fontStack = glyphStore.getFontStack(bucket_desc.geometry.font); + std::map shaping; // Shape and place all labels. { @@ -221,29 +222,30 @@ std::unique_ptr TileParser::createTextBucket(const VectorTileLayer& laye uint32_t i = 0; uint32_t x = 0; const std::string string = toString(it_prop->second); - std::map shaping; + Shaping shapedGlyphs; std::map metrics = fontStack.getMetrics(); // fprintf(stderr, "%s\n", string.c_str()); + // TODO: Shape label // Loop through all characters of this label and shape. for (uint32_t chr : string) { - // Can we reuse GlyphPlacement here? First arg is a faces index ... - GlyphPlacement shaped = GlyphPlacement(0, chr, x, 0); - // No idea how to properly put together a shaping std::map ... - shaping.emplace(i, shaped); + // Can we reuse GlyphPlacement here? First arg is a faces index. + GlyphPlacement glyph = GlyphPlacement(0, chr, x, 0); + // No idea how to properly put together a shapedGlyphs vector. + shapedGylphs.push(glyph); i++; x += metrics[chr].advance; } - // TODO: Shape label // TODO: Place label - // Can faces here be a std::map of fontstacks? - // It looks like nearly the same interface through the rest - // of the stack. - std::unique_ptr bucket = std::make_unique( - tile.textVertexBuffer, tile.triangleElementsBuffer, bucket_desc, placement); - addBucketFeatures(bucket, layer, bucket_desc, faces, shaping); + shaping.emplace(string, shaped); } - } + // Can faces here be a std::map of fontstacks? + // It looks like nearly the same interface through the rest + // of the stack. + std::unique_ptr bucket = std::make_unique( + tile.textVertexBuffer, tile.triangleElementsBuffer, bucket_desc, placement); + addBucketFeatures(bucket, layer, bucket_desc, faces, shaping); + return std::move(bucket); } From da0d436572fbf75bb5c0ba19e4cf4e2299696202 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Wed, 28 May 2014 20:28:37 -0700 Subject: [PATCH 19/29] missing semicolon --- include/llmr/util/utf.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/llmr/util/utf.hpp b/include/llmr/util/utf.hpp index f23f616f8..8aa85a90a 100644 --- a/include/llmr/util/utf.hpp +++ b/include/llmr/util/utf.hpp @@ -20,7 +20,7 @@ class utf8_to_utf32 { public: explicit utf8_to_utf32() {} std::u32string convert(std::string const& utf8) { - return boost::locale::conv::utf_to_utf(utf8) + return boost::locale::conv::utf_to_utf(utf8); } }; From d702152068affd0792fb6e4588559efafbcedcdf Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Wed, 28 May 2014 20:34:34 -0700 Subject: [PATCH 20/29] temporarily remove directory caching --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9dde1b9bb..3a3a66e52 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,8 +6,6 @@ compiler: cache: - apt -- directories: - - mapnik-packaging before_install: - sudo add-apt-repository --yes ppa:ubuntu-toolchain-r/test From cbd881c020549bcb8b6dcf633ff500e45e90c54b Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Wed, 28 May 2014 20:34:48 -0700 Subject: [PATCH 21/29] Revert "temporarily remove directory caching" This reverts commit d702152068affd0792fb6e4588559efafbcedcdf. --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 3a3a66e52..9dde1b9bb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,8 @@ compiler: cache: - apt +- directories: + - mapnik-packaging before_install: - sudo add-apt-repository --yes ppa:ubuntu-toolchain-r/test From 6e5ff8c1226965a31a22d5097d4a7de48f6dd6e1 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Wed, 28 May 2014 21:00:34 -0700 Subject: [PATCH 22/29] also find boost headers in include --- setup-libraries.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup-libraries.sh b/setup-libraries.sh index d43461111..fca19fd9c 100755 --- a/setup-libraries.sh +++ b/setup-libraries.sh @@ -71,7 +71,7 @@ source MacOSX.sh if [ ! -f out/build-cpp11-libcpp-x86_64/lib/libuv.a ] ; then ./scripts/build_libuv.sh ; fi if [ ! -f out/build-cpp11-libcpp-x86_64/lib/libssl.a ] ; then ./scripts/build_openssl.sh ; fi if [ ! -f out/build-cpp11-libcpp-x86_64/lib/libcurl.a ] ; then ./scripts/build_curl.sh ; fi - if [ ! -d out/build-cpp11-libcpp-x86_64/include/boost ] ; then ./scripts/build_boost.sh `pwd`/../../src/ `pwd`/../../linux/ `pwd`/../../common/ ; fi + if [ ! -d out/build-cpp11-libcpp-x86_64/include/boost ] ; then ./scripts/build_boost.sh `pwd`/../../src/ `pwd`/../../include/ `pwd`/../../linux/ `pwd`/../../common/ ; fi echo ' ...done' rm -rf out/packages @@ -91,7 +91,7 @@ source Linux.sh if [ ! -f out/build-cpp11-libstdcpp-gcc-x86_64/lib/libuv.a ] ; then ./scripts/build_libuv.sh ; fi if [ ! -f out/build-cpp11-libstdcpp-gcc-x86_64/lib/libssl.a ] ; then ./scripts/build_openssl.sh ; fi if [ ! -f out/build-cpp11-libstdcpp-gcc-x86_64/lib/libcurl.a ] ; then ./scripts/build_curl.sh ; fi - if [ ! -d out/build-cpp11-libstdcpp-gcc-x86_64/include/boost ] ; then ./scripts/build_boost.sh `pwd`/../../src/ `pwd`/../../linux/ `pwd`/../../common/ ; fi + if [ ! -d out/build-cpp11-libstdcpp-gcc-x86_64/include/boost ] ; then ./scripts/build_boost.sh `pwd`/../../src/ `pwd`/../../include/ `pwd`/../../linux/ `pwd`/../../common/ ; fi rm -rf out/packages From d82acee8c7ec8b5850131ded412bc6337b0371bd Mon Sep 17 00:00:00 2001 From: Young Hahn Date: Thu, 29 May 2014 05:05:48 -0400 Subject: [PATCH 23/29] Stash. --- include/llmr/geometry/glyph_atlas.hpp | 3 +- include/llmr/map/tile_parser.hpp | 3 ++ include/llmr/renderer/text_bucket.hpp | 3 +- include/llmr/text/glyph_store.hpp | 9 +++-- include/llmr/text/placement.hpp | 4 +-- src/geometry/glyph_atlas.cpp | 2 +- src/map/tile_parser.cpp | 48 +++++++++------------------ src/renderer/text_bucket.cpp | 7 ++-- src/text/glyph_store.cpp | 32 +++++++++++++++--- src/text/placement.cpp | 33 +++++++----------- 10 files changed, 76 insertions(+), 68 deletions(-) diff --git a/include/llmr/geometry/glyph_atlas.hpp b/include/llmr/geometry/glyph_atlas.hpp index 312572f5e..5bcd573f5 100644 --- a/include/llmr/geometry/glyph_atlas.hpp +++ b/include/llmr/geometry/glyph_atlas.hpp @@ -2,6 +2,7 @@ #define LLMR_GEOMETRY_GLYPH_ATLAS #include +#include #include #include @@ -30,7 +31,7 @@ public: Rect addGlyph(uint64_t tile_id, const std::string& face_name, - const VectorTileGlyph& glyph); + const SDFGlyph& glyph); void removeGlyphs(uint64_t tile_id); void bind(); diff --git a/include/llmr/map/tile_parser.hpp b/include/llmr/map/tile_parser.hpp index 644cc45b1..b4bccb67a 100644 --- a/include/llmr/map/tile_parser.hpp +++ b/include/llmr/map/tile_parser.hpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include namespace llmr { @@ -23,6 +25,7 @@ private: bool obsolete() const; void parseGlyphs(); void parseStyleLayers(const std::vector& layers); + void addGlyph(uint64_t tileid, const std::string stackname, const std::string &string, const FontStack &fontStack, GlyphAtlas &glyphAtlas, GlyphPositions &face); std::unique_ptr createBucket(const BucketDescription& bucket_desc); std::unique_ptr createFillBucket(const VectorTileLayer& layer, const BucketDescription& bucket_desc); std::unique_ptr createLineBucket(const VectorTileLayer& layer, const BucketDescription& bucket_desc); diff --git a/include/llmr/renderer/text_bucket.hpp b/include/llmr/renderer/text_bucket.hpp index 2b73aa74d..ad282bdd0 100644 --- a/include/llmr/renderer/text_bucket.hpp +++ b/include/llmr/renderer/text_bucket.hpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -36,7 +37,7 @@ public: PlacementRange placementRange, float zoom); void addFeature(const VectorTileFeature &feature, - const IndexedFaces &faces, + const GlyphPositions &face, const std::map &shapings); void drawGlyphs(TextShader &shader); diff --git a/include/llmr/text/glyph_store.hpp b/include/llmr/text/glyph_store.hpp index 90fa6d908..ea7c96dc4 100644 --- a/include/llmr/text/glyph_store.hpp +++ b/include/llmr/text/glyph_store.hpp @@ -4,7 +4,6 @@ #include #include - #include #include #include @@ -17,6 +16,8 @@ namespace llmr { class SDFGlyph { public: + uint32_t id = 0; + // A signed distance field of the glyph with a border of 3 pixels. std::string bitmap; @@ -26,13 +27,15 @@ public: class FontStack { public: - void insert(uint32_t id, const GlyphMetrics &glyphMetrics, const std::string &bitmap); - + void insert(uint32_t id, const SDFGlyph &glyph); const std::map &getMetrics() const; + const std::map &getSDFs() const; + const Shaping getShaping(const std::string &string) const; private: std::map bitmaps; std::map metrics; + std::map sdfs; mutable std::mutex mtx; }; diff --git a/include/llmr/text/placement.hpp b/include/llmr/text/placement.hpp index 8ec7df5e4..87c47345e 100644 --- a/include/llmr/text/placement.hpp +++ b/include/llmr/text/placement.hpp @@ -18,9 +18,9 @@ public: void addFeature(TextBucket &bucket, const std::vector &line, const BucketGeometryDescription &info, - const IndexedFaces &faces, + const GlyphPositions &face, const Shaping &shaping); - float measureText(const IndexedFaces &faces, + float measureText(const GlyphPositions &face, const Shaping &shaping); private: diff --git a/src/geometry/glyph_atlas.cpp b/src/geometry/glyph_atlas.cpp index 200d281f7..f4defac69 100644 --- a/src/geometry/glyph_atlas.cpp +++ b/src/geometry/glyph_atlas.cpp @@ -22,7 +22,7 @@ GlyphAtlas::~GlyphAtlas() { } Rect GlyphAtlas::addGlyph(uint64_t tile_id, const std::string& face_name, - const VectorTileGlyph& glyph) { + const SDFGlyph& glyph) { std::lock_guard lock(mtx); // Use constant value for now. diff --git a/src/map/tile_parser.cpp b/src/map/tile_parser.cpp index 553c8c79a..2cd10049a 100644 --- a/src/map/tile_parser.cpp +++ b/src/map/tile_parser.cpp @@ -23,7 +23,6 @@ TileParser::TileParser(const std::string& data, VectorTileData& tile, const Styl glyphStore(glyphStore), spriteAtlas(spriteAtlas), placement(tile.id.z) { - parseGlyphs(); parseStyleLayers(style.layers); } @@ -31,17 +30,13 @@ bool TileParser::obsolete() const { return tile.state == TileData::State::obsolete; } -void TileParser::parseGlyphs() { - for (const std::pair pair : vector_data.faces) { - const std::string &name = pair.first; - const VectorTileFace &face = pair.second; - - GlyphPositions &glyphs = faces[name]; - for (const VectorTileGlyph &glyph : face.glyphs) { - const Rect rect = - glyphAtlas.addGlyph(tile.id.to_uint64(), name, glyph); - glyphs.emplace(glyph.id, Glyph{rect, glyph.metrics}); - } +void TileParser::addGlyph(uint64_t tileid, const std::string stackname, const std::string &string, const FontStack &fontStack, GlyphAtlas &glyphAtlas, GlyphPositions &face) { + std::map sdfs = fontStack.getSDFs(); + // Loop through all characters and add glyph to atlas, positions. + for (uint32_t chr : string) { + const SDFGlyph sdf = sdfs[chr]; + const Rect rect = glyphAtlas.addGlyph(tileid, stackname, sdf); + face.emplace(chr, Glyph{rect, sdf.metrics}); } } @@ -202,6 +197,7 @@ std::unique_ptr TileParser::createTextBucket(const VectorTileLayer& laye // Create a copy! const FontStack &fontStack = glyphStore.getFontStack(bucket_desc.geometry.font); std::map shaping; + GlyphPositions face; // Shape and place all labels. { @@ -219,33 +215,21 @@ std::unique_ptr TileParser::createTextBucket(const VectorTileLayer& laye continue; } - uint32_t i = 0; - uint32_t x = 0; const std::string string = toString(it_prop->second); - Shaping shapedGlyphs; - std::map metrics = fontStack.getMetrics(); - // fprintf(stderr, "%s\n", string.c_str()); - // TODO: Shape label - // Loop through all characters of this label and shape. - for (uint32_t chr : string) { - // Can we reuse GlyphPlacement here? First arg is a faces index. - GlyphPlacement glyph = GlyphPlacement(0, chr, x, 0); - // No idea how to properly put together a shapedGlyphs vector. - shapedGylphs.push(glyph); - i++; - x += metrics[chr].advance; - } - // TODO: Place label + + // Shape labels. + const Shaping shaped = fontStack.getShaping(string); shaping.emplace(string, shaped); + + // Place labels. + addGlyph(tile.id.to_uint64(), bucket_desc.geometry.font, string, fontStack, glyphAtlas, face); } } - // Can faces here be a std::map of fontstacks? // It looks like nearly the same interface through the rest // of the stack. - std::unique_ptr bucket = std::make_unique( - tile.textVertexBuffer, tile.triangleElementsBuffer, bucket_desc, placement); - addBucketFeatures(bucket, layer, bucket_desc, faces, shaping); + addBucketFeatures(bucket, layer, bucket_desc, face, shaping); return std::move(bucket); } + diff --git a/src/renderer/text_bucket.cpp b/src/renderer/text_bucket.cpp index 310baf374..7e9633f4b 100644 --- a/src/renderer/text_bucket.cpp +++ b/src/renderer/text_bucket.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -99,7 +100,7 @@ void TextBucket::addGlyphs(const PlacedGlyphs &glyphs, float placementZoom, }; void TextBucket::addFeature(const VectorTileFeature &feature, - const IndexedFaces &faces, + const GlyphPositions &face, const std::map &shapings) { auto it_prop = feature.properties.find(geom_desc.field); if (it_prop == feature.properties.end()) { @@ -134,14 +135,14 @@ void TextBucket::addFeature(const VectorTileFeature &feature, while ((cmd = geometry.next(x, y)) != Geometry::end) { if (cmd == Geometry::move_to) { if (!line.empty()) { - placement.addFeature(*this, line, geom_desc, faces, shaping); + placement.addFeature(*this, line, geom_desc, face, shaping); line.clear(); } } line.emplace_back(x, y); } if (line.size()) { - placement.addFeature(*this, line, geom_desc, faces, shaping); + placement.addFeature(*this, line, geom_desc, face, shaping); } } diff --git a/src/text/glyph_store.cpp b/src/text/glyph_store.cpp index e2ebb97bd..88642907a 100644 --- a/src/text/glyph_store.cpp +++ b/src/text/glyph_store.cpp @@ -10,10 +10,11 @@ namespace llmr { -void FontStack::insert(uint32_t id, const GlyphMetrics &glyphMetrics, const std::string &bitmap) { +void FontStack::insert(uint32_t id, const SDFGlyph &glyph) { std::lock_guard lock(mtx); - metrics.emplace(id, glyphMetrics); - bitmaps.emplace(id, bitmap); + metrics.emplace(id, glyph.metrics); + bitmaps.emplace(id, glyph.bitmap); + sdfs.emplace(id, glyph); } const std::map &FontStack::getMetrics() const { @@ -21,6 +22,26 @@ const std::map &FontStack::getMetrics() const { return metrics; } +const std::map &FontStack::getSDFs() const { + std::lock_guard lock(mtx); + return sdfs; +} + +const Shaping FontStack::getShaping(const std::string &string) const { + std::lock_guard lock(mtx); + uint32_t i = 0; + uint32_t x = 0; + Shaping shaped; + // Loop through all characters of this label and shape. + for (uint32_t chr : string) { + GlyphPlacement glyph = GlyphPlacement(0, chr, x, 0); + shaped.push_back(glyph); + i++; + x += metrics.find(chr)->second.advance; + } + return shaped; +} + GlyphPBF::GlyphPBF(const std::string &fontStack, GlyphRange glyphRange) : future(promise.get_future().share()) { @@ -30,6 +51,8 @@ GlyphPBF::GlyphPBF(const std::string &fontStack, GlyphRange glyphRange) // TODO: Find more reliable URL normalization function std::replace(url.begin(), url.end(), ' ', '+'); + fprintf(stderr, url.c_str()); + platform::request_http(url, [&](platform::Response *res) { if (res->code != 200) { // Something went wrong with loading the glyph pbf. Pass on the error to the future listeners. @@ -75,6 +98,7 @@ void GlyphPBF::parse(FontStack &stack) { while (glyph_pbf.next()) { if (glyph_pbf.tag == 1) { // id id = glyph_pbf.varint(); + glyph.id = glyph_pbf.varint(); } else if (glyph_pbf.tag == 2) { // bitmap glyph.bitmap = glyph_pbf.string(); } else if (glyph_pbf.tag == 3) { // width @@ -92,7 +116,7 @@ void GlyphPBF::parse(FontStack &stack) { } } - stack.insert(id, glyph.metrics, glyph.bitmap); + stack.insert(id, glyph); } else { fontstack_pbf.skip(); } diff --git a/src/text/placement.cpp b/src/text/placement.cpp index 237fd05d7..310f457cd 100644 --- a/src/text/placement.cpp +++ b/src/text/placement.cpp @@ -35,21 +35,12 @@ bool byScale(const Anchor &a, const Anchor &b) { return a.scale < b.scale; } static const Glyph null_glyph; inline const Glyph &getGlyph(const GlyphPlacement &placed, - const IndexedFaces &faces) { - if (placed.face < faces.size()) { - const GlyphPositions &face = *faces[placed.face]; - if (&face) { - auto it = face.find(placed.glyph); - if (it != face.end()) { - return it->second; - } else { - fprintf(stderr, "glyph %d does not exist\n", placed.glyph); - } - } else { - fprintf(stderr, "face pointer is null\n"); - } + const GlyphPositions &face) { + auto it = face.find(placed.glyph); + if (it != face.end()) { + return it->second; } else { - fprintf(stderr, "face does not exist\n"); + fprintf(stderr, "glyph %d does not exist\n", placed.glyph); } return null_glyph; @@ -146,7 +137,7 @@ void getSegmentGlyphs(std::back_insert_iterator glyphs, } PlacedGlyphs getGlyphs(Anchor &anchor, float advance, const Shaping &shaping, - const IndexedFaces &faces, float fontScale, + const GlyphPositions &face, float fontScale, bool horizontal, const std::vector &line, float maxAngleDelta, float rotate) { // The total text advance is the width of this label. @@ -167,7 +158,7 @@ PlacedGlyphs getGlyphs(Anchor &anchor, float advance, const Shaping &shaping, const uint32_t buffer = 3; for (const GlyphPlacement &placed : shaping) { - const Glyph &glyph = getGlyph(placed, faces); + const Glyph &glyph = getGlyph(placed, face); if (!glyph) { // This glyph is empty and doesn't have any pixels that we'd need to // show. @@ -257,7 +248,7 @@ PlacedGlyphs getGlyphs(Anchor &anchor, float advance, const Shaping &shaping, void Placement::addFeature(TextBucket& bucket, const std::vector &line, const BucketGeometryDescription &info, - const IndexedFaces &faces, + const GlyphPositions &face, const Shaping &shaping) { const bool horizontal = info.path == TextPathType::Horizontal; @@ -268,7 +259,7 @@ void Placement::addFeature(TextBucket& bucket, const float fontScale = (tileExtent / util::tileSize) / (glyphSize / info.size); - const float advance = measureText(faces, shaping); + const float advance = measureText(face, shaping); Anchors anchors; // fprintf(stderr, "adding feature with advance %f\n", advance); @@ -290,7 +281,7 @@ void Placement::addFeature(TextBucket& bucket, for (Anchor anchor : anchors) { PlacedGlyphs glyphs = - getGlyphs(anchor, advance, shaping, faces, fontScale, horizontal, + getGlyphs(anchor, advance, shaping, face, fontScale, horizontal, line, maxAngleDelta, rotate); PlacementProperty place = collision.place(glyphs, anchor, anchor.scale, maxPlacementScale, @@ -302,14 +293,14 @@ void Placement::addFeature(TextBucket& bucket, } } -float Placement::measureText(const IndexedFaces &faces, +float Placement::measureText(const GlyphPositions &face, const Shaping &shaping) { float advance = 0; // TODO: advance is not calculated correctly. we should instead use the // bounding box of the glyph placement. for (const GlyphPlacement &shape : shaping) { - advance += getGlyph(shape, faces).metrics.advance; + advance += getGlyph(shape, face).metrics.advance; } return advance; From d5349f9cd31f60914dcacb3a8f540a1eb47c5790 Mon Sep 17 00:00:00 2001 From: Young Hahn Date: Thu, 29 May 2014 05:19:15 -0400 Subject: [PATCH 24/29] Don't call varint() more than once. --- src/text/glyph_store.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/text/glyph_store.cpp b/src/text/glyph_store.cpp index 88642907a..3e6c2bc74 100644 --- a/src/text/glyph_store.cpp +++ b/src/text/glyph_store.cpp @@ -93,11 +93,9 @@ void GlyphPBF::parse(FontStack &stack) { pbf glyph_pbf = fontstack_pbf.message(); SDFGlyph glyph; - uint32_t id = 0; while (glyph_pbf.next()) { if (glyph_pbf.tag == 1) { // id - id = glyph_pbf.varint(); glyph.id = glyph_pbf.varint(); } else if (glyph_pbf.tag == 2) { // bitmap glyph.bitmap = glyph_pbf.string(); @@ -116,7 +114,7 @@ void GlyphPBF::parse(FontStack &stack) { } } - stack.insert(id, glyph); + stack.insert(glyph.id, glyph); } else { fontstack_pbf.skip(); } From f48999dc13fec3fa024f9faa13b4d697a417fa41 Mon Sep 17 00:00:00 2001 From: Young Hahn Date: Thu, 29 May 2014 05:37:30 -0400 Subject: [PATCH 25/29] Treat out of range glyphs as belonging to 0-127 for now. --- src/text/glyph.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/text/glyph.cpp b/src/text/glyph.cpp index 039e3bc36..99e0169ca 100644 --- a/src/text/glyph.cpp +++ b/src/text/glyph.cpp @@ -95,7 +95,10 @@ GlyphRange getGlyphRange(uint32_t glyph) { if (glyph <= 65279) return { 65279, 65279 }; // Specials if (glyph <= 65519) return { 65280, 65519 }; // Halfwidth and Fullwidth Forms if (glyph <= 65533) return { 65520, 65533 }; // Specials - return { 0, 0 }; + // fprintf(stderr, "Glyph out of range %u\n", glyph); + // If glyph is out of range request basic latin for now... + // TODO: null range or way to prevent any range from being used at all? + return { 0, 127 }; } } From 8ba1acaa8361ec717ea6726131df25016645aa9f Mon Sep 17 00:00:00 2001 From: Young Hahn Date: Thu, 29 May 2014 06:09:54 -0400 Subject: [PATCH 26/29] Add second fontstack for testing. --- bin/style.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/style.js b/bin/style.js index 1a6aca8b7..e1f3cae08 100644 --- a/bin/style.js +++ b/bin/style.js @@ -731,7 +731,7 @@ module.exports = { "value": "city", "text_field": "name", "path": "horizontal", - "font": "Open Sans Regular, Arial Unicode MS Regular", + "font": "Open Sans Semibold, Arial Unicode MS Regular", "fontSize": 20, "feature_type": "point", "type": "text" @@ -743,7 +743,7 @@ module.exports = { "value": "town", "text_field": "name", "path": "horizontal", - "font": "Open Sans Regular, Arial Unicode MS Regular", + "font": "Open Sans Semibold, Arial Unicode MS Regular", "fontSize": 24, "feature_type": "point", "type": "text" @@ -755,7 +755,7 @@ module.exports = { "value": "village", "text_field": "name", "path": "horizontal", - "font": "Open Sans Regular, Arial Unicode MS Regular", + "font": "Open Sans Semibold, Arial Unicode MS Regular", "fontSize": 22, "feature_type": "point", "type": "text" @@ -771,7 +771,7 @@ module.exports = { ], "text_field": "name", "path": "horizontal", - "font": "Open Sans Regular, Arial Unicode MS Regular", + "font": "Open Sans Semibold, Arial Unicode MS Regular", "fontSize": 14, "feature_type": "point", "type": "text" From 99396478ff0579a50feb3cc4bb30dbfe4455b204 Mon Sep 17 00:00:00 2001 From: Young Hahn Date: Thu, 29 May 2014 11:41:35 -0400 Subject: [PATCH 27/29] Use 256 sized rangeglyphs. Switch to mapbox API VTs. --- src/map/map.cpp | 2 +- src/text/glyph.cpp | 97 +++------------------------------------- src/text/glyph_store.cpp | 2 +- 3 files changed, 7 insertions(+), 94 deletions(-) diff --git a/src/map/map.cpp b/src/map/map.cpp index e72c7e30b..7ac6dbe0e 100644 --- a/src/map/map.cpp +++ b/src/map/map.cpp @@ -163,7 +163,7 @@ void Map::setup() { sources.emplace("outdoors", std::unique_ptr(new Source(*this, painter, - "http://api-maps-gl.tilestream.net/v3/mapbox.mapbox-terrain-v1,mapbox.mapbox-streets-v5/%d/%d/%d.vector.pbf", + "http://a.tiles.mapbox.com/v3/mapbox.mapbox-terrain-v1,mapbox.mapbox-streets-v5/%d/%d/%d.vector.pbf", Source::Type::vector, {{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 }}, 512, diff --git a/src/text/glyph.cpp b/src/text/glyph.cpp index 99e0169ca..882a8c493 100644 --- a/src/text/glyph.cpp +++ b/src/text/glyph.cpp @@ -7,98 +7,11 @@ namespace llmr { // most of our lookups are going to be within the first 3 ranges listed here, so this is // likely faster. GlyphRange getGlyphRange(uint32_t glyph) { - if (glyph <= 127) return { 0, 127 }; // Basic Latin - if (glyph <= 255) return { 128, 255 }; // Latin-1 Supplement - if (glyph >= 8192 && glyph <= 8303) return { 8192, 8303 }; // General Punctuation - if (glyph <= 383) return { 256, 383 }; // Latin Extended-A - if (glyph <= 591) return { 384, 591 }; // Latin Extended-B - if (glyph <= 687) return { 592, 687 }; // IPA Extensions - if (glyph <= 767) return { 688, 767 }; // Spacing Modifier Letters - if (glyph <= 879) return { 768, 879 }; // Combining Diacritical Marks - if (glyph <= 1023) return { 880, 1023 }; // Greek - if (glyph <= 1279) return { 1024, 1279 }; // Cyrillic - if (glyph <= 1423) return { 1280, 1423 }; // Armenian - if (glyph <= 1535) return { 1424, 1535 }; // Hebrew - if (glyph <= 1791) return { 1536, 1791 }; // Arabic - if (glyph <= 1871) return { 1792, 1871 }; // Syriac - if (glyph <= 1983) return { 1872, 1983 }; // Thaana - if (glyph <= 2431) return { 1984, 2431 }; // Devanagari - if (glyph <= 2559) return { 2432, 2559 }; // Bengali - if (glyph <= 2687) return { 2560, 2687 }; // Gurmukhi - if (glyph <= 2815) return { 2688, 2815 }; // Gujarati - if (glyph <= 2943) return { 2816, 2943 }; // Oriya - if (glyph <= 3071) return { 2944, 3071 }; // Tamil - if (glyph <= 3199) return { 3072, 3199 }; // Telugu - if (glyph <= 3327) return { 3200, 3327 }; // Kannada - if (glyph <= 3455) return { 3328, 3455 }; // Malayalam - if (glyph <= 3583) return { 3456, 3583 }; // Sinhala - if (glyph <= 3711) return { 3584, 3711 }; // Thai - if (glyph <= 3839) return { 3712, 3839 }; // Lao - if (glyph <= 4095) return { 3840, 4095 }; // Tibetan - if (glyph <= 4255) return { 4096, 4255 }; // Myanmar - if (glyph <= 4351) return { 4256, 4351 }; // Georgian - if (glyph <= 4607) return { 4352, 4607 }; // Hangul Jamo - if (glyph <= 4991) return { 4608, 4991 }; // Ethiopic - if (glyph <= 5119) return { 4992, 5119 }; // Cherokee - if (glyph <= 5759) return { 5120, 5759 }; // Unified Canadian Aboriginal Syllabics - if (glyph <= 5791) return { 5760, 5791 }; // Ogham - if (glyph <= 5887) return { 5792, 5887 }; // Runic - if (glyph <= 6143) return { 5888, 6143 }; // Khmer - if (glyph <= 6319) return { 6144, 6319 }; // Mongolian - if (glyph <= 7935) return { 6320, 7935 }; // Latin Extended Additional - if (glyph <= 8191) return { 7936, 8191 }; // Greek Extended - if (glyph <= 8351) return { 8304, 8351 }; // Superscripts and Subscripts - if (glyph <= 8399) return { 8352, 8399 }; // Currency Symbols - if (glyph <= 8447) return { 8400, 8447 }; // Combining Marks for Symbols - if (glyph <= 8527) return { 8448, 8527 }; // Letterlike Symbols - if (glyph <= 8591) return { 8528, 8591 }; // Number Forms - if (glyph <= 8703) return { 8592, 8703 }; // Arrows - if (glyph <= 8959) return { 8704, 8959 }; // Mathematical Operators - if (glyph <= 9215) return { 8960, 9215 }; // Miscellaneous Technical - if (glyph <= 9279) return { 9216, 9279 }; // Control Pictures - if (glyph <= 9311) return { 9280, 9311 }; // Optical Character Recognition - if (glyph <= 9471) return { 9312, 9471 }; // Enclosed Alphanumerics - if (glyph <= 9599) return { 9472, 9599 }; // Box Drawing - if (glyph <= 9631) return { 9600, 9631 }; // Block Elements - if (glyph <= 9727) return { 9632, 9727 }; // Geometric Shapes - if (glyph <= 9983) return { 9728, 9983 }; // Miscellaneous Symbols - if (glyph <= 10175) return { 9984, 10175 }; // Dingbats - if (glyph <= 10495) return { 10176, 10495 }; // Braille Patterns - if (glyph <= 12031) return { 10496, 12031 }; // CJK Radicals Supplement - if (glyph <= 12255) return { 12032, 12255 }; // Kangxi Radicals - if (glyph <= 12287) return { 12256, 12287 }; // Ideographic Description Characters - if (glyph <= 12351) return { 12288, 12351 }; // CJK Symbols and Punctuation - if (glyph <= 12447) return { 12352, 12447 }; // Hiragana - if (glyph <= 12543) return { 12448, 12543 }; // Katakana - if (glyph <= 12591) return { 12544, 12591 }; // Bopomofo - if (glyph <= 12687) return { 12592, 12687 }; // Hangul Compatibility Jamo - if (glyph <= 12703) return { 12688, 12703 }; // Kanbun - if (glyph <= 12735) return { 12704, 12735 }; // Bopomofo Extended - if (glyph <= 13055) return { 12736, 13055 }; // Enclosed CJK Letters and Months - if (glyph <= 13311) return { 13056, 13311 }; // CJK Compatibility - if (glyph <= 19893) return { 13312, 19893 }; // CJK Unified Ideographs Extension A - if (glyph <= 40959) return { 19894, 40959 }; // CJK Unified Ideographs - if (glyph <= 42127) return { 40960, 42127 }; // Yi Syllables - if (glyph <= 42191) return { 42128, 42191 }; // Yi Radicals - if (glyph <= 55203) return { 42192, 55203 }; // Hangul Syllables - if (glyph <= 56191) return { 55204, 56191 }; // High Surrogates - if (glyph <= 56319) return { 56192, 56319 }; // High Private Use Surrogates - if (glyph <= 57343) return { 56320, 57343 }; // Low Surrogates - if (glyph <= 63743) return { 57344, 63743 }; // Private Use - if (glyph <= 64255) return { 63744, 64255 }; // CJK Compatibility Ideographs - if (glyph <= 64335) return { 64256, 64335 }; // Alphabetic Presentation Forms - if (glyph <= 65023) return { 64336, 65023 }; // Arabic Presentation Forms-A - if (glyph <= 65071) return { 65024, 65071 }; // Combining Half Marks - if (glyph <= 65103) return { 65072, 65103 }; // CJK Compatibility Forms - if (glyph <= 65135) return { 65104, 65135 }; // Small Form Variants - if (glyph <= 65278) return { 65136, 65278 }; // Arabic Presentation Forms-B - if (glyph <= 65279) return { 65279, 65279 }; // Specials - if (glyph <= 65519) return { 65280, 65519 }; // Halfwidth and Fullwidth Forms - if (glyph <= 65533) return { 65520, 65533 }; // Specials - // fprintf(stderr, "Glyph out of range %u\n", glyph); - // If glyph is out of range request basic latin for now... - // TODO: null range or way to prevent any range from being used at all? - return { 0, 127 }; + unsigned start = (glyph/256) * 256; + unsigned end = (start + 255); + if (start > 65280) start = 65280; + if (end > 65533) end = 65533; + return { start, end }; } } diff --git a/src/text/glyph_store.cpp b/src/text/glyph_store.cpp index dc77d18c0..e0404c08d 100644 --- a/src/text/glyph_store.cpp +++ b/src/text/glyph_store.cpp @@ -47,7 +47,7 @@ GlyphPBF::GlyphPBF(const std::string &fontStack, GlyphRange glyphRange) : future(promise.get_future().share()) { // Load the glyph set URL - std::string url = util::sprintf<255>("http://mapbox.s3.amazonaws.com/gl-glyphs/%s/%d-%d.pbf", fontStack.c_str(), glyphRange.first, glyphRange.second); + std::string url = util::sprintf<255>("http://mapbox.s3.amazonaws.com/gl-glyphs-256/%s/%d-%d.pbf", fontStack.c_str(), glyphRange.first, glyphRange.second); // TODO: Find more reliable URL normalization function std::replace(url.begin(), url.end(), ' ', '+'); From 67a3e4651f5f3bb52d2178eb00f931be105036fb Mon Sep 17 00:00:00 2001 From: Mike Morris Date: Thu, 29 May 2014 12:08:37 -0400 Subject: [PATCH 28/29] include locale, ref #270 --- include/llmr/util/utf.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/llmr/util/utf.hpp b/include/llmr/util/utf.hpp index 8aa85a90a..92da93f01 100644 --- a/include/llmr/util/utf.hpp +++ b/include/llmr/util/utf.hpp @@ -8,6 +8,7 @@ #include #else #include +#include #endif namespace llmr { From 72a71c6f70a14a166ccf29a09d2cb1c0d643e73f Mon Sep 17 00:00:00 2001 From: Young Hahn Date: Thu, 29 May 2014 12:13:56 -0400 Subject: [PATCH 29/29] Lookup shaping value by string key. Refs #272. --- src/renderer/text_bucket.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/renderer/text_bucket.cpp b/src/renderer/text_bucket.cpp index 7e9633f4b..5823607ac 100644 --- a/src/renderer/text_bucket.cpp +++ b/src/renderer/text_bucket.cpp @@ -112,7 +112,7 @@ void TextBucket::addFeature(const VectorTileFeature &feature, } const Value &value = it_prop->second; - auto it_shaping = shapings.find(value); + auto it_shaping = shapings.find(toString(value)); if (it_shaping == shapings.end()) { if (debug::shapingWarning) { fprintf(stderr, "[WARNING] missing shaping for '%s'\n", toString(value).c_str());