Merge branch 'glyphrange-boost-locale' into glyphrange-yh-guesses

Conflicts:
	src/map/tile_parser.cpp
	src/text/glyph_store.cpp

include/llmr/map/tile_parser.hpp

@@ -6,6 +6,7 @@
 #include <llmr/text/placement.hpp>
 #include <llmr/text/glyph_store.hpp>
 #include <llmr/text/glyph.hpp>
+#include <llmr/util/utf.hpp>
 
 namespace llmr {
 
@@ -25,7 +26,7 @@ private:
     bool obsolete() const;
     void parseGlyphs();
     void parseStyleLayers(const std::vector<LayerDescription>& layers);
-    void addGlyph(uint64_t tileid, const std::string stackname, const std::string &string, const FontStack &fontStack, GlyphAtlas &glyphAtlas, GlyphPositions &face);
+    void addGlyph(uint64_t tileid, const std::string stackname, const std::u32string &string, const FontStack &fontStack, GlyphAtlas &glyphAtlas, GlyphPositions &face);
     std::unique_ptr<Bucket> createBucket(const BucketDescription& bucket_desc);
     std::unique_ptr<Bucket> createFillBucket(const VectorTileLayer& layer, const BucketDescription& bucket_desc);
     std::unique_ptr<Bucket> createLineBucket(const VectorTileLayer& layer, const BucketDescription& bucket_desc);

include/llmr/text/glyph_store.hpp

@@ -30,7 +30,7 @@ public:
     void insert(uint32_t id, const SDFGlyph &glyph);
     const std::map<uint32_t, GlyphMetrics> &getMetrics() const;
     const std::map<uint32_t, SDFGlyph> &getSDFs() const;
-    const Shaping getShaping(const std::string &string) const;
+    const Shaping getShaping(const std::u32string &string) const;
 
 private:
     std::map<uint32_t, std::string> bitmaps;

include/llmr/util/utf.hpp

@@ -0,0 +1,44 @@
+#ifndef LLMR_UTIL_UTF
+#define LLMR_UTIL_UTF
+
+#include <memory>
+
+// g++/libstdc++ is missing c++11 codecvt support
+#ifdef __linux__
+#include <boost/locale.hpp>
+#else
+#include <codecvt>
+#endif
+
+namespace llmr {
+
+namespace util {
+
+#ifdef __linux__
+
+class utf8_to_utf32 {
+ public:
+    explicit utf8_to_utf32() {}
+    std::u32string convert(std::string const& utf8) {
+        return boost::locale::conv::utf_to_utf<char32_t>(utf8);
+    }
+};
+
+#else
+
+class utf8_to_utf32 {
+ public:
+    explicit utf8_to_utf32()
+        : utf32conv_() {}
+    std::u32string convert(std::string const& utf8) {
+        return utf32conv_.from_bytes(utf8);
+    }
+ private:
+    std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> utf32conv_;
+};
+
+#endif
+
+}}
+
+#endif

setup-libraries.sh

@@ -71,7 +71,7 @@ source MacOSX.sh
     if [ ! -f out/build-cpp11-libcpp-x86_64/lib/libuv.a ] ; then ./scripts/build_libuv.sh ; fi
     if [ ! -f out/build-cpp11-libcpp-x86_64/lib/libssl.a ] ; then ./scripts/build_openssl.sh ; fi
     if [ ! -f out/build-cpp11-libcpp-x86_64/lib/libcurl.a ] ; then ./scripts/build_curl.sh ; fi
-    if [ ! -d out/build-cpp11-libcpp-x86_64/include/boost ] ; then ./scripts/build_boost.sh `pwd`/../../src/ `pwd`/../../linux/ `pwd`/../../common/ ; fi
+    if [ ! -d out/build-cpp11-libcpp-x86_64/include/boost ] ; then ./scripts/build_boost.sh `pwd`/../../src/ `pwd`/../../include/ `pwd`/../../linux/ `pwd`/../../common/ ; fi
     echo '     ...done'
 
 rm -rf out/packages
@@ -91,7 +91,7 @@ source Linux.sh
     if [ ! -f out/build-cpp11-libstdcpp-gcc-x86_64/lib/libuv.a ] ; then ./scripts/build_libuv.sh ; fi
     if [ ! -f out/build-cpp11-libstdcpp-gcc-x86_64/lib/libssl.a ] ; then ./scripts/build_openssl.sh ; fi
     if [ ! -f out/build-cpp11-libstdcpp-gcc-x86_64/lib/libcurl.a ] ; then ./scripts/build_curl.sh ; fi
-    if [ ! -d out/build-cpp11-libstdcpp-gcc-x86_64/include/boost ] ; then ./scripts/build_boost.sh `pwd`/../../src/ `pwd`/../../linux/ `pwd`/../../common/ ; fi
+    if [ ! -d out/build-cpp11-libstdcpp-gcc-x86_64/include/boost ] ; then ./scripts/build_boost.sh `pwd`/../../src/ `pwd`/../../include/ `pwd`/../../linux/ `pwd`/../../common/ ; fi
 
 rm -rf out/packages
 

src/map/tile_parser.cpp

@@ -11,7 +11,9 @@
 #include <llmr/geometry/glyph_atlas.hpp>
 #include <llmr/text/glyph_store.hpp>
 #include <llmr/text/glyph.hpp>
+
 #include <llmr/util/std.hpp>
+#include <llmr/util/utf.hpp>
 
 using namespace llmr;
 
@@ -30,7 +32,7 @@ bool TileParser::obsolete() const {
     return tile.state == TileData::State::obsolete;
 }
 
-void TileParser::addGlyph(uint64_t tileid, const std::string stackname, const std::string &string, const FontStack &fontStack, GlyphAtlas &glyphAtlas, GlyphPositions &face) {
+void TileParser::addGlyph(uint64_t tileid, const std::string stackname, const std::u32string &string, const FontStack &fontStack, GlyphAtlas &glyphAtlas, GlyphPositions &face) {
     std::map<uint32_t, SDFGlyph> sdfs = fontStack.getSDFs();
     // Loop through all characters and add glyph to atlas, positions.
     for (uint32_t chr : string) {
@@ -164,6 +166,7 @@ std::unique_ptr<Bucket> TileParser::createTextBucket(const VectorTileLayer& laye
     std::unique_ptr<TextBucket> bucket = std::make_unique<TextBucket>(
         tile.textVertexBuffer, tile.triangleElementsBuffer, bucket_desc, placement);
 
+    util::utf8_to_utf32 ucs4conv;
 
     // Determine and load glyph ranges
     {
@@ -183,7 +186,7 @@ std::unique_ptr<Bucket> TileParser::createTextBucket(const VectorTileLayer& laye
                 continue;
             }
 
-            const std::string string = toString(it_prop->second);
+            const std::u32string string = ucs4conv.convert(toString(it_prop->second));
 
             // Loop through all characters of this text and collect unique codepoints.
             for (uint32_t chr : string) {
@@ -215,11 +218,11 @@ std::unique_ptr<Bucket> TileParser::createTextBucket(const VectorTileLayer& laye
                 continue;
             }
 
-            const std::string string = toString(it_prop->second);
+            const std::u32string string = ucs4conv.convert(toString(it_prop->second));
 
             // Shape labels.
             const Shaping shaped = fontStack.getShaping(string);
-            shaping.emplace(string, shaped);
+            shaping.emplace(toString(it_prop->second), shaped);
 
             // Place labels.
             addGlyph(tile.id.to_uint64(), bucket_desc.geometry.font, string, fontStack, glyphAtlas, face);

src/text/glyph_store.cpp

@@ -2,10 +2,11 @@
 
 #include <llmr/util/std.hpp>
 #include <llmr/util/string.hpp>
+#include <llmr/util/utf.hpp>
 #include <llmr/util/pbf.hpp>
 #include <llmr/platform/platform.hpp>
 #include <uv.h>
-#include <regex>
+#include <algorithm>
 
 namespace llmr {
 
@@ -27,7 +28,7 @@ const std::map<uint32_t, SDFGlyph> &FontStack::getSDFs() const {
     return sdfs;
 }
 
-const Shaping FontStack::getShaping(const std::string &string) const {
+const Shaping FontStack::getShaping(const std::u32string &string) const {
     std::lock_guard<std::mutex> lock(mtx);
     uint32_t i = 0;
     uint32_t x = 0;