From ea2416c7cf0bc1bc745dd4d219057d9d979a2dbe Mon Sep 17 00:00:00 2001 From: artemp Date: Fri, 31 Oct 2014 14:43:46 -0400 Subject: [PATCH 1/3] use nunicode for proper to_upper/to_lower string conversions --- configure | 38 +++++++++++++-------- gyp/mbgl-linux.gypi | 3 ++ platform/default/string_stdlib.cpp | 55 ++++++++++++++++++++++-------- 3 files changed, 68 insertions(+), 28 deletions(-) diff --git a/configure b/configure index afb94d479..6894a7b27 100755 --- a/configure +++ b/configure @@ -37,21 +37,23 @@ alias mason='~/.mason/mason' case $MASON_PLATFORM in 'ios') - SQLITE_VERSION=system - LIBPNG_VERSION=1.6.13 - LIBUV_VERSION=0.10.28 - ZLIB_VERSION=system - BOOST_VERSION=system - ;; + SQLITE_VERSION=system + LIBPNG_VERSION=1.6.13 + LIBUV_VERSION=0.10.28 + ZLIB_VERSION=system + BOOST_VERSION=system + NUNICODE_VERSION=1.4 + ;; *) - GLFW_VERSION=a21f2377 - SQLITE_VERSION=system - LIBPNG_VERSION=system - LIBCURL_VERSION=system - LIBUV_VERSION=0.10.28 - ZLIB_VERSION=system - BOOST_VERSION=system - ;; + GLFW_VERSION=a21f2377 + SQLITE_VERSION=system + LIBPNG_VERSION=system + LIBCURL_VERSION=system + LIBUV_VERSION=0.10.28 + ZLIB_VERSION=system + BOOST_VERSION=system + NUNICODE_VERSION=1.4 + ;; esac function abort { >&2 echo -e "\033[1m\033[31m$1\033[0m"; exit 1; } @@ -138,6 +140,14 @@ if [ ! -z ${ZLIB_VERSION} ]; then CONFIG+=" 'zlib_ldflags': $(quote_flags $(mason ldflags zlib ${ZLIB_VERSION})),"$LN fi +if [ ! -z ${NUNICODE_VERSION} ]; then + mason install nunicode ${NUNICODE_VERSION} + CONFIG+=" 'nu_static_libs': $(quote_flags $(mason static_libs nunicode ${NUNICODE_VERSION})),"$LN + CONFIG+=" 'nu_cflags': $(quote_flags $(mason cflags nunicode ${NUNICODE_VERSION})),"$LN + CONFIG+=" 'nu_ldflags': $(quote_flags $(mason ldflags nunicode ${NUNICODE_VERSION})),"$LN +fi + + CONFIG+=" } } " diff --git a/gyp/mbgl-linux.gypi b/gyp/mbgl-linux.gypi index 337e98039..9f01ba3bb 100644 --- a/gyp/mbgl-linux.gypi +++ b/gyp/mbgl-linux.gypi @@ -9,13 +9,16 @@ 'cflags_cc': [ '<@(uv_cflags)', '<@(curl_cflags)', + '<@(nu_cflags)', ], 'cflags': [ '<@(uv_cflags)', + '<@(nu_cflags)', ], 'ldflags': [ '<@(uv_ldflags)', '<@(curl_ldflags)', + '<@(nu_ldflags)', ], }, 'sources': [ diff --git a/platform/default/string_stdlib.cpp b/platform/default/string_stdlib.cpp index 1acbfa850..d2c02e288 100644 --- a/platform/default/string_stdlib.cpp +++ b/platform/default/string_stdlib.cpp @@ -1,24 +1,51 @@ #include - -#include +#include +#define NU_WITH_TOUPPER +#define NU_WITH_TOLOWER +#define NU_WITH_UTF8_WRITER +#include namespace mbgl { namespace platform { -std::string uppercase(const std::string& string) { - // TODO: Use a proper Unicode Special Casing-aware algorithm. - const auto &convert = std::use_facet>(std::locale()); - std::string converted = string; - convert.toupper(&converted[0], &converted[0] + converted.size()); - return converted; +std::string uppercase(const std::string& str) +{ + boost::u8_to_u32_iterator itr(str.begin()); + boost::u8_to_u32_iterator end(str.end()); + std::string output; + char buf[4]; + for (; itr!=end; ++itr) + { + char const* up = nu_toupper(*itr); + if (up != 0) output.append(up); + else + { + std::memset(buf, 0, 4); + nu_utf8_write(*itr, buf); + output.append(buf); + } + } + return output; } -std::string lowercase(const std::string& string) { - // TODO: Use a proper Unicode Special Casing-aware algorithm. - const auto &convert = std::use_facet>(std::locale()); - std::string converted = string; - convert.tolower(&converted[0], &converted[0] + converted.size()); - return converted; +std::string lowercase(const std::string& str) +{ + boost::u8_to_u32_iterator itr(str.begin()); + boost::u8_to_u32_iterator end(str.end()); + std::string output; + char buf[4]; + for (; itr!=end; ++itr) + { + char const* up = nu_tolower(*itr); + if (up != 0) output.append(up); + else + { + std::memset(buf, 0, 4); + nu_utf8_write(*itr, buf); + output.append(buf); + } + } + return output; } } From 318d08706fdf1ee8e32ed00c96d2e8506a9a9935 Mon Sep 17 00:00:00 2001 From: artemp Date: Fri, 31 Oct 2014 15:01:50 -0400 Subject: [PATCH 2/3] add for std::memset --- platform/default/string_stdlib.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/platform/default/string_stdlib.cpp b/platform/default/string_stdlib.cpp index d2c02e288..a506c761e 100644 --- a/platform/default/string_stdlib.cpp +++ b/platform/default/string_stdlib.cpp @@ -4,6 +4,7 @@ #define NU_WITH_TOLOWER #define NU_WITH_UTF8_WRITER #include +#include namespace mbgl { namespace platform { From 01f70f5db043cccdc4efdfd40ded68d7fea90853 Mon Sep 17 00:00:00 2001 From: artemp Date: Mon, 3 Nov 2014 12:29:18 +0000 Subject: [PATCH 3/3] increase buf to hold trailing '\0' + better names --- platform/default/string_stdlib.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/platform/default/string_stdlib.cpp b/platform/default/string_stdlib.cpp index a506c761e..9639cd7b9 100644 --- a/platform/default/string_stdlib.cpp +++ b/platform/default/string_stdlib.cpp @@ -14,14 +14,14 @@ std::string uppercase(const std::string& str) boost::u8_to_u32_iterator itr(str.begin()); boost::u8_to_u32_iterator end(str.end()); std::string output; - char buf[4]; + char buf[5]; for (; itr!=end; ++itr) { char const* up = nu_toupper(*itr); if (up != 0) output.append(up); else { - std::memset(buf, 0, 4); + std::memset(buf, 0, 5); nu_utf8_write(*itr, buf); output.append(buf); } @@ -34,14 +34,14 @@ std::string lowercase(const std::string& str) boost::u8_to_u32_iterator itr(str.begin()); boost::u8_to_u32_iterator end(str.end()); std::string output; - char buf[4]; + char buf[5]; for (; itr!=end; ++itr) { - char const* up = nu_tolower(*itr); - if (up != 0) output.append(up); + char const* lo = nu_tolower(*itr); + if (lo != 0) output.append(lo); else { - std::memset(buf, 0, 4); + std::memset(buf, 0, 5); nu_utf8_write(*itr, buf); output.append(buf); }