From 66d64c7c12648b3f15181eb9d64e55d1a23ade34 Mon Sep 17 00:00:00 2001 From: artemp Date: Mon, 17 Nov 2014 11:22:47 +0100 Subject: [PATCH 1/4] use nunicode conditional case mapping (lowercase) --- platform/default/string_stdlib.cpp | 21 ++++++++++++--------- test/text_conversions.cpp | 4 +--- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/platform/default/string_stdlib.cpp b/platform/default/string_stdlib.cpp index 9639cd7b9..3b89a4504 100644 --- a/platform/default/string_stdlib.cpp +++ b/platform/default/string_stdlib.cpp @@ -31,19 +31,22 @@ std::string uppercase(const std::string& str) std::string lowercase(const std::string& str) { - boost::u8_to_u32_iterator itr(str.begin()); - boost::u8_to_u32_iterator end(str.end()); std::string output; - char buf[5]; - for (; itr!=end; ++itr) + char lo[5]; + char const* itr = str.c_str(); + char const* end = itr + str.length(); + char const* buf = 0; + uint32_t code_point; + for ( ; itr!=end;) { - char const* lo = nu_tolower(*itr); - if (lo != 0) output.append(lo); + + itr = _nu_tolower(itr, end, nu_utf8_read, &code_point, &buf, 0); + if (buf != 0) output.append(buf); else { - std::memset(buf, 0, 5); - nu_utf8_write(*itr, buf); - output.append(buf); + std::memset(lo, 0, 5); + nu_utf8_write(code_point, lo); + output.append(lo); } } return output; diff --git a/test/text_conversions.cpp b/test/text_conversions.cpp index 4f71d6a61..756bc4db1 100644 --- a/test/text_conversions.cpp +++ b/test/text_conversions.cpp @@ -30,8 +30,6 @@ TEST(TextConversions, to_lower) { EXPECT_EQ(std::string("weisskopfseeadler"), platform::lowercase("weiSSkopfseeadler")); // DE EXPECT_EQ(std::string("azərbaycan"), platform::lowercase("AZƏRBAYCAN")); // AZ - - //EXPECT_EQ(std::string("ὀδυσσεύσ"), platform::lowercase("ὈΔΥΣΣΕΎΣ")); // GR - // nunicode can't map last Σ to word-ending ς + EXPECT_EQ(std::string("ὀδυσσεύς"), platform::lowercase("ὈΔΥΣΣΕΎΣ")); // GR } From b9ebe1622420d628d85c2ce26abaac59701b99f4 Mon Sep 17 00:00:00 2001 From: artemp Date: Mon, 17 Nov 2014 18:15:48 +0100 Subject: [PATCH 2/4] correct _nu_tolower/_nu_toupper usage --- platform/default/string_stdlib.cpp | 82 +++++++++++++++++++----------- 1 file changed, 51 insertions(+), 31 deletions(-) diff --git a/platform/default/string_stdlib.cpp b/platform/default/string_stdlib.cpp index 3b89a4504..ba5554220 100644 --- a/platform/default/string_stdlib.cpp +++ b/platform/default/string_stdlib.cpp @@ -6,51 +6,71 @@ #include #include -namespace mbgl { -namespace platform { +namespace mbgl { namespace platform { -std::string uppercase(const std::string& str) +std::string uppercase(std::string const& str) { - boost::u8_to_u32_iterator itr(str.begin()); - boost::u8_to_u32_iterator end(str.end()); - std::string output; - char buf[5]; - for (; itr!=end; ++itr) + std::stringstream output; + char const *itr = str.c_str(), *nitr = itr; + char const *end = itr + str.length(); + char lo[5] = { 0 }; + + for (; itr < end; itr = nitr) { - char const* up = nu_toupper(*itr); - if (up != 0) output.append(up); + uint32_t code_point = 0; + char const* buf = 0; + + nitr = _nu_toupper(itr, end, nu_utf8_read, &code_point, &buf, 0); + if (buf != 0) + { + do + { + buf = NU_CASEMAP_DECODING_FUNCTION(buf, &code_point); + if (code_point == 0) break; + output.write(lo, nu_utf8_write(code_point, lo) - lo); + } + while (code_point != 0); + } else { - std::memset(buf, 0, 5); - nu_utf8_write(*itr, buf); - output.append(buf); + output.write(itr, nitr - itr); } } - return output; + + return output.str(); + } -std::string lowercase(const std::string& str) +std::string lowercase(std::string const& str) { - std::string output; - char lo[5]; - char const* itr = str.c_str(); - char const* end = itr + str.length(); - char const* buf = 0; - uint32_t code_point; - for ( ; itr!=end;) - { + std::stringstream output; + char const *itr = str.c_str(), *nitr = itr; + char const *end = itr + str.length(); + char lo[5] = { 0 }; - itr = _nu_tolower(itr, end, nu_utf8_read, &code_point, &buf, 0); - if (buf != 0) output.append(buf); + for (; itr < end; itr = nitr) + { + uint32_t code_point = 0; + char const* buf = 0; + + nitr = _nu_tolower(itr, end, nu_utf8_read, &code_point, &buf, 0); + if (buf != 0) + { + do + { + buf = NU_CASEMAP_DECODING_FUNCTION(buf, &code_point); + if (code_point == 0) break; + output.write(lo, nu_utf8_write(code_point, lo) - lo); + } + while (code_point != 0); + } else { - std::memset(lo, 0, 5); - nu_utf8_write(code_point, lo); - output.append(lo); + output.write(itr, nitr - itr); } } - return output; + + return output.str(); } -} -} +}} From 9092cb42093aaff9031267a00f6ce36ba29f69d1 Mon Sep 17 00:00:00 2001 From: artemp Date: Fri, 21 Nov 2014 16:27:34 +0100 Subject: [PATCH 3/4] update nunicode version --- configure | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure b/configure index b0a5e5332..08b972b05 100755 --- a/configure +++ b/configure @@ -33,7 +33,7 @@ case ${MASON_PLATFORM} in LIBUV_VERSION=0.10.28 ZLIB_VERSION=system BOOST_VERSION=system - NUNICODE_VERSION=1.4 + NUNICODE_VERSION=1.5-dev ;; esac From 2586cfb9e305ce2da39fb06af62d71a911fbfba4 Mon Sep 17 00:00:00 2001 From: artemp Date: Thu, 27 Nov 2014 18:11:18 +0100 Subject: [PATCH 4/4] use ```const var&``` notation in public interface --- platform/default/string_stdlib.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platform/default/string_stdlib.cpp b/platform/default/string_stdlib.cpp index ba5554220..ac4c26234 100644 --- a/platform/default/string_stdlib.cpp +++ b/platform/default/string_stdlib.cpp @@ -8,7 +8,7 @@ namespace mbgl { namespace platform { -std::string uppercase(std::string const& str) +std::string uppercase(const std::string& str) { std::stringstream output; char const *itr = str.c_str(), *nitr = itr; @@ -41,7 +41,7 @@ std::string uppercase(std::string const& str) } -std::string lowercase(std::string const& str) +std::string lowercase(const std::string& str) { std::stringstream output; char const *itr = str.c_str(), *nitr = itr;