Make string expressions operate on unicode (#4344)

CMakeLists.txt

@@ -738,6 +738,8 @@ list(APPEND SRC_FILES
     ${PROJECT_SOURCE_DIR}/src/mbgl/style/expression/parsing_context.cpp
     ${PROJECT_SOURCE_DIR}/src/mbgl/style/expression/slice.cpp
     ${PROJECT_SOURCE_DIR}/src/mbgl/style/expression/step.cpp
+    ${PROJECT_SOURCE_DIR}/src/mbgl/style/expression/utf8_op_helpers.cpp
+    ${PROJECT_SOURCE_DIR}/src/mbgl/style/expression/utf8_op_helpers.hpp
     ${PROJECT_SOURCE_DIR}/src/mbgl/style/expression/util.cpp
     ${PROJECT_SOURCE_DIR}/src/mbgl/style/expression/util.hpp
     ${PROJECT_SOURCE_DIR}/src/mbgl/style/expression/value.cpp

bazel/core.bzl

@@ -395,6 +395,8 @@ MLN_CORE_SOURCE = [
     "src/mbgl/style/expression/parsing_context.cpp",
     "src/mbgl/style/expression/slice.cpp",
     "src/mbgl/style/expression/step.cpp",
+    "src/mbgl/style/expression/utf8_op_helpers.cpp",
+    "src/mbgl/style/expression/utf8_op_helpers.hpp",
     "src/mbgl/style/expression/util.cpp",
     "src/mbgl/style/expression/util.hpp",
     "src/mbgl/style/expression/value.cpp",

src/mbgl/style/expression/index_of.cpp

@@ -1,6 +1,7 @@
 #include "mbgl/style/expression/expression.hpp"
 #include <mbgl/style/conversion_impl.hpp>
 #include <mbgl/style/expression/index_of.hpp>
+#include <mbgl/style/expression/utf8_op_helpers.hpp>
 #include <mbgl/util/string.hpp>
 
 namespace mbgl {
@@ -77,10 +78,13 @@ EvaluationResult IndexOf::evaluateForStringInput(const std::string &string,
             assert(false);
             return "";
         });
-    size_t index = string.find(keywordString, fromIndexValue);
-    if (index == std::string::npos) {
+    size_t fromIndexByteOffset = getUnicodeCharacterOffsetOnValidatedUtf8(string, fromIndexValue);
+    size_t byte_index = string.find(keywordString, fromIndexByteOffset);
+    if (byte_index == std::string::npos) {
         return static_cast<double>(-1);
     }
+    std::string_view prefix(string.data(), byte_index);
+    size_t index = unicodeLengthOnValidatedUtf8(prefix);
     return static_cast<double>(index);
 }
 

src/mbgl/style/expression/length.cpp

@@ -1,5 +1,6 @@
 #include <mbgl/style/expression/length.hpp>
 #include <mbgl/style/conversion_impl.hpp>
+#include <mbgl/style/expression/utf8_op_helpers.hpp>
 #include <mbgl/util/string.hpp>
 
 namespace mbgl {
@@ -13,12 +14,13 @@ Length::Length(std::unique_ptr<Expression> input_)
 EvaluationResult Length::evaluate(const EvaluationContext& params) const {
     EvaluationResult value = input->evaluate(params);
     if (!value) return value;
-    return value->match([](const std::string& s) { return EvaluationResult{static_cast<double>(s.size())}; },
-                        [](const std::vector<Value>& v) { return EvaluationResult{static_cast<double>(v.size())}; },
-                        [&](const auto&) -> EvaluationResult {
-                            return EvaluationError{"Expected value to be of type string or array, but found " +
-                                                   toString(typeOf(*value)) + " instead."};
-                        });
+    return value->match(
+        [](const std::string& s) { return EvaluationResult{static_cast<double>(unicodeLengthOnValidatedUtf8(s))}; },
+        [](const std::vector<Value>& v) { return EvaluationResult{static_cast<double>(v.size())}; },
+        [&](const auto&) -> EvaluationResult {
+            return EvaluationError{"Expected value to be of type string or array, but found " +
+                                   toString(typeOf(*value)) + " instead."};
+        });
 }
 
 void Length::eachChild(const std::function<void(const Expression&)>& visit) const {

src/mbgl/style/expression/slice.cpp

@@ -2,6 +2,7 @@
 #include <limits>
 #include <mbgl/style/conversion_impl.hpp>
 #include <mbgl/style/expression/slice.hpp>
+#include <mbgl/style/expression/utf8_op_helpers.hpp>
 #include <mbgl/util/string.hpp>
 
 namespace mbgl {
@@ -89,17 +90,24 @@ EvaluationResult Slice::evaluateForArrayInput(const std::vector<Value> &array,
 }
 
 EvaluationResult Slice::evaluateForStringInput(const std::string &string, int fromIndexValue, int toIndexValue) const {
-    int length = static_cast<int>(string.size());
-    if (toIndexValue == std::numeric_limits<int>::max()) {
-        toIndexValue = length;
-    }
-    fromIndexValue = normalizeIndex(fromIndexValue, length);
-    toIndexValue = normalizeIndex(toIndexValue, length);
-    if (fromIndexValue >= length) {
+    size_t lengthBytes = string.size();
+    int length = static_cast<int>(unicodeLengthOnValidatedUtf8(string));
+    size_t toIndexValueBytes = ([&]() {
+        if (toIndexValue == std::numeric_limits<int>::max()) {
+            return lengthBytes;
+        } else {
+            int toIndexValueChars = normalizeIndex(toIndexValue, length);
+            return getUnicodeCharacterOffsetOnValidatedUtf8(string, toIndexValueChars);
+        }
+    })();
+    int fromIndexValueChars = normalizeIndex(fromIndexValue, length);
+    size_t fromIndexValueBytes = getUnicodeCharacterOffsetOnValidatedUtf8(string, fromIndexValueChars);
+
+    if ((fromIndexValueBytes >= lengthBytes) || (fromIndexValueBytes >= toIndexValueBytes)) {
         return std::string{};
     }
 
-    return string.substr(fromIndexValue, toIndexValue - fromIndexValue);
+    return string.substr(fromIndexValueBytes, toIndexValueBytes - fromIndexValueBytes);
 }
 
 void Slice::eachChild(const std::function<void(const Expression &)> &visit) const {

src/mbgl/style/expression/utf8_op_helpers.cpp

@@ -0,0 +1,51 @@
+#include <mbgl/style/expression/utf8_op_helpers.hpp>
+
+#include <cstdint>
+
+namespace mbgl {
+namespace style {
+namespace expression {
+
+namespace {
+bool isContinuationByte(uint8_t in) {
+    return (in & 0xC0) == 0x80;
+}
+} // namespace
+
+// get the number of unicode characters of an utf-8 string
+// precondition: the string must be valid utf-8 (otherwise, the result may be incorrect)
+size_t unicodeLengthOnValidatedUtf8(std::string_view str) {
+    size_t length = 0;
+    for (size_t i = 0; i < str.size(); i++) {
+        uint8_t byte = str[i];
+        if (!isContinuationByte(byte)) {
+            length++;
+        }
+    }
+    return length;
+}
+// get the byte offset of a unicode character offset in a utf-8 string
+// precondition: the string must be valid utf-8 (otherwise, the result may be incorrect)
+// parameters:
+// - str: input string
+// - char_offset: offset in the string with respect to the unicode character count
+// return value:
+// - byte offset of the (start of the) unicode character with the requested offset in the string
+// - if char_offset is larger than the number of unicode characters or otherwise invalid, the return value is str.size()
+size_t getUnicodeCharacterOffsetOnValidatedUtf8(std::string_view str, size_t char_offset) {
+    size_t cur_char = 0;
+    for (size_t i = 0; i < str.size(); i++) {
+        uint8_t byte = str[i];
+        if (!isContinuationByte(byte)) {
+            if (cur_char == char_offset) {
+                return i;
+            }
+            cur_char++;
+        }
+    }
+    return str.size();
+}
+
+} // namespace expression
+} // namespace style
+} // namespace mbgl

src/mbgl/style/expression/utf8_op_helpers.hpp

@@ -0,0 +1,14 @@
+#pragma once
+
+#include <string_view>
+
+namespace mbgl {
+namespace style {
+namespace expression {
+
+size_t unicodeLengthOnValidatedUtf8(std::string_view str);
+size_t getUnicodeCharacterOffsetOnValidatedUtf8(std::string_view str, size_t char_offset);
+
+} // namespace expression
+} // namespace style
+} // namespace mbgl

test/CMakeLists.txt

@@ -58,6 +58,7 @@ add_library(
     ${PROJECT_SOURCE_DIR}/test/style/expression/dependency.test.cpp
     ${PROJECT_SOURCE_DIR}/test/style/expression/expression.test.cpp
     ${PROJECT_SOURCE_DIR}/test/style/expression/util.test.cpp
+    ${PROJECT_SOURCE_DIR}/test/style/expression/utf8_op_helpers.test.cpp
     ${PROJECT_SOURCE_DIR}/test/style/filter.test.cpp
     ${PROJECT_SOURCE_DIR}/test/style/properties.test.cpp
     ${PROJECT_SOURCE_DIR}/test/style/property_expression.test.cpp

test/style/expression/utf8_op_helpers.test.cpp

@@ -0,0 +1,32 @@
+#include <mbgl/test/util.hpp>
+
+#include <mbgl/style/expression/utf8_op_helpers.hpp>
+
+using namespace mbgl::style::expression;
+
+TEST(Utf8OpHelpers, unicodeLengthOnValidatedUtf8) {
+    EXPECT_EQ(unicodeLengthOnValidatedUtf8(""), 0);
+    EXPECT_EQ(unicodeLengthOnValidatedUtf8("Hello, world!"), 13);
+    EXPECT_EQ(unicodeLengthOnValidatedUtf8("aäü"), 3);
+    EXPECT_EQ(unicodeLengthOnValidatedUtf8("オオオ"), 3);
+    EXPECT_EQ(unicodeLengthOnValidatedUtf8("𝄆𝄆"), 2);
+    EXPECT_EQ(unicodeLengthOnValidatedUtf8("aöオ𝄆オöa"), 7);
+}
+TEST(Utf8OpHelpers, getUnicodeCharacterOffsetOnValidatedUtf8) {
+    // edge case: empty string
+    EXPECT_EQ(getUnicodeCharacterOffsetOnValidatedUtf8("", 0), 0);
+
+    std::string str = "aöオ𝄆オöa";
+    EXPECT_EQ(getUnicodeCharacterOffsetOnValidatedUtf8(str, 0), 0);  // 'a'
+    EXPECT_EQ(getUnicodeCharacterOffsetOnValidatedUtf8(str, 1), 1);  // 'ö'
+    EXPECT_EQ(getUnicodeCharacterOffsetOnValidatedUtf8(str, 2), 3);  // 'オ'
+    EXPECT_EQ(getUnicodeCharacterOffsetOnValidatedUtf8(str, 3), 6);  // '𝄆'
+    EXPECT_EQ(getUnicodeCharacterOffsetOnValidatedUtf8(str, 4), 10); // 'オ'
+    EXPECT_EQ(getUnicodeCharacterOffsetOnValidatedUtf8(str, 5), 13); // 'ö'
+    EXPECT_EQ(getUnicodeCharacterOffsetOnValidatedUtf8(str, 6), 15); // 'a'
+
+    // Test out of bounds character offset (should return the string length)
+    EXPECT_EQ(getUnicodeCharacterOffsetOnValidatedUtf8(str, 7), 16);
+    EXPECT_EQ(getUnicodeCharacterOffsetOnValidatedUtf8(str, 12), 16);
+    EXPECT_EQ(getUnicodeCharacterOffsetOnValidatedUtf8(str, 50000), 16);
+}