diff --git a/CMakeLists.txt b/CMakeLists.txt index a34536e75..743ca6458 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -738,6 +738,8 @@ list(APPEND SRC_FILES ${PROJECT_SOURCE_DIR}/src/mbgl/style/expression/parsing_context.cpp ${PROJECT_SOURCE_DIR}/src/mbgl/style/expression/slice.cpp ${PROJECT_SOURCE_DIR}/src/mbgl/style/expression/step.cpp + ${PROJECT_SOURCE_DIR}/src/mbgl/style/expression/utf8_op_helpers.cpp + ${PROJECT_SOURCE_DIR}/src/mbgl/style/expression/utf8_op_helpers.hpp ${PROJECT_SOURCE_DIR}/src/mbgl/style/expression/util.cpp ${PROJECT_SOURCE_DIR}/src/mbgl/style/expression/util.hpp ${PROJECT_SOURCE_DIR}/src/mbgl/style/expression/value.cpp diff --git a/bazel/core.bzl b/bazel/core.bzl index 63acfe931..744c50c1a 100644 --- a/bazel/core.bzl +++ b/bazel/core.bzl @@ -395,6 +395,8 @@ MLN_CORE_SOURCE = [ "src/mbgl/style/expression/parsing_context.cpp", "src/mbgl/style/expression/slice.cpp", "src/mbgl/style/expression/step.cpp", + "src/mbgl/style/expression/utf8_op_helpers.cpp", + "src/mbgl/style/expression/utf8_op_helpers.hpp", "src/mbgl/style/expression/util.cpp", "src/mbgl/style/expression/util.hpp", "src/mbgl/style/expression/value.cpp", diff --git a/src/mbgl/style/expression/index_of.cpp b/src/mbgl/style/expression/index_of.cpp index e1e0e4543..358767bef 100644 --- a/src/mbgl/style/expression/index_of.cpp +++ b/src/mbgl/style/expression/index_of.cpp @@ -1,6 +1,7 @@ #include "mbgl/style/expression/expression.hpp" #include #include +#include #include namespace mbgl { @@ -77,10 +78,13 @@ EvaluationResult IndexOf::evaluateForStringInput(const std::string &string, assert(false); return ""; }); - size_t index = string.find(keywordString, fromIndexValue); - if (index == std::string::npos) { + size_t fromIndexByteOffset = getUnicodeCharacterOffsetOnValidatedUtf8(string, fromIndexValue); + size_t byte_index = string.find(keywordString, fromIndexByteOffset); + if (byte_index == std::string::npos) { return static_cast(-1); } + std::string_view prefix(string.data(), byte_index); + size_t index = unicodeLengthOnValidatedUtf8(prefix); return static_cast(index); } diff --git a/src/mbgl/style/expression/length.cpp b/src/mbgl/style/expression/length.cpp index 41815b978..11641d63b 100644 --- a/src/mbgl/style/expression/length.cpp +++ b/src/mbgl/style/expression/length.cpp @@ -1,5 +1,6 @@ #include #include +#include #include namespace mbgl { @@ -13,12 +14,13 @@ Length::Length(std::unique_ptr input_) EvaluationResult Length::evaluate(const EvaluationContext& params) const { EvaluationResult value = input->evaluate(params); if (!value) return value; - return value->match([](const std::string& s) { return EvaluationResult{static_cast(s.size())}; }, - [](const std::vector& v) { return EvaluationResult{static_cast(v.size())}; }, - [&](const auto&) -> EvaluationResult { - return EvaluationError{"Expected value to be of type string or array, but found " + - toString(typeOf(*value)) + " instead."}; - }); + return value->match( + [](const std::string& s) { return EvaluationResult{static_cast(unicodeLengthOnValidatedUtf8(s))}; }, + [](const std::vector& v) { return EvaluationResult{static_cast(v.size())}; }, + [&](const auto&) -> EvaluationResult { + return EvaluationError{"Expected value to be of type string or array, but found " + + toString(typeOf(*value)) + " instead."}; + }); } void Length::eachChild(const std::function& visit) const { diff --git a/src/mbgl/style/expression/slice.cpp b/src/mbgl/style/expression/slice.cpp index 90bed4eaf..d626bc292 100644 --- a/src/mbgl/style/expression/slice.cpp +++ b/src/mbgl/style/expression/slice.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include namespace mbgl { @@ -89,17 +90,24 @@ EvaluationResult Slice::evaluateForArrayInput(const std::vector &array, } EvaluationResult Slice::evaluateForStringInput(const std::string &string, int fromIndexValue, int toIndexValue) const { - int length = static_cast(string.size()); - if (toIndexValue == std::numeric_limits::max()) { - toIndexValue = length; - } - fromIndexValue = normalizeIndex(fromIndexValue, length); - toIndexValue = normalizeIndex(toIndexValue, length); - if (fromIndexValue >= length) { + size_t lengthBytes = string.size(); + int length = static_cast(unicodeLengthOnValidatedUtf8(string)); + size_t toIndexValueBytes = ([&]() { + if (toIndexValue == std::numeric_limits::max()) { + return lengthBytes; + } else { + int toIndexValueChars = normalizeIndex(toIndexValue, length); + return getUnicodeCharacterOffsetOnValidatedUtf8(string, toIndexValueChars); + } + })(); + int fromIndexValueChars = normalizeIndex(fromIndexValue, length); + size_t fromIndexValueBytes = getUnicodeCharacterOffsetOnValidatedUtf8(string, fromIndexValueChars); + + if ((fromIndexValueBytes >= lengthBytes) || (fromIndexValueBytes >= toIndexValueBytes)) { return std::string{}; } - return string.substr(fromIndexValue, toIndexValue - fromIndexValue); + return string.substr(fromIndexValueBytes, toIndexValueBytes - fromIndexValueBytes); } void Slice::eachChild(const std::function &visit) const { diff --git a/src/mbgl/style/expression/utf8_op_helpers.cpp b/src/mbgl/style/expression/utf8_op_helpers.cpp new file mode 100644 index 000000000..d6f2d3de9 --- /dev/null +++ b/src/mbgl/style/expression/utf8_op_helpers.cpp @@ -0,0 +1,51 @@ +#include + +#include + +namespace mbgl { +namespace style { +namespace expression { + +namespace { +bool isContinuationByte(uint8_t in) { + return (in & 0xC0) == 0x80; +} +} // namespace + +// get the number of unicode characters of an utf-8 string +// precondition: the string must be valid utf-8 (otherwise, the result may be incorrect) +size_t unicodeLengthOnValidatedUtf8(std::string_view str) { + size_t length = 0; + for (size_t i = 0; i < str.size(); i++) { + uint8_t byte = str[i]; + if (!isContinuationByte(byte)) { + length++; + } + } + return length; +} +// get the byte offset of a unicode character offset in a utf-8 string +// precondition: the string must be valid utf-8 (otherwise, the result may be incorrect) +// parameters: +// - str: input string +// - char_offset: offset in the string with respect to the unicode character count +// return value: +// - byte offset of the (start of the) unicode character with the requested offset in the string +// - if char_offset is larger than the number of unicode characters or otherwise invalid, the return value is str.size() +size_t getUnicodeCharacterOffsetOnValidatedUtf8(std::string_view str, size_t char_offset) { + size_t cur_char = 0; + for (size_t i = 0; i < str.size(); i++) { + uint8_t byte = str[i]; + if (!isContinuationByte(byte)) { + if (cur_char == char_offset) { + return i; + } + cur_char++; + } + } + return str.size(); +} + +} // namespace expression +} // namespace style +} // namespace mbgl diff --git a/src/mbgl/style/expression/utf8_op_helpers.hpp b/src/mbgl/style/expression/utf8_op_helpers.hpp new file mode 100644 index 000000000..41a3096af --- /dev/null +++ b/src/mbgl/style/expression/utf8_op_helpers.hpp @@ -0,0 +1,14 @@ +#pragma once + +#include + +namespace mbgl { +namespace style { +namespace expression { + +size_t unicodeLengthOnValidatedUtf8(std::string_view str); +size_t getUnicodeCharacterOffsetOnValidatedUtf8(std::string_view str, size_t char_offset); + +} // namespace expression +} // namespace style +} // namespace mbgl diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index e1a882d60..09cb79763 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -58,6 +58,7 @@ add_library( ${PROJECT_SOURCE_DIR}/test/style/expression/dependency.test.cpp ${PROJECT_SOURCE_DIR}/test/style/expression/expression.test.cpp ${PROJECT_SOURCE_DIR}/test/style/expression/util.test.cpp + ${PROJECT_SOURCE_DIR}/test/style/expression/utf8_op_helpers.test.cpp ${PROJECT_SOURCE_DIR}/test/style/filter.test.cpp ${PROJECT_SOURCE_DIR}/test/style/properties.test.cpp ${PROJECT_SOURCE_DIR}/test/style/property_expression.test.cpp diff --git a/test/style/expression/utf8_op_helpers.test.cpp b/test/style/expression/utf8_op_helpers.test.cpp new file mode 100644 index 000000000..58bc1c50b --- /dev/null +++ b/test/style/expression/utf8_op_helpers.test.cpp @@ -0,0 +1,32 @@ +#include + +#include + +using namespace mbgl::style::expression; + +TEST(Utf8OpHelpers, unicodeLengthOnValidatedUtf8) { + EXPECT_EQ(unicodeLengthOnValidatedUtf8(""), 0); + EXPECT_EQ(unicodeLengthOnValidatedUtf8("Hello, world!"), 13); + EXPECT_EQ(unicodeLengthOnValidatedUtf8("aäü"), 3); + EXPECT_EQ(unicodeLengthOnValidatedUtf8("オオオ"), 3); + EXPECT_EQ(unicodeLengthOnValidatedUtf8("𝄆𝄆"), 2); + EXPECT_EQ(unicodeLengthOnValidatedUtf8("aöオ𝄆オöa"), 7); +} +TEST(Utf8OpHelpers, getUnicodeCharacterOffsetOnValidatedUtf8) { + // edge case: empty string + EXPECT_EQ(getUnicodeCharacterOffsetOnValidatedUtf8("", 0), 0); + + std::string str = "aöオ𝄆オöa"; + EXPECT_EQ(getUnicodeCharacterOffsetOnValidatedUtf8(str, 0), 0); // 'a' + EXPECT_EQ(getUnicodeCharacterOffsetOnValidatedUtf8(str, 1), 1); // 'ö' + EXPECT_EQ(getUnicodeCharacterOffsetOnValidatedUtf8(str, 2), 3); // 'オ' + EXPECT_EQ(getUnicodeCharacterOffsetOnValidatedUtf8(str, 3), 6); // '𝄆' + EXPECT_EQ(getUnicodeCharacterOffsetOnValidatedUtf8(str, 4), 10); // 'オ' + EXPECT_EQ(getUnicodeCharacterOffsetOnValidatedUtf8(str, 5), 13); // 'ö' + EXPECT_EQ(getUnicodeCharacterOffsetOnValidatedUtf8(str, 6), 15); // 'a' + + // Test out of bounds character offset (should return the string length) + EXPECT_EQ(getUnicodeCharacterOffsetOnValidatedUtf8(str, 7), 16); + EXPECT_EQ(getUnicodeCharacterOffsetOnValidatedUtf8(str, 12), 16); + EXPECT_EQ(getUnicodeCharacterOffsetOnValidatedUtf8(str, 50000), 16); +}