tei/MapLibreParticleLab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[core] Use UTF-16 instead of UTF-32 for label features to avoid extra conversions and reduce in-memory size.

Browse Source 
Continue to use uint32 as glyph ID to maintain Glyph PBF, even though we're only using 16 bits of that uint32.
Use std::codecvt instead of boost::unicode_iterator for UTF8->UTF16 conversions.
This commit is contained in:
Chris Loer
2016-11-09 18:23:49 -08:00
committed by John Firebaugh
parent 5c7aeec5c7
commit 40b0c8d076
16 changed files with 40 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
platform/default/mbgl/storage/offline_download.cpp
View File

@@ -184,7 +184,7 @@ void OfflineDownload::activateDownload() {
if (!parser.glyphURL.empty()) {
for (const auto& fontStack : parser.fontStacks()) {
for (uint32_t i = 0; i < GLYPH_RANGES_PER_FONT_STACK; i++) {
for (char16_t i = 0; i < GLYPH_RANGES_PER_FONT_STACK; i++) {
queueResource(Resource::glyphs(parser.glyphURL, fontStack, getGlyphRange(i * GLYPHS_PER_GLYPH_RANGE)));
}
}

2
platform/default/string_stdlib.cpp
View File

@@ -1,10 +1,10 @@
#include <mbgl/platform/platform.hpp>
#include <mbgl/util/utf.hpp>
#define NU_WITH_TOUPPER
#define NU_WITH_TOLOWER
#define NU_WITH_UTF8_WRITER
#include <libnu/libnu.h>
#include <cstring>
#include <sstream>
namespace mbgl { namespace platform {

4
src/mbgl/layout/merge_lines.cpp
View File

@@ -47,10 +47,10 @@ enum class Side {
};
size_t
getKey(const std::u32string& text, const GeometryCollection& geom, Side side) {
getKey(const std::u16string& text, const GeometryCollection& geom, Side side) {
const GeometryCoordinate& coord = side == Side::Right ? geom[0].back() : geom[0].front();
auto hash = std::hash<std::u32string>()(text);
auto hash = std::hash<std::u16string>()(text);
boost::hash_combine(hash, coord.x);
boost::hash_combine(hash, coord.y);
return hash;

2
src/mbgl/layout/symbol_feature.hpp
View File

@@ -10,7 +10,7 @@ namespace mbgl {
class SymbolFeature {
public:
GeometryCollection geometry;
optional<std::u32string> text;
optional<std::u16string> text;
optional<std::string> icon;
std::size_t index;
};

6
src/mbgl/layout/symbol_layout.cpp
View File

@@ -90,10 +90,10 @@ SymbolLayout::SymbolLayout(std::string bucketName_,
u8string = platform::lowercase(u8string);
}
ft.text = util::utf8_to_utf32::convert(u8string);
ft.text = util::utf8_to_utf16::convert( u8string );
// Loop through all characters of this text and collect unique codepoints.
for (char32_t chr : *ft.text) {
for (char16_t chr : *ft.text) {
ranges.insert(getGlyphRange(chr));
}
}
@@ -309,7 +309,7 @@ void SymbolLayout::addFeature(const GeometryCollection &lines,
}
}
bool SymbolLayout::anchorIsTooClose(const std::u32string &text, const float repeatDistance, Anchor &anchor) {
bool SymbolLayout::anchorIsTooClose(const std::u16string &text, const float repeatDistance, Anchor &anchor) {
if (compareText.find(text) == compareText.end()) {
compareText.emplace(text, Anchors());
} else {

4
src/mbgl/layout/symbol_layout.hpp
View File

@@ -64,8 +64,8 @@ private:
const GlyphPositions& face,
const size_t index);
bool anchorIsTooClose(const std::u32string& text, const float repeatDistance, Anchor&);
std::map<std::u32string, std::vector<Anchor>> compareText;
bool anchorIsTooClose(const std::u16string& text, const float repeatDistance, Anchor&);
std::map<std::u16string, std::vector<Anchor>> compareText;
void addToDebugBuffers(CollisionTile&, SymbolBucket&);

2
src/mbgl/text/glyph.cpp
View File

@@ -3,7 +3,7 @@
namespace mbgl {
// Note: this only works for the BMP
GlyphRange getGlyphRange(char32_t glyph) {
GlyphRange getGlyphRange(char16_t glyph) {
unsigned start = (glyph/256) * 256;
unsigned end = (start + 255);
if (start > 65280) start = 65280;

6
src/mbgl/text/glyph.hpp
View File

@@ -11,7 +11,7 @@
namespace mbgl {
// Note: this only works for the BMP
GlyphRange getGlyphRange(char32_t glyph);
GlyphRange getGlyphRange(char16_t glyph);
struct GlyphMetrics {
explicit operator bool() const {
@@ -63,10 +63,10 @@ public:
class Shaping {
public:
explicit Shaping() : top(0), bottom(0), left(0), right(0) {}
explicit Shaping(float x, float y, std::u32string text_)
explicit Shaping(float x, float y, std::u16string text_)
: text(std::move(text_)), top(y), bottom(y), left(x), right(x) {}
std::vector<PositionedGlyph> positionedGlyphs;
std::u32string text;
std::u16string text;
int32_t top;
int32_t bottom;
int32_t left;

4
src/mbgl/text/glyph_atlas.cpp
View File

@@ -81,7 +81,7 @@ void GlyphAtlas::setObserver(GlyphAtlasObserver* observer_) {
}
void GlyphAtlas::addGlyphs(uintptr_t tileUID,
const std::u32string& text,
const std::u16string& text,
const FontStack& fontStack,
const GlyphSet& glyphSet,
GlyphPositions& face)
@@ -90,7 +90,7 @@ void GlyphAtlas::addGlyphs(uintptr_t tileUID,
const std::map<uint32_t, SDFGlyph>& sdfs = glyphSet.getSDFs();
for (uint32_t chr : text)
for (char16_t chr : text)
{
auto sdf_it = sdfs.find(chr);
if (sdf_it == sdfs.end()) {

2
src/mbgl/text/glyph_atlas.hpp
View File

@@ -55,7 +55,7 @@ public:
void setObserver(GlyphAtlasObserver* observer);
void addGlyphs(uintptr_t tileUID,
const std::u32string& text,
const std::u16string& text,
const FontStack&,
const GlyphSet&,
GlyphPositions&);

4
src/mbgl/text/glyph_set.cpp
View File

@@ -31,7 +31,7 @@ const std::map<uint32_t, SDFGlyph> &GlyphSet::getSDFs() const {
return sdfs;
}
const Shaping GlyphSet::getShaping(const std::u32string &string, const float maxWidth,
const Shaping GlyphSet::getShaping(const std::u16string &string, const float maxWidth,
const float lineHeight, const float horizontalAlign,
const float verticalAlign, const float justify,
const float spacing, const Point<float> &translate) const {
@@ -44,7 +44,7 @@ const Shaping GlyphSet::getShaping(const std::u32string &string, const float max
const float y = yOffset;
// Loop through all characters of this label and shape.
for (uint32_t chr : string) {
for (char16_t chr : string) {
auto it = sdfs.find(chr);
if (it != sdfs.end()) {
shaping.positionedGlyphs.emplace_back(chr, x, y);

2
src/mbgl/text/glyph_set.hpp
View File

@@ -9,7 +9,7 @@ class GlyphSet {
public:
void insert(uint32_t id, SDFGlyph&&);
const std::map<uint32_t, SDFGlyph> &getSDFs() const;
const Shaping getShaping(const std::u32string &string, float maxWidth, float lineHeight,
const Shaping getShaping(const std::u16string &string, float maxWidth, float lineHeight,
float horizontalAlign, float verticalAlign, float justify,
float spacing, const Point<float> &translate) const;
void lineWrap(Shaping &shaping, float lineHeight, float maxWidth, float horizontalAlign,

12
src/mbgl/util/i18n.cpp
View File

@@ -8,7 +8,7 @@ namespace {
@param last The last codepoint in the block, inclusive.
*/
#define DEFINE_IS_IN_UNICODE_BLOCK(name, first, last) \
inline bool isIn##name(uint32_t codepoint) { \
inline bool isIn##name(uint16_t codepoint) { \
return codepoint >= first && codepoint <= last; \
}
@@ -294,13 +294,13 @@ namespace mbgl {
namespace util {
namespace i18n {
bool isVisible(uint32_t chr) {
bool isVisible(uint16_t chr) {
return (chr == 0x0a /* newline */
|| chr == 0x20 /* space */
|| chr == 0x200b /* zero-width space */);
}
bool allowsWordBreaking(uint32_t chr) {
bool allowsWordBreaking(uint16_t chr) {
return (chr == 0x0a /* newline */
|| chr == 0x20 /* space */
|| chr == 0x26 /* ampersand */
@@ -314,8 +314,8 @@ bool allowsWordBreaking(uint32_t chr) {
|| chr == 0x2013 /* en dash */);
}
bool allowsIdeographicBreaking(const std::u32string& string) {
for (uint32_t chr : string) {
bool allowsIdeographicBreaking(const std::u16string& string) {
for (uint16_t chr : string) {
if (!allowsIdeographicBreaking(chr)) {
return false;
}
@@ -323,7 +323,7 @@ bool allowsIdeographicBreaking(const std::u32string& string) {
return true;
}
bool allowsIdeographicBreaking(uint32_t chr) {
bool allowsIdeographicBreaking(uint16_t chr) {
// Return early for characters outside all ideographic ranges.
if (chr < 0x2E80)
return false;

8
src/mbgl/util/i18n.hpp
View File

@@ -7,20 +7,20 @@ namespace util {
namespace i18n {
/** Returns whether a character is a visible character. */
bool isVisible(uint32_t chr);
bool isVisible(uint16_t chr);
/** Returns whether a line break can be inserted after the character indicated
by the given Unicode codepoint due to word breaking. */
bool allowsWordBreaking(uint32_t chr);
bool allowsWordBreaking(uint16_t chr);
/** Returns whether a line break can be inserted after any character in the
given string. If false, line breaking should occur on word boundaries
instead. */
bool allowsIdeographicBreaking(const std::u32string& string);
bool allowsIdeographicBreaking(const std::u16string& string);
/** Returns whether a line break can be inserted after the character indicated
by the given Unicode codepoint due to ideographic breaking. */
bool allowsIdeographicBreaking(uint32_t chr);
bool allowsIdeographicBreaking(uint16_t chr);
} // namespace i18n
} // namespace util

15
src/mbgl/util/utf.hpp
View File

@@ -2,19 +2,20 @@
#include <memory>
#include <boost/regex/pending/unicode_iterator.hpp>
#include <locale>
#include <codecvt>
namespace mbgl {
namespace util {
class utf8_to_utf32 {
public:
static std::u32string convert(std::string const& utf8)
class utf8_to_utf16 {
public:
static std::u16string convert(std::string const& utf8)
{
boost::u8_to_u32_iterator<std::string::const_iterator> begin(utf8.begin());
boost::u8_to_u32_iterator<std::string::const_iterator> end(utf8.end());
return std::u32string(begin,end);
std::wstring_convert<std::codecvt_utf8_utf16<char16_t>,char16_t> converter;
return converter.from_bytes( utf8 );
}
};
} // namespace util

4
test/util/merge_lines.test.cpp
View File

@@ -3,8 +3,8 @@
#include <mbgl/layout/merge_lines.hpp>
#include <mbgl/layout/symbol_feature.hpp>
const std::u32string aaa = U"a";
const std::u32string bbb = U"b";
const std::u16string aaa = u"a";
const std::u16string bbb = u"b";
TEST(MergeLines, SameText) {
// merges lines with the same text