tei/MapLibreParticleLab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add glyph store and load glyph pbfs

Browse Source
This commit is contained in:
Konstantin Käfer
2014-05-28 20:51:13 +02:00
parent 1749aa3de2
commit ed7b86a134
10 changed files with 334 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
include/llmr/map/map.hpp
View File

@@ -8,6 +8,7 @@
#include <llmr/map/transform.hpp>
#include <llmr/style/style.hpp>
#include <llmr/geometry/glyph_atlas.hpp>
#include <llmr/text/glyph_store.hpp>
#include <llmr/renderer/painter.hpp>
#include <llmr/util/noncopyable.hpp>
#include <llmr/util/texturepool.hpp>
@@ -98,6 +99,7 @@ public:
inline const TransformState &getState() const { return state; }
inline const Style &getStyle() const { return style; }
inline GlyphAtlas &getGlyphAtlas() { return glyphAtlas; }
inline GlyphStore &getGlyphStore() { return glyphStore; }
inline SpriteAtlas &getSpriteAtlas() { return spriteAtlas; }
inline uv_loop_t *getLoop() { return loop; }
inline time getAnimationTime() const { return animationTime; }
@@ -152,6 +154,7 @@ private:
Texturepool texturepool;
Style style;
GlyphAtlas glyphAtlas;
GlyphStore glyphStore;
SpriteAtlas spriteAtlas;
Painter painter;

4
include/llmr/map/tile_parser.hpp
View File

@@ -9,6 +9,7 @@ namespace llmr {
class Style;
class GlyphAtlas;
class GlyphStore;
class SpriteAtlas;
class LayerDescription;
@@ -16,7 +17,7 @@ class Bucket;
class TileParser {
public:
TileParser(const std::string& data, VectorTileData& tile, const Style& style, GlyphAtlas& glyphAtlas, SpriteAtlas &spriteAtlas);
TileParser(const std::string& data, VectorTileData& tile, const Style& style, GlyphAtlas& glyphAtlas, GlyphStore &glyphStore, SpriteAtlas &spriteAtlas);
private:
bool obsolete() const;
@@ -35,6 +36,7 @@ private:
VectorTileData& tile;
const Style& style;
GlyphAtlas& glyphAtlas;
GlyphStore &glyphStore;
SpriteAtlas &spriteAtlas;
Faces faces;
Placement placement;

4
include/llmr/platform/platform.hpp
View File

@@ -25,9 +25,11 @@ struct Response {
// Makes an HTTP request of a URL, preferrably on a background thread, and calls a function with the
// results in the original thread (which runs the libuv loop).
// If the loop pointer is NULL, the callback function will be called on an arbitrary thread.
// Returns a cancellable request.
std::shared_ptr<Request> request_http(const std::string &url,
std::function<void(Response *)> callback, uv_loop_t *loop);
std::function<void(Response *)> callback,
uv_loop_t *loop = nullptr);
// Cancels an HTTP request.
void cancel_request_http(const std::shared_ptr<Request> &req);

5
include/llmr/text/glyph.hpp
View File

@@ -9,6 +9,11 @@
namespace llmr {
typedef std::pair<uint16_t, uint16_t> GlyphRange;
// Note: this only works for the BMP
GlyphRange getGlyphRange(uint32_t glyph);
struct GlyphMetrics {
operator bool() const {
return width == 0 && height == 0 && advance == 0;

66
include/llmr/text/glyph_store.hpp Normal file
View File

@@ -0,0 +1,66 @@
#ifndef LLMR_TEXT_GLYPH_STORE
#define LLMR_TEXT_GLYPH_STORE
#include <llmr/text/glyph.hpp>
#include <llmr/util/pbf.hpp>
#include <cstdint>
#include <vector>
#include <future>
#include <map>
#include <set>
#include <unordered_map>
namespace llmr {
class SDFGlyph {
public:
SDFGlyph();
SDFGlyph(pbf data);
// A signed distance field of the glyph with a border of 3 pixels.
std::string bitmap;
// Glyph metrics
GlyphMetrics metrics;
};
class GlyphSet {
public:
GlyphSet(const std::string &fontStack, GlyphRange glyphRange);
void parse();
std::shared_future<GlyphSet &> getFuture();
private:
std::map<uint32_t, SDFGlyph> glyphs;
std::string data;
std::promise<GlyphSet &> promise;
std::shared_future<GlyphSet &> future;
std::mutex mtx;
};
// Manages Glyphrange PBF loading.
class GlyphStore {
public:
// Block until all specified GlyphRanges of the specified font stack are loaded.
void waitForGlyphRanges(const std::string &fontStack, const std::set<GlyphRange> &glyphRanges);
private:
// Loads an individual glyph range from the font stack and adds it to rangeSets
std::shared_future<GlyphSet &> loadGlyphRange(const std::string &fontStack, std::map<GlyphRange, std::unique_ptr<GlyphSet>> &rangeSets, GlyphRange range);
private:
std::unordered_map<std::string, std::map<GlyphRange, std::unique_ptr<GlyphSet>>> stacks;
std::mutex mtx;
};
}
#endif

67
src/map/tile_parser.cpp
View File

@@ -9,17 +9,21 @@
#include <llmr/util/raster.hpp>
#include <llmr/util/constants.hpp>
#include <llmr/geometry/glyph_atlas.hpp>
#include <llmr/text/glyph_store.hpp>
#include <llmr/util/std.hpp>
#include <codecvt>
using namespace llmr;
TileParser::TileParser(const std::string& data, VectorTileData& tile, const Style& style, GlyphAtlas& glyphAtlas, SpriteAtlas &spriteAtlas)
TileParser::TileParser(const std::string& data, VectorTileData& tile, const Style& style, GlyphAtlas& glyphAtlas, GlyphStore &glyphStore, SpriteAtlas &spriteAtlas)
: vector_data(pbf((const uint8_t *)data.data(), data.size())),
tile(tile),
style(style),
glyphAtlas(glyphAtlas),
glyphStore(glyphStore),
spriteAtlas(spriteAtlas),
placement(tile.id.z) {
parseGlyphs();
@@ -162,35 +166,42 @@ std::unique_ptr<Bucket> TileParser::createIconBucket(const VectorTileLayer& laye
return obsolete() ? nullptr : std::move(bucket);
}
typedef std::pair<uint16_t, uint16_t> GlyphRange;
std::unique_ptr<Bucket> TileParser::createTextBucket(const VectorTileLayer& layer, const BucketDescription& bucket_desc) {
// Determine the correct text stack.
if (!layer.shaping.size()) {
if (debug::missingFontStackWarning) {
fprintf(stderr, "[WARNING] missing font stack '%s'\n", bucket_desc.geometry.font.c_str());
}
return nullptr;
}
// TODO: currently hardcoded to use the first font stack.
const std::map<Value, Shaping>& shaping = layer.shaping.begin()->second;
const Faces& const_faces = faces;
IndexedFaces faces;
for (const std::string& face : layer.faces) {
auto it = const_faces.find(face);
if (it == const_faces.end()) {
// This layer references an unknown face.
if (debug::missingFontFaceWarning) {
fprintf(stderr, "[WARNING] missing font face '%s'\n", face.c_str());
}
return nullptr;
}
faces.push_back(&it->second);
}
std::unique_ptr<TextBucket> bucket = std::make_unique<TextBucket>(
tile.textVertexBuffer, tile.triangleElementsBuffer, bucket_desc, placement);
addBucketFeatures(bucket, layer, bucket_desc, faces, shaping);
std::set<GlyphRange> ranges;
std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> ucs4conv;
FilteredVectorTileLayer filtered_layer(layer, bucket_desc);
for (const pbf& feature_pbf : filtered_layer) {
if (obsolete()) return nullptr;
VectorTileFeature feature { feature_pbf, layer };
auto it_prop = feature.properties.find(bucket_desc.geometry.field);
if (it_prop == feature.properties.end()) {
// feature does not have the correct property
if (debug::labelTextMissingWarning) {
fprintf(stderr, "[WARNING] feature doesn't have property '%s' required for labelling\n", bucket_desc.geometry.field.c_str());
}
continue;
}
const std::u32string string = ucs4conv.from_bytes(toString(it_prop->second));
// Loop through all characters of this text and collect unique codepoints.
for (uint32_t chr : string) {
ranges.insert(getGlyphRange(chr));
}
}
glyphStore.waitForGlyphRanges(bucket_desc.geometry.font, ranges);
// TODO: Now that we have all the glyph ranges, actually perform the shaping and add features to
// the bucket, and add the required glyphs from the glyphStore to the glyphAtlas.
return std::move(bucket);
}

2
src/map/vector_tile_data.cpp
View File

@@ -22,7 +22,7 @@ void VectorTileData::parse() {
// Parsing creates state that is encapsulated in TileParser. While parsing,
// the TileParser object writes results into this objects. All other state
// is going to be discarded afterwards.
TileParser parser(data, *this, map.getStyle(), map.getGlyphAtlas(), map.getSpriteAtlas());
TileParser parser(data, *this, map.getStyle(), map.getGlyphAtlas(), map.getGlyphStore(), map.getSpriteAtlas());
} catch (const std::exception& ex) {
fprintf(stderr, "[%p] exception [%d/%d/%d]... failed: %s\n", this, id.z, id.x, id.y, ex.what());
cancel();

34
src/platform/request.cpp
View File

@@ -12,25 +12,33 @@ Request::Request(const std::string &url,
res(std::make_unique<Response>(callback)),
cancelled(false),
loop(loop) {
// Add a check handle without a callback to keep the default loop running.
// We don't have a real handler attached to the default loop right from the
// beginning, because we're using asynchronous messaging to perform the actual
// request in the request thread. Only after the request is complete, we
// create an actual work request that is attached to the default loop.
async = new uv_async_t();
async->data = new std::unique_ptr<Response>();
uv_async_init(loop, async, complete);
if (loop) {
// Add a check handle without a callback to keep the default loop running.
// We don't have a real handler attached to the default loop right from the
// beginning, because we're using asynchronous messaging to perform the actual
// request in the request thread. Only after the request is complete, we
// create an actual work request that is attached to the default loop.
async = new uv_async_t();
async->data = new std::unique_ptr<Response>();
uv_async_init(loop, async, complete);
}
}
Request::~Request() {
}
void Request::complete() {
// We're scheduling the response callback to be invoked in the event loop.
// Since the Request object will be deleted before the callback is invoked,
// we move over the Response object to be owned by the async handle.
((std::unique_ptr<Response> *)async->data)->swap(res);
uv_async_send(async);
if (loop) {
// We're scheduling the response callback to be invoked in the event loop.
// Since the Request object will be deleted before the callback is invoked,
// we move over the Response object to be owned by the async handle.
((std::unique_ptr<Response> *)async->data)->swap(res);
uv_async_send(async);
} else {
// We're calling the response callback immediately. We're currently on an
// arbitrary thread, but that's okay.
res->callback(res.get());
}
}
void Request::complete(uv_async_t *async) {

101
src/text/glyph.cpp Normal file
View File

@@ -0,0 +1,101 @@
#include <llmr/text/glyph.hpp>
namespace llmr {
// Note: this only works for the BMP
// Note: we could use a binary lookup table to get averaged constant time lookups, however,
// most of our lookups are going to be within the first 3 ranges listed here, so this is
// likely faster.
GlyphRange getGlyphRange(uint32_t glyph) {
if (glyph <= 127) return { 0, 127 }; // Basic Latin
if (glyph <= 255) return { 128, 255 }; // Latin-1 Supplement
if (glyph >= 8192 && glyph <= 8303) return { 8192, 8303 }; // General Punctuation
if (glyph <= 383) return { 256, 383 }; // Latin Extended-A
if (glyph <= 591) return { 384, 591 }; // Latin Extended-B
if (glyph <= 687) return { 592, 687 }; // IPA Extensions
if (glyph <= 767) return { 688, 767 }; // Spacing Modifier Letters
if (glyph <= 879) return { 768, 879 }; // Combining Diacritical Marks
if (glyph <= 1023) return { 880, 1023 }; // Greek
if (glyph <= 1279) return { 1024, 1279 }; // Cyrillic
if (glyph <= 1423) return { 1280, 1423 }; // Armenian
if (glyph <= 1535) return { 1424, 1535 }; // Hebrew
if (glyph <= 1791) return { 1536, 1791 }; // Arabic
if (glyph <= 1871) return { 1792, 1871 }; // Syriac
if (glyph <= 1983) return { 1872, 1983 }; // Thaana
if (glyph <= 2431) return { 1984, 2431 }; // Devanagari
if (glyph <= 2559) return { 2432, 2559 }; // Bengali
if (glyph <= 2687) return { 2560, 2687 }; // Gurmukhi
if (glyph <= 2815) return { 2688, 2815 }; // Gujarati
if (glyph <= 2943) return { 2816, 2943 }; // Oriya
if (glyph <= 3071) return { 2944, 3071 }; // Tamil
if (glyph <= 3199) return { 3072, 3199 }; // Telugu
if (glyph <= 3327) return { 3200, 3327 }; // Kannada
if (glyph <= 3455) return { 3328, 3455 }; // Malayalam
if (glyph <= 3583) return { 3456, 3583 }; // Sinhala
if (glyph <= 3711) return { 3584, 3711 }; // Thai
if (glyph <= 3839) return { 3712, 3839 }; // Lao
if (glyph <= 4095) return { 3840, 4095 }; // Tibetan
if (glyph <= 4255) return { 4096, 4255 }; // Myanmar
if (glyph <= 4351) return { 4256, 4351 }; // Georgian
if (glyph <= 4607) return { 4352, 4607 }; // Hangul Jamo
if (glyph <= 4991) return { 4608, 4991 }; // Ethiopic
if (glyph <= 5119) return { 4992, 5119 }; // Cherokee
if (glyph <= 5759) return { 5120, 5759 }; // Unified Canadian Aboriginal Syllabics
if (glyph <= 5791) return { 5760, 5791 }; // Ogham
if (glyph <= 5887) return { 5792, 5887 }; // Runic
if (glyph <= 6143) return { 5888, 6143 }; // Khmer
if (glyph <= 6319) return { 6144, 6319 }; // Mongolian
if (glyph <= 7935) return { 6320, 7935 }; // Latin Extended Additional
if (glyph <= 8191) return { 7936, 8191 }; // Greek Extended
if (glyph <= 8351) return { 8304, 8351 }; // Superscripts and Subscripts
if (glyph <= 8399) return { 8352, 8399 }; // Currency Symbols
if (glyph <= 8447) return { 8400, 8447 }; // Combining Marks for Symbols
if (glyph <= 8527) return { 8448, 8527 }; // Letterlike Symbols
if (glyph <= 8591) return { 8528, 8591 }; // Number Forms
if (glyph <= 8703) return { 8592, 8703 }; // Arrows
if (glyph <= 8959) return { 8704, 8959 }; // Mathematical Operators
if (glyph <= 9215) return { 8960, 9215 }; // Miscellaneous Technical
if (glyph <= 9279) return { 9216, 9279 }; // Control Pictures
if (glyph <= 9311) return { 9280, 9311 }; // Optical Character Recognition
if (glyph <= 9471) return { 9312, 9471 }; // Enclosed Alphanumerics
if (glyph <= 9599) return { 9472, 9599 }; // Box Drawing
if (glyph <= 9631) return { 9600, 9631 }; // Block Elements
if (glyph <= 9727) return { 9632, 9727 }; // Geometric Shapes
if (glyph <= 9983) return { 9728, 9983 }; // Miscellaneous Symbols
if (glyph <= 10175) return { 9984, 10175 }; // Dingbats
if (glyph <= 10495) return { 10176, 10495 }; // Braille Patterns
if (glyph <= 12031) return { 10496, 12031 }; // CJK Radicals Supplement
if (glyph <= 12255) return { 12032, 12255 }; // Kangxi Radicals
if (glyph <= 12287) return { 12256, 12287 }; // Ideographic Description Characters
if (glyph <= 12351) return { 12288, 12351 }; // CJK Symbols and Punctuation
if (glyph <= 12447) return { 12352, 12447 }; // Hiragana
if (glyph <= 12543) return { 12448, 12543 }; // Katakana
if (glyph <= 12591) return { 12544, 12591 }; // Bopomofo
if (glyph <= 12687) return { 12592, 12687 }; // Hangul Compatibility Jamo
if (glyph <= 12703) return { 12688, 12703 }; // Kanbun
if (glyph <= 12735) return { 12704, 12735 }; // Bopomofo Extended
if (glyph <= 13055) return { 12736, 13055 }; // Enclosed CJK Letters and Months
if (glyph <= 13311) return { 13056, 13311 }; // CJK Compatibility
if (glyph <= 19893) return { 13312, 19893 }; // CJK Unified Ideographs Extension A
if (glyph <= 40959) return { 19894, 40959 }; // CJK Unified Ideographs
if (glyph <= 42127) return { 40960, 42127 }; // Yi Syllables
if (glyph <= 42191) return { 42128, 42191 }; // Yi Radicals
if (glyph <= 55203) return { 42192, 55203 }; // Hangul Syllables
if (glyph <= 56191) return { 55204, 56191 }; // High Surrogates
if (glyph <= 56319) return { 56192, 56319 }; // High Private Use Surrogates
if (glyph <= 57343) return { 56320, 57343 }; // Low Surrogates
if (glyph <= 63743) return { 57344, 63743 }; // Private Use
if (glyph <= 64255) return { 63744, 64255 }; // CJK Compatibility Ideographs
if (glyph <= 64335) return { 64256, 64335 }; // Alphabetic Presentation Forms
if (glyph <= 65023) return { 64336, 65023 }; // Arabic Presentation Forms-A
if (glyph <= 65071) return { 65024, 65071 }; // Combining Half Marks
if (glyph <= 65103) return { 65072, 65103 }; // CJK Compatibility Forms
if (glyph <= 65135) return { 65104, 65135 }; // Small Form Variants
if (glyph <= 65278) return { 65136, 65278 }; // Arabic Presentation Forms-B
if (glyph <= 65279) return { 65279, 65279 }; // Specials
if (glyph <= 65519) return { 65280, 65519 }; // Halfwidth and Fullwidth Forms
if (glyph <= 65533) return { 65520, 65533 }; // Specials
return { 0, 0 };
}
}

92
src/text/glyph_store.cpp Normal file
View File

@@ -0,0 +1,92 @@
#include <llmr/text/glyph_store.hpp>
#include <llmr/util/std.hpp>
#include <llmr/util/string.hpp>
#include <llmr/platform/platform.hpp>
#include <uv.h>
namespace llmr {
GlyphSet::GlyphSet(const std::string &fontStack, GlyphRange glyphRange)
: future(promise.get_future().share())
{
// Load the glyph set URL
std::string url = util::sprintf<255>("http://mapbox.s3.amazonaws.com/gl-glyphs/%s/%d-%d.pbf", fontStack.c_str(), glyphRange.first, glyphRange.second);
// TODO: Find more reliable URL normalization function
std::replace(url.begin(), url.end(), ' ', '+');
platform::request_http(url, [&](platform::Response *res) {
if (res->code != 200) {
// Something went wrong with loading the glyph pbf. Pass on the error to the future listeners.
const std::string msg = util::sprintf<255>("[ERROR] failed to load glyphs (%d): %s\n", res->code, res->error_message.c_str());
promise.set_exception(std::make_exception_ptr(std::runtime_error(msg)));
} else {
// Transfer the data to the GlyphSet and signal its availability.
// Once it is available, the caller will need to call parse() to actually
// parse the data we received. We are not doing this here since this callback is being
// called from another (unknown) thread.
data.swap(res->body);
promise.set_value(*this);
}
});
}
std::shared_future<GlyphSet &> GlyphSet::getFuture() {
return future;
}
void GlyphSet::parse() {
std::lock_guard<std::mutex> lock(mtx);
if (!data.size()) {
// If there is no data, this means we either haven't received any data, or
// we have already parsed the data.
return;
}
// TODO: parse the glyphs
data.clear();
}
void GlyphStore::waitForGlyphRanges(const std::string &fontStack, const std::set<GlyphRange> &glyphRanges) {
// We are implementing a blocking wait with futures: Every GlyphSet has a future that we are
// waiting for until it is loaded.
std::vector<std::shared_future<GlyphSet &>> futures;
futures.reserve(glyphRanges.size());
{
std::lock_guard<std::mutex> lock(mtx);
auto &rangeSets = stacks[fontStack];
// Attempt to load the glyph range. If the GlyphSet already exists, we are getting back
// the same shared_future.
for (GlyphRange range : glyphRanges) {
futures.emplace_back(loadGlyphRange(fontStack, rangeSets, range));
}
}
// Now that we potentially created all GlyphSets, we are waiting for the results, one by one.
// When we get a result (or the GlyphSet is aready loaded), we are attempting to parse the
// GlyphSet.
for (std::shared_future<GlyphSet &> &future : futures) {
future.get().parse();
}
}
std::shared_future<GlyphSet &> GlyphStore::loadGlyphRange(const std::string &name, std::map<GlyphRange, std::unique_ptr<GlyphSet>> &rangeSets, const GlyphRange range) {
auto range_it = rangeSets.find(range);
if (range_it == rangeSets.end()) {
// We don't have this glyph set yet for this font stack.
range_it = rangeSets.emplace(range, std::make_unique<GlyphSet>(name, range)).first;
}
return range_it->second->getFuture();
}
}