tei/MapLibreParticleLab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[render-test] Implement fps benchmarking tests (#15803)

Browse Source
This commit is contained in:
Mikko Pulkki
2019-10-22 12:13:55 +03:00
committed by GitHub
parent bb0e5ffb2c
commit 0ca96fd8a4
27 changed files with 424 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
include/mbgl/util/monotonic_timer.hpp Normal file
View File

@@ -0,0 +1,24 @@
#pragma once
#include <chrono>
#include <functional>
#include <mbgl/util/noncopyable.hpp>
#include <memory>
namespace mbgl {
namespace util {
class MonotonicTimer {
public:
static std::chrono::duration<double> now();
template <typename F, typename... Args>
inline static std::chrono::duration<double> duration(F&& func, Args&&... args) {
auto start = now();
func(std::forward<Args>(args)...);
return now() - start;
}
};
} // namespace util
} // namespace mbgl

1
next/platform/android/android.cmake
View File

@@ -213,6 +213,7 @@ target_sources(
${MBGL_ROOT}/platform/default/src/mbgl/storage/sqlite3.cpp
${MBGL_ROOT}/platform/default/src/mbgl/text/bidi.cpp
${MBGL_ROOT}/platform/default/src/mbgl/util/compression.cpp
${MBGL_ROOT}/platform/default/src/mbgl/util/monotonic_timer.cpp
${MBGL_ROOT}/platform/default/src/mbgl/util/png_writer.cpp
${MBGL_ROOT}/platform/default/src/mbgl/util/thread_local.cpp
${MBGL_ROOT}/platform/default/src/mbgl/util/utf.cpp

1
next/platform/ios/ios.cmake
View File

@@ -39,6 +39,7 @@ target_sources(
${MBGL_ROOT}/platform/default/src/mbgl/storage/sqlite3.cpp
${MBGL_ROOT}/platform/default/src/mbgl/text/bidi.cpp
${MBGL_ROOT}/platform/default/src/mbgl/util/compression.cpp
${MBGL_ROOT}/platform/default/src/mbgl/util/monotonic_timer.cpp
${MBGL_ROOT}/platform/default/src/mbgl/util/png_writer.cpp
${MBGL_ROOT}/platform/default/src/mbgl/util/thread_local.cpp
${MBGL_ROOT}/platform/default/src/mbgl/util/utf.cpp

1
next/platform/linux/linux.cmake
View File

@@ -37,6 +37,7 @@ target_sources(
${MBGL_ROOT}/platform/default/src/mbgl/util/image.cpp
${MBGL_ROOT}/platform/default/src/mbgl/util/jpeg_reader.cpp
${MBGL_ROOT}/platform/default/src/mbgl/util/logging_stderr.cpp
${MBGL_ROOT}/platform/default/src/mbgl/util/monotonic_timer.cpp
${MBGL_ROOT}/platform/default/src/mbgl/util/png_reader.cpp
${MBGL_ROOT}/platform/default/src/mbgl/util/png_writer.cpp
${MBGL_ROOT}/platform/default/src/mbgl/util/run_loop.cpp

1
next/platform/macos/macos.cmake
View File

@@ -105,6 +105,7 @@ target_sources(
${MBGL_ROOT}/platform/default/src/mbgl/storage/sqlite3.cpp
${MBGL_ROOT}/platform/default/src/mbgl/text/bidi.cpp
${MBGL_ROOT}/platform/default/src/mbgl/util/compression.cpp
${MBGL_ROOT}/platform/default/src/mbgl/util/monotonic_timer.cpp
${MBGL_ROOT}/platform/default/src/mbgl/util/png_writer.cpp
${MBGL_ROOT}/platform/default/src/mbgl/util/thread_local.cpp
${MBGL_ROOT}/platform/default/src/mbgl/util/utf.cpp

1
next/platform/qt/qt.cmake
View File

@@ -43,6 +43,7 @@ target_sources(
${MBGL_ROOT}/platform/default/src/mbgl/storage/online_file_source.cpp
${MBGL_ROOT}/platform/default/src/mbgl/storage/sqlite3.cpp
${MBGL_ROOT}/platform/default/src/mbgl/util/compression.cpp
${MBGL_ROOT}/platform/default/src/mbgl/util/monotonic_timer.cpp
${MBGL_ROOT}/platform/qt/src/async_task.cpp
${MBGL_ROOT}/platform/qt/src/async_task_impl.hpp
${MBGL_ROOT}/platform/qt/src/number_format.cpp

1
platform/android/core-files.json
View File

@@ -93,6 +93,7 @@
"platform/default/src/mbgl/map/map_snapshotter.cpp",
"platform/default/src/mbgl/text/bidi.cpp",
"platform/default/src/mbgl/util/compression.cpp",
"platform/default/src/mbgl/util/monotonic_timer.cpp",
"platform/default/src/mbgl/util/png_writer.cpp",
"platform/default/src/mbgl/util/thread_local.cpp",
"platform/default/src/mbgl/util/utf.cpp",

10
platform/default/include/mbgl/gfx/headless_backend.hpp
View File

@@ -15,11 +15,13 @@ namespace gfx {
// of readStillImage.
class HeadlessBackend : public gfx::Renderable {
public:
enum class SwapBehaviour { NoFlush, Flush };
// Factory.
static std::unique_ptr<HeadlessBackend>
Create(const Size size = { 256, 256 },
const gfx::ContextMode contextMode = gfx::ContextMode::Unique) {
return Backend::Create<HeadlessBackend, Size, gfx::ContextMode>(size, contextMode);
static std::unique_ptr<HeadlessBackend> Create(const Size size = {256, 256},
SwapBehaviour swapBehavior = SwapBehaviour::NoFlush,
const gfx::ContextMode contextMode = gfx::ContextMode::Unique) {
return Backend::Create<HeadlessBackend, Size, SwapBehaviour, gfx::ContextMode>(size, swapBehavior, contextMode);
}
virtual PremultipliedImage readStillImage() = 0;

8
platform/default/include/mbgl/gfx/headless_frontend.hpp
View File

@@ -1,11 +1,12 @@
#pragma once
#include <mbgl/gfx/headless_backend.hpp>
#include <mbgl/map/camera.hpp>
#include <mbgl/renderer/renderer_frontend.hpp>
#include <mbgl/gfx/headless_backend.hpp>
#include <mbgl/util/async_task.hpp>
#include <mbgl/util/optional.hpp>
#include <atomic>
#include <memory>
namespace mbgl {
@@ -17,10 +18,12 @@ class TransformState;
class HeadlessFrontend : public RendererFrontend {
public:
HeadlessFrontend(float pixelRatio_,
gfx::HeadlessBackend::SwapBehaviour swapBehviour = gfx::HeadlessBackend::SwapBehaviour::NoFlush,
gfx::ContextMode mode = gfx::ContextMode::Unique,
const optional<std::string> localFontFamily = {});
HeadlessFrontend(Size,
float pixelRatio_,
gfx::HeadlessBackend::SwapBehaviour swapBehviour = gfx::HeadlessBackend::SwapBehaviour::NoFlush,
gfx::ContextMode mode = gfx::ContextMode::Unique,
const optional<std::string> localFontFamily = {});
~HeadlessFrontend() override;
@@ -29,6 +32,7 @@ public:
void update(std::shared_ptr<UpdateParameters>) override;
void setObserver(RendererObserver&) override;
double getFrameTime() const;
Size getSize() const;
void setSize(Size);
@@ -45,6 +49,7 @@ public:
PremultipliedImage readStillImage();
PremultipliedImage render(Map&);
void renderOnce(Map&);
optional<TransformState> getTransformState() const;
@@ -52,6 +57,7 @@ private:
Size size;
float pixelRatio;
std::atomic<double> frameTime;
std::unique_ptr<gfx::HeadlessBackend> backend;
util::AsyncTask asyncInvalidate;

7
platform/default/include/mbgl/gl/headless_backend.hpp
View File

@@ -10,13 +10,17 @@ namespace gl {
class HeadlessBackend final : public gl::RendererBackend, public gfx::HeadlessBackend {
public:
HeadlessBackend(Size = { 256, 256 }, gfx::ContextMode = gfx::ContextMode::Unique);
HeadlessBackend(Size = {256, 256},
SwapBehaviour = SwapBehaviour::NoFlush,
gfx::ContextMode = gfx::ContextMode::Unique);
~HeadlessBackend() override;
void updateAssumedState() override;
gfx::Renderable& getDefaultRenderable() override;
PremultipliedImage readStillImage() override;
RendererBackend* getRendererBackend() override;
void swap();
class Impl {
public:
virtual ~Impl() = default;
@@ -37,6 +41,7 @@ private:
private:
std::unique_ptr<Impl> impl;
bool active = false;
SwapBehaviour swapBehaviour = SwapBehaviour::NoFlush;
};
} // namespace gl

59
platform/default/src/mbgl/gfx/headless_frontend.cpp
View File

@@ -1,43 +1,50 @@
#include <mbgl/gfx/headless_frontend.hpp>
#include <mbgl/gfx/backend_scope.hpp>
#include <mbgl/gfx/headless_frontend.hpp>
#include <mbgl/map/map.hpp>
#include <mbgl/map/transform_state.hpp>
#include <mbgl/renderer/renderer.hpp>
#include <mbgl/renderer/renderer_state.hpp>
#include <mbgl/renderer/update_parameters.hpp>
#include <mbgl/map/map.hpp>
#include <mbgl/map/transform_state.hpp>
#include <mbgl/util/monotonic_timer.hpp>
#include <mbgl/util/run_loop.hpp>
namespace mbgl {
HeadlessFrontend::HeadlessFrontend(float pixelRatio_,
gfx::HeadlessBackend::SwapBehaviour swapBehavior,
const gfx::ContextMode contextMode,
const optional<std::string> localFontFamily)
: HeadlessFrontend(
{ 256, 256 }, pixelRatio_, contextMode, localFontFamily) {
}
: HeadlessFrontend({256, 256}, pixelRatio_, swapBehavior, contextMode, localFontFamily) {}
HeadlessFrontend::HeadlessFrontend(Size size_,
float pixelRatio_,
gfx::HeadlessBackend::SwapBehaviour swapBehavior,
const gfx::ContextMode contextMode,
const optional<std::string> localFontFamily)
: size(size_),
pixelRatio(pixelRatio_),
backend(gfx::HeadlessBackend::Create({ static_cast<uint32_t>(size.width * pixelRatio),
static_cast<uint32_t>(size.height * pixelRatio) }, contextMode)),
asyncInvalidate([this] {
if (renderer && updateParameters) {
gfx::BackendScope guard { *getBackend() };
frameTime(0),
backend(gfx::HeadlessBackend::Create(
{static_cast<uint32_t>(size.width * pixelRatio), static_cast<uint32_t>(size.height * pixelRatio)},
swapBehavior,
contextMode)),
asyncInvalidate([this] {
if (renderer && updateParameters) {
auto startTime = mbgl::util::MonotonicTimer::now();
gfx::BackendScope guard{*getBackend()};
// onStyleImageMissing might be called during a render. The user implemented method
// could trigger a call to MGLRenderFrontend#update which overwrites `updateParameters`.
// Copy the shared pointer here so that the parameters aren't destroyed while `render(...)` is
// still using them.
auto updateParameters_ = updateParameters;
renderer->render(*updateParameters_);
}
}),
renderer(std::make_unique<Renderer>(*getBackend(), pixelRatio, localFontFamily)) {
}
// onStyleImageMissing might be called during a render. The user implemented method
// could trigger a call to MGLRenderFrontend#update which overwrites `updateParameters`.
// Copy the shared pointer here so that the parameters aren't destroyed while `render(...)` is
// still using them.
auto updateParameters_ = updateParameters;
renderer->render(*updateParameters_);
auto endTime = mbgl::util::MonotonicTimer::now();
frameTime = (endTime - startTime).count();
}
}),
renderer(std::make_unique<Renderer>(*getBackend(), pixelRatio, localFontFamily)) {}
HeadlessFrontend::~HeadlessFrontend() = default;
@@ -56,6 +63,10 @@ void HeadlessFrontend::setObserver(RendererObserver& observer_) {
renderer->setObserver(&observer_);
}
double HeadlessFrontend::getFrameTime() const {
return frameTime;
}
Size HeadlessFrontend::getSize() const {
return size;
}
@@ -147,10 +158,14 @@ PremultipliedImage HeadlessFrontend::render(Map& map) {
if (error) {
std::rethrow_exception(error);
}
return result;
}
void HeadlessFrontend::renderOnce(Map&) {
util::RunLoop::Get()->runOnce();
}
optional<TransformState> HeadlessFrontend::getTransformState() const {
if (updateParameters) {
return updateParameters->transformState;

32
platform/default/src/mbgl/gl/headless_backend.cpp
View File

@@ -12,12 +12,12 @@ namespace gl {
class HeadlessRenderableResource final : public gl::RenderableResource {
public:
HeadlessRenderableResource(gl::Context& context_, Size size_)
: context(context_),
HeadlessRenderableResource(HeadlessBackend& backend_, gl::Context& context_, Size size_)
: backend(backend_),
context(context_),
color(context.createRenderbuffer<gfx::RenderbufferPixelType::RGBA>(size_)),
depthStencil(context.createRenderbuffer<gfx::RenderbufferPixelType::DepthStencil>(size_)),
framebuffer(context.createFramebuffer(color, depthStencil)) {
}
framebuffer(context.createFramebuffer(color, depthStencil)) {}
void bind() override {
context.bindFramebuffer = framebuffer.framebuffer;
@@ -25,18 +25,22 @@ public:
context.viewport = { 0, 0, framebuffer.size };
}
void swap() override { backend.swap(); }
HeadlessBackend& backend;
gl::Context& context;
gfx::Renderbuffer<gfx::RenderbufferPixelType::RGBA> color;
gfx::Renderbuffer<gfx::RenderbufferPixelType::DepthStencil> depthStencil;
gl::Framebuffer framebuffer;
};
HeadlessBackend::HeadlessBackend(const Size size_, const gfx::ContextMode contextMode_)
: mbgl::gl::RendererBackend(contextMode_), mbgl::gfx::HeadlessBackend(size_) {
}
HeadlessBackend::HeadlessBackend(const Size size_,
gfx::HeadlessBackend::SwapBehaviour swapBehaviour_,
const gfx::ContextMode contextMode_)
: mbgl::gl::RendererBackend(contextMode_), mbgl::gfx::HeadlessBackend(size_), swapBehaviour(swapBehaviour_) {}
HeadlessBackend::~HeadlessBackend() {
gfx::BackendScope guard { *this };
gfx::BackendScope guard{*this};
resource.reset();
// Explicitly reset the context so that it is destructed and cleaned up before we destruct
// the impl object.
@@ -67,11 +71,15 @@ void HeadlessBackend::deactivate() {
gfx::Renderable& HeadlessBackend::getDefaultRenderable() {
if (!resource) {
resource = std::make_unique<HeadlessRenderableResource>(static_cast<gl::Context&>(getContext()), size);
resource = std::make_unique<HeadlessRenderableResource>(*this, static_cast<gl::Context&>(getContext()), size);
}
return *this;
}
void HeadlessBackend::swap() {
if (swapBehaviour == SwapBehaviour::Flush) static_cast<gl::Context&>(getContext()).finish();
}
void HeadlessBackend::updateAssumedState() {
// no-op
}
@@ -89,9 +97,9 @@ RendererBackend* HeadlessBackend::getRendererBackend() {
namespace gfx {
template <>
std::unique_ptr<gfx::HeadlessBackend>
Backend::Create<gfx::Backend::Type::OpenGL>(const Size size, const gfx::ContextMode contextMode) {
return std::make_unique<gl::HeadlessBackend>(size, contextMode);
std::unique_ptr<gfx::HeadlessBackend> Backend::Create<gfx::Backend::Type::OpenGL>(
const Size size, gfx::HeadlessBackend::SwapBehaviour swapBehavior, const gfx::ContextMode contextMode) {
return std::make_unique<gl::HeadlessBackend>(size, swapBehavior, contextMode);
}
} // namespace gfx

4
platform/default/src/mbgl/map/map_snapshotter.cpp
View File

@@ -51,8 +51,8 @@ MapSnapshotter::Impl::Impl(const std::pair<bool, std::string> style,
const optional<LatLngBounds> region,
const optional<std::string> localFontFamily,
const ResourceOptions& resourceOptions)
: frontend(
size, pixelRatio, gfx::ContextMode::Unique, localFontFamily),
: frontend(
size, pixelRatio, gfx::HeadlessBackend::SwapBehaviour::NoFlush, gfx::ContextMode::Unique, localFontFamily),
map(frontend,
MapObserver::nullObserver(),
MapOptions().withMapMode(MapMode::Static).withSize(size).withPixelRatio(pixelRatio),

24
platform/default/src/mbgl/util/monotonic_timer.cpp Normal file
View File

@@ -0,0 +1,24 @@
#include <assert.h>
#include <chrono>
#include <mbgl/util/monotonic_timer.hpp>
namespace mbgl {
namespace util {
// Prefer high resolution timer if it is monotonic
template <typename T, std::enable_if_t<std::chrono::high_resolution_clock::is_steady, T>* = nullptr>
static T sample() {
return std::chrono::duration_cast<T>(std::chrono::high_resolution_clock::now().time_since_epoch());
}
template <typename T, std::enable_if_t<!std::chrono::high_resolution_clock::is_steady, T>* = nullptr>
static T sample() {
return std::chrono::duration_cast<T>(std::chrono::steady_clock::now().time_since_epoch());
}
std::chrono::duration<double> MonotonicTimer::now() {
return sample<std::chrono::duration<double>>();
}
} // namespace util
} // namespace mbgl

1
platform/ios/core-files.json
View File

@@ -17,6 +17,7 @@
"platform/default/src/mbgl/map/map_snapshotter.cpp",
"platform/default/src/mbgl/text/bidi.cpp",
"platform/default/src/mbgl/util/compression.cpp",
"platform/default/src/mbgl/util/monotonic_timer.cpp",
"platform/default/src/mbgl/util/png_writer.cpp",
"platform/default/src/mbgl/util/thread_local.cpp",
"platform/default/src/mbgl/util/utf.cpp"

1
platform/linux/config.cmake
View File

@@ -52,6 +52,7 @@ macro(mbgl_platform_core)
PRIVATE platform/default/src/mbgl/layermanager/layer_manager.cpp
PRIVATE platform/default/src/mbgl/util/compression.cpp
PRIVATE platform/default/src/mbgl/util/logging_stderr.cpp
PRIVATE platform/default/src/mbgl/util/monotonic_timer.cpp
PRIVATE platform/default/src/mbgl/util/string_stdlib.cpp
PRIVATE platform/default/src/mbgl/util/thread.cpp
PRIVATE platform/default/src/mbgl/util/thread_local.cpp

1
platform/macos/core-files.json
View File

@@ -16,6 +16,7 @@
"platform/default/src/mbgl/map/map_snapshotter.cpp",
"platform/default/src/mbgl/text/bidi.cpp",
"platform/default/src/mbgl/util/compression.cpp",
"platform/default/src/mbgl/util/monotonic_timer.cpp",
"platform/default/src/mbgl/util/png_writer.cpp",
"platform/default/src/mbgl/util/thread_local.cpp",
"platform/default/src/mbgl/util/utf.cpp"

10
render-test/metadata.hpp
View File

@@ -68,6 +68,12 @@ struct MemoryProbe {
}
};
struct FpsProbe {
float average = 0.0;
float minOnePc = 0.0;
float tolerance = 0.0f;
};
struct NetworkProbe {
NetworkProbe() = default;
NetworkProbe(size_t requests_, size_t transferred_) : requests(requests_), transferred(transferred_) {}
@@ -78,10 +84,11 @@ struct NetworkProbe {
class TestMetrics {
public:
bool isEmpty() const { return fileSize.empty() && memory.empty() && network.empty(); }
bool isEmpty() const { return fileSize.empty() && memory.empty() && network.empty() && fps.empty(); }
std::map<std::string, FileSizeProbe> fileSize;
std::map<std::string, MemoryProbe> memory;
std::map<std::string, NetworkProbe> network;
std::map<std::string, FpsProbe> fps;
};
struct TestMetadata {
@@ -90,6 +97,7 @@ struct TestMetadata {
TestPaths paths;
mbgl::JSDocument document;
bool renderTest = true;
bool outputsImage = true;
mbgl::Size size{ 512u, 512u };
float pixelRatio = 1.0f;

75
render-test/parser.cpp
View File

@@ -303,6 +303,22 @@ std::string serializeMetrics(const TestMetrics& metrics) {
writer.EndArray();
}
if (!metrics.fps.empty()) {
// Start fps section
writer.Key("fps");
writer.StartArray();
for (const auto& fpsProbe : metrics.fps) {
assert(!fpsProbe.first.empty());
writer.StartArray();
writer.String(fpsProbe.first.c_str());
writer.Double(fpsProbe.second.average);
writer.Double(fpsProbe.second.minOnePc);
writer.EndArray();
}
writer.EndArray();
// End fps section
}
writer.EndObject();
return s.GetString();
@@ -433,6 +449,7 @@ ArgumentsTuple parseArguments(int argc, char** argv) {
if (testFilterValue && !std::regex_match(testPath.path().string(), args::get(testFilterValue))) {
continue;
}
if (testPath.path().filename() == "style.json") {
testPaths.emplace_back(testPath, getTestExpectations(testPath, path, expectationsPaths));
}
@@ -549,6 +566,23 @@ TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path) {
}
}
if (document.HasMember("fps")) {
const mbgl::JSValue& fpsValue = document["fps"];
assert(fpsValue.IsArray());
for (auto& probeValue : fpsValue.GetArray()) {
assert(probeValue.IsArray());
assert(probeValue.Size() >= 4u);
assert(probeValue[0].IsString());
assert(probeValue[1].IsNumber()); // Average
assert(probeValue[2].IsNumber()); // Minimum
assert(probeValue[3].IsNumber()); // Tolerance
const std::string mark{probeValue[0].GetString(), probeValue[0].GetStringLength()};
assert(!mark.empty());
result.fps.insert(
{std::move(mark), {probeValue[1].GetFloat(), probeValue[2].GetFloat(), probeValue[3].GetFloat()}});
}
}
return result;
}
@@ -607,8 +641,21 @@ TestMetadata parseTestMetadata(const TestPaths& paths) {
}
if (testValue.HasMember("mapMode")) {
metadata.outputsImage = true;
assert(testValue["mapMode"].IsString());
metadata.mapMode = testValue["mapMode"].GetString() == std::string("tile") ? mbgl::MapMode::Tile : mbgl::MapMode::Static;
std::string mapModeStr = testValue["mapMode"].GetString();
if (mapModeStr == "tile")
metadata.mapMode = mbgl::MapMode::Tile;
else if (mapModeStr == "continuous") {
metadata.mapMode = mbgl::MapMode::Continuous;
metadata.outputsImage = false;
} else if (mapModeStr == "static")
metadata.mapMode = mbgl::MapMode::Static;
else {
mbgl::Log::Warning(
mbgl::Event::ParseStyle, "Unknown map mode: %s. Falling back to static mode", mapModeStr.c_str());
metadata.mapMode = mbgl::MapMode::Static;
}
}
// Test operations handled in runner.cpp.
@@ -703,19 +750,21 @@ std::string createResultItem(const TestMetadata& metadata, bool hasFailedTests)
html.append("<div class=\"test " + metadata.status + (shouldHide ? " hide" : "") + "\">\n");
html.append(R"(<h2><span class="label" style="background: )" + metadata.color + "\">" + metadata.status + "</span> " + metadata.id + "</h2>\n");
if (metadata.status != "errored") {
if (metadata.renderTest) {
html.append("<img width=" + mbgl::util::toString(metadata.size.width));
html.append(" height=" + mbgl::util::toString(metadata.size.height));
html.append(" src=\"data:image/png;base64," + encodeBase64(metadata.actual) + "\"");
html.append(" data-alt-src=\"data:image/png;base64," + encodeBase64(metadata.expected) + "\">\n");
if (metadata.outputsImage) {
if (metadata.renderTest) {
html.append("<img width=" + mbgl::util::toString(metadata.size.width));
html.append(" height=" + mbgl::util::toString(metadata.size.height));
html.append(" src=\"data:image/png;base64," + encodeBase64(metadata.actual) + "\"");
html.append(" data-alt-src=\"data:image/png;base64," + encodeBase64(metadata.expected) + "\">\n");
html.append("<img width=" + mbgl::util::toString(metadata.size.width));
html.append(" height=" + mbgl::util::toString(metadata.size.height));
html.append(" src=\"data:image/png;base64," + encodeBase64(metadata.diff) + "\">\n");
} else {
html.append("<img width=" + mbgl::util::toString(metadata.size.width));
html.append(" height=" + mbgl::util::toString(metadata.size.height));
html.append(" src=\"data:image/png;base64," + encodeBase64(metadata.actual) + "\">\n");
html.append("<img width=" + mbgl::util::toString(metadata.size.width));
html.append(" height=" + mbgl::util::toString(metadata.size.height));
html.append(" src=\"data:image/png;base64," + encodeBase64(metadata.diff) + "\">\n");
} else {
html.append("<img width=" + mbgl::util::toString(metadata.size.width));
html.append(" height=" + mbgl::util::toString(metadata.size.height));
html.append(" src=\"data:image/png;base64," + encodeBase64(metadata.actual) + "\">\n");
}
}
} else {
assert(!metadata.errorMessage.empty());

3
render-test/render_test.cpp
View File

@@ -100,7 +100,8 @@ int runRenderTests(int argc, char** argv) {
errored = !runner.run(metadata) || !metadata.errorMessage.empty();
}
bool passed = !errored && !metadata.diff.empty() && metadata.difference <= metadata.allowed;
bool passed =
!errored && (!metadata.outputsImage || !metadata.diff.empty()) && metadata.difference <= metadata.allowed;
if (shouldIgnore) {
if (passed) {

264
render-test/runner.cpp
View File

@@ -1,6 +1,7 @@
#include <mbgl/map/camera.hpp>
#include <mbgl/map/map_observer.hpp>
#include <mbgl/renderer/renderer.hpp>
#include <mbgl/renderer/renderer_observer.hpp>
#include <mbgl/style/conversion/filter.hpp>
#include <mbgl/style/conversion/layer.hpp>
#include <mbgl/style/conversion/light.hpp>
@@ -33,12 +34,41 @@
using namespace mbgl;
// static
class TestRunnerMapObserver : public MapObserver {
public:
TestRunnerMapObserver() : mapLoadFailure(false), finishRenderingMap(false), idle(false) {}
void onDidFailLoadingMap(MapLoadError, const std::string&) override { mapLoadFailure = true; }
void onDidFinishRenderingMap(RenderMode mode) override final {
if (!finishRenderingMap) finishRenderingMap = mode == RenderMode::Full;
}
void onDidBecomeIdle() override final { idle = true; }
void reset() {
mapLoadFailure = false;
finishRenderingMap = false;
idle = false;
}
bool mapLoadFailure;
bool finishRenderingMap;
bool idle;
};
// static
const std::string& TestRunner::getBasePath() {
const static std::string result = std::string(TEST_RUNNER_ROOT_PATH).append("/mapbox-gl-js/test/integration");
return result;
}
// static
gfx::HeadlessBackend::SwapBehaviour swapBehavior(MapMode mode) {
return mode == MapMode::Continuous ? gfx::HeadlessBackend::SwapBehaviour::Flush
: gfx::HeadlessBackend::SwapBehaviour::NoFlush;
}
std::string simpleDiff(const Value& result, const Value& expected) {
std::vector<std::string> resultTokens{tokenize(toJSON(result, 2, false))};
std::vector<std::string> expectedTokens{tokenize(toJSON(expected, 2, false))};
@@ -159,81 +189,97 @@ bool TestRunner::checkRenderTestResults(mbgl::PremultipliedImage&& actualImage,
const std::string& base = metadata.paths.defaultExpectations();
const std::vector<mbgl::filesystem::path>& expectations = metadata.paths.expectations;
metadata.actual = mbgl::encodePNG(actualImage);
if (metadata.outputsImage) {
metadata.actual = mbgl::encodePNG(actualImage);
if (actualImage.size.isEmpty()) {
metadata.errorMessage = "Invalid size for actual image";
return false;
if (actualImage.size.isEmpty()) {
metadata.errorMessage = "Invalid size for actual image";
return false;
}
#if !TEST_READ_ONLY
if (getenv("UPDATE_PLATFORM")) {
mbgl::filesystem::create_directories(expectations.back());
mbgl::util::write_file(expectations.back().string() + "/expected.png", mbgl::encodePNG(actualImage));
return true;
} else if (getenv("UPDATE_DEFAULT")) {
mbgl::util::write_file(base + "/expected.png", mbgl::encodePNG(actualImage));
return true;
}
mbgl::util::write_file(base + "/actual.png", metadata.actual);
#endif
mbgl::PremultipliedImage expectedImage{actualImage.size};
mbgl::PremultipliedImage imageDiff{actualImage.size};
double pixels = 0.0;
std::vector<std::string> expectedImagesPaths;
for (auto rit = expectations.rbegin(); rit != expectations.rend(); ++rit) {
if (mbgl::filesystem::exists(*rit)) {
expectedImagesPaths = readExpectedImageEntries(*rit);
if (!expectedImagesPaths.empty()) break;
}
}
if (expectedImagesPaths.empty()) {
metadata.errorMessage = "Failed to find expectations for: " + metadata.paths.stylePath.string();
return false;
}
for (const auto& entry : expectedImagesPaths) {
mbgl::optional<std::string> maybeExpectedImage = mbgl::util::readFile(entry);
if (!maybeExpectedImage) {
metadata.errorMessage = "Failed to load expected image " + entry;
return false;
}
metadata.expected = *maybeExpectedImage;
expectedImage = mbgl::decodeImage(*maybeExpectedImage);
pixels = // implicitly converting from uint64_t
mapbox::pixelmatch(actualImage.data.get(),
expectedImage.data.get(),
expectedImage.size.width,
expectedImage.size.height,
imageDiff.data.get(),
0.1285); // Defined in GL JS
metadata.diff = mbgl::encodePNG(imageDiff);
#if !TEST_READ_ONLY
mbgl::util::write_file(base + "/diff.png", metadata.diff);
#endif
metadata.difference = pixels / expectedImage.size.area();
if (metadata.difference <= metadata.allowed) {
break;
}
}
}
#if !TEST_READ_ONLY
if (getenv("UPDATE_PLATFORM")) {
mbgl::filesystem::create_directories(expectations.back());
mbgl::util::write_file(expectations.back().string() + "/expected.png", mbgl::encodePNG(actualImage));
return true;
} else if (getenv("UPDATE_DEFAULT")) {
mbgl::util::write_file(base + "/expected.png", mbgl::encodePNG(actualImage));
return true;
} else if (getenv("UPDATE_METRICS")) {
if (getenv("UPDATE_METRICS")) {
if (!metadata.metrics.isEmpty()) {
mbgl::filesystem::create_directories(expectations.back());
mbgl::util::write_file(expectations.back().string() + "/metrics.json", serializeMetrics(metadata.metrics));
return true;
}
}
mbgl::util::write_file(base + "/actual.png", metadata.actual);
#endif
mbgl::PremultipliedImage expectedImage { actualImage.size };
mbgl::PremultipliedImage imageDiff { actualImage.size };
double pixels = 0.0;
std::vector<std::string> expectedImagesPaths;
mbgl::filesystem::path expectedMetricsPath;
for (auto rit = expectations.rbegin(); rit!= expectations.rend(); ++rit) {
for (auto rit = expectations.rbegin(); rit != expectations.rend(); ++rit) {
if (mbgl::filesystem::exists(*rit)) {
if (metadata.expectedMetrics.isEmpty()) {
mbgl::filesystem::path maybeExpectedMetricsPath{ *rit };
mbgl::filesystem::path maybeExpectedMetricsPath{*rit};
maybeExpectedMetricsPath.replace_filename("metrics.json");
metadata.expectedMetrics = readExpectedMetrics(maybeExpectedMetricsPath);
}
expectedImagesPaths = readExpectedImageEntries(*rit);
if (!expectedImagesPaths.empty()) break;
}
}
if (expectedImagesPaths.empty()) {
metadata.errorMessage = "Failed to find expectations for: " + metadata.paths.stylePath.string();
return false;
}
for (const auto& entry: expectedImagesPaths) {
mbgl::optional<std::string> maybeExpectedImage = mbgl::util::readFile(entry);
if (!maybeExpectedImage) {
metadata.errorMessage = "Failed to load expected image " + entry;
return false;
}
metadata.expected = *maybeExpectedImage;
expectedImage = mbgl::decodeImage(*maybeExpectedImage);
pixels = // implicitly converting from uint64_t
mapbox::pixelmatch(actualImage.data.get(), expectedImage.data.get(), expectedImage.size.width,
expectedImage.size.height, imageDiff.data.get(), 0.1285); // Defined in GL JS
metadata.diff = mbgl::encodePNG(imageDiff);
#if !TEST_READ_ONLY
mbgl::util::write_file(base + "/diff.png", metadata.diff);
#endif
metadata.difference = pixels / expectedImage.size.area();
if (metadata.difference <= metadata.allowed) {
break;
}
}
// Check file size metrics.
for (const auto& expected : metadata.expectedMetrics.fileSize) {
auto actual = metadata.metrics.fileSize.find(expected.first);
@@ -317,16 +363,38 @@ bool TestRunner::checkRenderTestResults(mbgl::PremultipliedImage&& actualImage,
}
}
#endif // !defined(SANITIZE)
// Check fps metrics
for (const auto& expected : metadata.expectedMetrics.fps) {
auto actual = metadata.metrics.fps.find(expected.first);
if (actual == metadata.metrics.fps.end()) {
metadata.errorMessage = "Failed to find fps probe: " + expected.first;
return false;
}
auto result = checkValue(expected.second.average, actual->second.average, expected.second.tolerance);
if (!std::get<bool>(result)) {
std::stringstream ss;
ss << "Average fps at probe \"" << expected.first << "\" is " << actual->second.average
<< ", expected to be " << expected.second.average << " with tolerance of " << expected.second.tolerance;
metadata.errorMessage = ss.str();
return false;
}
result = checkValue(expected.second.minOnePc, actual->second.minOnePc, expected.second.tolerance);
if (!std::get<bool>(result)) {
std::stringstream ss;
ss << "Minimum(1%) fps at probe \"" << expected.first << "\" is " << actual->second.minOnePc
<< ", expected to be " << expected.second.minOnePc << " with tolerance of " << expected.second.tolerance;
metadata.errorMessage = ss.str();
return false;
}
}
return true;
}
bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
if (!metadata.document.HasMember("metadata") ||
!metadata.document["metadata"].HasMember("test") ||
if (!metadata.document.HasMember("metadata") || !metadata.document["metadata"].HasMember("test") ||
!metadata.document["metadata"]["test"].HasMember("operations")) {
return true;
}
assert(metadata.document["metadata"]["test"]["operations"].IsArray());
const auto& operationsArray = metadata.document["metadata"]["test"]["operations"].GetArray();
@@ -342,6 +410,7 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
auto& frontend = maps[key]->frontend;
auto& map = maps[key]->map;
auto& observer = maps[key]->observer;
static const std::string waitOp("wait");
static const std::string sleepOp("sleep");
@@ -372,6 +441,7 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
static const std::string setFeatureStateOp("setFeatureState");
static const std::string getFeatureStateOp("getFeatureState");
static const std::string removeFeatureStateOp("removeFeatureState");
static const std::string panGestureOp("panGesture");
if (operationArray[0].GetString() == waitOp) {
// wait
@@ -813,9 +883,76 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
return false;
}
frontend.getRenderer()->removeFeatureState(sourceID, sourceLayer, featureID, stateKey);
} else if (operationArray[0].GetString() == panGestureOp) {
// benchmarkPanGesture
assert(operationArray.Size() >= 4u);
assert(operationArray[1].IsString()); // identifier
assert(operationArray[2].IsNumber()); // duration
assert(operationArray[3].IsArray()); // start [lat, lng, zoom]
assert(operationArray[4].IsArray()); // end [lat, lng, zoom]
if (metadata.mapMode != mbgl::MapMode::Continuous) {
metadata.errorMessage = "Map mode must be Continous for " + panGestureOp + " operation";
return false;
}
std::string mark = operationArray[1].GetString();
int duration = operationArray[2].GetFloat();
LatLng startPos, endPos;
double startZoom, endZoom;
std::vector<float> samples;
auto parsePosition = [](auto arr) -> std::tuple<LatLng, double> {
assert(arr.Size() >= 3);
return {{arr[1].GetDouble(), arr[0].GetDouble()}, arr[2].GetDouble()};
};
std::tie(startPos, startZoom) = parsePosition(operationArray[3].GetArray());
std::tie(endPos, endZoom) = parsePosition(operationArray[4].GetArray());
// Jump to the starting point of the segment and make sure there's something to render
map.jumpTo(mbgl::CameraOptions().withCenter(startPos).withZoom(startZoom));
observer->reset();
while (!observer->finishRenderingMap) {
frontend.renderOnce(map);
}
if (observer->mapLoadFailure) return false;
size_t frames = 0;
float totalTime = 0.0;
bool transitionFinished = false;
mbgl::AnimationOptions animationOptions(mbgl::Milliseconds(duration * 1000));
animationOptions.minZoom = util::min(startZoom, endZoom);
animationOptions.transitionFinishFn = [&]() { transitionFinished = true; };
map.flyTo(mbgl::CameraOptions().withCenter(endPos).withZoom(endZoom), animationOptions);
for (; !transitionFinished; frames++) {
frontend.renderOnce(map);
float frameTime = (float)frontend.getFrameTime();
totalTime += frameTime;
samples.push_back(frameTime);
}
float averageFps = totalTime > 0.0 ? frames / totalTime : 0.0;
float minFrameTime = 0.0;
// Use 1% of the longest frames to compute the minimum fps
std::sort(samples.begin(), samples.end());
int sampleCount = util::max(1, (int)samples.size() / 100);
for (auto it = samples.rbegin(); it != samples.rbegin() + sampleCount; it++) minFrameTime += *it;
float minOnePcFps = sampleCount / minFrameTime;
metadata.metrics.fps.insert({std::move(mark), {averageFps, minOnePcFps, 0.0f}});
} else {
metadata.errorMessage = std::string("Unsupported operation: ") + operationArray[0].GetString();
metadata.errorMessage = std::string("Unsupported operation: ") + operationArray[0].GetString();
return false;
}
@@ -824,9 +961,10 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
}
TestRunner::Impl::Impl(const TestMetadata& metadata)
: frontend(metadata.size, metadata.pixelRatio),
: observer(std::make_unique<TestRunnerMapObserver>()),
frontend(metadata.size, metadata.pixelRatio, swapBehavior(metadata.mapMode)),
map(frontend,
mbgl::MapObserver::nullObserver(),
*observer.get(),
mbgl::MapOptions()
.withMapMode(metadata.mapMode)
.withSize(metadata.size)
@@ -834,6 +972,8 @@ TestRunner::Impl::Impl(const TestMetadata& metadata)
.withCrossSourceCollisions(metadata.crossSourceCollisions),
mbgl::ResourceOptions().withCacheOnlyRequestsSupport(false)) {}
TestRunner::Impl::~Impl() {}
bool TestRunner::run(TestMetadata& metadata) {
AllocationIndex::setActive(false);
AllocationIndex::reset();
@@ -864,7 +1004,7 @@ bool TestRunner::run(TestMetadata& metadata) {
mbgl::PremultipliedImage image;
try {
image = frontend.render(map);
if (metadata.outputsImage) image = frontend.render(map);
} catch (const std::exception&) {
return false;
}

3
render-test/runner.hpp
View File

@@ -5,6 +5,7 @@
#include <memory>
class TestRunnerMapObserver;
struct TestMetadata;
class TestRunner {
@@ -26,7 +27,9 @@ private:
struct Impl {
Impl(const TestMetadata&);
~Impl();
std::unique_ptr<TestRunnerMapObserver> observer;
mbgl::HeadlessFrontend frontend;
mbgl::Map map;
};

1
src/core-files.json
View File

@@ -483,6 +483,7 @@
"mbgl/util/indexed_tuple.hpp": "include/mbgl/util/indexed_tuple.hpp",
"mbgl/util/interpolate.hpp": "include/mbgl/util/interpolate.hpp",
"mbgl/util/logging.hpp": "include/mbgl/util/logging.hpp",
"mbgl/util/monotonic_timer.hpp": "include/mbgl/util/monotonic_timer.hpp",
"mbgl/util/noncopyable.hpp": "include/mbgl/util/noncopyable.hpp",
"mbgl/util/optional.hpp": "include/mbgl/util/optional.hpp",
"mbgl/util/platform.hpp": "include/mbgl/util/platform.hpp",

4
src/mbgl/gl/context.cpp
View File

@@ -583,6 +583,10 @@ std::unique_ptr<gfx::CommandEncoder> Context::createCommandEncoder() {
return std::make_unique<gl::CommandEncoder>(*this);
}
void Context::finish() {
MBGL_CHECK_ERROR(glFinish());
}
void Context::draw(const gfx::DrawMode& drawMode,
std::size_t indexOffset,
std::size_t indexLength) {

2
src/mbgl/gl/context.hpp
View File

@@ -92,6 +92,8 @@ public:
std::size_t indexOffset,
std::size_t indexLength);
void finish();
// Actually remove the objects we marked as abandoned with the above methods.
// Only call this while the OpenGL context is exclusive to this thread.
void performCleanup() override;

5
test/gl/context.test.cpp
View File

@@ -91,9 +91,10 @@ TEST(GLContextMode, Shared) {
util::RunLoop loop;
HeadlessFrontend frontend { 1, gfx::ContextMode::Shared };
HeadlessFrontend frontend{1, gfx::HeadlessBackend::SwapBehaviour::NoFlush, gfx::ContextMode::Shared};
Map map(frontend, MapObserver::nullObserver(),
Map map(frontend,
MapObserver::nullObserver(),
MapOptions().withMapMode(MapMode::Static).withSize(frontend.getSize()),
ResourceOptions().withCachePath(":memory:").withAssetPath("test/fixtures/api/assets"));
map.getStyle().loadJSON(util::read_file("test/fixtures/api/water.json"));

4
test/text/local_glyph_rasterizer.test.cpp
View File

@@ -33,9 +33,7 @@ namespace {
class LocalGlyphRasterizerTest {
public:
LocalGlyphRasterizerTest(const optional<std::string> fontFamily)
: frontend(1, gfx::ContextMode::Unique, fontFamily)
{
}
: frontend(1, gfx::HeadlessBackend::SwapBehaviour::NoFlush, gfx::ContextMode::Unique, fontFamily) {}
util::RunLoop loop;
std::shared_ptr<StubFileSource> fileSource = std::make_shared<StubFileSource>();