[render-test] Add support for collecting network metrics

This commit is contained in:
Juha Alanen
2019-10-10 12:17:34 +03:00
parent ee2b803315
commit b53093b522
6 changed files with 204 additions and 2 deletions

View File

@@ -4,7 +4,8 @@ add_library(
${MBGL_ROOT}/expression-test/test_runner_common.hpp
${MBGL_ROOT}/render-test/allocation_index.cpp
${MBGL_ROOT}/render-test/allocation_index.hpp
${MBGL_ROOT}/render-test/filesystem.hpp
${MBGL_ROOT}/render-test/file_source.cpp
${MBGL_ROOT}/render-test/file_source.hpp
${MBGL_ROOT}/render-test/filesystem.hpp
${MBGL_ROOT}/render-test/include/mbgl/render_test.hpp
${MBGL_ROOT}/render-test/metadata.hpp

View File

@@ -0,0 +1,65 @@
#include <mbgl/storage/resource_options.hpp>
#include "file_source.hpp"
namespace mbgl {
std::atomic_size_t requestCount{0};
std::atomic_size_t transferredSize{0};
std::atomic_bool active{false};
ProxyFileSource::ProxyFileSource(const std::string& cachePath,
const std::string& assetPath,
bool supportCacheOnlyRequests_)
: DefaultFileSource(cachePath, assetPath, supportCacheOnlyRequests_) {}
ProxyFileSource::ProxyFileSource(const std::string& cachePath,
std::unique_ptr<FileSource>&& assetFileSource_,
bool supportCacheOnlyRequests_)
: DefaultFileSource(cachePath, std::move(assetFileSource_), supportCacheOnlyRequests_) {}
ProxyFileSource::~ProxyFileSource() = default;
std::unique_ptr<AsyncRequest> ProxyFileSource::request(const Resource& resource, Callback callback) {
auto result = DefaultFileSource::request(resource, [=](Response response) {
std::size_t size = response.data != nullptr ? response.data->size() : 0;
if (active) {
requestCount++;
transferredSize += size;
}
callback(response);
});
return result;
}
std::shared_ptr<FileSource> FileSource::createPlatformFileSource(const ResourceOptions& options) {
auto fileSource = std::make_shared<ProxyFileSource>(
options.cachePath(), options.assetPath(), options.supportsCacheOnlyRequests());
fileSource->setAccessToken(options.accessToken());
fileSource->setAPIBaseURL(options.baseURL());
return fileSource;
}
// static
void ProxyFileSource::setTrackingActive(bool active_) {
active = active_;
requestCount = 0;
transferredSize = 0;
}
// static
bool ProxyFileSource::isTrackingActive() {
return active;
}
// static
size_t ProxyFileSource::getRequestCount() {
return requestCount;
}
// static
size_t ProxyFileSource::getTransferredSize() {
return transferredSize;
}
} // namespace mbgl

View File

@@ -0,0 +1,40 @@
#pragma once
#include <mbgl/storage/default_file_source.hpp>
namespace mbgl {
class ProxyFileSource : public DefaultFileSource {
public:
ProxyFileSource(const std::string& cachePath, const std::string& assetPath, bool supportCacheOnlyRequests = true);
ProxyFileSource(const std::string& cachePath,
std::unique_ptr<FileSource>&& assetFileSource,
bool supportCacheOnlyRequests = true);
~ProxyFileSource();
std::unique_ptr<AsyncRequest> request(const Resource&, Callback);
/**
* @brief Starts/stops metrics tracking.
*/
static void setTrackingActive(bool);
/**
* @brief Returns metrics tracking status.
*/
static bool isTrackingActive();
/**
* @brief Returns the total amount of requests.
*
* @return size_t
*/
static size_t getRequestCount();
/**
* @brief Returns the size of transferred data (in bytes).
*
* @return size_t
*/
static size_t getTransferredSize();
};
} // namespace mbgl

View File

@@ -57,11 +57,20 @@ struct MemoryProbe {
size_t allocations;
};
struct NetworkProbe {
NetworkProbe() = default;
NetworkProbe(size_t requests_, size_t transferred_) : requests(requests_), transferred(transferred_) {}
size_t requests;
size_t transferred;
};
class TestMetrics {
public:
bool isEmpty() const { return fileSize.empty() && memory.empty(); }
bool isEmpty() const { return fileSize.empty() && memory.empty() && network.empty(); }
std::map<std::string, FileSizeProbe> fileSize;
std::map<std::string, MemoryProbe> memory;
std::map<std::string, NetworkProbe> network;
};
struct TestMetadata {

View File

@@ -304,6 +304,21 @@ std::string serializeMetrics(const TestMetrics& metrics) {
writer.EndArray();
}
// Start network section
if (!metrics.network.empty()) {
writer.Key("network");
writer.StartArray();
for (const auto& networkProbe : metrics.network) {
assert(!networkProbe.first.empty());
writer.StartArray();
writer.String(networkProbe.first.c_str());
writer.Uint64(networkProbe.second.requests);
writer.Uint64(networkProbe.second.transferred);
writer.EndArray();
}
writer.EndArray();
}
writer.EndObject();
return s.GetString();
@@ -500,6 +515,25 @@ TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path) {
}
}
if (document.HasMember("network")) {
const mbgl::JSValue& networkValue = document["network"];
assert(networkValue.IsArray());
for (auto& probeValue : networkValue.GetArray()) {
assert(probeValue.IsArray());
assert(probeValue.Size() >= 3u);
assert(probeValue[0].IsString());
assert(probeValue[1].IsNumber());
assert(probeValue[2].IsNumber());
std::string mark{probeValue[0].GetString(), probeValue[0].GetStringLength()};
assert(!mark.empty());
result.network.emplace(std::piecewise_construct,
std::forward_as_tuple(std::move(mark)),
std::forward_as_tuple(probeValue[1].GetUint64(), probeValue[2].GetUint64()));
}
}
return result;
}

View File

@@ -21,6 +21,7 @@
#include <../expression-test/test_runner_common.hpp>
#include "allocation_index.hpp"
#include "file_source.hpp"
#include "metadata.hpp"
#include "parser.hpp"
#include "runner.hpp"
@@ -294,6 +295,35 @@ bool TestRunner::checkRenderTestResults(mbgl::PremultipliedImage&& actualImage,
}
}
// Check network metrics.
for (const auto& expected : metadata.expectedMetrics.network) {
auto actual = metadata.metrics.network.find(expected.first);
if (actual == metadata.metrics.network.end()) {
metadata.errorMessage = "Failed to find network probe: " + expected.first;
return false;
}
bool failed = false;
if (actual->second.requests != expected.second.requests) {
std::stringstream ss;
ss << "Number of requests at probe \"" << expected.first << "\" is " << actual->second.requests
<< ", expected is " << expected.second.requests << ". ";
metadata.errorMessage = ss.str();
failed = true;
}
if (actual->second.transferred != expected.second.transferred) {
std::stringstream ss;
ss << "Transferred data at probe \"" << expected.first << "\" is " << actual->second.transferred
<< " bytes, expected is " << expected.second.transferred << " bytes.";
metadata.errorMessage += ss.str();
failed = true;
}
if (failed) {
return false;
}
}
return true;
}
@@ -343,6 +373,9 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
static const std::string memoryProbeOp("probeMemory");
static const std::string memoryProbeStartOp("probeMemoryStart");
static const std::string memoryProbeEndOp("probeMemoryEnd");
static const std::string networkProbeOp("probeNetwork");
static const std::string networkProbeStartOp("probeNetworkStart");
static const std::string networkProbeEndOp("probeNetworkEnd");
static const std::string setFeatureStateOp("setFeatureState");
static const std::string getFeatureStateOp("getFeatureState");
static const std::string removeFeatureStateOp("removeFeatureState");
@@ -619,6 +652,25 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
assert(AllocationIndex::isActive());
AllocationIndex::setActive(false);
AllocationIndex::reset();
} else if (operationArray[0].GetString() == networkProbeStartOp) {
// probeNetworkStart
assert(!ProxyFileSource::isTrackingActive());
ProxyFileSource::setTrackingActive(true);
} else if (operationArray[0].GetString() == networkProbeOp) {
// probeNetwork
assert(ProxyFileSource::isTrackingActive());
assert(operationArray.Size() >= 2u);
assert(operationArray[1].IsString());
std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength());
metadata.metrics.network.emplace(
std::piecewise_construct,
std::forward_as_tuple(std::move(mark)),
std::forward_as_tuple(ProxyFileSource::getRequestCount(), ProxyFileSource::getTransferredSize()));
} else if (operationArray[0].GetString() == networkProbeEndOp) {
// probeNetworkEnd
assert(ProxyFileSource::isTrackingActive());
ProxyFileSource::setTrackingActive(false);
} else if (operationArray[0].GetString() == setFeatureStateOp) {
// setFeatureState
assert(operationArray.Size() >= 3u);
@@ -786,6 +838,7 @@ TestRunner::Impl::Impl(const TestMetadata& metadata)
bool TestRunner::run(TestMetadata& metadata) {
AllocationIndex::setActive(false);
AllocationIndex::reset();
ProxyFileSource::setTrackingActive(false);
std::string key = mbgl::util::toString(uint32_t(metadata.mapMode))
+ "/" + mbgl::util::toString(metadata.pixelRatio)
+ "/" + mbgl::util::toString(uint32_t(metadata.crossSourceCollisions));