diff options
author | Mikhail Pozdnyakov <mikhail.pozdnyakov@mapbox.com> | 2019-10-11 10:43:02 +0300 |
---|---|---|
committer | Mikhail Pozdnyakov <mikhail.pozdnyakov@mapbox.com> | 2019-10-21 16:10:45 +0300 |
commit | 9adc78134c261d5e236faac3b537e1cfd937880a (patch) | |
tree | b9a12b9fe8ddc5e828e588efc04187148e042e77 | |
parent | f41f274ca16032da8c41186b2181f0c800d777df (diff) |
[test runner] Threshold for memory metrics
-rw-r--r-- | render-test/metadata.hpp | 13 | ||||
-rw-r--r-- | render-test/parser.cpp | 2 | ||||
-rw-r--r-- | render-test/runner.cpp | 28 |
3 files changed, 30 insertions, 13 deletions
diff --git a/render-test/metadata.hpp b/render-test/metadata.hpp index c26fde1d1..20cc440fd 100644 --- a/render-test/metadata.hpp +++ b/render-test/metadata.hpp @@ -49,12 +49,19 @@ struct FileSizeProbe { struct MemoryProbe { MemoryProbe() = default; - MemoryProbe(size_t peak_, size_t allocations_) - : peak(peak_) - , allocations(allocations_) {} + MemoryProbe(size_t peak_, size_t allocations_) : peak(peak_), allocations(allocations_), tolerance(0.0f) {} size_t peak; size_t allocations; + float tolerance; + + static std::tuple<bool, float> checkPeak(const MemoryProbe& expected, const MemoryProbe& actual) { + return checkValue(expected.peak, actual.peak, actual.tolerance); + } + + static std::tuple<bool, float> checkAllocations(const MemoryProbe& expected, const MemoryProbe& actual) { + return checkValue(expected.allocations, actual.allocations, actual.tolerance); + } }; struct NetworkProbe { diff --git a/render-test/parser.cpp b/render-test/parser.cpp index 41c9b8329..7300530b0 100644 --- a/render-test/parser.cpp +++ b/render-test/parser.cpp @@ -507,7 +507,7 @@ TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path) { assert(probeValue[1].IsNumber()); assert(probeValue[2].IsNumber()); - const std::string mark { probeValue[0].GetString(), probeValue[0].GetStringLength() }; + std::string mark{probeValue[0].GetString(), probeValue[0].GetStringLength()}; assert(!mark.empty()); result.memory.emplace(std::piecewise_construct, std::forward_as_tuple(std::move(mark)), diff --git a/render-test/runner.cpp b/render-test/runner.cpp index 810806d23..925599dfb 100644 --- a/render-test/runner.cpp +++ b/render-test/runner.cpp @@ -276,19 +276,23 @@ bool TestRunner::checkRenderTestResults(mbgl::PremultipliedImage&& actualImage, metadata.errorMessage = "Failed to find memory probe: " + expected.first; return false; } - if (actual->second.peak > expected.second.peak) { + bool passed{false}; + float delta{0.0f}; + std::tie(passed, delta) = MemoryProbe::checkPeak(expected.second, actual->second); + if (!passed) { std::stringstream ss; - ss << "Allocated memory peak size at probe \"" << expected.first << "\" is " - << actual->second.peak << " bytes, expected is " << expected.second.peak << " bytes."; + ss << "Allocated memory peak size at probe \"" << expected.first << "\" is " << actual->second.peak + << " bytes, expected is " << expected.second.peak << "±" << delta << " bytes."; metadata.errorMessage = ss.str(); return false; } - if (actual->second.allocations > expected.second.allocations) { + std::tie(passed, delta) = MemoryProbe::checkAllocations(expected.second, actual->second); + if (!passed) { std::stringstream ss; - ss << "Number of allocations at probe \"" << expected.first << "\" is " - << actual->second.allocations << ", expected is " << expected.second.allocations << "."; + ss << "Number of allocations at probe \"" << expected.first << "\" is " << actual->second.allocations + << ", expected is " << expected.second.allocations << "±" << delta << " allocations."; metadata.errorMessage = ss.str(); return false; @@ -644,9 +648,15 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { assert(operationArray[1].IsString()); std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength()); - metadata.metrics.memory.emplace(std::piecewise_construct, - std::forward_as_tuple(std::move(mark)), - std::forward_as_tuple(AllocationIndex::getAllocatedSizePeak(), AllocationIndex::getAllocationsCount())); + auto emplaced = metadata.metrics.memory.emplace( + std::piecewise_construct, + std::forward_as_tuple(std::move(mark)), + std::forward_as_tuple(AllocationIndex::getAllocatedSizePeak(), AllocationIndex::getAllocationsCount())); + assert(emplaced.second); + if (operationArray.Size() >= 3u) { + assert(operationArray[2].IsNumber()); + emplaced.first->second.tolerance = float(operationArray[2].GetDouble()); + } } else if (operationArray[0].GetString() == memoryProbeEndOp) { // probeMemoryEnd assert(AllocationIndex::isActive()); |