wehub-resource-sync
cb66c5d68e
bazel / bazel.ubuntu-latest (push) Failing after 1s
build-and-test-perfcounters / ubuntu-latest.Debug (push) Failing after 0s
build-and-test-perfcounters / ubuntu-latest.Release (push) Failing after 1s
doxygen / Build HTML documentation (push) Failing after 1s
python + Bazel pre-commit checks / pre-commit (push) Failing after 1s
sanitizer / tsan.Debug (push) Failing after 1s
sanitizer / ubsan.Debug (push) Failing after 0s
sanitizer / ubsan.RelWithDebInfo (push) Failing after 1s
build-and-test-min-cmake / ubuntu-latest.min-cmake (push) Failing after 1s
clang-format-lint / check-clang-format (push) Failing after 1s
sanitizer / asan.Debug (push) Failing after 1s
sanitizer / msan.RelWithDebInfo (push) Failing after 1s
clang-tidy / run-clang-tidy (push) Failing after 1s
sanitizer / msan.Debug (push) Failing after 1s
sanitizer / asan.RelWithDebInfo (push) Failing after 0s
sanitizer / tsan.RelWithDebInfo (push) Failing after 0s
test-bindings / Test Rust bindings on ubuntu-latest (push) Failing after 0s
test-bindings / Test Python 3.10 bindings on ubuntu-latest (push) Failing after 0s
test-bindings / Test Python 3.11 bindings on ubuntu-latest (push) Failing after 0s
test-bindings / Test Python 3.12 bindings on ubuntu-latest (push) Failing after 1s
test-bindings / Test Python 3.13 bindings on ubuntu-latest (push) Failing after 0s
build-and-test / macos-latest.Debug.shared.clang++ (push) Has been cancelled
build-and-test / ubuntu-22.04.Debug.shared.clang++ (push) Has been cancelled
build-and-test / ubuntu-24.04.Debug.shared.clang++ (push) Has been cancelled
build-and-test / ubuntu-24.04-arm.Debug.shared.clang++ (push) Has been cancelled
bazel / bazel.macos-latest (push) Has been cancelled
bazel / bazel.windows-latest (push) Has been cancelled
build-and-test-min-cmake / macos-latest.min-cmake (push) Has been cancelled
build-and-test / ubuntu-22.04.Debug.static.clang++ (push) Has been cancelled
build-and-test / macos-latest.Debug.shared.g++ (push) Has been cancelled
build-and-test / ubuntu-22.04.Debug.shared.g++ (push) Has been cancelled
build-and-test / ubuntu-24.04.Debug.shared.g++ (push) Has been cancelled
build-and-test / ubuntu-24.04-arm.Debug.shared.g++ (push) Has been cancelled
build-and-test / macos-latest.Debug.static.g++ (push) Has been cancelled
build-and-test / ubuntu-22.04.Debug.static.g++ (push) Has been cancelled
build-and-test / ubuntu-24.04.Debug.static.g++ (push) Has been cancelled
build-and-test / ubuntu-24.04-arm.Debug.static.g++ (push) Has been cancelled
build-and-test / macos-latest.Release.shared.clang++ (push) Has been cancelled
build-and-test / ubuntu-22.04.Release.shared.clang++ (push) Has been cancelled
build-and-test / ubuntu-24.04.Release.shared.clang++ (push) Has been cancelled
build-and-test / ubuntu-24.04-arm.Release.shared.clang++ (push) Has been cancelled
build-and-test / macos-latest.Release.static.clang++ (push) Has been cancelled
build-and-test / ubuntu-22.04.Release.static.clang++ (push) Has been cancelled
build-and-test / macos-latest.Debug.static.clang++ (push) Has been cancelled
build-and-test / ubuntu-24.04.Debug.static.clang++ (push) Has been cancelled
build-and-test / ubuntu-24.04-arm.Debug.static.clang++ (push) Has been cancelled
build-and-test / macos-latest.Release.static.g++ (push) Has been cancelled
build-and-test / ubuntu-22.04.Release.static.g++ (push) Has been cancelled
build-and-test / windows-2025.Debug.static.VS-18-2025 (push) Has been cancelled
build-and-test / windows-2022.Release.shared.VS-17-2022 (push) Has been cancelled
build-and-test / windows-2022.Release.static.VS-17-2022 (push) Has been cancelled
build-and-test / windows-2025.Release.shared.VS-18-2025 (push) Has been cancelled
build-and-test / windows-2025.Release.static.VS-18-2025 (push) Has been cancelled
build-and-test / windows-latest.Debug.shared.CLANG64 (push) Has been cancelled
build-and-test / windows-latest.Debug.shared.MINGW64 (push) Has been cancelled
build-and-test / windows-latest.Debug.shared.UCRT64 (push) Has been cancelled
build-and-test / windows-latest.Debug.static.CLANG64 (push) Has been cancelled
build-and-test / windows-latest.Debug.static.MINGW64 (push) Has been cancelled
build-and-test / windows-latest.Debug.static.UCRT64 (push) Has been cancelled
build-and-test / ubuntu-24.04.Release.static.clang++ (push) Has been cancelled
build-and-test / ubuntu-24.04-arm.Release.static.clang++ (push) Has been cancelled
build-and-test / macos-latest.Release.shared.g++ (push) Has been cancelled
build-and-test / ubuntu-22.04.Release.shared.g++ (push) Has been cancelled
build-and-test / ubuntu-24.04.Release.shared.g++ (push) Has been cancelled
build-and-test / ubuntu-24.04-arm.Release.shared.g++ (push) Has been cancelled
build-and-test / ubuntu-24.04.Release.static.g++ (push) Has been cancelled
build-and-test / ubuntu-24.04-arm.Release.static.g++ (push) Has been cancelled
build-and-test / windows-2022.Debug.shared.VS-17-2022 (push) Has been cancelled
build-and-test / windows-2022.Debug.static.VS-17-2022 (push) Has been cancelled
build-and-test / windows-2025.Debug.shared.VS-18-2025 (push) Has been cancelled
build-and-test / windows-latest.Release.static.MINGW64 (push) Has been cancelled
build-and-test / windows-latest.Release.static.UCRT64 (push) Has been cancelled
test-bindings / Test Python 3.10 bindings on macos-latest (push) Has been cancelled
test-bindings / Test Python 3.11 bindings on macos-latest (push) Has been cancelled
test-bindings / Test Python 3.12 bindings on macos-latest (push) Has been cancelled
test-bindings / Test Python 3.13 bindings on macos-latest (push) Has been cancelled
build-and-test / windows-latest.Release.shared.CLANG64 (push) Has been cancelled
build-and-test / windows-latest.Release.shared.MINGW64 (push) Has been cancelled
build-and-test / windows-latest.Release.shared.UCRT64 (push) Has been cancelled
build-and-test / windows-latest.Release.static.CLANG64 (push) Has been cancelled
test-bindings / Test Python 3.12 bindings on windows-latest (push) Has been cancelled
test-bindings / Test Python 3.13 bindings on windows-latest (push) Has been cancelled
test-bindings / Test Python 3.10 bindings on windows-latest (push) Has been cancelled
test-bindings / Test Python 3.11 bindings on windows-latest (push) Has been cancelled
test-bindings / Test Rust bindings on macos-latest (push) Has been cancelled
test-bindings / Test Rust bindings on windows-latest (push) Has been cancelled
573 lines
28 KiB
C++
573 lines
28 KiB
C++
|
|
#undef NDEBUG
|
|
|
|
#include "benchmark/benchmark_api.h"
|
|
#include "benchmark/counter.h"
|
|
#include "benchmark/registration.h"
|
|
#include "benchmark/state.h"
|
|
#include "benchmark/utils.h"
|
|
#include "output_test.h"
|
|
|
|
namespace {
|
|
// @todo: <jpmag> this checks the full output at once; the rule for
|
|
// CounterSet1 was failing because it was not matching "^[-]+$".
|
|
// @todo: <jpmag> check that the counters are vertically aligned.
|
|
ADD_CASES(TC_ConsoleOut,
|
|
{
|
|
// keeping these lines long improves readability, so:
|
|
// clang-format off
|
|
{"^[-]+$", MR_Next},
|
|
{"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Bat %s Baz %s Foo %s Frob %s Lob$", MR_Next},
|
|
{"^[-]+$", MR_Next},
|
|
{"^BM_Counters_Tabular/repeats:2/threads:1 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_Counters_Tabular/repeats:2/threads:1 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_Counters_Tabular/repeats:2/threads:1_mean %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_Counters_Tabular/repeats:2/threads:1_median %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_Counters_Tabular/repeats:2/threads:1_stddev %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_Counters_Tabular/repeats:2/threads:1_cv %console_percentage_report [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*%$", MR_Next},
|
|
{"^BM_Counters_Tabular/repeats:2/threads:2 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_Counters_Tabular/repeats:2/threads:2 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_Counters_Tabular/repeats:2/threads:2_mean %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_Counters_Tabular/repeats:2/threads:2_median %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_Counters_Tabular/repeats:2/threads:2_stddev %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_Counters_Tabular/repeats:2/threads:2_cv %console_percentage_report [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*% [ ]*%percentage[ ]*%$", MR_Next},
|
|
{"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
|
|
{"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
|
|
{"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
|
|
{"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
|
|
{"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
|
|
{"^[-]+$", MR_Next},
|
|
{"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Baz %s Foo$", MR_Next},
|
|
{"^[-]+$", MR_Next},
|
|
{"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^[-]+$", MR_Next},
|
|
{"^Benchmark %s Time %s CPU %s Iterations %s Bat %s Baz %s Foo$", MR_Next},
|
|
{"^[-]+$", MR_Next},
|
|
{"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$"},
|
|
// clang-format on
|
|
});
|
|
ADD_CASES(TC_CSVOut, {{"%csv_header,"
|
|
"\"Bar\",\"Bat\",\"Baz\",\"Foo\",\"Frob\",\"Lob\""}});
|
|
|
|
// ========================================================================= //
|
|
// ------------------------- Tabular Counters Output ----------------------- //
|
|
// ========================================================================= //
|
|
|
|
void BM_Counters_Tabular(benchmark::State& state) {
|
|
for (auto _ : state) {
|
|
// This test requires a non-zero CPU time to avoid divide-by-zero
|
|
auto iterations = static_cast<double>(state.iterations()) *
|
|
static_cast<double>(state.iterations());
|
|
benchmark::DoNotOptimize(iterations);
|
|
}
|
|
namespace bm = benchmark;
|
|
state.counters.insert({
|
|
{"Foo", {1, bm::Counter::kAvgThreads}},
|
|
{"Bar", {2, bm::Counter::kAvgThreads}},
|
|
{"Baz", {4, bm::Counter::kAvgThreads}},
|
|
{"Bat", {8, bm::Counter::kAvgThreads}},
|
|
{"Frob", {16, bm::Counter::kAvgThreads}},
|
|
{"Lob", {32, bm::Counter::kAvgThreads}},
|
|
});
|
|
}
|
|
BENCHMARK(BM_Counters_Tabular)->ThreadRange(1, 2)->Repetitions(2);
|
|
ADD_CASES(TC_JSONOut,
|
|
{{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$"},
|
|
{"\"family_index\": 0,$", MR_Next},
|
|
{"\"per_family_instance_index\": 0,$", MR_Next},
|
|
{"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$",
|
|
MR_Next},
|
|
{"\"run_type\": \"iteration\",$", MR_Next},
|
|
{"\"repetitions\": 2,$", MR_Next},
|
|
{"\"repetition_index\": 0,$", MR_Next},
|
|
{"\"threads\": 1,$", MR_Next},
|
|
{"\"iterations\": %int,$", MR_Next},
|
|
{"\"real_time\": %float,$", MR_Next},
|
|
{"\"cpu_time\": %float,$", MR_Next},
|
|
{"\"time_unit\": \"ns\",$", MR_Next},
|
|
{"\"Bar\": %float,$", MR_Next},
|
|
{"\"Bat\": %float,$", MR_Next},
|
|
{"\"Baz\": %float,$", MR_Next},
|
|
{"\"Foo\": %float,$", MR_Next},
|
|
{"\"Frob\": %float,$", MR_Next},
|
|
{"\"Lob\": %float$", MR_Next},
|
|
{"}", MR_Next}});
|
|
ADD_CASES(TC_JSONOut,
|
|
{{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$"},
|
|
{"\"family_index\": 0,$", MR_Next},
|
|
{"\"per_family_instance_index\": 0,$", MR_Next},
|
|
{"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$",
|
|
MR_Next},
|
|
{"\"run_type\": \"iteration\",$", MR_Next},
|
|
{"\"repetitions\": 2,$", MR_Next},
|
|
{"\"repetition_index\": 1,$", MR_Next},
|
|
{"\"threads\": 1,$", MR_Next},
|
|
{"\"iterations\": %int,$", MR_Next},
|
|
{"\"real_time\": %float,$", MR_Next},
|
|
{"\"cpu_time\": %float,$", MR_Next},
|
|
{"\"time_unit\": \"ns\",$", MR_Next},
|
|
{"\"Bar\": %float,$", MR_Next},
|
|
{"\"Bat\": %float,$", MR_Next},
|
|
{"\"Baz\": %float,$", MR_Next},
|
|
{"\"Foo\": %float,$", MR_Next},
|
|
{"\"Frob\": %float,$", MR_Next},
|
|
{"\"Lob\": %float$", MR_Next},
|
|
{"}", MR_Next}});
|
|
ADD_CASES(TC_JSONOut,
|
|
{{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1_mean\",$"},
|
|
{"\"family_index\": 0,$", MR_Next},
|
|
{"\"per_family_instance_index\": 0,$", MR_Next},
|
|
{"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$",
|
|
MR_Next},
|
|
{"\"run_type\": \"aggregate\",$", MR_Next},
|
|
{"\"repetitions\": 2,$", MR_Next},
|
|
{"\"threads\": 1,$", MR_Next},
|
|
{"\"aggregate_name\": \"mean\",$", MR_Next},
|
|
{"\"aggregate_unit\": \"time\",$", MR_Next},
|
|
{"\"iterations\": %int,$", MR_Next},
|
|
{"\"real_time\": %float,$", MR_Next},
|
|
{"\"cpu_time\": %float,$", MR_Next},
|
|
{"\"time_unit\": \"ns\",$", MR_Next},
|
|
{"\"Bar\": %float,$", MR_Next},
|
|
{"\"Bat\": %float,$", MR_Next},
|
|
{"\"Baz\": %float,$", MR_Next},
|
|
{"\"Foo\": %float,$", MR_Next},
|
|
{"\"Frob\": %float,$", MR_Next},
|
|
{"\"Lob\": %float$", MR_Next},
|
|
{"}", MR_Next}});
|
|
ADD_CASES(TC_JSONOut,
|
|
{{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1_median\",$"},
|
|
{"\"family_index\": 0,$", MR_Next},
|
|
{"\"per_family_instance_index\": 0,$", MR_Next},
|
|
{"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$",
|
|
MR_Next},
|
|
{"\"run_type\": \"aggregate\",$", MR_Next},
|
|
{"\"repetitions\": 2,$", MR_Next},
|
|
{"\"threads\": 1,$", MR_Next},
|
|
{"\"aggregate_name\": \"median\",$", MR_Next},
|
|
{"\"aggregate_unit\": \"time\",$", MR_Next},
|
|
{"\"iterations\": %int,$", MR_Next},
|
|
{"\"real_time\": %float,$", MR_Next},
|
|
{"\"cpu_time\": %float,$", MR_Next},
|
|
{"\"time_unit\": \"ns\",$", MR_Next},
|
|
{"\"Bar\": %float,$", MR_Next},
|
|
{"\"Bat\": %float,$", MR_Next},
|
|
{"\"Baz\": %float,$", MR_Next},
|
|
{"\"Foo\": %float,$", MR_Next},
|
|
{"\"Frob\": %float,$", MR_Next},
|
|
{"\"Lob\": %float$", MR_Next},
|
|
{"}", MR_Next}});
|
|
ADD_CASES(TC_JSONOut,
|
|
{{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1_stddev\",$"},
|
|
{"\"family_index\": 0,$", MR_Next},
|
|
{"\"per_family_instance_index\": 0,$", MR_Next},
|
|
{"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$",
|
|
MR_Next},
|
|
{"\"run_type\": \"aggregate\",$", MR_Next},
|
|
{"\"repetitions\": 2,$", MR_Next},
|
|
{"\"threads\": 1,$", MR_Next},
|
|
{"\"aggregate_name\": \"stddev\",$", MR_Next},
|
|
{"\"aggregate_unit\": \"time\",$", MR_Next},
|
|
{"\"iterations\": %int,$", MR_Next},
|
|
{"\"real_time\": %float,$", MR_Next},
|
|
{"\"cpu_time\": %float,$", MR_Next},
|
|
{"\"time_unit\": \"ns\",$", MR_Next},
|
|
{"\"Bar\": %float,$", MR_Next},
|
|
{"\"Bat\": %float,$", MR_Next},
|
|
{"\"Baz\": %float,$", MR_Next},
|
|
{"\"Foo\": %float,$", MR_Next},
|
|
{"\"Frob\": %float,$", MR_Next},
|
|
{"\"Lob\": %float$", MR_Next},
|
|
{"}", MR_Next}});
|
|
ADD_CASES(TC_JSONOut,
|
|
{{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1_cv\",$"},
|
|
{"\"family_index\": 0,$", MR_Next},
|
|
{"\"per_family_instance_index\": 0,$", MR_Next},
|
|
{"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$",
|
|
MR_Next},
|
|
{"\"run_type\": \"aggregate\",$", MR_Next},
|
|
{"\"repetitions\": 2,$", MR_Next},
|
|
{"\"threads\": 1,$", MR_Next},
|
|
{"\"aggregate_name\": \"cv\",$", MR_Next},
|
|
{"\"aggregate_unit\": \"percentage\",$", MR_Next},
|
|
{"\"iterations\": %int,$", MR_Next},
|
|
{"\"real_time\": %float,$", MR_Next},
|
|
{"\"cpu_time\": %float,$", MR_Next},
|
|
{"\"time_unit\": \"ns\",$", MR_Next},
|
|
{"\"Bar\": %float,$", MR_Next},
|
|
{"\"Bat\": %float,$", MR_Next},
|
|
{"\"Baz\": %float,$", MR_Next},
|
|
{"\"Foo\": %float,$", MR_Next},
|
|
{"\"Frob\": %float,$", MR_Next},
|
|
{"\"Lob\": %float$", MR_Next},
|
|
{"}", MR_Next}});
|
|
|
|
ADD_CASES(TC_JSONOut,
|
|
{{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$"},
|
|
{"\"family_index\": 0,$", MR_Next},
|
|
{"\"per_family_instance_index\": 1,$", MR_Next},
|
|
{"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$",
|
|
MR_Next},
|
|
{"\"run_type\": \"iteration\",$", MR_Next},
|
|
{"\"repetitions\": 2,$", MR_Next},
|
|
{"\"repetition_index\": 0,$", MR_Next},
|
|
{"\"threads\": 2,$", MR_Next},
|
|
{"\"iterations\": %int,$", MR_Next},
|
|
{"\"real_time\": %float,$", MR_Next},
|
|
{"\"cpu_time\": %float,$", MR_Next},
|
|
{"\"time_unit\": \"ns\",$", MR_Next},
|
|
{"\"Bar\": %float,$", MR_Next},
|
|
{"\"Bat\": %float,$", MR_Next},
|
|
{"\"Baz\": %float,$", MR_Next},
|
|
{"\"Foo\": %float,$", MR_Next},
|
|
{"\"Frob\": %float,$", MR_Next},
|
|
{"\"Lob\": %float$", MR_Next},
|
|
{"}", MR_Next}});
|
|
ADD_CASES(TC_JSONOut,
|
|
{{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$"},
|
|
{"\"family_index\": 0,$", MR_Next},
|
|
{"\"per_family_instance_index\": 1,$", MR_Next},
|
|
{"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$",
|
|
MR_Next},
|
|
{"\"run_type\": \"iteration\",$", MR_Next},
|
|
{"\"repetitions\": 2,$", MR_Next},
|
|
{"\"repetition_index\": 1,$", MR_Next},
|
|
{"\"threads\": 2,$", MR_Next},
|
|
{"\"iterations\": %int,$", MR_Next},
|
|
{"\"real_time\": %float,$", MR_Next},
|
|
{"\"cpu_time\": %float,$", MR_Next},
|
|
{"\"time_unit\": \"ns\",$", MR_Next},
|
|
{"\"Bar\": %float,$", MR_Next},
|
|
{"\"Bat\": %float,$", MR_Next},
|
|
{"\"Baz\": %float,$", MR_Next},
|
|
{"\"Foo\": %float,$", MR_Next},
|
|
{"\"Frob\": %float,$", MR_Next},
|
|
{"\"Lob\": %float$", MR_Next},
|
|
{"}", MR_Next}});
|
|
ADD_CASES(TC_JSONOut,
|
|
{{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2_median\",$"},
|
|
{"\"family_index\": 0,$", MR_Next},
|
|
{"\"per_family_instance_index\": 1,$", MR_Next},
|
|
{"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$",
|
|
MR_Next},
|
|
{"\"run_type\": \"aggregate\",$", MR_Next},
|
|
{"\"repetitions\": 2,$", MR_Next},
|
|
{"\"threads\": 2,$", MR_Next},
|
|
{"\"aggregate_name\": \"median\",$", MR_Next},
|
|
{"\"aggregate_unit\": \"time\",$", MR_Next},
|
|
{"\"iterations\": %int,$", MR_Next},
|
|
{"\"real_time\": %float,$", MR_Next},
|
|
{"\"cpu_time\": %float,$", MR_Next},
|
|
{"\"time_unit\": \"ns\",$", MR_Next},
|
|
{"\"Bar\": %float,$", MR_Next},
|
|
{"\"Bat\": %float,$", MR_Next},
|
|
{"\"Baz\": %float,$", MR_Next},
|
|
{"\"Foo\": %float,$", MR_Next},
|
|
{"\"Frob\": %float,$", MR_Next},
|
|
{"\"Lob\": %float$", MR_Next},
|
|
{"}", MR_Next}});
|
|
ADD_CASES(TC_JSONOut,
|
|
{{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2_stddev\",$"},
|
|
{"\"family_index\": 0,$", MR_Next},
|
|
{"\"per_family_instance_index\": 1,$", MR_Next},
|
|
{"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$",
|
|
MR_Next},
|
|
{"\"run_type\": \"aggregate\",$", MR_Next},
|
|
{"\"repetitions\": 2,$", MR_Next},
|
|
{"\"threads\": 2,$", MR_Next},
|
|
{"\"aggregate_name\": \"stddev\",$", MR_Next},
|
|
{"\"aggregate_unit\": \"time\",$", MR_Next},
|
|
{"\"iterations\": %int,$", MR_Next},
|
|
{"\"real_time\": %float,$", MR_Next},
|
|
{"\"cpu_time\": %float,$", MR_Next},
|
|
{"\"time_unit\": \"ns\",$", MR_Next},
|
|
{"\"Bar\": %float,$", MR_Next},
|
|
{"\"Bat\": %float,$", MR_Next},
|
|
{"\"Baz\": %float,$", MR_Next},
|
|
{"\"Foo\": %float,$", MR_Next},
|
|
{"\"Frob\": %float,$", MR_Next},
|
|
{"\"Lob\": %float$", MR_Next},
|
|
{"}", MR_Next}});
|
|
ADD_CASES(TC_JSONOut,
|
|
{{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2_cv\",$"},
|
|
{"\"family_index\": 0,$", MR_Next},
|
|
{"\"per_family_instance_index\": 1,$", MR_Next},
|
|
{"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$",
|
|
MR_Next},
|
|
{"\"run_type\": \"aggregate\",$", MR_Next},
|
|
{"\"repetitions\": 2,$", MR_Next},
|
|
{"\"threads\": 2,$", MR_Next},
|
|
{"\"aggregate_name\": \"cv\",$", MR_Next},
|
|
{"\"aggregate_unit\": \"percentage\",$", MR_Next},
|
|
{"\"iterations\": %int,$", MR_Next},
|
|
{"\"real_time\": %float,$", MR_Next},
|
|
{"\"cpu_time\": %float,$", MR_Next},
|
|
{"\"time_unit\": \"ns\",$", MR_Next},
|
|
{"\"Bar\": %float,$", MR_Next},
|
|
{"\"Bat\": %float,$", MR_Next},
|
|
{"\"Baz\": %float,$", MR_Next},
|
|
{"\"Foo\": %float,$", MR_Next},
|
|
{"\"Frob\": %float,$", MR_Next},
|
|
{"\"Lob\": %float$", MR_Next},
|
|
{"}", MR_Next}});
|
|
ADD_CASES(TC_CSVOut,
|
|
{{"^\"BM_Counters_Tabular/repeats:2/threads:1\",%csv_report,"
|
|
"%float,%float,%float,%float,%float,%float$"}});
|
|
ADD_CASES(TC_CSVOut,
|
|
{{"^\"BM_Counters_Tabular/repeats:2/threads:1\",%csv_report,"
|
|
"%float,%float,%float,%float,%float,%float$"}});
|
|
ADD_CASES(TC_CSVOut,
|
|
{{"^\"BM_Counters_Tabular/repeats:2/threads:1_mean\",%csv_report,"
|
|
"%float,%float,%float,%float,%float,%float$"}});
|
|
ADD_CASES(TC_CSVOut,
|
|
{{"^\"BM_Counters_Tabular/repeats:2/threads:1_median\",%csv_report,"
|
|
"%float,%float,%float,%float,%float,%float$"}});
|
|
ADD_CASES(TC_CSVOut,
|
|
{{"^\"BM_Counters_Tabular/repeats:2/threads:1_stddev\",%csv_report,"
|
|
"%float,%float,%float,%float,%float,%float$"}});
|
|
ADD_CASES(TC_CSVOut,
|
|
{{"^\"BM_Counters_Tabular/repeats:2/threads:1_cv\",%csv_cv_report,"
|
|
"%float,%float,%float,%float,%float,%float$"}});
|
|
ADD_CASES(TC_CSVOut,
|
|
{{"^\"BM_Counters_Tabular/repeats:2/threads:2\",%csv_report,"
|
|
"%float,%float,%float,%float,%float,%float$"}});
|
|
ADD_CASES(TC_CSVOut,
|
|
{{"^\"BM_Counters_Tabular/repeats:2/threads:2\",%csv_report,"
|
|
"%float,%float,%float,%float,%float,%float$"}});
|
|
ADD_CASES(TC_CSVOut,
|
|
{{"^\"BM_Counters_Tabular/repeats:2/threads:2_mean\",%csv_report,"
|
|
"%float,%float,%float,%float,%float,%float$"}});
|
|
ADD_CASES(TC_CSVOut,
|
|
{{"^\"BM_Counters_Tabular/repeats:2/threads:2_median\",%csv_report,"
|
|
"%float,%float,%float,%float,%float,%float$"}});
|
|
ADD_CASES(TC_CSVOut,
|
|
{{"^\"BM_Counters_Tabular/repeats:2/threads:2_stddev\",%csv_report,"
|
|
"%float,%float,%float,%float,%float,%float$"}});
|
|
ADD_CASES(TC_CSVOut,
|
|
{{"^\"BM_Counters_Tabular/repeats:2/threads:2_cv\",%csv_cv_report,"
|
|
"%float,%float,%float,%float,%float,%float$"}});
|
|
// VS2013 does not allow this function to be passed as a lambda argument
|
|
// to CHECK_BENCHMARK_RESULTS()
|
|
void CheckTabular(Results const& e) {
|
|
CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 1);
|
|
CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 2);
|
|
CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 4);
|
|
CHECK_COUNTER_VALUE(e, int, "Bat", EQ, 8);
|
|
CHECK_COUNTER_VALUE(e, int, "Frob", EQ, 16);
|
|
CHECK_COUNTER_VALUE(e, int, "Lob", EQ, 32);
|
|
}
|
|
CHECK_BENCHMARK_RESULTS("BM_Counters_Tabular/repeats:2/threads:1$",
|
|
&CheckTabular);
|
|
CHECK_BENCHMARK_RESULTS("BM_Counters_Tabular/repeats:2/threads:2$",
|
|
&CheckTabular);
|
|
|
|
// ========================================================================= //
|
|
// -------------------- Tabular+Rate Counters Output ----------------------- //
|
|
// ========================================================================= //
|
|
|
|
void BM_CounterRates_Tabular(benchmark::State& state) {
|
|
for (auto _ : state) {
|
|
// This test requires a non-zero CPU time to avoid divide-by-zero
|
|
auto iterations = static_cast<double>(state.iterations()) *
|
|
static_cast<double>(state.iterations());
|
|
benchmark::DoNotOptimize(iterations);
|
|
}
|
|
namespace bm = benchmark;
|
|
state.counters.insert({
|
|
{"Foo", {1, bm::Counter::kAvgThreadsRate}},
|
|
{"Bar", {2, bm::Counter::kAvgThreadsRate}},
|
|
{"Baz", {4, bm::Counter::kAvgThreadsRate}},
|
|
{"Bat", {8, bm::Counter::kAvgThreadsRate}},
|
|
{"Frob", {16, bm::Counter::kAvgThreadsRate}},
|
|
{"Lob", {32, bm::Counter::kAvgThreadsRate}},
|
|
});
|
|
}
|
|
BENCHMARK(BM_CounterRates_Tabular)->ThreadRange(1, 16);
|
|
ADD_CASES(TC_JSONOut,
|
|
{{"\"name\": \"BM_CounterRates_Tabular/threads:%int\",$"},
|
|
{"\"family_index\": 1,$", MR_Next},
|
|
{"\"per_family_instance_index\": 0,$", MR_Next},
|
|
{"\"run_name\": \"BM_CounterRates_Tabular/threads:%int\",$",
|
|
MR_Next},
|
|
{"\"run_type\": \"iteration\",$", MR_Next},
|
|
{"\"repetitions\": 1,$", MR_Next},
|
|
{"\"repetition_index\": 0,$", MR_Next},
|
|
{"\"threads\": 1,$", MR_Next},
|
|
{"\"iterations\": %int,$", MR_Next},
|
|
{"\"real_time\": %float,$", MR_Next},
|
|
{"\"cpu_time\": %float,$", MR_Next},
|
|
{"\"time_unit\": \"ns\",$", MR_Next},
|
|
{"\"Bar\": %float,$", MR_Next},
|
|
{"\"Bat\": %float,$", MR_Next},
|
|
{"\"Baz\": %float,$", MR_Next},
|
|
{"\"Foo\": %float,$", MR_Next},
|
|
{"\"Frob\": %float,$", MR_Next},
|
|
{"\"Lob\": %float$", MR_Next},
|
|
{"}", MR_Next}});
|
|
ADD_CASES(TC_CSVOut, {{"^\"BM_CounterRates_Tabular/threads:%int\",%csv_report,"
|
|
"%float,%float,%float,%float,%float,%float$"}});
|
|
// VS2013 does not allow this function to be passed as a lambda argument
|
|
// to CHECK_BENCHMARK_RESULTS()
|
|
void CheckTabularRate(Results const& e) {
|
|
double t = e.DurationCPUTime() / e.NumThreads();
|
|
CHECK_FLOAT_COUNTER_VALUE(e, "Foo", EQ, 1. / t, 0.001);
|
|
CHECK_FLOAT_COUNTER_VALUE(e, "Bar", EQ, 2. / t, 0.001);
|
|
CHECK_FLOAT_COUNTER_VALUE(e, "Baz", EQ, 4. / t, 0.001);
|
|
CHECK_FLOAT_COUNTER_VALUE(e, "Bat", EQ, 8. / t, 0.001);
|
|
CHECK_FLOAT_COUNTER_VALUE(e, "Frob", EQ, 16. / t, 0.001);
|
|
CHECK_FLOAT_COUNTER_VALUE(e, "Lob", EQ, 32. / t, 0.001);
|
|
}
|
|
CHECK_BENCHMARK_RESULTS("BM_CounterRates_Tabular/threads:%int",
|
|
&CheckTabularRate);
|
|
|
|
// ========================================================================= //
|
|
// ------------------------- Tabular Counters Output ----------------------- //
|
|
// ========================================================================= //
|
|
|
|
// set only some of the counters
|
|
void BM_CounterSet0_Tabular(benchmark::State& state) {
|
|
for (auto _ : state) {
|
|
}
|
|
namespace bm = benchmark;
|
|
state.counters.insert({
|
|
{"Foo", {10, bm::Counter::kAvgThreads}},
|
|
{"Bar", {20, bm::Counter::kAvgThreads}},
|
|
{"Baz", {40, bm::Counter::kAvgThreads}},
|
|
});
|
|
}
|
|
BENCHMARK(BM_CounterSet0_Tabular)->ThreadRange(1, 16);
|
|
ADD_CASES(TC_JSONOut,
|
|
{{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$"},
|
|
{"\"family_index\": 2,$", MR_Next},
|
|
{"\"per_family_instance_index\": 0,$", MR_Next},
|
|
{"\"run_name\": \"BM_CounterSet0_Tabular/threads:%int\",$", MR_Next},
|
|
{"\"run_type\": \"iteration\",$", MR_Next},
|
|
{"\"repetitions\": 1,$", MR_Next},
|
|
{"\"repetition_index\": 0,$", MR_Next},
|
|
{"\"threads\": 1,$", MR_Next},
|
|
{"\"iterations\": %int,$", MR_Next},
|
|
{"\"real_time\": %float,$", MR_Next},
|
|
{"\"cpu_time\": %float,$", MR_Next},
|
|
{"\"time_unit\": \"ns\",$", MR_Next},
|
|
{"\"Bar\": %float,$", MR_Next},
|
|
{"\"Baz\": %float,$", MR_Next},
|
|
{"\"Foo\": %float$", MR_Next},
|
|
{"}", MR_Next}});
|
|
ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet0_Tabular/threads:%int\",%csv_report,"
|
|
"%float,,%float,%float,,"}});
|
|
// VS2013 does not allow this function to be passed as a lambda argument
|
|
// to CHECK_BENCHMARK_RESULTS()
|
|
void CheckSet0(Results const& e) {
|
|
CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 10);
|
|
CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 20);
|
|
CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 40);
|
|
}
|
|
CHECK_BENCHMARK_RESULTS("BM_CounterSet0_Tabular", &CheckSet0);
|
|
|
|
// again.
|
|
void BM_CounterSet1_Tabular(benchmark::State& state) {
|
|
for (auto _ : state) {
|
|
}
|
|
namespace bm = benchmark;
|
|
state.counters.insert({
|
|
{"Foo", {15, bm::Counter::kAvgThreads}},
|
|
{"Bar", {25, bm::Counter::kAvgThreads}},
|
|
{"Baz", {45, bm::Counter::kAvgThreads}},
|
|
});
|
|
}
|
|
BENCHMARK(BM_CounterSet1_Tabular)->ThreadRange(1, 16);
|
|
ADD_CASES(TC_JSONOut,
|
|
{{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$"},
|
|
{"\"family_index\": 3,$", MR_Next},
|
|
{"\"per_family_instance_index\": 0,$", MR_Next},
|
|
{"\"run_name\": \"BM_CounterSet1_Tabular/threads:%int\",$", MR_Next},
|
|
{"\"run_type\": \"iteration\",$", MR_Next},
|
|
{"\"repetitions\": 1,$", MR_Next},
|
|
{"\"repetition_index\": 0,$", MR_Next},
|
|
{"\"threads\": 1,$", MR_Next},
|
|
{"\"iterations\": %int,$", MR_Next},
|
|
{"\"real_time\": %float,$", MR_Next},
|
|
{"\"cpu_time\": %float,$", MR_Next},
|
|
{"\"time_unit\": \"ns\",$", MR_Next},
|
|
{"\"Bar\": %float,$", MR_Next},
|
|
{"\"Baz\": %float,$", MR_Next},
|
|
{"\"Foo\": %float$", MR_Next},
|
|
{"}", MR_Next}});
|
|
ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet1_Tabular/threads:%int\",%csv_report,"
|
|
"%float,,%float,%float,,"}});
|
|
// VS2013 does not allow this function to be passed as a lambda argument
|
|
// to CHECK_BENCHMARK_RESULTS()
|
|
void CheckSet1(Results const& e) {
|
|
CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 15);
|
|
CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 25);
|
|
CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 45);
|
|
}
|
|
CHECK_BENCHMARK_RESULTS("BM_CounterSet1_Tabular/threads:%int", &CheckSet1);
|
|
|
|
// ========================================================================= //
|
|
// ------------------------- Tabular Counters Output ----------------------- //
|
|
// ========================================================================= //
|
|
|
|
// set only some of the counters, different set now.
|
|
void BM_CounterSet2_Tabular(benchmark::State& state) {
|
|
for (auto _ : state) {
|
|
}
|
|
namespace bm = benchmark;
|
|
state.counters.insert({
|
|
{"Foo", {10, bm::Counter::kAvgThreads}},
|
|
{"Bat", {30, bm::Counter::kAvgThreads}},
|
|
{"Baz", {40, bm::Counter::kAvgThreads}},
|
|
});
|
|
}
|
|
BENCHMARK(BM_CounterSet2_Tabular)->ThreadRange(1, 16);
|
|
ADD_CASES(TC_JSONOut,
|
|
{{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$"},
|
|
{"\"family_index\": 4,$", MR_Next},
|
|
{"\"per_family_instance_index\": 0,$", MR_Next},
|
|
{"\"run_name\": \"BM_CounterSet2_Tabular/threads:%int\",$", MR_Next},
|
|
{"\"run_type\": \"iteration\",$", MR_Next},
|
|
{"\"repetitions\": 1,$", MR_Next},
|
|
{"\"repetition_index\": 0,$", MR_Next},
|
|
{"\"threads\": 1,$", MR_Next},
|
|
{"\"iterations\": %int,$", MR_Next},
|
|
{"\"real_time\": %float,$", MR_Next},
|
|
{"\"cpu_time\": %float,$", MR_Next},
|
|
{"\"time_unit\": \"ns\",$", MR_Next},
|
|
{"\"Bat\": %float,$", MR_Next},
|
|
{"\"Baz\": %float,$", MR_Next},
|
|
{"\"Foo\": %float$", MR_Next},
|
|
{"}", MR_Next}});
|
|
ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet2_Tabular/threads:%int\",%csv_report,"
|
|
",%float,%float,%float,,"}});
|
|
// VS2013 does not allow this function to be passed as a lambda argument
|
|
// to CHECK_BENCHMARK_RESULTS()
|
|
void CheckSet2(Results const& e) {
|
|
CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 10);
|
|
CHECK_COUNTER_VALUE(e, int, "Bat", EQ, 30);
|
|
CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 40);
|
|
}
|
|
CHECK_BENCHMARK_RESULTS("BM_CounterSet2_Tabular", &CheckSet2);
|
|
} // end namespace
|
|
|
|
// ========================================================================= //
|
|
// --------------------------- TEST CASES END ------------------------------ //
|
|
// ========================================================================= //
|
|
|
|
int main(int argc, char* argv[]) {
|
|
benchmark::MaybeReenterWithoutASLR(argc, argv);
|
|
RunOutputTests(argc, argv);
|
|
}
|