Skip to content

Commit 58a6248

Browse files
committed
partial bitcoin#26158: add "priority level" to the benchmark framework
excludes: - 3da7cd2
1 parent 160b3fc commit 58a6248

File tree

4 files changed

+90
-27
lines changed

4 files changed

+90
-27
lines changed

src/Makefile.test.include

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -399,8 +399,8 @@ endif
399399
if TARGET_WINDOWS
400400
else
401401
if ENABLE_BENCH
402-
@echo "Running bench/bench_dash (one iteration sanity check)..."
403-
$(BENCH_BINARY) --sanity-check > /dev/null
402+
@echo "Running bench/bench_dash (one iteration sanity check, only high priority)..."
403+
$(BENCH_BINARY) -sanity-check -priority-level=high > /dev/null
404404
endif
405405
endif
406406
$(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C secp256k1 check

src/bench/bench.cpp

Lines changed: 44 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
#include <fs.h>
88
#include <test/util/setup_common.h>
9+
#include <util/string.h>
910

1011
#include <algorithm>
1112
#include <chrono>
@@ -42,18 +43,42 @@ void GenerateTemplateResults(const std::vector<ankerl::nanobench::Result>& bench
4243

4344
} // namespace
4445

45-
benchmark::BenchRunner::BenchmarkMap& benchmark::BenchRunner::benchmarks()
46+
namespace benchmark {
47+
48+
// map a label to one or multiple priority levels
49+
std::map<std::string, uint8_t> map_label_priority = {
50+
{"high", PriorityLevel::HIGH},
51+
{"low", PriorityLevel::LOW},
52+
{"all", 0xff}
53+
};
54+
55+
std::string ListPriorities()
4656
{
47-
static std::map<std::string, BenchFunction> benchmarks_map;
57+
using item_t = std::pair<std::string, uint8_t>;
58+
auto sort_by_priority = [](item_t a, item_t b){ return a.second < b.second; };
59+
std::set<item_t, decltype(sort_by_priority)> sorted_priorities(map_label_priority.begin(), map_label_priority.end(), sort_by_priority);
60+
return Join(sorted_priorities, ',', [](const auto& entry){ return entry.first; });
61+
}
62+
63+
uint8_t StringToPriority(const std::string& str)
64+
{
65+
auto it = map_label_priority.find(str);
66+
if (it == map_label_priority.end()) throw std::runtime_error(strprintf("Unknown priority level %s", str));
67+
return it->second;
68+
}
69+
70+
BenchRunner::BenchmarkMap& BenchRunner::benchmarks()
71+
{
72+
static BenchmarkMap benchmarks_map;
4873
return benchmarks_map;
4974
}
5075

51-
benchmark::BenchRunner::BenchRunner(std::string name, benchmark::BenchFunction func)
76+
BenchRunner::BenchRunner(std::string name, BenchFunction func, PriorityLevel level)
5277
{
53-
benchmarks().insert(std::make_pair(name, func));
78+
benchmarks().insert(std::make_pair(name, std::make_pair(func, level)));
5479
}
5580

56-
void benchmark::BenchRunner::RunAll(const Args& args)
81+
void BenchRunner::RunAll(const Args& args)
5782
{
5883
std::regex reFilter(args.regex_filter);
5984
std::smatch baseMatch;
@@ -63,33 +88,39 @@ void benchmark::BenchRunner::RunAll(const Args& args)
6388
}
6489

6590
std::vector<ankerl::nanobench::Result> benchmarkResults;
66-
for (const auto& p : benchmarks()) {
67-
if (!std::regex_match(p.first, baseMatch, reFilter)) {
91+
for (const auto& [name, bench_func] : benchmarks()) {
92+
const auto& [func, priority_level] = bench_func;
93+
94+
if (!(priority_level & args.priority)) {
95+
continue;
96+
}
97+
98+
if (!std::regex_match(name, baseMatch, reFilter)) {
6899
continue;
69100
}
70101

71102
if (args.is_list_only) {
72-
std::cout << p.first << std::endl;
103+
std::cout << name << std::endl;
73104
continue;
74105
}
75106

76107
Bench bench;
77108
if (args.sanity_check) {
78109
bench.epochs(1).epochIterations(1);
79110
}
80-
bench.name(p.first);
111+
bench.name(name);
81112
if (args.min_time > 0ms) {
82113
// convert to nanos before dividing to reduce rounding errors
83114
std::chrono::nanoseconds min_time_ns = args.min_time;
84115
bench.minEpochTime(min_time_ns / bench.epochs());
85116
}
86117

87118
if (args.asymptote.empty()) {
88-
p.second(bench);
119+
func(bench);
89120
} else {
90121
for (auto n : args.asymptote) {
91122
bench.complexityN(n);
92-
p.second(bench);
123+
func(bench);
93124
}
94125
std::cout << bench.complexityBigO() << std::endl;
95126
}
@@ -104,3 +135,5 @@ void benchmark::BenchRunner::RunAll(const Args& args)
104135
"{{/result}}");
105136
GenerateTemplateResults(benchmarkResults, args.output_json, ankerl::nanobench::templates::json());
106137
}
138+
139+
} // namespace benchmark

src/bench/bench.h

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,16 @@ using ankerl::nanobench::Bench;
4141

4242
typedef std::function<void(Bench&)> BenchFunction;
4343

44+
enum PriorityLevel : uint8_t
45+
{
46+
LOW = 1 << 0,
47+
HIGH = 1 << 2,
48+
};
49+
50+
// List priority labels, comma-separated and sorted by increasing priority
51+
std::string ListPriorities();
52+
uint8_t StringToPriority(const std::string& str);
53+
4454
struct Args {
4555
bool is_list_only;
4656
bool sanity_check;
@@ -49,22 +59,24 @@ struct Args {
4959
fs::path output_csv;
5060
fs::path output_json;
5161
std::string regex_filter;
62+
uint8_t priority;
5263
};
5364

5465
class BenchRunner
5566
{
56-
typedef std::map<std::string, BenchFunction> BenchmarkMap;
67+
// maps from "name" -> (function, priority_level)
68+
typedef std::map<std::string, std::pair<BenchFunction, PriorityLevel>> BenchmarkMap;
5769
static BenchmarkMap& benchmarks();
5870

5971
public:
60-
BenchRunner(std::string name, BenchFunction func);
72+
BenchRunner(std::string name, BenchFunction func, PriorityLevel level);
6173

6274
static void RunAll(const Args& args);
6375
};
6476
} // namespace benchmark
6577

66-
// BENCHMARK(foo) expands to: benchmark::BenchRunner bench_11foo("foo", foo);
78+
// BENCHMARK(foo) expands to: benchmark::BenchRunner bench_11foo("foo", foo, priority_level);
6779
#define BENCHMARK(n) \
68-
benchmark::BenchRunner PASTE2(bench_, PASTE2(__LINE__, n))(STRINGIZE(n), n);
80+
benchmark::BenchRunner PASTE2(bench_, PASTE2(__LINE__, n))(STRINGIZE(n), n, benchmark::PriorityLevel::HIGH);
6981

7082
#endif // BITCOIN_BENCH_BENCH_H

src/bench/bench_bitcoin.cpp

Lines changed: 28 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818

1919
static const char* DEFAULT_BENCH_FILTER = ".*";
2020
static constexpr int64_t DEFAULT_MIN_TIME_MS{10};
21+
/** Priority level default value, run "all" priority levels */
22+
static const std::string DEFAULT_PRIORITY{"all"};
2123

2224
static void SetupBenchArgs(ArgsManager& argsman)
2325
{
@@ -30,6 +32,8 @@ static void SetupBenchArgs(ArgsManager& argsman)
3032
argsman.AddArg("-output-csv=<output.csv>", "Generate CSV file with the most important benchmark results", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
3133
argsman.AddArg("-output-json=<output.json>", "Generate JSON file with all benchmark results", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
3234
argsman.AddArg("-sanity-check", "Run benchmarks for only one iteration", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
35+
argsman.AddArg("-priority-level=<l1,l2,l3>", strprintf("Run benchmarks of one or multiple priority level(s) (%s), default: '%s'",
36+
benchmark::ListPriorities(), DEFAULT_PRIORITY), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
3337
}
3438

3539
// parses a comma separated list like "10,20,30,50"
@@ -45,6 +49,14 @@ static std::vector<double> parseAsymptote(const std::string& str) {
4549
return numbers;
4650
}
4751

52+
static uint8_t parsePriorityLevel(const std::string& str) {
53+
uint8_t levels{0};
54+
for (const auto& level: SplitString(str, ',')) {
55+
levels |= benchmark::StringToPriority(level);
56+
}
57+
return levels;
58+
}
59+
4860
int main(int argc, char** argv)
4961
{
5062
ArgsManager argsman;
@@ -106,16 +118,22 @@ int main(int argc, char** argv)
106118
return EXIT_SUCCESS;
107119
}
108120

109-
benchmark::Args args;
110-
args.asymptote = parseAsymptote(argsman.GetArg("-asymptote", ""));
111-
args.is_list_only = argsman.GetBoolArg("-list", false);
112-
args.min_time = std::chrono::milliseconds(argsman.GetIntArg("-min-time", DEFAULT_MIN_TIME_MS));
113-
args.output_csv = argsman.GetPathArg("-output-csv");
114-
args.output_json = argsman.GetPathArg("-output-json");
115-
args.regex_filter = argsman.GetArg("-filter", DEFAULT_BENCH_FILTER);
116-
args.sanity_check = argsman.GetBoolArg("-sanity-check", false);
121+
try {
122+
benchmark::Args args;
123+
args.asymptote = parseAsymptote(argsman.GetArg("-asymptote", ""));
124+
args.is_list_only = argsman.GetBoolArg("-list", false);
125+
args.min_time = std::chrono::milliseconds(argsman.GetIntArg("-min-time", DEFAULT_MIN_TIME_MS));
126+
args.output_csv = argsman.GetPathArg("-output-csv");
127+
args.output_json = argsman.GetPathArg("-output-json");
128+
args.regex_filter = argsman.GetArg("-filter", DEFAULT_BENCH_FILTER);
129+
args.sanity_check = argsman.GetBoolArg("-sanity-check", false);
130+
args.priority = parsePriorityLevel(argsman.GetArg("-priority-level", DEFAULT_PRIORITY));
117131

118-
benchmark::BenchRunner::RunAll(args);
132+
benchmark::BenchRunner::RunAll(args);
119133

120-
return EXIT_SUCCESS;
134+
return EXIT_SUCCESS;
135+
} catch (const std::exception& e) {
136+
tfm::format(std::cerr, "Error: %s\n", e.what());
137+
return EXIT_FAILURE;
138+
}
121139
}

0 commit comments

Comments
 (0)