Skip to content

test: Fix state test format benchmarking #1043

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 2 commits into
base: master
Choose a base branch
from
+131 −136
Draft

test: Fix state test format benchmarking #1043

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b7ebdcf
Apply minimal changes to run bls bench state tests
rodiazet Oct 9, 2024
ffb29ae
WIP
rodiazet May 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
224 changes: 96 additions & 128 deletions test/bench/bench.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@
// Copyright 2019 The evmone Authors.
// SPDX-License-Identifier: Apache-2.0

#include "../statetest/statetest.hpp"
#include "helpers.hpp"
#include "synthetic_benchmarks.hpp"
#include <benchmark/benchmark.h>
#include <evmc/evmc.hpp>
#include <evmc/loader.h>
#include <evmone/evmone.h>
#include <gtest/gtest.h>
#include <test/statetest/statetest.hpp>
#include <filesystem>
#include <fstream>
#include <iostream>
Expand All @@ -26,70 +26,67 @@
{
struct BenchmarkCase
{
struct Input
{
std::string name;
bytes input;
bytes expected_output;

Input(std::string _name, bytes _input, bytes _expected_output = {}) noexcept
: name{std::move(_name)},
input{std::move(_input)},
expected_output{std::move(_expected_output)}
{}
};

StateTransitionTest state_test;
std::string name;
bytes code;
std::vector<Input> inputs;
};

/// Loads the benchmark case's inputs from the inputs file at the given path.
std::vector<BenchmarkCase::Input> load_inputs(const StateTransitionTest& state_test)
{
std::vector<BenchmarkCase::Input> inputs;
inputs.reserve(state_test.multi_tx.inputs.size());
for (size_t i = 0; i < state_test.multi_tx.inputs.size(); ++i)
inputs.emplace_back(state_test.input_labels.at(i), state_test.multi_tx.inputs[i]);
return inputs;
}

/// Loads a benchmark case from a file at `path` and all its inputs from the matching inputs file.
BenchmarkCase load_benchmark(const fs::path& path, const std::string& name_prefix)
std::vector<BenchmarkCase> load_benchmark(const fs::path& path, const std::string& name_prefix)

Check warning on line 34 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L34 

Added line #L34 was not covered by tests
{
std::ifstream f{path};
auto state_test = std::move(load_state_tests(f).at(0));
std::vector<BenchmarkCase> result;

Check warning on line 37 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L37 

Added line #L37 was not covered by tests

const auto name = name_prefix + path.stem().string();
const auto code = state_test.pre_state.get(state_test.multi_tx.to.value()).code;
const auto inputs = load_inputs(state_test);
auto state_tests = load_state_tests(f);
result.reserve(state_tests.size());
for (const auto& state_test : state_tests)

Check warning on line 41 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L39-L41 

Added lines #L39 - L41 were not covered by tests
{
result.emplace_back(
BenchmarkCase{state_test, name_prefix + path.stem().string() + "/" + state_test.name});

Check warning on line 44 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L43-L44 

Added lines #L43 - L44 were not covered by tests
}

return BenchmarkCase{name, code, inputs};
return result;

Check warning on line 47 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L47 

Added line #L47 was not covered by tests
}

/// Loads all benchmark cases from the given directory and all its subdirectories.
std::vector<BenchmarkCase> load_benchmarks_from_dir( // NOLINT(misc-no-recursion)
const fs::path& path, const std::string& name_prefix = {})
{
std::vector<fs::path> subdirs;
std::vector<fs::path> code_files;
std::vector<fs::path> files;

Check warning on line 55 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L55 

Added line #L55 was not covered by tests

for (auto& e : fs::directory_iterator{path})
if (fs::is_directory(path))

Check warning on line 57 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L57 

Added line #L57 was not covered by tests
{
if (e.is_directory())
subdirs.emplace_back(e);
else if (e.path().extension() == ".json")
code_files.emplace_back(e);
for (auto& e : fs::directory_iterator{path})

Check warning on line 59 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L59 

Added line #L59 was not covered by tests
{
if (e.is_directory())
subdirs.emplace_back(e);
else if (e.path().extension() == ".json")
files.emplace_back(e);
}

Check warning on line 65 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L61-L65 

Added lines #L61 - L65 were not covered by tests
}
else
{
if (path.extension() == ".json")

Check warning on line 69 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L69 

Added line #L69 was not covered by tests
{
if (fs::exists(path))
files.emplace_back(path);

Check warning on line 72 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L71-L72 

Added lines #L71 - L72 were not covered by tests
else
throw std::invalid_argument{"Path '" + path.string() + "' does not exist."};

Check warning on line 74 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L74 

Added line #L74 was not covered by tests
}
}

std::ranges::sort(subdirs);
std::ranges::sort(code_files);
std::ranges::sort(files);

Check warning on line 79 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L79 

Added line #L79 was not covered by tests

std::vector<BenchmarkCase> benchmark_cases;

benchmark_cases.reserve(std::size(code_files));
for (const auto& f : code_files)
benchmark_cases.emplace_back(load_benchmark(f, name_prefix));
for (const auto& f : files)

Check warning on line 83 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L83 

Added line #L83 was not covered by tests
{
auto file_benchmarks = load_benchmark(f, name_prefix);
benchmark_cases.insert(benchmark_cases.end(),

Check warning on line 86 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L85-L86 

Added lines #L85 - L86 were not covered by tests
std::make_move_iterator(file_benchmarks.begin()),
std::make_move_iterator(file_benchmarks.end()));
}

Check warning on line 89 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L89 

Added line #L89 was not covered by tests

for (const auto& d : subdirs)
{
Expand All @@ -105,66 +102,54 @@
{
evmc::VM* advanced_vm = nullptr;
evmc::VM* baseline_vm = nullptr;
evmc::VM* basel_cg_vm = nullptr;
if (const auto it = registered_vms.find("advanced"); it != registered_vms.end())
advanced_vm = &it->second;
if (const auto it = registered_vms.find("baseline"); it != registered_vms.end())
baseline_vm = &it->second;
if (const auto it = registered_vms.find("bnocgoto"); it != registered_vms.end())
basel_cg_vm = &it->second;

evmc::VM check_test_vm = evmc::VM{evmc_create_evmone()};

Check warning on line 110 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L110 

Added line #L110 was not covered by tests

for (const auto& b : benchmark_cases)
{
if (advanced_vm != nullptr)
{
RegisterBenchmark("advanced/analyse/" + b.name, [&b](State& state) {
bench_analyse<advanced::AdvancedCodeAnalysis, advanced_analyse>(
state, default_revision, b.code);
})->Unit(kMicrosecond);
}
run_state_test(b.state_test, check_test_vm, false);

Check warning on line 114 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L114 

Added line #L114 was not covered by tests

if (baseline_vm != nullptr)
{
RegisterBenchmark("baseline/analyse/" + b.name, [&b](State& state) {
bench_analyse<baseline::CodeAnalysis, baseline_analyse>(
state, default_revision, b.code);
})->Unit(kMicrosecond);
}
if (::testing::Test::HasFailure())
throw std::invalid_argument{"State test you want to bench failed."};

Check warning on line 117 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L116-L117 

Added lines #L116 - L117 were not covered by tests

for (const auto& input : b.inputs)
const auto code = b.state_test.pre_state.get_account_code(b.state_test.multi_tx.to.value());
for (const auto& [rev, cases, block_info] : b.state_test.cases)

Check warning on line 120 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L119-L120 

Added lines #L119 - L120 were not covered by tests
{
const auto case_name = b.name + (!input.name.empty() ? '/' + input.name : "");

if (advanced_vm != nullptr)
{
const auto name = "advanced/execute/" + case_name;
RegisterBenchmark(name, [&vm = *advanced_vm, &b, &input](State& state) {
bench_advanced_execute(state, vm, b.code, input.input, input.expected_output);
RegisterBenchmark("advanced/analyse/" + b.name, [code, &rev](State& state) {
bench_analyse<advanced::AdvancedCodeAnalysis, advanced_analyse>(

Check warning on line 125 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L124-L125 

Added lines #L124 - L125 were not covered by tests
state, rev, code);
})->Unit(kMicrosecond);
}

if (baseline_vm != nullptr)
{
const auto name = "baseline/execute/" + case_name;
RegisterBenchmark(name, [&vm = *baseline_vm, &b, &input](State& state) {
bench_baseline_execute(state, vm, b.code, input.input, input.expected_output);
RegisterBenchmark("baseline/analyse/" + b.name, [code, &rev](State& state) {
bench_analyse<baseline::CodeAnalysis, baseline_analyse>(state, rev, code);

Check warning on line 133 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L132-L133 

Added lines #L132 - L133 were not covered by tests
})->Unit(kMicrosecond);
}

if (basel_cg_vm != nullptr)
for (size_t case_index = 0; case_index != cases.size(); ++case_index)

Check warning on line 137 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L137 

Added line #L137 was not covered by tests
{
const auto name = "bnocgoto/execute/" + case_name;
RegisterBenchmark(name, [&vm = *basel_cg_vm, &b, &input](State& state) {
bench_baseline_execute(state, vm, b.code, input.input, input.expected_output);
})->Unit(kMicrosecond);
}

for (auto& [vm_name, vm] : registered_vms)
{
const auto name = std::string{vm_name} + "/total/" + case_name;
RegisterBenchmark(name, [&vm, &b, &input](State& state) {
bench_evmc_execute(state, vm, b.code, input.input, input.expected_output);
})->Unit(kMicrosecond);
const auto& expected = cases[case_index];
const auto tx = b.state_test.multi_tx.get(expected.indexes);

Check warning on line 140 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L139-L140 

Added lines #L139 - L140 were not covered by tests

for (auto& [vm_name, vm] : registered_vms)

Check warning on line 142 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L142 

Added line #L142 was not covered by tests
{
const auto name = std::string{vm_name} + "/execute/" + b.name + '/' +
std::to_string(case_index);

Check warning on line 145 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L144-L145 

Added lines #L144 - L145 were not covered by tests

// `tx` is temporary.
RegisterBenchmark(name, [tx, &vm, &b, &block_info, &rev](State& state) {
bench_transition(state, vm, tx, b.state_test.pre_state, block_info,
b.state_test.block_hashes, rev);
})->Unit(kMicrosecond);
}

Check warning on line 152 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L148-L152 

Added lines #L148 - L152 were not covered by tests
}
}
}
Expand All @@ -183,20 +168,19 @@
/// Uses evmone VMs, only synthetic benchmarks are available.
/// 2: evmone-bench benchmarks_dir
/// Uses evmone VMs, loads all benchmarks from benchmarks_dir.
/// NO LONGER SUPPORTED
/// 3: evmone-bench evmc_config benchmarks_dir
/// The same as (2) but loads additional custom EVMC VM.
/// 4: evmone-bench code_hex_file input_hex expected_output_hex.
/// Uses evmone VMs, registers custom benchmark with the code from the given file,
/// and the given input. The benchmark will compare the output with the provided
/// expected one.
std::tuple<int, std::vector<BenchmarkCase>> parseargs(int argc, char** argv)
std::variant<int, std::vector<BenchmarkCase>> parseargs(int argc, char** argv)

Check warning on line 178 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L178 

Added line #L178 was not covered by tests
{
// Arguments' placeholders:
std::string evmc_config;
std::string benchmarks_dir;
std::string code_hex_file;
std::string input_hex;
std::string expected_output_hex;

switch (argc)
{
Expand All @@ -212,48 +196,24 @@
break;
case 4:
code_hex_file = argv[1];
input_hex = argv[2];
expected_output_hex = argv[3];
break;
default:
std::cerr << "Too many arguments\n";
return {cli_parsing_error, {}};
return cli_parsing_error;

Check warning on line 202 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L202 

Added line #L202 was not covered by tests
}

if (!evmc_config.empty())
{
auto ec = evmc_loader_error_code{};
registered_vms["external"] = evmc::VM{evmc_load_and_configure(evmc_config.c_str(), &ec)};

if (ec != EVMC_LOADER_SUCCESS)
{
if (const auto error = evmc_last_error_msg())
std::cerr << "EVMC loading error: " << error << "\n";
else
std::cerr << "EVMC loading error " << ec << "\n";
return {static_cast<int>(ec), {}};
}

std::cout << "External VM: " << evmc_config << "\n";
}
assert(false); // Unsupported. Benchmarks should be run as a state tests.

Check warning on line 206 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L206 

Added line #L206 was not covered by tests

if (!benchmarks_dir.empty())
{
return {0, load_benchmarks_from_dir(benchmarks_dir)};
return load_benchmarks_from_dir(benchmarks_dir);

Check warning on line 210 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L210 

Added line #L210 was not covered by tests
}

if (!code_hex_file.empty())
{
std::ifstream file{code_hex_file};
return {0, {BenchmarkCase{code_hex_file,
from_spaced_hex(
std::istreambuf_iterator<char>{file}, std::istreambuf_iterator<char>{})
.value(),
{BenchmarkCase::Input{"", from_hex(input_hex).value(),
from_hex(expected_output_hex).value()}}}}};
}
assert(false); // Unsupported. Benchmarks should be run as a state tests.

Check warning on line 214 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L214 

Added line #L214 was not covered by tests

return {0, {}};
return std::vector<BenchmarkCase>{};

Check warning on line 216 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L216 

Added line #L216 was not covered by tests
}
} // namespace
} // namespace evmone::test
Expand All @@ -264,20 +224,28 @@
try
{
Initialize(&argc, argv); // Consumes --benchmark_ options.
const auto [ec, benchmark_cases] = parseargs(argc, argv);
if (ec == cli_parsing_error && ReportUnrecognizedArguments(argc, argv))
return ec;

if (ec != 0)
return ec;

registered_vms["advanced"] = evmc::VM{evmc_create_evmone(), {{"advanced", ""}}};
registered_vms["baseline"] = evmc::VM{evmc_create_evmone()};
registered_vms["bnocgoto"] = evmc::VM{evmc_create_evmone(), {{"cgoto", "no"}}};
register_benchmarks(benchmark_cases);
register_synthetic_benchmarks();
RunSpecifiedBenchmarks();
return 0;
const auto ec_or_benchmark_cases = parseargs(argc, argv);
if (std::holds_alternative<int>(ec_or_benchmark_cases))

Check warning on line 228 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L227-L228 

Added lines #L227 - L228 were not covered by tests
{
const auto ec = std::get<int>(ec_or_benchmark_cases);
if (ec == cli_parsing_error && ReportUnrecognizedArguments(argc, argv))
return ec;

Check warning on line 232 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L230-L232 

Added lines #L230 - L232 were not covered by tests

if (ec != 0)
return ec;

Check warning on line 235 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L234-L235 

Added lines #L234 - L235 were not covered by tests
}
else
{
const auto benchmark_cases =
std::get<std::vector<BenchmarkCase>>(ec_or_benchmark_cases);
registered_vms["advanced"] = evmc::VM{evmc_create_evmone(), {{"advanced", ""}}};
registered_vms["baseline"] = evmc::VM{evmc_create_evmone()};
registered_vms["bnocgoto"] = evmc::VM{evmc_create_evmone(), {{"cgoto", "no"}}};
register_benchmarks(benchmark_cases);
register_synthetic_benchmarks();
RunSpecifiedBenchmarks();
return 0;
}

Check warning on line 248 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L240-L248 

Added lines #L240 - L248 were not covered by tests
}
catch (const std::exception& ex)
{
Expand Down
Loading