Skip to content

test: Handle full state tests in evmone-bench #1043

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

test: Handle full state tests in evmone-bench #1043

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
74a4607
test: Move `statetest_runner.cpp` to statetestutils
rodiazet May 19, 2025
a815b5f
test: Handle full state tests in evmone-becnh
rodiazet May 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
258 changes: 108 additions & 150 deletions test/bench/bench.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@
// Copyright 2019 The evmone Authors.
// SPDX-License-Identifier: Apache-2.0

#include "../statetest/statetest.hpp"
#include "helpers.hpp"
#include "synthetic_benchmarks.hpp"
#include <benchmark/benchmark.h>
#include <evmc/evmc.hpp>
#include <evmc/loader.h>
#include <evmone/evmone.h>
#include <gtest/gtest.h>
#include <test/statetest/statetest.hpp>
#include <filesystem>
#include <fstream>
#include <iostream>
Expand All @@ -26,70 +26,55 @@
{
struct BenchmarkCase
{
struct Input
{
std::string name;
bytes input;
bytes expected_output;

Input(std::string _name, bytes _input, bytes _expected_output = {}) noexcept
: name{std::move(_name)},
input{std::move(_input)},
expected_output{std::move(_expected_output)}
{}
};

StateTransitionTest state_test;
std::string name;
bytes code;
std::vector<Input> inputs;
};

/// Loads the benchmark case's inputs from the inputs file at the given path.
std::vector<BenchmarkCase::Input> load_inputs(const StateTransitionTest& state_test)
{
std::vector<BenchmarkCase::Input> inputs;
inputs.reserve(state_test.multi_tx.inputs.size());
for (size_t i = 0; i < state_test.multi_tx.inputs.size(); ++i)
inputs.emplace_back(state_test.input_labels.at(i), state_test.multi_tx.inputs[i]);
return inputs;
}

/// Loads a benchmark case from a file at `path` and all its inputs from the matching inputs file.
BenchmarkCase load_benchmark(const fs::path& path, const std::string& name_prefix)
std::vector<BenchmarkCase> load_benchmark(const fs::path& path, const std::string& name_prefix)

Check warning on line 34 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L34 

Added line #L34 was not covered by tests
{
std::ifstream f{path};
auto state_test = std::move(load_state_tests(f).at(0));
std::vector<BenchmarkCase> result;

Check warning on line 37 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L37 

Added line #L37 was not covered by tests

const auto name = name_prefix + path.stem().string();
const auto code = state_test.pre_state.get(state_test.multi_tx.to.value()).code;
const auto inputs = load_inputs(state_test);
auto state_tests = load_state_tests(f);
result.reserve(state_tests.size());
for (const auto& state_test : state_tests)

Check warning on line 41 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L39-L41 

Added lines #L39 - L41 were not covered by tests
{
// If a file has a single test, skip its name.
auto name = name_prefix + path.stem().string() +
(state_tests.size() > 1 ? "/" + state_test.name : "");
result.emplace_back(BenchmarkCase{state_test, std::move(name)});
}

Check warning on line 47 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L44-L47 

Added lines #L44 - L47 were not covered by tests

return BenchmarkCase{name, code, inputs};
return result;

Check warning on line 49 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L49 

Added line #L49 was not covered by tests
}

/// Loads all benchmark cases from the given directory and all its subdirectories.
std::vector<BenchmarkCase> load_benchmarks_from_dir( // NOLINT(misc-no-recursion)
const fs::path& path, const std::string& name_prefix = {})
{
std::vector<fs::path> subdirs;
std::vector<fs::path> code_files;
std::vector<fs::path> files;

Check warning on line 57 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L57 

Added line #L57 was not covered by tests

for (auto& e : fs::directory_iterator{path})
{
if (e.is_directory())
subdirs.emplace_back(e);
else if (e.path().extension() == ".json")
code_files.emplace_back(e);
files.emplace_back(e);

Check warning on line 64 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L64 

Added line #L64 was not covered by tests
}

std::ranges::sort(subdirs);
std::ranges::sort(code_files);
std::ranges::sort(files);

Check warning on line 68 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L68 

Added line #L68 was not covered by tests

std::vector<BenchmarkCase> benchmark_cases;

benchmark_cases.reserve(std::size(code_files));
for (const auto& f : code_files)
benchmark_cases.emplace_back(load_benchmark(f, name_prefix));
for (const auto& f : files)

Check warning on line 72 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L72 

Added line #L72 was not covered by tests
{
auto t = load_benchmark(f, name_prefix);
benchmark_cases.insert(benchmark_cases.end(), std::make_move_iterator(t.begin()),

Check warning on line 75 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L74-L75 

Added lines #L74 - L75 were not covered by tests
std::make_move_iterator(t.end()));
}

Check warning on line 77 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L77 

Added line #L77 was not covered by tests

for (const auto& d : subdirs)
{
Expand All @@ -105,66 +90,75 @@
{
evmc::VM* advanced_vm = nullptr;
evmc::VM* baseline_vm = nullptr;
evmc::VM* basel_cg_vm = nullptr;
if (const auto it = registered_vms.find("advanced"); it != registered_vms.end())
advanced_vm = &it->second;
if (const auto it = registered_vms.find("baseline"); it != registered_vms.end())
baseline_vm = &it->second;
if (const auto it = registered_vms.find("bnocgoto"); it != registered_vms.end())
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why was this VM removed?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Because now all VMs are tested in the loop below. First two VMs are used to perform analysis. The bnocgoto did not have analysis run before. After moving all VMs to the loop this became unused.

basel_cg_vm = &it->second;

evmc::VM check_test_vm = evmc::VM{evmc_create_evmone()};

Check warning on line 98 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L98 

Added line #L98 was not covered by tests

for (const auto& b : benchmark_cases)
{
if (advanced_vm != nullptr)
{
RegisterBenchmark("advanced/analyse/" + b.name, [&b](State& state) {
bench_analyse<advanced::AdvancedCodeAnalysis, advanced_analyse>(
state, default_revision, b.code);
})->Unit(kMicrosecond);
}
run_state_test(b.state_test, check_test_vm, false);

Check warning on line 102 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L102 

Added line #L102 was not covered by tests

if (baseline_vm != nullptr)
{
RegisterBenchmark("baseline/analyse/" + b.name, [&b](State& state) {
bench_analyse<baseline::CodeAnalysis, baseline_analyse>(
state, default_revision, b.code);
})->Unit(kMicrosecond);
}
if (::testing::Test::HasFailure())
throw std::invalid_argument{"State test you want to bench failed."};

Check warning on line 105 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L104-L105 

Added lines #L104 - L105 were not covered by tests

for (const auto& input : b.inputs)
const auto code = b.state_test.pre_state.get_account_code(b.state_test.multi_tx.to.value());
for (const auto& [rev, cases, block_info] : b.state_test.cases)

Check warning on line 108 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L107-L108 

Added lines #L107 - L108 were not covered by tests
{
const auto case_name = b.name + (!input.name.empty() ? '/' + input.name : "");

if (advanced_vm != nullptr)
{
const auto name = "advanced/execute/" + case_name;
RegisterBenchmark(name, [&vm = *advanced_vm, &b, &input](State& state) {
bench_advanced_execute(state, vm, b.code, input.input, input.expected_output);
RegisterBenchmark("advanced/analyse/" + b.name, [code, &rev](State& state) {
Copy link
Preview

Copilot AI May 20, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Consider capturing 'rev' by value instead of by reference in the lambda to ensure that its value is preserved correctly in the benchmark callback.

Suggested change
RegisterBenchmark("advanced/analyse/" + b.name, [code, &rev](State& state) {
RegisterBenchmark("advanced/analyse/" + b.name, [code, rev](State& state) {

Copilot uses AI. Check for mistakes.

bench_analyse<advanced::AdvancedCodeAnalysis, advanced_analyse>(

Check warning on line 113 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L112-L113 

Added lines #L112 - L113 were not covered by tests
state, rev, code);
})->Unit(kMicrosecond);
}

if (baseline_vm != nullptr)
{
const auto name = "baseline/execute/" + case_name;
RegisterBenchmark(name, [&vm = *baseline_vm, &b, &input](State& state) {
bench_baseline_execute(state, vm, b.code, input.input, input.expected_output);
})->Unit(kMicrosecond);
}

if (basel_cg_vm != nullptr)
{
const auto name = "bnocgoto/execute/" + case_name;
RegisterBenchmark(name, [&vm = *basel_cg_vm, &b, &input](State& state) {
bench_baseline_execute(state, vm, b.code, input.input, input.expected_output);
RegisterBenchmark("baseline/analyse/" + b.name, [code, &rev](State& state) {
bench_analyse<baseline::CodeAnalysis, baseline_analyse>(state, rev, code);

Check warning on line 121 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L120-L121 

Added lines #L120 - L121 were not covered by tests
})->Unit(kMicrosecond);
}

for (auto& [vm_name, vm] : registered_vms)
for (size_t case_index = 0; case_index != cases.size(); ++case_index)

Check warning on line 125 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L125 

Added line #L125 was not covered by tests
{
const auto name = std::string{vm_name} + "/total/" + case_name;
RegisterBenchmark(name, [&vm, &b, &input](State& state) {
bench_evmc_execute(state, vm, b.code, input.input, input.expected_output);
})->Unit(kMicrosecond);
const auto& expected = cases[case_index];
const auto tx = b.state_test.multi_tx.get(expected.indexes);

Check warning on line 128 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L127-L128 

Added lines #L127 - L128 were not covered by tests

std::string case_name;
if (const auto it = b.state_test.input_labels.find(case_index);
it != b.state_test.input_labels.end())

Check warning on line 132 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L130-L132 

Added lines #L130 - L132 were not covered by tests
{
case_name = it->second;

Check warning on line 134 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L134 

Added line #L134 was not covered by tests
}
else
case_name = std::to_string(case_index);

Check warning on line 137 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L137 

Added line #L137 was not covered by tests


for (auto& [vm_name, vm] : registered_vms)

Check warning on line 140 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L140 

Added line #L140 was not covered by tests
{
const auto name = std::string{vm_name} + "/execute/" + b.name + '/' + case_name;

Check warning on line 142 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L142 

Added line #L142 was not covered by tests

const auto tx_props_or_error = state::validate_transaction(
b.state_test.pre_state, block_info, tx, rev, block_info.gas_limit,
static_cast<int64_t>(state::max_blob_gas_per_block(rev)));
if (const auto err = get_if<std::error_code>(&tx_props_or_error))

Check warning on line 147 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L144-L147 

Added lines #L144 - L147 were not covered by tests
{
throw std::invalid_argument{
"Transaction validation failure: " + err->message()};

Check warning on line 150 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L150 

Added line #L150 was not covered by tests
}

const auto tx_props = get<state::TransactionProperties>(tx_props_or_error);

Check warning on line 153 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L153 

Added line #L153 was not covered by tests

// `tx` and `tx_props` are temporary.
RegisterBenchmark(name, [tx, tx_props, &vm, &b, &block_info, &rev](

Check warning on line 156 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L156 

Added line #L156 was not covered by tests
State& state) {
bench_transition(state, vm, tx, tx_props, b.state_test.pre_state,
block_info, b.state_test.block_hashes, rev);
})->Unit(kMicrosecond);
}

Check warning on line 161 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L158-L161 

Added lines #L158 - L161 were not covered by tests
}
}
}
Expand All @@ -181,79 +175,35 @@
///
/// 1: evmone-bench
/// Uses evmone VMs, only synthetic benchmarks are available.
/// 2: evmone-bench benchmarks_dir
/// Uses evmone VMs, loads all benchmarks from benchmarks_dir.
/// 3: evmone-bench evmc_config benchmarks_dir
/// The same as (2) but loads additional custom EVMC VM.
/// 4: evmone-bench code_hex_file input_hex expected_output_hex.
/// Uses evmone VMs, registers custom benchmark with the code from the given file,
/// and the given input. The benchmark will compare the output with the provided
/// expected one.
std::tuple<int, std::vector<BenchmarkCase>> parseargs(int argc, char** argv)
/// 2: evmone-bench benchmarks_path
/// Uses evmone VMs, loads all benchmarks from benchmarks_path. If benchmarks_path is a `json`
/// file, single test is run.
std::variant<int, std::vector<BenchmarkCase>> parseargs(int argc, char** argv)

Check warning on line 181 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L181 

Added line #L181 was not covered by tests
{
// Arguments' placeholders:
std::string evmc_config;
std::string benchmarks_dir;
std::string code_hex_file;
std::string input_hex;
std::string expected_output_hex;
// Argument's placeholder:
std::string benchmarks_path;

Check warning on line 184 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L184 

Added line #L184 was not covered by tests

switch (argc)
{
case 1:
// Run with built-in synthetic benchmarks only.
break;
case 2:
benchmarks_dir = argv[1];
break;
case 3:
evmc_config = argv[1];
benchmarks_dir = argv[2];
break;
case 4:
code_hex_file = argv[1];
input_hex = argv[2];
expected_output_hex = argv[3];
benchmarks_path = argv[1];

Check warning on line 192 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L192 

Added line #L192 was not covered by tests
break;
default:
std::cerr << "Too many arguments\n";
return {cli_parsing_error, {}};
}

if (!evmc_config.empty())
{
auto ec = evmc_loader_error_code{};
registered_vms["external"] = evmc::VM{evmc_load_and_configure(evmc_config.c_str(), &ec)};

if (ec != EVMC_LOADER_SUCCESS)
{
if (const auto error = evmc_last_error_msg())
std::cerr << "EVMC loading error: " << error << "\n";
else
std::cerr << "EVMC loading error " << ec << "\n";
return {static_cast<int>(ec), {}};
}

std::cout << "External VM: " << evmc_config << "\n";
}

if (!benchmarks_dir.empty())
{
return {0, load_benchmarks_from_dir(benchmarks_dir)};
return cli_parsing_error;

Check warning on line 195 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L195 

Added line #L195 was not covered by tests
}

if (!code_hex_file.empty())
if (!benchmarks_path.empty())

Check warning on line 198 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L198 

Added line #L198 was not covered by tests
{
std::ifstream file{code_hex_file};
return {0, {BenchmarkCase{code_hex_file,
from_spaced_hex(
std::istreambuf_iterator<char>{file}, std::istreambuf_iterator<char>{})
.value(),
{BenchmarkCase::Input{"", from_hex(input_hex).value(),
from_hex(expected_output_hex).value()}}}}};
if (fs::is_directory(benchmarks_path))
return load_benchmarks_from_dir(benchmarks_path);

Check warning on line 201 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L200-L201 

Added lines #L200 - L201 were not covered by tests
else
return load_benchmark(benchmarks_path, {});

Check warning on line 203 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L203 

Added line #L203 was not covered by tests
}

return {0, {}};
return std::vector<BenchmarkCase>{};

Check warning on line 206 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L206 

Added line #L206 was not covered by tests
}
} // namespace
} // namespace evmone::test
Expand All @@ -266,20 +216,28 @@
try
{
Initialize(&argc, argv); // Consumes --benchmark_ options.
const auto [ec, benchmark_cases] = parseargs(argc, argv);
if (ec == cli_parsing_error && ReportUnrecognizedArguments(argc, argv))
return ec;

if (ec != 0)
return ec;

registered_vms["advanced"] = evmc::VM{evmc_create_evmone(), {{"advanced", ""}}};
registered_vms["baseline"] = evmc::VM{evmc_create_evmone()};
registered_vms["bnocgoto"] = evmc::VM{evmc_create_evmone(), {{"cgoto", "no"}}};
register_benchmarks(benchmark_cases);
register_synthetic_benchmarks();
RunSpecifiedBenchmarks();
return 0;
const auto ec_or_benchmark_cases = parseargs(argc, argv);
if (std::holds_alternative<int>(ec_or_benchmark_cases))

Check warning on line 220 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L219-L220 

Added lines #L219 - L220 were not covered by tests
{
const auto ec = std::get<int>(ec_or_benchmark_cases);
if (ec == cli_parsing_error && ReportUnrecognizedArguments(argc, argv))
return ec;

Check warning on line 224 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L222-L224 

Added lines #L222 - L224 were not covered by tests

if (ec != 0)
return ec;

Check warning on line 227 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L226-L227 

Added lines #L226 - L227 were not covered by tests
}
else
{
const auto benchmark_cases =
std::get<std::vector<BenchmarkCase>>(ec_or_benchmark_cases);
registered_vms["advanced"] = evmc::VM{evmc_create_evmone(), {{"advanced", ""}}};
registered_vms["baseline"] = evmc::VM{evmc_create_evmone()};
registered_vms["bnocgoto"] = evmc::VM{evmc_create_evmone(), {{"cgoto", "no"}}};
register_benchmarks(benchmark_cases);
register_synthetic_benchmarks();
RunSpecifiedBenchmarks();
return 0;
}

Check warning on line 240 in test/bench/bench.cpp

View check run for this annotation

Codecov / codecov/patch

test/bench/bench.cpp#L232-L240 

Added lines #L232 - L240 were not covered by tests
}
catch (const std::exception& ex)
{
Expand Down
Loading