Skip to content

Commit c56c2fc

Browse files
sthahasthaha
authored
Merge pull request #1631 from husky-parul/validation
feat: Export validation result as json object.
2 parents 8421eea + 6a6017b commit c56c2fc

File tree

6 files changed

+366
-9
lines changed

6 files changed

+366
-9
lines changed

‎e2e/tools/validator/html/index.html

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
<!DOCTYPE html>
2+
<html lang="en">
3+
<head>
4+
<meta charset="UTF-8">
5+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
6+
<title>Chart with JSON Data</title>
7+
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
8+
<script defer src="script.js"></script>
9+
<style>
10+
body {
11+
display: flex;
12+
}
13+
#chart-container {
14+
width: 70%;
15+
padding: 20px;
16+
}
17+
#info-container {
18+
width: 30%;
19+
padding: 20px;
20+
}
21+
ul {
22+
list-style-type: none;
23+
}
24+
</style>
25+
</head>
26+
<body>
27+
<div id="chart-container">
28+
<canvas id="myChart"></canvas>
29+
</div>
30+
<div id="info-container">
31+
<div id="build-info">
32+
<h3>Build Info</h3>
33+
<ul></ul>
34+
</div>
35+
<div id="machine-spec">
36+
<h3>Machine Spec</h3>
37+
<ul></ul>
38+
</div>
39+
</div>
40+
</body>
41+
</html>

‎e2e/tools/validator/html/script.js

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
async function fetchJSONFile(filePath) {
2+
try {
3+
const response = await fetch(filePath);
4+
const data = await response.json();
5+
return data;
6+
} catch (error) {
7+
console.error('Error fetching JSON file:', error);
8+
}
9+
}
10+
11+
function processJSONData(data) {
12+
const revisions = [];
13+
const mseValues = [];
14+
const mapeValues = [];
15+
const labels = [];
16+
17+
data.build_info.forEach(buildInfo => {
18+
const buildInfoObj = JSON.parse(buildInfo);
19+
const revision = buildInfoObj.revision;
20+
if (!revisions.includes(revision)) {
21+
revisions.push(revision);
22+
}
23+
});
24+
25+
data.result.forEach(result => {
26+
labels.push(result['metric-name']);
27+
mseValues.push(parseFloat(result.value.mse));
28+
mapeValues.push(parseFloat(result.value.mape));
29+
});
30+
31+
return { revisions, mseValues, mapeValues, labels };
32+
}
33+
34+
function plotChart(revisions, mseValues, mapeValues, labels) {
35+
const ctx = document.getElementById('myChart').getContext('2d');
36+
new Chart(ctx, {
37+
type: 'line',
38+
data: {
39+
labels: labels,
40+
datasets: [{
41+
label: `MSE - ${revisions.join(', ')}`,
42+
data: mseValues,
43+
borderColor: 'rgba(75, 192, 192, 1)',
44+
backgroundColor: 'rgba(75, 192, 192, 0.2)',
45+
fill: false
46+
}, {
47+
label: `MAPE - ${revisions.join(', ')}`,
48+
data: mapeValues,
49+
borderColor: 'rgba(255, 99, 132, 1)',
50+
backgroundColor: 'rgba(255, 99, 132, 0.2)',
51+
fill: false
52+
}]
53+
},
54+
options: {
55+
responsive: true,
56+
scales: {
57+
x: {
58+
beginAtZero: true
59+
},
60+
y: {
61+
beginAtZero: true,
62+
title: {
63+
display: true,
64+
text: 'Values'
65+
}
66+
}
67+
},
68+
plugins: {
69+
legend: {
70+
display: true,
71+
position: 'top'
72+
},
73+
tooltip: {
74+
callbacks: {
75+
label: function(context) {
76+
let label = context.dataset.label || '';
77+
if (label) {
78+
label += ': ';
79+
}
80+
if (context.parsed.y !== null) {
81+
label += new Intl.NumberFormat('en-US', { maximumFractionDigits: 2 }).format(context.parsed.y);
82+
}
83+
return label;
84+
}
85+
}
86+
}
87+
}
88+
}
89+
});
90+
}
91+
92+
function displayBuildInfo(data) {
93+
const buildInfoContainer = document.getElementById('build-info').querySelector('ul');
94+
data.build_info.forEach(info => {
95+
const buildInfoObj = JSON.parse(info);
96+
const listItem = document.createElement('li');
97+
listItem.textContent = `Revision: ${buildInfoObj.revision}, Version: ${buildInfoObj.version}`;
98+
buildInfoContainer.appendChild(listItem);
99+
});
100+
}
101+
102+
function displayMachineSpec(data) {
103+
const machineSpecContainer = document.getElementById('machine-spec').querySelector('ul');
104+
data.machine_spec.forEach(spec => {
105+
const listItem = document.createElement('li');
106+
listItem.innerHTML = `
107+
<strong>Type:</strong> ${spec.type}<br>
108+
<strong>Model:</strong> ${spec.model}<br>
109+
<strong>Cores:</strong> ${spec.cores}<br>
110+
<strong>Threads:</strong> ${spec.threads}<br>
111+
<strong>Sockets:</strong> ${spec.sockets}<br>
112+
<strong>DRAM:</strong> ${spec.dram}
113+
`;
114+
machineSpecContainer.appendChild(listItem);
115+
});
116+
}
117+
118+
// Path to the single JSON file
119+
const filePath = '/tmp/v0.2-2315-g859d9bf1/v0.2-2316-gc607a6df.json'; // Change to the actual path of your JSON file
120+
121+
fetchJSONFile(filePath)
122+
.then(data => {
123+
const { revisions, mseValues, mapeValues, labels } = processJSONData(data);
124+
plotChart(revisions, mseValues, mapeValues, labels);
125+
displayBuildInfo(data);
126+
displayMachineSpec(data);
127+
})
128+
.catch(error => console.error('Error processing data:', error));

‎e2e/tools/validator/html/style.css

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
body {
2+
font-family: Arial, sans-serif;
3+
display: flex;
4+
justify-content: center;
5+
align-items: center;
6+
height: 100vh;
7+
margin: 0;
8+
background-color: #f4f4f4;
9+
}
10+
11+
#dashboard {
12+
width: 80%;
13+
max-width: 800px;
14+
background: white;
15+
padding: 20px;
16+
box-shadow: 0 0 10px rgba(0, 0, 0, 0.1);
17+
border-radius: 8px;
18+
}
19+
20+
canvas {
21+
max-width: 100%;
22+
height: auto;
23+
}

‎e2e/tools/validator/src/validator/cli/__init__.py

Lines changed: 78 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import json
77
import logging
88
import os
9+
import re
910
import subprocess
1011
import time
1112
import typing
@@ -17,12 +18,14 @@
1718
from validator.__about__ import __version__
1819
from validator.cli import options
1920
from validator.prometheus import Comparator, PrometheusClient, Series, ValueOrError
21+
from validator.report import CustomEncoder, JsonTemplate
2022
from validator.specs import MachineSpec, get_host_spec, get_vm_spec
2123
from validator.stresser import Remote, ScriptResult
2224
from validator.validations import Loader, QueryTemplate, Validation
2325

2426
logger = logging.getLogger(__name__)
2527
pass_config = click.make_pass_decorator(config.Validator)
28+
data_dict = {}
2629

2730

2831
@dataclass
@@ -277,6 +280,8 @@ def stress(cfg: config.Validator, script_path: str, report_dir: str):
277280
click.secho(" * Generating report dir and tag", fg="green")
278281
results_dir, tag = create_report_dir(report_dir)
279282
click.secho(f"\tresults dir: {results_dir}, tag: {tag}", fg="bright_green")
283+
filepath = results_dir + "/" + tag + ".json"
284+
data_dict.update({"file_path": filepath})
280285

281286
res = TestResult(tag)
282287

@@ -297,7 +302,7 @@ def stress(cfg: config.Validator, script_path: str, report_dir: str):
297302
time.sleep(10)
298303

299304
res.validations = run_validations(cfg, stress_test, results_dir)
300-
305+
create_json(res)
301306
write_md_report(results_dir, res)
302307

303308

@@ -450,3 +455,75 @@ def validate_acpi(cfg: config.Validator, duration: datetime.timedelta, report_di
450455
write_md_report(results_dir, res)
451456

452457
return int(res.validations.passed)
458+
459+
460+
def create_json(res):
461+
def update_list_json(new_value: list, new_key: str):
462+
data_dict[new_key] = new_value
463+
464+
def custom_encode(input_string):
465+
pattern = re.compile(r'(\w+)=("[^"]*"|[^,]+)')
466+
matches = pattern.findall(input_string)
467+
parsed_dict = {key: value.strip('"') for key, value in matches}
468+
return json.dumps(parsed_dict)
469+
470+
result = []
471+
472+
for i in res.validations.results:
473+
value = {}
474+
if i.mse_passed:
475+
value["mse"] = float(i.mse.value)
476+
else:
477+
value["mse"] = float(i.mse.error)
478+
if i.mape_passed:
479+
value["mape"] = float(i.mape.value)
480+
else:
481+
value["mape"] = float(i.mape.error)
482+
value["status"] = "mape passed: " + str(i.mape_passed) + ", mse passed: " + str(i.mse_passed)
483+
m_name = i.name.replace(" - ", "_")
484+
485+
result.append({m_name: value})
486+
487+
build_info = []
488+
for i in res.build_info:
489+
tmp = i.replace("kepler_exporter_build_info", "")
490+
build_info.append(custom_encode(tmp))
491+
492+
node_info = []
493+
for i in res.node_info:
494+
tmp = i.replace("kepler_exporter_node_info", "")
495+
node_info.append(custom_encode(tmp))
496+
497+
update_list_json(build_info, "build_info")
498+
update_list_json(node_info, "node_info")
499+
500+
machine_spec = []
501+
machine_spec.append(
502+
{
503+
"type": "host",
504+
"model": res.host_spec[0][0],
505+
"cores": res.host_spec[0][1],
506+
"threads": res.host_spec[0][2],
507+
"sockets": res.host_spec[0][3],
508+
"flags": res.host_spec[0][4],
509+
"dram": res.host_spec[1],
510+
}
511+
)
512+
machine_spec.append(
513+
{
514+
"type": "vm",
515+
"model": res.vm_spec[0][0],
516+
"cores": res.vm_spec[0][1],
517+
"threads": res.vm_spec[0][2],
518+
"sockets": res.vm_spec[0][3],
519+
"flags": res.vm_spec[0][4],
520+
"dram": res.vm_spec[1],
521+
}
522+
)
523+
update_list_json(machine_spec, "machine_spec")
524+
525+
update_list_json(result, "result")
526+
json_template = JsonTemplate(**data_dict)
527+
file_name = data_dict["file_path"]
528+
with open(file_name, "w") as file:
529+
json.dump(json_template, file, cls=CustomEncoder, indent=2)

‎e2e/tools/validator/src/validator/report/__init__.py

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
import json
2+
from typing import Any
3+
4+
5+
class Value:
6+
def __init__(self, mse: str = "", mape: str = "", status: str = ""):
7+
self.mse = mse
8+
self.mape = mape
9+
self.status = status
10+
11+
def to_dict(self):
12+
return {"mse": self.mse, "mape": self.mape, "status": self.status}
13+
14+
def __repr__(self):
15+
return f"Value(mse='{self.mse}', mape='{self.mape}', status='{self.status}')"
16+
17+
18+
class Result:
19+
def __init__(self, metric_name: str, value: dict[str, Any]):
20+
if value is None:
21+
value = {}
22+
self.metric_name = metric_name
23+
self.value = Value(**value)
24+
25+
def to_dict(self):
26+
return {"metric-name": self.metric_name, "value": self.value.to_dict()}
27+
28+
def __repr__(self):
29+
return f"Result(metric_name='{self.metric_name}', value={self.value})"
30+
31+
32+
class JsonTemplate:
33+
def __init__(
34+
self,
35+
file_path: str,
36+
build_info: list[Any],
37+
node_info: list[Any],
38+
machine_spec: list[Any],
39+
result: list[dict[str, Any]],
40+
):
41+
if build_info is None:
42+
build_info = []
43+
if node_info is None:
44+
node_info = []
45+
if machine_spec is None:
46+
machine_spec = []
47+
if result is None:
48+
result = []
49+
50+
self.file_path = file_path
51+
self.build_info = build_info
52+
self.node_info = node_info
53+
self.machine_spec = machine_spec
54+
self.result = []
55+
for res in result:
56+
for key, value in res.items():
57+
self.result.append(Result(key, value))
58+
59+
def to_dict(self):
60+
return {
61+
"file_path": self.file_path,
62+
"build_info": self.build_info,
63+
"node_info": self.node_info,
64+
"machine_spec": self.machine_spec,
65+
"result": [res.to_dict() for res in self.result],
66+
}
67+
68+
def __repr__(self):
69+
return (
70+
f"JsonTemplate(file_path='{self.file_path}', build_info={self.build_info}, "
71+
f"node_info={self.node_info}, machine_spec={self.machine_spec}, "
72+
f"result={self.result})"
73+
)
74+
75+
76+
class CustomEncoder(json.JSONEncoder):
77+
def default(self, obj):
78+
if hasattr(obj, "to_dict"):
79+
return obj.to_dict()
80+
return super().default(obj)

0 commit comments

Comments
 (0)