Update dependencies, add scripts to generate report and updated README

README.md

@@ -15,25 +15,43 @@ These benchmarks can be executed on a vanilla Kubernetes or OpenShift cluster. I
   * Non-OpenShift Deployments Only; Optional: [Cadvisor](https://github.com/google/cadvisor)
 
 * Notes
-   * Clone git repositories into sibling directories to the `loki-benchmarks` one.
-   * Recommended cluster size: `m4.16xlarge`
+  * Clone git repositories into sibling directories to the `loki-benchmarks` one.
+  * Recommended cluster size: `m4.16xlarge`
 
 ## Configuring Tests
 
 To change the testing configuration, see the files in the [config](./config) directory.
 
-Use the `scenarios/benchmarks.yaml` file to add, modify, or remove configurations. Modify the `generator.yaml`, `metrics.yaml`, or `querier.yaml` in the prefered deployment method directory to change these soruces.
+Different scenarios can be customized under `config/benchmarks/scenarios/benchmarks`. Current the benchmarks support two testing scenarios:
+
+* Ingestion path scenarios: [suppored configuration](https://github.com/observatorium/loki-benchmarks/blob/1a0a9e8f6190475b6c1bfacb5a31a88bd76cbb36/internal/config/config.go#L76-L81), this test will generate X amount of logs throughout a 30 minute window that's supposed to represent a full day of log ingestion.
+* Query path scenarios: [supported configuration](https://github.com/observatorium/loki-benchmarks/blob/1a0a9e8f6190475b6c1bfacb5a31a88bd76cbb36/internal/config/config.go#L102-L108), the theory behind this test is to generate the amount of data that would be queried before it starts running the queries.
 
 ## Running Benchmarks
 
-Use the `make run-rhobs-benchmarks` or `make run-operator-benchmarks` to execute the benchmark program with the RHOBS or operator deployment styles on OpenShift respectively. Upon successful completion, a JSON and XML file will be created in the `reports/date+time` directory with the results of the tests.
+### Prerequisites
 
-## Troubleshooting
+The `run-operator-benchmarks` expects the following two env vars to be set `LOKI_OPERATOR_REGISTRY` `LOKI_STORAGE_BUCKET`.
+E.g
+
+```shell
+export LOKI_OPERATOR_REGISTRY=jmarcal
+export LOKI_STORAGE_BUCKET=jmarcal-loki-benchmark-storage
+```
 
-During benchmark execution, use [hack/scripts/ocp-deploy-grafana.sh](hack/scripts/ocp-deploy-grafana.sh) to deploy grafna and connect to Loki as a datasource: 
-- Use a web browser to access grafana UI. The URL, username and password are printed by the script 
-- In the UI, under settings -> data-sources hit `Save & test` to verify that Loki data-source is connected and that there are no errors
-- In explore tab change the data-source to `Loki` and use `{client="promtail"}` query to visualize log lines
-- Use additional queries such as `rate({client="promtail"}[1m])` to verify the behaviour of Loki and the benchmark
+### Steps
+
+1. Use the `make run-rhobs-benchmarks` or `make run-operator-benchmarks` to execute the benchmark program with the RHOBS or operator deployment styles on OpenShift respectively.
+Both commands will run all the scenarios under `config/benchmarks/scenarios/benchmarks`.
+2. Upon successful completion of each scenario, a JSON and XML file will be created in the `reports/date+time/schenario_name` directory with the results of the tests.
+3. Once all scenarios have been run we can run `python3 hack/scripts/generate_report.py $PATH_TO_SCENARIO_1 $PATH_TO_SCENARIO_2 $PATH_TO_SCENARIO_...` to compile a report that helps compare the different scenarios.
+4. To share the report on gDoc you can run `pthon3 hack/scripts/create-gdoc.py $PATH_TO_THE_REPORT` this will generate a docx file that can then be shared.
+
+## Troubleshooting
 
+During benchmark execution, use [hack/scripts/ocp-deploy-grafana.sh](hack/scripts/ocp-deploy-grafana.sh) to deploy grafna and connect to Loki as a datasource:
 
+* Use a web browser to access grafana UI. The URL, username and password are printed by the script
+* In the UI, under settings -> data-sources hit `Save & test` to verify that Loki data-source is connected and that there are no errors
+* In explore tab change the data-source to `Loki` and use `{client="promtail"}` query to visualize log lines
+* Use additional queries such as `rate({client="promtail"}[1m])` to verify the behaviour of Loki and the benchmark

benchmarks/ingestion_path_test.go

@@ -62,10 +62,18 @@ var _ = Describe("Ingestion Path", func() {
 				err := metricsClient.MeasureIngestionVerificationMetrics(e, generatorDpl.GetName(), samplingRange)
 				Expect(err).Should(Succeed(), fmt.Sprintf("Failed - %v", err))
 
+				// Gateways
+				job := benchCfg.Metrics.Jobs.Gateway
+				annotation := metrics.GatewayAnnotation
+				err = metricsClient.MeasureResourceUsageMetrics(e, job, samplingRange, annotation)
+				Expect(err).Should(Succeed(), fmt.Sprintf("Failed - %v", err))
+
 				// Distributors
-				job := benchCfg.Metrics.Jobs.Distributor
-				annotation := metrics.DistributorAnnotation
+				job = benchCfg.Metrics.Jobs.Distributor
+				annotation = metrics.DistributorAnnotation
 
+				err = metricsClient.MeasureResourceUsageMetrics(e, job, samplingRange, annotation)
+				Expect(err).Should(Succeed(), fmt.Sprintf("Failed - %v", err))
 				err = metricsClient.MeasureHTTPRequestMetrics(e, metrics.WriteRequestPath, job, samplingRange, annotation)
 				Expect(err).Should(Succeed(), fmt.Sprintf("Failed - %v", err))
 
@@ -79,7 +87,7 @@ var _ = Describe("Ingestion Path", func() {
 				Expect(err).Should(Succeed(), fmt.Sprintf("Failed - %v", err))
 				err = metricsClient.MeasureGRPCRequestMetrics(e, metrics.WriteRequestPath, job, samplingRange, annotation)
 				Expect(err).Should(Succeed(), fmt.Sprintf("Failed - %v", err))
-				err = metricsClient.MeasureBoltDBShipperRequestMetrics(e, metrics.WriteRequestPath, job, samplingRange)
+				err = metricsClient.MeasureIndexRequestMetrics(e, metrics.WriteRequestPath, job, samplingRange)
 				Expect(err).Should(Succeed(), fmt.Sprintf("Failed - %v", err))
 			}, samplingCfg)
 		})

benchmarks/query_path_test.go

@@ -122,7 +122,7 @@ var _ = Describe("Query Path", func() {
 				Expect(err).Should(Succeed(), fmt.Sprintf("Failed - %v", err))
 				err = metricsClient.MeasureGRPCRequestMetrics(e, metrics.ReadRequestPath, job, samplingRange, annotation)
 				Expect(err).Should(Succeed(), fmt.Sprintf("Failed - %v", err))
-				err = metricsClient.MeasureBoltDBShipperRequestMetrics(e, metrics.ReadRequestPath, job, samplingRange)
+				err = metricsClient.MeasureIndexRequestMetrics(e, metrics.ReadRequestPath, job, samplingRange)
 				Expect(err).Should(Succeed(), fmt.Sprintf("Failed - %v", err))
 			}, samplingCfg)
 		})

hack/scripts/create-gdoc.py

@@ -0,0 +1,117 @@
+import os
+import sys
+import argparse
+from docx import Document
+from docx.shared import Inches
+from docx.oxml.ns import qn
+from docx.oxml import OxmlElement
+import markdown
+from bs4 import BeautifulSoup
+
+def add_hyperlink(paragraph, url, text, color="0000FF", underline=True):
+    part = paragraph.part
+    r_id = part.relate_to(url, 'http://schemas.openxmlformats.org/officeDocument/2006/relationships/hyperlink', is_external=True)
+
+    hyperlink = OxmlElement('w:hyperlink')
+    hyperlink.set(qn('r:id'), r_id)
+
+    new_run = OxmlElement('w:r')
+    rPr = OxmlElement('w:rPr')
+
+    if color:
+        c = OxmlElement('w:color')
+        c.set(qn('w:val'), color)
+        rPr.append(c)
+
+    if underline:
+        u = OxmlElement('w:u')
+        u.set(qn('w:val'), 'single')
+        rPr.append(u)
+
+    new_run.append(rPr)
+    new_run.text = text
+    hyperlink.append(new_run)
+
+    paragraph._p.append(hyperlink)
+    return hyperlink
+
+def add_table_of_contents(soup, doc):
+    toc = soup.find('ul')
+    if toc:
+        for li in toc.find_all('li'):
+            link = li.find('a')
+            if link and link['href'].startswith('#'):
+                heading_text = link.text
+                toc_paragraph = doc.add_paragraph()
+                add_hyperlink(toc_paragraph, f'#{heading_text}', heading_text)
+
+def add_markdown_to_docx(md_content, doc, base_path):
+    html = markdown.markdown(md_content)
+    soup = BeautifulSoup(html, 'html.parser')
+
+    heading_map = {}
+    toc_inserted = False
+
+    for element in soup:
+        if element.name == 'h1':
+            paragraph = doc.add_heading(element.text, level=1)
+            heading_map[element.text] = paragraph
+        elif element.name == 'h2':
+            paragraph = doc.add_heading(element.text, level=2)
+            heading_map[element.text] = paragraph
+            if element.text.lower() == 'table of contents' and not toc_inserted:
+                add_table_of_contents(soup, doc)
+                toc_inserted = True
+        elif element.name == 'h3':
+            paragraph = doc.add_heading(element.text, level=3)
+            heading_map[element.text] = paragraph
+        elif element.name == 'p':
+            paragraph = doc.add_paragraph(element.text)
+            for img in element.find_all('img'):
+                img_src = img['src'].lstrip('./')
+                img_path = os.path.join(base_path, img_src)
+                if os.path.exists(img_path):
+                    doc.add_picture(img_path, width=Inches(5.0))
+                else:
+                    paragraph.add_run(f"[Image not found: {img_path}]")
+        elif element.name == 'ul' and not toc_inserted:
+            for li in element.find_all('li'):
+                doc.add_paragraph(li.text, style='ListBullet')
+        elif element.name == 'ol':
+            for li in element.find_all('li'):
+                doc.add_paragraph(li.text, style='ListNumber')
+        elif element.name == 'a':
+            paragraph = doc.add_paragraph()
+            add_hyperlink(paragraph, element['href'], element.text)
+
+    for heading_text, paragraph in heading_map.items():
+        bookmark = OxmlElement('w:bookmarkStart')
+        bookmark.set(qn('w:id'), str(hash(heading_text)))
+        bookmark.set(qn('w:name'), heading_text)
+        paragraph._p.insert(0, bookmark)
+        bookmark_end = OxmlElement('w:bookmarkEnd')
+        bookmark_end.set(qn('w:id'), str(hash(heading_text)))
+        paragraph._p.append(bookmark_end)
+
+def convert_readme_to_docx(readme_dir, output_path):
+    readme_path = os.path.join(readme_dir, 'README.md')
+    if not os.path.exists(readme_path):
+        print(f"README.md not found in {readme_dir}")
+        return
+
+    with open(readme_path, 'r') as file:
+        md_content = file.read()
+
+    doc = Document()
+    add_markdown_to_docx(md_content, doc, readme_dir)
+    doc.save(output_path)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Convert a README.md file to a DOCX file.')
+    parser.add_argument('readme_dir', type=str, help='Directory containing the README.md file')
+    args = parser.parse_args()
+
+    readme_dir = args.readme_dir
+    output_path = os.path.join(readme_dir, 'README.docx')
+    convert_readme_to_docx(readme_dir, output_path)
+    print(f"Converted README.md in {readme_dir} to {output_path}")

hack/scripts/create-s3-bucket.sh

@@ -2,8 +2,8 @@
 
 set -eou pipefail
 
+export AWS_PAGER=""
 BUCKET_NAME=$1
-
 REGION=$(aws configure get region)
 
 create_bucket() {

hack/scripts/generate_report.py

@@ -0,0 +1,87 @@
+import json
+import matplotlib.pyplot as plt
+from jinja2 import Template
+import os
+import argparse
+import yaml
+
+# Parse command-line arguments
+parser = argparse.ArgumentParser(description='Generate benchmark report from measurements.json')
+parser.add_argument('dir_paths', type=str, nargs='+', help='Paths to the directories containing the measurements.json files')
+args = parser.parse_args()
+
+# Function to load benchmark description from benchmark.yaml
+def load_benchmark_description(dir_path):
+    yaml_path = os.path.join(dir_path, 'benchmark.yaml')
+    with open(yaml_path) as f:
+        benchmark_data = yaml.safe_load(f)
+    return benchmark_data.get('scenarios', {}).get('ingestionPath', {}).get('description', 'Unknown Benchmark')
+
+# Function to plot a measurement and save as image
+def plot_measurement(measurements, output_dir, plot_index):
+    plt.figure(figsize=(10, 6))
+    for measurement, description in measurements:
+        name = measurement['Name']
+        values = measurement['Values']
+        units = measurement['Units']
+        annotations = measurement.get('Annotations', [])
+
+        # Generate time values for x-axis starting from 3 minutes
+        time_values = [(i + 1) * 3 for i in range(len(values))]
+
+        plt.plot(time_values, values, marker='o', label=description)
+
+    plt.title(f'{name}')
+    plt.xlabel('Time (minutes)')
+    plt.ylabel(f'{units}')
+    plt.legend()
+    plt.grid(True)
+
+    plot_filename = os.path.join(output_dir, f'plot_{plot_index}.png')
+    plt.savefig(plot_filename)
+    plt.close()
+
+    return f'./plots/plot_{plot_index}.png', f'{name}'
+
+# Collect all measurements from the provided directories
+all_measurements = {}
+for dir_path in args.dir_paths:
+    json_path = os.path.join(dir_path, 'measurements.json')
+    with open(json_path) as f:
+        data = json.load(f)
+
+    benchmark_description = load_benchmark_description(dir_path)
+    measurements = data[0]['Measurements']
+
+    for measurement in measurements:
+        name = measurement['Name']
+        if name not in all_measurements:
+            all_measurements[name] = []
+        all_measurements[name].append((measurement, benchmark_description))
+
+# Determine the parent directory for the README and plots
+parent_dir = os.path.commonpath(args.dir_paths)
+output_dir = os.path.join(parent_dir, 'plots')
+os.makedirs(output_dir, exist_ok=True)
+
+# Plot all measurements and save images
+plot_files = []
+for plot_index, (name, measurements) in enumerate(all_measurements.items()):
+    plot_file, plot_title = plot_measurement(measurements, output_dir, plot_index)
+    plot_files.append((plot_title, plot_file))
+
+# Load README template
+template_path = 'reports/README.template'
+with open(template_path) as f:
+    template_content = f.read()
+
+# Render README with plots
+template = Template(template_content)
+rendered_readme = template.render(plots=plot_files)
+
+# Save rendered README
+readme_path = os.path.join(parent_dir, 'README.md')
+with open(readme_path, 'w') as f:
+    f.write(rendered_readme)
+
+print(f"Plots and README.md generated successfully in {parent_dir}.")

internal/config/config.go

@@ -42,6 +42,7 @@ type Jobs struct {
 	Querier       string `yaml:"querier"`
 	QueryFrontend string `yaml:"queryFrontend"`
 	IndexGateway  string `yaml:"indexGateway"`
+	Gateway       string `yaml:"gateway"`
 }
 
 type Scenarios struct {

internal/metrics/client.go

@@ -110,17 +110,17 @@ func (c *Client) MeasureGRPCRequestMetrics(
 	}
 }
 
-func (c *Client) MeasureBoltDBShipperRequestMetrics(
+func (c *Client) MeasureIndexRequestMetrics(
 	e *gmeasure.Experiment,
 	path RequestPath,
 	job string,
 	sampleRange model.Duration,
 ) error {
 	switch path {
 	case WriteRequestPath:
-		return c.Measure(e, RequestBoltDBShipperRequestRate(BoltDBShipperWriteName, job, BoltDBWriteOperation, "2.*", sampleRange))
+		return c.Measure(e, RequestIndexRequestRate(IndexWriteName, job, WriteOperation, "2.*", sampleRange))
 	case ReadRequestPath:
-		return c.Measure(e, RequestBoltDBShipperRequestRate(BoltDBShipperReadName, job, BoltDBReadOperation, "2.*", sampleRange))
+		return c.Measure(e, RequestIndexRequestRate(IndexReadName, job, ReadOperation, "2.*", sampleRange))
 	default:
 		return fmt.Errorf("error unknown path specified: %d", path)
 	}
@@ -140,6 +140,10 @@ func (c *Client) MeasureResourceUsageMetrics(
 		if err := c.Measure(e, ContainerMemoryWorkingSetBytes(job, sampleRange, annotation)); err != nil {
 			return err
 		}
+
+		if err := c.Measure(e, ContainerGoMemstatsHeapInuse(job, sampleRange, annotation)); err != nil {
+			return err
+		}
 	}
 
 	return nil
@@ -245,29 +249,39 @@ func (c *Client) measureCommonRequestMetrics(
 	sampleRange model.Duration,
 	annotation gmeasure.Annotation,
 ) error {
-	var name, code, requestRateName string
+	var name, code, badCode, requestRateName, badRequestRateName string
 
 	if method == GRPCMethod {
-		name = fmt.Sprintf("successful GRPC %s", route)
+		name = fmt.Sprintf("%s successful GRPC %s", annotation, route)
 		code = "success"
-
 		requestRateName = name
 		if pathRoutes == GRPCReadPathRoutes {
 			requestRateName = "successful GRPC reads"
 		}
 	} else {
-		name = fmt.Sprintf("2xx %s", route)
+		name = fmt.Sprintf("%s 2xx %s", annotation, route)
 		code = "2.*"
-
 		requestRateName = name
 		if pathRoutes == HTTPReadPathRoutes {
 			requestRateName = "2xx reads"
 		}
+
+		badCode = "5.*"
+		badRequestRateName = fmt.Sprintf("%s 5xx %s", annotation, route)
+		if pathRoutes == HTTPReadPathRoutes {
+			badRequestRateName = "5xx reads"
+		}
 	}
 
+	// Rate request of 200 or success
 	if err := c.Measure(e, RequestRate(requestRateName, job, pathRoutes, code, sampleRange, annotation)); err != nil {
 		return err
 	}
+	if method != GRPCMethod {
+		if err := c.Measure(e, RequestRate(badRequestRateName, job, pathRoutes, badCode, sampleRange, annotation)); err != nil {
+			return err
+		}
+	}
 	if err := c.Measure(e, RequestDurationAverage(name, job, method, route, code, sampleRange, annotation)); err != nil {
 		return err
 	}