Skip to content

Commit ffa5ab6

Browse files
vimalk78vimalk78
authored
feat(compose): sampling vs probing (#1719)
Signed-off-by: Vimal Kumar <[email protected]>
1 parent c56c2fc commit ffa5ab6

File tree

8 files changed

+299
-0
lines changed

8 files changed

+299
-0
lines changed

‎manifests/compose/sampling/compose.yaml

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
name: sampling
2+
include:
3+
- path:
4+
- ../monitoring/compose.yaml
5+
- ./override.yaml
6+
7+
services:
8+
### 📦 kepler created from the current repo with sampling enabled
9+
kepler-sampling:
10+
build:
11+
context: ../../../
12+
dockerfile: build/Dockerfile
13+
args:
14+
INSTALL_DCGM: false
15+
INSTALL_HABANA: false
16+
17+
ports:
18+
# NOTE: use 9888 to keep the host 8888 port free for any local testing
19+
# e.g. sudo ./bin/kepler
20+
- 9188:8888
21+
privileged: true
22+
pid: host
23+
depends_on:
24+
- prometheus
25+
26+
volumes:
27+
- type: bind
28+
source: /proc
29+
target: /proc
30+
- type: bind
31+
source: /sys
32+
target: /sys
33+
- type: bind
34+
source: ./sampling/kepler/etc/kepler
35+
target: /etc/kepler
36+
37+
# NOTE: use the weights from the local repo
38+
- type: bind
39+
source: ../../../data
40+
target: /var/lib/kepler/data
41+
42+
entrypoint: [/usr/bin/bash, -c]
43+
44+
command:
45+
- |
46+
echo Starting kepler with sampling;
47+
# NOTE: uncomment to wait for estimator container
48+
# echo waiting for estimator socket to be ready;
49+
# until [[ -e /tmp/estimator.sock ]]; do
50+
# echo " ... waiting for socket";
51+
# sleep 1;
52+
# done;
53+
set -x;
54+
/usr/bin/kepler \
55+
-address "0.0.0.0:8888" \
56+
-v "8" \
57+
-enable-gpu=false
58+
networks:
59+
- kepler-network
60+
cap_add:
61+
- ALL
62+
63+
### 📦 kepler created from the current repo without sampling enabled
64+
kepler-probing:
65+
build:
66+
context: ../../../
67+
dockerfile: build/Dockerfile
68+
args:
69+
INSTALL_DCGM: false
70+
INSTALL_HABANA: false
71+
72+
ports:
73+
# NOTE: use 9888 to keep the host 8888 port free for any local testing
74+
# e.g. sudo ./bin/kepler
75+
- 9288:8888
76+
privileged: true
77+
pid: host
78+
depends_on:
79+
- prometheus
80+
81+
volumes:
82+
- type: bind
83+
source: /proc
84+
target: /proc
85+
- type: bind
86+
source: /sys
87+
target: /sys
88+
- type: bind
89+
source: ./probing/kepler/etc/kepler
90+
target: /etc/kepler
91+
92+
# NOTE: use the weights from the local repo
93+
- type: bind
94+
source: ../../../data
95+
target: /var/lib/kepler/data
96+
97+
entrypoint: [/usr/bin/bash, -c]
98+
99+
command:
100+
- |
101+
echo Starting kepler with probing;
102+
# NOTE: uncomment to wait for estimator container
103+
# echo waiting for estimator socket to be ready;
104+
# until [[ -e /tmp/estimator.sock ]]; do
105+
# echo " ... waiting for socket";
106+
# sleep 1;
107+
# done;
108+
set -x;
109+
/usr/bin/kepler \
110+
-address "0.0.0.0:8888" \
111+
-v "8" \
112+
-enable-gpu=false
113+
networks:
114+
- kepler-network
115+
cap_add:
116+
- ALL
117+
118+
networks:
119+
kepler-network:

‎manifests/compose/sampling/grafana/dashboards/sampling/dashboard.json

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
{
2+
"annotations": {
3+
"list": [
4+
{
5+
"builtIn": 1,
6+
"datasource": {
7+
"type": "grafana",
8+
"uid": "-- Grafana --"
9+
},
10+
"enable": true,
11+
"hide": true,
12+
"iconColor": "rgba(0, 211, 255, 1)",
13+
"name": "Annotations & Alerts",
14+
"type": "dashboard"
15+
}
16+
]
17+
},
18+
"editable": true,
19+
"fiscalYearStartMonth": 0,
20+
"graphTooltip": 0,
21+
"id": 1,
22+
"links": [],
23+
"panels": [
24+
{
25+
"datasource": {
26+
"type": "prometheus",
27+
"uid": "PDE6745920139CE56"
28+
},
29+
"fieldConfig": {
30+
"defaults": {
31+
"color": {
32+
"mode": "palette-classic"
33+
},
34+
"custom": {
35+
"axisBorderShow": false,
36+
"axisCenteredZero": false,
37+
"axisColorMode": "text",
38+
"axisLabel": "",
39+
"axisPlacement": "auto",
40+
"barAlignment": 0,
41+
"drawStyle": "line",
42+
"fillOpacity": 0,
43+
"gradientMode": "none",
44+
"hideFrom": {
45+
"legend": false,
46+
"tooltip": false,
47+
"viz": false
48+
},
49+
"insertNulls": false,
50+
"lineInterpolation": "linear",
51+
"lineWidth": 1,
52+
"pointSize": 1,
53+
"scaleDistribution": {
54+
"type": "linear"
55+
},
56+
"showPoints": "auto",
57+
"spanNulls": false,
58+
"stacking": {
59+
"group": "A",
60+
"mode": "none"
61+
},
62+
"thresholdsStyle": {
63+
"mode": "off"
64+
}
65+
},
66+
"mappings": [],
67+
"thresholds": {
68+
"mode": "absolute",
69+
"steps": [
70+
{
71+
"color": "green",
72+
"value": null
73+
},
74+
{
75+
"color": "red",
76+
"value": 80
77+
}
78+
]
79+
}
80+
},
81+
"overrides": []
82+
},
83+
"gridPos": {
84+
"h": 8,
85+
"w": 12,
86+
"x": 0,
87+
"y": 0
88+
},
89+
"id": 1,
90+
"options": {
91+
"legend": {
92+
"calcs": [],
93+
"displayMode": "list",
94+
"placement": "bottom",
95+
"showLegend": true
96+
},
97+
"tooltip": {
98+
"mode": "single",
99+
"sort": "none"
100+
}
101+
},
102+
"targets": [
103+
{
104+
"datasource": {
105+
"type": "prometheus",
106+
"uid": "PDE6745920139CE56"
107+
},
108+
"editorMode": "code",
109+
"expr": "sum(rate(kepler_process_bpf_cpu_time_ms_total{job=\"probing\"}[$__rate_interval]))",
110+
"instant": false,
111+
"legendFormat": "__auto",
112+
"range": true,
113+
"refId": "A"
114+
},
115+
{
116+
"datasource": {
117+
"type": "prometheus",
118+
"uid": "PDE6745920139CE56"
119+
},
120+
"editorMode": "code",
121+
"expr": "sum(rate(kepler_process_bpf_cpu_time_ms_total{job=\"sampling\"}[$__rate_interval]))",
122+
"hide": false,
123+
"instant": false,
124+
"legendFormat": "__auto",
125+
"range": true,
126+
"refId": "B"
127+
}
128+
],
129+
"title": "sampling vs probing bpf cpu time, SAMPLING RATE = 1000",
130+
"type": "timeseries"
131+
}
132+
],
133+
"schemaVersion": 39,
134+
"tags": [],
135+
"templating": {
136+
"list": []
137+
},
138+
"time": {
139+
"from": "now-30m",
140+
"to": "now"
141+
},
142+
"timepicker": {},
143+
"timezone": "browser",
144+
"title": "sampling vs probing",
145+
"uid": "bdvhzahkl46psd",
146+
"version": 1,
147+
"weekStart": ""
148+
}

‎manifests/compose/sampling/override.yaml

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
services:
2+
grafana:
3+
environment:
4+
GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/sampling/dashboard.json
5+
6+
volumes:
7+
- type: bind
8+
source: ../sampling/grafana/dashboards/sampling/
9+
target: /var/lib/grafana/dashboards/sampling
10+
11+
prometheus:
12+
volumes:
13+
- type: bind
14+
source: ../sampling/sampling/prometheus/scrape-configs/sampling.yaml
15+
target: /etc/prometheus/scrape-configs/sampling.yaml
16+
- type: bind
17+
source: ../sampling/probing/prometheus/scrape-configs/probing.yaml
18+
target: /etc/prometheus/scrape-configs/probing.yaml
19+
20+
networks:
21+
- kepler-network

‎manifests/compose/sampling/probing/kepler/etc/kepler/kepler.config/ENABLE_PROCESS_METRICS

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
true

‎manifests/compose/sampling/probing/prometheus/scrape-configs/probing.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
scrape_configs:
2+
- job_name: probing
3+
static_configs:
4+
- targets: [kepler-probing:8888]

‎manifests/compose/sampling/sampling/kepler/etc/kepler/kepler.config/ENABLE_PROCESS_METRICS

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
true

‎manifests/compose/sampling/sampling/kepler/etc/kepler/kepler.config/EXPERIMENTAL_BPF_SAMPLE_RATE

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
1000

‎manifests/compose/sampling/sampling/prometheus/scrape-configs/sampling.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
scrape_configs:
2+
- job_name: sampling
3+
static_configs:
4+
- targets: [kepler-sampling:8888]

0 commit comments

Comments
 (0)