Skip to content

Commit afda7c0

Browse files
committed
Add a kvcache example deployment
- Enables redis and ENABLE_KVCACHE_AWARE_SCORER Signed-off-by: Brent Salisbury <[email protected]>
1 parent 65433e3 commit afda7c0

File tree

2 files changed

+64
-0
lines changed

2 files changed

+64
-0
lines changed
Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
sampleApplication:
2+
baseConfigMapRefName: basic-gpu-with-nixl-and-redis-lookup-preset
3+
model:
4+
modelArtifactURI: hf://meta-llama/Llama-3.2-3B-Instruct
5+
modelName: "meta-llama/Llama-3.2-3B-Instruct"
6+
redis:
7+
enabled: true
8+
modelservice:
9+
epp:
10+
defaultEnvVarsOverride:
11+
- name: ENABLE_KVCACHE_AWARE_SCORER
12+
value: "true"
13+
- name: ENABLE_PREFIX_AWARE_SCORER
14+
value: "false"
15+
- name: ENABLE_LOAD_AWARE_SCORER
16+
value: "true"
17+
- name: ENABLE_SESSION_AWARE_SCORER
18+
value: "false"
19+
- name: PD_ENABLED
20+
value: "false"
21+
- name: PD_PROMPT_LEN_THRESHOLD
22+
value: "10"
23+
- name: PREFILL_ENABLE_KVCACHE_AWARE_SCORER
24+
value: "false"
25+
- name: PREFILL_ENABLE_LOAD_AWARE_SCORER
26+
value: "false"
27+
- name: PREFILL_ENABLE_PREFIX_AWARE_SCORER
28+
value: "false"
29+
- name: PREFILL_ENABLE_SESSION_AWARE_SCORER
30+
value: "false"
Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
sampleApplication:
2+
baseConfigMapRefName: basic-gpu-with-nixl-and-redis-lookup-preset
3+
model:
4+
modelArtifactURI: hf://Qwen/Qwen3-0.6B
5+
modelName: "Qwen/Qwen3-0.6B"
6+
prefill:
7+
replicas: 0
8+
decode:
9+
replicas: 1
10+
redis:
11+
enabled: true
12+
modelservice:
13+
epp:
14+
defaultEnvVarsOverride:
15+
- name: ENABLE_KVCACHE_AWARE_SCORER
16+
value: "true"
17+
- name: ENABLE_PREFIX_AWARE_SCORER
18+
value: "false"
19+
- name: ENABLE_LOAD_AWARE_SCORER
20+
value: "true"
21+
- name: ENABLE_SESSION_AWARE_SCORER
22+
value: "false"
23+
- name: PD_ENABLED
24+
value: "false"
25+
- name: PD_PROMPT_LEN_THRESHOLD
26+
value: "10"
27+
- name: PREFILL_ENABLE_KVCACHE_AWARE_SCORER
28+
value: "false"
29+
- name: PREFILL_ENABLE_LOAD_AWARE_SCORER
30+
value: "false"
31+
- name: PREFILL_ENABLE_PREFIX_AWARE_SCORER
32+
value: "false"
33+
- name: PREFILL_ENABLE_SESSION_AWARE_SCORER
34+
value: "false"

0 commit comments

Comments
 (0)