-
Notifications
You must be signed in to change notification settings - Fork 276
/
Copy pathbenchmark_chatqna.yaml
112 lines (99 loc) · 3.33 KB
/
benchmark_chatqna.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
# Copyright (C) 2025 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
deploy:
device: gaudi
version: 1.2.0
modelUseHostPath: /mnt/models
HUGGINGFACEHUB_API_TOKEN: "" # mandatory
node: [1, 2, 4, 8]
namespace: ""
timeout: 1000 # timeout in seconds for services to be ready, default 30 minutes
interval: 5 # interval in seconds between service ready checks, default 5 seconds
services:
backend:
resources:
enabled: False
cores_per_instance: "16"
memory_capacity: "8000Mi"
replicaCount: [1, 2, 4, 8]
teirerank:
enabled: True
model_id: ""
resources:
enabled: False
cards_per_instance: 1
replicaCount: [1, 1, 1, 1]
tei:
model_id: ""
resources:
enabled: False
cores_per_instance: "80"
memory_capacity: "20000Mi"
replicaCount: [1, 2, 4, 8]
llm:
engine: vllm # or tgi
model_id: "meta-llama/Meta-Llama-3-8B-Instruct" # mandatory
replicaCount:
with_teirerank: [7, 15, 31, 63] # When teirerank.enabled is True
without_teirerank: [8, 16, 32, 64] # When teirerank.enabled is False
resources:
enabled: False
cards_per_instance: 1
model_params:
vllm: # VLLM specific parameters
batch_params:
enabled: True
max_num_seqs: [1, 2, 4, 8] # Each value triggers an LLM service upgrade
token_params:
enabled: False
max_input_length: ""
max_total_tokens: ""
max_batch_total_tokens: ""
max_batch_prefill_tokens: ""
tgi: # TGI specific parameters
batch_params:
enabled: True
max_batch_size: [1, 2, 4, 8] # Each value triggers an LLM service upgrade
token_params:
enabled: False
max_input_length: "1280"
max_total_tokens: "2048"
max_batch_total_tokens: "65536"
max_batch_prefill_tokens: "4096"
data-prep:
resources:
enabled: False
cores_per_instance: ""
memory_capacity: ""
replicaCount: [1, 1, 1, 1]
retriever-usvc:
resources:
enabled: False
cores_per_instance: "8"
memory_capacity: "8000Mi"
replicaCount: [1, 2, 4, 8]
redis-vector-db:
resources:
enabled: False
cores_per_instance: ""
memory_capacity: ""
replicaCount: [1, 1, 1, 1]
chatqna-ui:
replicaCount: [1, 1, 1, 1]
nginx:
replicaCount: [1, 1, 1, 1]
benchmark:
# http request behavior related fields
user_queries: [640]
concurrency: [128]
load_shape_type: "constant" # "constant" or "poisson"
poisson_arrival_rate: 1.0 # only used when load_shape_type is "poisson"
warmup_iterations: 10
seed: 1024
# workload, all of the test cases will run for benchmark
bench_target: [chatqnafixed, chatqna_qlist_pubmed] # specify the bench_target for benchmark
dataset: ["/home/sdp/upload_file.txt", "/home/sdp/pubmed_10000.txt"] # specify the absolute path to the dataset file
prompt: [10, 1000] # set the prompt length for the chatqna_qlist_pubmed workload, set to 10 for chatqnafixed workload
llm:
# specify the llm output token size
max_token_size: [128, 256]