Enable multi-image support benchmarking for serving

benchmarks/backend_request_func.py

@@ -362,16 +362,30 @@ async def async_request_openai_chat_completions(
     async with aiohttp.ClientSession(
         trust_env=True, timeout=AIOHTTP_TIMEOUT
     ) as session:
-        content = [{"type": "text", "text": request_func_input.prompt}]
-        if request_func_input.multi_modal_content:
-            content.append(request_func_input.multi_modal_content)
+        prompt_text = request_func_input.prompt
+        mm_content = request_func_input.multi_modal_content
+
+        content_list = [{"type": "text", "text": prompt_text}]
+
+        # Check if image exist
+        if mm_content:
+            if isinstance(mm_content, list):
+                # Multi Images exist. Extend the list with all images.
+                content_list.extend(mm_content)
+            elif isinstance(mm_content, dict):
+                # Only single-image case. Append the one image.
+                content_list.append(mm_content)
+
+            messages = [{"role": "user", "content": content_list}]
+        else:
+            # text-only
+            messages = [{"role": "user", "content": prompt_text}]
+
         payload = {
             "model": request_func_input.model_name
             if request_func_input.model_name
             else request_func_input.model,
-            "messages": [
-                {"role": "user", "content": content},
-            ],
+            "messages": messages,
             "temperature": 0.0,
             "max_completion_tokens": request_func_input.output_len,
             "stream": True,

benchmarks/benchmark_dataset.py

@@ -838,6 +838,62 @@ def sample(
         return sampled_requests
 
 
+# -----------------------------------------------------------------------------
+# MuirBenchDataset Implementation
+# -----------------------------------------------------------------------------
+
+
+class MuirBenchDataset(HuggingFaceDataset):
+    """
+    MUIRBENCH Dataset.
+    https://huggingface.co/datasets/MUIRBENCH/MUIRBENCH
+    """
+
+    DEFAULT_OUTPUT_LEN = 128
+    SUPPORTED_DATASET_PATHS = {
+        "MUIRBENCH/MUIRBENCH": lambda x: x["question"],
+    }
+    IS_MULTIMODAL = True
+
+    def sample(
+        self,
+        tokenizer: PreTrainedTokenizerBase,
+        num_requests: int,
+        output_len: Optional[int] = None,
+        enable_multimodal_chat: bool = False,
+        **kwargs,
+    ) -> list:
+        output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
+        sampled_requests = []
+
+        for item in self.data:
+            if len(sampled_requests) >= num_requests:
+                break
+            parser_fn = self.SUPPORTED_DATASET_PATHS.get(self.dataset_path)
+            if parser_fn is None:
+                raise ValueError(f"Unsupported dataset path: {self.dataset_path}")
+
+            prompt = parser_fn(item)
+            prompt_len = len(tokenizer(prompt).input_ids)
+
+            mm_content = [process_image(img) for img in item["image_list"]]
+
+            if enable_multimodal_chat:
+                prompt = self.apply_multimodal_chat_transformation(prompt, mm_content)
+
+            sampled_requests.append(
+                SampleRequest(
+                    prompt=prompt,
+                    prompt_len=prompt_len,
+                    expected_output_len=output_len,
+                    multi_modal_data=mm_content,
+                )
+            )
+
+        self.maybe_oversample_requests(sampled_requests, num_requests)
+        return sampled_requests
+
+
 # -----------------------------------------------------------------------------
 # Instruct Coder Dataset Implementation
 # -----------------------------------------------------------------------------

benchmarks/benchmark_serving.py

@@ -65,6 +65,7 @@
     HuggingFaceDataset,
     InstructCoderDataset,
     MTBenchDataset,
+    MuirBenchDataset,
     NextEditPredictionDataset,
     RandomDataset,
     SampleRequest,
@@ -357,7 +358,7 @@ async def benchmark(
         input_requests[0].multi_modal_data,
     )
 
-    assert test_mm_content is None or isinstance(test_mm_content, dict)
+    assert test_mm_content is None or isinstance(test_mm_content, (dict, list))
     test_input = RequestFuncInput(
         model=model_id,
         model_name=model_name,
@@ -795,6 +796,9 @@ def main(args: argparse.Namespace):
         elif args.dataset_path in ASRDataset.SUPPORTED_DATASET_PATHS:
             dataset_class = ASRDataset
             args.hf_split = "train"
+        elif args.dataset_path in MuirBenchDataset.SUPPORTED_DATASET_PATHS:
+            dataset_class = MuirBenchDataset
+            args.hf_split = "test"
         else:
             supported_datasets = set(
                 [