Qwen3 VLM: handle scenario specifc latency setting and dataset sample count to submission checker

tools/submission/submission_checker/checks/accuracy_check.py

@@ -113,7 +113,7 @@ def accuracy_result_check(self):
             return False
 
         patterns, acc_targets, acc_types, acc_limits, up_patterns, acc_upper_limit = self.config.get_accuracy_values(
-            self.model
+            self.model, self.scenario_fixed
         )
         acc = None
         hash_val = None
@@ -242,7 +242,7 @@ def dataset_check(self):
             )
             return True
         expected_qsl_total_count = self.config.get_accuracy_sample_count(
-            self.model)
+            self.model, self.scenario_fixed)
         if "effective_accuracy_sample_count" in self.mlperf_log.get_keys():
             qsl_total_count = self.mlperf_log["effective_accuracy_sample_count"]
         else:

tools/submission/submission_checker/checks/compliance_check.py

@@ -255,9 +255,9 @@ def accuracy_check(self):
                             is_valid = False
                         else:
                             target = self.config.get_accuracy_target(
-                                self.model)
+                                self.model, self.submission_logs.loader_data.get("scenario", ""))
                             patterns, acc_targets, acc_types, acc_limits, up_patterns, acc_upper_limit = self.config.get_accuracy_values(
-                                self.model)
+                                self.model, self.submission_logs.loader_data.get("scenario", ""))
                             acc_limit_check = True
 
                             acc_seen = [False for _ in acc_targets]

tools/submission/submission_checker/checks/performance_check.py

@@ -56,10 +56,19 @@ def __init__(self, log, path, config: Config,
         self.is_endpoints = self.submission_logs.loader_data.get(
             "is_endpoints_submission", False)
         if self.is_endpoints:
-            if self.scenario.lower() == "online":
-                self.scenario = "Server"
+            self.scenario = self._normalize_endpoints_scenario(self.scenario)
         self.setup_checks()
 
+    def _normalize_endpoints_scenario(self, scenario):
+        if str(scenario).lower() == "online":
+            # endpoint doesn't distinguish interactive mode and server mode, they 
+            # are all categorized into online mode, but in mlperf submission,
+            # the rule distinguish Server and Interactive
+            if str(self.scenario_fixed).lower() == "interactive":
+                return "Interactive"
+            return "Server"
+        return "Offline"
+
     def setup_checks(self):
         """Register individual performance-related checks.
 
@@ -521,9 +530,7 @@ def get_performance_metric_check(self):
             is_valid = True
         scenario = self.mlperf_log["effective_scenario"]
         if self.is_endpoints:
-            if scenario.lower() == "online":
-                scenario = "Server"
-            scenario = scenario.capitalize()
+            scenario = self._normalize_endpoints_scenario(scenario)
 
         res = float(self.mlperf_log[RESULT_FIELD_NEW[version][scenario]])
         if (

tools/submission/submission_checker/configuration/configuration.py

@@ -120,23 +120,28 @@ def get_optional(self, model):
             return set()
         return set(self.optional[model])
 
-    def get_accuracy_target(self, model):
+    def get_accuracy_target(self, model, scenario=None):
         if model not in self.accuracy_target:
             raise ValueError("model not known: " + model)
+        if model == "qwen3-vl-235b-a22b":
+            # qwen model specically has offline, server using 1 threshold, interactive uses another threshold
+            # due to smaller dataset, hence accuracy target need both model and scenario to retrive the correct one
+            assert scenario in self.accuracy_target[model].keys()
+            return self.accuracy_target[model][scenario]
         return self.accuracy_target[model]
 
     def get_accuracy_upper_limit(self, model):
         return self.accuracy_upper_limit.get(model, None)
 
-    def get_accuracy_values(self, model):
+    def get_accuracy_values(self, model, scenario=None):
         patterns = []
         acc_targets = []
         acc_types = []
         acc_limits = []
         up_patterns = []
         acc_limit_check = False
 
-        target = self.get_accuracy_target(model)
+        target = self.get_accuracy_target(model, scenario)
         acc_upper_limit = self.get_accuracy_upper_limit(model)
         if acc_upper_limit is not None:
             for i in range(0, len(acc_upper_limit), 2):
@@ -158,8 +163,11 @@ def get_performance_sample_count(self, model):
             raise ValueError("model not known: " + model)
         return self.performance_sample_count[model]
 
-    def get_accuracy_sample_count(self, model):
+    def get_accuracy_sample_count(self, model, scenario=None):
         model = self.get_mlperf_model(model)
+        if model == "qwen3-vl-235b-a22b":
+            assert scenario in self.accuracy_sample_count[model].keys()
+            return self.accuracy_sample_count[model][scenario]
         if model not in self.accuracy_sample_count:
             return self.get_dataset_size(model)
         return self.accuracy_sample_count[model]

tools/submission/submission_checker/constants.py

@@ -148,7 +148,12 @@
             "deepseek-r1": ("exact_match", 0.99 * 81.3582, "TOKENS_PER_SAMPLE", 0.9 * 3886.2274),
             "whisper": ("ACCURACY", (100.0 - 2.0671) * 0.99),
             "gpt-oss-120b": ("exact_match", 83.13 * 0.99),
-            "qwen3-vl-235b-a22b": ("F1_HIERARCHICAL", 0.7903 * 0.99),
+            "qwen3-vl-235b-a22b": {
+                # offline and server use large dataset, interactive scenario uses small dataset, hence score is different
+                "Offline": ("F1_HIERARCHICAL", 0.7903 * 0.99),
+                "Server": ("F1_HIERARCHICAL", 0.7903 * 0.99),
+                "Interactive": ("F1_HIERARCHICAL", 0.78777 * 0.99),
+            },
             "dlrm-v3": (
                 "DLRM_NE",
                 0.86687 * 0.999,
@@ -207,6 +212,11 @@
         },
         "accuracy-sample-count": {
             "gpt-oss-120b": 4395,
+            "qwen3-vl-235b-a22b": {
+                "Offline": 48289,
+                "Server": 48289,
+                "Interactive": 8000,
+            },
             "wan-2.2-t2v-a14b": 248,
         },
         "dataset-size": {
@@ -257,7 +267,10 @@
             "llama2-70b-99.9": {"Server": 20000000000},
             "deepseek-r1": {"Server": 60000000000},
             "gpt-oss-120b": {"Server": 60000000000},
-            "qwen3-vl-235b-a22b": {"Server": 60000000000},
+            "qwen3-vl-235b-a22b": {
+                "Server": 12000000000,
+                "Interactive": 1500000000,
+            },
             "dlrm-v3": {"Server": 60000000000},
         },
         "min-queries": {
@@ -512,8 +525,11 @@
             "deepseek-r1": ("exact_match", 0.99 * 81.3582, "TOKENS_PER_SAMPLE", 0.9 * 3886.2274),
             "whisper": ("ACCURACY", (100.0 - 2.0671) * 0.99),
             "gpt-oss-120b": ("exact_match", 83.13 * 0.99),
-            # TODO: Placeholder for now
-            "qwen3-vl-235b-a22b": ("F1_HIERARCHICAL", 0.7903 * 0.99),
+            "qwen3-vl-235b-a22b": {
+                "Offline": ("F1_HIERARCHICAL", 0.7903 * 0.99),
+                "Server": ("F1_HIERARCHICAL", 0.7903 * 0.99),
+                "Interactive": ("F1_HIERARCHICAL", 0.78777 * 0.99),
+            },
             "dlrm-v3": (
                 "DLRM_NE",
                 0.86687 * 0.999,
@@ -1573,6 +1589,7 @@
         "SingleStream": "early_stopping_latency_ss",
         "MultiStream": "early_stopping_latency_ms",
         "Server": "result_completed_samples_per_sec",
+        "Interactive": "result_completed_samples_per_sec",
     },
 }