diff --git a/tools/submission/submission_checker/checks/accuracy_check.py b/tools/submission/submission_checker/checks/accuracy_check.py index ac94679c74..f37ac062aa 100644 --- a/tools/submission/submission_checker/checks/accuracy_check.py +++ b/tools/submission/submission_checker/checks/accuracy_check.py @@ -113,7 +113,7 @@ def accuracy_result_check(self): return False patterns, acc_targets, acc_types, acc_limits, up_patterns, acc_upper_limit = self.config.get_accuracy_values( - self.model + self.model, self.scenario_fixed ) acc = None hash_val = None @@ -242,7 +242,7 @@ def dataset_check(self): ) return True expected_qsl_total_count = self.config.get_accuracy_sample_count( - self.model) + self.model, self.scenario_fixed) if "effective_accuracy_sample_count" in self.mlperf_log.get_keys(): qsl_total_count = self.mlperf_log["effective_accuracy_sample_count"] else: diff --git a/tools/submission/submission_checker/checks/compliance_check.py b/tools/submission/submission_checker/checks/compliance_check.py index e110af9c40..0ad6831ac0 100644 --- a/tools/submission/submission_checker/checks/compliance_check.py +++ b/tools/submission/submission_checker/checks/compliance_check.py @@ -255,9 +255,9 @@ def accuracy_check(self): is_valid = False else: target = self.config.get_accuracy_target( - self.model) + self.model, self.submission_logs.loader_data.get("scenario", "")) patterns, acc_targets, acc_types, acc_limits, up_patterns, acc_upper_limit = self.config.get_accuracy_values( - self.model) + self.model, self.submission_logs.loader_data.get("scenario", "")) acc_limit_check = True acc_seen = [False for _ in acc_targets] diff --git a/tools/submission/submission_checker/checks/performance_check.py b/tools/submission/submission_checker/checks/performance_check.py index ed8ad18bec..75bd2dd0eb 100644 --- a/tools/submission/submission_checker/checks/performance_check.py +++ b/tools/submission/submission_checker/checks/performance_check.py @@ -56,10 +56,19 @@ def __init__(self, log, path, config: Config, self.is_endpoints = self.submission_logs.loader_data.get( "is_endpoints_submission", False) if self.is_endpoints: - if self.scenario.lower() == "online": - self.scenario = "Server" + self.scenario = self._normalize_endpoints_scenario(self.scenario) self.setup_checks() + def _normalize_endpoints_scenario(self, scenario): + if str(scenario).lower() == "online": + # endpoint doesn't distinguish interactive mode and server mode, they + # are all categorized into online mode, but in mlperf submission, + # the rule distinguish Server and Interactive + if str(self.scenario_fixed).lower() == "interactive": + return "Interactive" + return "Server" + return "Offline" + def setup_checks(self): """Register individual performance-related checks. @@ -521,9 +530,7 @@ def get_performance_metric_check(self): is_valid = True scenario = self.mlperf_log["effective_scenario"] if self.is_endpoints: - if scenario.lower() == "online": - scenario = "Server" - scenario = scenario.capitalize() + scenario = self._normalize_endpoints_scenario(scenario) res = float(self.mlperf_log[RESULT_FIELD_NEW[version][scenario]]) if ( diff --git a/tools/submission/submission_checker/configuration/configuration.py b/tools/submission/submission_checker/configuration/configuration.py index 37abb6b802..46bf6a59e6 100644 --- a/tools/submission/submission_checker/configuration/configuration.py +++ b/tools/submission/submission_checker/configuration/configuration.py @@ -120,15 +120,20 @@ def get_optional(self, model): return set() return set(self.optional[model]) - def get_accuracy_target(self, model): + def get_accuracy_target(self, model, scenario=None): if model not in self.accuracy_target: raise ValueError("model not known: " + model) + if model == "qwen3-vl-235b-a22b": + # qwen model specically has offline, server using 1 threshold, interactive uses another threshold + # due to smaller dataset, hence accuracy target need both model and scenario to retrive the correct one + assert scenario in self.accuracy_target[model].keys() + return self.accuracy_target[model][scenario] return self.accuracy_target[model] def get_accuracy_upper_limit(self, model): return self.accuracy_upper_limit.get(model, None) - def get_accuracy_values(self, model): + def get_accuracy_values(self, model, scenario=None): patterns = [] acc_targets = [] acc_types = [] @@ -136,7 +141,7 @@ def get_accuracy_values(self, model): up_patterns = [] acc_limit_check = False - target = self.get_accuracy_target(model) + target = self.get_accuracy_target(model, scenario) acc_upper_limit = self.get_accuracy_upper_limit(model) if acc_upper_limit is not None: for i in range(0, len(acc_upper_limit), 2): @@ -158,8 +163,11 @@ def get_performance_sample_count(self, model): raise ValueError("model not known: " + model) return self.performance_sample_count[model] - def get_accuracy_sample_count(self, model): + def get_accuracy_sample_count(self, model, scenario=None): model = self.get_mlperf_model(model) + if model == "qwen3-vl-235b-a22b": + assert scenario in self.accuracy_sample_count[model].keys() + return self.accuracy_sample_count[model][scenario] if model not in self.accuracy_sample_count: return self.get_dataset_size(model) return self.accuracy_sample_count[model] diff --git a/tools/submission/submission_checker/constants.py b/tools/submission/submission_checker/constants.py index b4000a1d86..f0223f8270 100644 --- a/tools/submission/submission_checker/constants.py +++ b/tools/submission/submission_checker/constants.py @@ -148,7 +148,12 @@ "deepseek-r1": ("exact_match", 0.99 * 81.3582, "TOKENS_PER_SAMPLE", 0.9 * 3886.2274), "whisper": ("ACCURACY", (100.0 - 2.0671) * 0.99), "gpt-oss-120b": ("exact_match", 83.13 * 0.99), - "qwen3-vl-235b-a22b": ("F1_HIERARCHICAL", 0.7903 * 0.99), + "qwen3-vl-235b-a22b": { + # offline and server use large dataset, interactive scenario uses small dataset, hence score is different + "Offline": ("F1_HIERARCHICAL", 0.7903 * 0.99), + "Server": ("F1_HIERARCHICAL", 0.7903 * 0.99), + "Interactive": ("F1_HIERARCHICAL", 0.78777 * 0.99), + }, "dlrm-v3": ( "DLRM_NE", 0.86687 * 0.999, @@ -207,6 +212,11 @@ }, "accuracy-sample-count": { "gpt-oss-120b": 4395, + "qwen3-vl-235b-a22b": { + "Offline": 48289, + "Server": 48289, + "Interactive": 8000, + }, "wan-2.2-t2v-a14b": 248, }, "dataset-size": { @@ -257,7 +267,10 @@ "llama2-70b-99.9": {"Server": 20000000000}, "deepseek-r1": {"Server": 60000000000}, "gpt-oss-120b": {"Server": 60000000000}, - "qwen3-vl-235b-a22b": {"Server": 60000000000}, + "qwen3-vl-235b-a22b": { + "Server": 12000000000, + "Interactive": 1500000000, + }, "dlrm-v3": {"Server": 60000000000}, }, "min-queries": { @@ -512,8 +525,11 @@ "deepseek-r1": ("exact_match", 0.99 * 81.3582, "TOKENS_PER_SAMPLE", 0.9 * 3886.2274), "whisper": ("ACCURACY", (100.0 - 2.0671) * 0.99), "gpt-oss-120b": ("exact_match", 83.13 * 0.99), - # TODO: Placeholder for now - "qwen3-vl-235b-a22b": ("F1_HIERARCHICAL", 0.7903 * 0.99), + "qwen3-vl-235b-a22b": { + "Offline": ("F1_HIERARCHICAL", 0.7903 * 0.99), + "Server": ("F1_HIERARCHICAL", 0.7903 * 0.99), + "Interactive": ("F1_HIERARCHICAL", 0.78777 * 0.99), + }, "dlrm-v3": ( "DLRM_NE", 0.86687 * 0.999, @@ -1573,6 +1589,7 @@ "SingleStream": "early_stopping_latency_ss", "MultiStream": "early_stopping_latency_ms", "Server": "result_completed_samples_per_sec", + "Interactive": "result_completed_samples_per_sec", }, }