Merge branch 'master' into arav-patch-016

Author: pgmpablo157321

.github/workflows/auto-update-dev.yml

@@ -3,7 +3,7 @@ name: Auto-Update Dev Branches from Master
 on:
   push:
     branches:
-      - master  # Trigger workflow on commits to 'master' branch.
+      - master  # Trigger workflow on commits to 'master' branch
   workflow_dispatch: {}
 
 jobs:

.github/workflows/build_wheels.yml

@@ -1,5 +1,6 @@
 name: Build loadgen wheels and release them into PYPI
 
+
 on:
   release:
     types: [published]

.github/workflows/test-bert.yml

@@ -5,7 +5,7 @@ name: Test for MLPerf inference bert submission generation using MLC script auto
 
 on:
   pull_request:
-    branches: [ "master_off", "dev_off" ]
+    branches: [ "master", "dev" ]
     paths:
       - language/bert/**
       - tools/submission/**
@@ -22,8 +22,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [ "3.9" ]
-        backend: [ "deepsparse" ]
+        python-version: [ "3.10" ]
+        backend: [ "onnxruntime" ]
 
     steps:
     - uses: actions/checkout@v3
@@ -36,4 +36,4 @@ jobs:
         python3 -m pip install mlc-scripts
     - name: Test BERT and end to end submission generation
       run: |
-        mlcr run,mlperf,inference,generate-run-cmds,_submission,_short --quiet --submitter="MLCommons" --hw_name=default --model=bert-99 --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --adr.compiler.tags=gcc  --adr.inference-src.version=custom --adr.inference-src.tags=_repo.${{ github.event.pull_request.head.repo.html_url }},_branch.$PR_HEAD_REF --adr.inference-src-loadgen.tags=_branch.$PR_HEAD_REF,_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src-loadgen.version=custom --adr.loadgen.version=custom
+        mlcr run,mlperf,inference,generate-run-cmds,_submission,_short --quiet --submitter="MLCommons" --hw_name=default --model=bert-99 --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --adr.compiler.tags=gcc  --adr.inference-src.version=custom --adr.inference-src.tags=_repo.${{ github.event.pull_request.head.repo.html_url }},_branch.$PR_HEAD_REF --adr.inference-src-loadgen.tags=_branch.$PR_HEAD_REF,_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src-loadgen.version=custom --adr.loadgen.version=custom 

.gitmodules

@@ -10,6 +10,6 @@
 [submodule "language/deepseek-r1/submodules/LiveCodeBench"]
 	path = language/deepseek-r1/submodules/LiveCodeBench
 	url = https://github.com/LiveCodeBench/LiveCodeBench
-[submodule "text_to_video/wan2.2-t2v-14b/submodules/VBench"]
-	path = text_to_video/wan2.2-t2v-14b/submodules/VBench
+[submodule "text_to_video/wan-2.2-t2v-14b/submodules/VBench"]
+	path = text_to_video/wan-2.2-t2v-a14b/submodules/VBench
 	url = https://github.com/Vchitect/VBench

README.md

@@ -17,7 +17,7 @@ Please see the [MLPerf Inference benchmark paper](https://arxiv.org/abs/1911.025
 
 Please see [here](https://docs.mlcommons.org/inference/benchmarks/) for the MLPerf inference documentation website which includes automated commands to run MLPerf inference benchmarks using different implementations.
 
-## MLPerf Inference v6.0 (submission deadline February 13, 2025)
+## MLPerf Inference v6.0 (submission deadline February 13, 2026)
 
 For submissions, please use the master branch and any commit since the [6.0 seed release](https://github.com/mlcommons/inference/commit/f131a0d29ccae9a967d93ffe96f66b1be3537d3b) although it is best to use the latest commit in the [master branch](https://github.com/mlcommons/inference).
 

compliance/README.md

@@ -47,4 +47,4 @@ The `run_verification.py` found in each test directory will copy the test files
 | DeepSeek-r1 | [TEST06](./TEST06/) |
 | Whisper | [TEST01](./TEST01/) |
 | gpt-oss-120b | [TEST07](./TEST07/), [TEST09](./TEST09/) |
-| wan2.2-t2v-14b | [TEST01](./TEST01/), [TEST04](./TEST04/) |
+| wan2.2-t2v-14b | [TEST04](./TEST04/) |

…iance/TEST01/wan2.2-t2v-14b/audit.config …nce/TEST01/wan-2.2-t2v-a14b/audit.configcompliance/TEST01/wan2.2-t2v-14b/audit.config renamed to compliance/TEST01/wan-2.2-t2v-a14b/audit.config compliance/TEST01/wan2.2-t2v-14b/audit.config renamed to compliance/TEST01/wan-2.2-t2v-a14b/audit.config

@@ -19,6 +19,8 @@
 *.Server.performance_issue_same = 1
 *.Server.performance_issue_same_index = 3
 stable-diffusion-xl.Offline.min_query_count = 500
+wan-2.2-t2v-a14b.Offline.min_query_count = 20
+wan-2.2-t2v-a14b.SingleStream.min_query_count = 20
 # You can optionally set the target qps to match the expected query count 
 # in the min duration with the min_query_count. But take into account you
 # system expected qps, the min duration for this test is 10 minutes

compliance/TEST04/audit.config

@@ -12,9 +12,9 @@ This repository provides the config files and scripts to run and verify TEST07 -
 
 ## Applicable Benchmarks
 
-| Model | Accuracy Threshold | Score Pattern | Dataset Size |
-|-------|-------------------|---------------|--------------|
-| gpt-oss-120b | 60.698 | `'exact_match': <score>` | 990 |
+| Model        | Accuracy Threshold | Score Pattern            | Dataset Size |
+| ------------ | ------------------ | ------------------------ | ------------ |
+| gpt-oss-120b | 60.698             | `'exact_match': <score>` | 990          |
 
 ## Introduction
 
@@ -89,14 +89,18 @@ python3 run_verification.py \
 
 ### Example: gpt-oss-120b
 
+**Dataset:** Use `acc/acc_eval_compliance_gpqa.parquet` from the gpt-oss dataset download for TEST07 compliance runs.
+
+**Generation Config:** Use perf generation config (`max_output_len=10240`, `reasoning_effort=low`).
+
 ```bash
 python3 compliance/TEST07/run_verification.py \
     -c /path/to/compliance/run/logs/ \
     -o /path/to/submission/compliance/gpt-oss-120b/Offline \
     --audit-config compliance/TEST07/gpt-oss-120b/audit.config \
     --accuracy-script "python3 language/gpt-oss-120b/eval_mlperf_accuracy.py \
         --mlperf-log {accuracy_log} \
-        --reference-data /path/to/accuracy_eval_data.pkl \
+        --reference-data /path/to/acc/acc_eval_compliance_gpqa.parquet \
         --tokenizer openai/gpt-oss-120b"
 ```
 

compliance/TEST07/README.md

@@ -3,6 +3,7 @@
 This repository provides the config files and scripts to run and verify TEST09 - Verify output token length in performance mode for LLM workloads.
 
 # Table of Contents
+
 1. [Applicable Benchmarks](#applicable-benchmarks)
 2. [Introduction](#introduction)
 3. [Prerequisites](#prerequisites)
@@ -11,21 +12,21 @@ This repository provides the config files and scripts to run and verify TEST09 -
 
 ## Applicable Benchmarks
 
-| Model | Min Output Tokens | Max Output Tokens | Dataset Size | Notes |
-|-------|-------------------|-------------------|--------------|-------|
-| gpt-oss-120b | 1150.38 | 1406.02 | 4395 | ±10% of 1278.20 reference mean |
+| Model        | Min Output Tokens | Max Output Tokens | Dataset Size | Notes                          |
+| ------------ | ----------------- | ----------------- | ------------ | ------------------------------ |
+| gpt-oss-120b | 1150.38           | 1406.02           | 6396         | ±10% of 1278.20 reference mean |
 
 ## Introduction
 
 The purpose of this test is to ensure that models are generating outputs of expected length during performance runs. This prevents cheating by truncating outputs to artificially improve throughput metrics.
 
 **Key Verification:**
 
-| Metric | Description |
-|--------|-------------|
-| Mean output tokens | Average number of output tokens across all samples |
-| Min threshold | Benchmark-specific minimum - ensures outputs are not truncated |
-| Max threshold | Benchmark-specific maximum - ensures outputs are not artificially padded |
+| Metric             | Description                                                              |
+| ------------------ | ------------------------------------------------------------------------ |
+| Mean output tokens | Average number of output tokens across all samples                       |
+| Min threshold      | Benchmark-specific minimum - ensures outputs are not truncated           |
+| Max threshold      | Benchmark-specific maximum - ensures outputs are not artificially padded |
 
 The compliance thresholds are defined in the benchmark's `audit.config` file via the `test09_min_output_tokens` and `test09_max_output_tokens` fields. Each benchmark defines its own bounds based on the reference implementation.
 
@@ -51,7 +52,7 @@ The `audit.config` contains both LoadGen settings and the compliance thresholds:
 # LoadGen settings
 *.*.mode = 2
 *.*.accuracy_log_sampling_target = 10000
-*.*.min_query_count = 4395
+*.*.min_query_count = 6396
 ...
 
 # TEST09 Compliance Thresholds (read by run_verification.py)
@@ -83,18 +84,22 @@ python3 run_verification.py \
 
 **Arguments:**
 
-| Argument | Required | Description |
-|----------|----------|-------------|
-| `-c`, `--compliance_dir` | Yes | Path to compliance test logs (contains `mlperf_log_accuracy.json`) |
-| `-o`, `--output_dir` | Yes | Output directory for submission artifacts |
-| `--audit-config` | No* | Path to audit.config containing thresholds |
-| `--min-output-tokens` | No* | Override minimum threshold (CLI takes precedence) |
-| `--max-output-tokens` | No* | Override maximum threshold (CLI takes precedence) |
+| Argument                 | Required | Description                                                        |
+| ------------------------ | -------- | ------------------------------------------------------------------ |
+| `-c`, `--compliance_dir` | Yes      | Path to compliance test logs (contains `mlperf_log_accuracy.json`) |
+| `-o`, `--output_dir`     | Yes      | Output directory for submission artifacts                          |
+| `--audit-config`         | No\*     | Path to audit.config containing thresholds                         |
+| `--min-output-tokens`    | No\*     | Override minimum threshold (CLI takes precedence)                  |
+| `--max-output-tokens`    | No\*     | Override maximum threshold (CLI takes precedence)                  |
 
-*At least one of `--audit-config` or both `--min-output-tokens` and `--max-output-tokens` must be provided.
+\*At least one of `--audit-config` or both `--min-output-tokens` and `--max-output-tokens` must be provided.
 
 ### Example: gpt-oss-120b
 
+**Dataset:** Use `perf/perf_eval_ref.parquet` (the performance dataset) from the gpt-oss dataset download for TEST09 compliance runs.
+
+**Generation Config:** Use performance generation config as-is (`max_output_len=10240`, `reasoning_effort=low`). This test verifies mean output sequence length is within ±10% of reference (1278.20 tokens).
+
 ```bash
 python3 compliance/TEST09/run_verification.py \
     -c /path/to/compliance/run/logs/ \
@@ -118,14 +123,14 @@ Using thresholds:
 ================================================================================
 
 Parsing MLPerf accuracy log...
-Loaded 4395 entries as JSON array
+Loaded 6396 entries as JSON array
 
-Computing output token lengths for 4395 samples...
+Computing output token lengths for 6396 samples...
 
 ================================================================================
 Output Token Length Statistics
 ================================================================================
-Total samples: 4395
+Total samples: 6396
 Mean output tokens: 1278.20
 Min output tokens: 518
 Max output tokens: 3154
@@ -183,6 +188,7 @@ Create `compliance/TEST09/<benchmark>/audit.config`:
 ### 2. Update this README
 
 Add the benchmark to the "Applicable Benchmarks" table with:
+
 - Model name
 - Min output tokens threshold
 - Max output tokens threshold