Initial ScanOSS Licensing Checker

.github/workflows/scanoss.yml

@@ -0,0 +1,161 @@
+name: "SCANOSS License & BOM Check"
+
+# Runs only when a PR is APPROVED — i.e. after review and all other checks are
+# green — so the scan happens once, last, right before merge. Scans only the
+# files the PR changed (not the whole tree). Both choices minimize osskb quota
+# (limits are per-file: 5000/day, 7500/week, 15000/month).
+on:
+  pull_request_review:
+    types: [submitted]
+  workflow_dispatch:
+
+env:
+  REPORTS_DIR: scanoss-reports
+
+jobs:
+  scanoss:
+    runs-on: ubuntu-latest
+    # Only on an approving review of a PR targeting master/main (or manual dispatch).
+    if: >-
+      github.event_name == 'workflow_dispatch' ||
+      (github.event.review.state == 'approved' &&
+       (github.event.pull_request.base.ref == 'master' ||
+        github.event.pull_request.base.ref == 'main'))
+    permissions:
+      contents: read
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          # pull_request_review checks out the base branch by default; force the
+          # PR head so we scan the code being merged. fetch-depth 0 so we can
+          # diff against the base branch. (Empty ref on manual dispatch -> the
+          # dispatched ref.)
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install scanoss-py
+        run: pip install scanoss
+
+      - name: Create reports directory
+        run: mkdir -p ${{ env.REPORTS_DIR }}
+
+      - name: Run SCANOSS scan (changed files only)
+        run: |
+          set -euo pipefail
+          OUT=${{ env.REPORTS_DIR }}/results.json
+          BASE_REF="${{ github.event.pull_request.base.ref }}"
+          if [ -z "$BASE_REF" ]; then
+            # Manual dispatch: no PR context -> scan the whole tree.
+            echo "No PR context (manual dispatch): scanning full tree."
+            scanoss-py scan --apiurl https://api.osskb.org -o "$OUT" . || true
+          else
+            git fetch --no-tags --quiet origin "$BASE_REF" || true
+            BASE_SHA=$(git merge-base "origin/$BASE_REF" HEAD 2>/dev/null || echo "${{ github.event.pull_request.base.sha }}")
+            git diff --name-only --diff-filter=ACMR "$BASE_SHA" HEAD > changed_files.txt || true
+            echo "Changed files in this PR ($(wc -l < changed_files.txt)):"
+            cat changed_files.txt || true
+            if [ -s changed_files.txt ]; then
+              # scanoss-py applies scanoss.json skip patterns to this list too,
+              # so tests/vendor/generated/docs are filtered out automatically.
+              scanoss-py scan \
+                --apiurl https://api.osskb.org \
+                --files-from changed_files.txt \
+                -o "$OUT" || true
+            else
+              echo "No changed files to scan."
+            fi
+          fi
+          # Ensure a valid (possibly empty) results file exists for downstream steps.
+          [ -s "$OUT" ] || echo '{}' > "$OUT"
+
+      - name: Generate CycloneDX BOM (convert, no re-scan)
+        run: |
+          scanoss-py convert \
+            -i ${{ env.REPORTS_DIR }}/results.json \
+            -f cyclonedx \
+            -o ${{ env.REPORTS_DIR }}/bom_cyclonedx.json || true
+
+      - name: Inspect - Copyleft
+        run: |
+          scanoss-py inspect raw copyleft \
+            -i ${{ env.REPORTS_DIR }}/results.json \
+            -f md \
+            -o ${{ env.REPORTS_DIR }}/copyleft.md \
+            -s ${{ env.REPORTS_DIR }}/copyleft_status.md
+
+      - name: Inspect - License Summary
+        run: |
+          scanoss-py inspect raw license-summary \
+            -i ${{ env.REPORTS_DIR }}/results.json \
+            -o ${{ env.REPORTS_DIR }}/license_summary.md
+
+      - name: Inspect - Component Summary
+        run: |
+          scanoss-py inspect raw component-summary \
+            -i ${{ env.REPORTS_DIR }}/results.json \
+            -o ${{ env.REPORTS_DIR }}/component_summary.md
+
+      - name: Inspect - Undeclared Components
+        run: |
+          scanoss-py inspect raw undeclared \
+            -i ${{ env.REPORTS_DIR }}/results.json \
+            -f md \
+            -o ${{ env.REPORTS_DIR }}/undeclared.md \
+            -s ${{ env.REPORTS_DIR }}/undeclared_status.md
+
+      - name: Check for non-allowed licenses
+        run: |
+          python3 - <<'PYEOF'
+          import json, re, sys
+          from collections import defaultdict
+
+          ALLOWED = {'MIT', 'Apache-2.0', 'BSD-3-Clause'}
+
+          def spdx_tokens(expr):
+              return set(re.split(r'\s+(?:OR|AND|WITH)\s+|\(|\)', expr)) - {''}
+
+          with open('${{ env.REPORTS_DIR }}/results.json') as f:
+              data = json.load(f)
+
+          violations = defaultdict(set)
+          for file_path, matches in data.items():
+              for match in matches:
+                  for lic in match.get('licenses', []):
+                      name = lic.get('name', '')
+                      if name and not spdx_tokens(name).issubset(ALLOWED):
+                          violations[name].add(file_path)
+
+          if violations:
+              print("::error::Non-allowed licenses found:")
+              for lic, files in sorted(violations.items()):
+                  print(f"  {lic}: {len(files)} file(s)")
+                  for f in sorted(files)[:5]:
+                      print(f"    - {f}")
+              sys.exit(1)
+          else:
+              print("License check passed — all licenses within MIT, Apache-2.0, BSD-3-Clause")
+          PYEOF
+
+      - name: Check for copyleft
+        run: |
+          STATUS=$(cat ${{ env.REPORTS_DIR }}/copyleft_status.md)
+          echo "$STATUS"
+          if echo "$STATUS" | grep -qv "^0 "; then
+            echo "::error::Copyleft licenses detected"
+            exit 1
+          fi
+
+      - name: Upload reports
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: scanoss-reports
+          path: ${{ env.REPORTS_DIR }}/
+          retention-days: 30

e2e-rag/QSL.py

@@ -85,7 +85,8 @@ def __init__(self, dataset_path, perf_count=None, skip_qsl=False):
 
         print(f"Dataset loaded: {self.count} queries")
         if perf_count is not None:
-            print(f"  (limited to first {perf_count} queries for performance testing)")
+            print(
+                f"  (limited to first {perf_count} queries for performance testing)")
 
     def load_query_samples(self, sample_list):
         """
@@ -161,15 +162,12 @@ def __init__(self, dataset_path, perf_count=None):
 # limitations under the License.
 # =============================================================================
 
+
 """
 Query Sample Library for RAG-QnA workload.
 Loads queries from frames_dataset.tsv and provides them to MLPerf Loadgen.
 """
 
-import os
-import pandas as pd
-import mlperf_loadgen as lg
-
 
 class E2EQSL:
     """Query Sample Library for RAG-QnA multi-hop RAG benchmark."""
@@ -233,7 +231,8 @@ def __init__(self, dataset_path, perf_count=None, skip_qsl=False):
 
         print(f"Dataset loaded: {self.count} queries")
         if perf_count is not None:
-            print(f"  (limited to first {perf_count} queries for performance testing)")
+            print(
+                f"  (limited to first {perf_count} queries for performance testing)")
 
     def load_query_samples(self, sample_list):
         """

e2e-rag/accuracy_eval.py

@@ -34,9 +34,10 @@
 # OpenRouter configuration
 DEFAULT_JUDGE_URL = "http://127.0.0.1:8123/v1/chat/completions"
 DEFAULT_JUDGE_MODEL = "gpt-oss-20b"
-# Masked API key (set OPENROUTER_API_KEY environment variable to use OpenRouter)
+# Masked API key (set OPENROUTER_API_KEY environment variable to use
+# OpenRouter)
 OPENROUTER_API_KEY = os.environ.get('OPENROUTER_API_KEY',
-    'sk-or-v1-****')
+                                    'sk-or-v1-****')
 
 
 JUDGE_PROMPT = """You are an expert evaluator comparing LLM-generated answers to ground truth answers.
@@ -83,7 +84,11 @@ def call_judge(question: str, ground_truth: str, llm_answer: str,
     }
 
     try:
-        response = requests.post(service_url, json=payload, headers=headers, timeout=60)
+        response = requests.post(
+            service_url,
+            json=payload,
+            headers=headers,
+            timeout=60)
         response.raise_for_status()
         result = response.json()
 
@@ -105,7 +110,8 @@ def call_judge(question: str, ground_truth: str, llm_answer: str,
         return {"correct": False, "reasoning": f"Judge error: {e}"}
 
 
-def calculate_retrieval_metrics(retrieved_urls: List[str], expected_urls: List[str]) -> Dict:
+def calculate_retrieval_metrics(
+        retrieved_urls: List[str], expected_urls: List[str]) -> Dict:
     """Calculate precision, recall, F1 for retrieval."""
 
     retrieved_set = set(retrieved_urls)
@@ -118,7 +124,8 @@ def calculate_retrieval_metrics(retrieved_urls: List[str], expected_urls: List[s
 
     precision = len(correct) / len(retrieved_set) if retrieved_set else 0.0
     recall = len(correct) / len(expected_set) if expected_set else 0.0
-    f1 = 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0.0
+    f1 = 2 * precision * recall / \
+        (precision + recall) if (precision + recall) > 0 else 0.0
 
     return {
         "precision": precision,
@@ -128,8 +135,8 @@ def calculate_retrieval_metrics(retrieved_urls: List[str], expected_urls: List[s
 
 
 def evaluate_results(results: Dict, dataset_path: str, num_workers: int = 4,
-                    judge_service_url: str = DEFAULT_JUDGE_URL,
-                    judge_model: str = DEFAULT_JUDGE_MODEL) -> Dict:
+                     judge_service_url: str = DEFAULT_JUDGE_URL,
+                     judge_model: str = DEFAULT_JUDGE_MODEL) -> Dict:
     """
     Evaluate loadgen results.
 
@@ -190,12 +197,13 @@ def evaluate_single_query(query_id, result):
         expected_urls = gt_data['expected_urls']
 
         # Calculate retrieval metrics
-        retrieval_metrics = calculate_retrieval_metrics(retrieved_urls, expected_urls)
+        retrieval_metrics = calculate_retrieval_metrics(
+            retrieved_urls, expected_urls)
 
         # Judge answer correctness
         judge_result = call_judge(query, ground_truth, llm_answer,
-                                 service_url=judge_service_url,
-                                 model_name=judge_model)
+                                  service_url=judge_service_url,
+                                  model_name=judge_model)
         answer_correct = judge_result.get('correct', False)
 
         return {
@@ -229,7 +237,8 @@ def evaluate_single_query(query_id, result):
                     total_queries += 1
 
                     if total_queries % 10 == 0:
-                        print(f"  Evaluated {total_queries}/{len(results)} queries...")
+                        print(
+                            f"  Evaluated {total_queries}/{len(results)} queries...")
             except Exception as e:
                 print(f"Error evaluating query: {e}")
 
@@ -257,14 +266,37 @@ def evaluate_single_query(query_id, result):
 
 
 def main():
-    parser = argparse.ArgumentParser(description="Evaluate RAG-QnA loadgen accuracy")
-    parser.add_argument('--log_dir', required=True, help='Loadgen log directory')
-    parser.add_argument('--results_file', required=True, help='SUT results JSON file')
-    parser.add_argument('--dataset_path', required=True, help='Path to frames_dataset.tsv')
-    parser.add_argument('--num_workers', type=int, default=4, help='Number of parallel judge workers')
-    parser.add_argument('--output', default='accuracy_results.json', help='Output file for detailed results')
-    parser.add_argument('--judge_service_url', default=DEFAULT_JUDGE_URL, help='Judge LLM service URL')
-    parser.add_argument('--judge_model', default=DEFAULT_JUDGE_MODEL, help='Judge LLM model name')
+    parser = argparse.ArgumentParser(
+        description="Evaluate RAG-QnA loadgen accuracy")
+    parser.add_argument(
+        '--log_dir',
+        required=True,
+        help='Loadgen log directory')
+    parser.add_argument(
+        '--results_file',
+        required=True,
+        help='SUT results JSON file')
+    parser.add_argument(
+        '--dataset_path',
+        required=True,
+        help='Path to frames_dataset.tsv')
+    parser.add_argument(
+        '--num_workers',
+        type=int,
+        default=4,
+        help='Number of parallel judge workers')
+    parser.add_argument(
+        '--output',
+        default='accuracy_results.json',
+        help='Output file for detailed results')
+    parser.add_argument(
+        '--judge_service_url',
+        default=DEFAULT_JUDGE_URL,
+        help='Judge LLM service URL')
+    parser.add_argument(
+        '--judge_model',
+        default=DEFAULT_JUDGE_MODEL,
+        help='Judge LLM model name')
     args = parser.parse_args()
 
     # Load results
@@ -280,17 +312,17 @@ def main():
                                judge_model=args.judge_model)
 
     # Print summary
-    print("\n" + "="*80)
+    print("\n" + "=" * 80)
     print("ACCURACY EVALUATION RESULTS")
-    print("="*80)
+    print("=" * 80)
     print(f"Total Queries:        {metrics['total_queries']}")
     print(f"\nRetrieval Metrics:")
     print(f"  Precision@N:        {metrics['retrieval_precision']:.3f}")
     print(f"  Recall@N:           {metrics['retrieval_recall']:.3f}")
     print(f"  F1@N:               {metrics['retrieval_f1']:.3f}")
     print(f"\nAnswer Quality:")
     print(f"  LLM Judge Accuracy: {metrics['answer_accuracy']:.3f}")
-    print("="*80 + "\n")
+    print("=" * 80 + "\n")
 
     # Save detailed results
     with open(args.output, 'w') as f: