Merge branch 'release/2.5.0'

Author: NicoLaval

.github/workflows/ci.yml

@@ -27,31 +27,8 @@ jobs:
             ${{ runner.os }}-maven-
       - name: Verify code format with Spotless
         run: mvn spotless:check
-  test-excluding-vtl-sdmx:
-    name: Run Trevas tests excluding vtl-sdmx module
-    runs-on: ubuntu-latest
-    needs: format
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          fetch-depth: 0
-      - name: Set up Maven Central Repository
-        uses: actions/setup-java@v5
-        with:
-          java-version: 17
-          distribution: "adopt"
-      # Also exclude vtl-prov because of vtl-sdmx scope test dependency
-      - name: Build parser stack (vtl-antlr + vtl-parser)
-        run: mvn install -pl vtl-antlr,vtl-parser -am -DskipTests
-      - name: Test
-        run: mvn test -pl '!vtl-sdmx,!vtl-prov'
   test:
     name: Run Trevas tests
-    if: (github.repository != 'InseeFr/Trevas' &&
-      github.event_name == 'push') ||
-      (github.event.pull_request.head.repo.fork == true ||
-      (github.event.pull_request.head.repo.fork == false &&
-      github.event.pull_request.merged == false))
     runs-on: ubuntu-latest
     needs: format
     steps:
@@ -63,15 +40,6 @@ jobs:
         with:
           java-version: 17
           distribution: "adopt"
-      - uses: s4u/maven-settings-action@v4.0.0
-        with:
-          githubServer: false
-          servers: |
-            [{
-                "id": "Github",
-                "username": "${{ secrets.GH_PACKAGES_USERNAME }}",
-                "password": "${{ secrets.GH_PACKAGES_PASSWORD }}"
-            }]
       - name: Build parser stack (vtl-antlr + vtl-parser)
         run: mvn install -pl vtl-antlr,vtl-parser -am -DskipTests
       - name: Test
@@ -110,15 +78,6 @@ jobs:
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-maven-
-      - uses: s4u/maven-settings-action@v4.0.0
-        with:
-          githubServer: false
-          servers: |
-            [{
-                "id": "Github",
-                "username": "${{ secrets.GH_PACKAGES_USERNAME }}",
-                "password": "${{ secrets.GH_PACKAGES_PASSWORD }}"
-            }]
       # Install vtl-antlr + vtl-parser first, then exclude vtl-parser from the reactor so
       # dependent modules resolve the installed jar, not target/classes.
       - name: Test ${{ matrix.module }} (and upstream modules)
@@ -138,15 +97,6 @@ jobs:
         with:
           java-version: 17
           distribution: "adopt"
-      - uses: s4u/maven-settings-action@v4.0.0
-        with:
-          githubServer: false
-          servers: |
-            [{
-                "id": "Github",
-                "username": "${{ secrets.GH_PACKAGES_USERNAME }}",
-                "password": "${{ secrets.GH_PACKAGES_PASSWORD }}"
-            }]
       - name: Test
         run: mvn package
   #  test-sonar-package:
@@ -168,15 +118,6 @@ jobs:
   #        with:
   #          java-version: 17
   #          distribution: "adopt"
-  #      - uses: s4u/maven-settings-action@v3.0.0
-  #        with:
-  #          githubServer: false
-  #          servers: |
-  #            [{
-  #                "id": "Github",
-  #                "username": "${{ secrets.GH_PACKAGES_USERNAME }}",
-  #                "password": "${{ secrets.GH_PACKAGES_PASSWORD }}"
-  #            }]
   #      - name: Test, package and analyze with maven & SonarCloud
   #        run: mvn verify sonar:sonar -Pcoverage -Dsonar.projectKey=InseeFr_Trevas -Dsonar.organization=inseefr -Dsonar.host.url=https://sonarcloud.io
   #        env:
@@ -200,11 +141,6 @@ jobs:
           githubServer: false
           servers: |
             [
-              {
-                  "id": "Github",
-                  "username": "${{ secrets.GH_PACKAGES_USERNAME }}",
-                  "password": "${{ secrets.GH_PACKAGES_PASSWORD }}"
-              },
               {
                   "id": "central",
                   "username": "${{ secrets.MAVEN_CENTRAL_USERNAME }}",

.github/workflows/tck-vtl-tf-spark3.yml

@@ -12,15 +12,16 @@ jobs:
     runs-on: ubuntu-latest
     env:
       TCK_REPORT_TITLE: 'TCK VTL v2.1 (Spark 3)'
+      VTL_TCK_BRANCH: fix/doc-examples
 
     steps:
       - name: Checkout main project
         uses: actions/checkout@v6
         with:
           fetch-depth: 0
 
-      - name: Clone vtl spec repo (branch fix/tck-2.1)
-        run: git clone --branch fix/tck-2.1 https://github.com/sdmx-twg/vtl.git
+      - name: Clone vtl spec repo
+        run: git clone --depth 1 --branch "${VTL_TCK_BRANCH}" https://github.com/sdmx-twg/vtl.git
 
       - name: Install Python 3
         uses: actions/setup-python@v6
@@ -42,16 +43,6 @@ jobs:
           distribution: 'temurin'
           java-version: '17'
 
-      - uses: s4u/maven-settings-action@v4.0.0
-        with:
-          githubServer: false
-          servers: |
-            [{
-                "id": "Github",
-                "username": "${{ secrets.GH_PACKAGES_USERNAME }}",
-                "password": "${{ secrets.GH_PACKAGES_PASSWORD }}"
-            }]
-
       - name: Cache Maven packages
         uses: actions/cache@v5
         with:
@@ -75,7 +66,8 @@ jobs:
         if: always()
         with:
           name: JUnit (Spark 3)
-          path: coverage/target/surefire-reports/*.xml
+          # TCK suite only (exclude coverage unit tests: TckDatasetComparisonTest, …)
+          path: coverage/target/surefire-reports/TEST-fr.insee.vtl.coverage.TCKTest.xml
           reporter: java-junit
           only-summary: 'true'
           max-annotations: 0

.github/workflows/tck-vtl-tf-spark4.yml

@@ -12,15 +12,16 @@ jobs:
     runs-on: ubuntu-latest
     env:
       TCK_REPORT_TITLE: 'TCK VTL v2.1 (Spark 4)'
+      VTL_TCK_BRANCH: fix/doc-examples
 
     steps:
       - name: Checkout main project
         uses: actions/checkout@v6
         with:
           fetch-depth: 0
 
-      - name: Clone vtl spec repo (branch fix/tck-2.1)
-        run: git clone --branch fix/tck-2.1 https://github.com/sdmx-twg/vtl.git
+      - name: Clone vtl spec repo
+        run: git clone --depth 1 --branch "${VTL_TCK_BRANCH}" https://github.com/sdmx-twg/vtl.git
 
       - name: Install Python 3
         uses: actions/setup-python@v6
@@ -42,16 +43,6 @@ jobs:
           distribution: 'temurin'
           java-version: '17'
 
-      - uses: s4u/maven-settings-action@v4.0.0
-        with:
-          githubServer: false
-          servers: |
-            [{
-                "id": "Github",
-                "username": "${{ secrets.GH_PACKAGES_USERNAME }}",
-                "password": "${{ secrets.GH_PACKAGES_PASSWORD }}"
-            }]
-
       - name: Cache Maven packages
         uses: actions/cache@v5
         with:
@@ -75,7 +66,8 @@ jobs:
         if: always()
         with:
           name: JUnit (Spark 4)
-          path: coverage/target/surefire-reports/*.xml
+          # TCK suite only (exclude coverage unit tests: TckDatasetComparisonTest, …)
+          path: coverage/target/surefire-reports/TEST-fr.insee.vtl.coverage.TCKTest.xml
           reporter: java-junit
           only-summary: 'true'
           max-annotations: 0

.gitignore

@@ -1,4 +1,5 @@
 target/
+.cache/
 **/.DS_Store
 **/*.iml
 **/.idea

coverage/README.md

@@ -1,33 +1,120 @@
-# 📊 Coverage
+# Coverage — TCK VTL local
 
-## 🧪 TCK (Technology Compatibility Kit)
+Module Maven qui exécute le **Technology Compatibility Kit (TCK)** VTL v2.1 contre Trevas (moteur + backend Spark).
 
-We are working on a suite of compatibility tests to ensure conformance with the VTL specification across implementations.
+**Point d’entrée JUnit :** `fr.insee.vtl.coverage.TCKTest`  
+**Fichier requis :** `coverage/src/main/resources/v2.1.zip` (non versionné — à générer une fois)
 
-🛠️ _Work in Progress_
+---
 
-### Issues/discussions to follow
+## Démarrage rapide (depuis la racine du dépôt Trevas)
 
-- [junit5](https://github.com/junit-team/junit5/discussions/4504#discussioncomment-13046641)
-- [surefire](https://github.com/apache/maven-surefire/issues/835)
-- [dorny/test-reporter](https://github.com/dorny/test-reporter/issues/580)
+Prérequis : **JDK 17+**, **Maven 3.9+**, **Python 3**, accès réseau (clone GitHub la première fois).
 
-### Temporary run procedure
+```shell
+# 1) Générer le zip TCK (une fois, ou après mise à jour des exemples SDMX-VTL)
+./coverage/scripts/refresh_tck_zip.sh
+
+# 2) Compiler les modules dont dépend coverage (dont vtl-csv, vtl-spark, vtl-engine…)
+mvn clean install -pl coverage -am -DskipTests --batch-mode
+
+# 3) Lancer le TCK (Spark 3 par défaut)
+mvn test -pl coverage --batch-mode
+```
 
-While [TCK](https://github.com/sdmx-twg/vtl/pull/565) is not automated in the VTL TF repository, we have to build the input source manually.
+Vérifier que le zip est bien là avant l’étape 3 :
 
 ```shell
-git clone https://github.com/sdmx-twg/vtl.git
-cd vtl/scripts
-DOC_VERSION=v2.1 python3 generate_tck_files.py
+test -f coverage/src/main/resources/v2.1.zip && echo OK || echo "manquant — relancer refresh_tck_zip.sh"
 ```
 
-A zip will be created at `tck/v2.1.zip`.
+### Spark 4
 
-Move it in Trevas resources:
+```shell
+mvn clean install -pl coverage -am -DskipTests --batch-mode -Pspark4-tck
+mvn test -pl coverage --batch-mode -Pspark4-tck
+```
+
+### Rapport lisible après les tests
 
 ```shell
-mv vtl/tck/v2.1.zip trevas/coverage/src/main/resources
+python3 coverage/scripts/prettify_tck_surefire_xml.py
+python3 coverage/scripts/render_tck_job_summary.py
+# → coverage/target/tck-scripts-report.md
 ```
 
-You are now able to run `TCKTest`.
+XML Surefire principal : `coverage/target/surefire-reports/TEST-fr.insee.vtl.coverage.TCKTest.xml`
+
+---
+
+## Pourquoi `mvn test -pl coverage` seul échoue souvent
+
+| Symptôme | Cause | Correctif |
+|----------|--------|-----------|
+| `FileNotFoundException` / zip introuvable | Pas de `v2.1.zip` | `./coverage/scripts/refresh_tck_zip.sh` |
+| `ClassNotFoundException: fr.insee.vtl.csv.CsvDatasetValidator` | Module `vtl-csv` non installé | `mvn install -pl vtl-csv -am -DskipTests` ou **toujours** utiliser `-am` avec `coverage` |
+| Tests TCK absents / 0 test | Classpath incomplet | `mvn clean install -pl coverage -am -DskipTests` puis `mvn test -pl coverage` |
+
+**Règle :** pour `coverage`, utiliser **`-pl coverage -am`** (`-am` = *also make* : construit tous les modules requis du réacteur, dont `vtl-csv`).
+
+---
+
+## D’où vient le TCK ?
+
+Les cas sont extraits des exemples du manuel VTL SDMX  
+(`v2.1/docs/reference_manual/operators/**/examples`).
+
+Branche Git pinée (fixtures CSV/JSON corrigées) : **`fix/doc-examples`**  
+https://github.com/sdmx-twg/vtl/tree/fix/doc-examples
+
+Le script local reproduit ce que fait la CI :
+
+```shell
+./coverage/scripts/refresh_tck_zip.sh
+```
+
+→ clone `sdmx-twg/vtl`, exécute `scripts/generate_tck_files.py`, copie `tck/v2.1.zip` vers `coverage/src/main/resources/v2.1.zip`.
+
+Variables optionnelles :
+
+```shell
+VTL_TCK_BRANCH=fix/doc-examples ./coverage/scripts/refresh_tck_zip.sh
+DOC_VERSION=v2.1 VTL_TCK_BRANCH=master ./coverage/scripts/refresh_tck_zip.sh
+```
+
+### Génération manuelle (équivalent)
+
+```shell
+git clone --depth 1 --branch fix/doc-examples https://github.com/sdmx-twg/vtl.git /tmp/vtl-tck
+DOC_VERSION=v2.1 python3 /tmp/vtl-tck/scripts/generate_tck_files.py
+mkdir -p coverage/src/main/resources
+cp /tmp/vtl-tck/tck/v2.1.zip coverage/src/main/resources/v2.1.zip
+```
+
+---
+
+## Lancer un sous-ensemble (debug)
+
+```shell
+# un seul cas (adapter le nom affiché dans les logs / XML)
+mvn test -pl coverage -Dtest='TCKTest#leafCases' --batch-mode
+
+# tests unitaires du module coverage (hors TCK long)
+mvn test -pl coverage -Dtest='!TCKTest' --batch-mode
+```
+
+---
+
+## CI
+
+Workflows : `.github/workflows/tck-vtl-tf-spark3.yml`, `tck-vtl-tf-spark4.yml`
+
+En résumé : checkout Trevas → génération `v2.1.zip` → `mvn clean install -pl coverage -am -DskipTests` → `mvn test -pl coverage` → rapport Surefire / résumé Markdown.
+
+---
+
+## Liens
+
+- [TCK automation PR (VTL TF)](https://github.com/sdmx-twg/vtl/pull/565)
+- [Notes chargeur CSV TCK](../roadmap/tck-csv-loader.md)
+- JUnit / reporting : [junit5](https://github.com/junit-team/junit5/discussions/4504#discussioncomment-13046641), [surefire](https://github.com/apache/maven-surefire/issues/835), [dorny/test-reporter](https://github.com/dorny/test-reporter/issues/580)