add extended crrcascade tests

tests/functional/ctst/common/common.ts

@@ -274,7 +274,7 @@ Given('a transition workflow to {string} location', async function (this: Zenko,
 
 Given('a replication configuration to {string} location',
     async function (this: Zenko, replicationLocation: string) {
-        this.addToSaved('replicationLocation', replicationLocation);
+        this.addToSaved('replicationLocations', [replicationLocation]);
         await putBucketReplication.call(this, this.getSaved<string>('bucketName'), replicationLocation);
     });
 

tests/functional/ctst/features/replication/crrCascade.feature

@@ -40,6 +40,46 @@ Feature: CRR Cascade Replication
         When I wait 15 seconds
         Then the cascade replication states should be settled
 
+    @2.15.0
+    @PreMerge
+    @ReplicationTest
+    @CRRCascade
+    Scenario: Cascade replication loop with non-empty object : A -> B -> C -> A
+        Given cascade replication accounts are registered
+        And a versioned bucket exists in location "crr-location-a"
+        And a versioned bucket exists in location "crr-location-b"
+        And a versioned bucket exists in location "crr-location-c"
+        And replication is configured from location "crr-location-a" to "crr-location-b"
+        And replication is configured from location "crr-location-b" to "crr-location-c"
+        And replication is configured from location "crr-location-c" to "crr-location-a"
+        When a non-empty object "cascade-loop-nonempty-obj" is put in location "crr-location-a"
+        Then the object should replicate to location "crr-location-b" within 300 seconds
+        And the object should replicate to location "crr-location-c" within 300 seconds
+        And the object at location "crr-location-a" should never have replication status PENDING within 30 seconds
+        When I wait 15 seconds
+        Then the cascade replication states should be settled
+
+    @2.15.0
+    @PreMerge
+    @ReplicationTest
+    @CRRCascade
+    Scenario: Cascade tag update replicates through a loop without bouncing back : A -> B -> C -> A
+        Given cascade replication accounts are registered
+        And a versioned bucket exists in location "crr-location-a"
+        And a versioned bucket exists in location "crr-location-b"
+        And a versioned bucket exists in location "crr-location-c"
+        And replication is configured from location "crr-location-a" to "crr-location-b"
+        And replication is configured from location "crr-location-b" to "crr-location-c"
+        And replication is configured from location "crr-location-c" to "crr-location-a"
+        When a non-empty object "cascade-tag-loop-obj" is put in location "crr-location-a"
+        Then the object should replicate to location "crr-location-b" within 300 seconds
+        And the object should replicate to location "crr-location-c" within 300 seconds
+        When tags are put on the object "cascade-tag-loop-obj" in location "crr-location-a"
+        Then the object at location "crr-location-c" should have the expected tags within 300 seconds
+        And the object at location "crr-location-a" should never have replication status PENDING within 30 seconds
+        When I wait 15 seconds
+        Then the cascade replication states should be settled
+
     @2.15.0
     @PreMerge
     @ReplicationTest

tests/functional/ctst/features/replication/crrReplicationS3utils.feature

@@ -20,7 +20,7 @@ Feature: Replication
         And a replication configuration to "awsbackendmismatch" location
         When the job to replicate existing objects with status "NEW" is executed
         Then the object replication should "succeed" within 300 seconds
-        And the replicated object should be the same as the source object
+        And the replicated object should match the source on every configured destination
 
     @2.12.0
     @PreMerge
@@ -35,4 +35,4 @@ Feature: Replication
         When the destination bucket on the location is created again
         And the job to replicate existing objects with status "FAILED" is executed
         Then the object replication should "succeed" within 300 seconds
-        And the replicated object should be the same as the source object
+        And the replicated object should match the source on every configured destination

tests/functional/ctst/features/replication/replication.feature

@@ -1,26 +1,167 @@
-Feature: Live Object Replication
-    This feature tests live replication of objects across different sizes,
-    verifying that oplog location stripping does not affect replication
-    correctness.
-
-    The objectRefreshSizeThresholdMB is set to 2 in the Zenko CR, meaning
-    objects >= 2 MiB have their location stripped from the MongoDB change
-    stream event. The replication consumer detects this and re-fetches
-    metadata from the source before replicating.
+Feature: Bucket Replication
+    End-to-end coverage of Zenko's bucket replication, covering both
+    cloud destinations (AWS / GCP / Azure backends) and CRR loopback
+    destinations (replication between Zenko accounts).
 
     @2.14.0
     @PreMerge
     @ReplicationTest
+    # Targets the location-stripping threshold: objectRefreshSizeThresholdMB
+    # is set to 2 in the Zenko CR, so objects >= 2 MiB have their location
+    # stripped from the MongoDB change stream event. The replication consumer
+    # detects this and re-fetches metadata from the source before replicating.
     Scenario Outline: Object of <objectDescription> replicates with location stripping threshold
         Given an existing bucket "repl-strip-<sizeBytes>" "with" versioning, "without" ObjectLock "without" retention mode
         And a replication configuration to "awsbackendmismatch" location
         And 1 objects "repl-strip-obj" of size <sizeBytes> bytes
         Then the object replication should "succeed" within 300 seconds
-        And the replicated object should be the same as the source object
+        And the replicated object should match the source on every configured destination
 
         Examples:
             | sizeBytes | objectDescription                  |
             |         0 | 0 bytes (zero-byte)                |
             |   1048576 | 1 MiB (below threshold)            |
             |   2097152 | 2 MiB (at threshold, stripped)     |
             |   4194304 | 4 MiB (above threshold, stripped)  |
+
+    @2.15.0
+    @PreMerge
+    @ReplicationTest
+    # Multi-destination replication: a single replication configuration
+    # carries several rules targeting distinct destinations, optionally with
+    # overlapping prefixes resolved by Priority. The legacy V1 format and
+    # the non-standard comma-separated StorageClass form remain supported
+    # for backward compatibility and round-trip unchanged through
+    # getBucketReplication.
+    Scenario: V2 replication fans out to two cloud destinations
+        Given an existing bucket "multi-cloud-2" "with" versioning, "without" ObjectLock "without" retention mode
+        And a valid multi-destination replication configuration with rules:
+            | id     | prefix | priority | location              |
+            | rule-a |        | 1        | awsbackendmismatch    |
+            | rule-b |        | 2        | awsbackend    |
+        And 1 objects "multi-cloud-obj" of size 1024 bytes
+        Then the object replication should "succeed" within 300 seconds
+        And the replicated object should match the source on every configured destination
+
+    @2.15.0
+    @PreMerge
+    @ReplicationTest
+    Scenario: V2 replication fans out to three cloud destinations
+        Given an existing bucket "multi-cloud-3" "with" versioning, "without" ObjectLock "without" retention mode
+        And a valid multi-destination replication configuration with rules:
+            | id     | prefix | priority | location              |
+            | rule-a |        | 1        | awsbackendmismatch    |
+            | rule-b |        | 2        | gcpbackendmismatch    |
+            | rule-c |        | 3        | azurebackendmismatch  |
+        And 1 objects "multi-cloud-obj" of size 1024 bytes
+        Then the object replication should "succeed" within 300 seconds
+        And the replicated object should match the source on every configured destination
+
+    @2.15.0
+    @PreMerge
+    @ReplicationTest
+    Scenario: CRR + cloud mixed destination replication
+        Given cascade replication accounts are registered
+        And a versioned bucket exists in location "crr-location-a"
+        And an existing bucket "multi-mixed" "with" versioning, "without" ObjectLock "without" retention mode
+        And a valid multi-destination replication configuration with rules:
+            | id     | prefix | priority | location                  |
+            | rule-c |        | 1        | crr-location-a    |
+            | rule-x |        | 2        | awsbackendmismatch        |
+        And 1 objects "mixed-obj" of size 1024 bytes
+        Then the object replication should "succeed" within 300 seconds
+        And the replicated object should match the source on every configured destination
+
+    @2.15.0
+    @PreMerge
+    @ReplicationTest
+    Scenario: V2 rules with overlapping prefixes to different destinations all apply
+        Given an existing bucket "multi-overlap" "with" versioning, "without" ObjectLock "without" retention mode
+        And a valid multi-destination replication configuration with rules:
+            | id     | prefix | priority | location              |
+            | broad  |        | 1        | awsbackendmismatch    |
+            | narrow | docs-  | 2        | awsbackend            |
+        And 1 objects "docs-report" of size 1024 bytes
+        Then the object replication should "succeed" within 300 seconds
+        And the replicated object should match the source on every configured destination
+
+    @2.15.0
+    @PreMerge
+    @ReplicationTest
+    Scenario: Disabled rule does not replicate
+        Given an existing bucket "multi-disabled" "with" versioning, "without" ObjectLock "without" retention mode
+        And a valid multi-destination replication configuration with rules:
+            | id     | prefix | priority | status   | location              |
+            | active |        | 1        | Enabled  | awsbackendmismatch    |
+            | off    |        | 2        | Disabled | awsbackend    |
+        And 1 objects "disabled-obj" of size 1024 bytes
+        Then the object replication should "succeed" within 300 seconds
+        And the object should not be replicated to "awsbackend"
+        And the replicated object on "awsbackendmismatch" should match the source
+
+    @2.15.0
+    @PreMerge
+    @ReplicationTest
+    Scenario: Overlap on same destination without distinct priorities is rejected
+        Given an existing bucket "multi-bad-overlap" "with" versioning, "without" ObjectLock "without" retention mode
+        And an invalid multi-destination replication configuration with rules:
+            | id    | prefix | priority | location              |
+            | dup-a |        | 1        | awsbackendmismatch    |
+            | dup-b | docs-| 1        | awsbackendmismatch    |
+        Then the replication configuration request should be rejected with "InvalidRequest"
+
+    @2.15.0
+    @PreMerge
+    @ReplicationTest
+    Scenario: Same destination overlap with distinct priorities is accepted
+        Given an existing bucket "multi-priority-dedup" "with" versioning, "without" ObjectLock "without" retention mode
+        And a valid multi-destination replication configuration with rules:
+            | id    | prefix | priority | location              |
+            | lo    |        | 1        | awsbackendmismatch    |
+            | hi    | docs-| 2        | awsbackendmismatch    |
+        And 1 objects "docs-winner" of size 1024 bytes
+        Then the object replication should "succeed" within 300 seconds
+        And the replicated object on "awsbackendmismatch" should match the source
+
+    @2.15.0
+    @PreMerge
+    @ReplicationTest
+    Scenario: V1 configuration replicates and round-trips through getBucketReplication
+        Given an existing bucket "multi-v1-compat" "with" versioning, "without" ObjectLock "without" retention mode
+        And a valid multi-destination replication configuration with rules:
+            | id      | prefix | location              |
+            | v1-only |        | awsbackendmismatch    |
+        And 1 objects "v1-obj" of size 1024 bytes
+        Then the object replication should "succeed" within 300 seconds
+        And the replicated object on "awsbackendmismatch" should match the source
+        And getBucketReplication should return a "V1" configuration with 1 rules
+
+    @2.15.0
+    @PreMerge
+    @ReplicationTest
+    Scenario: Legacy comma-separated StorageClass replicates to multiple sites
+        Given an existing bucket "multi-legacy-comma" "with" versioning, "without" ObjectLock "without" retention mode
+        And a legacy comma-StorageClass replication configuration to locations "awsbackendmismatch,awsbackend"
+        And 1 objects "legacy-obj" of size 1024 bytes
+        Then the object replication should "succeed" within 300 seconds
+        And the replicated object should match the source on every configured destination
+        And getBucketReplication should return a "V1" configuration with 1 rules
+        And getBucketReplication rule 0 should have StorageClass "awsbackendmismatch,awsbackend"
+
+    @2.15.0
+    @PreMerge
+    @ReplicationTest
+    @MultiCRR
+    Scenario: Two CRR destinations replicate independently
+        Given cascade replication accounts are registered
+        And a versioned bucket exists in location "crr-location-a"
+        And a versioned bucket exists in location "crr-location-b"
+        And an existing bucket "multi-crr-pair" "with" versioning, "without" ObjectLock "without" retention mode
+        And a valid multi-destination replication configuration with rules:
+            | id     | prefix | priority | location                |
+            | crr-a  |        | 1        | crr-location-a  |
+            | crr-b  |        | 2        | crr-location-b  |
+        And 1 objects "crr-pair-obj" of size 1024 bytes
+        Then the object replication should "succeed" within 300 seconds
+        And the replicated object on "crr-location-a" should match the source
+        And the replicated object on "crr-location-b" should match the source

tests/functional/ctst/steps/crrCascade.ts

@@ -1,17 +1,19 @@
 import { Given, Then, When } from '@cucumber/cucumber';
 import {
     CreateBucketCommand,
+    GetObjectTaggingCommand,
     HeadObjectCommand,
     PutBucketReplicationCommand,
     PutBucketVersioningCommand,
     PutObjectCommand,
+    PutObjectTaggingCommand,
     StorageClass,
 } from '@aws-sdk/client-s3';
 import assert from 'assert';
 import { Identity, IdentityEnum, Utils } from 'cli-testing';
 import Zenko from 'world/Zenko';
 
-interface CRRAccountInfo {
+export interface CRRAccountInfo {
     AccessKeyId: string;
     SecretAccessKey: string;
     SessionToken?: string;
@@ -106,6 +108,66 @@ When('an object {string} is put in location {string}',
         this.addToSaved('cascadeLastMarker', marker);
     });
 
+When('a non-empty object {string} is put in location {string}',
+    async function (this: Zenko, objectName: string, location: string) {
+        const cascadeBuckets = this.getSaved<Record<string, string>>('cascadeBuckets');
+        const marker = Utils.randomString().toLowerCase();
+        const body = Buffer.from('cascade-test-body');
+        Identity.useIdentity(IdentityEnum.ACCOUNT, location);
+        await this.createS3Client().send(new PutObjectCommand({
+            Bucket: cascadeBuckets[location],
+            Key: objectName,
+            Body: body,
+            ContentType: 'text/plain',
+            Metadata: { marker },
+        }));
+        this.addToSaved('cascadeObjectName', objectName);
+        this.addToSaved('cascadeSourceLocation', location);
+        this.addToSaved('cascadeLastMarker', marker);
+        this.addToSaved('cascadeExpectedContentLength', body.length);
+    });
+
+When('tags are put on the object {string} in location {string}',
+    async function (this: Zenko, objectName: string, location: string) {
+        const cascadeBuckets = this.getSaved<Record<string, string>>('cascadeBuckets');
+        const tagValue = Utils.randomString().toLowerCase();
+        Identity.useIdentity(IdentityEnum.ACCOUNT, location);
+        await this.createS3Client().send(new PutObjectTaggingCommand({
+            Bucket: cascadeBuckets[location],
+            Key: objectName,
+            Tagging: { TagSet: [{ Key: 'cascade-test-tag', Value: tagValue }] },
+        }));
+        this.addToSaved('cascadeTagValue', tagValue);
+    });
+
+Then(
+    'the object at location {string} should have the expected tags within {int} seconds',
+    { timeout: 300_000 },
+    async function (this: Zenko, location: string, timeoutSeconds: number) {
+        const bucket = this.getSaved<Record<string, string>>('cascadeBuckets')[location];
+        const objectName = this.getSaved<string>('cascadeObjectName');
+        const tagValue = this.getSaved<string>('cascadeTagValue');
+        const deadline = Date.now() + timeoutSeconds * 1000;
+        Identity.useIdentity(IdentityEnum.ACCOUNT, location);
+        const client = this.createS3Client();
+        while (Date.now() < deadline) {
+            const res = await client.send(
+                new GetObjectTaggingCommand({ Bucket: bucket, Key: objectName }),
+            );
+            const found = res.TagSet?.some(
+                tag => tag.Key === 'cascade-test-tag' && tag.Value === tagValue,
+            );
+            if (found) {
+                return;
+            }
+            await new Promise(resolve => setTimeout(resolve, 3000));
+        }
+        assert.fail(
+            `Timeout: tag 'cascade-test-tag=${tagValue}' not found at '${location}' after ${timeoutSeconds}s`,
+        );
+    },
+);
+
 Then(
     'the object should replicate to location {string} within {int} seconds',
     { timeout: 300_000 },
@@ -115,6 +177,7 @@ Then(
         const deadline = Date.now() + timeoutSeconds * 1000;
         Identity.useIdentity(IdentityEnum.ACCOUNT, location);
         const client = this.createS3Client();
+        const expectedContentLength = this.getSaved<number | undefined>('cascadeExpectedContentLength');
         while (Date.now() < deadline) {
             try {
                 const res = await client.send(
@@ -124,6 +187,12 @@ Then(
                     res.ReplicationStatus, 'REPLICA',
                     `Expected ReplicationStatus to be REPLICA at '${location}', got '${res.ReplicationStatus}'`,
                 );
+                if (expectedContentLength !== undefined) {
+                    assert.strictEqual(
+                        res.ContentLength, expectedContentLength,
+                        `Expected ContentLength ${expectedContentLength} at '${location}', got ${res.ContentLength}`,
+                    );
+                }
                 return;
             } catch (err: unknown) {
                 const status = (err as { $metadata?: { httpStatusCode?: number } }).$metadata?.httpStatusCode;