diff --git a/.github/scripts/end2end/configure-e2e.sh b/.github/scripts/end2end/configure-e2e.sh index 23c76a079c..f8b8586c76 100755 --- a/.github/scripts/end2end/configure-e2e.sh +++ b/.github/scripts/end2end/configure-e2e.sh @@ -96,6 +96,12 @@ CRR_SOURCE_LOCATION_NAME=${CRR_SOURCE_LOCATION_NAME} \ CRR_DESTINATION_LOCATION_NAME=${CRR_DESTINATION_LOCATION_NAME} \ CRR_SOURCE_ACCOUNT_NAME=${CRR_SOURCE_ACCOUNT_NAME} \ CRR_DESTINATION_ACCOUNT_NAME=${CRR_DESTINATION_ACCOUNT_NAME} \ +CRR_LOCATION_A_NAME=${CRR_LOCATION_A_NAME} \ +CRR_LOCATION_B_NAME=${CRR_LOCATION_B_NAME} \ +CRR_LOCATION_C_NAME=${CRR_LOCATION_C_NAME} \ +CRR_ACCOUNT_A_NAME=${CRR_ACCOUNT_A_NAME} \ +CRR_ACCOUNT_B_NAME=${CRR_ACCOUNT_B_NAME} \ +CRR_ACCOUNT_C_NAME=${CRR_ACCOUNT_C_NAME} \ CRR_ROLE_NAME=${CRR_ROLE_NAME} \ DEPLOY_CRR_LOCATIONS=${DEPLOY_CRR_LOCATIONS} \ yarn --cwd "$FUNCTIONAL_TESTS_DIR" initializeFunctionalTestsResources diff --git a/.github/scripts/end2end/setup-e2e-env.sh b/.github/scripts/end2end/setup-e2e-env.sh index a458b73eed..c27a6eb1b4 100755 --- a/.github/scripts/end2end/setup-e2e-env.sh +++ b/.github/scripts/end2end/setup-e2e-env.sh @@ -55,6 +55,9 @@ export ZENKO_SESSION_TOKEN=$(kubectl get secret ${ZENKO_NAME}-account-zenko -o j # CRR account credentials _src_secret="${ZENKO_NAME}-account-${CRR_SOURCE_ACCOUNT_NAME:-crr-source-account}" _dst_secret="${ZENKO_NAME}-account-${CRR_DESTINATION_ACCOUNT_NAME:-crr-destination-account}" +_crr_a_secret="${ZENKO_NAME}-account-${CRR_ACCOUNT_A_NAME:-crr-account-a}" +_crr_b_secret="${ZENKO_NAME}-account-${CRR_ACCOUNT_B_NAME:-crr-account-b}" +_crr_c_secret="${ZENKO_NAME}-account-${CRR_ACCOUNT_C_NAME:-crr-account-c}" SOURCE_ACCESS_KEY=$(kubectl get secret "${_src_secret}" -o jsonpath='{.data.AccessKeyId}' | base64 -d) SOURCE_SECRET_KEY=$(kubectl get secret "${_src_secret}" -o jsonpath='{.data.SecretAccessKey}' | base64 -d) @@ -68,6 +71,24 @@ DESTINATION_SESSION_TOKEN=$(kubectl get secret "${_dst_secret}" -o jsonpath='{.d DESTINATION_ACCOUNT_ID=$(kubectl get secret "${_dst_secret}" -o jsonpath='{.data.AccountId}' | base64 -d) export CRR_DESTINATION_INFO="{\"AccessKeyId\":\"${DESTINATION_ACCESS_KEY}\",\"SecretAccessKey\":\"${DESTINATION_SECRET_KEY}\",\"SessionToken\":\"${DESTINATION_SESSION_TOKEN}\",\"AccountId\":\"${DESTINATION_ACCOUNT_ID}\"}" +CRR_A_ACCESS_KEY=$(kubectl get secret "${_crr_a_secret}" -o jsonpath='{.data.AccessKeyId}' | base64 -d) +CRR_A_SECRET_KEY=$(kubectl get secret "${_crr_a_secret}" -o jsonpath='{.data.SecretAccessKey}' | base64 -d) +CRR_A_SESSION_TOKEN=$(kubectl get secret "${_crr_a_secret}" -o jsonpath='{.data.SessionToken}' | base64 -d) +CRR_A_ACCOUNT_ID=$(kubectl get secret "${_crr_a_secret}" -o jsonpath='{.data.AccountId}' | base64 -d) +export CRR_INFO_A="{\"AccessKeyId\":\"${CRR_A_ACCESS_KEY}\",\"SecretAccessKey\":\"${CRR_A_SECRET_KEY}\",\"SessionToken\":\"${CRR_A_SESSION_TOKEN}\",\"AccountId\":\"${CRR_A_ACCOUNT_ID}\"}" + +CRR_B_ACCESS_KEY=$(kubectl get secret "${_crr_b_secret}" -o jsonpath='{.data.AccessKeyId}' | base64 -d) +CRR_B_SECRET_KEY=$(kubectl get secret "${_crr_b_secret}" -o jsonpath='{.data.SecretAccessKey}' | base64 -d) +CRR_B_SESSION_TOKEN=$(kubectl get secret "${_crr_b_secret}" -o jsonpath='{.data.SessionToken}' | base64 -d) +CRR_B_ACCOUNT_ID=$(kubectl get secret "${_crr_b_secret}" -o jsonpath='{.data.AccountId}' | base64 -d) +export CRR_INFO_B="{\"AccessKeyId\":\"${CRR_B_ACCESS_KEY}\",\"SecretAccessKey\":\"${CRR_B_SECRET_KEY}\",\"SessionToken\":\"${CRR_B_SESSION_TOKEN}\",\"AccountId\":\"${CRR_B_ACCOUNT_ID}\"}" + +CRR_C_ACCESS_KEY=$(kubectl get secret "${_crr_c_secret}" -o jsonpath='{.data.AccessKeyId}' | base64 -d) +CRR_C_SECRET_KEY=$(kubectl get secret "${_crr_c_secret}" -o jsonpath='{.data.SecretAccessKey}' | base64 -d) +CRR_C_SESSION_TOKEN=$(kubectl get secret "${_crr_c_secret}" -o jsonpath='{.data.SessionToken}' | base64 -d) +CRR_C_ACCOUNT_ID=$(kubectl get secret "${_crr_c_secret}" -o jsonpath='{.data.AccountId}' | base64 -d) +export CRR_INFO_C="{\"AccessKeyId\":\"${CRR_C_ACCESS_KEY}\",\"SecretAccessKey\":\"${CRR_C_SECRET_KEY}\",\"SessionToken\":\"${CRR_C_SESSION_TOKEN}\",\"AccountId\":\"${CRR_C_ACCOUNT_ID}\"}" + # --- 6. Keycloak test user --- export KEYCLOAK_TEST_USER="${OIDC_USERNAME}-norights" export KEYCLOAK_TEST_PASSWORD=${OIDC_PASSWORD} @@ -103,6 +124,8 @@ export RING_S3C_BACKEND_SOURCE_NON_VERSIONED_LOCATION export RING_S3C_INGESTION_SRC_NON_VERSIONED_BUCKET_NAME export RING_S3C_INGESTION_NON_VERSIONED_OBJECT_COUNT_PER_TYPE export CRR_SOURCE_LOCATION_NAME CRR_DESTINATION_LOCATION_NAME CRR_ROLE_NAME +export CRR_LOCATION_A_NAME CRR_LOCATION_B_NAME CRR_LOCATION_C_NAME +export CRR_INFO_A CRR_INFO_B CRR_INFO_C export MOCHA_FILE=${MOCHA_FILE:-_reports/test-results-[hash].xml} # Ensure test results dir exists for Mocha JUnit reporter @@ -399,8 +422,14 @@ if [ -n "${GITHUB_ENV:-}" ]; then # Don't do it for Codespace echo "RING_S3C_ENDPOINT=$RING_S3C_ENDPOINT" >> "$GITHUB_ENV" echo "CRR_SOURCE_LOCATION_NAME=$CRR_SOURCE_LOCATION_NAME" >> "$GITHUB_ENV" echo "CRR_DESTINATION_LOCATION_NAME=$CRR_DESTINATION_LOCATION_NAME" >> "$GITHUB_ENV" + echo "CRR_LOCATION_A_NAME=$CRR_LOCATION_A_NAME" >> "$GITHUB_ENV" + echo "CRR_LOCATION_B_NAME=$CRR_LOCATION_B_NAME" >> "$GITHUB_ENV" + echo "CRR_LOCATION_C_NAME=$CRR_LOCATION_C_NAME" >> "$GITHUB_ENV" echo "CRR_ROLE_NAME=$CRR_ROLE_NAME" >> "$GITHUB_ENV" # JSON vars need the heredoc delimiter syntax (values contain '=') printf 'CRR_SOURCE_INFO<> "$GITHUB_ENV" printf 'CRR_DESTINATION_INFO<> "$GITHUB_ENV" + printf 'CRR_INFO_A<> "$GITHUB_ENV" + printf 'CRR_INFO_B<> "$GITHUB_ENV" + printf 'CRR_INFO_C<> "$GITHUB_ENV" fi \ No newline at end of file diff --git a/.github/workflows/end2end.yaml b/.github/workflows/end2end.yaml index 80e927184d..5ac0270687 100644 --- a/.github/workflows/end2end.yaml +++ b/.github/workflows/end2end.yaml @@ -77,6 +77,12 @@ env: CRR_DESTINATION_LOCATION_NAME: crr-destination-location CRR_SOURCE_ACCOUNT_NAME: crr-source-account CRR_DESTINATION_ACCOUNT_NAME: crr-destination-account + CRR_LOCATION_A_NAME: crr-location-a + CRR_LOCATION_B_NAME: crr-location-b + CRR_LOCATION_C_NAME: crr-location-c + CRR_ACCOUNT_A_NAME: crr-account-a + CRR_ACCOUNT_B_NAME: crr-account-b + CRR_ACCOUNT_C_NAME: crr-account-c CRR_ROLE_NAME: crr-role # Enable this for Ring tests ENABLE_RING_TESTS: "false" diff --git a/tests/functional/ctst/features/replication/crrCascade.feature b/tests/functional/ctst/features/replication/crrCascade.feature new file mode 100644 index 0000000000..c06c4f47b8 --- /dev/null +++ b/tests/functional/ctst/features/replication/crrCascade.feature @@ -0,0 +1,56 @@ +Feature: CRR Cascade Replication + + Cross-Region Replication Cascade allows replication of an object from + a first location to carry on through multiple locations without + stopping at first destination. + + @2.15.0 + @PreMerge + @ReplicationTest + @CRRCascade + Scenario: Cascade replication nominal case : A -> B -> C + Given cascade replication accounts are registered + And a versioned bucket exists in location "crr-location-a" + And a versioned bucket exists in location "crr-location-b" + And a versioned bucket exists in location "crr-location-c" + And replication is configured from location "crr-location-b" to "crr-location-c" + And replication is configured from location "crr-location-a" to "crr-location-b" + When an object "cascade-obj" is put in location "crr-location-a" + Then the object should replicate to location "crr-location-b" within 300 seconds + And the object should replicate to location "crr-location-c" within 300 seconds + When I wait 15 seconds + Then the cascade replication states should be settled + + @2.15.0 + @PreMerge + @ReplicationTest + @CRRCascade + Scenario: Cascade replication with loop : A -> B -> C -> A + Given cascade replication accounts are registered + And a versioned bucket exists in location "crr-location-a" + And a versioned bucket exists in location "crr-location-b" + And a versioned bucket exists in location "crr-location-c" + And replication is configured from location "crr-location-a" to "crr-location-b" + And replication is configured from location "crr-location-b" to "crr-location-c" + And replication is configured from location "crr-location-c" to "crr-location-a" + When an object "cascade-loop-obj" is put in location "crr-location-a" + Then the object should replicate to location "crr-location-b" within 300 seconds + And the object should replicate to location "crr-location-c" within 300 seconds + And the object at location "crr-location-a" should never have replication status PENDING within 30 seconds + When I wait 15 seconds + Then the cascade replication states should be settled + + @2.15.0 + @PreMerge + @ReplicationTest + @CRRCascade + Scenario: Multiple writes across a loop converge to the last write + Given cascade replication accounts are registered + And a versioned bucket exists in location "crr-location-a" + And a versioned bucket exists in location "crr-location-b" + And a versioned bucket exists in location "crr-location-c" + And replication is configured from location "crr-location-b" to "crr-location-c" + And replication is configured from location "crr-location-c" to "crr-location-a" + And replication is configured from location "crr-location-a" to "crr-location-b" + When the object "cascade-convergence-obj" is concurrently written 10 times to every cascade location + Then all cascade locations should converge to the same metadata marker within 300 seconds diff --git a/tests/functional/ctst/features/crrReplicationS3utils.feature b/tests/functional/ctst/features/replication/crrReplicationS3utils.feature similarity index 100% rename from tests/functional/ctst/features/crrReplicationS3utils.feature rename to tests/functional/ctst/features/replication/crrReplicationS3utils.feature diff --git a/tests/functional/ctst/features/replication.feature b/tests/functional/ctst/features/replication/replication.feature similarity index 100% rename from tests/functional/ctst/features/replication.feature rename to tests/functional/ctst/features/replication/replication.feature diff --git a/tests/functional/ctst/steps/crrCascade.ts b/tests/functional/ctst/steps/crrCascade.ts new file mode 100644 index 0000000000..4642f3eadc --- /dev/null +++ b/tests/functional/ctst/steps/crrCascade.ts @@ -0,0 +1,266 @@ +import { Given, Then, When } from '@cucumber/cucumber'; +import { + CreateBucketCommand, + HeadObjectCommand, + PutBucketReplicationCommand, + PutBucketVersioningCommand, + PutObjectCommand, + StorageClass, +} from '@aws-sdk/client-s3'; +import assert from 'assert'; +import { Identity, IdentityEnum, Utils } from 'cli-testing'; +import Zenko from 'world/Zenko'; + +interface CRRAccountInfo { + AccessKeyId: string; + SecretAccessKey: string; + SessionToken?: string; + AccountId: string; +} + +Given('cascade replication accounts are registered', function (this: Zenko) { + const roleName = process.env.CRR_ROLE_NAME; + assert.ok(roleName, 'CRR_ROLE_NAME must be set'); + + const locationNames = [ + process.env.CRR_LOCATION_A_NAME, + process.env.CRR_LOCATION_B_NAME, + process.env.CRR_LOCATION_C_NAME, + ]; + assert.ok(locationNames.every(Boolean), 'CRR_LOCATION_A/B/C_NAME must be set'); + + const infoByLocation: Record = {}; + for (const [i, loc] of locationNames.entries()) { + const varName = `CRR_INFO_${['A', 'B', 'C'][i]}`; + const raw = process.env[varName]; + assert.ok(raw, `${varName} must be set`); + const info = JSON.parse(raw) as CRRAccountInfo; + Identity.addIdentity(IdentityEnum.ACCOUNT, loc!, { + accessKeyId: info.AccessKeyId, + secretAccessKey: info.SecretAccessKey, + sessionToken: info.SessionToken, + }); + infoByLocation[loc!] = info; + } + + this.addToSaved('cascadeInfoByLocation', infoByLocation); + this.addToSaved('cascadeRoleName', roleName); + this.addToSaved('cascadeBuckets', {} as Record); +}); + +Given('a versioned bucket exists in location {string}', async function (this: Zenko, location: string) { + const bucket = `cascade-${Utils.randomString().toLowerCase()}`; + Identity.useIdentity(IdentityEnum.ACCOUNT, location); + const client = this.createS3Client(); + await client.send(new CreateBucketCommand({ Bucket: bucket })); + await client.send(new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Enabled' }, + })); + const cascadeBuckets = this.getSaved>('cascadeBuckets'); + cascadeBuckets[location] = bucket; +}); + +Given( + 'replication is configured from location {string} to {string}', + async function (this: Zenko, srcLocation: string, dstLocation: string) { + const infoByLocation = this.getSaved>('cascadeInfoByLocation'); + const cascadeBuckets = this.getSaved>('cascadeBuckets'); + const roleName = this.getSaved('cascadeRoleName'); + const role = [ + 'arn:aws:iam::root:role/s3-replication-role', + `arn:aws:iam::${infoByLocation[dstLocation].AccountId}:role/${roleName}`, + ].join(','); + + Identity.useIdentity(IdentityEnum.ACCOUNT, srcLocation); + await this.createS3Client().send(new PutBucketReplicationCommand({ + Bucket: cascadeBuckets[srcLocation], + ReplicationConfiguration: { + Role: role, + Rules: [{ + Status: 'Enabled', + Prefix: '', + Destination: { + Bucket: `arn:aws:s3:::${cascadeBuckets[dstLocation]}`, + StorageClass: dstLocation as StorageClass, + }, + }], + }, + })); + }, +); + +When('an object {string} is put in location {string}', + async function (this: Zenko, objectName: string, location: string) { + const cascadeBuckets = this.getSaved>('cascadeBuckets'); + const marker = Utils.randomString().toLowerCase(); + Identity.useIdentity(IdentityEnum.ACCOUNT, location); + await this.createS3Client().send(new PutObjectCommand({ + Bucket: cascadeBuckets[location], + Key: objectName, + Body: new Uint8Array(0), + Metadata: { marker }, + })); + this.addToSaved('cascadeObjectName', objectName); + this.addToSaved('cascadeSourceLocation', location); + this.addToSaved('cascadeLastMarker', marker); + }); + +Then( + 'the object should replicate to location {string} within {int} seconds', + { timeout: 300_000 }, + async function (this: Zenko, location: string, timeoutSeconds: number) { + const bucket = this.getSaved>('cascadeBuckets')[location]; + const objectName = this.getSaved('cascadeObjectName'); + const deadline = Date.now() + timeoutSeconds * 1000; + Identity.useIdentity(IdentityEnum.ACCOUNT, location); + const client = this.createS3Client(); + while (Date.now() < deadline) { + try { + const res = await client.send( + new HeadObjectCommand({ Bucket: bucket, Key: objectName }), + ); + assert.strictEqual( + res.ReplicationStatus, 'REPLICA', + `Expected ReplicationStatus to be REPLICA at '${location}', got '${res.ReplicationStatus}'`, + ); + return; + } catch (err: unknown) { + const status = (err as { $metadata?: { httpStatusCode?: number } }).$metadata?.httpStatusCode; + if (status !== 404) { + throw err; + } + await new Promise(resolve => setTimeout(resolve, 3000)); + } + } + assert.fail(`Timeout: object '${objectName}' not found in bucket '${bucket}' after ${timeoutSeconds}s`); + }, +); + +Then( + 'the object at location {string} should never have replication status PENDING within {int} seconds', + { timeout: 120_000 }, + async function (this: Zenko, location: string, waitSeconds: number) { + const cascadeBuckets = this.getSaved>('cascadeBuckets'); + const objectName = this.getSaved('cascadeObjectName'); + const bucket = cascadeBuckets[location]; + const deadline = Date.now() + waitSeconds * 1000; + Identity.useIdentity(IdentityEnum.ACCOUNT, location); + const client = this.createS3Client(); + while (Date.now() < deadline) { + const result = await client.send( + new HeadObjectCommand({ Bucket: bucket, Key: objectName }), + ); + assert.notStrictEqual( + result.ReplicationStatus, + 'PENDING', + `Object at '${location}' was found with ReplicationStatus=PENDING, ` + + 'indicating the cascade loop wrote back to the source.', + ); + await new Promise(resolve => setTimeout(resolve, 1000)); + } + }, +); + +Then( + 'the cascade replication states should be settled', + { timeout: 30_000 }, + async function (this: Zenko) { + const cascadeBuckets = this.getSaved>('cascadeBuckets'); + const objectName = this.getSaved('cascadeObjectName'); + const sourceLocation = this.getSaved('cascadeSourceLocation'); + + for (const [location, bucket] of Object.entries(cascadeBuckets)) { + Identity.useIdentity(IdentityEnum.ACCOUNT, location); + const res = await this.createS3Client().send( + new HeadObjectCommand({ Bucket: bucket, Key: objectName }), + ); + + const expectedStatus = location === sourceLocation ? 'COMPLETED' : 'REPLICA'; + assert.strictEqual( + res.ReplicationStatus, + expectedStatus, + `Expected ReplicationStatus '${expectedStatus}' at '${location}', got '${res.ReplicationStatus}'`, + ); + + const pendingBackends = Object.entries(res.Metadata ?? {}) + .filter(([key, value]) => key.endsWith('-replication-status') && value === 'PENDING'); + assert.strictEqual( + pendingBackends.length, 0, + `Location '${location}' still has PENDING backends after settling: ${ + pendingBackends.map(([k, v]) => `${k}=${v}`).join(', ') + }`, + ); + } + }, +); + +When('the object {string} is concurrently written {int} times to every cascade location', + async function (this: Zenko, objectName: string, writesPerLocation: number) { + const cascadeBuckets = this.getSaved>('cascadeBuckets'); + + const clientByLocation = Object.fromEntries( + Object.keys(cascadeBuckets).map(location => { + Identity.useIdentity(IdentityEnum.ACCOUNT, location); + return [location, this.createS3Client()]; + }), + ); + + await Promise.all( + Object.entries(cascadeBuckets).flatMap(([location, bucket]) => + Array.from({ length: writesPerLocation }, () => { + const marker = Utils.randomString().toLowerCase(); + return clientByLocation[location].send(new PutObjectCommand({ + Bucket: bucket, + Key: objectName, + Body: new Uint8Array(0), + Metadata: { marker }, + })); + }), + ), + ); + + this.addToSaved('cascadeObjectName', objectName); + }); + +Then( + 'all cascade locations should converge to the same metadata marker within {int} seconds', + { timeout: 600_000 }, + async function (this: Zenko, timeoutSeconds: number) { + const cascadeBuckets = this.getSaved>('cascadeBuckets'); + const objectName = this.getSaved('cascadeObjectName'); + const deadline = Date.now() + timeoutSeconds * 1000; + const clientByLocation = Object.fromEntries( + Object.keys(cascadeBuckets).map(location => { + Identity.useIdentity(IdentityEnum.ACCOUNT, location); + return [location, this.createS3Client()]; + }), + ); + + while (Date.now() < deadline) { + const markers: string[] = []; + for (const [location, bucket] of Object.entries(cascadeBuckets)) { + try { + const res = await clientByLocation[location].send( + new HeadObjectCommand({ Bucket: bucket, Key: objectName }), + ); + if (res.Metadata?.marker) { + markers.push(res.Metadata.marker); + } + } catch (err: unknown) { + const status = (err as { $metadata?: { httpStatusCode?: number } }).$metadata?.httpStatusCode; + if (status !== 404) { + throw err; + } + } + } + const uniqueMarkers = new Set(markers); + if (markers.length === Object.keys(cascadeBuckets).length && uniqueMarkers.size === 1) { + return; + } + await new Promise(resolve => setTimeout(resolve, 3000)); + } + + assert.fail(`Cascade locations did not converge to the same marker within ${timeoutSeconds}s`); + }, +); diff --git a/tests/functional/ctst/world/Zenko.ts b/tests/functional/ctst/world/Zenko.ts index d2ed6db10d..9f5afdde0f 100644 --- a/tests/functional/ctst/world/Zenko.ts +++ b/tests/functional/ctst/world/Zenko.ts @@ -966,6 +966,7 @@ export default class Zenko extends World { credentials: { accessKeyId: credentials.accessKeyId, secretAccessKey: credentials.secretAccessKey, + sessionToken: credentials.sessionToken || undefined, }, forcePathStyle: true, }); diff --git a/tests/functional/testResourcesSetup/config.ts b/tests/functional/testResourcesSetup/config.ts index c95e04676d..7992df449d 100644 --- a/tests/functional/testResourcesSetup/config.ts +++ b/tests/functional/testResourcesSetup/config.ts @@ -67,6 +67,12 @@ export interface Env { CRR_SOURCE_ACCOUNT_NAME?: string; CRR_DESTINATION_ACCOUNT_NAME?: string; CRR_DESTINATION_LOCATION_NAME?: string; + CRR_ACCOUNT_A_NAME?: string; + CRR_ACCOUNT_B_NAME?: string; + CRR_ACCOUNT_C_NAME?: string; + CRR_LOCATION_A_NAME?: string; + CRR_LOCATION_B_NAME?: string; + CRR_LOCATION_C_NAME?: string; // Cloud provider credentials — optional since a provider may not be configured AWS_ACCESS_KEY?: string; AWS_SECRET_KEY?: string; @@ -111,6 +117,12 @@ export function loadEnv(): Env { CRR_SOURCE_ACCOUNT_NAME: process.env.CRR_SOURCE_ACCOUNT_NAME, CRR_DESTINATION_ACCOUNT_NAME: process.env.CRR_DESTINATION_ACCOUNT_NAME, CRR_DESTINATION_LOCATION_NAME: process.env.CRR_DESTINATION_LOCATION_NAME, + CRR_ACCOUNT_A_NAME: process.env.CRR_ACCOUNT_A_NAME, + CRR_ACCOUNT_B_NAME: process.env.CRR_ACCOUNT_B_NAME, + CRR_ACCOUNT_C_NAME: process.env.CRR_ACCOUNT_C_NAME, + CRR_LOCATION_A_NAME: process.env.CRR_LOCATION_A_NAME, + CRR_LOCATION_B_NAME: process.env.CRR_LOCATION_B_NAME, + CRR_LOCATION_C_NAME: process.env.CRR_LOCATION_C_NAME, AWS_ACCESS_KEY: process.env.AWS_ACCESS_KEY, AWS_SECRET_KEY: process.env.AWS_SECRET_KEY, AWS_ENDPOINT: process.env.AWS_ENDPOINT, diff --git a/tests/functional/testResourcesSetup/resources.yaml.template b/tests/functional/testResourcesSetup/resources.yaml.template index fb4b7529c3..8534dfa9db 100644 --- a/tests/functional/testResourcesSetup/resources.yaml.template +++ b/tests/functional/testResourcesSetup/resources.yaml.template @@ -2,6 +2,9 @@ accounts: - "zenko" - "${CRR_SOURCE_ACCOUNT_NAME}" - "${CRR_DESTINATION_ACCOUNT_NAME}" + - "${CRR_ACCOUNT_A_NAME}" + - "${CRR_ACCOUNT_B_NAME}" + - "${CRR_ACCOUNT_C_NAME}" endpoints: [] locations: - name: "${AWS_BACKEND_DESTINATION_LOCATION}" @@ -133,3 +136,30 @@ locations: secretKey: "" bucketMatch: false repoId: [] + - name: "${CRR_LOCATION_A_NAME}" + locationType: "location-scality-crr-v1" + details: + endpoint: "http://s3.${SUBDOMAIN}:80" + stsEndpoint: "http://sts.${SUBDOMAIN}:80" + accessKey: "" + secretKey: "" + bucketMatch: false + repoId: [] + - name: "${CRR_LOCATION_B_NAME}" + locationType: "location-scality-crr-v1" + details: + endpoint: "http://s3.${SUBDOMAIN}:80" + stsEndpoint: "http://sts.${SUBDOMAIN}:80" + accessKey: "" + secretKey: "" + bucketMatch: false + repoId: [] + - name: "${CRR_LOCATION_C_NAME}" + locationType: "location-scality-crr-v1" + details: + endpoint: "http://s3.${SUBDOMAIN}:80" + stsEndpoint: "http://sts.${SUBDOMAIN}:80" + accessKey: "" + secretKey: "" + bucketMatch: false + repoId: [] diff --git a/tests/functional/testResourcesSetup/setup/locations.ts b/tests/functional/testResourcesSetup/setup/locations.ts index 361d4a6323..b4b377cffe 100644 --- a/tests/functional/testResourcesSetup/setup/locations.ts +++ b/tests/functional/testResourcesSetup/setup/locations.ts @@ -97,9 +97,20 @@ export async function createLocations( 'CRR_DESTINATION_LOCATION_NAME are required when DEPLOY_CRR_LOCATIONS=true', ); } - const accountName = location.name === env.CRR_DESTINATION_LOCATION_NAME - ? env.CRR_DESTINATION_ACCOUNT_NAME - : env.CRR_SOURCE_ACCOUNT_NAME; + const locationAccountMap: Record = { + [env.CRR_DESTINATION_LOCATION_NAME ?? '']: env.CRR_DESTINATION_ACCOUNT_NAME, + [env.CRR_LOCATION_A_NAME ?? '']: env.CRR_ACCOUNT_A_NAME, + [env.CRR_LOCATION_B_NAME ?? '']: env.CRR_ACCOUNT_B_NAME, + [env.CRR_LOCATION_C_NAME ?? '']: env.CRR_ACCOUNT_C_NAME, + }; + const accountName = locationAccountMap[location.name] ?? env.CRR_SOURCE_ACCOUNT_NAME; + if (!accountName || !accountsCreds[accountName]) { + throw new Error( + `No credentials found for CRR location "${location.name}" ` + + `(resolved account: "${accountName ?? 'undefined'}"). ` + + 'Ensure the corresponding CRR_*_ACCOUNT_NAME env var and k8s secret are set.', + ); + } const userCreds = await setupCRRIamResources(accountsCreds[accountName], env); location.details.accessKey = userCreds.accessKey; location.details.secretKey = userCreds.secretKey;