From 14aa3a0a4d7a8f72bdcc7d498804d1cb2429fef1 Mon Sep 17 00:00:00 2001 From: sylvain senechal Date: Thu, 28 May 2026 19:32:59 +0200 Subject: [PATCH] Implement CRR Cascaded feature Issue: CLDSRV-897 --- lib/api/apiUtils/object/getReplicationInfo.js | 14 +- lib/routes/routeBackbeat.js | 170 +++++++- package.json | 2 +- tests/functional/backbeat/putData.js | 126 ++++++ tests/functional/backbeat/putMetadata.js | 395 ++++++++++++++++++ tests/unit/api/apiUtils/getReplicationInfo.js | 43 ++ tests/unit/routes/routeBackbeat.js | 43 ++ yarn.lock | 13 +- 8 files changed, 788 insertions(+), 18 deletions(-) create mode 100644 tests/functional/backbeat/putData.js create mode 100644 tests/functional/backbeat/putMetadata.js diff --git a/lib/api/apiUtils/object/getReplicationInfo.js b/lib/api/apiUtils/object/getReplicationInfo.js index 2320c56e69..ab21143b90 100644 --- a/lib/api/apiUtils/object/getReplicationInfo.js +++ b/lib/api/apiUtils/object/getReplicationInfo.js @@ -66,9 +66,12 @@ function _canUserReplicate(authInfo) { * @param {string} operationType - The type of operation to replicate * @param {object} objectMD - The object metadata * @param {AuthInfo} [authInfo] - authentication info of object owner + * @param {string[]} [blockedSiteTypes=[]] - location types to exclude from the returned backends * @return {object|undefined} */ -function getReplicationInfo(s3config, objKey, bucketMD, isMD, objSize, operationType, objectMD, authInfo) { +function getReplicationInfo( + s3config, objKey, bucketMD, isMD, objSize, + operationType, objectMD, authInfo, blockedSiteTypes = []) { const config = bucketMD.getReplicationConfiguration(); if (!config || !_canUserReplicate(authInfo)) { return undefined; @@ -76,13 +79,20 @@ function getReplicationInfo(s3config, objKey, bucketMD, isMD, objSize, operation const isCloud = site => !!replicationBackends[s3config.locationConstraints[site]?.type]; const rules = _withDefaultStorageClass(config.rules || [], s3config); - const backends = ReplicationConfiguration.resolveBackends( + const allBackends = ReplicationConfiguration.resolveBackends( { ...config, rules }, objKey, isCloud, objectMD?.replicationInfo?.backends, ); + const backends = blockedSiteTypes.length > 0 + ? allBackends.filter(b => { + const loc = s3config.locationConstraints[b.site]; + return !loc || !blockedSiteTypes.includes(loc.type); + }) + : allBackends; + if (backends.length === 0) { return undefined; } diff --git a/lib/routes/routeBackbeat.js b/lib/routes/routeBackbeat.js index cde9c63a50..d32863590d 100644 --- a/lib/routes/routeBackbeat.js +++ b/lib/routes/routeBackbeat.js @@ -1,3 +1,4 @@ +const { constants: { HTTP_STATUS_CONFLICT } } = require('http2'); const url = require('url'); const async = require('async'); const httpProxy = require('http-proxy'); @@ -7,7 +8,13 @@ const joi = require('@hapi/joi'); const backbeatProxy = httpProxy.createProxyServer({ ignorePath: true, }); -const { auth, errors, errorInstances, s3middleware, s3routes, models, storage } = require('arsenal'); +const { auth, errors, errorInstances, s3middleware, s3routes, models, storage, versioning } = require('arsenal'); +const { decode, encode } = versioning.VersionID; +const { + VersionIdCollisionException, + StaleMicroVersionIdException, + MicroVersionIdAlreadyStoredException, +} = require('@scality/cloudserverclient'); const { responseJSONBody } = s3routes.routesUtils; const { getSubPartIds } = s3middleware.azureHelper.mpuUtils; @@ -21,6 +28,7 @@ const locationStorageCheck = require('../api/apiUtils/object/locationStorageChec const { dataStore } = require('../api/apiUtils/object/storeObject'); const prepareRequestContexts = require('../api/apiUtils/authorization/prepareRequestContexts'); const { decodeVersionId } = require('../api/apiUtils/object/versioning'); +const getReplicationInfo = require('../api/apiUtils/object/getReplicationInfo'); const locationKeysHaveChanged = require('../api/apiUtils/object/locationKeysHaveChanged'); const { standardMetadataValidateBucketAndObj, metadataGetObject } = require('../metadata/metadataUtils'); const { config } = require('../Config'); @@ -32,6 +40,7 @@ const { } = require('../api/apiUtils/integrity/validateChecksums'); const { BackendInfo } = models; const { pushReplicationMetric } = require('./utilities/pushReplicationMetric'); +const writeContinue = require('../utilities/writeContinue'); const kms = require('../kms/wrapper'); const { listLifecycleCurrents } = require('../api/backbeat/listLifecycleCurrents'); const { listLifecycleNonCurrents } = require('../api/backbeat/listLifecycleNonCurrents'); @@ -93,7 +102,7 @@ function _isObjectRequest(req) { return ['data', 'metadata', 'multiplebackenddata', 'multiplebackendmetadata'].includes(req.resourceType); } -function _respondWithHeaders(response, payload, extraHeaders, log, callback) { +function _respondWithHeaders(response, payload, extraHeaders, log, callback, statusCode = 200) { let body = ''; if (typeof payload === 'string') { body = payload; @@ -115,10 +124,10 @@ function _respondWithHeaders(response, payload, extraHeaders, log, callback) { // eslint-disable-next-line no-param-reassign response.serverAccessLog.endTurnAroundTime = process.hrtime.bigint(); } - response.writeHead(200, httpHeaders); + response.writeHead(statusCode, httpHeaders); response.end(body, 'utf8', () => { log.end().info('responded with payload', { - httpCode: 200, + httpCode: statusCode, contentLength: Buffer.byteLength(body), }); callback(); @@ -129,6 +138,15 @@ function _respond(response, payload, log, callback) { _respondWithHeaders(response, payload, {}, log, callback); } +function _respondWithHeaderCrrConflict(response, log, callback, code, message, mvId) { + return _respondWithHeaders( + response, + { code, message }, + { 'x-scal-micro-version-id': mvId ? encode(mvId) : '' }, + log, callback, HTTP_STATUS_CONFLICT, + ); +} + function _getRequestPayload(req, cb) { const payload = []; let payloadLen = 0; @@ -414,6 +432,38 @@ function putData(request, response, bucketInfo, objMd, log, callback) { log.error(errMessage); return callback(errorInstances.BadRequest.customizeDescription(errMessage)); } + + const incomingVersionIdEncoded = request.headers['x-scal-version-id']; + if (incomingVersionIdEncoded != null && incomingVersionIdEncoded !== '') { + const incomingVersionIdDecoded = decode(incomingVersionIdEncoded); + if (incomingVersionIdDecoded instanceof Error) { + log.error('crr putData: failed to decode x-scal-version-id header', { + method: 'putData', + error: incomingVersionIdDecoded.message, + }); + return callback(errorInstances.BadRequest.customizeDescription( + 'bad request: invalid x-scal-version-id header')); + } + if (objMd && objMd.versionId === incomingVersionIdDecoded) { + // Data already at destination for this version; return 409 with the existing + // microVersionId so backbeat can decide if putMetadata is still needed. + log.debug('crr putData: version already at destination', { + method: 'putData', + bucketName: request.bucketName, + objectKey: request.objectKey, + hasMicroVersionId: !!objMd.microVersionId, + }); + request.resume(); + return _respondWithHeaderCrrConflict( + response, log, callback, + VersionIdCollisionException.name, + 'version id already at destination', + objMd.microVersionId, + ); + } + } + + writeContinue(request, response); const context = { bucketName: request.bucketName, owner: canonicalID, @@ -539,6 +589,64 @@ function getCanonicalIdsByAccountId(accountId, log, cb) { } function putMetadata(request, response, bucketInfo, objMd, log, callback) { + const { bucketName, objectKey } = request; + + const encodedMicroVersionId = request.headers['x-scal-micro-version-id']; + // When the header is present this is a conditional write: accept only if + // the incoming microVersionId is newer than the one already stored. + // '' is a valid value meaning the source has no microVersionId (null revision). + // Note: '' is falsy so the presence check must be !== undefined, not truthy. + const hasMicroVersionId = encodedMicroVersionId !== undefined; + let incomingMicroVersionId = null; + if (hasMicroVersionId && objMd) { + // '' means source has no microVersionId, treated as older revision + incomingMicroVersionId = encodedMicroVersionId === '' ? null : decode(encodedMicroVersionId); + if (incomingMicroVersionId instanceof Error) { + log.error('putMetadata: failed to decode x-scal-micro-version-id header', { + method: 'putMetadata', + error: incomingMicroVersionId.message, + }); + return callback(errorInstances.BadRequest.customizeDescription( + 'bad request: invalid x-scal-micro-version-id header')); + } + + // null = oldest revision (original putObject, before any metadata update). + // VersionID strings are in reverse chronological order: a lexicographically + // larger string is an older revision. + const isIncomingOlderThanCurrent = (incoming, current) => + current !== null && + (incoming === null || incoming > current); + + const objectMicroVersionId = objMd.microVersionId || null; + if (incomingMicroVersionId === objectMicroVersionId) { + log.debug('putMetadata: incoming microVersionId already stored, skipping write', { + method: 'putMetadata', + bucketName, + objectKey, + }); + request.resume(); + return _respondWithHeaderCrrConflict( + response, log, callback, + MicroVersionIdAlreadyStoredException.name, + 'incoming microVersionId already at destination', + ); + } + if (isIncomingOlderThanCurrent(incomingMicroVersionId, objectMicroVersionId)) { + log.debug('putMetadata: incoming microVersionId is older than stored, rejecting', { + method: 'putMetadata', + bucketName, + objectKey, + }); + request.resume(); + return _respondWithHeaderCrrConflict( + response, log, callback, + StaleMicroVersionIdException.name, + 'incoming revision is older than destination', + objMd?.microVersionId, + ); + } + } + return _getRequestPayload(request, (err, payload) => { if (err) { return callback(err); @@ -552,7 +660,12 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) { return callback(errors.MalformedPOSTRequest); } - const { headers, bucketName, objectKey } = request; + if (incomingMicroVersionId !== null && incomingMicroVersionId !== (omVal.microVersionId ?? null)) { + return callback(errors.BadRequest.customizeDescription( + 'bad request: x-scal-micro-version-id header does not match body microVersionId')); + } + + const { headers } = request; // Destination-side delete-marker replication. // We need the REPLICA status to distinguish from @@ -560,7 +673,7 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) { if ( omVal.isDeleteMarker && omVal.replicationInfo && - omVal.replicationInfo.status === 'REPLICA' && + (omVal.replicationInfo.isReplica === true || omVal.replicationInfo.status === 'REPLICA') && request.serverAccessLog ) { // eslint-disable-next-line no-param-reassign @@ -576,7 +689,7 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) { // The REPLICA status excludes source-side replication-status updates. if ( omVal.replicationInfo && - omVal.replicationInfo.status === 'REPLICA' && + (omVal.replicationInfo.isReplica === true || omVal.replicationInfo.status === 'REPLICA') && (omVal.originOp === 's3:ObjectTagging:Put' || omVal.originOp === 's3:ObjectTagging:Delete') && request.serverAccessLog ) { @@ -593,7 +706,7 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) { // The REPLICA status excludes source-side replication-status updates. if ( omVal.replicationInfo && - omVal.replicationInfo.status === 'REPLICA' && + (omVal.replicationInfo.isReplica === true || omVal.replicationInfo.status === 'REPLICA') && omVal.originOp === 's3:ObjectAcl:Put' && request.serverAccessLog ) { @@ -672,7 +785,8 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) { // then we want to create a version for the replica object even though // none was provided in the object metadata value. if (omVal.replicationInfo.isNFS) { - const { isReplica } = omVal.replicationInfo; + const isReplica = omVal.replicationInfo.isReplica === true + || omVal.replicationInfo.status === 'REPLICA'; versioning = isReplica; omVal.replicationInfo.isNFS = !isReplica; } @@ -724,6 +838,44 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) { options.isNull = isNull; } + const isReplicationWrite = !!headers['x-scal-replication-content']; + if (isReplicationWrite) { + const isMDOnly = headers['x-scal-replication-content'] === 'METADATA'; + const objSize = omVal['content-length'] || 0; + + // These S3-compatible Scality locations are excluded as cascade + // targets because they use the MultiBackend S3 path which bypasses + // the putData/putMetadata routes, so loop detection cannot fire + // on those destinations. + const cascadeBlockedLocationTypes = [ + 'location-scality-ring-s3-v1', + 'location-scality-artesca-s3-v1', + ]; + const nextReplInfo = getReplicationInfo( + config, objectKey, bucketInfo, isMDOnly, objSize, + null, null, null, cascadeBlockedLocationTypes); + + const hasNextHop = nextReplInfo && nextReplInfo.backends.length > 0; + + // Replicating further requires x-scal-micro-version-id for loop detection. + if (hasNextHop && !hasMicroVersionId) { + return callback(errors.InternalError.customizeDescription( + 'x-scal-micro-version-id is required when replication would trigger a next hop')); + } + + omVal.replicationInfo = hasNextHop ? nextReplInfo : { + status: '', + backends: [], + content: [], + destination: '', + storageClass: '', + role: '', + storageType: '', + dataStoreVersionId: '', + }; + omVal.replicationInfo.isReplica = true; + } + return async.series( [ // Zenko's CRR delegates replacing the account diff --git a/package.json b/package.json index c0e3dfaf94..0a623f4d37 100644 --- a/package.json +++ b/package.json @@ -65,11 +65,11 @@ "vaultclient": "scality/vaultclient#8.5.3", "werelogs": "scality/werelogs#semver:^8.2.4", "ws": "^8.18.0", + "@scality/cloudserverclient": "1.0.9", "xml2js": "^0.6.2" }, "devDependencies": { "@eslint/compat": "^1.2.2", - "@scality/cloudserverclient": "1.0.7", "@scality/eslint-config-scality": "scality/Guidelines#8.3.1", "eslint": "^9.14.0", "eslint-plugin-import": "^2.31.0", diff --git a/tests/functional/backbeat/putData.js b/tests/functional/backbeat/putData.js new file mode 100644 index 0000000000..40f5509c93 --- /dev/null +++ b/tests/functional/backbeat/putData.js @@ -0,0 +1,126 @@ +'use strict'; + +const assert = require('assert'); +const { createHash } = require('crypto'); +const { v4: uuidv4 } = require('uuid'); +const { + CreateBucketCommand, + PutBucketVersioningCommand, + PutObjectCommand, +} = require('@aws-sdk/client-s3'); + +const { versioning } = require('arsenal'); +const BucketUtility = require('../aws-node-sdk/lib/utility/bucket-util'); + +const { + BackbeatRoutesClient, + PutDataCommand, + VersionIdCollisionException, +} = require('@scality/cloudserverclient'); + +const { generateVersionId, encode: encodeVersionId } = versioning.VersionID; + +const TEST_BUCKET = `bucket-putdata-${uuidv4().split('-')[0]}`; +const OBJECT_BODY = 'imAboutToBeCascadedWitNoParachuteInMyBack'; +const OBJECT_MD5_HEX = createHash('md5').update(OBJECT_BODY).digest('hex'); +const CANONICAL_ID = '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be'; +const bucketUtil = new BucketUtility('default', {}); +const s3 = bucketUtil.s3; + +let backbeatClient; + +async function putData(key, { versionId } = {}) { + return backbeatClient.send(new PutDataCommand({ + Bucket: TEST_BUCKET, + Key: key, + ContentMD5: OBJECT_MD5_HEX, + CanonicalID: CANONICAL_ID, + VersioningRequired: true, + VersionId: versionId || undefined, + Body: Buffer.from(OBJECT_BODY), + })); +} + +before(async () => { + const creds = await s3.config.credentials(); + backbeatClient = new BackbeatRoutesClient({ + endpoint: `http://${process.env.IP || '127.0.0.1'}:8000`, + region: 'us-east-1', + credentials: { + accessKeyId: creds.accessKeyId, + secretAccessKey: creds.secretAccessKey, + }, + forcePathStyle: true, + }); + + await s3.send(new CreateBucketCommand({ Bucket: TEST_BUCKET })); + await s3.send(new PutBucketVersioningCommand({ + Bucket: TEST_BUCKET, + VersioningConfiguration: { Status: 'Enabled' }, + })); +}); + +describe('putData : VersionId collision detection', () => { + it('should throw VersionIdCollisionException with microVersionId when versionId matches current master', + async () => { + const key = 'putdata-collision'; + const putResult = await s3.send(new PutObjectCommand({ + Bucket: TEST_BUCKET, + Key: key, + Body: Buffer.from(OBJECT_BODY), + ContentType: 'text/plain', + })); + const encodedVersionId = putResult.VersionId; + assert.ok(encodedVersionId, 'PutObject should return a VersionId'); + + try { + await putData(key, { versionId: encodedVersionId }); + assert.fail('expected VersionIdCollisionException'); + } catch (err) { + assert.ok(err instanceof VersionIdCollisionException, + `expected VersionIdCollisionException, got ${err.constructor.name}`); + // microVersionId in the error lets backbeat decide whether to proceed + // with metadata-only replication or skip entirely (loop/stale detection). + // '' signals the existing object has no microVersionId (original write state). + assert.strictEqual(err.microVersionId, '', + 'microVersionId in exception should be empty when object has no microVersionId'); + } + }); + + it('should write data normally when VersionId does not match the current master', async () => { + const key = 'putdata-no-collision'; + + await s3.send(new PutObjectCommand({ + Bucket: TEST_BUCKET, + Key: key, + Body: Buffer.from(OBJECT_BODY), + ContentType: 'text/plain', + })); + + const differentVersionId = encodeVersionId(generateVersionId('different-instance', 'RG001')); + const output = await putData(key, { versionId: differentVersionId }); + assert.ok(output.Location, 'should return a Location when data is written normally'); + }); +}); + +describe('putData : baseline (no cascade headers)', () => { + it('should succeed normally when putData has no VersionId header', async () => { + const key = 'putdata-baseline-no-version-id'; + const output = await putData(key); + assert.ok(output.Location, 'putData without VersionId should return a Location'); + }); + + it('should succeed when putData is sent without Expect: 100-continue (old-client compat)', async () => { + const key = 'putdata-baseline-no-expect-continue'; + const output = await backbeatClient.send(new PutDataCommand({ + Bucket: TEST_BUCKET, + Key: key, + ContentMD5: OBJECT_MD5_HEX, + CanonicalID: CANONICAL_ID, + VersioningRequired: true, + Body: Buffer.from(OBJECT_BODY), + })); + assert.ok(output.Location, + 'putData without Expect: 100-continue should return a Location'); + }); +}); diff --git a/tests/functional/backbeat/putMetadata.js b/tests/functional/backbeat/putMetadata.js new file mode 100644 index 0000000000..4478eb659f --- /dev/null +++ b/tests/functional/backbeat/putMetadata.js @@ -0,0 +1,395 @@ +'use strict'; + +const assert = require('assert'); +const { createHash } = require('crypto'); +const { v4: uuidv4 } = require('uuid'); +const { + CreateBucketCommand, + PutBucketReplicationCommand, + PutBucketVersioningCommand, + PutObjectCommand, + PutObjectTaggingCommand, +} = require('@aws-sdk/client-s3'); + +const { versioning, models } = require('arsenal'); +const { ObjectMD } = models; +const BucketUtility = require('../aws-node-sdk/lib/utility/bucket-util'); + +const { + BackbeatRoutesClient, + GetMetadataCommand, + PutMetadataCommand, + MicroVersionIdAlreadyStoredException, + StaleMicroVersionIdException, +} = require('@scality/cloudserverclient'); + +const { generateVersionId, encode: encodeVersionId } = versioning.VersionID; + +const TEST_BUCKET = `bucket-putmetadata-${uuidv4().split('-')[0]}`; +const TEST_BUCKET_CRR = `bucket-putmetadata-crr-${uuidv4().split('-')[0]}`; +const DEST_BUCKET = `bucket-putmetadata-dest-${uuidv4().split('-')[0]}`; +const OBJECT_BODY = 'imAboutToBeCascadedWitNoParachuteInMyBack'; +const OBJECT_MD5_HEX = createHash('md5').update(OBJECT_BODY).digest('hex'); +const CANONICAL_ID = '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be'; +const bucketUtil = new BucketUtility('default', {}); +const s3 = bucketUtil.s3; + +let backbeatClient; + +function makeMicroVersionId() { + const raw = generateVersionId('test-instance', 'RG001'); + return { raw, encoded: encodeVersionId(raw) }; +} + +function buildMetadataBody(overrides) { + const obj = Object.assign({ + 'content-length': Buffer.byteLength(OBJECT_BODY), + 'content-type': 'text/plain', + 'last-modified': new Date().toISOString(), + 'x-amz-version-id': 'null', + 'owner-id': CANONICAL_ID, + 'owner-display-name': 'test', + 'content-md5': OBJECT_MD5_HEX, + replicationInfo: { + status: 'REPLICA', + isReplica: true, + backends: [], + content: [], + destination: '', + storageClass: '', + role: '', + storageType: '', + dataStoreVersionId: '', + }, + }, overrides || {}); + return Buffer.from(JSON.stringify(obj)); +} + +// putMetadata without x-scal-replication-content — tests pure conditional update semantics +async function putMetadata(key, mvId) { + const bodyOverrides = mvId ? { microVersionId: mvId.raw } : {}; + return backbeatClient.send(new PutMetadataCommand({ + Bucket: TEST_BUCKET, + Key: key, + MicroVersionId: mvId ? mvId.encoded : undefined, + Body: buildMetadataBody(bodyOverrides), + })); +} + +before(async () => { + const creds = await s3.config.credentials(); + backbeatClient = new BackbeatRoutesClient({ + endpoint: `http://${process.env.IP || '127.0.0.1'}:8000`, + region: 'us-east-1', + credentials: { + accessKeyId: creds.accessKeyId, + secretAccessKey: creds.secretAccessKey, + }, + forcePathStyle: true, + }); + + await s3.send(new CreateBucketCommand({ Bucket: TEST_BUCKET })); + await s3.send(new PutBucketVersioningCommand({ + Bucket: TEST_BUCKET, + VersioningConfiguration: { Status: 'Enabled' }, + })); + + await s3.send(new CreateBucketCommand({ Bucket: DEST_BUCKET })); + await s3.send(new PutBucketVersioningCommand({ + Bucket: DEST_BUCKET, + VersioningConfiguration: { Status: 'Enabled' }, + })); + + await s3.send(new CreateBucketCommand({ Bucket: TEST_BUCKET_CRR })); + await s3.send(new PutBucketVersioningCommand({ + Bucket: TEST_BUCKET_CRR, + VersioningConfiguration: { Status: 'Enabled' }, + })); + await s3.send(new PutBucketReplicationCommand({ + Bucket: TEST_BUCKET_CRR, + ReplicationConfiguration: { + Role: 'arn:aws:iam::account-id:role/src-resource,' + + 'arn:aws:iam::account-id:role/dest-resource', + Rules: [{ + Status: 'Enabled', + Prefix: '', + Destination: { + Bucket: `arn:aws:s3:::${DEST_BUCKET}`, + StorageClass: 'zenko', + }, + }], + }, + })); +}); + +// These tests send no x-scal-replication-content header — microVersionId is used +// purely as a conditional update mechanism, independent of replication. +describe('putMetadata : microVersionId conditional updates (no replication content)', () => { + it('should succeed on first write when destination has no microVersionId', async () => { + const key = 'putmetadata-cond-first-write'; + const mvId = makeMicroVersionId(); + await putMetadata(key, mvId); + const { Body } = await backbeatClient.send( + new GetMetadataCommand({ Bucket: TEST_BUCKET, Key: key })); + assert.strictEqual(new ObjectMD(JSON.parse(Body)).getMicroVersionId(), mvId.raw); + }); + + it('should succeed on first write when source has no microVersionId (putObject replication)', async () => { + const key = 'putmetadata-cond-first-write-null-mvid'; + // '' = source has no microVersionId (putObject state): must not be treated as a loop + await backbeatClient.send(new PutMetadataCommand({ + Bucket: TEST_BUCKET, + Key: key, + MicroVersionId: '', + Body: buildMetadataBody({}), + })); + const { Body } = await backbeatClient.send( + new GetMetadataCommand({ Bucket: TEST_BUCKET, Key: key })); + assert.strictEqual(new ObjectMD(JSON.parse(Body)).getMicroVersionId(), undefined, + 'stored object should have no microVersionId when source had none'); + }); + + it('should throw MicroVersionIdAlreadyStoredException on second write with the same microVersionId', + async () => { + const key = 'putmetadata-cond-loop'; + const mvId = makeMicroVersionId(); + await putMetadata(key, mvId); + await assert.rejects( + () => putMetadata(key, mvId), + err => { + assert.ok(err instanceof MicroVersionIdAlreadyStoredException, + 'second write with same id should throw MicroVersionIdAlreadyStoredException'); + return true; + }, + ); + }); + + it('should throw StaleMicroVersionIdException when writing with an older microVersionId', async () => { + const key = 'putmetadata-cond-stale'; + const olderMvId = makeMicroVersionId(); + const newerMvId = makeMicroVersionId(); + await putMetadata(key, newerMvId); + await assert.rejects( + () => putMetadata(key, olderMvId), + err => { + assert.ok(err instanceof StaleMicroVersionIdException, + `expected StaleMicroVersionIdException, got ${err.constructor.name}`); + return true; + }, + ); + }); + + it('should succeed and overwrite when writing with a newer microVersionId', async () => { + const key = 'putmetadata-cond-forward'; + const olderMvId = makeMicroVersionId(); + const newerMvId = makeMicroVersionId(); + await putMetadata(key, olderMvId); + await putMetadata(key, newerMvId); + const { Body } = await backbeatClient.send( + new GetMetadataCommand({ Bucket: TEST_BUCKET, Key: key })); + assert.strictEqual(new ObjectMD(JSON.parse(Body)).getMicroVersionId(), newerMvId.raw, + 'stored microVersionId should be the newer one'); + }); + + it('should return 400 when MicroVersionId header does not match body microVersionId', async () => { + const key = 'putmetadata-cond-header-body-mismatch'; + // The mismatch check only fires when the object already exists (objMd non-null), + // because incomingMicroVersionId is only decoded in that path. + const initialMvId = makeMicroVersionId(); + await putMetadata(key, initialMvId); + const headerMvId = makeMicroVersionId(); + const bodyMvId = makeMicroVersionId(); + await assert.rejects( + () => backbeatClient.send(new PutMetadataCommand({ + Bucket: TEST_BUCKET, + Key: key, + MicroVersionId: headerMvId.encoded, + Body: buildMetadataBody({ microVersionId: bodyMvId.raw }), + })), + err => { + assert.strictEqual(err.$metadata.httpStatusCode, 400, + 'mismatched header and body microVersionId should return 400'); + return true; + }, + ); + }); + + it('should throw StaleMicroVersionIdException after objectPutTagging bumped microVersionId', + async () => { + const key = 'putmetadata-cond-stale-tagging'; + + await s3.send(new PutObjectCommand({ + Bucket: TEST_BUCKET_CRR, + Key: key, + Body: Buffer.from(OBJECT_BODY), + ContentType: 'text/plain', + })); + + const olderMvId = makeMicroVersionId(); + + await s3.send(new PutObjectTaggingCommand({ + Bucket: TEST_BUCKET_CRR, + Key: key, + Tagging: { TagSet: [{ Key: 'crr', Value: 'cascade' }] }, + })); + + const { Body } = await backbeatClient.send( + new GetMetadataCommand({ Bucket: TEST_BUCKET_CRR, Key: key })); + assert.ok(new ObjectMD(JSON.parse(Body)).getMicroVersionId(), + 'objectPutTagging should have set a microVersionId'); + + await assert.rejects( + () => backbeatClient.send(new PutMetadataCommand({ + Bucket: TEST_BUCKET_CRR, + Key: key, + MicroVersionId: olderMvId.encoded, + Body: buildMetadataBody({ microVersionId: olderMvId.raw }), + })), + err => { + assert.ok(err instanceof StaleMicroVersionIdException, + `expected StaleMicroVersionIdException, got ${err.constructor.name}`); + return true; + }, + ); + }); +}); + +// These tests send x-scal-replication-content to simulate backbeat replication writes. +// microVersionId conditional update semantics apply on top of the replication behavior. +describe('putMetadata : cascade replication behavior (with replication content)', () => { + it('should succeed on first write of a zero-byte object without replication content', async () => { + const key = 'putmetadata-crr-zero-byte'; + await backbeatClient.send(new PutMetadataCommand({ + Bucket: TEST_BUCKET, + Key: key, + MicroVersionId: '', + Body: buildMetadataBody({ 'content-length': 0, location: null }), + })); + const { Body } = await backbeatClient.send( + new GetMetadataCommand({ Bucket: TEST_BUCKET, Key: key })); + assert.strictEqual(new ObjectMD(JSON.parse(Body)).getContentLength(), 0, + 'zero-byte object should be stored correctly'); + }); + + it('should throw MicroVersionIdAlreadyStoredException on loop even with replication content', + async () => { + // Verifies that microVersionId loop detection fires regardless of whether + // x-scal-replication-content is set + const key = 'putmetadata-crr-loop'; + const mvId = makeMicroVersionId(); + await putMetadata(key, mvId); + await assert.rejects( + () => backbeatClient.send(new PutMetadataCommand({ + Bucket: TEST_BUCKET, + Key: key, + MicroVersionId: mvId.encoded, + ReplicationContent: 'METADATA', + Body: buildMetadataBody({ microVersionId: mvId.raw }), + })), + err => { + assert.ok(err instanceof MicroVersionIdAlreadyStoredException, + 'loop detection should fire even with x-scal-replication-content set'); + return true; + }, + ); + }); + + it('should throw StaleMicroVersionIdException on stale even with replication content', async () => { + const key = 'putmetadata-crr-stale'; + const olderMvId = makeMicroVersionId(); + const newerMvId = makeMicroVersionId(); + await putMetadata(key, newerMvId); + await assert.rejects( + () => backbeatClient.send(new PutMetadataCommand({ + Bucket: TEST_BUCKET, + Key: key, + MicroVersionId: olderMvId.encoded, + ReplicationContent: 'METADATA', + Body: buildMetadataBody({ microVersionId: olderMvId.raw }), + })), + err => { + assert.ok(err instanceof StaleMicroVersionIdException, + 'stale detection should fire even with x-scal-replication-content set'); + return true; + }, + ); + }); + + it('should clear replicationInfo when no CRR rules match on destination bucket', async () => { + const key = 'putmetadata-crr-no-rules'; + const olderMvId = makeMicroVersionId(); + const newerMvId = makeMicroVersionId(); + // First write creates the object (required before METADATA-content writes). + await putMetadata(key, olderMvId); + // Second write with x-scal-replication-content triggers the cascade block. + await backbeatClient.send(new PutMetadataCommand({ + Bucket: TEST_BUCKET, + Key: key, + MicroVersionId: newerMvId.encoded, + ReplicationContent: 'METADATA', + Body: buildMetadataBody({ microVersionId: newerMvId.raw }), + })); + const { Body } = await backbeatClient.send( + new GetMetadataCommand({ Bucket: TEST_BUCKET, Key: key })); + const storedMd = new ObjectMD(JSON.parse(Body)); + assert.strictEqual(storedMd.getReplicationStatus(), '', + 'replication status should be cleared when no CRR rules match'); + assert.deepStrictEqual(storedMd.getReplicationBackends(), [], + 'replication backends should be empty when no CRR rules match'); + assert.strictEqual(storedMd.getReplicationIsReplica(), true, + 'isReplica should be preserved regardless of cascade triggering'); + }); + + it('should set replication status to PENDING and preserve isReplica when bucket has CRR rules', + async () => { + const key = 'putmetadata-crr-next-hop'; + const olderMvId = makeMicroVersionId(); + const newerMvId = makeMicroVersionId(); + + await backbeatClient.send(new PutMetadataCommand({ + Bucket: TEST_BUCKET_CRR, + Key: key, + MicroVersionId: olderMvId.encoded, + Body: buildMetadataBody({ microVersionId: olderMvId.raw }), + })); + await backbeatClient.send(new PutMetadataCommand({ + Bucket: TEST_BUCKET_CRR, + Key: key, + MicroVersionId: newerMvId.encoded, + ReplicationContent: 'METADATA', + Body: buildMetadataBody({ microVersionId: newerMvId.raw }), + })); + + const { Body } = await backbeatClient.send( + new GetMetadataCommand({ Bucket: TEST_BUCKET_CRR, Key: key })); + const storedMd = new ObjectMD(JSON.parse(Body)); + assert.strictEqual(storedMd.getMicroVersionId(), newerMvId.raw, + 'stored microVersionId should be the newer one'); + assert.strictEqual(storedMd.getReplicationStatus(), 'PENDING', + 'replication status should be PENDING when a CRR rule matches'); + assert.ok(storedMd.getReplicationBackends().length > 0, + 'replication backends should be populated when a CRR rule matches'); + assert.strictEqual(storedMd.getReplicationIsReplica(), true, + 'isReplica should be preserved regardless of cascade triggering'); + }); +}); + +describe('putMetadata : baseline (no cascade headers)', () => { + it('should succeed normally when putMetadata has no MicroVersionId header', async () => { + const key = 'putmetadata-baseline-no-mvid'; + await putMetadata(key, null); + }); + + it('should not set a microVersionId on a regular S3 PutObject', async () => { + const key = 'putmetadata-baseline-s3put'; + await s3.send(new PutObjectCommand({ + Bucket: TEST_BUCKET, + Key: key, + Body: Buffer.from(OBJECT_BODY), + })); + const { Body } = await backbeatClient.send( + new GetMetadataCommand({ Bucket: TEST_BUCKET, Key: key })); + assert.strictEqual(new ObjectMD(JSON.parse(Body)).getMicroVersionId(), undefined, + 'a regular S3 PutObject should not set a microVersionId'); + }); +}); diff --git a/tests/unit/api/apiUtils/getReplicationInfo.js b/tests/unit/api/apiUtils/getReplicationInfo.js index 2cb4f8ae67..56236661a7 100644 --- a/tests/unit/api/apiUtils/getReplicationInfo.js +++ b/tests/unit/api/apiUtils/getReplicationInfo.js @@ -609,4 +609,47 @@ describe('getReplicationInfo helper', () => { assert.strictEqual(cloudBackend.storageType, undefined); }); }); + + describe('blockedSiteTypes filtering', () => { + const RING_TYPE = 'location-scality-ring-s3-v1'; + const configWithRing = { + ...TEST_CONFIG, + locationConstraints: { + ...TEST_CONFIG.locationConstraints, + 'ring-site': { type: RING_TYPE }, + }, + }; + + it('should exclude backends whose location type is blocked', () => { + const replicationConfig = { + role: TWO_PART_ROLE, + rules: [ + { prefix: '', enabled: true, storageClass: 'awsbackend' }, + { prefix: '', enabled: true, storageClass: 'ring-site' }, + ], + destination: 'tosomewhere', + }; + const info = getReplicationInfo( + configWithRing, 'fookey', + new BucketInfo('b', 'id', 'name', new Date().toJSON(), + null, null, null, null, null, null, null, null, null, replicationConfig), + true, 123, null, null, null, [RING_TYPE]); + assert.strictEqual(info.backends.length, 1); + assert.strictEqual(info.backends[0].site, 'awsbackend'); + }); + + it('should return undefined when all backends are blocked', () => { + const replicationConfig = { + role: TWO_PART_ROLE, + rules: [{ prefix: '', enabled: true, storageClass: 'ring-site' }], + destination: 'tosomewhere', + }; + const info = getReplicationInfo( + configWithRing, 'fookey', + new BucketInfo('b', 'id', 'name', new Date().toJSON(), + null, null, null, null, null, null, null, null, null, replicationConfig), + true, 123, null, null, null, [RING_TYPE]); + assert.strictEqual(info, undefined); + }); + }); }); diff --git a/tests/unit/routes/routeBackbeat.js b/tests/unit/routes/routeBackbeat.js index 6a581bfb82..4107651fee 100644 --- a/tests/unit/routes/routeBackbeat.js +++ b/tests/unit/routes/routeBackbeat.js @@ -20,6 +20,7 @@ const bucketPutVersioning = require('../../../lib/api/bucketPutVersioning'); const objectPut = require('../../../lib/api/objectPut'); const { objectDelete } = require('../../../lib/api/objectDelete'); const bucketPutPolicy = require('../../../lib/api/bucketPutPolicy'); +const { versioning } = require('arsenal'); const log = new DummyRequestLogger(); @@ -180,10 +181,52 @@ describe('routeBackbeat', () => { assert.deepStrictEqual(mockResponse.body, [{}]); }); + it('should return 409 VersionIdCollisionException when versionId matches master, no microVersionId', async () => { + // Old objects without microVersionId: data is already at destination. + // Cloudserver returns 409 VersionIdCollisionException with empty x-scal-micro-version-id. + // Backbeat sees empty MicroVersionId and proceeds to putMetadata (partAlreadyAtDest). + const rawVersionId = versioning.VersionID.generateVersionId('test', 'RG001'); + const encodedVersionId = versioning.VersionID.encode(rawVersionId); + + mockRequest = prepareDummyRequest({ + 'x-scal-canonical-id': 'id', + 'content-md5': '1234', + 'content-length': '0', + 'x-scal-versioning-required': 'true', + 'x-scal-version-id': encodedVersionId, + }); + mockRequest.method = 'PUT'; + mockRequest.url = '/_/backbeat/data/bucket0/key0'; + mockRequest.destroy = () => {}; + + metadataUtils.standardMetadataValidateBucketAndObj.callsFake( + (params, denies, log, callback) => { + callback(null, { + getVersioningConfiguration: () => ({ Status: 'Enabled' }), + isVersioningEnabled: () => true, + getLocationConstraint: () => undefined, + }, { + versionId: rawVersionId, + // no microVersionId — old-format object + }); + }); + + routeBackbeat('127.0.0.1', mockRequest, mockResponse, log); + void await endPromise; + + sinon.assert.notCalled(storeObject.dataStore); + assert.strictEqual(mockResponse.statusCode, 409); + assert.strictEqual(mockResponse.body.code, 'VersionIdCollisionException'); + const [, responseHeaders] = mockResponse.writeHead.firstCall.args; + assert.strictEqual(responseHeaders['x-scal-micro-version-id'], '', + 'should have empty x-scal-micro-version-id header for old-format objects'); + }); + describe('putMetadata', () => { const bucketInfo = { getVersioningConfiguration: () => ({ Status: 'Enabled' }), isVersioningEnabled: () => true, + getReplicationConfiguration: () => null, }; let dataDeleteSpy; diff --git a/yarn.lock b/yarn.lock index e465d55da2..586c43f42b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3662,14 +3662,15 @@ resolved "https://registry.yarnpkg.com/@rtsao/scc/-/scc-1.1.0.tgz#927dd2fae9bc3361403ac2c7a00c32ddce9ad7e8" integrity sha512-zt6OdqaDoOnJ1ZYsCYGt9YmWzDXl4vQdKTyJev62gFhRGKdx7mcT54V9KIjg+d2wi9EXsPvAPKe7i7WjfVWB8g== -"@scality/cloudserverclient@1.0.7": - version "1.0.7" - resolved "https://registry.yarnpkg.com/@scality/cloudserverclient/-/cloudserverclient-1.0.7.tgz#ee9eed09cc7da5e97d5ad8359f429218a0a30859" - integrity sha512-gMDtI/ufRDVqWJlYkvqxXRfZOChBCw9uXt2lsaIvMuiqx/pDTNZMVLfPyRN5vx0tb6HP+5ZAe2FyPdtKXG24ng== +"@scality/cloudserverclient@1.0.9": + version "1.0.9" + resolved "https://registry.yarnpkg.com/@scality/cloudserverclient/-/cloudserverclient-1.0.9.tgz#0a333e39436e1f1e74279c3b5a11645be1fffbc1" + integrity sha512-ZGqH4G535opDAEP2PxdYDFG7kjZ/eBrzP3ZmO49goFVjRhyDCZ1gT0Pask3/wZcyysZ2Au1qylorZSLPiZmB2A== dependencies: "@aws-sdk/client-s3" "^3.1009.0" + "@aws-sdk/middleware-expect-continue" "^3.972.8" JSONStream "^1.3.5" - fast-xml-parser "^4.3.2" + fast-xml-parser "^5.5.7" "@scality/eslint-config-scality@scality/Guidelines#8.3.1": version "8.3.1" @@ -8312,7 +8313,7 @@ fast-xml-builder@^1.1.4: dependencies: path-expression-matcher "^1.1.3" -fast-xml-parser@5.2.5, fast-xml-parser@5.3.6, fast-xml-parser@5.5.6, fast-xml-parser@^4.3.2, fast-xml-parser@^5.0.7, fast-xml-parser@^5.5.6: +fast-xml-parser@5.2.5, fast-xml-parser@5.3.6, fast-xml-parser@5.5.6, fast-xml-parser@^5.0.7, fast-xml-parser@^5.5.6, fast-xml-parser@^5.5.7: version "5.5.7" resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-5.5.7.tgz#e1ddc86662d808450a19cf2fb6ccc9c3c9933c5d" integrity sha512-LteOsISQ2GEiDHZch6L9hB0+MLoYVLToR7xotrzU0opCICBkxOPgHAy1HxAvtxfJNXDJpgAsQN30mkrfpO2Prg==