diff --git a/CHANGELOG.md b/CHANGELOG.md index 75c6a3fd19c..0e9f825ffd9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,14 @@ # Changelog +## Unreleased + > [!WARNING] > **The minimum macOS deployment target will be raised to macOS 12 (Monterey)** with the upcoming release that adopts Xcode 27. Xcode 27 no longer supports deployment targets below macOS 12. If your app must support macOS 11 or earlier, please stay on the last SDK version released before this change. See [#8113](https://github.com/getsentry/sentry-cocoa/issues/8113) for full details. +### Fixes + +- Session replay video assembly: drop empty video segments, avoid duplicating frames at segment boundaries, and keep video timing stable when captured frames are skipped or unreadable (#8041) + ## 9.18.0 ### Features diff --git a/Sources/Swift/Integrations/SessionReplay/SentryOnDemandReplay.swift b/Sources/Swift/Integrations/SessionReplay/SentryOnDemandReplay.swift index e3ffcf52e78..5e9116c5f0d 100644 --- a/Sources/Swift/Integrations/SessionReplay/SentryOnDemandReplay.swift +++ b/Sources/Swift/Integrations/SessionReplay/SentryOnDemandReplay.swift @@ -9,6 +9,17 @@ import CoreMedia import Foundation import UIKit +func removeReplayFile(at fileURL: URL) { + guard FileManager.default.fileExists(atPath: fileURL.path) else { return } + + do { + try FileManager.default.removeItem(at: fileURL) + SentrySDKLog.debug("[Session Replay] Removed replay file at: \(fileURL.path)") + } catch { + SentrySDKLog.warning("[Session Replay] Could not delete replay file at: \(fileURL.path), reason: \(error)") + } +} + // swiftlint:disable type_body_length @objcMembers @_spi(Private) public class SentryOnDemandReplay: NSObject, SentryReplayVideoMaker { @@ -18,6 +29,10 @@ import UIKit private let processingQueue: SentryDispatchQueueWrapper private let assetWorkerQueue: SentryDispatchQueueWrapper private var _frames = [SentryReplayFrame]() + /// Guards `retainedFrameBeforeCurrentFrames`. All other accesses to it happen on + /// `processingQueue`; the lock only exists because `deinit` can run on any thread. + private let retainedFrameLock = NSLock() + private var retainedFrameBeforeCurrentFrames: SentryReplayFrame? #if SENTRY_TEST || SENTRY_TEST_CI || DEBUG //This is exposed only for tests, no need to make it thread safe. @@ -41,6 +56,19 @@ import UIKit self.processingQueue = processingQueue self.assetWorkerQueue = assetWorkerQueue } + + deinit { + // Clean shutdown removes the retained file. If the app crashes, the file stays + // in the replay folder and is loaded during crash recovery on the next launch. + let retainedFrame = retainedFrameLock.synchronized { + let frame = retainedFrameBeforeCurrentFrames + retainedFrameBeforeCurrentFrames = nil + return frame + } + if let retainedFrame = retainedFrame { + removeReplayFile(at: URL(fileURLWithPath: retainedFrame.imagePath)) + } + } public convenience init( withContentFrom outputPath: String, @@ -130,20 +158,26 @@ import UIKit SentrySDKLog.debug("[Session Replay] Releasing frames until date: \(date)") while let first = self._frames.first, first.time < date { self._frames.removeFirst() - let fileUrl = URL(fileURLWithPath: first.imagePath) - do { - try FileManager.default.removeItem(at: fileUrl) - SentrySDKLog.debug("[Session Replay] Removed frame at url: \(fileUrl.path)") - } catch { - SentrySDKLog.error("[Session Replay] Failed to remove frame at: \(fileUrl.path), reason: \(error), ignoring error") + // Retain the released frame so the next segment can still render the screen + // state at its window start; delete the previously retained frame's file once + // it is replaced. + let frameToRemove = self.retainedFrameLock.synchronized { () -> SentryReplayFrame? in + let previousFrame = self.retainedFrameBeforeCurrentFrames + self.retainedFrameBeforeCurrentFrames = first + guard previousFrame?.imagePath != first.imagePath else { return nil } + return previousFrame + } + if let frameToRemove = frameToRemove { + removeReplayFile(at: URL(fileURLWithPath: frameToRemove.imagePath)) } } SentrySDKLog.debug("[Session Replay] Frames released, remaining frames count: \(self._frames.count)") } } - public var oldestFrameDate: Date? { - return _frames.first?.time + /// Used by replay recovery after `init(withContentFrom:)` loaded surviving frames from disk. + var oldestRecoveredFrameDate: Date? { + _frames.first?.time } public func createVideoInBackgroundWith(beginning: Date, end: Date, completion: @escaping ([SentryVideoInfo]) -> Void) { @@ -162,7 +196,23 @@ import UIKit // Note: In previous implementations this method was wrapped by a sync call to the processing queue. // As this method is already called from the processing queue, we must remove the sync call. - let videoFrames = self._frames.filter { $0.time >= beginning && $0.time <= end } + guard end > beginning else { return [] } + + // Select the frames in the half-open window [beginning, end). When captures were skipped, + // hold the last frame captured before the window at the window start so the segment still + // begins with the correct screen state. + var videoFrames = _frames.filter { $0.time >= beginning && $0.time < end } + if let firstFrame = videoFrames.first { + if firstFrame.time > beginning { + let frameToHold = frameBefore(beginning) ?? firstFrame + videoFrames.insert(SentryReplayFrame(imagePath: frameToHold.imagePath, time: beginning, screenName: frameToHold.screenName), at: 0) + } + } else if let previousFrame = frameBefore(beginning) { + videoFrames = [SentryReplayFrame(imagePath: previousFrame.imagePath, time: beginning, screenName: previousFrame.screenName)] + } else { + return [] + } + var frameCount = 0 var videos = [SentryVideoInfo]() @@ -177,7 +227,7 @@ import UIKit var currentError: Error? group.enter() - self.renderVideo(with: videoFrames, from: frameCount, at: outputFileURL) { result in + self.renderVideo(with: videoFrames, fromIndex: frameCount, until: end, at: outputFileURL) { result in switch result { case .success(let videoResult): // Set the frame count/offset to the new index that is returned by the completion block. @@ -221,16 +271,33 @@ import UIKit return videos } + private func frameBefore(_ date: Date) -> SentryReplayFrame? { + let retainedFrame = retainedFrameLock.synchronized { + retainedFrameBeforeCurrentFrames + }.flatMap { $0.time < date ? $0 : nil } + let currentFrame = _frames.last(where: { $0.time < date }) + + guard let retained = retainedFrame else { return currentFrame } + guard let current = currentFrame else { return retained } + return retained.time > current.time ? retained : current + } + // swiftlint:disable function_body_length cyclomatic_complexity - private func renderVideo(with videoFrames: [SentryReplayFrame], from: Int, at outputFileURL: URL, completion: @escaping (Result) -> Void) { - SentrySDKLog.debug("[Session Replay] Rendering video with \(videoFrames.count) frames, from index: \(from), to output url: \(outputFileURL)") + private func renderVideo( + with videoFrames: [SentryReplayFrame], + fromIndex: Int, + until videoEnd: Date, + at outputFileURL: URL, + completion: @escaping (Result) -> Void + ) { + SentrySDKLog.debug("[Session Replay] Rendering video with \(videoFrames.count) frames, from index: \(fromIndex), to output url: \(outputFileURL)") - guard from < videoFrames.count else { + guard fromIndex < videoFrames.count else { SentrySDKLog.error("[Session Replay] Failed to render video, reason: index out of bounds") return completion(.failure(SentryOnDemandReplayError.indexOutOfBounds)) } - guard let image = UIImage(contentsOfFile: videoFrames[from].imagePath) else { - SentrySDKLog.error("[Session Replay] Failed to render video, reason: can't read image at path: \(videoFrames[from].imagePath)") + guard let image = UIImage(contentsOfFile: videoFrames[fromIndex].imagePath) else { + SentrySDKLog.error("[Session Replay] Failed to render video, reason: can't read image at path: \(videoFrames[fromIndex].imagePath)") return completion(.failure(SentryOnDemandReplayError.cantReadImage)) } @@ -265,8 +332,9 @@ import UIKit videoHeight: videoHeight, videoWidth: videoWidth, frameRate: frameRate, - initialFrameIndex: from, - initialImageSize: image.size + initialFrameIndex: fromIndex, + initialImageSize: image.size, + videoEnd: videoEnd ) // Append frames to the video writer input in a pull-style manner when the input is ready to receive more media data. diff --git a/Sources/Swift/Integrations/SessionReplay/SentryVideoFrameProcessor.swift b/Sources/Swift/Integrations/SessionReplay/SentryVideoFrameProcessor.swift index d31c2e62a16..a6c651aaa78 100644 --- a/Sources/Swift/Integrations/SessionReplay/SentryVideoFrameProcessor.swift +++ b/Sources/Swift/Integrations/SessionReplay/SentryVideoFrameProcessor.swift @@ -7,6 +7,16 @@ import Foundation import UIKit class SentryVideoFrameProcessor { + private enum AppendFrameResult { + case success + case notReady + case failure + } + + /// Tolerance applied when mapping capture timestamps to presentation frame indices, to + /// absorb floating-point error at frame boundaries. + private static let frameIndexEpsilon = 0.000001 + let videoFrames: [SentryReplayFrame] let videoWriter: AVAssetWriter let currentPixelBuffer: SentryAppendablePixelBuffer @@ -14,11 +24,16 @@ class SentryVideoFrameProcessor { let videoHeight: CGFloat let videoWidth: CGFloat let frameRate: Int + let videoEnd: Date? var frameIndex: Int var lastImageSize: CGSize var usedFrames: [SentryReplayFrame] var isFinished: Bool + private var outputFrameIndex: Int + private var videoStart: Date? + private var lastAppendedImage: UIImage? + private var lastAppendedFrame: SentryReplayFrame? init( videoFrames: [SentryReplayFrame], @@ -29,7 +44,8 @@ class SentryVideoFrameProcessor { videoWidth: CGFloat, frameRate: Int, initialFrameIndex: Int, - initialImageSize: CGSize + initialImageSize: CGSize, + videoEnd: Date? = nil ) { self.videoFrames = videoFrames self.videoWriter = videoWriter @@ -38,14 +54,19 @@ class SentryVideoFrameProcessor { self.videoHeight = videoHeight self.videoWidth = videoWidth self.frameRate = frameRate + self.videoEnd = videoEnd self.frameIndex = initialFrameIndex + self.outputFrameIndex = 0 self.lastImageSize = initialImageSize self.usedFrames = [] self.isFinished = false } + // swiftlint:disable function_body_length cyclomatic_complexity func processFrames(videoWriterInput: AVAssetWriterInput, onCompletion: @escaping (Result) -> Void) { + guard !isFinished else { return } + // Use the recommended loop pattern for AVAssetWriterInput // See https://developer.apple.com/documentation/avfoundation/avassetwriterinput/requestmediadatawhenready(on:using:)#Discussion // This could lead to an infinite loop if we don't make sure to mark the input as finished when the video is finished either by the end of the frames or by an error. @@ -53,57 +74,149 @@ class SentryVideoFrameProcessor { SentrySDKLog.debug("[Session Replay] Video writer input is ready, status: \(videoWriter.status)") guard videoWriter.status == .writing else { SentrySDKLog.error("[Session Replay] Video writer is not writing anymore, cancelling the writing session, reason: \(videoWriter.error?.localizedDescription ?? "Unknown error")") + isFinished = true videoWriter.inputs.forEach { $0.markAsFinished() } videoWriter.cancelWriting() return onCompletion(.failure(videoWriter.error ?? SentryOnDemandReplayError.errorRenderingVideo)) } guard frameIndex < videoFrames.count else { + // When rendering a bounded segment, keep the last captured frame on screen + // until `videoEnd`. Without bounds, finish at the last captured frame. + if let videoEnd = videoEnd, let videoStart = videoStart { + let elapsed = max(0, videoEnd.timeIntervalSince(videoStart)) + let targetFrameIndex = max(0, Int(ceil(elapsed * Double(frameRate) - Self.frameIndexEpsilon))) + do { + guard try shouldContinueProcessing( + after: repeatLastFrame(untilFrameIndex: targetFrameIndex, videoWriterInput: videoWriterInput) + ) else { return } + } catch { + return onCompletion(.failure(error)) + } + } + SentrySDKLog.debug("[Session Replay] No more frames available to process, finishing the video") return finishVideo(frameIndex: self.frameIndex, onCompletion: onCompletion) } - + let frame = videoFrames[frameIndex] - defer { - // Increment the frame index even if the image could not be appended to the pixel buffer. - // This is important to avoid an infinite loop. - frameIndex += 1 - } guard let image = UIImage(contentsOfFile: frame.imagePath) else { - // Continue with the next frame + if lastAppendedImage != nil { + if let videoStart = videoStart { + let elapsed = max(0, frame.time.timeIntervalSince(videoStart)) + let targetFrameIndex = max(0, Int(floor(elapsed * Double(frameRate) + Self.frameIndexEpsilon))) + do { + guard try shouldContinueProcessing( + after: repeatLastFrame(untilFrameIndex: targetFrameIndex, videoWriterInput: videoWriterInput) + ) else { return } + } catch { + return onCompletion(.failure(error)) + } + } + } else { + SentrySDKLog.warning("[Session Replay] Could not load initial replay frame image, skipping frame.") + } + frameIndex += 1 continue } - + + if let videoStart = videoStart { + let elapsed = max(0, frame.time.timeIntervalSince(videoStart)) + let targetFrameIndex = max(0, Int(floor(elapsed * Double(frameRate) + Self.frameIndexEpsilon))) + do { + guard try shouldContinueProcessing( + after: repeatLastFrame(untilFrameIndex: targetFrameIndex, videoWriterInput: videoWriterInput) + ) else { return } + } catch { + return onCompletion(.failure(error)) + } + } + SentrySDKLog.debug("[Session Replay] Image at index \(frameIndex) is ready, size: \(image.size)") guard lastImageSize == image.size else { SentrySDKLog.debug("[Session Replay] Image size has changed, finishing video") - return finishVideo(frameIndex: self.frameIndex, onCompletion: onCompletion) + finishVideo(frameIndex: self.frameIndex, onCompletion: onCompletion) + return } lastImageSize = image.size - - let presentTime = SentryOnDemandReplay.calculatePresentationTime( - forFrameAtIndex: frameIndex, - frameRate: frameRate - ).timeValue - guard currentPixelBuffer.append(image: image, presentationTime: presentTime) else { - SentrySDKLog.error("[Session Replay] Failed to append image to pixel buffer, cancelling the writing session, reason: \(String(describing: videoWriter.error))") - videoWriter.inputs.forEach { $0.markAsFinished() } - videoWriter.cancelWriting() - return onCompletion(.failure(videoWriter.error ?? SentryOnDemandReplayError.errorRenderingVideo)) + + do { + guard try shouldContinueProcessing( + after: append(image: image, forFrame: frame, videoWriterInput: videoWriterInput) + ) else { return } + } catch { + return onCompletion(.failure(error)) } - usedFrames.append(frame) + + frameIndex += 1 } } - // swiftlint:enable function_body_length cyclomatic_complexity + + private func shouldContinueProcessing(after result: AppendFrameResult) throws -> Bool { + switch result { + case .success: + return true + case .notReady: + return false + case .failure: + SentrySDKLog.error("[Session Replay] Failed to append image to pixel buffer, cancelling the writing session, reason: \(String(describing: videoWriter.error))") + isFinished = true + videoWriter.inputs.forEach { $0.markAsFinished() } + videoWriter.cancelWriting() + throw videoWriter.error ?? SentryOnDemandReplayError.errorRenderingVideo + } + } + + /// Repeats the most recently appended image until the output reaches `targetFrameIndex`. + private func repeatLastFrame(untilFrameIndex targetFrameIndex: Int, videoWriterInput: AVAssetWriterInput) -> AppendFrameResult { + guard let lastAppendedImage = lastAppendedImage, + let lastAppendedFrame = lastAppendedFrame + else { return .success } + + while outputFrameIndex < targetFrameIndex { + switch append(image: lastAppendedImage, forFrame: lastAppendedFrame, videoWriterInput: videoWriterInput) { + case .success: + break + case .notReady: + return .notReady + case .failure: + return .failure + } + } + + return .success + } + + private func append(image: UIImage, forFrame frame: SentryReplayFrame, videoWriterInput: AVAssetWriterInput) -> AppendFrameResult { + guard videoWriterInput.isReadyForMoreMediaData else { return .notReady } + + if videoStart == nil { + videoStart = frame.time + } + + let presentTime = SentryOnDemandReplay.calculatePresentationTime( + forFrameAtIndex: outputFrameIndex, + frameRate: frameRate + ).timeValue + guard currentPixelBuffer.append(image: image, presentationTime: presentTime) else { return .failure } + + usedFrames.append(frame) + lastAppendedImage = image + lastAppendedFrame = frame + outputFrameIndex += 1 + return .success + } + func finishVideo(frameIndex: Int, onCompletion completion: @escaping (Result) -> Void) { + guard !isFinished else { return }; isFinished = true + // Note: This method is expected to be called from the asset worker queue and *not* the processing queue. SentrySDKLog.info("[Session Replay] Finishing video with output file URL: \(outputFileURL), used frames count: \(usedFrames.count), video height: \(videoHeight), video width: \(videoWidth)") videoWriter.inputs.forEach { $0.markAsFinished() } videoWriter.finishWriting { [weak self] in guard let self = self else { SentrySDKLog.warning("[Session Replay] On-demand replay is deallocated, completing writing session without output video info") - let videoResult = SentryRenderVideoResult(info: nil, finalFrameIndex: frameIndex) - return completion(.success(videoResult)) + return completion(.success(SentryRenderVideoResult(info: nil, finalFrameIndex: frameIndex))) } SentrySDKLog.debug("[Session Replay] Finished video writing, status: \(self.videoWriter.status)") @@ -118,6 +231,13 @@ class SentryVideoFrameProcessor { return completion(.success(videoResult)) case .completed: SentrySDKLog.debug("[Session Replay] Finish writing video was completed, creating video info from file attributes.") + guard !self.usedFrames.isEmpty else { + SentrySDKLog.debug("[Session Replay] Finished video writing without appended frames, completing with no video info") + removeReplayFile(at: self.outputFileURL) + let videoResult = SentryRenderVideoResult(info: nil, finalFrameIndex: frameIndex) + return completion(.success(videoResult)) + } + do { let videoInfo = try self.getVideoInfo( from: self.outputFileURL, @@ -158,7 +278,7 @@ class SentryVideoFrameProcessor { SentrySDKLog.warning("[Session Replay] Failed to read video start time from used frames, reason: no frames found") throw SentryOnDemandReplayError.cantReadVideoStartTime } - let duration = TimeInterval(usedFrames.count / self.frameRate) + let duration = TimeInterval(usedFrames.count) / TimeInterval(self.frameRate) return SentryVideoInfo( path: outputFileURL, height: videoHeight, diff --git a/Sources/Swift/Integrations/SessionReplay/SessionReplayRecovery.swift b/Sources/Swift/Integrations/SessionReplay/SessionReplayRecovery.swift index a8a5fd4abb1..ebc2176c061 100644 --- a/Sources/Swift/Integrations/SessionReplay/SessionReplayRecovery.swift +++ b/Sources/Swift/Integrations/SessionReplay/SessionReplayRecovery.swift @@ -115,7 +115,7 @@ struct SessionReplayRecovery { if hasCrashInfo { beginning = Date(timeIntervalSinceReferenceDate: crashInfo.lastSegmentEnd) } else { - guard let oldestFrame = resumeReplayMaker.oldestFrameDate else { + guard let oldestFrame = resumeReplayMaker.oldestRecoveredFrameDate else { SentrySDKLog.debug("[Session Replay] No frames to send, dropping replay") return nil } diff --git a/Tests/SentryTests/Integrations/SessionReplay/SentryOnDemandReplayTests.swift b/Tests/SentryTests/Integrations/SessionReplay/SentryOnDemandReplayTests.swift index 63cd9bd793f..de86c3f570a 100644 --- a/Tests/SentryTests/Integrations/SessionReplay/SentryOnDemandReplayTests.swift +++ b/Tests/SentryTests/Integrations/SessionReplay/SentryOnDemandReplayTests.swift @@ -64,6 +64,34 @@ class SentryOnDemandReplayTests: XCTestCase { XCTAssertEqual(frames.first?.time, start.addingTimeInterval(5)) XCTAssertEqual(frames.last?.time, start.addingTimeInterval(9)) } + + func testDeinit_whenFrameIsRetainedBeforeCurrentFrames_shouldRemoveRetainedFrameFile() throws { + let start = Date(timeIntervalSinceReferenceDate: 0) + let retainedFramePath = outputPath + .appendingPathComponent("\(start.timeIntervalSinceReferenceDate)") + .appendingPathExtension("png") + .path + + let processingQueue = SentryDispatchQueueWrapper() + let workerQueue = SentryDispatchQueueWrapper() + var sut: SentryOnDemandReplay? = SentryOnDemandReplay( + outputPath: outputPath.path, + processingQueue: processingQueue, + assetWorkerQueue: workerQueue + ) + + sut?.addFrameAsync(timestamp: start, maskedViewImage: UIImage.add) + sut?.addFrameAsync(timestamp: start.addingTimeInterval(1), maskedViewImage: UIImage.add) + processingQueue.queue.sync {} + sut?.releaseFramesUntil(start.addingTimeInterval(1)) + processingQueue.queue.sync {} + + XCTAssertTrue(FileManager.default.fileExists(atPath: retainedFramePath)) + + sut = nil + + XCTAssertFalse(FileManager.default.fileExists(atPath: retainedFramePath)) + } func testFramesWithScreenName() { let sut = getSut() @@ -113,6 +141,130 @@ class SentryOnDemandReplayTests: XCTestCase { try FileManager.default.removeItem(at: videoPath) wait(for: [videoExpectation], timeout: 1) } + + func testGenerateVideo_whenFramesHaveGap_shouldHoldPreviousFrame() throws { + // -- Arrange -- + let sut = getSut() + + let start = Date(timeIntervalSinceReferenceDate: 0) + sut.addFrameAsync(timestamp: start, maskedViewImage: UIImage.add) + sut.addFrameAsync(timestamp: start.addingTimeInterval(3), maskedViewImage: UIImage.add) + + // -- Act -- + let videos = sut.createVideoWith(beginning: start, end: start.addingTimeInterval(5)) + + // -- Assert -- + XCTAssertEqual(videos.count, 1) + let info = try XCTUnwrap(videos.first) + + XCTAssertEqual(info.duration, 5) + XCTAssertEqual(info.frameCount, 5) + XCTAssertEqual(info.start, start) + XCTAssertEqual(info.end, start.addingTimeInterval(5)) + + try FileManager.default.removeItem(at: info.path) + } + + func testGenerateVideo_whenFirstFrameIsAfterBeginning_shouldKeepSegmentDuration() throws { + // -- Arrange -- + let sut = getSut() + + let start = Date(timeIntervalSinceReferenceDate: 0) + sut.addFrameAsync(timestamp: start.addingTimeInterval(2), maskedViewImage: UIImage.add) + + // -- Act -- + let videos = sut.createVideoWith(beginning: start, end: start.addingTimeInterval(5)) + + // -- Assert -- + XCTAssertEqual(videos.count, 1) + let info = try XCTUnwrap(videos.first) + + XCTAssertEqual(info.duration, 5) + XCTAssertEqual(info.frameCount, 5) + XCTAssertEqual(info.start, start) + XCTAssertEqual(info.end, start.addingTimeInterval(5)) + + try FileManager.default.removeItem(at: info.path) + } + + func testGenerateVideo_whenFrameExistsAtEnd_shouldKeepSegmentDuration() throws { + // -- Arrange -- + let sut = getSut() + + let start = Date(timeIntervalSinceReferenceDate: 0) + for i in 0...5 { + sut.addFrameAsync(timestamp: start.addingTimeInterval(TimeInterval(i)), maskedViewImage: UIImage.add) + } + + // -- Act -- + let firstSegment = sut.createVideoWith(beginning: start, end: start.addingTimeInterval(5)) + let secondSegment = sut.createVideoWith(beginning: start.addingTimeInterval(5), end: start.addingTimeInterval(10)) + + // -- Assert -- + XCTAssertEqual(firstSegment.count, 1) + let firstInfo = try XCTUnwrap(firstSegment.first) + XCTAssertEqual(firstInfo.duration, 5) + XCTAssertEqual(firstInfo.frameCount, 5) + XCTAssertEqual(firstInfo.start, start) + XCTAssertEqual(firstInfo.end, start.addingTimeInterval(5)) + + XCTAssertEqual(secondSegment.count, 1) + let secondInfo = try XCTUnwrap(secondSegment.first) + XCTAssertEqual(secondInfo.duration, 5) + XCTAssertEqual(secondInfo.frameCount, 5) + XCTAssertEqual(secondInfo.start, start.addingTimeInterval(5)) + XCTAssertEqual(secondInfo.end, start.addingTimeInterval(10)) + + try FileManager.default.removeItem(at: firstInfo.path) + try FileManager.default.removeItem(at: secondInfo.path) + } + + func testGenerateVideo_whenOnlyPreviousFrameExists_shouldHoldPreviousFrameFromBeginning() throws { + // -- Arrange -- + let sut = getSut() + + let start = Date(timeIntervalSinceReferenceDate: 0) + sut.addFrameAsync(timestamp: start.addingTimeInterval(4), maskedViewImage: UIImage.add) + sut.releaseFramesUntil(start.addingTimeInterval(5)) + + // -- Act -- + let videos = sut.createVideoWith(beginning: start.addingTimeInterval(5), end: start.addingTimeInterval(10)) + + // -- Assert -- + XCTAssertEqual(videos.count, 1) + let info = try XCTUnwrap(videos.first) + + XCTAssertEqual(info.duration, 5) + XCTAssertEqual(info.frameCount, 5) + XCTAssertEqual(info.start, start.addingTimeInterval(5)) + XCTAssertEqual(info.end, start.addingTimeInterval(10)) + + try FileManager.default.removeItem(at: info.path) + } + + func testGenerateVideo_whenFirstFrameAfterBeginning_shouldHoldPreviousFrameUntilFirstFrame() throws { + // -- Arrange -- + let sut = getSut() + + let start = Date(timeIntervalSinceReferenceDate: 0) + sut.addFrameAsync(timestamp: start.addingTimeInterval(4), maskedViewImage: UIImage.add) + sut.addFrameAsync(timestamp: start.addingTimeInterval(7), maskedViewImage: UIImage.add) + sut.releaseFramesUntil(start.addingTimeInterval(5)) + + // -- Act -- + let videos = sut.createVideoWith(beginning: start.addingTimeInterval(5), end: start.addingTimeInterval(10)) + + // -- Assert -- + XCTAssertEqual(videos.count, 1) + let info = try XCTUnwrap(videos.first) + + XCTAssertEqual(info.duration, 5) + XCTAssertEqual(info.frameCount, 5) + XCTAssertEqual(info.start, start.addingTimeInterval(5)) + XCTAssertEqual(info.end, start.addingTimeInterval(10)) + + try FileManager.default.removeItem(at: info.path) + } func testAddFrameIsThreadSafe() { let processingQueue = SentryDispatchQueueWrapper() diff --git a/Tests/SentryTests/Integrations/SessionReplay/SentrySessionReplayIntegrationTests.swift b/Tests/SentryTests/Integrations/SessionReplay/SentrySessionReplayIntegrationTests.swift index 165a2d5077d..6bcdc03978c 100644 --- a/Tests/SentryTests/Integrations/SessionReplay/SentrySessionReplayIntegrationTests.swift +++ b/Tests/SentryTests/Integrations/SessionReplay/SentrySessionReplayIntegrationTests.swift @@ -294,9 +294,9 @@ class SentrySessionReplayIntegrationTests: XCTestCase { let replayInfo = try XCTUnwrap(hub.capturedReplayRecordingVideo.first) XCTAssertEqual(replayInfo.replay.replayType, SentryReplayType.session) XCTAssertEqual(replayInfo.recording.segmentId, 2) - XCTAssertEqual(replayInfo.replay.replayStartTimestamp, Date(timeIntervalSinceReferenceDate: 5)) + XCTAssertEqual(replayInfo.replay.replayStartTimestamp, Date(timeIntervalSinceReferenceDate: 4)) } - + func testBufferReplayForCrash() throws { try createLastSessionReplay(writeSessionInfo: false) diff --git a/Tests/SentryTests/Integrations/SessionReplay/SentryVideoFrameProcessorTests.swift b/Tests/SentryTests/Integrations/SessionReplay/SentryVideoFrameProcessorTests.swift index 806d1af32e8..8ab776e3da0 100644 --- a/Tests/SentryTests/Integrations/SessionReplay/SentryVideoFrameProcessorTests.swift +++ b/Tests/SentryTests/Integrations/SessionReplay/SentryVideoFrameProcessorTests.swift @@ -28,6 +28,7 @@ class SentryVideoFrameProcessorTests: XCTestCase { var errorOverride: Error? var cancelWritingCalled = false var finishWritingCalled = false + var finishWritingInvocations = Invocations() var trackedInputs: [AVAssetWriterInput] = [] override var status: AVAssetWriter.Status { @@ -44,6 +45,7 @@ class SentryVideoFrameProcessorTests: XCTestCase { override func finishWriting(completionHandler: @escaping () -> Void) { finishWritingCalled = true + finishWritingInvocations.record(Void()) completionHandler() } @@ -63,6 +65,7 @@ class SentryVideoFrameProcessorTests: XCTestCase { override func finishWriting(completionHandler: @escaping () -> Void) { finishWritingCalled = true + finishWritingInvocations.record(Void()) if shouldExecuteCompletionImmediately { completionHandler() } else { @@ -213,6 +216,112 @@ class SentryVideoFrameProcessorTests: XCTestCase { XCTAssertEqual(sut.usedFrames.count, 3) } + func testProcessFrames_WhenFramesHaveGap_ShouldHoldPreviousFrame() { + let videoWriterInput = TestAVAssetWriterInput(mediaType: .video, outputSettings: nil) + fixture.videoWriter.add(videoWriterInput) + + let frames = [ + SentryReplayFrame( + imagePath: fixture.videoFrames[0].imagePath, + time: Date(timeIntervalSinceReferenceDate: 0), + screenName: "A" + ), + SentryReplayFrame( + imagePath: fixture.videoFrames[0].imagePath, + time: Date(timeIntervalSinceReferenceDate: 3), + screenName: "B" + ) + ] + let sut = SentryVideoFrameProcessor( + videoFrames: frames, + videoWriter: fixture.videoWriter, + currentPixelBuffer: fixture.currentPixelBuffer, + outputFileURL: fixture.outputFileURL, + videoHeight: fixture.videoHeight, + videoWidth: fixture.videoWidth, + frameRate: fixture.frameRate, + initialFrameIndex: 0, + initialImageSize: fixture.initialImageSize, + videoEnd: Date(timeIntervalSinceReferenceDate: 5) + ) + + sut.processFrames(videoWriterInput: videoWriterInput) { _ in } + + XCTAssertEqual(fixture.currentPixelBuffer.appendInvocations.count, 5) + let presentationTimes = fixture.currentPixelBuffer.appendInvocations.invocations.map { $0.presentationTime.seconds } + XCTAssertEqual(presentationTimes, [0, 1, 2, 3, 4]) + XCTAssertEqual(sut.usedFrames.compactMap(\.screenName), ["A", "A", "A", "B", "B"]) + } + + func testProcessFrames_WhenFramesHaveFractionalGap_ShouldNotOverExpandDuration() { + let videoWriterInput = TestAVAssetWriterInput(mediaType: .video, outputSettings: nil) + fixture.videoWriter.add(videoWriterInput) + + let frames = [ + SentryReplayFrame( + imagePath: fixture.videoFrames[0].imagePath, + time: Date(timeIntervalSinceReferenceDate: 0), + screenName: "A" + ), + SentryReplayFrame( + imagePath: fixture.videoFrames[0].imagePath, + time: Date(timeIntervalSinceReferenceDate: 2.4), + screenName: "B" + ) + ] + let sut = SentryVideoFrameProcessor( + videoFrames: frames, + videoWriter: fixture.videoWriter, + currentPixelBuffer: fixture.currentPixelBuffer, + outputFileURL: fixture.outputFileURL, + videoHeight: fixture.videoHeight, + videoWidth: fixture.videoWidth, + frameRate: fixture.frameRate, + initialFrameIndex: 0, + initialImageSize: fixture.initialImageSize, + videoEnd: Date(timeIntervalSinceReferenceDate: 2.4) + ) + + sut.processFrames(videoWriterInput: videoWriterInput) { _ in } + + XCTAssertEqual(fixture.currentPixelBuffer.appendInvocations.count, 3) + let presentationTimes = fixture.currentPixelBuffer.appendInvocations.invocations.map { $0.presentationTime.seconds } + XCTAssertEqual(presentationTimes, [0, 1, 2]) + XCTAssertEqual(sut.usedFrames.compactMap(\.screenName), ["A", "A", "B"]) + } + + func testProcessFrames_WhenVideoEndHasFractionalGap_ShouldNotCompressDuration() { + let videoWriterInput = TestAVAssetWriterInput(mediaType: .video, outputSettings: nil) + fixture.videoWriter.add(videoWriterInput) + + let frames = [ + SentryReplayFrame( + imagePath: fixture.videoFrames[0].imagePath, + time: Date(timeIntervalSinceReferenceDate: 0), + screenName: "A" + ) + ] + let sut = SentryVideoFrameProcessor( + videoFrames: frames, + videoWriter: fixture.videoWriter, + currentPixelBuffer: fixture.currentPixelBuffer, + outputFileURL: fixture.outputFileURL, + videoHeight: fixture.videoHeight, + videoWidth: fixture.videoWidth, + frameRate: fixture.frameRate, + initialFrameIndex: 0, + initialImageSize: fixture.initialImageSize, + videoEnd: Date(timeIntervalSinceReferenceDate: 2.4) + ) + + sut.processFrames(videoWriterInput: videoWriterInput) { _ in } + + XCTAssertEqual(fixture.currentPixelBuffer.appendInvocations.count, 3) + let presentationTimes = fixture.currentPixelBuffer.appendInvocations.invocations.map { $0.presentationTime.seconds } + XCTAssertEqual(presentationTimes, [0, 1, 2]) + XCTAssertEqual(sut.usedFrames.compactMap(\.screenName), ["A", "A", "A"]) + } + func testProcessFrames_WhenVideoWriterNotWriting_ShouldCancelWriting() { let sut = fixture.getSut() let videoWriterInput = TestAVAssetWriterInput(mediaType: .video, outputSettings: nil) @@ -251,6 +360,31 @@ class SentryVideoFrameProcessorTests: XCTestCase { XCTAssertEqual(completionInvocations.count, 1) } + func testProcessFrames_WhenCalledAgainWhileFinishing_ShouldFinishOnce() throws { + let videoWriter = try XCTUnwrap(DelayedTestAVAssetWriter(url: fixture.outputFileURL, fileType: .mp4)) + let videoWriterInput = TestAVAssetWriterInput(mediaType: .video, outputSettings: nil) + videoWriter.add(videoWriterInput) + let completionInvocations = Invocations>() + let sut = SentryVideoFrameProcessor( + videoFrames: fixture.videoFrames, + videoWriter: videoWriter, + currentPixelBuffer: fixture.currentPixelBuffer, + outputFileURL: fixture.outputFileURL, + videoHeight: fixture.videoHeight, + videoWidth: fixture.videoWidth, + frameRate: fixture.frameRate, + initialFrameIndex: fixture.initialFrameIndex, + initialImageSize: fixture.initialImageSize + ) + + sut.processFrames(videoWriterInput: videoWriterInput) { completionInvocations.record($0) } + sut.processFrames(videoWriterInput: videoWriterInput) { completionInvocations.record($0) } + + XCTAssertEqual(videoWriterInput.markAsFinishedInvocations.count, 1) + XCTAssertEqual(videoWriter.finishWritingInvocations.count, 1) + XCTAssertEqual(completionInvocations.count, 0) + } + func testProcessFrames_WhenImageSizeChanges_ShouldFinishVideo() { let videoWriterInput = TestAVAssetWriterInput(mediaType: .video, outputSettings: nil) fixture.videoWriter.add(videoWriterInput) @@ -299,13 +433,13 @@ class SentryVideoFrameProcessorTests: XCTestCase { } } - func testProcessFrames_WhenImageCannotBeLoaded_ShouldSkipFrame() { + func testProcessFrames_WhenInitialImageCannotBeLoaded_ShouldSkipFrame() { let videoWriterInput = TestAVAssetWriterInput(mediaType: .video, outputSettings: nil) let completionInvocations = Invocations>() // Create frames with non-existent image paths let nonExistentFrames = [ - SentryReplayFrame(imagePath: "/another/non/existent/path.png", time: Date(), screenName: "Screen2") + SentryReplayFrame(imagePath: "/another/non/existent/path.png", time: Date(timeIntervalSinceReferenceDate: 1), screenName: "Screen2") ] let sutWithNonExistentFrames = SentryVideoFrameProcessor( @@ -322,9 +456,45 @@ class SentryVideoFrameProcessorTests: XCTestCase { sutWithNonExistentFrames.processFrames(videoWriterInput: videoWriterInput) { completionInvocations.record($0) } - // Should still increment frame index even if image can't be loaded XCTAssertEqual(sutWithNonExistentFrames.frameIndex, 1) XCTAssertEqual(sutWithNonExistentFrames.usedFrames.count, 0) + XCTAssertEqual(fixture.currentPixelBuffer.appendInvocations.count, 0) + } + + func testProcessFrames_WhenTrailingImageCannotBeLoaded_ShouldHoldPreviousFrame() throws { + let videoWriterInput = TestAVAssetWriterInput(mediaType: .video, outputSettings: nil) + fixture.videoWriter.add(videoWriterInput) + + let frames = [ + SentryReplayFrame( + imagePath: try fixture.createTestImage(), + time: Date(timeIntervalSinceReferenceDate: 0), + screenName: "A" + ), + SentryReplayFrame( + imagePath: "/non/existent/path.png", + time: Date(timeIntervalSinceReferenceDate: 3), + screenName: "Missing" + ) + ] + let sut = SentryVideoFrameProcessor( + videoFrames: frames, + videoWriter: fixture.videoWriter, + currentPixelBuffer: fixture.currentPixelBuffer, + outputFileURL: fixture.outputFileURL, + videoHeight: fixture.videoHeight, + videoWidth: fixture.videoWidth, + frameRate: fixture.frameRate, + initialFrameIndex: 0, + initialImageSize: fixture.initialImageSize + ) + + sut.processFrames(videoWriterInput: videoWriterInput) { _ in } + + XCTAssertEqual(fixture.currentPixelBuffer.appendInvocations.count, 3) + let presentationTimes = fixture.currentPixelBuffer.appendInvocations.invocations.map { $0.presentationTime.seconds } + XCTAssertEqual(presentationTimes, [0, 1, 2]) + XCTAssertEqual(sut.usedFrames.compactMap(\.screenName), ["A", "A", "A"]) } // MARK: - Finish Video Tests @@ -369,6 +539,35 @@ class SentryVideoFrameProcessorTests: XCTestCase { } } + func testFinishVideo_WhenWriterCompletedWithoutUsedFrames_ShouldReturnNilVideoInfo() throws { + let sut = fixture.getSut() + let videoWriterInput = TestAVAssetWriterInput(mediaType: .video, outputSettings: nil) + fixture.videoWriter.add(videoWriterInput) + fixture.videoWriter.statusOverride = .completed + let completionInvocations = Invocations>() + + try Data("empty video data".utf8).write(to: fixture.outputFileURL) + + sut.finishVideo(frameIndex: 1) { result in + completionInvocations.record(result) + } + + XCTAssertEqual(videoWriterInput.markAsFinishedInvocations.count, 1) + XCTAssertEqual(completionInvocations.count, 1) + XCTAssertFalse(FileManager.default.fileExists(atPath: fixture.outputFileURL.path)) + + let result = completionInvocations.invocations.first + XCTAssertNotNil(result) + + switch result { + case .success(let videoResult): + XCTAssertEqual(videoResult.finalFrameIndex, 1) + XCTAssertNil(videoResult.info) + default: + XCTFail("Expected success result with nil info") + } + } + func testFinishVideo_WhenWriterCancelled_ShouldReturnNilVideoInfo() { let sut = fixture.getSut() let videoWriterInput = TestAVAssetWriterInput(mediaType: .video, outputSettings: nil) @@ -540,6 +739,32 @@ class SentryVideoFrameProcessorTests: XCTestCase { XCTAssertEqual(videoInfo.screens, ["Screen1", "Screen2", "Screen3"]) } + func testGetVideoInfo_WithMoreThanOneFPS_ShouldUseFractionalDuration() throws { + let sut = SentryVideoFrameProcessor( + videoFrames: fixture.videoFrames, + videoWriter: fixture.videoWriter, + currentPixelBuffer: fixture.currentPixelBuffer, + outputFileURL: fixture.outputFileURL, + videoHeight: fixture.videoHeight, + videoWidth: fixture.videoWidth, + frameRate: 2, + initialFrameIndex: fixture.initialFrameIndex, + initialImageSize: fixture.initialImageSize + ) + let testData = Data("test video data".utf8) + try testData.write(to: fixture.outputFileURL) + + let videoInfo = try sut.getVideoInfo( + from: fixture.outputFileURL, + usedFrames: fixture.videoFrames, + videoWidth: 200, + videoHeight: 100 + ) + + XCTAssertEqual(videoInfo.duration, 1.5) + XCTAssertEqual(videoInfo.end, Date(timeIntervalSinceReferenceDate: 1.5)) + } + func testGetVideoInfo_WithNonExistentFile_ShouldThrowError() { let sut = fixture.getSut() @@ -615,11 +840,10 @@ class SentryVideoFrameProcessorTests: XCTestCase { let videoWriterInput = TestAVAssetWriterInput(mediaType: .video, outputSettings: nil) let completionInvocations = Invocations>() - // Create mixed frames (valid and invalid) let mixedFrames = [ - SentryReplayFrame(imagePath: try fixture.createTestImage(), time: Date(), screenName: "Valid1"), - SentryReplayFrame(imagePath: "/non/existent/path.png", time: Date(), screenName: "Invalid"), - SentryReplayFrame(imagePath: try fixture.createTestImage(), time: Date(), screenName: "Valid2") + SentryReplayFrame(imagePath: try fixture.createTestImage(), time: Date(timeIntervalSinceReferenceDate: 0), screenName: "Valid1"), + SentryReplayFrame(imagePath: "/non/existent/path.png", time: Date(timeIntervalSinceReferenceDate: 1), screenName: "Invalid"), + SentryReplayFrame(imagePath: try fixture.createTestImage(), time: Date(timeIntervalSinceReferenceDate: 2), screenName: "Valid2") ] let sutWithMixedFrames = SentryVideoFrameProcessor( @@ -636,9 +860,10 @@ class SentryVideoFrameProcessorTests: XCTestCase { sutWithMixedFrames.processFrames(videoWriterInput: videoWriterInput) { completionInvocations.record($0) } - // Should process valid frames and skip invalid ones XCTAssertEqual(sutWithMixedFrames.frameIndex, 3) - XCTAssertEqual(sutWithMixedFrames.usedFrames.count, 2) // Only valid frames + // 3 used frames: Valid1 at t=0, gap-fill (holding Valid1) at t=1, Valid2 at t=2 + XCTAssertEqual(sutWithMixedFrames.usedFrames.count, 3) + XCTAssertEqual(sutWithMixedFrames.usedFrames.compactMap(\.screenName), ["Valid1", "Valid1", "Valid2"]) } }