fix: fix a bug which will lost audio when clip is too much

Author: moonrailgun

apps/web/src/core/managers/audio-manager.ts

@@ -1,35 +1,20 @@
 import type { EditorCore } from "@/core";
 import type { AudioClipSource } from "@/lib/media/audio";
 import { createAudioContext, collectAudioClips } from "@/lib/media/audio";
-import {
-	ALL_FORMATS,
-	AudioBufferSink,
-	BlobSource,
-	Input,
-	type WrappedAudioBuffer,
-} from "mediabunny";
 
 export class AudioManager {
 	private audioContext: AudioContext | null = null;
 	private masterGain: GainNode | null = null;
 	private playbackStartTime = 0;
 	private playbackStartContextTime = 0;
-	private scheduleTimer: number | null = null;
-	private lookaheadSeconds = 2;
-	private scheduleIntervalMs = 500;
 	private clips: AudioClipSource[] = [];
-	private sinks = new Map<string, AudioBufferSink>();
-	private inputs = new Map<string, Input>();
-	private activeClipIds = new Set<string>();
-	private clipIterators = new Map<
-		string,
-		AsyncGenerator<WrappedAudioBuffer, void, unknown>
-	>();
+	private decodedBuffers = new Map<string, AudioBuffer>();
 	private queuedSources = new Set<AudioBufferSourceNode>();
 	private playbackSessionId = 0;
 	private lastIsPlaying = false;
 	private lastVolume = 1;
 	private unsubscribers: Array<() => void> = [];
+	private timelineChangeTimer: number | null = null;
 
 	constructor(private editor: EditorCore) {
 		this.lastVolume = this.editor.playback.getVolume();
@@ -46,14 +31,18 @@ export class AudioManager {
 
 	dispose(): void {
 		this.stopPlayback();
+		if (this.timelineChangeTimer !== null) {
+			window.clearTimeout(this.timelineChangeTimer);
+			this.timelineChangeTimer = null;
+		}
 		for (const unsub of this.unsubscribers) {
 			unsub();
 		}
 		this.unsubscribers = [];
 		if (typeof window !== "undefined") {
 			window.removeEventListener("playback-seek", this.handleSeek);
 		}
-		this.disposeSinks();
+		this.decodedBuffers.clear();
 		if (this.audioContext) {
 			void this.audioContext.close();
 			this.audioContext = null;
@@ -100,11 +89,20 @@ export class AudioManager {
 	};
 
 	private handleTimelineChange = (): void => {
-		this.disposeSinks();
+		if (this.timelineChangeTimer !== null) {
+			window.clearTimeout(this.timelineChangeTimer);
+		}
 
-		if (!this.editor.playback.getIsPlaying()) return;
+		this.timelineChangeTimer = window.setTimeout(() => {
+			this.timelineChangeTimer = null;
+			this.decodedBuffers.clear();
+
+			if (!this.editor.playback.getIsPlaying()) return;
 
-		void this.startPlayback({ time: this.editor.playback.getCurrentTime() });
+			void this.startPlayback({
+				time: this.editor.playback.getCurrentTime(),
+			});
+		}, 300);
 	};
 
 	private ensureAudioContext(): AudioContext | null {
@@ -123,18 +121,13 @@ export class AudioManager {
 		this.masterGain.gain.value = this.lastVolume;
 	}
 
-	private getPlaybackTime(): number {
-		if (!this.audioContext) return this.playbackStartTime;
-		const elapsed = this.audioContext.currentTime - this.playbackStartContextTime;
-		return this.playbackStartTime + elapsed;
-	}
-
 	private async startPlayback({ time }: { time: number }): Promise<void> {
 		const audioContext = this.ensureAudioContext();
 		if (!audioContext) return;
 
 		this.stopPlayback();
 		this.playbackSessionId++;
+		const sessionId = this.playbackSessionId;
 
 		const tracks = this.editor.timeline.getTracks();
 		const mediaAssets = this.editor.media.getAssets();
@@ -148,195 +141,123 @@ export class AudioManager {
 
 		this.clips = await collectAudioClips({ tracks, mediaAssets });
 		if (!this.editor.playback.getIsPlaying()) return;
+		if (sessionId !== this.playbackSessionId) return;
 
 		this.playbackStartTime = time;
 		this.playbackStartContextTime = audioContext.currentTime;
 
-		this.scheduleUpcomingClips();
-
-		if (typeof window !== "undefined") {
-			this.scheduleTimer = window.setInterval(() => {
-				this.scheduleUpcomingClips();
-			}, this.scheduleIntervalMs);
-		}
+		await this.scheduleAllClips({ time, sessionId });
 	}
 
-	private scheduleUpcomingClips(): void {
-		if (!this.editor.playback.getIsPlaying()) return;
-
-		const currentTime = this.getPlaybackTime();
-		const windowEnd = currentTime + this.lookaheadSeconds;
+	private async scheduleAllClips({
+		time,
+		sessionId,
+	}: {
+		time: number;
+		sessionId: number;
+	}): Promise<void> {
+		const audioContext = this.audioContext;
+		if (!audioContext) return;
 
 		for (const clip of this.clips) {
 			if (clip.muted) continue;
-			if (this.activeClipIds.has(clip.id)) continue;
 
 			const clipEnd = clip.startTime + clip.duration;
-			if (clipEnd <= currentTime) continue;
-			if (clip.startTime > windowEnd) continue;
-
-			this.activeClipIds.add(clip.id);
-			void this.runClipIterator({ clip, startTime: currentTime, sessionId: this.playbackSessionId });
-		}
-	}
-
-	private stopPlayback(): void {
-		if (this.scheduleTimer && typeof window !== "undefined") {
-			window.clearInterval(this.scheduleTimer);
-		}
-		this.scheduleTimer = null;
-
-		for (const iterator of this.clipIterators.values()) {
-			void iterator.return();
-		}
-		this.clipIterators.clear();
-		this.activeClipIds.clear();
+			if (clipEnd <= time) continue;
+			if (sessionId !== this.playbackSessionId) return;
 
-		for (const source of this.queuedSources) {
 			try {
-				source.stop();
-			} catch {}
-			source.disconnect();
+				const buffer = await this.getDecodedBuffer({ clip });
+				if (!buffer) continue;
+				if (sessionId !== this.playbackSessionId) return;
+				if (!this.editor.playback.getIsPlaying()) return;
+
+				this.scheduleClipNode({ clip, buffer, time });
+			} catch (error) {
+				console.warn("Failed to schedule audio clip:", clip.id, error);
+			}
 		}
-		this.queuedSources.clear();
 	}
 
-	private async runClipIterator({
+	private scheduleClipNode({
 		clip,
-		startTime,
-		sessionId,
+		buffer,
+		time,
 	}: {
 		clip: AudioClipSource;
-		startTime: number;
-		sessionId: number;
-	}): Promise<void> {
-		const audioContext = this.ensureAudioContext();
-		if (!audioContext) return;
-
-		const sink = await this.getAudioSink({ clip });
-		if (!sink || !this.editor.playback.getIsPlaying()) return;
-		if (sessionId !== this.playbackSessionId) return;
+		buffer: AudioBuffer;
+		time: number;
+	}): void {
+		const audioContext = this.audioContext;
+		if (!audioContext || !this.masterGain) return;
 
-		const clipStart = clip.startTime;
-		const clipEnd = clip.startTime + clip.duration;
 		const rate = clip.playbackRate;
-
-		const iteratorStartTime = Math.max(startTime, clipStart);
-		const sourceStartTime =
-			clip.trimStart + (iteratorStartTime - clip.startTime) * rate;
-
-		const iterator = sink.buffers(sourceStartTime);
-		this.clipIterators.set(clip.id, iterator);
-
-		for await (const { buffer, timestamp } of iterator) {
-			if (!this.editor.playback.getIsPlaying()) return;
-			if (sessionId !== this.playbackSessionId) return;
-
-			const timelineTime = clip.startTime + (timestamp - clip.trimStart) / rate;
-			if (timelineTime >= clipEnd) break;
-
-			const node = audioContext.createBufferSource();
-			node.buffer = buffer;
-			node.playbackRate.value = rate;
-			node.connect(this.masterGain ?? audioContext.destination);
-
-			const startTimestamp =
-				this.playbackStartContextTime +
-				(timelineTime - this.playbackStartTime);
-
-			if (startTimestamp >= audioContext.currentTime) {
-				node.start(startTimestamp);
+		const elapsed = Math.max(0, time - clip.startTime);
+		const sourceOffset = clip.trimStart + elapsed * rate;
+		const remainingDuration = clip.duration - elapsed;
+
+		if (remainingDuration <= 0) return;
+
+		const timelineStart = Math.max(clip.startTime, time);
+		const scheduleTime =
+			this.playbackStartContextTime +
+			(timelineStart - this.playbackStartTime);
+
+		const node = audioContext.createBufferSource();
+		node.buffer = buffer;
+		node.playbackRate.value = rate;
+		node.connect(this.masterGain);
+
+		if (scheduleTime >= audioContext.currentTime) {
+			node.start(scheduleTime, sourceOffset, remainingDuration);
+		} else {
+			const late = audioContext.currentTime - scheduleTime;
+			const adjustedOffset = sourceOffset + late * rate;
+			const adjustedDuration = remainingDuration - late;
+			if (adjustedDuration > 0) {
+				node.start(audioContext.currentTime, adjustedOffset, adjustedDuration);
 			} else {
-				const offset = audioContext.currentTime - startTimestamp;
-				if (offset < buffer.duration) {
-					node.start(audioContext.currentTime, offset);
-				} else {
-					continue;
-				}
-			}
-
-			this.queuedSources.add(node);
-			node.addEventListener("ended", () => {
-				node.disconnect();
-				this.queuedSources.delete(node);
-			});
-
-			const aheadTime = timelineTime - this.getPlaybackTime();
-			if (aheadTime >= 1) {
-				await this.waitUntilCaughtUp({ timelineTime, targetAhead: 1 });
-				if (sessionId !== this.playbackSessionId) return;
+				return;
 			}
 		}
 
-		this.clipIterators.delete(clip.id);
-		// don't remove from activeClipIds - prevents scheduler from restarting this clip
-		// the set is cleared on stopPlayback anyway
-	}
-
-	private waitUntilCaughtUp({
-		timelineTime,
-		targetAhead,
-	}: {
-		timelineTime: number;
-		targetAhead: number;
-	}): Promise<void> {
-		return new Promise((resolve) => {
-			const checkInterval = setInterval(() => {
-				if (!this.editor.playback.getIsPlaying()) {
-					clearInterval(checkInterval);
-					resolve();
-					return;
-				}
-
-				const playbackTime = this.getPlaybackTime();
-				if (timelineTime - playbackTime < targetAhead) {
-					clearInterval(checkInterval);
-					resolve();
-				}
-			}, 100);
+		this.queuedSources.add(node);
+		node.addEventListener("ended", () => {
+			node.disconnect();
+			this.queuedSources.delete(node);
 		});
 	}
 
-	private disposeSinks(): void {
-		for (const iterator of this.clipIterators.values()) {
-			void iterator.return();
-		}
-		this.clipIterators.clear();
-		this.activeClipIds.clear();
-
-		for (const input of this.inputs.values()) {
-			input.dispose();
-		}
-		this.inputs.clear();
-		this.sinks.clear();
-	}
-
-	private async getAudioSink({
+	private async getDecodedBuffer({
 		clip,
 	}: {
 		clip: AudioClipSource;
-	}): Promise<AudioBufferSink | null> {
-		const existingSink = this.sinks.get(clip.sourceKey);
-		if (existingSink) return existingSink;
+	}): Promise<AudioBuffer | null> {
+		const cached = this.decodedBuffers.get(clip.sourceKey);
+		if (cached) return cached;
 
-		try {
-			const input = new Input({
-				source: new BlobSource(clip.file),
-				formats: ALL_FORMATS,
-			});
-			const audioTrack = await input.getPrimaryAudioTrack();
-			if (!audioTrack) {
-				input.dispose();
-				return null;
-			}
+		const audioContext = this.audioContext;
+		if (!audioContext) return null;
 
-			const sink = new AudioBufferSink(audioTrack);
-			this.inputs.set(clip.sourceKey, input);
-			this.sinks.set(clip.sourceKey, sink);
-			return sink;
+		try {
+			const arrayBuffer = await clip.file.arrayBuffer();
+			// .slice(0) avoids the detached-buffer error on repeated decodes
+			const buffer = await audioContext.decodeAudioData(arrayBuffer.slice(0));
+			this.decodedBuffers.set(clip.sourceKey, buffer);
+			return buffer;
 		} catch (error) {
-			console.warn("Failed to initialize audio sink:", error);
+			console.warn("Failed to decode audio:", clip.sourceKey, error);
 			return null;
 		}
 	}
+
+	private stopPlayback(): void {
+		for (const source of this.queuedSources) {
+			try {
+				source.stop();
+			} catch {}
+			source.disconnect();
+		}
+		this.queuedSources.clear();
+	}
 }