Refactor broadcast page

src/main/resources/static/js/broadcast/assetKinds.js

@@ -0,0 +1,39 @@
+import { isAudioAsset } from "../media/audio.js";
+import { AssetKind } from "./constants.js";
+
+export function isCodeAsset(asset) {
+    if (asset?.assetType) {
+        return asset.assetType === "SCRIPT";
+    }
+    const type = (asset?.mediaType || asset?.originalMediaType || "").toLowerCase();
+    return type.startsWith("application/javascript") || type.startsWith("text/javascript");
+}
+
+export function isVideoAsset(asset) {
+    if (asset?.assetType) {
+        return asset.assetType === "VIDEO";
+    }
+    return asset?.mediaType?.startsWith("video/");
+}
+
+export function isVideoElement(element) {
+    return element?.tagName === "VIDEO";
+}
+
+export function isGifAsset(asset) {
+    return asset?.mediaType?.toLowerCase() === "image/gif";
+}
+
+export function getAssetKind(asset) {
+    if (isAudioAsset(asset)) {
+        return AssetKind.AUDIO;
+    }
+    if (isCodeAsset(asset)) {
+        return AssetKind.CODE;
+    }
+    return AssetKind.VISUAL;
+}
+
+export function isVisualAsset(asset) {
+    return getAssetKind(asset) === AssetKind.VISUAL;
+}

src/main/resources/static/js/broadcast/audioManager.js

@@ -0,0 +1,216 @@
+const audioUnlockEvents = ["pointerdown", "keydown", "touchstart"];
+
+export function createAudioManager({ assets, globalScope = globalThis }) {
+    const audioControllers = new Map();
+    const pendingAudioUnlock = new Set();
+
+    audioUnlockEvents.forEach((eventName) => {
+        globalScope.addEventListener(eventName, () => {
+            if (!pendingAudioUnlock.size) return;
+            pendingAudioUnlock.forEach((controller) => safePlay(controller, pendingAudioUnlock));
+            pendingAudioUnlock.clear();
+        });
+    });
+
+    function ensureAudioController(asset) {
+        const cached = audioControllers.get(asset.id);
+        if (cached && cached.src === asset.url) {
+            applyAudioSettings(cached, asset);
+            return cached;
+        }
+
+        if (cached) {
+            clearAudio(asset.id);
+        }
+
+        const element = new Audio(asset.url);
+        element.autoplay = true;
+        element.preload = "auto";
+        element.controls = false;
+        element.addEventListener("loadedmetadata", () => recordDuration(asset.id, element.duration));
+        const controller = {
+            id: asset.id,
+            src: asset.url,
+            element,
+            delayTimeout: null,
+            loopEnabled: false,
+            loopActive: true,
+            delayMs: 0,
+            baseDelayMs: 0,
+        };
+        element.onended = () => handleAudioEnded(asset.id);
+        audioControllers.set(asset.id, controller);
+        applyAudioSettings(controller, asset, true);
+        return controller;
+    }
+
+    function applyAudioSettings(controller, asset, resetPosition = false) {
+        controller.loopEnabled = !!asset.audioLoop;
+        controller.loopActive = controller.loopEnabled && controller.loopActive !== false;
+        controller.baseDelayMs = Math.max(0, asset.audioDelayMillis || 0);
+        controller.delayMs = controller.baseDelayMs;
+        applyAudioElementSettings(controller.element, asset);
+        if (resetPosition) {
+            controller.element.currentTime = 0;
+            controller.element.pause();
+        }
+    }
+
+    function applyAudioElementSettings(element, asset) {
+        const speed = Math.max(0.25, asset.audioSpeed || 1);
+        const pitch = Math.max(0.5, asset.audioPitch || 1);
+        element.playbackRate = speed * pitch;
+        const volume = Math.max(0, Math.min(2, asset.audioVolume ?? 1));
+        element.volume = Math.min(volume, 1);
+    }
+
+    function getAssetVolume(asset) {
+        return Math.max(0, Math.min(2, asset?.audioVolume ?? 1));
+    }
+
+    function applyMediaVolume(element, asset) {
+        if (!element) return 1;
+        const volume = getAssetVolume(asset);
+        element.volume = Math.min(volume, 1);
+        return volume;
+    }
+
+    function handleAudioEnded(assetId) {
+        const controller = audioControllers.get(assetId);
+        if (!controller) return;
+        controller.element.currentTime = 0;
+        if (controller.delayTimeout) {
+            clearTimeout(controller.delayTimeout);
+        }
+        if (controller.loopEnabled && controller.loopActive) {
+            controller.delayTimeout = setTimeout(() => {
+                safePlay(controller, pendingAudioUnlock);
+            }, controller.delayMs);
+        } else {
+            controller.element.pause();
+        }
+    }
+
+    function stopAudio(assetId) {
+        const controller = audioControllers.get(assetId);
+        if (!controller) return;
+        if (controller.delayTimeout) {
+            clearTimeout(controller.delayTimeout);
+        }
+        controller.element.pause();
+        controller.element.currentTime = 0;
+        controller.delayTimeout = null;
+        controller.delayMs = controller.baseDelayMs;
+        controller.loopActive = false;
+    }
+
+    function playAudioImmediately(asset) {
+        const controller = ensureAudioController(asset);
+        if (controller.delayTimeout) {
+            clearTimeout(controller.delayTimeout);
+            controller.delayTimeout = null;
+        }
+        controller.element.currentTime = 0;
+        const originalDelay = controller.delayMs;
+        controller.delayMs = 0;
+        safePlay(controller, pendingAudioUnlock);
+        controller.delayMs = controller.baseDelayMs ?? originalDelay ?? 0;
+    }
+
+    function playOverlappingAudio(asset) {
+        const temp = new Audio(asset.url);
+        temp.autoplay = true;
+        temp.preload = "auto";
+        temp.controls = false;
+        applyAudioElementSettings(temp, asset);
+        const controller = { element: temp };
+        temp.onended = () => {
+            temp.remove();
+        };
+        safePlay(controller, pendingAudioUnlock);
+    }
+
+    function handleAudioPlay(asset, shouldPlay) {
+        const controller = ensureAudioController(asset);
+        controller.loopActive = !!shouldPlay;
+        if (!shouldPlay) {
+            stopAudio(asset.id);
+            return;
+        }
+        if (asset.audioLoop) {
+            controller.delayMs = controller.baseDelayMs;
+            safePlay(controller, pendingAudioUnlock);
+        } else {
+            playOverlappingAudio(asset);
+        }
+    }
+
+    function autoStartAudio(asset) {
+        if (asset.hidden) {
+            return;
+        }
+        const controller = ensureAudioController(asset);
+        if (!controller.loopEnabled || !controller.loopActive) {
+            return;
+        }
+        if (!controller.element.paused && !controller.element.ended) {
+            return;
+        }
+        if (controller.delayTimeout) {
+            return;
+        }
+        controller.delayTimeout = setTimeout(() => {
+            safePlay(controller, pendingAudioUnlock);
+        }, controller.delayMs);
+    }
+
+    function recordDuration(assetId, seconds) {
+        if (!Number.isFinite(seconds) || seconds <= 0) {
+            return;
+        }
+        const asset = assets.get(assetId);
+        if (!asset) {
+            return;
+        }
+        const nextMs = Math.round(seconds * 1000);
+        if (asset.durationMs === nextMs) {
+            return;
+        }
+        asset.durationMs = nextMs;
+    }
+
+    function clearAudio(assetId) {
+        const audio = audioControllers.get(assetId);
+        if (!audio) {
+            return;
+        }
+        if (audio.delayTimeout) {
+            clearTimeout(audio.delayTimeout);
+        }
+        audio.element.pause();
+        audio.element.currentTime = 0;
+        audio.element.src = "";
+        audio.element.remove();
+        audioControllers.delete(assetId);
+    }
+
+    return {
+        ensureAudioController,
+        applyMediaVolume,
+        handleAudioPlay,
+        stopAudio,
+        playAudioImmediately,
+        autoStartAudio,
+        clearAudio,
+    };
+}
+
+function safePlay(controller, pendingUnlock) {
+    if (!controller?.element) return;
+    const playPromise = controller.element.play();
+    if (playPromise?.catch) {
+        playPromise.catch(() => {
+            pendingUnlock.add(controller);
+        });
+    }
+}

src/main/resources/static/js/broadcast/constants.js

@@ -0,0 +1,9 @@
+export const TARGET_FPS = 60;
+export const MIN_FRAME_TIME = 1000 / TARGET_FPS;
+export const VISIBILITY_THRESHOLD = 0.01;
+
+export const AssetKind = Object.freeze({
+    AUDIO: "audio",
+    CODE: "code",
+    VISUAL: "visual",
+});

src/main/resources/static/js/broadcast/layers.js

@@ -0,0 +1,44 @@
+import { isVisualAsset } from "./assetKinds.js";
+
+export function ensureLayerPosition(state, assetId, placement = "keep") {
+    const asset = state.assets.get(assetId);
+    if (asset && !isVisualAsset(asset)) {
+        return;
+    }
+    const existingIndex = state.layerOrder.indexOf(assetId);
+    if (existingIndex !== -1 && placement === "keep") {
+        return;
+    }
+    if (existingIndex !== -1) {
+        state.layerOrder.splice(existingIndex, 1);
+    }
+    if (placement === "append") {
+        state.layerOrder.push(assetId);
+    } else {
+        state.layerOrder.unshift(assetId);
+    }
+    state.layerOrder = state.layerOrder.filter((id) => state.assets.has(id));
+}
+
+export function getLayerOrder(state) {
+    state.layerOrder = state.layerOrder.filter((id) => {
+        const asset = state.assets.get(id);
+        return asset && isVisualAsset(asset);
+    });
+    state.assets.forEach((asset, id) => {
+        if (!isVisualAsset(asset)) {
+            return;
+        }
+        if (!state.layerOrder.includes(id)) {
+            state.layerOrder.unshift(id);
+        }
+    });
+    return state.layerOrder;
+}
+
+export function getRenderOrder(state) {
+    return [...getLayerOrder(state)]
+        .reverse()
+        .map((id) => state.assets.get(id))
+        .filter(Boolean);
+}

src/main/resources/static/js/broadcast/mediaManager.js

@@ -0,0 +1,334 @@
+import { isAudioAsset } from "../media/audio.js";
+import { isGifAsset, isVideoAsset, isVideoElement } from "./assetKinds.js";
+
+export function createMediaManager({
+    state,
+    audioManager,
+    draw,
+    obsBrowser,
+    supportsAnimatedDecode,
+    canPlayProbe,
+}) {
+    const { mediaCache, animatedCache, blobCache, animationFailures, videoPlaybackStates } = state;
+
+    function clearMedia(assetId) {
+        const element = mediaCache.get(assetId);
+        if (isVideoElement(element)) {
+            element.src = "";
+            element.remove();
+        }
+        mediaCache.delete(assetId);
+        const animated = animatedCache.get(assetId);
+        if (animated) {
+            animated.cancelled = true;
+            clearTimeout(animated.timeout);
+            animated.bitmap?.close?.();
+            animated.decoder?.close?.();
+            animatedCache.delete(assetId);
+        }
+        animationFailures.delete(assetId);
+        const cachedBlob = blobCache.get(assetId);
+        if (cachedBlob?.objectUrl) {
+            URL.revokeObjectURL(cachedBlob.objectUrl);
+        }
+        blobCache.delete(assetId);
+        audioManager.clearAudio(assetId);
+    }
+
+    function ensureMedia(asset) {
+        const cached = mediaCache.get(asset.id);
+        const cachedSource = getCachedSource(cached);
+        if (cached && cachedSource !== asset.url) {
+            clearMedia(asset.id);
+        }
+        if (cached && cachedSource === asset.url) {
+            applyMediaSettings(cached, asset);
+            return cached;
+        }
+
+        if (isAudioAsset(asset)) {
+            audioManager.ensureAudioController(asset);
+            mediaCache.delete(asset.id);
+            return null;
+        }
+
+        if (isGifAsset(asset) && supportsAnimatedDecode) {
+            const animated = ensureAnimatedImage(asset);
+            if (animated) {
+                mediaCache.set(asset.id, animated);
+                return animated;
+            }
+        }
+
+        const element = isVideoAsset(asset) ? document.createElement("video") : new Image();
+        element.dataset.sourceUrl = asset.url;
+        element.crossOrigin = "anonymous";
+        if (isVideoElement(element)) {
+            if (!canPlayVideoType(asset.mediaType)) {
+                return null;
+            }
+            element.loop = true;
+            element.playsInline = true;
+            element.autoplay = true;
+            element.controls = false;
+            element.onloadeddata = draw;
+            element.onloadedmetadata = () => recordDuration(asset.id, element.duration);
+            element.preload = "auto";
+            element.addEventListener("error", () => clearMedia(asset.id));
+            const playbackState = getVideoPlaybackState(element);
+            element.addEventListener("playing", () => {
+                playbackState.playRequested = false;
+                if (playbackState.unmuteOnPlay) {
+                    element.muted = false;
+                    playbackState.unmuteOnPlay = false;
+                }
+            });
+            element.addEventListener("pause", () => {
+                playbackState.playRequested = false;
+            });
+            audioManager.applyMediaVolume(element, asset);
+            element.muted = true;
+            setVideoSource(element, asset);
+        } else {
+            element.onload = draw;
+            element.src = asset.url;
+        }
+        mediaCache.set(asset.id, element);
+        return element;
+    }
+
+    function ensureAnimatedImage(asset) {
+        const failedAt = animationFailures.get(asset.id);
+        if (failedAt && Date.now() - failedAt < 15000) {
+            return null;
+        }
+        const cached = animatedCache.get(asset.id);
+        if (cached && cached.url === asset.url) {
+            return cached;
+        }
+
+        animationFailures.delete(asset.id);
+
+        if (cached) {
+            clearMedia(asset.id);
+        }
+
+        const controller = {
+            id: asset.id,
+            url: asset.url,
+            src: asset.url,
+            decoder: null,
+            bitmap: null,
+            timeout: null,
+            cancelled: false,
+            isAnimated: true,
+        };
+
+        fetchAssetBlob(asset)
+            .then((blob) => new ImageDecoder({ data: blob, type: blob.type || "image/gif" }))
+            .then((decoder) => {
+                if (controller.cancelled) {
+                    decoder.close?.();
+                    return null;
+                }
+                controller.decoder = decoder;
+                scheduleNextFrame(controller);
+                return controller;
+            })
+            .catch(() => {
+                animatedCache.delete(asset.id);
+                animationFailures.set(asset.id, Date.now());
+            });
+
+        animatedCache.set(asset.id, controller);
+        return controller;
+    }
+
+    function fetchAssetBlob(asset) {
+        const cached = blobCache.get(asset.id);
+        if (cached && cached.url === asset.url && cached.blob) {
+            return Promise.resolve(cached.blob);
+        }
+        if (cached && cached.url === asset.url && cached.pending) {
+            return cached.pending;
+        }
+
+        const pending = fetch(asset.url)
+            .then((r) => r.blob())
+            .then((blob) => {
+                const previous = blobCache.get(asset.id);
+                const existingUrl = previous?.url === asset.url ? previous.objectUrl : null;
+                const objectUrl = existingUrl || URL.createObjectURL(blob);
+                blobCache.set(asset.id, { url: asset.url, blob, objectUrl });
+                return blob;
+            });
+        blobCache.set(asset.id, { url: asset.url, pending });
+        return pending;
+    }
+
+    function setVideoSource(element, asset) {
+        if (!shouldUseBlobUrl(asset)) {
+            applyVideoSource(element, asset.url, asset);
+            return;
+        }
+
+        const cached = blobCache.get(asset.id);
+        if (cached?.url === asset.url && cached.objectUrl) {
+            applyVideoSource(element, cached.objectUrl, asset);
+            return;
+        }
+
+        fetchAssetBlob(asset)
+            .then(() => {
+                const next = blobCache.get(asset.id);
+                if (next?.url !== asset.url || !next.objectUrl) {
+                    return;
+                }
+                applyVideoSource(element, next.objectUrl, asset);
+            })
+            .catch(() => {});
+    }
+
+    function applyVideoSource(element, objectUrl, asset) {
+        element.src = objectUrl;
+        startVideoPlayback(element, asset);
+    }
+
+    function shouldUseBlobUrl(asset) {
+        return !obsBrowser && asset?.mediaType && canPlayVideoType(asset.mediaType);
+    }
+
+    function canPlayVideoType(mediaType) {
+        if (!mediaType) {
+            return true;
+        }
+        const support = canPlayProbe.canPlayType(mediaType);
+        return support === "probably" || support === "maybe";
+    }
+
+    function getCachedSource(element) {
+        return element?.dataset?.sourceUrl || element?.src;
+    }
+
+    function scheduleNextFrame(controller) {
+        if (controller.cancelled || !controller.decoder) {
+            return;
+        }
+        controller.decoder
+            .decode()
+            .then(({ image, complete }) => {
+                if (controller.cancelled) {
+                    image.close?.();
+                    return;
+                }
+                controller.bitmap?.close?.();
+                createImageBitmap(image)
+                    .then((bitmap) => {
+                        controller.bitmap = bitmap;
+                        draw();
+                    })
+                    .finally(() => image.close?.());
+
+                const durationMicros = image.duration || 0;
+                const delay = durationMicros > 0 ? durationMicros / 1000 : 100;
+                const hasMore = !complete;
+                controller.timeout = setTimeout(() => {
+                    if (controller.cancelled) {
+                        return;
+                    }
+                    if (hasMore) {
+                        scheduleNextFrame(controller);
+                    } else {
+                        controller.decoder.reset();
+                        scheduleNextFrame(controller);
+                    }
+                }, delay);
+            })
+            .catch(() => {
+                animatedCache.delete(controller.id);
+                animationFailures.set(controller.id, Date.now());
+            });
+    }
+
+    function applyMediaSettings(element, asset) {
+        if (!isVideoElement(element)) {
+            return;
+        }
+        startVideoPlayback(element, asset);
+    }
+
+    function startVideoPlayback(element, asset) {
+        const playbackState = getVideoPlaybackState(element);
+        const nextSpeed = asset.speed ?? 1;
+        const effectiveSpeed = Math.max(nextSpeed, 0.01);
+        if (element.playbackRate !== effectiveSpeed) {
+            element.playbackRate = effectiveSpeed;
+        }
+        const volume = audioManager.applyMediaVolume(element, asset);
+        const shouldUnmute = volume > 0;
+        element.muted = true;
+
+        if (effectiveSpeed === 0) {
+            element.pause();
+            playbackState.playRequested = false;
+            playbackState.unmuteOnPlay = false;
+            return;
+        }
+
+        element.play();
+
+        if (shouldUnmute) {
+            if (!element.paused && element.readyState >= 2) {
+                element.muted = false;
+            } else {
+                playbackState.unmuteOnPlay = true;
+            }
+        }
+
+        if (element.paused || element.ended) {
+            if (!playbackState.playRequested) {
+                playbackState.playRequested = true;
+                const playPromise = element.play();
+                if (playPromise?.catch) {
+                    playPromise.catch(() => {
+                        playbackState.playRequested = false;
+                    });
+                }
+            }
+        }
+    }
+
+    function recordDuration(assetId, seconds) {
+        if (!Number.isFinite(seconds) || seconds <= 0) {
+            return;
+        }
+        const asset = state.assets.get(assetId);
+        if (!asset) {
+            return;
+        }
+        const nextMs = Math.round(seconds * 1000);
+        if (asset.durationMs === nextMs) {
+            return;
+        }
+        asset.durationMs = nextMs;
+    }
+
+    function getVideoPlaybackState(element) {
+        if (!element) {
+            return { playRequested: false, unmuteOnPlay: false };
+        }
+        let playbackState = videoPlaybackStates.get(element);
+        if (!playbackState) {
+            playbackState = { playRequested: false, unmuteOnPlay: false };
+            videoPlaybackStates.set(element, playbackState);
+        }
+        return playbackState;
+    }
+
+    return {
+        clearMedia,
+        ensureMedia,
+        applyMediaSettings,
+        canPlayVideoType,
+    };
+}

src/main/resources/static/js/broadcast/renderer.js

@@ -0,0 +1,385 @@
+import { AssetKind, MIN_FRAME_TIME, VISIBILITY_THRESHOLD } from "./constants.js";
+import { createBroadcastState } from "./state.js";
+import { getAssetKind, isCodeAsset, isVisualAsset, isVideoElement } from "./assetKinds.js";
+import { ensureLayerPosition, getLayerOrder, getRenderOrder } from "./layers.js";
+import { getVisibilityState, smoothState } from "./visibility.js";
+import { createAudioManager } from "./audioManager.js";
+import { createMediaManager } from "./mediaManager.js";
+
+export class BroadcastRenderer {
+    constructor({ canvas, broadcaster, showToast }) {
+        this.canvas = canvas;
+        this.ctx = canvas.getContext("2d");
+        this.broadcaster = broadcaster;
+        this.showToast = showToast;
+        this.state = createBroadcastState();
+        this.lastRenderTime = 0;
+        this.frameScheduled = false;
+        this.pendingDraw = false;
+        this.renderIntervalId = null;
+
+        this.obsBrowser = !!globalThis.obsstudio;
+        this.supportsAnimatedDecode =
+            typeof ImageDecoder !== "undefined" &&
+            typeof createImageBitmap === "function" &&
+            !this.obsBrowser;
+        this.canPlayProbe = document.createElement("video");
+
+        this.audioManager = createAudioManager({ assets: this.state.assets });
+        this.mediaManager = createMediaManager({
+            state: this.state,
+            audioManager: this.audioManager,
+            draw: () => this.draw(),
+            obsBrowser: this.obsBrowser,
+            supportsAnimatedDecode: this.supportsAnimatedDecode,
+            canPlayProbe: this.canPlayProbe,
+        });
+
+        this.applyCanvasSettings(this.state.canvasSettings);
+        globalThis.addEventListener("resize", () => {
+            this.resizeCanvas();
+        });
+    }
+
+    start() {
+        this.fetchCanvasSettings().finally(() => {
+            this.resizeCanvas();
+            this.startRenderLoop();
+            this.connect();
+        });
+    }
+
+    connect() {
+        const socket = new SockJS("/ws");
+        const stompClient = Stomp.over(socket);
+        stompClient.connect({}, () => {
+            stompClient.subscribe(`/topic/channel/${this.broadcaster}`, (payload) => {
+                const body = JSON.parse(payload.body);
+                this.handleEvent(body);
+            });
+            fetch(`/api/channels/${this.broadcaster}/assets`)
+                .then((r) => {
+                    if (!r.ok) {
+                        throw new Error("Failed to load assets");
+                    }
+                    return r.json();
+                })
+                .then((assets) => this.renderAssets(assets))
+                .catch(() =>
+                    this.showToast("Unable to load overlay assets. Retrying may help.", "error"),
+                );
+        });
+    }
+
+    renderAssets(list) {
+        this.state.layerOrder = [];
+        list.forEach((asset) => {
+            this.storeAsset(asset, "append");
+            if (isCodeAsset(asset)) {
+                this.spawnUserJavaScriptWorker(asset);
+            }
+        });
+        this.draw();
+    }
+
+    storeAsset(asset, placement = "keep") {
+        if (!asset) return;
+        console.info(`Storing asset: ${asset.id}`);
+        const wasExisting = this.state.assets.has(asset.id);
+        this.state.assets.set(asset.id, asset);
+        ensureLayerPosition(this.state, asset.id, placement);
+        if (!wasExisting && !this.state.visibilityStates.has(asset.id)) {
+            const initialAlpha = 0; // Fade in newly discovered assets
+            this.state.visibilityStates.set(asset.id, {
+                alpha: initialAlpha,
+                targetHidden: !!asset.hidden,
+            });
+        }
+    }
+
+    removeAsset(assetId) {
+        this.state.assets.delete(assetId);
+        this.state.layerOrder = this.state.layerOrder.filter((id) => id !== assetId);
+        this.mediaManager.clearMedia(assetId);
+        this.stopUserJavaScriptWorker(assetId);
+        this.state.renderStates.delete(assetId);
+        this.state.visibilityStates.delete(assetId);
+    }
+
+    async fetchCanvasSettings() {
+        return fetch(`/api/channels/${encodeURIComponent(this.broadcaster)}/canvas`)
+            .then((r) => {
+                if (!r.ok) {
+                    throw new Error("Failed to load canvas");
+                }
+                return r.json();
+            })
+            .then((settings) => {
+                this.applyCanvasSettings(settings);
+            })
+            .catch(() => {
+                this.resizeCanvas();
+                this.showToast("Using default canvas size. Unable to load saved settings.", "warning");
+            });
+    }
+
+    applyCanvasSettings(settings) {
+        if (!settings) {
+            return;
+        }
+        const width = Number.isFinite(settings.width)
+            ? settings.width
+            : this.state.canvasSettings.width;
+        const height = Number.isFinite(settings.height)
+            ? settings.height
+            : this.state.canvasSettings.height;
+        this.state.canvasSettings = { width, height };
+        this.resizeCanvas();
+    }
+
+    resizeCanvas() {
+        if (
+            Number.isFinite(this.state.canvasSettings.width) &&
+            Number.isFinite(this.state.canvasSettings.height)
+        ) {
+            this.canvas.width = this.state.canvasSettings.width;
+            this.canvas.height = this.state.canvasSettings.height;
+            this.canvas.style.width = `${this.state.canvasSettings.width}px`;
+            this.canvas.style.height = `${this.state.canvasSettings.height}px`;
+        }
+        this.draw();
+    }
+
+    handleEvent(event) {
+        if (event.type === "CANVAS" && event.payload) {
+            this.applyCanvasSettings(event.payload);
+            return;
+        }
+        const assetId = event.assetId || event?.patch?.id || event?.payload?.id;
+        if (event.type === "VISIBILITY") {
+            this.handleVisibilityEvent(event);
+            return;
+        }
+        if (event.type === "DELETED") {
+            this.removeAsset(assetId);
+        } else if (event.patch) {
+            this.applyPatch(assetId, event.patch);
+            if (event.payload) {
+                const payload = this.normalizePayload(event.payload);
+                if (payload.hidden) {
+                    this.hideAssetWithTransition(payload);
+                } else if (!this.state.assets.has(payload.id)) {
+                    this.upsertVisibleAsset(payload, "append");
+                }
+            }
+        } else if (event.type === "PLAY" && event.payload) {
+            const payload = this.normalizePayload(event.payload);
+            this.storeAsset(payload);
+            if (getAssetKind(payload) === AssetKind.AUDIO) {
+                this.audioManager.handleAudioPlay(payload, event.play !== false);
+            }
+        } else if (event.payload && !event.payload.hidden) {
+            const payload = this.normalizePayload(event.payload);
+            this.upsertVisibleAsset(payload);
+        } else if (event.payload && event.payload.hidden) {
+            this.hideAssetWithTransition(event.payload);
+        }
+        this.draw();
+    }
+
+    normalizePayload(payload) {
+        return { ...payload };
+    }
+
+    hideAssetWithTransition(asset) {
+        const payload = asset ? this.normalizePayload(asset) : null;
+        if (!payload?.id) {
+            return;
+        }
+        const existing = this.state.assets.get(payload.id);
+        if (
+            !existing &&
+            (!Number.isFinite(payload.x) ||
+                !Number.isFinite(payload.y) ||
+                !Number.isFinite(payload.width) ||
+                !Number.isFinite(payload.height))
+        ) {
+            return;
+        }
+        const merged = this.normalizePayload({ ...(existing || {}), ...payload, hidden: true });
+        this.storeAsset(merged);
+        this.stopUserJavaScriptWorker(merged.id);
+        this.audioManager.stopAudio(payload.id);
+    }
+
+    upsertVisibleAsset(asset, placement = "keep") {
+        const payload = asset ? this.normalizePayload(asset) : null;
+        if (!payload?.id) {
+            return;
+        }
+        const placementMode = this.state.assets.has(payload.id) ? "keep" : placement;
+        this.storeAsset(payload, placementMode);
+        this.mediaManager.ensureMedia(payload);
+        const kind = getAssetKind(payload);
+        if (kind === AssetKind.AUDIO) {
+            this.audioManager.playAudioImmediately(payload);
+        } else if (kind === AssetKind.CODE) {
+            this.spawnUserJavaScriptWorker(payload);
+        }
+    }
+
+    handleVisibilityEvent(event) {
+        const payload = event.payload ? this.normalizePayload(event.payload) : null;
+        const patch = event.patch;
+        const id = payload?.id || patch?.id || event.assetId;
+
+        if (payload?.hidden || patch?.hidden) {
+            this.hideAssetWithTransition({ id, ...payload, ...patch });
+            this.draw();
+            return;
+        }
+
+        if (payload) {
+            const placement = this.state.assets.has(payload.id) ? "keep" : "append";
+            this.upsertVisibleAsset(payload, placement);
+        }
+
+        if (patch && id) {
+            this.applyPatch(id, patch);
+        }
+
+        this.draw();
+    }
+
+    applyPatch(assetId, patch) {
+        if (!assetId || !patch) {
+            return;
+        }
+        const sanitizedPatch = Object.fromEntries(
+            Object.entries(patch).filter(([, value]) => value !== null && value !== undefined),
+        );
+        const existing = this.state.assets.get(assetId);
+        if (!existing) {
+            return;
+        }
+        const merged = this.normalizePayload({ ...existing, ...sanitizedPatch });
+        console.log(merged);
+        const isVisual = isVisualAsset(merged);
+        if (sanitizedPatch.hidden) {
+            this.hideAssetWithTransition(merged);
+            return;
+        }
+        const targetLayer = Number.isFinite(patch.layer)
+            ? patch.layer
+            : Number.isFinite(patch.zIndex)
+              ? patch.zIndex
+              : null;
+        if (isVisual && Number.isFinite(targetLayer)) {
+            const currentOrder = getLayerOrder(this.state).filter((id) => id !== assetId);
+            const insertIndex = Math.max(0, currentOrder.length - Math.round(targetLayer));
+            currentOrder.splice(insertIndex, 0, assetId);
+            this.state.layerOrder = currentOrder;
+        }
+        this.storeAsset(merged);
+        this.mediaManager.ensureMedia(merged);
+        if (isCodeAsset(merged)) {
+            console.info(`Spawning JS worker for patched asset: ${merged.id}`);
+            this.spawnUserJavaScriptWorker(merged);
+        }
+    }
+
+    draw() {
+        if (this.frameScheduled) {
+            this.pendingDraw = true;
+            return;
+        }
+        this.frameScheduled = true;
+        requestAnimationFrame((timestamp) => {
+            const elapsed = timestamp - this.lastRenderTime;
+            const delay = MIN_FRAME_TIME - elapsed;
+            const shouldRender = elapsed >= MIN_FRAME_TIME;
+
+            if (shouldRender) {
+                this.lastRenderTime = timestamp;
+                this.renderFrame();
+            }
+
+            this.frameScheduled = false;
+            if (this.pendingDraw || !shouldRender) {
+                this.pendingDraw = false;
+                setTimeout(() => this.draw(), Math.max(0, delay));
+            }
+        });
+    }
+
+    renderFrame() {
+        this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);
+        getRenderOrder(this.state).forEach((asset) => this.drawAsset(asset));
+    }
+
+    drawAsset(asset) {
+        const visibility = getVisibilityState(this.state, asset);
+        if (visibility.alpha <= VISIBILITY_THRESHOLD && asset.hidden) {
+            return;
+        }
+        const renderState = smoothState(this.state, asset);
+        const halfWidth = renderState.width / 2;
+        const halfHeight = renderState.height / 2;
+        this.ctx.save();
+        this.ctx.globalAlpha = Math.max(0, Math.min(1, visibility.alpha));
+        this.ctx.translate(renderState.x + halfWidth, renderState.y + halfHeight);
+        this.ctx.rotate((renderState.rotation * Math.PI) / 180);
+
+        const kind = getAssetKind(asset);
+        if (kind === AssetKind.CODE) {
+            this.ctx.restore();
+            return;
+        }
+
+        if (kind === AssetKind.AUDIO) {
+            if (!asset.hidden) {
+                this.audioManager.autoStartAudio(asset);
+            }
+            this.ctx.restore();
+            return;
+        }
+
+        const media = this.mediaManager.ensureMedia(asset);
+        const drawSource = media?.isAnimated ? media.bitmap : media;
+        const ready = this.isDrawable(media);
+        if (ready && drawSource) {
+            this.ctx.drawImage(drawSource, -halfWidth, -halfHeight, renderState.width, renderState.height);
+        }
+
+        this.ctx.restore();
+    }
+
+    isDrawable(element) {
+        if (!element) {
+            return false;
+        }
+        if (element.isAnimated) {
+            return !!element.bitmap;
+        }
+        if (isVideoElement(element)) {
+            return element.readyState >= 2;
+        }
+        if (typeof ImageBitmap !== "undefined" && element instanceof ImageBitmap) {
+            return true;
+        }
+        return !!element.complete;
+    }
+
+    startRenderLoop() {
+        if (this.renderIntervalId) {
+            return;
+        }
+        this.renderIntervalId = setInterval(() => {
+            this.draw();
+        }, MIN_FRAME_TIME);
+    }
+
+    spawnUserJavaScriptWorker() {}
+
+    stopUserJavaScriptWorker() {}
+}

src/main/resources/static/js/broadcast/state.js

@@ -0,0 +1,14 @@
+export function createBroadcastState() {
+    return {
+        canvasSettings: { width: 1920, height: 1080 },
+        assets: new Map(),
+        mediaCache: new Map(),
+        renderStates: new Map(),
+        visibilityStates: new Map(),
+        animatedCache: new Map(),
+        blobCache: new Map(),
+        animationFailures: new Map(),
+        videoPlaybackStates: new WeakMap(),
+        layerOrder: [],
+    };
+}

src/main/resources/static/js/broadcast/visibility.js

@@ -0,0 +1,33 @@
+export function getVisibilityState(state, asset) {
+    const current = state.visibilityStates.get(asset.id) || {};
+    const targetAlpha = asset.hidden ? 0 : 1;
+    const startingAlpha = Number.isFinite(current.alpha) ? current.alpha : 0;
+    const factor = asset.hidden ? 0.18 : 0.2;
+    const nextAlpha = lerp(startingAlpha, targetAlpha, factor);
+    const nextState = { alpha: nextAlpha, targetHidden: !!asset.hidden };
+    state.visibilityStates.set(asset.id, nextState);
+    return nextState;
+}
+
+export function smoothState(state, asset) {
+    const previous = state.renderStates.get(asset.id) || { ...asset };
+    const factor = 0.15;
+    const next = {
+        x: lerp(previous.x, asset.x, factor),
+        y: lerp(previous.y, asset.y, factor),
+        width: lerp(previous.width, asset.width, factor),
+        height: lerp(previous.height, asset.height, factor),
+        rotation: smoothAngle(previous.rotation, asset.rotation, factor),
+    };
+    state.renderStates.set(asset.id, next);
+    return next;
+}
+
+function smoothAngle(current, target, factor) {
+    const delta = ((target - current + 180) % 360) - 180;
+    return current + delta * factor;
+}
+
+function lerp(a, b, t) {
+    return a + (b - a) * t;
+}