Temp refactor

src/main/resources/static/js/broadcast/const.js

@@ -0,0 +1,4 @@
+const TARGET_FPS = 60;
+
+export const MIN_FRAME_TIME = 1000 / TARGET_FPS;
+export const VISIBILITY_THRESHOLD = 0.01;

src/main/resources/static/js/broadcast/dom.js

@@ -0,0 +1,2 @@
+export const canvas = document.getElementById("broadcast-canvas");
+export const canPlayProbe = document.createElement("video");

src/main/resources/static/js/broadcast/gfx.js

@@ -0,0 +1,34 @@
+import { MIN_FRAME_TIME } from "./const.js"
+
+let frameScheduled = false;
+let lastRenderTime = 0;
+let pendingDraw = false;
+
+export function draw(ctx) {
+    if (frameScheduled) {
+        pendingDraw = true;
+        return;
+    }
+    frameScheduled = true;
+    requestAnimationFrame((timestamp) => {
+        const elapsed = timestamp - lastRenderTime;
+        const delay = MIN_FRAME_TIME - elapsed;
+        const shouldRender = elapsed >= MIN_FRAME_TIME;
+
+        if (shouldRender) {
+            lastRenderTime = timestamp;
+            renderFrame(ctx);
+        }
+
+        frameScheduled = false;
+        if (pendingDraw || !shouldRender) {
+            pendingDraw = false;
+            setTimeout(draw, Math.max(0, delay));
+        }
+    });
+}
+
+function renderFrame(ctx) {
+    ctx.clearRect(0, 0, canvas.width, canvas.height);
+    getRenderOrder().forEach(drawAsset);
+}

src/main/resources/static/js/broadcast/jsworkers.js

@@ -0,0 +1,15 @@
+export async function spawnUserJavaScriptWorker(asset) {
+    // let assetSource;
+    // try {
+    //     assetSource = await fetch(asset.url).then((r) => r.text());
+    // } catch (error) {
+    //     console.error(`Unable to fetch asset with id:${id} from url:${asset.url}`, error);
+    //     return;
+    // }
+    // const blob = new Blob([assetSource], { type: 'application/javascript' });
+    // const worker = new Worker(URL.createObjectURL(blob));
+    // worker.onmessage = (event) => {
+    //     console.log('Message from worker:', event.data);
+    // }
+    // worker.postMessage(data);
+}

src/main/resources/static/js/broadcast/main.js

@@ -0,0 +1,929 @@
+import { spawnUserJavaScriptWorker } from "./jsworkers.js"
+import { canvas, canPlayProbe } from "./dom.js";
+import { MIN_FRAME_TIME, VISIBILITY_THRESHOLD } from "./const.js"
+import { wsconnect } from "./ws.js";
+
+const obsBrowser = !!globalThis.obsstudio;
+const supportsAnimatedDecode =
+    typeof ImageDecoder !== "undefined" && typeof createImageBitmap === "function" && !obsBrowser;
+const ctx = canvas.getContext("2d");
+let canvasSettings = { width: 1920, height: 1080 };
+const assets = new Map();
+const mediaCache = new Map();
+const renderStates = new Map();
+const visibilityStates = new Map();
+const animatedCache = new Map();
+const blobCache = new Map();
+const animationFailures = new Map();
+const audioControllers = new Map();
+const videoPlaybackStates = new WeakMap();
+const pendingAudioUnlock = new Set();
+let lastRenderTime = 0;
+let frameScheduled = false;
+let pendingDraw = false;
+let renderIntervalId = null;
+let layerOrder = [];
+
+applyCanvasSettings(canvasSettings);
+
+["pointerdown", "keydown", "touchstart"].forEach((eventName) => {
+    globalThis.addEventListener(eventName, () => {
+        if (!pendingAudioUnlock.size) return;
+        pendingAudioUnlock.forEach((controller) => safePlay(controller));
+        pendingAudioUnlock.clear();
+    });
+});
+
+function ensureLayerPosition(assetId, placement = "keep") {
+    const asset = assets.get(assetId);
+    if (asset && (isAudioAsset(asset) || isCodeAsset(asset))) {
+        return;
+    }
+    const existingIndex = layerOrder.indexOf(assetId);
+    if (existingIndex !== -1 && placement === "keep") {
+        return;
+    }
+    if (existingIndex !== -1) {
+        layerOrder.splice(existingIndex, 1);
+    }
+    if (placement === "append") {
+        layerOrder.push(assetId);
+    } else {
+        layerOrder.unshift(assetId);
+    }
+    layerOrder = layerOrder.filter((id) => assets.has(id));
+}
+
+function getLayerOrder() {
+    layerOrder = layerOrder.filter((id) => {
+        const asset = assets.get(id);
+        return asset && !isAudioAsset(asset) && !isCodeAsset(asset);
+    });
+    assets.forEach((asset, id) => {
+        if (isAudioAsset(asset) || isCodeAsset(asset)) {
+            return;
+        }
+        if (!layerOrder.includes(id)) {
+            layerOrder.unshift(id);
+        }
+    });
+    return layerOrder;
+}
+
+function getRenderOrder() {
+    return [...getLayerOrder()]
+        .reverse()
+        .map((id) => assets.get(id))
+        .filter(Boolean);
+}
+
+function removeAsset(assetId) {
+    assets.delete(assetId);
+    layerOrder = layerOrder.filter((id) => id !== assetId);
+    clearMedia(assetId);
+    renderStates.delete(assetId);
+    visibilityStates.delete(assetId);
+}
+
+function renderAssets(list) {
+    layerOrder = [];
+    list.forEach((asset) => {
+        storeAsset(asset, "append");
+        if (isCodeAsset(asset)) {
+            spawnUserJavaScriptWorker(asset);
+        }
+    });
+    draw();
+}
+
+function storeAsset(asset, placement = "keep") {
+    if (!asset) return;
+    console.info(`Storing asset: ${asset.id}`);
+    const wasExisting = assets.has(asset.id);
+    assets.set(asset.id, asset);
+    ensureLayerPosition(asset.id, placement);
+    if (!wasExisting && !visibilityStates.has(asset.id)) {
+        const initialAlpha = 0; // Fade in newly discovered assets
+        visibilityStates.set(asset.id, { alpha: initialAlpha, targetHidden: !!asset.hidden });
+    }
+}
+
+async function fetchCanvasSettings() {
+    return fetch(`/api/channels/${encodeURIComponent(broadcaster)}/canvas`)
+        .then((r) => {
+            if (!r.ok) {
+                throw new Error("Failed to load canvas");
+            }
+            return r.json();
+        })
+        .then((settings) => {
+            applyCanvasSettings(settings);
+        })
+        .catch(() => {
+            resizeCanvas();
+            showToast("Using default canvas size. Unable to load saved settings.", "warning");
+        });
+}
+
+function applyCanvasSettings(settings) {
+    if (!settings) {
+        return;
+    }
+    const width = Number.isFinite(settings.width) ? settings.width : canvasSettings.width;
+    const height = Number.isFinite(settings.height) ? settings.height : canvasSettings.height;
+    canvasSettings = { width, height };
+    resizeCanvas();
+}
+
+function resizeCanvas() {
+    if (Number.isFinite(canvasSettings.width) && Number.isFinite(canvasSettings.height)) {
+        canvas.width = canvasSettings.width;
+        canvas.height = canvasSettings.height;
+        canvas.style.width = `${canvasSettings.width}px`;
+        canvas.style.height = `${canvasSettings.height}px`;
+    }
+    draw();
+}
+
+function onWebsocketEvent(event) {
+    if (event.type === "CANVAS" && event.payload) {
+        applyCanvasSettings(event.payload);
+        return;
+    }
+    const assetId = event.assetId || event?.patch?.id || event?.payload?.id;
+    if (event.type === "VISIBILITY") {
+        handleVisibilityEvent(event);
+        return;
+    }
+    if (event.type === "DELETED") {
+        removeAsset(assetId);
+    } else if (event.patch) {
+        applyPatch(assetId, event.patch);
+        if (event.payload) {
+            const payload = normalizePayload(event.payload);
+            if (payload.hidden) {
+                hideAssetWithTransition(payload);
+            } else if (!assets.has(payload.id)) {
+                upsertVisibleAsset(payload, "append");
+            }
+        }
+    } else if (event.type === "PLAY" && event.payload) {
+        const payload = normalizePayload(event.payload);
+        storeAsset(payload);
+        if (isAudioAsset(payload)) {
+            handleAudioPlay(payload, event.play !== false);
+        }
+    } else if (event.payload && !event.payload.hidden) {
+        const payload = normalizePayload(event.payload);
+        upsertVisibleAsset(payload);
+    } else if (event.payload && event.payload.hidden) {
+        hideAssetWithTransition(event.payload);
+    }
+    draw();
+}
+
+function normalizePayload(payload) {
+    return { ...payload };
+}
+
+function hideAssetWithTransition(asset) {
+    const payload = asset ? normalizePayload(asset) : null;
+    if (!payload?.id) {
+        return;
+    }
+    const existing = assets.get(payload.id);
+    if (
+        !existing &&
+        (!Number.isFinite(payload.x) ||
+            !Number.isFinite(payload.y) ||
+            !Number.isFinite(payload.width) ||
+            !Number.isFinite(payload.height))
+    ) {
+        return;
+    }
+    const merged = normalizePayload({ ...(existing || {}), ...payload, hidden: true });
+    storeAsset(merged);
+    stopAudio(payload.id);
+}
+
+function upsertVisibleAsset(asset, placement = "keep") {
+    const payload = asset ? normalizePayload(asset) : null;
+    if (!payload?.id) {
+        return;
+    }
+    const placementMode = assets.has(payload.id) ? "keep" : placement;
+    storeAsset(payload, placementMode);
+    ensureMedia(payload);
+    if (isAudioAsset(payload)) {
+        playAudioImmediately(payload);
+    } else if (isCodeAsset(payload)) {
+        spawnUserJavaScriptWorker(payload);
+    }
+}
+
+function handleVisibilityEvent(event) {
+    const payload = event.payload ? normalizePayload(event.payload) : null;
+    const patch = event.patch;
+    const id = payload?.id || patch?.id || event.assetId;
+
+    if (payload?.hidden || patch?.hidden) {
+        hideAssetWithTransition({ id, ...payload, ...patch });
+        draw();
+        return;
+    }
+
+    if (payload) {
+        const placement = assets.has(payload.id) ? "keep" : "append";
+        upsertVisibleAsset(payload, placement);
+    }
+
+    if (patch && id) {
+        applyPatch(id, patch);
+    }
+
+    draw();
+}
+
+function applyPatch(assetId, patch) {
+    if (!assetId || !patch) {
+        return;
+    }
+    const sanitizedPatch = Object.fromEntries(
+        Object.entries(patch).filter(([, value]) => value !== null && value !== undefined),
+    );
+    const existing = assets.get(assetId);
+    if (!existing) {
+        return;
+    }
+    const merged = normalizePayload({ ...existing, ...sanitizedPatch });
+    console.log(merged)
+    const isAudio = isAudioAsset(merged);
+    if (sanitizedPatch.hidden) {
+        hideAssetWithTransition(merged);
+        return;
+    }
+    const targetLayer = Number.isFinite(patch.layer)
+        ? patch.layer
+        : Number.isFinite(patch.zIndex)
+            ? patch.zIndex
+            : null;
+    if (!isAudio && Number.isFinite(targetLayer)) {
+        const currentOrder = getLayerOrder().filter((id) => id !== assetId);
+        const insertIndex = Math.max(0, currentOrder.length - Math.round(targetLayer));
+        currentOrder.splice(insertIndex, 0, assetId);
+        layerOrder = currentOrder;
+    }
+    storeAsset(merged);
+    ensureMedia(merged);
+    if (isCodeAsset(merged)) {
+        console.info(`Spawning JS worker for patched asset: ${merged.id}`);
+        spawnUserJavaScriptWorker(merged);
+    }
+}
+
+function draw() {
+    if (frameScheduled) {
+        pendingDraw = true;
+        return;
+    }
+    frameScheduled = true;
+    requestAnimationFrame((timestamp) => {
+        const elapsed = timestamp - lastRenderTime;
+        const delay = MIN_FRAME_TIME - elapsed;
+        const shouldRender = elapsed >= MIN_FRAME_TIME;
+
+        if (shouldRender) {
+            lastRenderTime = timestamp;
+            renderFrame();
+        }
+
+        frameScheduled = false;
+        if (pendingDraw || !shouldRender) {
+            pendingDraw = false;
+            setTimeout(draw, Math.max(0, delay));
+        }
+    });
+}
+
+function renderFrame() {
+    ctx.clearRect(0, 0, canvas.width, canvas.height);
+    getRenderOrder().forEach(drawAsset);
+}
+
+function drawAsset(asset) {
+    const visibility = getVisibilityState(asset);
+    if (visibility.alpha <= VISIBILITY_THRESHOLD && asset.hidden) {
+        return;
+    }
+    const renderState = smoothState(asset);
+    const halfWidth = renderState.width / 2;
+    const halfHeight = renderState.height / 2;
+    ctx.save();
+    ctx.globalAlpha = Math.max(0, Math.min(1, visibility.alpha));
+    ctx.translate(renderState.x + halfWidth, renderState.y + halfHeight);
+    ctx.rotate((renderState.rotation * Math.PI) / 180);
+
+    if (isCodeAsset(asset)) {
+        ctx.restore();
+        return;
+    }
+
+    if (isAudioAsset(asset)) {
+        if (!asset.hidden) {
+            autoStartAudio(asset);
+        }
+        ctx.restore();
+        return;
+    }
+
+    const media = ensureMedia(asset);
+    const drawSource = media?.isAnimated ? media.bitmap : media;
+    const ready = isDrawable(media);
+    if (ready && drawSource) {
+        ctx.drawImage(drawSource, -halfWidth, -halfHeight, renderState.width, renderState.height);
+    }
+
+    ctx.restore();
+}
+
+function getVisibilityState(asset) {
+    const current = visibilityStates.get(asset.id) || {};
+    const targetAlpha = asset.hidden ? 0 : 1;
+    const startingAlpha = Number.isFinite(current.alpha) ? current.alpha : 0;
+    const factor = asset.hidden ? 0.18 : 0.2;
+    const nextAlpha = lerp(startingAlpha, targetAlpha, factor);
+    const state = { alpha: nextAlpha, targetHidden: !!asset.hidden };
+    visibilityStates.set(asset.id, state);
+    return state;
+}
+
+function smoothState(asset) {
+    const previous = renderStates.get(asset.id) || { ...asset };
+    const factor = 0.15;
+    const next = {
+        x: lerp(previous.x, asset.x, factor),
+        y: lerp(previous.y, asset.y, factor),
+        width: lerp(previous.width, asset.width, factor),
+        height: lerp(previous.height, asset.height, factor),
+        rotation: smoothAngle(previous.rotation, asset.rotation, factor),
+    };
+    renderStates.set(asset.id, next);
+    return next;
+}
+
+function smoothAngle(current, target, factor) {
+    const delta = ((target - current + 180) % 360) - 180;
+    return current + delta * factor;
+}
+
+function lerp(a, b, t) {
+    return a + (b - a) * t;
+}
+
+function queueAudioForUnlock(controller) {
+    if (!controller) return;
+    pendingAudioUnlock.add(controller);
+}
+
+function safePlay(controller) {
+    if (!controller?.element) return;
+    const playPromise = controller.element.play();
+    if (playPromise?.catch) {
+        playPromise.catch(() => queueAudioForUnlock(controller));
+    }
+}
+
+function recordDuration(assetId, seconds) {
+    if (!Number.isFinite(seconds) || seconds <= 0) {
+        return;
+    }
+    const asset = assets.get(assetId);
+    if (!asset) {
+        return;
+    }
+    const nextMs = Math.round(seconds * 1000);
+    if (asset.durationMs === nextMs) {
+        return;
+    }
+    asset.durationMs = nextMs;
+}
+
+function isVideoAsset(asset) {
+    return asset?.mediaType?.startsWith("video/");
+}
+
+function isAudioAsset(asset) {
+    return asset?.mediaType?.startsWith("audio/");
+}
+function getVideoPlaybackState(element) {
+    if (!element) {
+        return { playRequested: false, unmuteOnPlay: false };
+    }
+    let state = videoPlaybackStates.get(element);
+    if (!state) {
+        state = { playRequested: false, unmuteOnPlay: false };
+        videoPlaybackStates.set(element, state);
+    }
+    return state;
+}
+
+function isCodeAsset(asset) {
+    const type = (asset?.mediaType || asset?.originalMediaType || "").toLowerCase();
+    return type.startsWith("application/javascript") || type.startsWith("text/javascript");
+}
+
+function isVideoElement(element) {
+    return element?.tagName === "VIDEO";
+}
+
+function isGifAsset(asset) {
+    return asset?.mediaType?.toLowerCase() === "image/gif";
+}
+
+function isDrawable(element) {
+    if (!element) {
+        return false;
+    }
+    if (element.isAnimated) {
+        return !!element.bitmap;
+    }
+    if (isVideoElement(element)) {
+        return element.readyState >= 2;
+    }
+    if (typeof ImageBitmap !== "undefined" && element instanceof ImageBitmap) {
+        return true;
+    }
+    return !!element.complete;
+}
+
+function clearMedia(assetId) {
+    const element = mediaCache.get(assetId);
+    if (isVideoElement(element)) {
+        element.src = "";
+        element.remove();
+    }
+    mediaCache.delete(assetId);
+    const animated = animatedCache.get(assetId);
+    if (animated) {
+        animated.cancelled = true;
+        clearTimeout(animated.timeout);
+        animated.bitmap?.close?.();
+        animated.decoder?.close?.();
+        animatedCache.delete(assetId);
+    }
+    animationFailures.delete(assetId);
+    const cachedBlob = blobCache.get(assetId);
+    if (cachedBlob?.objectUrl) {
+        URL.revokeObjectURL(cachedBlob.objectUrl);
+    }
+    blobCache.delete(assetId);
+    const audio = audioControllers.get(assetId);
+    if (audio) {
+        if (audio.delayTimeout) {
+            clearTimeout(audio.delayTimeout);
+        }
+        audio.element.pause();
+        audio.element.currentTime = 0;
+        audio.element.src = "";
+        audio.element.remove();
+        audioControllers.delete(assetId);
+    }
+}
+
+function ensureAudioController(asset) {
+    const cached = audioControllers.get(asset.id);
+    if (cached && cached.src === asset.url) {
+        applyAudioSettings(cached, asset);
+        return cached;
+    }
+
+    if (cached) {
+        clearMedia(asset.id);
+    }
+
+    const element = new Audio(asset.url);
+    element.autoplay = true;
+    element.preload = "auto";
+    element.controls = false;
+    element.addEventListener("loadedmetadata", () => recordDuration(asset.id, element.duration));
+    const controller = {
+        id: asset.id,
+        src: asset.url,
+        element,
+        delayTimeout: null,
+        loopEnabled: false,
+        loopActive: true,
+        delayMs: 0,
+        baseDelayMs: 0,
+    };
+    element.onended = () => handleAudioEnded(asset.id);
+    audioControllers.set(asset.id, controller);
+    applyAudioSettings(controller, asset, true);
+    return controller;
+}
+
+function applyAudioSettings(controller, asset, resetPosition = false) {
+    controller.loopEnabled = !!asset.audioLoop;
+    controller.loopActive = controller.loopEnabled && controller.loopActive !== false;
+    controller.baseDelayMs = Math.max(0, asset.audioDelayMillis || 0);
+    controller.delayMs = controller.baseDelayMs;
+    applyAudioElementSettings(controller.element, asset);
+    if (resetPosition) {
+        controller.element.currentTime = 0;
+        controller.element.pause();
+    }
+}
+
+function applyAudioElementSettings(element, asset) {
+    const speed = Math.max(0.25, asset.audioSpeed || 1);
+    const pitch = Math.max(0.5, asset.audioPitch || 1);
+    element.playbackRate = speed * pitch;
+    const volume = Math.max(0, Math.min(2, asset.audioVolume ?? 1));
+    element.volume = Math.min(volume, 1);
+}
+
+function getAssetVolume(asset) {
+    return Math.max(0, Math.min(2, asset?.audioVolume ?? 1));
+}
+
+function applyMediaVolume(element, asset) {
+    if (!element) return 1;
+    const volume = getAssetVolume(asset);
+    element.volume = Math.min(volume, 1);
+    return volume;
+}
+
+function handleAudioEnded(assetId) {
+    const controller = audioControllers.get(assetId);
+    if (!controller) return;
+    controller.element.currentTime = 0;
+    if (controller.delayTimeout) {
+        clearTimeout(controller.delayTimeout);
+    }
+    if (controller.loopEnabled && controller.loopActive) {
+        controller.delayTimeout = setTimeout(() => {
+            safePlay(controller);
+        }, controller.delayMs);
+    } else {
+        controller.element.pause();
+    }
+}
+
+function stopAudio(assetId) {
+    const controller = audioControllers.get(assetId);
+    if (!controller) return;
+    if (controller.delayTimeout) {
+        clearTimeout(controller.delayTimeout);
+    }
+    controller.element.pause();
+    controller.element.currentTime = 0;
+    controller.delayTimeout = null;
+    controller.delayMs = controller.baseDelayMs;
+    controller.loopActive = false;
+}
+
+function playAudioImmediately(asset) {
+    const controller = ensureAudioController(asset);
+    if (controller.delayTimeout) {
+        clearTimeout(controller.delayTimeout);
+        controller.delayTimeout = null;
+    }
+    controller.element.currentTime = 0;
+    const originalDelay = controller.delayMs;
+    controller.delayMs = 0;
+    safePlay(controller);
+    controller.delayMs = controller.baseDelayMs ?? originalDelay ?? 0;
+}
+
+function playOverlappingAudio(asset) {
+    const temp = new Audio(asset.url);
+    temp.autoplay = true;
+    temp.preload = "auto";
+    temp.controls = false;
+    applyAudioElementSettings(temp, asset);
+    const controller = { element: temp };
+    temp.onended = () => {
+        temp.remove();
+    };
+    safePlay(controller);
+}
+
+function handleAudioPlay(asset, shouldPlay) {
+    const controller = ensureAudioController(asset);
+    controller.loopActive = !!shouldPlay;
+    if (!shouldPlay) {
+        stopAudio(asset.id);
+        return;
+    }
+    if (asset.audioLoop) {
+        controller.delayMs = controller.baseDelayMs;
+        safePlay(controller);
+    } else {
+        playOverlappingAudio(asset);
+    }
+}
+
+function autoStartAudio(asset) {
+    if (!isAudioAsset(asset) || asset.hidden) {
+        return;
+    }
+    const controller = ensureAudioController(asset);
+    if (!controller.loopEnabled || !controller.loopActive) {
+        return;
+    }
+    if (!controller.element.paused && !controller.element.ended) {
+        return;
+    }
+    if (controller.delayTimeout) {
+        return;
+    }
+    controller.delayTimeout = setTimeout(() => {
+        safePlay(controller);
+    }, controller.delayMs);
+}
+
+function ensureMedia(asset) {
+    const cached = mediaCache.get(asset.id);
+    const cachedSource = getCachedSource(cached);
+    if (cached && cachedSource !== asset.url) {
+        clearMedia(asset.id);
+    }
+    if (cached && cachedSource === asset.url) {
+        applyMediaSettings(cached, asset);
+        return cached;
+    }
+
+    if (isAudioAsset(asset)) {
+        ensureAudioController(asset);
+        mediaCache.delete(asset.id);
+        return null;
+    }
+
+    if (isGifAsset(asset) && supportsAnimatedDecode) {
+        const animated = ensureAnimatedImage(asset);
+        if (animated) {
+            mediaCache.set(asset.id, animated);
+            return animated;
+        }
+    }
+
+    const element = isVideoAsset(asset) ? document.createElement("video") : new Image();
+    element.dataset.sourceUrl = asset.url;
+    element.crossOrigin = "anonymous";
+    if (isVideoElement(element)) {
+        if (!canPlayVideoType(asset.mediaType)) {
+            return null;
+        }
+        element.loop = true;
+        element.playsInline = true;
+        element.autoplay = true;
+        element.controls = false;
+        element.onloadeddata = draw;
+        element.onloadedmetadata = () => recordDuration(asset.id, element.duration);
+        element.preload = "auto";
+        element.addEventListener("error", () => clearMedia(asset.id));
+        const playbackState = getVideoPlaybackState(element);
+        element.addEventListener("playing", () => {
+            playbackState.playRequested = false;
+            if (playbackState.unmuteOnPlay) {
+                element.muted = false;
+                playbackState.unmuteOnPlay = false;
+            }
+        });
+        element.addEventListener("pause", () => {
+            playbackState.playRequested = false;
+        });
+        applyMediaVolume(element, asset);
+        element.muted = true;
+        setVideoSource(element, asset);
+    } else {
+        element.onload = draw;
+        element.src = asset.url;
+    }
+    mediaCache.set(asset.id, element);
+    return element;
+}
+
+function ensureAnimatedImage(asset) {
+    const failedAt = animationFailures.get(asset.id);
+    if (failedAt && Date.now() - failedAt < 15000) {
+        return null;
+    }
+    const cached = animatedCache.get(asset.id);
+    if (cached && cached.url === asset.url) {
+        return cached;
+    }
+
+    animationFailures.delete(asset.id);
+
+    if (cached) {
+        clearMedia(asset.id);
+    }
+
+    const controller = {
+        id: asset.id,
+        url: asset.url,
+        src: asset.url,
+        decoder: null,
+        bitmap: null,
+        timeout: null,
+        cancelled: false,
+        isAnimated: true,
+    };
+
+    fetchAssetBlob(asset)
+        .then((blob) => new ImageDecoder({ data: blob, type: blob.type || "image/gif" }))
+        .then((decoder) => {
+            if (controller.cancelled) {
+                decoder.close?.();
+                return null;
+            }
+            controller.decoder = decoder;
+            scheduleNextFrame(controller);
+            return controller;
+        })
+        .catch(() => {
+            animatedCache.delete(asset.id);
+            animationFailures.set(asset.id, Date.now());
+        });
+
+    animatedCache.set(asset.id, controller);
+    return controller;
+}
+
+function fetchAssetBlob(asset) {
+    const cached = blobCache.get(asset.id);
+    if (cached && cached.url === asset.url && cached.blob) {
+        return Promise.resolve(cached.blob);
+    }
+    if (cached && cached.url === asset.url && cached.pending) {
+        return cached.pending;
+    }
+
+    const pending = fetch(asset.url)
+        .then((r) => r.blob())
+        .then((blob) => {
+            const previous = blobCache.get(asset.id);
+            const existingUrl = previous?.url === asset.url ? previous.objectUrl : null;
+            const objectUrl = existingUrl || URL.createObjectURL(blob);
+            blobCache.set(asset.id, { url: asset.url, blob, objectUrl });
+            return blob;
+        });
+    blobCache.set(asset.id, { url: asset.url, pending });
+    return pending;
+}
+
+function setVideoSource(element, asset) {
+    if (!shouldUseBlobUrl(asset)) {
+        applyVideoSource(element, asset.url, asset);
+        return;
+    }
+
+    const cached = blobCache.get(asset.id);
+    if (cached?.url === asset.url && cached.objectUrl) {
+        applyVideoSource(element, cached.objectUrl, asset);
+        return;
+    }
+
+    fetchAssetBlob(asset)
+        .then(() => {
+            const next = blobCache.get(asset.id);
+            if (next?.url !== asset.url || !next.objectUrl) {
+                return;
+            }
+            applyVideoSource(element, next.objectUrl, asset);
+        })
+        .catch(() => { });
+}
+
+function applyVideoSource(element, objectUrl, asset) {
+    element.src = objectUrl;
+    startVideoPlayback(element, asset);
+}
+
+function shouldUseBlobUrl(asset) {
+    return !obsBrowser && asset?.mediaType && canPlayVideoType(asset.mediaType);
+}
+
+function canPlayVideoType(mediaType) {
+    if (!mediaType) {
+        return true;
+    }
+    const support = canPlayProbe.canPlayType(mediaType);
+    return support === "probably" || support === "maybe";
+}
+
+function getCachedSource(element) {
+    return element?.dataset?.sourceUrl || element?.src;
+}
+
+function scheduleNextFrame(controller) {
+    if (controller.cancelled || !controller.decoder) {
+        return;
+    }
+    controller.decoder
+        .decode()
+        .then(({ image, complete }) => {
+            if (controller.cancelled) {
+                image.close?.();
+                return;
+            }
+            controller.bitmap?.close?.();
+            createImageBitmap(image)
+                .then((bitmap) => {
+                    controller.bitmap = bitmap;
+                    draw();
+                })
+                .finally(() => image.close?.());
+
+            const durationMicros = image.duration || 0;
+            const delay = durationMicros > 0 ? durationMicros / 1000 : 100;
+            const hasMore = !complete;
+            controller.timeout = setTimeout(() => {
+                if (controller.cancelled) {
+                    return;
+                }
+                if (hasMore) {
+                    scheduleNextFrame(controller);
+                } else {
+                    controller.decoder.reset();
+                    scheduleNextFrame(controller);
+                }
+            }, delay);
+        })
+        .catch(() => {
+            // If decoding fails, clear animated cache so static fallback is used next render
+            animatedCache.delete(controller.id);
+            animationFailures.set(controller.id, Date.now());
+        });
+}
+
+function applyMediaSettings(element, asset) {
+    if (!isVideoElement(element)) {
+        return;
+    }
+    startVideoPlayback(element, asset);
+}
+
+function startVideoPlayback(element, asset) {
+    const playbackState = getVideoPlaybackState(element);
+    const nextSpeed = asset.speed ?? 1;
+    const effectiveSpeed = Math.max(nextSpeed, 0.01);
+    if (element.playbackRate !== effectiveSpeed) {
+        element.playbackRate = effectiveSpeed;
+    }
+    const volume = applyMediaVolume(element, asset);
+    const shouldUnmute = volume > 0;
+    element.muted = true;
+
+    if (effectiveSpeed === 0) {
+        element.pause();
+        playbackState.playRequested = false;
+        playbackState.unmuteOnPlay = false;
+        return;
+    }
+
+    element.play();
+
+    if (shouldUnmute) {
+        if (!element.paused && element.readyState >= 2) {
+            element.muted = false;
+        } else {
+            playbackState.unmuteOnPlay = true;
+        }
+    }
+
+    if (element.paused || element.ended) {
+        if (!playbackState.playRequested) {
+            playbackState.playRequested = true;
+            const playPromise = element.play();
+            if (playPromise?.catch) {
+                playPromise.catch(() => {
+                    playbackState.playRequested = false;
+                });
+            }
+        }
+    }
+}
+
+function startRenderLoop() {
+    if (renderIntervalId) {
+        return;
+    }
+    renderIntervalId = setInterval(() => {
+        draw();
+    }, MIN_FRAME_TIME);
+}
+
+globalThis.addEventListener("resize", () => {
+    resizeCanvas();
+});
+
+fetchCanvasSettings().finally(() => {
+    resizeCanvas();
+    startRenderLoop();
+    wsconnect(broadcaster, {
+        onFetch: renderAssets,
+        onWsEvent: onWebsocketEvent,
+    });
+});

src/main/resources/static/js/broadcast/ws.js

@@ -0,0 +1,20 @@
+export function wsconnect(broadcaster, { onFetch, onWsEvent }) {
+    const socket = new SockJS("/ws");
+    const stompClient = Stomp.over(socket);
+    stompClient.connect({}, () => {
+        stompClient.subscribe(`/topic/channel/${broadcaster}`, (payload) => onWsEvent(JSON.parse(payload.body)));
+        fetch(`/api/channels/${broadcaster}/assets`)
+            .then((r) => {
+                if (!r.ok) {
+                    throw new Error("Failed to load assets");
+                }
+                return r.json();
+            })
+            .then(onFetch)
+            .catch((error) => {
+                console.error(error);
+                showToast("Unable to load overlay assets. Retrying may help.", "error")
+            });
+    });
+}
+