From 4a96efd310d3c3a070d7b02c5eb23d6aefb3e06f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Kr=C3=BChlmann?= Date: Thu, 8 Jan 2026 15:15:03 +0100 Subject: [PATCH] Temp refactor --- .../resources/static/js/broadcast/const.js | 4 + src/main/resources/static/js/broadcast/dom.js | 2 + src/main/resources/static/js/broadcast/gfx.js | 34 + .../resources/static/js/broadcast/globals.js | 0 .../static/js/broadcast/jsworkers.js | 15 + .../resources/static/js/broadcast/main.js | 929 ++++++++++++++++++ src/main/resources/static/js/broadcast/ws.js | 20 + 7 files changed, 1004 insertions(+) create mode 100644 src/main/resources/static/js/broadcast/const.js create mode 100644 src/main/resources/static/js/broadcast/dom.js create mode 100644 src/main/resources/static/js/broadcast/gfx.js create mode 100644 src/main/resources/static/js/broadcast/globals.js create mode 100644 src/main/resources/static/js/broadcast/jsworkers.js create mode 100644 src/main/resources/static/js/broadcast/main.js create mode 100644 src/main/resources/static/js/broadcast/ws.js diff --git a/src/main/resources/static/js/broadcast/const.js b/src/main/resources/static/js/broadcast/const.js new file mode 100644 index 0000000..a7c4f78 --- /dev/null +++ b/src/main/resources/static/js/broadcast/const.js @@ -0,0 +1,4 @@ +const TARGET_FPS = 60; + +export const MIN_FRAME_TIME = 1000 / TARGET_FPS; +export const VISIBILITY_THRESHOLD = 0.01; diff --git a/src/main/resources/static/js/broadcast/dom.js b/src/main/resources/static/js/broadcast/dom.js new file mode 100644 index 0000000..d615525 --- /dev/null +++ b/src/main/resources/static/js/broadcast/dom.js @@ -0,0 +1,2 @@ +export const canvas = document.getElementById("broadcast-canvas"); +export const canPlayProbe = document.createElement("video"); diff --git a/src/main/resources/static/js/broadcast/gfx.js b/src/main/resources/static/js/broadcast/gfx.js new file mode 100644 index 0000000..b69eaa1 --- /dev/null +++ b/src/main/resources/static/js/broadcast/gfx.js @@ -0,0 +1,34 @@ +import { MIN_FRAME_TIME } from "./const.js" + +let frameScheduled = false; +let lastRenderTime = 0; +let pendingDraw = false; + +export function draw(ctx) { + if (frameScheduled) { + pendingDraw = true; + return; + } + frameScheduled = true; + requestAnimationFrame((timestamp) => { + const elapsed = timestamp - lastRenderTime; + const delay = MIN_FRAME_TIME - elapsed; + const shouldRender = elapsed >= MIN_FRAME_TIME; + + if (shouldRender) { + lastRenderTime = timestamp; + renderFrame(ctx); + } + + frameScheduled = false; + if (pendingDraw || !shouldRender) { + pendingDraw = false; + setTimeout(draw, Math.max(0, delay)); + } + }); +} + +function renderFrame(ctx) { + ctx.clearRect(0, 0, canvas.width, canvas.height); + getRenderOrder().forEach(drawAsset); +} diff --git a/src/main/resources/static/js/broadcast/globals.js b/src/main/resources/static/js/broadcast/globals.js new file mode 100644 index 0000000..e69de29 diff --git a/src/main/resources/static/js/broadcast/jsworkers.js b/src/main/resources/static/js/broadcast/jsworkers.js new file mode 100644 index 0000000..873eada --- /dev/null +++ b/src/main/resources/static/js/broadcast/jsworkers.js @@ -0,0 +1,15 @@ +export async function spawnUserJavaScriptWorker(asset) { + // let assetSource; + // try { + // assetSource = await fetch(asset.url).then((r) => r.text()); + // } catch (error) { + // console.error(`Unable to fetch asset with id:${id} from url:${asset.url}`, error); + // return; + // } + // const blob = new Blob([assetSource], { type: 'application/javascript' }); + // const worker = new Worker(URL.createObjectURL(blob)); + // worker.onmessage = (event) => { + // console.log('Message from worker:', event.data); + // } + // worker.postMessage(data); +} diff --git a/src/main/resources/static/js/broadcast/main.js b/src/main/resources/static/js/broadcast/main.js new file mode 100644 index 0000000..c2776bd --- /dev/null +++ b/src/main/resources/static/js/broadcast/main.js @@ -0,0 +1,929 @@ +import { spawnUserJavaScriptWorker } from "./jsworkers.js" +import { canvas, canPlayProbe } from "./dom.js"; +import { MIN_FRAME_TIME, VISIBILITY_THRESHOLD } from "./const.js" +import { wsconnect } from "./ws.js"; + +const obsBrowser = !!globalThis.obsstudio; +const supportsAnimatedDecode = + typeof ImageDecoder !== "undefined" && typeof createImageBitmap === "function" && !obsBrowser; +const ctx = canvas.getContext("2d"); +let canvasSettings = { width: 1920, height: 1080 }; +const assets = new Map(); +const mediaCache = new Map(); +const renderStates = new Map(); +const visibilityStates = new Map(); +const animatedCache = new Map(); +const blobCache = new Map(); +const animationFailures = new Map(); +const audioControllers = new Map(); +const videoPlaybackStates = new WeakMap(); +const pendingAudioUnlock = new Set(); +let lastRenderTime = 0; +let frameScheduled = false; +let pendingDraw = false; +let renderIntervalId = null; +let layerOrder = []; + +applyCanvasSettings(canvasSettings); + +["pointerdown", "keydown", "touchstart"].forEach((eventName) => { + globalThis.addEventListener(eventName, () => { + if (!pendingAudioUnlock.size) return; + pendingAudioUnlock.forEach((controller) => safePlay(controller)); + pendingAudioUnlock.clear(); + }); +}); + +function ensureLayerPosition(assetId, placement = "keep") { + const asset = assets.get(assetId); + if (asset && (isAudioAsset(asset) || isCodeAsset(asset))) { + return; + } + const existingIndex = layerOrder.indexOf(assetId); + if (existingIndex !== -1 && placement === "keep") { + return; + } + if (existingIndex !== -1) { + layerOrder.splice(existingIndex, 1); + } + if (placement === "append") { + layerOrder.push(assetId); + } else { + layerOrder.unshift(assetId); + } + layerOrder = layerOrder.filter((id) => assets.has(id)); +} + +function getLayerOrder() { + layerOrder = layerOrder.filter((id) => { + const asset = assets.get(id); + return asset && !isAudioAsset(asset) && !isCodeAsset(asset); + }); + assets.forEach((asset, id) => { + if (isAudioAsset(asset) || isCodeAsset(asset)) { + return; + } + if (!layerOrder.includes(id)) { + layerOrder.unshift(id); + } + }); + return layerOrder; +} + +function getRenderOrder() { + return [...getLayerOrder()] + .reverse() + .map((id) => assets.get(id)) + .filter(Boolean); +} + +function removeAsset(assetId) { + assets.delete(assetId); + layerOrder = layerOrder.filter((id) => id !== assetId); + clearMedia(assetId); + renderStates.delete(assetId); + visibilityStates.delete(assetId); +} + +function renderAssets(list) { + layerOrder = []; + list.forEach((asset) => { + storeAsset(asset, "append"); + if (isCodeAsset(asset)) { + spawnUserJavaScriptWorker(asset); + } + }); + draw(); +} + +function storeAsset(asset, placement = "keep") { + if (!asset) return; + console.info(`Storing asset: ${asset.id}`); + const wasExisting = assets.has(asset.id); + assets.set(asset.id, asset); + ensureLayerPosition(asset.id, placement); + if (!wasExisting && !visibilityStates.has(asset.id)) { + const initialAlpha = 0; // Fade in newly discovered assets + visibilityStates.set(asset.id, { alpha: initialAlpha, targetHidden: !!asset.hidden }); + } +} + +async function fetchCanvasSettings() { + return fetch(`/api/channels/${encodeURIComponent(broadcaster)}/canvas`) + .then((r) => { + if (!r.ok) { + throw new Error("Failed to load canvas"); + } + return r.json(); + }) + .then((settings) => { + applyCanvasSettings(settings); + }) + .catch(() => { + resizeCanvas(); + showToast("Using default canvas size. Unable to load saved settings.", "warning"); + }); +} + +function applyCanvasSettings(settings) { + if (!settings) { + return; + } + const width = Number.isFinite(settings.width) ? settings.width : canvasSettings.width; + const height = Number.isFinite(settings.height) ? settings.height : canvasSettings.height; + canvasSettings = { width, height }; + resizeCanvas(); +} + +function resizeCanvas() { + if (Number.isFinite(canvasSettings.width) && Number.isFinite(canvasSettings.height)) { + canvas.width = canvasSettings.width; + canvas.height = canvasSettings.height; + canvas.style.width = `${canvasSettings.width}px`; + canvas.style.height = `${canvasSettings.height}px`; + } + draw(); +} + +function onWebsocketEvent(event) { + if (event.type === "CANVAS" && event.payload) { + applyCanvasSettings(event.payload); + return; + } + const assetId = event.assetId || event?.patch?.id || event?.payload?.id; + if (event.type === "VISIBILITY") { + handleVisibilityEvent(event); + return; + } + if (event.type === "DELETED") { + removeAsset(assetId); + } else if (event.patch) { + applyPatch(assetId, event.patch); + if (event.payload) { + const payload = normalizePayload(event.payload); + if (payload.hidden) { + hideAssetWithTransition(payload); + } else if (!assets.has(payload.id)) { + upsertVisibleAsset(payload, "append"); + } + } + } else if (event.type === "PLAY" && event.payload) { + const payload = normalizePayload(event.payload); + storeAsset(payload); + if (isAudioAsset(payload)) { + handleAudioPlay(payload, event.play !== false); + } + } else if (event.payload && !event.payload.hidden) { + const payload = normalizePayload(event.payload); + upsertVisibleAsset(payload); + } else if (event.payload && event.payload.hidden) { + hideAssetWithTransition(event.payload); + } + draw(); +} + +function normalizePayload(payload) { + return { ...payload }; +} + +function hideAssetWithTransition(asset) { + const payload = asset ? normalizePayload(asset) : null; + if (!payload?.id) { + return; + } + const existing = assets.get(payload.id); + if ( + !existing && + (!Number.isFinite(payload.x) || + !Number.isFinite(payload.y) || + !Number.isFinite(payload.width) || + !Number.isFinite(payload.height)) + ) { + return; + } + const merged = normalizePayload({ ...(existing || {}), ...payload, hidden: true }); + storeAsset(merged); + stopAudio(payload.id); +} + +function upsertVisibleAsset(asset, placement = "keep") { + const payload = asset ? normalizePayload(asset) : null; + if (!payload?.id) { + return; + } + const placementMode = assets.has(payload.id) ? "keep" : placement; + storeAsset(payload, placementMode); + ensureMedia(payload); + if (isAudioAsset(payload)) { + playAudioImmediately(payload); + } else if (isCodeAsset(payload)) { + spawnUserJavaScriptWorker(payload); + } +} + +function handleVisibilityEvent(event) { + const payload = event.payload ? normalizePayload(event.payload) : null; + const patch = event.patch; + const id = payload?.id || patch?.id || event.assetId; + + if (payload?.hidden || patch?.hidden) { + hideAssetWithTransition({ id, ...payload, ...patch }); + draw(); + return; + } + + if (payload) { + const placement = assets.has(payload.id) ? "keep" : "append"; + upsertVisibleAsset(payload, placement); + } + + if (patch && id) { + applyPatch(id, patch); + } + + draw(); +} + +function applyPatch(assetId, patch) { + if (!assetId || !patch) { + return; + } + const sanitizedPatch = Object.fromEntries( + Object.entries(patch).filter(([, value]) => value !== null && value !== undefined), + ); + const existing = assets.get(assetId); + if (!existing) { + return; + } + const merged = normalizePayload({ ...existing, ...sanitizedPatch }); + console.log(merged) + const isAudio = isAudioAsset(merged); + if (sanitizedPatch.hidden) { + hideAssetWithTransition(merged); + return; + } + const targetLayer = Number.isFinite(patch.layer) + ? patch.layer + : Number.isFinite(patch.zIndex) + ? patch.zIndex + : null; + if (!isAudio && Number.isFinite(targetLayer)) { + const currentOrder = getLayerOrder().filter((id) => id !== assetId); + const insertIndex = Math.max(0, currentOrder.length - Math.round(targetLayer)); + currentOrder.splice(insertIndex, 0, assetId); + layerOrder = currentOrder; + } + storeAsset(merged); + ensureMedia(merged); + if (isCodeAsset(merged)) { + console.info(`Spawning JS worker for patched asset: ${merged.id}`); + spawnUserJavaScriptWorker(merged); + } +} + +function draw() { + if (frameScheduled) { + pendingDraw = true; + return; + } + frameScheduled = true; + requestAnimationFrame((timestamp) => { + const elapsed = timestamp - lastRenderTime; + const delay = MIN_FRAME_TIME - elapsed; + const shouldRender = elapsed >= MIN_FRAME_TIME; + + if (shouldRender) { + lastRenderTime = timestamp; + renderFrame(); + } + + frameScheduled = false; + if (pendingDraw || !shouldRender) { + pendingDraw = false; + setTimeout(draw, Math.max(0, delay)); + } + }); +} + +function renderFrame() { + ctx.clearRect(0, 0, canvas.width, canvas.height); + getRenderOrder().forEach(drawAsset); +} + +function drawAsset(asset) { + const visibility = getVisibilityState(asset); + if (visibility.alpha <= VISIBILITY_THRESHOLD && asset.hidden) { + return; + } + const renderState = smoothState(asset); + const halfWidth = renderState.width / 2; + const halfHeight = renderState.height / 2; + ctx.save(); + ctx.globalAlpha = Math.max(0, Math.min(1, visibility.alpha)); + ctx.translate(renderState.x + halfWidth, renderState.y + halfHeight); + ctx.rotate((renderState.rotation * Math.PI) / 180); + + if (isCodeAsset(asset)) { + ctx.restore(); + return; + } + + if (isAudioAsset(asset)) { + if (!asset.hidden) { + autoStartAudio(asset); + } + ctx.restore(); + return; + } + + const media = ensureMedia(asset); + const drawSource = media?.isAnimated ? media.bitmap : media; + const ready = isDrawable(media); + if (ready && drawSource) { + ctx.drawImage(drawSource, -halfWidth, -halfHeight, renderState.width, renderState.height); + } + + ctx.restore(); +} + +function getVisibilityState(asset) { + const current = visibilityStates.get(asset.id) || {}; + const targetAlpha = asset.hidden ? 0 : 1; + const startingAlpha = Number.isFinite(current.alpha) ? current.alpha : 0; + const factor = asset.hidden ? 0.18 : 0.2; + const nextAlpha = lerp(startingAlpha, targetAlpha, factor); + const state = { alpha: nextAlpha, targetHidden: !!asset.hidden }; + visibilityStates.set(asset.id, state); + return state; +} + +function smoothState(asset) { + const previous = renderStates.get(asset.id) || { ...asset }; + const factor = 0.15; + const next = { + x: lerp(previous.x, asset.x, factor), + y: lerp(previous.y, asset.y, factor), + width: lerp(previous.width, asset.width, factor), + height: lerp(previous.height, asset.height, factor), + rotation: smoothAngle(previous.rotation, asset.rotation, factor), + }; + renderStates.set(asset.id, next); + return next; +} + +function smoothAngle(current, target, factor) { + const delta = ((target - current + 180) % 360) - 180; + return current + delta * factor; +} + +function lerp(a, b, t) { + return a + (b - a) * t; +} + +function queueAudioForUnlock(controller) { + if (!controller) return; + pendingAudioUnlock.add(controller); +} + +function safePlay(controller) { + if (!controller?.element) return; + const playPromise = controller.element.play(); + if (playPromise?.catch) { + playPromise.catch(() => queueAudioForUnlock(controller)); + } +} + +function recordDuration(assetId, seconds) { + if (!Number.isFinite(seconds) || seconds <= 0) { + return; + } + const asset = assets.get(assetId); + if (!asset) { + return; + } + const nextMs = Math.round(seconds * 1000); + if (asset.durationMs === nextMs) { + return; + } + asset.durationMs = nextMs; +} + +function isVideoAsset(asset) { + return asset?.mediaType?.startsWith("video/"); +} + +function isAudioAsset(asset) { + return asset?.mediaType?.startsWith("audio/"); +} +function getVideoPlaybackState(element) { + if (!element) { + return { playRequested: false, unmuteOnPlay: false }; + } + let state = videoPlaybackStates.get(element); + if (!state) { + state = { playRequested: false, unmuteOnPlay: false }; + videoPlaybackStates.set(element, state); + } + return state; +} + +function isCodeAsset(asset) { + const type = (asset?.mediaType || asset?.originalMediaType || "").toLowerCase(); + return type.startsWith("application/javascript") || type.startsWith("text/javascript"); +} + +function isVideoElement(element) { + return element?.tagName === "VIDEO"; +} + +function isGifAsset(asset) { + return asset?.mediaType?.toLowerCase() === "image/gif"; +} + +function isDrawable(element) { + if (!element) { + return false; + } + if (element.isAnimated) { + return !!element.bitmap; + } + if (isVideoElement(element)) { + return element.readyState >= 2; + } + if (typeof ImageBitmap !== "undefined" && element instanceof ImageBitmap) { + return true; + } + return !!element.complete; +} + +function clearMedia(assetId) { + const element = mediaCache.get(assetId); + if (isVideoElement(element)) { + element.src = ""; + element.remove(); + } + mediaCache.delete(assetId); + const animated = animatedCache.get(assetId); + if (animated) { + animated.cancelled = true; + clearTimeout(animated.timeout); + animated.bitmap?.close?.(); + animated.decoder?.close?.(); + animatedCache.delete(assetId); + } + animationFailures.delete(assetId); + const cachedBlob = blobCache.get(assetId); + if (cachedBlob?.objectUrl) { + URL.revokeObjectURL(cachedBlob.objectUrl); + } + blobCache.delete(assetId); + const audio = audioControllers.get(assetId); + if (audio) { + if (audio.delayTimeout) { + clearTimeout(audio.delayTimeout); + } + audio.element.pause(); + audio.element.currentTime = 0; + audio.element.src = ""; + audio.element.remove(); + audioControllers.delete(assetId); + } +} + +function ensureAudioController(asset) { + const cached = audioControllers.get(asset.id); + if (cached && cached.src === asset.url) { + applyAudioSettings(cached, asset); + return cached; + } + + if (cached) { + clearMedia(asset.id); + } + + const element = new Audio(asset.url); + element.autoplay = true; + element.preload = "auto"; + element.controls = false; + element.addEventListener("loadedmetadata", () => recordDuration(asset.id, element.duration)); + const controller = { + id: asset.id, + src: asset.url, + element, + delayTimeout: null, + loopEnabled: false, + loopActive: true, + delayMs: 0, + baseDelayMs: 0, + }; + element.onended = () => handleAudioEnded(asset.id); + audioControllers.set(asset.id, controller); + applyAudioSettings(controller, asset, true); + return controller; +} + +function applyAudioSettings(controller, asset, resetPosition = false) { + controller.loopEnabled = !!asset.audioLoop; + controller.loopActive = controller.loopEnabled && controller.loopActive !== false; + controller.baseDelayMs = Math.max(0, asset.audioDelayMillis || 0); + controller.delayMs = controller.baseDelayMs; + applyAudioElementSettings(controller.element, asset); + if (resetPosition) { + controller.element.currentTime = 0; + controller.element.pause(); + } +} + +function applyAudioElementSettings(element, asset) { + const speed = Math.max(0.25, asset.audioSpeed || 1); + const pitch = Math.max(0.5, asset.audioPitch || 1); + element.playbackRate = speed * pitch; + const volume = Math.max(0, Math.min(2, asset.audioVolume ?? 1)); + element.volume = Math.min(volume, 1); +} + +function getAssetVolume(asset) { + return Math.max(0, Math.min(2, asset?.audioVolume ?? 1)); +} + +function applyMediaVolume(element, asset) { + if (!element) return 1; + const volume = getAssetVolume(asset); + element.volume = Math.min(volume, 1); + return volume; +} + +function handleAudioEnded(assetId) { + const controller = audioControllers.get(assetId); + if (!controller) return; + controller.element.currentTime = 0; + if (controller.delayTimeout) { + clearTimeout(controller.delayTimeout); + } + if (controller.loopEnabled && controller.loopActive) { + controller.delayTimeout = setTimeout(() => { + safePlay(controller); + }, controller.delayMs); + } else { + controller.element.pause(); + } +} + +function stopAudio(assetId) { + const controller = audioControllers.get(assetId); + if (!controller) return; + if (controller.delayTimeout) { + clearTimeout(controller.delayTimeout); + } + controller.element.pause(); + controller.element.currentTime = 0; + controller.delayTimeout = null; + controller.delayMs = controller.baseDelayMs; + controller.loopActive = false; +} + +function playAudioImmediately(asset) { + const controller = ensureAudioController(asset); + if (controller.delayTimeout) { + clearTimeout(controller.delayTimeout); + controller.delayTimeout = null; + } + controller.element.currentTime = 0; + const originalDelay = controller.delayMs; + controller.delayMs = 0; + safePlay(controller); + controller.delayMs = controller.baseDelayMs ?? originalDelay ?? 0; +} + +function playOverlappingAudio(asset) { + const temp = new Audio(asset.url); + temp.autoplay = true; + temp.preload = "auto"; + temp.controls = false; + applyAudioElementSettings(temp, asset); + const controller = { element: temp }; + temp.onended = () => { + temp.remove(); + }; + safePlay(controller); +} + +function handleAudioPlay(asset, shouldPlay) { + const controller = ensureAudioController(asset); + controller.loopActive = !!shouldPlay; + if (!shouldPlay) { + stopAudio(asset.id); + return; + } + if (asset.audioLoop) { + controller.delayMs = controller.baseDelayMs; + safePlay(controller); + } else { + playOverlappingAudio(asset); + } +} + +function autoStartAudio(asset) { + if (!isAudioAsset(asset) || asset.hidden) { + return; + } + const controller = ensureAudioController(asset); + if (!controller.loopEnabled || !controller.loopActive) { + return; + } + if (!controller.element.paused && !controller.element.ended) { + return; + } + if (controller.delayTimeout) { + return; + } + controller.delayTimeout = setTimeout(() => { + safePlay(controller); + }, controller.delayMs); +} + +function ensureMedia(asset) { + const cached = mediaCache.get(asset.id); + const cachedSource = getCachedSource(cached); + if (cached && cachedSource !== asset.url) { + clearMedia(asset.id); + } + if (cached && cachedSource === asset.url) { + applyMediaSettings(cached, asset); + return cached; + } + + if (isAudioAsset(asset)) { + ensureAudioController(asset); + mediaCache.delete(asset.id); + return null; + } + + if (isGifAsset(asset) && supportsAnimatedDecode) { + const animated = ensureAnimatedImage(asset); + if (animated) { + mediaCache.set(asset.id, animated); + return animated; + } + } + + const element = isVideoAsset(asset) ? document.createElement("video") : new Image(); + element.dataset.sourceUrl = asset.url; + element.crossOrigin = "anonymous"; + if (isVideoElement(element)) { + if (!canPlayVideoType(asset.mediaType)) { + return null; + } + element.loop = true; + element.playsInline = true; + element.autoplay = true; + element.controls = false; + element.onloadeddata = draw; + element.onloadedmetadata = () => recordDuration(asset.id, element.duration); + element.preload = "auto"; + element.addEventListener("error", () => clearMedia(asset.id)); + const playbackState = getVideoPlaybackState(element); + element.addEventListener("playing", () => { + playbackState.playRequested = false; + if (playbackState.unmuteOnPlay) { + element.muted = false; + playbackState.unmuteOnPlay = false; + } + }); + element.addEventListener("pause", () => { + playbackState.playRequested = false; + }); + applyMediaVolume(element, asset); + element.muted = true; + setVideoSource(element, asset); + } else { + element.onload = draw; + element.src = asset.url; + } + mediaCache.set(asset.id, element); + return element; +} + +function ensureAnimatedImage(asset) { + const failedAt = animationFailures.get(asset.id); + if (failedAt && Date.now() - failedAt < 15000) { + return null; + } + const cached = animatedCache.get(asset.id); + if (cached && cached.url === asset.url) { + return cached; + } + + animationFailures.delete(asset.id); + + if (cached) { + clearMedia(asset.id); + } + + const controller = { + id: asset.id, + url: asset.url, + src: asset.url, + decoder: null, + bitmap: null, + timeout: null, + cancelled: false, + isAnimated: true, + }; + + fetchAssetBlob(asset) + .then((blob) => new ImageDecoder({ data: blob, type: blob.type || "image/gif" })) + .then((decoder) => { + if (controller.cancelled) { + decoder.close?.(); + return null; + } + controller.decoder = decoder; + scheduleNextFrame(controller); + return controller; + }) + .catch(() => { + animatedCache.delete(asset.id); + animationFailures.set(asset.id, Date.now()); + }); + + animatedCache.set(asset.id, controller); + return controller; +} + +function fetchAssetBlob(asset) { + const cached = blobCache.get(asset.id); + if (cached && cached.url === asset.url && cached.blob) { + return Promise.resolve(cached.blob); + } + if (cached && cached.url === asset.url && cached.pending) { + return cached.pending; + } + + const pending = fetch(asset.url) + .then((r) => r.blob()) + .then((blob) => { + const previous = blobCache.get(asset.id); + const existingUrl = previous?.url === asset.url ? previous.objectUrl : null; + const objectUrl = existingUrl || URL.createObjectURL(blob); + blobCache.set(asset.id, { url: asset.url, blob, objectUrl }); + return blob; + }); + blobCache.set(asset.id, { url: asset.url, pending }); + return pending; +} + +function setVideoSource(element, asset) { + if (!shouldUseBlobUrl(asset)) { + applyVideoSource(element, asset.url, asset); + return; + } + + const cached = blobCache.get(asset.id); + if (cached?.url === asset.url && cached.objectUrl) { + applyVideoSource(element, cached.objectUrl, asset); + return; + } + + fetchAssetBlob(asset) + .then(() => { + const next = blobCache.get(asset.id); + if (next?.url !== asset.url || !next.objectUrl) { + return; + } + applyVideoSource(element, next.objectUrl, asset); + }) + .catch(() => { }); +} + +function applyVideoSource(element, objectUrl, asset) { + element.src = objectUrl; + startVideoPlayback(element, asset); +} + +function shouldUseBlobUrl(asset) { + return !obsBrowser && asset?.mediaType && canPlayVideoType(asset.mediaType); +} + +function canPlayVideoType(mediaType) { + if (!mediaType) { + return true; + } + const support = canPlayProbe.canPlayType(mediaType); + return support === "probably" || support === "maybe"; +} + +function getCachedSource(element) { + return element?.dataset?.sourceUrl || element?.src; +} + +function scheduleNextFrame(controller) { + if (controller.cancelled || !controller.decoder) { + return; + } + controller.decoder + .decode() + .then(({ image, complete }) => { + if (controller.cancelled) { + image.close?.(); + return; + } + controller.bitmap?.close?.(); + createImageBitmap(image) + .then((bitmap) => { + controller.bitmap = bitmap; + draw(); + }) + .finally(() => image.close?.()); + + const durationMicros = image.duration || 0; + const delay = durationMicros > 0 ? durationMicros / 1000 : 100; + const hasMore = !complete; + controller.timeout = setTimeout(() => { + if (controller.cancelled) { + return; + } + if (hasMore) { + scheduleNextFrame(controller); + } else { + controller.decoder.reset(); + scheduleNextFrame(controller); + } + }, delay); + }) + .catch(() => { + // If decoding fails, clear animated cache so static fallback is used next render + animatedCache.delete(controller.id); + animationFailures.set(controller.id, Date.now()); + }); +} + +function applyMediaSettings(element, asset) { + if (!isVideoElement(element)) { + return; + } + startVideoPlayback(element, asset); +} + +function startVideoPlayback(element, asset) { + const playbackState = getVideoPlaybackState(element); + const nextSpeed = asset.speed ?? 1; + const effectiveSpeed = Math.max(nextSpeed, 0.01); + if (element.playbackRate !== effectiveSpeed) { + element.playbackRate = effectiveSpeed; + } + const volume = applyMediaVolume(element, asset); + const shouldUnmute = volume > 0; + element.muted = true; + + if (effectiveSpeed === 0) { + element.pause(); + playbackState.playRequested = false; + playbackState.unmuteOnPlay = false; + return; + } + + element.play(); + + if (shouldUnmute) { + if (!element.paused && element.readyState >= 2) { + element.muted = false; + } else { + playbackState.unmuteOnPlay = true; + } + } + + if (element.paused || element.ended) { + if (!playbackState.playRequested) { + playbackState.playRequested = true; + const playPromise = element.play(); + if (playPromise?.catch) { + playPromise.catch(() => { + playbackState.playRequested = false; + }); + } + } + } +} + +function startRenderLoop() { + if (renderIntervalId) { + return; + } + renderIntervalId = setInterval(() => { + draw(); + }, MIN_FRAME_TIME); +} + +globalThis.addEventListener("resize", () => { + resizeCanvas(); +}); + +fetchCanvasSettings().finally(() => { + resizeCanvas(); + startRenderLoop(); + wsconnect(broadcaster, { + onFetch: renderAssets, + onWsEvent: onWebsocketEvent, + }); +}); diff --git a/src/main/resources/static/js/broadcast/ws.js b/src/main/resources/static/js/broadcast/ws.js new file mode 100644 index 0000000..fd1584a --- /dev/null +++ b/src/main/resources/static/js/broadcast/ws.js @@ -0,0 +1,20 @@ +export function wsconnect(broadcaster, { onFetch, onWsEvent }) { + const socket = new SockJS("/ws"); + const stompClient = Stomp.over(socket); + stompClient.connect({}, () => { + stompClient.subscribe(`/topic/channel/${broadcaster}`, (payload) => onWsEvent(JSON.parse(payload.body))); + fetch(`/api/channels/${broadcaster}/assets`) + .then((r) => { + if (!r.ok) { + throw new Error("Failed to load assets"); + } + return r.json(); + }) + .then(onFetch) + .catch((error) => { + console.error(error); + showToast("Unable to load overlay assets. Retrying may help.", "error") + }); + }); +} +