server/src/main/resources/static/js/broadcast/renderer.js

import { AssetKind, MIN_FRAME_TIME, VISIBILITY_THRESHOLD } from "./constants.js";
import { createBroadcastState } from "./state.js";
import { getAssetKind, isCodeAsset, isVisualAsset, isVideoElement } from "./assetKinds.js";
import { ensureLayerPosition, getLayerOrder, getRenderOrder } from "./layers.js";
import { getVisibilityState, smoothState } from "./visibility.js";
import { createAudioManager } from "./audioManager.js";
import { createMediaManager } from "./mediaManager.js";

export class BroadcastRenderer {
    constructor({ canvas, scriptCanvas, broadcaster, showToast }) {
        this.canvas = canvas;
        this.ctx = canvas.getContext("2d");
        this.scriptCanvas = scriptCanvas;
        this.broadcaster = broadcaster;
        this.showToast = showToast;
        this.state = createBroadcastState();
        this.lastRenderTime = 0;
        this.frameScheduled = false;
        this.pendingDraw = false;
        this.renderIntervalId = null;
        this.scriptWorker = null;
        this.scriptWorkerReady = false;

        this.obsBrowser = !!globalThis.obsstudio;
        this.supportsAnimatedDecode =
            typeof ImageDecoder !== "undefined" && typeof createImageBitmap === "function" && !this.obsBrowser;
        this.canPlayProbe = document.createElement("video");

        this.audioManager = createAudioManager({ assets: this.state.assets });
        this.mediaManager = createMediaManager({
            state: this.state,
            audioManager: this.audioManager,
            draw: () => this.draw(),
            obsBrowser: this.obsBrowser,
            supportsAnimatedDecode: this.supportsAnimatedDecode,
            canPlayProbe: this.canPlayProbe,
        });

        this.applyCanvasSettings(this.state.canvasSettings);
        globalThis.addEventListener("resize", () => {
            this.resizeCanvas();
        });
    }

    start() {
        this.fetchCanvasSettings().finally(() => {
            this.resizeCanvas();
            this.startRenderLoop();
            this.connect();
        });
    }

    connect() {
        const socket = new SockJS("/ws");
        const stompClient = Stomp.over(socket);
        stompClient.connect({}, () => {
            stompClient.subscribe(`/topic/channel/${this.broadcaster}`, (payload) => {
                const body = JSON.parse(payload.body);
                this.handleEvent(body);
            });
            fetch(`/api/channels/${this.broadcaster}/assets`)
                .then((r) => {
                    if (!r.ok) {
                        throw new Error("Failed to load assets");
                    }
                    return r.json();
                })
                .then((assets) => this.renderAssets(assets))
                .catch(() => this.showToast("Unable to load overlay assets. Retrying may help.", "error"));
        });
    }

    renderAssets(list) {
        this.state.layerOrder = [];
        list.forEach((asset) => {
            this.storeAsset(asset, "append");
            if (isCodeAsset(asset)) {
                this.spawnUserJavaScriptWorker(asset);
            }
        });
        this.draw();
    }

    storeAsset(asset, placement = "keep") {
        if (!asset) return;
        console.info(`Storing asset: ${asset.id}`);
        const wasExisting = this.state.assets.has(asset.id);
        this.state.assets.set(asset.id, asset);
        ensureLayerPosition(this.state, asset.id, placement);
        if (!wasExisting && !this.state.visibilityStates.has(asset.id)) {
            const initialAlpha = 0; // Fade in newly discovered assets
            this.state.visibilityStates.set(asset.id, {
                alpha: initialAlpha,
                targetHidden: !!asset.hidden,
            });
        }
    }

    removeAsset(assetId) {
        this.state.assets.delete(assetId);
        this.state.layerOrder = this.state.layerOrder.filter((id) => id !== assetId);
        this.mediaManager.clearMedia(assetId);
        this.stopUserJavaScriptWorker(assetId);
        this.state.renderStates.delete(assetId);
        this.state.visibilityStates.delete(assetId);
    }

    async fetchCanvasSettings() {
        return fetch(`/api/channels/${encodeURIComponent(this.broadcaster)}/canvas`)
            .then((r) => {
                if (!r.ok) {
                    throw new Error("Failed to load canvas");
                }
                return r.json();
            })
            .then((settings) => {
                this.applyCanvasSettings(settings);
            })
            .catch(() => {
                this.resizeCanvas();
                this.showToast("Using default canvas size. Unable to load saved settings.", "warning");
            });
    }

    applyCanvasSettings(settings) {
        if (!settings) {
            return;
        }
        const width = Number.isFinite(settings.width) ? settings.width : this.state.canvasSettings.width;
        const height = Number.isFinite(settings.height) ? settings.height : this.state.canvasSettings.height;
        this.state.canvasSettings = { width, height };
        this.resizeCanvas();
    }

    resizeCanvas() {
        if (Number.isFinite(this.state.canvasSettings.width) && Number.isFinite(this.state.canvasSettings.height)) {
            this.canvas.width = this.state.canvasSettings.width;
            this.canvas.height = this.state.canvasSettings.height;
            this.canvas.style.width = `${this.state.canvasSettings.width}px`;
            this.canvas.style.height = `${this.state.canvasSettings.height}px`;
            if (this.scriptCanvas) {
                this.scriptCanvas.width = this.state.canvasSettings.width;
                this.scriptCanvas.height = this.state.canvasSettings.height;
                this.scriptCanvas.style.width = `${this.state.canvasSettings.width}px`;
                this.scriptCanvas.style.height = `${this.state.canvasSettings.height}px`;
            }
        }
        this.updateScriptWorkerCanvas();
        this.draw();
    }

    handleEvent(event) {
        if (event.type === "CANVAS" && event.payload) {
            this.applyCanvasSettings(event.payload);
            return;
        }
        const assetId = event.assetId || event?.patch?.id || event?.payload?.id;
        if (event.type === "VISIBILITY") {
            this.handleVisibilityEvent(event);
            return;
        }
        if (event.type === "DELETED") {
            this.removeAsset(assetId);
        } else if (event.patch) {
            this.applyPatch(assetId, event.patch);
            if (event.payload) {
                const payload = this.normalizePayload(event.payload);
                if (payload.hidden) {
                    this.hideAssetWithTransition(payload);
                } else if (!this.state.assets.has(payload.id)) {
                    this.upsertVisibleAsset(payload, "append");
                }
            }
        } else if (event.type === "PLAY" && event.payload) {
            const payload = this.normalizePayload(event.payload);
            this.storeAsset(payload);
            if (getAssetKind(payload) === AssetKind.AUDIO) {
                this.audioManager.handleAudioPlay(payload, event.play !== false);
            }
        } else if (event.payload && !event.payload.hidden) {
            const payload = this.normalizePayload(event.payload);
            this.upsertVisibleAsset(payload);
        } else if (event.payload && event.payload.hidden) {
            this.hideAssetWithTransition(event.payload);
        }
        this.draw();
    }

    normalizePayload(payload) {
        return { ...payload };
    }

    hideAssetWithTransition(asset) {
        const payload = asset ? this.normalizePayload(asset) : null;
        if (!payload?.id) {
            return;
        }
        const existing = this.state.assets.get(payload.id);
        if (
            !existing &&
            (!Number.isFinite(payload.x) ||
                !Number.isFinite(payload.y) ||
                !Number.isFinite(payload.width) ||
                !Number.isFinite(payload.height))
        ) {
            return;
        }
        const merged = this.normalizePayload({ ...(existing || {}), ...payload, hidden: true });
        this.storeAsset(merged);
        this.stopUserJavaScriptWorker(merged.id);
        this.audioManager.stopAudio(payload.id);
    }

    upsertVisibleAsset(asset, placement = "keep") {
        const payload = asset ? this.normalizePayload(asset) : null;
        if (!payload?.id) {
            return;
        }
        const placementMode = this.state.assets.has(payload.id) ? "keep" : placement;
        this.storeAsset(payload, placementMode);
        this.mediaManager.ensureMedia(payload);
        const kind = getAssetKind(payload);
        if (kind === AssetKind.AUDIO) {
            this.audioManager.playAudioImmediately(payload);
        } else if (kind === AssetKind.CODE) {
            this.spawnUserJavaScriptWorker(payload);
        }
    }

    handleVisibilityEvent(event) {
        const payload = event.payload ? this.normalizePayload(event.payload) : null;
        const patch = event.patch;
        const id = payload?.id || patch?.id || event.assetId;

        if (payload?.hidden || patch?.hidden) {
            this.hideAssetWithTransition({ id, ...payload, ...patch });
            this.draw();
            return;
        }

        if (payload) {
            const placement = this.state.assets.has(payload.id) ? "keep" : "append";
            this.upsertVisibleAsset(payload, placement);
        }

        if (patch && id) {
            this.applyPatch(id, patch);
        }

        this.draw();
    }

    applyPatch(assetId, patch) {
        if (!assetId || !patch) {
            return;
        }
        const sanitizedPatch = Object.fromEntries(
            Object.entries(patch).filter(([, value]) => value !== null && value !== undefined),
        );
        const existing = this.state.assets.get(assetId);
        if (!existing) {
            return;
        }
        const merged = this.normalizePayload({ ...existing, ...sanitizedPatch });
        console.log(merged);
        const isVisual = isVisualAsset(merged);
        if (sanitizedPatch.hidden) {
            this.hideAssetWithTransition(merged);
            return;
        }
        const targetLayer = Number.isFinite(patch.layer)
            ? patch.layer
            : Number.isFinite(patch.zIndex)
              ? patch.zIndex
              : null;
        if (isVisual && Number.isFinite(targetLayer)) {
            const currentOrder = getLayerOrder(this.state).filter((id) => id !== assetId);
            const insertIndex = Math.max(0, currentOrder.length - Math.round(targetLayer));
            currentOrder.splice(insertIndex, 0, assetId);
            this.state.layerOrder = currentOrder;
        }
        this.storeAsset(merged);
        this.mediaManager.ensureMedia(merged);
        if (isCodeAsset(merged)) {
            console.info(`Spawning JS worker for patched asset: ${merged.id}`);
            this.spawnUserJavaScriptWorker(merged);
        }
    }

    draw() {
        if (this.frameScheduled) {
            this.pendingDraw = true;
            return;
        }
        this.frameScheduled = true;
        requestAnimationFrame((timestamp) => {
            const elapsed = timestamp - this.lastRenderTime;
            const delay = MIN_FRAME_TIME - elapsed;
            const shouldRender = elapsed >= MIN_FRAME_TIME;

            if (shouldRender) {
                this.lastRenderTime = timestamp;
                this.renderFrame();
            }

            this.frameScheduled = false;
            if (this.pendingDraw || !shouldRender) {
                this.pendingDraw = false;
                setTimeout(() => this.draw(), Math.max(0, delay));
            }
        });
    }

    renderFrame() {
        this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);
        getRenderOrder(this.state).forEach((asset) => this.drawAsset(asset));
    }

    drawAsset(asset) {
        const visibility = getVisibilityState(this.state, asset);
        if (visibility.alpha <= VISIBILITY_THRESHOLD && asset.hidden) {
            return;
        }
        const renderState = smoothState(this.state, asset);
        const halfWidth = renderState.width / 2;
        const halfHeight = renderState.height / 2;
        this.ctx.save();
        this.ctx.globalAlpha = Math.max(0, Math.min(1, visibility.alpha));
        this.ctx.translate(renderState.x + halfWidth, renderState.y + halfHeight);
        this.ctx.rotate((renderState.rotation * Math.PI) / 180);

        const kind = getAssetKind(asset);
        if (kind === AssetKind.CODE) {
            this.ctx.restore();
            return;
        }

        if (kind === AssetKind.AUDIO) {
            if (!asset.hidden) {
                this.audioManager.autoStartAudio(asset);
            }
            this.ctx.restore();
            return;
        }

        const media = this.mediaManager.ensureMedia(asset);
        const drawSource = media?.isAnimated ? media.bitmap : media;
        const ready = this.isDrawable(media);
        if (ready && drawSource) {
            this.ctx.drawImage(drawSource, -halfWidth, -halfHeight, renderState.width, renderState.height);
        }

        this.ctx.restore();
    }

    isDrawable(element) {
        if (!element) {
            return false;
        }
        if (element.isAnimated) {
            return !!element.bitmap;
        }
        if (isVideoElement(element)) {
            return element.readyState >= 2;
        }
        if (typeof ImageBitmap !== "undefined" && element instanceof ImageBitmap) {
            return true;
        }
        return !!element.complete;
    }

    startRenderLoop() {
        if (this.renderIntervalId) {
            return;
        }
        this.renderIntervalId = setInterval(() => {
            this.draw();
        }, MIN_FRAME_TIME);
    }

    ensureScriptWorker() {
        if (this.scriptWorker || !this.scriptCanvas) {
            return;
        }
        if (typeof this.scriptCanvas.transferControlToOffscreen !== "function") {
            console.warn("OffscreenCanvas is not supported in this environment.");
            return;
        }
        const offscreen = this.scriptCanvas.transferControlToOffscreen();
        this.scriptWorker = new Worker("/js/broadcast/script-worker.js");
        this.scriptWorker.postMessage(
            {
                type: "init",
                payload: {
                    canvas: offscreen,
                    width: this.scriptCanvas.width,
                    height: this.scriptCanvas.height,
                    channelName: this.broadcaster,
                },
            },
            [offscreen],
        );
        this.scriptWorkerReady = true;
    }

    updateScriptWorkerCanvas() {
        if (!this.scriptWorker || !this.scriptWorkerReady || !this.scriptCanvas) {
            return;
        }
        this.scriptWorker.postMessage({
            type: "resize",
            payload: {
                width: this.scriptCanvas.width,
                height: this.scriptCanvas.height,
            },
        });
    }

    async spawnUserJavaScriptWorker(asset) {
        if (!asset?.id || !asset?.url) {
            return;
        }
        this.ensureScriptWorker();
        if (!this.scriptWorkerReady) {
            return;
        }
        let assetSource;
        try {
            const response = await fetch(asset.url);
            if (!response.ok) {
                throw new Error(`Failed to load script asset ${asset.id}`);
            }
            assetSource = await response.text();
        } catch (error) {
            console.error(`Unable to fetch asset ${asset.id} from ${asset.url}`, error);
            return;
        }
        this.scriptWorker.postMessage({
            type: "addScript",
            payload: {
                id: asset.id,
                source: assetSource,
            },
        });
    }

    stopUserJavaScriptWorker(assetId) {
        if (!this.scriptWorker || !assetId) {
            return;
        }
        this.scriptWorker.postMessage({
            type: "removeScript",
            payload: { id: assetId },
        });
    }
}