From 519ebbaaff8bc773d5b9678b506be94fcd753b3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Kr=C3=BChlmann?= Date: Wed, 10 Dec 2025 16:30:42 +0100 Subject: [PATCH] Improve media packets --- .../com/imgfloat/app/model/AssetEvent.java | 29 +++-- .../com/imgfloat/app/model/AssetPatch.java | 63 +++++++++++ .../app/service/ChannelDirectoryService.java | 7 +- src/main/resources/static/js/admin.js | 30 ++++- src/main/resources/static/js/broadcast.js | 104 +++++++++++++++--- 5 files changed, 198 insertions(+), 35 deletions(-) create mode 100644 src/main/java/com/imgfloat/app/model/AssetPatch.java diff --git a/src/main/java/com/imgfloat/app/model/AssetEvent.java b/src/main/java/com/imgfloat/app/model/AssetEvent.java index c96436f..71b1134 100644 --- a/src/main/java/com/imgfloat/app/model/AssetEvent.java +++ b/src/main/java/com/imgfloat/app/model/AssetEvent.java @@ -14,6 +14,7 @@ public class AssetEvent { private AssetView payload; private String assetId; private Boolean play; + private AssetPatch patch; public static AssetEvent created(String channel, AssetView asset) { AssetEvent event = new AssetEvent(); @@ -24,6 +25,15 @@ public class AssetEvent { return event; } + public static AssetEvent updated(String channel, AssetPatch patch) { + AssetEvent event = new AssetEvent(); + event.type = Type.UPDATED; + event.channel = channel; + event.assetId = patch.id(); + event.patch = patch; + return event; + } + public static AssetEvent play(String channel, AssetView asset, boolean play) { AssetEvent event = new AssetEvent(); event.type = Type.PLAY; @@ -34,21 +44,12 @@ public class AssetEvent { return event; } - public static AssetEvent updated(String channel, AssetView asset) { - AssetEvent event = new AssetEvent(); - event.type = Type.UPDATED; - event.channel = channel; - event.payload = asset; - event.assetId = asset.id(); - return event; - } - - public static AssetEvent visibility(String channel, AssetView asset) { + public static AssetEvent visibility(String channel, AssetPatch patch) { AssetEvent event = new AssetEvent(); event.type = Type.VISIBILITY; event.channel = channel; - event.payload = asset; - event.assetId = asset.id(); + event.patch = patch; + event.assetId = patch.id(); return event; } @@ -79,4 +80,8 @@ public class AssetEvent { public Boolean getPlay() { return play; } + + public AssetPatch getPatch() { + return patch; + } } diff --git a/src/main/java/com/imgfloat/app/model/AssetPatch.java b/src/main/java/com/imgfloat/app/model/AssetPatch.java new file mode 100644 index 0000000..f08bb10 --- /dev/null +++ b/src/main/java/com/imgfloat/app/model/AssetPatch.java @@ -0,0 +1,63 @@ +package com.imgfloat.app.model; + +/** + * Represents a partial update for an {@link Asset}. Only the fields that changed + * for a given operation are populated to reduce payload sizes sent over WebSocket. + */ +public record AssetPatch( + String id, + Double x, + Double y, + Double width, + Double height, + Double rotation, + Double speed, + Boolean muted, + Integer zIndex, + Boolean hidden, + Boolean audioLoop, + Integer audioDelayMillis, + Double audioSpeed, + Double audioPitch, + Double audioVolume +) { + public static AssetPatch fromTransform(Asset asset) { + return new AssetPatch( + asset.getId(), + asset.getX(), + asset.getY(), + asset.getWidth(), + asset.getHeight(), + asset.getRotation(), + asset.getSpeed(), + asset.isMuted(), + asset.getZIndex(), + null, + asset.isAudioLoop(), + asset.getAudioDelayMillis(), + asset.getAudioSpeed(), + asset.getAudioPitch(), + asset.getAudioVolume() + ); + } + + public static AssetPatch fromVisibility(Asset asset) { + return new AssetPatch( + asset.getId(), + null, + null, + null, + null, + null, + null, + null, + null, + asset.isHidden(), + null, + null, + null, + null, + null + ); + } +} diff --git a/src/main/java/com/imgfloat/app/service/ChannelDirectoryService.java b/src/main/java/com/imgfloat/app/service/ChannelDirectoryService.java index baedc2f..270bf1e 100644 --- a/src/main/java/com/imgfloat/app/service/ChannelDirectoryService.java +++ b/src/main/java/com/imgfloat/app/service/ChannelDirectoryService.java @@ -2,6 +2,7 @@ package com.imgfloat.app.service; import com.imgfloat.app.model.Asset; import com.imgfloat.app.model.AssetEvent; +import com.imgfloat.app.model.AssetPatch; import com.imgfloat.app.model.Channel; import com.imgfloat.app.model.AssetView; import com.imgfloat.app.model.CanvasSettingsRequest; @@ -191,7 +192,8 @@ public class ChannelDirectoryService { } assetRepository.save(asset); AssetView view = AssetView.from(normalized, asset); - messagingTemplate.convertAndSend(topicFor(broadcaster), AssetEvent.updated(broadcaster, view)); + AssetPatch patch = AssetPatch.fromTransform(asset); + messagingTemplate.convertAndSend(topicFor(broadcaster), AssetEvent.updated(broadcaster, patch)); return view; }); } @@ -216,7 +218,8 @@ public class ChannelDirectoryService { asset.setHidden(request.isHidden()); assetRepository.save(asset); AssetView view = AssetView.from(normalized, asset); - messagingTemplate.convertAndSend(topicFor(broadcaster), AssetEvent.visibility(broadcaster, view)); + AssetPatch patch = AssetPatch.fromVisibility(asset); + messagingTemplate.convertAndSend(topicFor(broadcaster), AssetEvent.visibility(broadcaster, patch)); return view; }); } diff --git a/src/main/resources/static/js/admin.js b/src/main/resources/static/js/admin.js index aec7e93..c290699 100644 --- a/src/main/resources/static/js/admin.js +++ b/src/main/resources/static/js/admin.js @@ -314,15 +314,18 @@ function updateRenderState(asset) { } function handleEvent(event) { + const assetId = event.assetId || event?.patch?.id || event?.payload?.id; if (event.type === 'DELETED') { - assets.delete(event.assetId); + assets.delete(assetId); zOrderDirty = true; - clearMedia(event.assetId); - renderStates.delete(event.assetId); - loopPlaybackState.delete(event.assetId); - if (selectedAssetId === event.assetId) { + clearMedia(assetId); + renderStates.delete(assetId); + loopPlaybackState.delete(assetId); + if (selectedAssetId === assetId) { selectedAssetId = null; } + } else if (event.patch) { + applyPatch(assetId, event.patch); } else if (event.payload) { storeAsset(event.payload); if (!event.payload.hidden) { @@ -338,6 +341,23 @@ function handleEvent(event) { drawAndList(); } +function applyPatch(assetId, patch) { + if (!assetId || !patch) { + return; + } + const existing = assets.get(assetId); + if (!existing) { + return; + } + const merged = { ...existing, ...patch }; + if (patch.hidden) { + clearMedia(assetId); + loopPlaybackState.delete(assetId); + } + storeAsset(merged); + updateRenderState(merged); +} + function drawAndList() { requestDraw(); renderAssetList(); diff --git a/src/main/resources/static/js/broadcast.js b/src/main/resources/static/js/broadcast.js index 2998b02..a926da1 100644 --- a/src/main/resources/static/js/broadcast.js +++ b/src/main/resources/static/js/broadcast.js @@ -96,18 +96,21 @@ function resizeCanvas() { } function handleEvent(event) { + const assetId = event.assetId || event?.patch?.id || event?.payload?.id; if (event.type === 'DELETED') { - assets.delete(event.assetId); - clearMedia(event.assetId); - renderStates.delete(event.assetId); + assets.delete(assetId); + clearMedia(assetId); + renderStates.delete(assetId); + } else if (event.patch) { + applyPatch(assetId, event.patch); } else if (event.type === 'PLAY' && event.payload) { - const payload = { ...event.payload, zIndex: Math.max(1, event.payload.zIndex ?? 1) }; + const payload = normalizePayload(event.payload); assets.set(payload.id, payload); if (isAudioAsset(payload)) { handleAudioPlay(payload, event.play !== false); } } else if (event.payload && !event.payload.hidden) { - const payload = { ...event.payload, zIndex: Math.max(1, event.payload.zIndex ?? 1) }; + const payload = normalizePayload(event.payload); assets.set(payload.id, payload); ensureMedia(payload); if (isAudioAsset(payload)) { @@ -122,6 +125,40 @@ function handleEvent(event) { draw(); } +function normalizePayload(payload) { + return { ...payload, zIndex: Math.max(1, payload.zIndex ?? 1) }; +} + +function applyPatch(assetId, patch) { + if (!assetId || !patch) { + return; + } + const existing = assets.get(assetId); + if (!existing) { + return; + } + const merged = normalizePayload({ ...existing, ...patch }); + if (patch.hidden) { + assets.delete(assetId); + clearMedia(assetId); + renderStates.delete(assetId); + return; + } + assets.set(assetId, merged); + ensureMedia(merged); + renderStates.set(assetId, { ...renderStates.get(assetId), ...pickTransform(merged) }); +} + +function pickTransform(asset) { + return { + x: asset.x, + y: asset.y, + width: asset.width, + height: asset.height, + rotation: asset.rotation + }; +} + function draw() { if (frameScheduled) { pendingDraw = true; @@ -286,6 +323,10 @@ function clearMedia(assetId) { animatedCache.delete(assetId); } animationFailures.delete(assetId); + const cachedBlob = blobCache.get(assetId); + if (cachedBlob?.objectUrl) { + URL.revokeObjectURL(cachedBlob.objectUrl); + } blobCache.delete(assetId); const audio = audioControllers.get(assetId); if (audio) { @@ -441,10 +482,11 @@ function autoStartAudio(asset) { function ensureMedia(asset) { const cached = mediaCache.get(asset.id); - if (cached && cached.src !== asset.url) { + const cachedSource = getCachedSource(cached); + if (cached && cachedSource !== asset.url) { clearMedia(asset.id); } - if (cached && cached.src === asset.url) { + if (cached && cachedSource === asset.url) { applyMediaSettings(cached, asset); return cached; } @@ -464,6 +506,7 @@ function ensureMedia(asset) { } const element = isVideoAsset(asset) ? document.createElement('video') : new Image(); + element.dataset.sourceUrl = asset.url; element.crossOrigin = 'anonymous'; if (isVideoElement(element)) { element.loop = true; @@ -472,14 +515,9 @@ function ensureMedia(asset) { element.autoplay = true; element.onloadeddata = draw; element.onloadedmetadata = () => recordDuration(asset.id, element.duration); - element.src = asset.url; - const playback = asset.speed ?? 1; - element.playbackRate = Math.max(playback, 0.01); - if (playback === 0) { - element.pause(); - } else { - element.play().catch(() => {}); - } + element.preload = 'auto'; + element.addEventListener('error', () => clearMedia(asset.id)); + setVideoSource(element, asset); } else { element.onload = draw; element.src = asset.url; @@ -547,13 +585,47 @@ function fetchAssetBlob(asset) { const pending = fetch(asset.url) .then((r) => r.blob()) .then((blob) => { - blobCache.set(asset.id, { url: asset.url, blob }); + const previous = blobCache.get(asset.id); + const existingUrl = previous?.url === asset.url ? previous.objectUrl : null; + const objectUrl = existingUrl || URL.createObjectURL(blob); + blobCache.set(asset.id, { url: asset.url, blob, objectUrl }); return blob; }); blobCache.set(asset.id, { url: asset.url, pending }); return pending; } +function setVideoSource(element, asset) { + const cached = blobCache.get(asset.id); + if (cached?.url === asset.url && cached.objectUrl) { + applyVideoSource(element, cached.objectUrl, asset); + return; + } + + fetchAssetBlob(asset).then(() => { + const next = blobCache.get(asset.id); + if (next?.url !== asset.url || !next.objectUrl) { + return; + } + applyVideoSource(element, next.objectUrl, asset); + }).catch(() => {}); +} + +function applyVideoSource(element, objectUrl, asset) { + element.src = objectUrl; + const playback = asset.speed ?? 1; + element.playbackRate = Math.max(playback, 0.01); + if (playback === 0) { + element.pause(); + } else { + element.play().catch(() => {}); + } +} + +function getCachedSource(element) { + return element?.dataset?.sourceUrl || element?.src; +} + function scheduleNextFrame(controller) { if (controller.cancelled || !controller.decoder) { return;