Add audio

This commit is contained in:
2025-12-10 09:38:38 +01:00
parent 65b8baabdc
commit b178c68434
9 changed files with 614 additions and 13 deletions

View File

@@ -61,6 +61,11 @@ public class SchemaMigration implements ApplicationRunner {
addColumnIfMissing("assets", columns, "speed", "REAL", "1.0"); addColumnIfMissing("assets", columns, "speed", "REAL", "1.0");
addColumnIfMissing("assets", columns, "muted", "BOOLEAN", "0"); addColumnIfMissing("assets", columns, "muted", "BOOLEAN", "0");
addColumnIfMissing("assets", columns, "media_type", "TEXT", "'application/octet-stream'"); addColumnIfMissing("assets", columns, "media_type", "TEXT", "'application/octet-stream'");
addColumnIfMissing("assets", columns, "audio_loop", "BOOLEAN", "0");
addColumnIfMissing("assets", columns, "audio_delay_millis", "INTEGER", "0");
addColumnIfMissing("assets", columns, "audio_speed", "REAL", "1.0");
addColumnIfMissing("assets", columns, "audio_pitch", "REAL", "1.0");
addColumnIfMissing("assets", columns, "audio_volume", "REAL", "1.0");
} }
private void addColumnIfMissing(String tableName, List<String> existingColumns, String columnName, String dataType, String defaultValue) { private void addColumnIfMissing(String tableName, List<String> existingColumns, String columnName, String dataType, String defaultValue) {

View File

@@ -35,6 +35,11 @@ public class Asset {
private String mediaType; private String mediaType;
private String originalMediaType; private String originalMediaType;
private Integer zIndex; private Integer zIndex;
private Boolean audioLoop;
private Integer audioDelayMillis;
private Double audioSpeed;
private Double audioPitch;
private Double audioVolume;
private boolean hidden; private boolean hidden;
private Instant createdAt; private Instant createdAt;
@@ -80,6 +85,21 @@ public class Asset {
if (this.zIndex == null || this.zIndex < 1) { if (this.zIndex == null || this.zIndex < 1) {
this.zIndex = 1; this.zIndex = 1;
} }
if (this.audioLoop == null) {
this.audioLoop = Boolean.FALSE;
}
if (this.audioDelayMillis == null) {
this.audioDelayMillis = 0;
}
if (this.audioSpeed == null) {
this.audioSpeed = 1.0;
}
if (this.audioPitch == null) {
this.audioPitch = 1.0;
}
if (this.audioVolume == null) {
this.audioVolume = 1.0;
}
} }
public String getId() { public String getId() {
@@ -210,6 +230,46 @@ public class Asset {
this.zIndex = zIndex == null ? null : Math.max(1, zIndex); this.zIndex = zIndex == null ? null : Math.max(1, zIndex);
} }
public boolean isAudioLoop() {
return audioLoop != null && audioLoop;
}
public void setAudioLoop(Boolean audioLoop) {
this.audioLoop = audioLoop;
}
public Integer getAudioDelayMillis() {
return audioDelayMillis == null ? 0 : Math.max(0, audioDelayMillis);
}
public void setAudioDelayMillis(Integer audioDelayMillis) {
this.audioDelayMillis = audioDelayMillis;
}
public double getAudioSpeed() {
return audioSpeed == null ? 1.0 : Math.max(0.1, audioSpeed);
}
public void setAudioSpeed(Double audioSpeed) {
this.audioSpeed = audioSpeed;
}
public double getAudioPitch() {
return audioPitch == null ? 1.0 : Math.max(0.5, audioPitch);
}
public void setAudioPitch(Double audioPitch) {
this.audioPitch = audioPitch;
}
public double getAudioVolume() {
return audioVolume == null ? 1.0 : Math.max(0.0, Math.min(1.0, audioVolume));
}
public void setAudioVolume(Double audioVolume) {
this.audioVolume = audioVolume;
}
private static String normalize(String value) { private static String normalize(String value) {
return value == null ? null : value.toLowerCase(Locale.ROOT); return value == null ? null : value.toLowerCase(Locale.ROOT);
} }

View File

@@ -17,6 +17,11 @@ public record AssetView(
String mediaType, String mediaType,
String originalMediaType, String originalMediaType,
Integer zIndex, Integer zIndex,
Boolean audioLoop,
Integer audioDelayMillis,
Double audioSpeed,
Double audioPitch,
Double audioVolume,
boolean hidden, boolean hidden,
Instant createdAt Instant createdAt
) { ) {
@@ -36,6 +41,11 @@ public record AssetView(
asset.getMediaType(), asset.getMediaType(),
asset.getOriginalMediaType(), asset.getOriginalMediaType(),
asset.getZIndex(), asset.getZIndex(),
asset.isAudioLoop(),
asset.getAudioDelayMillis(),
asset.getAudioSpeed(),
asset.getAudioPitch(),
asset.getAudioVolume(),
asset.isHidden(), asset.isHidden(),
asset.getCreatedAt() asset.getCreatedAt()
); );

View File

@@ -9,6 +9,11 @@ public class TransformRequest {
private Double speed; private Double speed;
private Boolean muted; private Boolean muted;
private Integer zIndex; private Integer zIndex;
private Boolean audioLoop;
private Integer audioDelayMillis;
private Double audioSpeed;
private Double audioPitch;
private Double audioVolume;
public double getX() { public double getX() {
return x; return x;
@@ -73,4 +78,44 @@ public class TransformRequest {
public void setZIndex(Integer zIndex) { public void setZIndex(Integer zIndex) {
this.zIndex = zIndex; this.zIndex = zIndex;
} }
public Boolean getAudioLoop() {
return audioLoop;
}
public void setAudioLoop(Boolean audioLoop) {
this.audioLoop = audioLoop;
}
public Integer getAudioDelayMillis() {
return audioDelayMillis;
}
public void setAudioDelayMillis(Integer audioDelayMillis) {
this.audioDelayMillis = audioDelayMillis;
}
public Double getAudioSpeed() {
return audioSpeed;
}
public void setAudioSpeed(Double audioSpeed) {
this.audioSpeed = audioSpeed;
}
public Double getAudioPitch() {
return audioPitch;
}
public void setAudioPitch(Double audioPitch) {
this.audioPitch = audioPitch;
}
public Double getAudioVolume() {
return audioVolume;
}
public void setAudioVolume(Double audioVolume) {
this.audioVolume = audioVolume;
}
} }

View File

@@ -125,13 +125,18 @@ public class ChannelDirectoryService {
.orElse("Asset " + System.currentTimeMillis()); .orElse("Asset " + System.currentTimeMillis());
String dataUrl = "data:" + optimized.mediaType() + ";base64," + Base64.getEncoder().encodeToString(optimized.bytes()); String dataUrl = "data:" + optimized.mediaType() + ";base64," + Base64.getEncoder().encodeToString(optimized.bytes());
double width = optimized.width() > 0 ? optimized.width() : 640; double width = optimized.width() > 0 ? optimized.width() : (optimized.mediaType().startsWith("audio/") ? 400 : 640);
double height = optimized.height() > 0 ? optimized.height() : 360; double height = optimized.height() > 0 ? optimized.height() : (optimized.mediaType().startsWith("audio/") ? 80 : 360);
Asset asset = new Asset(channel.getBroadcaster(), name, dataUrl, width, height); Asset asset = new Asset(channel.getBroadcaster(), name, dataUrl, width, height);
asset.setOriginalMediaType(mediaType); asset.setOriginalMediaType(mediaType);
asset.setMediaType(optimized.mediaType()); asset.setMediaType(optimized.mediaType());
asset.setSpeed(1.0); asset.setSpeed(1.0);
asset.setMuted(optimized.mediaType().startsWith("video/")); asset.setMuted(optimized.mediaType().startsWith("video/"));
asset.setAudioLoop(false);
asset.setAudioDelayMillis(0);
asset.setAudioSpeed(1.0);
asset.setAudioPitch(1.0);
asset.setAudioVolume(1.0);
asset.setZIndex(nextZIndex(channel.getBroadcaster())); asset.setZIndex(nextZIndex(channel.getBroadcaster()));
assetRepository.save(asset); assetRepository.save(asset);
@@ -159,6 +164,22 @@ public class ChannelDirectoryService {
if (request.getMuted() != null && asset.isVideo()) { if (request.getMuted() != null && asset.isVideo()) {
asset.setMuted(request.getMuted()); asset.setMuted(request.getMuted());
} }
if (request.getAudioLoop() != null) {
asset.setAudioLoop(request.getAudioLoop());
}
if (request.getAudioDelayMillis() != null && request.getAudioDelayMillis() >= 0) {
asset.setAudioDelayMillis(request.getAudioDelayMillis());
}
if (request.getAudioSpeed() != null && request.getAudioSpeed() >= 0) {
asset.setAudioSpeed(request.getAudioSpeed());
}
if (request.getAudioPitch() != null && request.getAudioPitch() > 0) {
asset.setAudioPitch(request.getAudioPitch());
}
if (request.getAudioVolume() != null && request.getAudioVolume() >= 0) {
double clamped = Math.max(0.0, Math.min(1.0, request.getAudioVolume()));
asset.setAudioVolume(clamped);
}
assetRepository.save(asset); assetRepository.save(asset);
AssetView view = AssetView.from(normalized, asset); AssetView view = AssetView.from(normalized, asset);
messagingTemplate.convertAndSend(topicFor(broadcaster), AssetEvent.updated(broadcaster, view)); messagingTemplate.convertAndSend(topicFor(broadcaster), AssetEvent.updated(broadcaster, view));
@@ -289,6 +310,9 @@ public class ChannelDirectoryService {
case "mp4" -> "video/mp4"; case "mp4" -> "video/mp4";
case "webm" -> "video/webm"; case "webm" -> "video/webm";
case "mov" -> "video/quicktime"; case "mov" -> "video/quicktime";
case "mp3" -> "audio/mpeg";
case "wav" -> "audio/wav";
case "ogg" -> "audio/ogg";
default -> "application/octet-stream"; default -> "application/octet-stream";
}) })
.orElse("application/octet-stream"); .orElse("application/octet-stream");
@@ -324,6 +348,10 @@ public class ChannelDirectoryService {
return new OptimizedAsset(bytes, mediaType, dimensions.width(), dimensions.height()); return new OptimizedAsset(bytes, mediaType, dimensions.width(), dimensions.height());
} }
if (mediaType.startsWith("audio/")) {
return new OptimizedAsset(bytes, mediaType, 0, 0);
}
BufferedImage image = ImageIO.read(new ByteArrayInputStream(bytes)); BufferedImage image = ImageIO.read(new ByteArrayInputStream(bytes));
if (image != null) { if (image != null) {
return new OptimizedAsset(bytes, mediaType, image.getWidth(), image.getHeight()); return new OptimizedAsset(bytes, mediaType, image.getWidth(), image.getHeight());

View File

@@ -476,7 +476,6 @@ body {
background: radial-gradient(circle at 15% 20%, rgba(124, 58, 237, 0.08), transparent 40%), background: radial-gradient(circle at 15% 20%, rgba(124, 58, 237, 0.08), transparent 40%),
radial-gradient(circle at 85% 0%, rgba(59, 130, 246, 0.06), transparent 45%), radial-gradient(circle at 85% 0%, rgba(59, 130, 246, 0.06), transparent 45%),
#0b1220; #0b1220;
border: 1px solid #312e81;
box-shadow: 0 18px 45px rgba(0, 0, 0, 0.4), inset 0 1px 0 rgba(255, 255, 255, 0.03); box-shadow: 0 18px 45px rgba(0, 0, 0, 0.4), inset 0 1px 0 rgba(255, 255, 255, 0.03);
} }

View File

@@ -10,6 +10,7 @@ const assets = new Map();
const mediaCache = new Map(); const mediaCache = new Map();
const renderStates = new Map(); const renderStates = new Map();
const animatedCache = new Map(); const animatedCache = new Map();
const audioControllers = new Map();
let drawPending = false; let drawPending = false;
let zOrderDirty = true; let zOrderDirty = true;
let zOrderCache = []; let zOrderCache = [];
@@ -27,8 +28,15 @@ const speedInput = document.getElementById('asset-speed');
const muteInput = document.getElementById('asset-muted'); const muteInput = document.getElementById('asset-muted');
const selectedZLabel = document.getElementById('asset-z-level'); const selectedZLabel = document.getElementById('asset-z-level');
const playbackSection = document.getElementById('playback-section'); const playbackSection = document.getElementById('playback-section');
const audioSection = document.getElementById('audio-section');
const audioLoopInput = document.getElementById('asset-audio-loop');
const audioDelayInput = document.getElementById('asset-audio-delay');
const audioSpeedInput = document.getElementById('asset-audio-speed');
const audioPitchInput = document.getElementById('asset-audio-pitch');
const audioVolumeInput = document.getElementById('asset-audio-volume');
const controlsPlaceholder = document.getElementById('asset-controls-placeholder'); const controlsPlaceholder = document.getElementById('asset-controls-placeholder');
const fileNameLabel = document.getElementById('asset-file-name'); const fileNameLabel = document.getElementById('asset-file-name');
const audioPlaceholder = 'data:image/svg+xml;utf8,' + encodeURIComponent('<svg xmlns="http://www.w3.org/2000/svg" width="320" height="80"><rect width="100%" height="100%" fill="#1f2937" rx="8"/><g fill="#fbbf24" transform="translate(20 20)"><circle cx="15" cy="20" r="6"/><rect x="28" y="5" width="12" height="30" rx="2"/><rect x="45" y="10" width="140" height="5" fill="#fef3c7"/><rect x="45" y="23" width="110" height="5" fill="#fef3c7"/></g><text x="20" y="70" fill="#e5e7eb" font-family="sans-serif" font-size="14">Audio</text></svg>');
const aspectLockState = new Map(); const aspectLockState = new Map();
const commitSizeChange = debounce(() => applyTransformFromInputs(), 180); const commitSizeChange = debounce(() => applyTransformFromInputs(), 180);
@@ -46,6 +54,11 @@ if (heightInput) heightInput.addEventListener('input', () => handleSizeInputChan
if (heightInput) heightInput.addEventListener('change', () => commitSizeChange()); if (heightInput) heightInput.addEventListener('change', () => commitSizeChange());
if (speedInput) speedInput.addEventListener('input', updatePlaybackFromInputs); if (speedInput) speedInput.addEventListener('input', updatePlaybackFromInputs);
if (muteInput) muteInput.addEventListener('change', updateMuteFromInput); if (muteInput) muteInput.addEventListener('change', updateMuteFromInput);
if (audioLoopInput) audioLoopInput.addEventListener('change', updateAudioSettingsFromInputs);
if (audioDelayInput) audioDelayInput.addEventListener('input', updateAudioSettingsFromInputs);
if (audioSpeedInput) audioSpeedInput.addEventListener('input', updateAudioSettingsFromInputs);
if (audioPitchInput) audioPitchInput.addEventListener('input', updateAudioSettingsFromInputs);
if (audioVolumeInput) audioVolumeInput.addEventListener('input', updateAudioSettingsFromInputs);
function connect() { function connect() {
const socket = new SockJS('/ws'); const socket = new SockJS('/ws');
stompClient = Stomp.over(socket); stompClient = Stomp.over(socket);
@@ -137,7 +150,14 @@ function handleEvent(event) {
} }
} else if (event.payload) { } else if (event.payload) {
storeAsset(event.payload); storeAsset(event.payload);
if (!event.payload.hidden) {
ensureMedia(event.payload); ensureMedia(event.payload);
if (isAudioAsset(event.payload) && event.type === 'VISIBILITY') {
playAudioFromCanvas(event.payload, true);
}
} else {
clearMedia(event.payload.id);
}
} }
drawAndList(); drawAndList();
} }
@@ -194,8 +214,11 @@ function drawAsset(asset) {
const media = ensureMedia(asset); const media = ensureMedia(asset);
const drawSource = media?.isAnimated ? media.bitmap : media; const drawSource = media?.isAnimated ? media.bitmap : media;
const ready = isDrawable(media); const ready = isAudioAsset(asset) || isDrawable(media);
if (ready) { if (isAudioAsset(asset)) {
autoStartAudio(asset);
}
if (ready && drawSource) {
ctx.globalAlpha = asset.hidden ? 0.35 : 0.9; ctx.globalAlpha = asset.hidden ? 0.35 : 0.9;
ctx.drawImage(drawSource, -halfWidth, -halfHeight, renderState.width, renderState.height); ctx.drawImage(drawSource, -halfWidth, -halfHeight, renderState.width, renderState.height);
} else { } else {
@@ -214,6 +237,9 @@ function drawAsset(asset) {
ctx.lineWidth = asset.id === selectedAssetId ? 2 : 1; ctx.lineWidth = asset.id === selectedAssetId ? 2 : 1;
ctx.setLineDash(asset.id === selectedAssetId ? [6, 4] : []); ctx.setLineDash(asset.id === selectedAssetId ? [6, 4] : []);
ctx.strokeRect(-halfWidth, -halfHeight, renderState.width, renderState.height); ctx.strokeRect(-halfWidth, -halfHeight, renderState.width, renderState.height);
if (isAudioAsset(asset)) {
drawAudioIndicators(asset, halfWidth, halfHeight);
}
if (asset.id === selectedAssetId) { if (asset.id === selectedAssetId) {
drawSelectionOverlay(renderState); drawSelectionOverlay(renderState);
} }
@@ -277,6 +303,59 @@ function drawHandle(x, y, isRotation) {
ctx.restore(); ctx.restore();
} }
function drawAudioIndicators(asset, halfWidth, halfHeight) {
const controller = audioControllers.get(asset.id);
const isPlaying = controller && !controller.element.paused && !controller.element.ended;
const hasDelay = !!(controller && controller.delayTimeout);
if (!isPlaying && !hasDelay) {
return;
}
const indicatorSize = 18;
const padding = 10;
let x = -halfWidth + padding + indicatorSize / 2;
const y = -halfHeight + padding + indicatorSize / 2;
ctx.save();
ctx.setLineDash([]);
if (isPlaying) {
ctx.fillStyle = 'rgba(52, 211, 153, 0.9)';
ctx.strokeStyle = '#0f172a';
ctx.lineWidth = 1;
ctx.beginPath();
ctx.arc(x, y, indicatorSize / 2, 0, Math.PI * 2);
ctx.fill();
ctx.stroke();
ctx.fillStyle = '#0f172a';
ctx.beginPath();
const radius = indicatorSize * 0.22;
ctx.moveTo(x - radius, y - radius * 1.1);
ctx.lineTo(x + radius * 1.2, y);
ctx.lineTo(x - radius, y + radius * 1.1);
ctx.closePath();
ctx.fill();
x += indicatorSize + 4;
}
if (hasDelay) {
ctx.fillStyle = 'rgba(251, 191, 36, 0.9)';
ctx.strokeStyle = '#0f172a';
ctx.lineWidth = 1;
ctx.beginPath();
ctx.arc(x, y, indicatorSize / 2, 0, Math.PI * 2);
ctx.fill();
ctx.stroke();
ctx.strokeStyle = '#0f172a';
ctx.beginPath();
ctx.moveTo(x, y);
ctx.lineTo(x, y - indicatorSize * 0.22);
ctx.moveTo(x, y);
ctx.lineTo(x + indicatorSize * 0.22, y);
ctx.stroke();
}
ctx.restore();
}
function getHandlePositions(asset) { function getHandlePositions(asset) {
return [ return [
{ x: 0, y: 0, type: 'nw' }, { x: 0, y: 0, type: 'nw' },
@@ -434,6 +513,11 @@ function isVideoAsset(asset) {
return type.startsWith('video/'); return type.startsWith('video/');
} }
function isAudioAsset(asset) {
const type = asset?.mediaType || asset?.originalMediaType || '';
return type.startsWith('audio/');
}
function isVideoElement(element) { function isVideoElement(element) {
return element && element.tagName === 'VIDEO'; return element && element.tagName === 'VIDEO';
} }
@@ -477,6 +561,112 @@ function clearMedia(assetId) {
animated.decoder?.close?.(); animated.decoder?.close?.();
animatedCache.delete(assetId); animatedCache.delete(assetId);
} }
const audio = audioControllers.get(assetId);
if (audio) {
if (audio.delayTimeout) {
clearTimeout(audio.delayTimeout);
}
audio.element.pause();
audio.element.currentTime = 0;
audioControllers.delete(assetId);
}
}
function ensureAudioController(asset) {
const cached = audioControllers.get(asset.id);
if (cached && cached.src === asset.url) {
applyAudioSettings(cached, asset);
return cached;
}
if (cached) {
clearMedia(asset.id);
}
const element = new Audio(asset.url);
element.controls = true;
element.preload = 'auto';
const controller = {
id: asset.id,
src: asset.url,
element,
delayTimeout: null,
loopEnabled: false,
delayMs: 0,
baseDelayMs: 0
};
element.onended = () => handleAudioEnded(asset.id);
audioControllers.set(asset.id, controller);
applyAudioSettings(controller, asset, true);
return controller;
}
function applyAudioSettings(controller, asset, resetPosition = false) {
controller.loopEnabled = !!asset.audioLoop;
controller.baseDelayMs = Math.max(0, asset.audioDelayMillis || 0);
controller.delayMs = controller.baseDelayMs;
const speed = Math.max(0.25, asset.audioSpeed || 1);
const pitch = Math.max(0.5, asset.audioPitch || 1);
controller.element.playbackRate = speed * pitch;
const volume = Math.max(0, Math.min(1, asset.audioVolume ?? 1));
controller.element.volume = volume;
if (resetPosition) {
controller.element.currentTime = 0;
controller.element.pause();
}
}
function handleAudioEnded(assetId) {
const controller = audioControllers.get(assetId);
if (!controller) return;
controller.element.currentTime = 0;
if (controller.delayTimeout) {
clearTimeout(controller.delayTimeout);
}
if (controller.loopEnabled) {
controller.delayTimeout = setTimeout(() => {
controller.element.play().catch(() => {});
}, controller.delayMs);
} else {
controller.element.pause();
}
}
function stopAudio(assetId) {
const controller = audioControllers.get(assetId);
if (!controller) return;
if (controller.delayTimeout) {
clearTimeout(controller.delayTimeout);
}
controller.element.pause();
controller.element.currentTime = 0;
controller.delayTimeout = null;
controller.delayMs = controller.baseDelayMs;
}
function playAudioFromCanvas(asset, resetDelay = false) {
const controller = ensureAudioController(asset);
if (controller.delayTimeout) {
clearTimeout(controller.delayTimeout);
controller.delayTimeout = null;
}
controller.element.currentTime = 0;
controller.delayMs = resetDelay ? 0 : controller.baseDelayMs;
controller.element.play().catch(() => {});
controller.delayMs = controller.baseDelayMs;
requestDraw();
}
function autoStartAudio(asset) {
if (!isAudioAsset(asset) || asset.hidden) {
return;
}
const controller = ensureAudioController(asset);
if (controller.loopEnabled && controller.element.paused && !controller.delayTimeout) {
controller.delayTimeout = setTimeout(() => {
controller.element.play().catch(() => {});
}, controller.delayMs);
}
} }
function ensureMedia(asset) { function ensureMedia(asset) {
@@ -486,6 +676,14 @@ function ensureMedia(asset) {
return cached; return cached;
} }
if (isAudioAsset(asset)) {
ensureAudioController(asset);
const placeholder = new Image();
placeholder.src = audioPlaceholder;
mediaCache.set(asset.id, placeholder);
return placeholder;
}
if (isGifAsset(asset) && 'ImageDecoder' in window) { if (isGifAsset(asset) && 'ImageDecoder' in window) {
const animated = ensureAnimatedImage(asset); const animated = ensureAnimatedImage(asset);
if (animated) { if (animated) {
@@ -727,6 +925,14 @@ function createBadge(label, extraClass = '') {
} }
function createPreviewElement(asset) { function createPreviewElement(asset) {
if (isAudioAsset(asset)) {
const audio = document.createElement('audio');
audio.className = 'asset-preview audio-preview';
audio.src = asset.url;
audio.controls = true;
audio.preload = 'metadata';
return audio;
}
if (isVideoAsset(asset)) { if (isVideoAsset(asset)) {
const video = document.createElement('video'); const video = document.createElement('video');
video.className = 'asset-preview'; video.className = 'asset-preview';
@@ -782,6 +988,17 @@ function updateSelectedAssetControls(asset = getSelectedAsset()) {
muteInput.disabled = !isVideoAsset(asset); muteInput.disabled = !isVideoAsset(asset);
muteInput.parentElement?.classList.toggle('disabled', !isVideoAsset(asset)); muteInput.parentElement?.classList.toggle('disabled', !isVideoAsset(asset));
} }
if (audioSection) {
const showAudio = isAudioAsset(asset);
audioSection.classList.toggle('hidden', !showAudio);
if (showAudio) {
audioLoopInput.checked = !!asset.audioLoop;
audioDelayInput.value = Math.max(0, asset.audioDelayMillis ?? 0);
audioSpeedInput.value = Math.round(Math.max(0.25, asset.audioSpeed ?? 1) * 100);
audioPitchInput.value = Math.round(Math.max(0.5, asset.audioPitch ?? 1) * 100);
audioVolumeInput.value = Math.round(Math.max(0, Math.min(1, asset.audioVolume ?? 1)) * 100);
}
}
} }
function applyTransformFromInputs() { function applyTransformFromInputs() {
@@ -836,6 +1053,20 @@ function updateMuteFromInput() {
drawAndList(); drawAndList();
} }
function updateAudioSettingsFromInputs() {
const asset = getSelectedAsset();
if (!asset || !isAudioAsset(asset)) return;
asset.audioLoop = !!audioLoopInput?.checked;
asset.audioDelayMillis = Math.max(0, parseInt(audioDelayInput?.value || '0', 10));
asset.audioSpeed = Math.max(0.25, (parseInt(audioSpeedInput?.value || '100', 10) / 100));
asset.audioPitch = Math.max(0.5, (parseInt(audioPitchInput?.value || '100', 10) / 100));
asset.audioVolume = Math.max(0, Math.min(1, (parseInt(audioVolumeInput?.value || '100', 10) / 100)));
const controller = ensureAudioController(asset);
applyAudioSettings(controller, asset);
persistTransform(asset);
drawAndList();
}
function nudgeRotation(delta) { function nudgeRotation(delta) {
const asset = getSelectedAsset(); const asset = getSelectedAsset();
if (!asset) return; if (!asset) return;
@@ -976,6 +1207,11 @@ function updateVisibility(asset, hidden) {
body: JSON.stringify({ hidden }) body: JSON.stringify({ hidden })
}).then((r) => r.json()).then((updated) => { }).then((r) => r.json()).then((updated) => {
storeAsset(updated); storeAsset(updated);
if (updated.hidden) {
stopAudio(updated.id);
} else if (isAudioAsset(updated)) {
playAudioFromCanvas(updated, true);
}
updateRenderState(updated); updateRenderState(updated);
drawAndList(); drawAndList();
}); });
@@ -983,8 +1219,8 @@ function updateVisibility(asset, hidden) {
function deleteAsset(asset) { function deleteAsset(asset) {
fetch(`/api/channels/${broadcaster}/assets/${asset.id}`, { method: 'DELETE' }).then(() => { fetch(`/api/channels/${broadcaster}/assets/${asset.id}`, { method: 'DELETE' }).then(() => {
clearMedia(asset.id);
assets.delete(asset.id); assets.delete(asset.id);
mediaCache.delete(asset.id);
renderStates.delete(asset.id); renderStates.delete(asset.id);
zOrderDirty = true; zOrderDirty = true;
if (selectedAssetId === asset.id) { if (selectedAssetId === asset.id) {
@@ -1005,7 +1241,7 @@ function handleFileSelection(input) {
function uploadAsset() { function uploadAsset() {
const fileInput = document.getElementById('asset-file'); const fileInput = document.getElementById('asset-file');
if (!fileInput || !fileInput.files || fileInput.files.length === 0) { if (!fileInput || !fileInput.files || fileInput.files.length === 0) {
alert('Please choose an image, GIF, or video to upload.'); alert('Please choose an image, GIF, video, or audio file to upload.');
return; return;
} }
const data = new FormData(); const data = new FormData();
@@ -1060,7 +1296,12 @@ function persistTransform(asset, silent = false) {
rotation: asset.rotation, rotation: asset.rotation,
speed: asset.speed, speed: asset.speed,
muted: asset.muted, muted: asset.muted,
zIndex: asset.zIndex zIndex: asset.zIndex,
audioLoop: asset.audioLoop,
audioDelayMillis: asset.audioDelayMillis,
audioSpeed: asset.audioSpeed,
audioPitch: asset.audioPitch,
audioVolume: asset.audioVolume
}) })
}).then((r) => r.json()).then((updated) => { }).then((r) => r.json()).then((updated) => {
storeAsset(updated); storeAsset(updated);
@@ -1097,6 +1338,13 @@ canvas.addEventListener('mousedown', (event) => {
const hit = findAssetAtPoint(point.x, point.y); const hit = findAssetAtPoint(point.x, point.y);
if (hit) { if (hit) {
if (isAudioAsset(hit) && !handle && event.detail >= 2) {
selectedAssetId = hit.id;
updateRenderState(hit);
playAudioFromCanvas(hit);
drawAndList();
return;
}
selectedAssetId = hit.id; selectedAssetId = hit.id;
updateRenderState(hit); updateRenderState(hit);
interactionState = { interactionState = {

View File

@@ -7,7 +7,9 @@ const assets = new Map();
const mediaCache = new Map(); const mediaCache = new Map();
const renderStates = new Map(); const renderStates = new Map();
const animatedCache = new Map(); const animatedCache = new Map();
const audioControllers = new Map();
let animationFrameId = null; let animationFrameId = null;
const audioPlaceholder = 'data:image/svg+xml;utf8,' + encodeURIComponent('<svg xmlns="http://www.w3.org/2000/svg" width="320" height="80"><rect width="100%" height="100%" fill="#0f172a" rx="8"/><g fill="#22d3ee" transform="translate(20 20)"><circle cx="15" cy="20" r="6"/><rect x="28" y="5" width="12" height="30" rx="2"/><rect x="45" y="10" width="140" height="5" fill="#a5f3fc"/><rect x="45" y="23" width="110" height="5" fill="#a5f3fc"/></g><text x="20" y="70" fill="#e5e7eb" font-family="sans-serif" font-size="14">Audio</text></svg>');
function connect() { function connect() {
const socket = new SockJS('/ws'); const socket = new SockJS('/ws');
@@ -61,6 +63,9 @@ function handleEvent(event) {
const payload = { ...event.payload, zIndex: Math.max(1, event.payload.zIndex ?? 1) }; const payload = { ...event.payload, zIndex: Math.max(1, event.payload.zIndex ?? 1) };
assets.set(payload.id, payload); assets.set(payload.id, payload);
ensureMedia(payload); ensureMedia(payload);
if (isAudioAsset(payload)) {
playAudioImmediately(payload);
}
} else if (event.payload && event.payload.hidden) { } else if (event.payload && event.payload.hidden) {
assets.delete(event.payload.id); assets.delete(event.payload.id);
clearMedia(event.payload.id); clearMedia(event.payload.id);
@@ -97,11 +102,18 @@ function drawAsset(asset) {
const media = ensureMedia(asset); const media = ensureMedia(asset);
const drawSource = media?.isAnimated ? media.bitmap : media; const drawSource = media?.isAnimated ? media.bitmap : media;
const ready = isDrawable(media); const ready = isAudioAsset(asset) || isDrawable(media);
if (ready) { if (isAudioAsset(asset)) {
autoStartAudio(asset);
}
if (ready && drawSource) {
ctx.drawImage(drawSource, -halfWidth, -halfHeight, renderState.width, renderState.height); ctx.drawImage(drawSource, -halfWidth, -halfHeight, renderState.width, renderState.height);
} }
if (isAudioAsset(asset)) {
drawAudioIndicators(asset, halfWidth, halfHeight);
}
ctx.restore(); ctx.restore();
} }
@@ -132,6 +144,10 @@ function isVideoAsset(asset) {
return asset?.mediaType?.startsWith('video/'); return asset?.mediaType?.startsWith('video/');
} }
function isAudioAsset(asset) {
return asset?.mediaType?.startsWith('audio/');
}
function isVideoElement(element) { function isVideoElement(element) {
return element && element.tagName === 'VIDEO'; return element && element.tagName === 'VIDEO';
} }
@@ -140,6 +156,59 @@ function isGifAsset(asset) {
return asset?.mediaType?.toLowerCase() === 'image/gif'; return asset?.mediaType?.toLowerCase() === 'image/gif';
} }
function drawAudioIndicators(asset, halfWidth, halfHeight) {
const controller = audioControllers.get(asset.id);
const isPlaying = controller && !controller.element.paused && !controller.element.ended;
const hasDelay = !!(controller && controller.delayTimeout);
if (!isPlaying && !hasDelay) {
return;
}
const indicatorSize = 18;
const padding = 8;
let x = -halfWidth + padding + indicatorSize / 2;
const y = -halfHeight + padding + indicatorSize / 2;
ctx.save();
ctx.setLineDash([]);
if (isPlaying) {
ctx.fillStyle = 'rgba(34, 197, 94, 0.9)';
ctx.strokeStyle = '#020617';
ctx.lineWidth = 1;
ctx.beginPath();
ctx.arc(x, y, indicatorSize / 2, 0, Math.PI * 2);
ctx.fill();
ctx.stroke();
ctx.fillStyle = '#020617';
ctx.beginPath();
const radius = indicatorSize * 0.22;
ctx.moveTo(x - radius, y - radius * 1.1);
ctx.lineTo(x + radius * 1.2, y);
ctx.lineTo(x - radius, y + radius * 1.1);
ctx.closePath();
ctx.fill();
x += indicatorSize + 4;
}
if (hasDelay) {
ctx.fillStyle = 'rgba(234, 179, 8, 0.9)';
ctx.strokeStyle = '#020617';
ctx.lineWidth = 1;
ctx.beginPath();
ctx.arc(x, y, indicatorSize / 2, 0, Math.PI * 2);
ctx.fill();
ctx.stroke();
ctx.strokeStyle = '#020617';
ctx.beginPath();
ctx.moveTo(x, y);
ctx.lineTo(x, y - indicatorSize * 0.22);
ctx.moveTo(x, y);
ctx.lineTo(x + indicatorSize * 0.22, y);
ctx.stroke();
}
ctx.restore();
}
function isDrawable(element) { function isDrawable(element) {
if (!element) { if (!element) {
return false; return false;
@@ -166,6 +235,104 @@ function clearMedia(assetId) {
animated.decoder?.close?.(); animated.decoder?.close?.();
animatedCache.delete(assetId); animatedCache.delete(assetId);
} }
const audio = audioControllers.get(assetId);
if (audio) {
if (audio.delayTimeout) {
clearTimeout(audio.delayTimeout);
}
audio.element.pause();
audio.element.currentTime = 0;
audioControllers.delete(assetId);
}
}
function ensureAudioController(asset) {
const cached = audioControllers.get(asset.id);
if (cached && cached.src === asset.url) {
applyAudioSettings(cached, asset);
return cached;
}
if (cached) {
clearMedia(asset.id);
}
const element = new Audio(asset.url);
element.preload = 'auto';
element.controls = false;
const controller = {
id: asset.id,
src: asset.url,
element,
delayTimeout: null,
loopEnabled: false,
delayMs: 0,
baseDelayMs: 0
};
element.onended = () => handleAudioEnded(asset.id);
audioControllers.set(asset.id, controller);
applyAudioSettings(controller, asset, true);
return controller;
}
function applyAudioSettings(controller, asset, resetPosition = false) {
controller.loopEnabled = !!asset.audioLoop;
controller.baseDelayMs = Math.max(0, asset.audioDelayMillis || 0);
controller.delayMs = controller.baseDelayMs;
const speed = Math.max(0.25, asset.audioSpeed || 1);
const pitch = Math.max(0.5, asset.audioPitch || 1);
controller.element.playbackRate = speed * pitch;
const volume = Math.max(0, Math.min(1, asset.audioVolume ?? 1));
controller.element.volume = volume;
if (resetPosition) {
controller.element.currentTime = 0;
controller.element.pause();
}
}
function handleAudioEnded(assetId) {
const controller = audioControllers.get(assetId);
if (!controller) return;
controller.element.currentTime = 0;
if (controller.delayTimeout) {
clearTimeout(controller.delayTimeout);
}
if (controller.loopEnabled) {
controller.delayTimeout = setTimeout(() => {
controller.element.play().catch(() => {});
}, controller.delayMs);
} else {
controller.element.pause();
}
}
function playAudioImmediately(asset) {
const controller = ensureAudioController(asset);
if (controller.delayTimeout) {
clearTimeout(controller.delayTimeout);
controller.delayTimeout = null;
}
controller.element.currentTime = 0;
const originalDelay = controller.delayMs;
controller.delayMs = 0;
controller.element.play().catch(() => {});
controller.delayMs = controller.baseDelayMs ?? originalDelay ?? 0;
}
function autoStartAudio(asset) {
if (!isAudioAsset(asset) || asset.hidden) {
return;
}
const controller = ensureAudioController(asset);
if (!controller.element.paused && !controller.element.ended) {
return;
}
if (controller.delayTimeout) {
return;
}
controller.delayTimeout = setTimeout(() => {
controller.element.play().catch(() => {});
}, controller.delayMs);
} }
function ensureMedia(asset) { function ensureMedia(asset) {
@@ -175,6 +342,14 @@ function ensureMedia(asset) {
return cached; return cached;
} }
if (isAudioAsset(asset)) {
ensureAudioController(asset);
const placeholder = new Image();
placeholder.src = audioPlaceholder;
mediaCache.set(asset.id, placeholder);
return placeholder;
}
if (isGifAsset(asset) && 'ImageDecoder' in window) { if (isGifAsset(asset) && 'ImageDecoder' in window) {
const animated = ensureAnimatedImage(asset); const animated = ensureAnimatedImage(asset);
if (animated) { if (animated) {

View File

@@ -28,11 +28,11 @@
<h3>Overlay assets</h3> <h3>Overlay assets</h3>
<p>Upload overlay visuals and adjust them inline.</p> <p>Upload overlay visuals and adjust them inline.</p>
<div class="upload-row"> <div class="upload-row">
<input id="asset-file" class="file-input-field" type="file" accept="image/*,video/*" onchange="handleFileSelection(this)" /> <input id="asset-file" class="file-input-field" type="file" accept="image/*,video/*,audio/*" onchange="handleFileSelection(this)" />
<label for="asset-file" class="file-input-trigger"> <label for="asset-file" class="file-input-trigger">
<span class="file-input-icon"><i class="fa-solid fa-cloud-arrow-up"></i></span> <span class="file-input-icon"><i class="fa-solid fa-cloud-arrow-up"></i></span>
<span class="file-input-copy"> <span class="file-input-copy">
<strong>Select an image, GIF, or video</strong> <strong>Select an image, GIF, video, or audio</strong>
<small id="asset-file-name">No file chosen</small> <small id="asset-file-name">No file chosen</small>
</span> </span>
</label> </label>
@@ -94,6 +94,37 @@
</label> </label>
</div> </div>
</div> </div>
<div class="panel-section hidden" id="audio-section">
<div class="section-header">
<h5>Audio</h5>
</div>
<div class="control-grid condensed three-col">
<label class="checkbox-inline">
<input id="asset-audio-loop" type="checkbox" />
Loop
</label>
<label>
Delay (ms)
<input id="asset-audio-delay" class="number-input" type="number" min="0" step="100" />
</label>
<label>
Pitch (%)
<input id="asset-audio-pitch" class="range-input" type="range" min="50" max="200" step="5" value="100" />
</label>
<label>
Volume (%)
<input id="asset-audio-volume" class="range-input" type="range" min="0" max="100" step="1" value="100" />
</label>
</div>
<div class="control-grid condensed">
<label>
Playback speed
<input id="asset-audio-speed" class="range-input" type="range" min="25" max="400" step="5" value="100" />
</label>
<div class="range-meta"><span>0.25x</span><span>4x</span></div>
</div>
</div>
</div> </div>
</div> </div>
</section> </section>