Improve rdd proc

This commit is contained in:
2025-12-10 15:46:04 +01:00
parent 2bec770b4e
commit 0bca3283fa
11 changed files with 413 additions and 75 deletions

View File

@@ -3,7 +3,10 @@ APP_NAME=imgfloat
.PHONY: run test package docker-build docker-run ssl .PHONY: run test package docker-build docker-run ssl
run: run:
test -f .env && . ./.env; mvn spring-boot:run -Dspring-boot.run.fork=true test -f .env && . ./.env; mvn spring-boot:run
dev:
test -f .env && . ./.env; ./devserver
test: test:
mvn test mvn test

26
devserver Executable file
View File

@@ -0,0 +1,26 @@
#!/usr/bin/env bash
set -e
cleanup() {
echo "Stopping dev server..."
kill "$SERVER_PID" 2>/dev/null || true
exit
}
trap cleanup INT TERM
echo "Starting Spring Boot dev server..."
mvn spring-boot:run &
SERVER_PID=$!
echo "Dev server PID: $SERVER_PID"
echo "Watching for file changes..."
while kill -0 "$SERVER_PID" 2>/dev/null; do
find src/main/java -name "*.java" |
entr -d mvn -q compile
done
echo "Dev server exited."
exit 0

View File

@@ -66,6 +66,7 @@ public class SchemaMigration implements ApplicationRunner {
addColumnIfMissing("assets", columns, "audio_speed", "REAL", "1.0"); addColumnIfMissing("assets", columns, "audio_speed", "REAL", "1.0");
addColumnIfMissing("assets", columns, "audio_pitch", "REAL", "1.0"); addColumnIfMissing("assets", columns, "audio_pitch", "REAL", "1.0");
addColumnIfMissing("assets", columns, "audio_volume", "REAL", "1.0"); addColumnIfMissing("assets", columns, "audio_volume", "REAL", "1.0");
addColumnIfMissing("assets", columns, "preview", "TEXT", "NULL");
} }
private void addColumnIfMissing(String tableName, List<String> existingColumns, String columnName, String dataType, String defaultValue) { private void addColumnIfMissing(String tableName, List<String> existingColumns, String columnName, String dataType, String defaultValue) {

View File

@@ -231,6 +231,33 @@ public class ChannelApiController {
.orElseThrow(() -> new ResponseStatusException(FORBIDDEN, "Asset not available")); .orElseThrow(() -> new ResponseStatusException(FORBIDDEN, "Asset not available"));
} }
@GetMapping("/assets/{assetId}/preview")
public ResponseEntity<byte[]> getAssetPreview(@PathVariable("broadcaster") String broadcaster,
@PathVariable("assetId") String assetId,
OAuth2AuthenticationToken authentication) {
boolean authorized = false;
if (authentication != null) {
String login = TwitchUser.from(authentication).login();
authorized = channelDirectoryService.isBroadcaster(broadcaster, login)
|| channelDirectoryService.isAdmin(broadcaster, login);
}
if (authorized) {
LOG.debug("Serving preview for asset {} for broadcaster {}", assetId, broadcaster);
return channelDirectoryService.getAssetPreview(broadcaster, assetId, true)
.map(content -> ResponseEntity.ok()
.contentType(MediaType.parseMediaType(content.mediaType()))
.body(content.bytes()))
.orElseThrow(() -> new ResponseStatusException(NOT_FOUND, "Preview not found"));
}
return channelDirectoryService.getAssetPreview(broadcaster, assetId, false)
.map(content -> ResponseEntity.ok()
.contentType(MediaType.parseMediaType(content.mediaType()))
.body(content.bytes()))
.orElseThrow(() -> new ResponseStatusException(FORBIDDEN, "Preview not available"));
}
@DeleteMapping("/assets/{assetId}") @DeleteMapping("/assets/{assetId}")
public ResponseEntity<?> delete(@PathVariable("broadcaster") String broadcaster, public ResponseEntity<?> delete(@PathVariable("broadcaster") String broadcaster,
@PathVariable("assetId") String assetId, @PathVariable("assetId") String assetId,

View File

@@ -34,6 +34,8 @@ public class Asset {
private Boolean muted; private Boolean muted;
private String mediaType; private String mediaType;
private String originalMediaType; private String originalMediaType;
@Column(columnDefinition = "TEXT")
private String preview;
private Integer zIndex; private Integer zIndex;
private Boolean audioLoop; private Boolean audioLoop;
private Integer audioDelayMillis; private Integer audioDelayMillis;
@@ -202,6 +204,14 @@ public class Asset {
this.originalMediaType = originalMediaType; this.originalMediaType = originalMediaType;
} }
public String getPreview() {
return preview;
}
public void setPreview(String preview) {
this.preview = preview;
}
public boolean isVideo() { public boolean isVideo() {
return mediaType != null && mediaType.toLowerCase(Locale.ROOT).startsWith("video/"); return mediaType != null && mediaType.toLowerCase(Locale.ROOT).startsWith("video/");
} }

View File

@@ -7,6 +7,7 @@ public record AssetView(
String broadcaster, String broadcaster,
String name, String name,
String url, String url,
String previewUrl,
double x, double x,
double y, double y,
double width, double width,
@@ -23,6 +24,7 @@ public record AssetView(
Double audioPitch, Double audioPitch,
Double audioVolume, Double audioVolume,
boolean hidden, boolean hidden,
boolean hasPreview,
Instant createdAt Instant createdAt
) { ) {
public static AssetView from(String broadcaster, Asset asset) { public static AssetView from(String broadcaster, Asset asset) {
@@ -31,6 +33,7 @@ public record AssetView(
asset.getBroadcaster(), asset.getBroadcaster(),
asset.getName(), asset.getName(),
"/api/channels/" + broadcaster + "/assets/" + asset.getId() + "/content", "/api/channels/" + broadcaster + "/assets/" + asset.getId() + "/content",
asset.getPreview() != null ? "/api/channels/" + broadcaster + "/assets/" + asset.getId() + "/preview" : null,
asset.getX(), asset.getX(),
asset.getY(), asset.getY(),
asset.getWidth(), asset.getWidth(),
@@ -47,6 +50,7 @@ public record AssetView(
asset.getAudioPitch(), asset.getAudioPitch(),
asset.getAudioVolume(), asset.getAudioVolume(),
asset.isHidden(), asset.isHidden(),
asset.getPreview() != null,
asset.getCreatedAt() asset.getCreatedAt()
); );
} }

View File

@@ -15,6 +15,7 @@ import org.jcodec.api.JCodecException;
import org.jcodec.api.awt.AWTSequenceEncoder; import org.jcodec.api.awt.AWTSequenceEncoder;
import org.jcodec.common.io.ByteBufferSeekableByteChannel; import org.jcodec.common.io.ByteBufferSeekableByteChannel;
import org.jcodec.common.model.Picture; import org.jcodec.common.model.Picture;
import org.jcodec.scale.AWTUtil;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.messaging.simp.SimpMessagingTemplate; import org.springframework.messaging.simp.SimpMessagingTemplate;
@@ -131,6 +132,7 @@ public class ChannelDirectoryService {
Asset asset = new Asset(channel.getBroadcaster(), name, dataUrl, width, height); Asset asset = new Asset(channel.getBroadcaster(), name, dataUrl, width, height);
asset.setOriginalMediaType(mediaType); asset.setOriginalMediaType(mediaType);
asset.setMediaType(optimized.mediaType()); asset.setMediaType(optimized.mediaType());
asset.setPreview(optimized.previewDataUrl());
asset.setSpeed(1.0); asset.setSpeed(1.0);
asset.setMuted(optimized.mediaType().startsWith("video/")); asset.setMuted(optimized.mediaType().startsWith("video/"));
asset.setAudioLoop(false); asset.setAudioLoop(false);
@@ -240,6 +242,24 @@ public class ChannelDirectoryService {
.flatMap(this::decodeAssetData); .flatMap(this::decodeAssetData);
} }
public Optional<AssetContent> getAssetPreview(String broadcaster, String assetId, boolean includeHidden) {
String normalized = normalize(broadcaster);
return assetRepository.findById(assetId)
.filter(asset -> normalized.equals(asset.getBroadcaster()))
.filter(asset -> includeHidden || !asset.isHidden())
.map(asset -> {
Optional<AssetContent> preview = decodeDataUrl(asset.getPreview());
if (preview.isPresent()) {
return preview.get();
}
if (asset.getMediaType() != null && asset.getMediaType().startsWith("image/")) {
return decodeAssetData(asset).orElse(null);
}
return null;
})
.flatMap(Optional::ofNullable);
}
public boolean isBroadcaster(String broadcaster, String username) { public boolean isBroadcaster(String broadcaster, String username) {
return broadcaster != null && broadcaster.equalsIgnoreCase(username); return broadcaster != null && broadcaster.equalsIgnoreCase(username);
} }
@@ -279,23 +299,30 @@ public class ChannelDirectoryService {
} }
private Optional<AssetContent> decodeAssetData(Asset asset) { private Optional<AssetContent> decodeAssetData(Asset asset) {
String url = asset.getUrl(); return decodeDataUrl(asset.getUrl())
if (url == null || !url.startsWith("data:")) { .or(() -> {
logger.warn("Unable to decode asset data for {}", asset.getId());
return Optional.empty();
});
}
private Optional<AssetContent> decodeDataUrl(String dataUrl) {
if (dataUrl == null || !dataUrl.startsWith("data:")) {
return Optional.empty(); return Optional.empty();
} }
int commaIndex = url.indexOf(','); int commaIndex = dataUrl.indexOf(',');
if (commaIndex < 0) { if (commaIndex < 0) {
return Optional.empty(); return Optional.empty();
} }
String metadata = url.substring(5, commaIndex); String metadata = dataUrl.substring(5, commaIndex);
String[] parts = metadata.split(";", 2); String[] parts = metadata.split(";", 2);
String mediaType = parts.length > 0 && !parts[0].isBlank() ? parts[0] : "application/octet-stream"; String mediaType = parts.length > 0 && !parts[0].isBlank() ? parts[0] : "application/octet-stream";
String encoded = url.substring(commaIndex + 1); String encoded = dataUrl.substring(commaIndex + 1);
try { try {
byte[] bytes = Base64.getDecoder().decode(encoded); byte[] bytes = Base64.getDecoder().decode(encoded);
return Optional.of(new AssetContent(bytes, mediaType)); return Optional.of(new AssetContent(bytes, mediaType));
} catch (IllegalArgumentException e) { } catch (IllegalArgumentException e) {
logger.warn("Unable to decode asset data for {}", asset.getId(), e); logger.warn("Unable to decode data url", e);
return Optional.empty(); return Optional.empty();
} }
} }
@@ -353,7 +380,7 @@ public class ChannelDirectoryService {
return null; return null;
} }
byte[] compressed = compressPng(image); byte[] compressed = compressPng(image);
return new OptimizedAsset(compressed, "image/png", image.getWidth(), image.getHeight()); return new OptimizedAsset(compressed, "image/png", image.getWidth(), image.getHeight(), null);
} }
if (mediaType.startsWith("image/")) { if (mediaType.startsWith("image/")) {
@@ -361,21 +388,22 @@ public class ChannelDirectoryService {
if (image == null) { if (image == null) {
return null; return null;
} }
return new OptimizedAsset(bytes, mediaType, image.getWidth(), image.getHeight()); return new OptimizedAsset(bytes, mediaType, image.getWidth(), image.getHeight(), null);
} }
if (mediaType.startsWith("video/")) { if (mediaType.startsWith("video/")) {
var dimensions = extractVideoDimensions(bytes); var dimensions = extractVideoDimensions(bytes);
return new OptimizedAsset(bytes, mediaType, dimensions.width(), dimensions.height()); String preview = extractVideoPreview(bytes, mediaType);
return new OptimizedAsset(bytes, mediaType, dimensions.width(), dimensions.height(), preview);
} }
if (mediaType.startsWith("audio/")) { if (mediaType.startsWith("audio/")) {
return new OptimizedAsset(bytes, mediaType, 0, 0); return new OptimizedAsset(bytes, mediaType, 0, 0, null);
} }
BufferedImage image = ImageIO.read(new ByteArrayInputStream(bytes)); BufferedImage image = ImageIO.read(new ByteArrayInputStream(bytes));
if (image != null) { if (image != null) {
return new OptimizedAsset(bytes, mediaType, image.getWidth(), image.getHeight()); return new OptimizedAsset(bytes, mediaType, image.getWidth(), image.getHeight(), null);
} }
return null; return null;
} }
@@ -404,7 +432,7 @@ public class ChannelDirectoryService {
encoder.finish(); encoder.finish();
BufferedImage cover = frames.get(0).image(); BufferedImage cover = frames.get(0).image();
byte[] video = Files.readAllBytes(temp.toPath()); byte[] video = Files.readAllBytes(temp.toPath());
return new OptimizedAsset(video, "video/mp4", cover.getWidth(), cover.getHeight()); return new OptimizedAsset(video, "video/mp4", cover.getWidth(), cover.getHeight(), encodePreview(cover));
} finally { } finally {
Files.deleteIfExists(temp.toPath()); Files.deleteIfExists(temp.toPath());
} }
@@ -498,6 +526,19 @@ public class ChannelDirectoryService {
} }
} }
private String encodePreview(BufferedImage image) {
if (image == null) {
return null;
}
try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
ImageIO.write(image, "png", baos);
return "data:image/png;base64," + Base64.getEncoder().encodeToString(baos.toByteArray());
} catch (IOException e) {
logger.warn("Unable to encode preview image", e);
return null;
}
}
private Dimension extractVideoDimensions(byte[] bytes) { private Dimension extractVideoDimensions(byte[] bytes) {
try (var channel = new ByteBufferSeekableByteChannel(ByteBuffer.wrap(bytes), bytes.length)) { try (var channel = new ByteBufferSeekableByteChannel(ByteBuffer.wrap(bytes), bytes.length)) {
FrameGrab grab = FrameGrab.createFrameGrab(channel); FrameGrab grab = FrameGrab.createFrameGrab(channel);
@@ -511,9 +552,24 @@ public class ChannelDirectoryService {
return new Dimension(640, 360); return new Dimension(640, 360);
} }
private String extractVideoPreview(byte[] bytes, String mediaType) {
try (var channel = new ByteBufferSeekableByteChannel(ByteBuffer.wrap(bytes), bytes.length)) {
FrameGrab grab = FrameGrab.createFrameGrab(channel);
Picture frame = grab.getNativeFrame();
if (frame == null) {
return null;
}
BufferedImage image = AWTUtil.toBufferedImage(frame);
return encodePreview(image);
} catch (IOException | JCodecException e) {
logger.warn("Unable to capture video preview frame for {}", mediaType, e);
return null;
}
}
public record AssetContent(byte[] bytes, String mediaType) { } public record AssetContent(byte[] bytes, String mediaType) { }
private record OptimizedAsset(byte[] bytes, String mediaType, int width, int height) { } private record OptimizedAsset(byte[] bytes, String mediaType, int width, int height, String previewDataUrl) { }
private record GifFrame(BufferedImage image, int delayMs) { } private record GifFrame(BufferedImage image, int delayMs) { }

View File

@@ -695,6 +695,11 @@ body {
margin: 6px 0 0; margin: 6px 0 0;
} }
.subtle-text {
color: #94a3b8;
font-size: 12px;
}
.panel-section { .panel-section {
margin-top: 12px; margin-top: 12px;
padding: 14px; padding: 14px;
@@ -720,6 +725,31 @@ body {
margin: 0; margin: 0;
} }
.field-note {
margin: 0;
color: #94a3b8;
font-size: 12px;
}
.stacked-field {
display: flex;
flex-direction: column;
gap: 6px;
margin-bottom: 10px;
}
.label-row {
display: flex;
align-items: center;
justify-content: space-between;
gap: 10px;
}
.value-hint {
color: #cbd5e1;
font-size: 12px;
}
.asset-list { .asset-list {
display: flex; display: flex;
flex-direction: column; flex-direction: column;
@@ -938,6 +968,11 @@ body {
margin-top: 8px; margin-top: 8px;
} }
.control-grid.split-row {
grid-template-columns: repeat(auto-fit, minmax(140px, 1fr));
margin-top: 6px;
}
.control-grid.three-col { .control-grid.three-col {
grid-template-columns: repeat(auto-fit, minmax(140px, 1fr)); grid-template-columns: repeat(auto-fit, minmax(140px, 1fr));
} }
@@ -949,6 +984,11 @@ body {
color: #cbd5e1; color: #cbd5e1;
} }
.control-grid .inline-toggle {
align-items: center;
justify-content: space-between;
}
.control-grid input[type="number"], .control-grid input[type="number"],
.control-grid input[type="range"] { .control-grid input[type="range"] {
padding: 8px; padding: 8px;

View File

@@ -15,6 +15,7 @@ const audioControllers = new Map();
const pendingAudioUnlock = new Set(); const pendingAudioUnlock = new Set();
const loopPlaybackState = new Map(); const loopPlaybackState = new Map();
const previewCache = new Map(); const previewCache = new Map();
const previewImageCache = new Map();
let drawPending = false; let drawPending = false;
let zOrderDirty = true; let zOrderDirty = true;
let zOrderCache = []; let zOrderCache = [];
@@ -30,6 +31,7 @@ const heightInput = document.getElementById('asset-height');
const aspectLockInput = document.getElementById('maintain-aspect'); const aspectLockInput = document.getElementById('maintain-aspect');
const speedInput = document.getElementById('asset-speed'); const speedInput = document.getElementById('asset-speed');
const muteInput = document.getElementById('asset-muted'); const muteInput = document.getElementById('asset-muted');
const speedLabel = document.getElementById('asset-speed-label');
const selectedZLabel = document.getElementById('asset-z-level'); const selectedZLabel = document.getElementById('asset-z-level');
const playbackSection = document.getElementById('playback-section'); const playbackSection = document.getElementById('playback-section');
const audioSection = document.getElementById('audio-section'); const audioSection = document.getElementById('audio-section');
@@ -37,6 +39,7 @@ const layoutSection = document.getElementById('layout-section');
const audioLoopInput = document.getElementById('asset-audio-loop'); const audioLoopInput = document.getElementById('asset-audio-loop');
const audioDelayInput = document.getElementById('asset-audio-delay'); const audioDelayInput = document.getElementById('asset-audio-delay');
const audioSpeedInput = document.getElementById('asset-audio-speed'); const audioSpeedInput = document.getElementById('asset-audio-speed');
const audioSpeedLabel = document.getElementById('asset-audio-speed-label');
const audioPitchInput = document.getElementById('asset-audio-pitch'); const audioPitchInput = document.getElementById('asset-audio-pitch');
const audioVolumeInput = document.getElementById('asset-audio-volume'); const audioVolumeInput = document.getElementById('asset-audio-volume');
const controlsPlaceholder = document.getElementById('asset-controls-placeholder'); const controlsPlaceholder = document.getElementById('asset-controls-placeholder');
@@ -44,6 +47,7 @@ const fileNameLabel = document.getElementById('asset-file-name');
const assetInspector = document.getElementById('asset-inspector'); const assetInspector = document.getElementById('asset-inspector');
const selectedAssetName = document.getElementById('selected-asset-name'); const selectedAssetName = document.getElementById('selected-asset-name');
const selectedAssetMeta = document.getElementById('selected-asset-meta'); const selectedAssetMeta = document.getElementById('selected-asset-meta');
const selectedAssetIdLabel = document.getElementById('selected-asset-id');
const selectedAssetBadges = document.getElementById('selected-asset-badges'); const selectedAssetBadges = document.getElementById('selected-asset-badges');
const selectedVisibilityBtn = document.getElementById('selected-asset-visibility'); const selectedVisibilityBtn = document.getElementById('selected-asset-visibility');
const selectedDeleteBtn = document.getElementById('selected-asset-delete'); const selectedDeleteBtn = document.getElementById('selected-asset-delete');
@@ -144,6 +148,18 @@ function getDurationBadge(asset) {
return formatDurationLabel(asset.durationMs); return formatDurationLabel(asset.durationMs);
} }
function setSpeedLabel(percent) {
if (!speedLabel) return;
speedLabel.textContent = `${Math.round(percent)}%`;
}
function setAudioSpeedLabel(percentValue) {
if (!audioSpeedLabel) return;
const multiplier = Math.max(0, percentValue) / 100;
const formatted = multiplier >= 10 ? multiplier.toFixed(0) : multiplier.toFixed(2);
audioSpeedLabel.textContent = `${formatted}x`;
}
function queueAudioForUnlock(controller) { function queueAudioForUnlock(controller) {
if (!controller) return; if (!controller) return;
pendingAudioUnlock.add(controller); pendingAudioUnlock.add(controller);
@@ -266,20 +282,22 @@ function renderAssets(list) {
function storeAsset(asset) { function storeAsset(asset) {
if (!asset) return; if (!asset) return;
const existing = assets.get(asset.id); const existing = assets.get(asset.id);
if (existing && existing.url !== asset.url) { const merged = existing ? { ...existing, ...asset } : { ...asset };
const mediaChanged = existing && existing.url !== merged.url;
const previewChanged = existing && existing.previewUrl !== merged.previewUrl;
if (mediaChanged || previewChanged) {
clearMedia(asset.id); clearMedia(asset.id);
previewCache.delete(asset.id);
} }
asset.zIndex = Math.max(1, asset.zIndex ?? 1); merged.zIndex = Math.max(1, merged.zIndex ?? 1);
const parsedCreatedAt = asset.createdAt ? new Date(asset.createdAt).getTime() : NaN; const parsedCreatedAt = merged.createdAt ? new Date(merged.createdAt).getTime() : NaN;
const hasCreatedAtMs = typeof asset.createdAtMs === 'number' && Number.isFinite(asset.createdAtMs); const hasCreatedAtMs = typeof merged.createdAtMs === 'number' && Number.isFinite(merged.createdAtMs);
if (!hasCreatedAtMs) { if (!hasCreatedAtMs) {
asset.createdAtMs = Number.isFinite(parsedCreatedAt) ? parsedCreatedAt : Date.now(); merged.createdAtMs = Number.isFinite(parsedCreatedAt) ? parsedCreatedAt : Date.now();
} }
assets.set(asset.id, asset); assets.set(asset.id, merged);
zOrderDirty = true; zOrderDirty = true;
if (!renderStates.has(asset.id)) { if (!renderStates.has(asset.id)) {
renderStates.set(asset.id, { ...asset }); renderStates.set(asset.id, { ...merged });
} }
resolvePendingUploadByName(asset.name); resolvePendingUploadByName(asset.name);
} }
@@ -376,9 +394,18 @@ function drawAsset(asset) {
return; return;
} }
const media = ensureMedia(asset); let drawSource = null;
const drawSource = media?.isAnimated ? media.bitmap : media; let ready = false;
const ready = isDrawable(media); let showPlayOverlay = false;
if (isVideoAsset(asset) || isGifAsset(asset)) {
drawSource = ensureCanvasPreview(asset);
ready = isDrawable(drawSource);
showPlayOverlay = true;
} else {
const media = ensureMedia(asset);
drawSource = media?.isAnimated ? media.bitmap : media;
ready = isDrawable(media);
}
if (ready && drawSource) { if (ready && drawSource) {
ctx.globalAlpha = asset.hidden ? 0.35 : 0.9; ctx.globalAlpha = asset.hidden ? 0.35 : 0.9;
ctx.drawImage(drawSource, -halfWidth, -halfHeight, renderState.width, renderState.height); ctx.drawImage(drawSource, -halfWidth, -halfHeight, renderState.width, renderState.height);
@@ -398,6 +425,9 @@ function drawAsset(asset) {
ctx.lineWidth = asset.id === selectedAssetId ? 2 : 1; ctx.lineWidth = asset.id === selectedAssetId ? 2 : 1;
ctx.setLineDash(asset.id === selectedAssetId ? [6, 4] : []); ctx.setLineDash(asset.id === selectedAssetId ? [6, 4] : []);
ctx.strokeRect(-halfWidth, -halfHeight, renderState.width, renderState.height); ctx.strokeRect(-halfWidth, -halfHeight, renderState.width, renderState.height);
if (showPlayOverlay) {
drawPlayOverlay(renderState);
}
if (asset.id === selectedAssetId) { if (asset.id === selectedAssetId) {
drawSelectionOverlay(renderState); drawSelectionOverlay(renderState);
} }
@@ -425,6 +455,24 @@ function lerp(a, b, t) {
return a + (b - a) * t; return a + (b - a) * t;
} }
function drawPlayOverlay(asset) {
const size = Math.max(24, Math.min(asset.width, asset.height) * 0.2);
ctx.save();
ctx.fillStyle = 'rgba(15, 23, 42, 0.35)';
ctx.beginPath();
ctx.arc(0, 0, size * 0.75, 0, Math.PI * 2);
ctx.fill();
ctx.fillStyle = '#ffffff';
ctx.beginPath();
ctx.moveTo(-size * 0.3, -size * 0.45);
ctx.lineTo(size * 0.55, 0);
ctx.lineTo(-size * 0.3, size * 0.45);
ctx.closePath();
ctx.fill();
ctx.restore();
}
function drawSelectionOverlay(asset) { function drawSelectionOverlay(asset) {
const halfWidth = asset.width / 2; const halfWidth = asset.width / 2;
const halfHeight = asset.height / 2; const halfHeight = asset.height / 2;
@@ -658,7 +706,12 @@ function isDrawable(element) {
function clearMedia(assetId) { function clearMedia(assetId) {
mediaCache.delete(assetId); mediaCache.delete(assetId);
const cachedPreview = previewCache.get(assetId);
if (cachedPreview && cachedPreview.startsWith('blob:')) {
URL.revokeObjectURL(cachedPreview);
}
previewCache.delete(assetId); previewCache.delete(assetId);
previewImageCache.delete(assetId);
const animated = animatedCache.get(assetId); const animated = animatedCache.get(assetId);
if (animated) { if (animated) {
animated.cancelled = true; animated.cancelled = true;
@@ -964,10 +1017,12 @@ function renderAssetList() {
const badges = document.createElement('div'); const badges = document.createElement('div');
badges.className = 'badge-row asset-meta-badges'; badges.className = 'badge-row asset-meta-badges';
badges.appendChild(createBadge(asset.hidden ? 'Hidden' : 'Visible', asset.hidden ? 'danger' : ''));
badges.appendChild(createBadge(getDisplayMediaType(asset))); badges.appendChild(createBadge(getDisplayMediaType(asset)));
badges.appendChild(createBadge(`Z ${asset.zIndex ?? 1}`)); if (!isAudioAsset(asset)) {
const aspectLabel = formatAspectRatioLabel(asset); badges.appendChild(createBadge(asset.hidden ? 'Hidden' : 'Visible', asset.hidden ? 'danger' : ''));
badges.appendChild(createBadge(`Z ${asset.zIndex ?? 1}`));
}
const aspectLabel = !isAudioAsset(asset) ? formatAspectRatioLabel(asset) : '';
if (aspectLabel) { if (aspectLabel) {
badges.appendChild(createBadge(aspectLabel, 'subtle')); badges.appendChild(createBadge(aspectLabel, 'subtle'));
} }
@@ -980,17 +1035,6 @@ function renderAssetList() {
const actions = document.createElement('div'); const actions = document.createElement('div');
actions.className = 'actions'; actions.className = 'actions';
const toggleBtn = document.createElement('button');
toggleBtn.type = 'button';
toggleBtn.className = 'ghost icon-button';
toggleBtn.innerHTML = `<i class="fa-solid ${asset.hidden ? 'fa-eye' : 'fa-eye-slash'}"></i>`;
toggleBtn.title = asset.hidden ? 'Show asset' : 'Hide asset';
toggleBtn.addEventListener('click', (e) => {
e.stopPropagation();
selectedAssetId = asset.id;
updateVisibility(asset, !asset.hidden);
});
if (isAudioAsset(asset)) { if (isAudioAsset(asset)) {
const playBtn = document.createElement('button'); const playBtn = document.createElement('button');
playBtn.type = 'button'; playBtn.type = 'button';
@@ -1016,6 +1060,20 @@ function renderAssetList() {
actions.appendChild(playBtn); actions.appendChild(playBtn);
} }
if (!isAudioAsset(asset)) {
const toggleBtn = document.createElement('button');
toggleBtn.type = 'button';
toggleBtn.className = 'ghost icon-button';
toggleBtn.innerHTML = `<i class="fa-solid ${asset.hidden ? 'fa-eye' : 'fa-eye-slash'}"></i>`;
toggleBtn.title = asset.hidden ? 'Show asset' : 'Hide asset';
toggleBtn.addEventListener('click', (e) => {
e.stopPropagation();
selectedAssetId = asset.id;
updateVisibility(asset, !asset.hidden);
});
actions.appendChild(toggleBtn);
}
const deleteBtn = document.createElement('button'); const deleteBtn = document.createElement('button');
deleteBtn.type = 'button'; deleteBtn.type = 'button';
deleteBtn.className = 'ghost danger icon-button'; deleteBtn.className = 'ghost danger icon-button';
@@ -1026,7 +1084,6 @@ function renderAssetList() {
deleteAsset(asset); deleteAsset(asset);
}); });
actions.appendChild(toggleBtn);
actions.appendChild(deleteBtn); actions.appendChild(deleteBtn);
row.appendChild(preview); row.appendChild(preview);
@@ -1136,26 +1193,50 @@ function createPreviewElement(asset) {
return img; return img;
} }
function loadPreviewFrame(asset, element) { function fetchPreviewData(asset) {
if (!asset || !element) return; if (!asset) return Promise.resolve(null);
const cached = previewCache.get(asset.id); const cached = previewCache.get(asset.id);
if (cached) { if (cached) {
applyPreviewFrame(element, cached); return Promise.resolve(cached);
return;
} }
const source = isVideoAsset(asset) const primary = asset.previewUrl
? captureVideoFrame(asset) ? fetch(asset.previewUrl)
: isGifAsset(asset) .then((r) => {
? captureGifFrame(asset) if (!r.ok) throw new Error('preview fetch failed');
: Promise.resolve(null); return r.blob();
})
.then((blob) => URL.createObjectURL(blob))
.catch(() => null)
: Promise.resolve(null);
source return primary
.then((dataUrl) => { .then((dataUrl) => {
if (!dataUrl) { if (dataUrl) {
return; previewCache.set(asset.id, dataUrl);
return dataUrl;
} }
previewCache.set(asset.id, dataUrl); const fallback = isVideoAsset(asset)
? captureVideoFrame(asset)
: isGifAsset(asset)
? captureGifFrame(asset)
: Promise.resolve(null);
return fallback.then((result) => {
if (!result) {
return null;
}
previewCache.set(asset.id, result);
return result;
});
})
.catch(() => null);
}
function loadPreviewFrame(asset, element) {
if (!asset || !element) return;
fetchPreviewData(asset)
.then((dataUrl) => {
if (!dataUrl) return;
applyPreviewFrame(element, dataUrl); applyPreviewFrame(element, dataUrl);
}) })
.catch(() => { }); .catch(() => { });
@@ -1167,6 +1248,36 @@ function applyPreviewFrame(element, dataUrl) {
element.classList.add('has-image'); element.classList.add('has-image');
} }
function ensureCanvasPreview(asset) {
const cachedData = previewCache.get(asset.id);
const cachedImage = previewImageCache.get(asset.id);
if (cachedData && cachedImage?.src === cachedData) {
return cachedImage.image;
}
if (cachedData) {
const img = new Image();
img.crossOrigin = 'anonymous';
img.onload = requestDraw;
img.src = cachedData;
previewImageCache.set(asset.id, { src: cachedData, image: img });
return img;
}
fetchPreviewData(asset)
.then((dataUrl) => {
if (!dataUrl) return;
const img = new Image();
img.crossOrigin = 'anonymous';
img.onload = requestDraw;
img.src = dataUrl;
previewImageCache.set(asset.id, { src: dataUrl, image: img });
})
.catch(() => { });
return null;
}
function captureVideoFrame(asset) { function captureVideoFrame(asset) {
return new Promise((resolve) => { return new Promise((resolve) => {
const video = document.createElement('video'); const video = document.createElement('video');
@@ -1273,6 +1384,7 @@ function updateSelectedAssetControls(asset = getSelectedAsset()) {
if (speedInput) { if (speedInput) {
const percent = Math.round((asset.speed ?? 1) * 100); const percent = Math.round((asset.speed ?? 1) * 100);
speedInput.value = Math.min(1000, Math.max(0, percent)); speedInput.value = Math.min(1000, Math.max(0, percent));
setSpeedLabel(speedInput.value);
} }
if (playbackSection) { if (playbackSection) {
const shouldShowPlayback = isVideoAsset(asset); const shouldShowPlayback = isVideoAsset(asset);
@@ -1297,6 +1409,7 @@ function updateSelectedAssetControls(asset = getSelectedAsset()) {
audioLoopInput.checked = !!asset.audioLoop; audioLoopInput.checked = !!asset.audioLoop;
audioDelayInput.value = Math.max(0, asset.audioDelayMillis ?? 0); audioDelayInput.value = Math.max(0, asset.audioDelayMillis ?? 0);
audioSpeedInput.value = Math.round(Math.max(0.25, asset.audioSpeed ?? 1) * 100); audioSpeedInput.value = Math.round(Math.max(0.25, asset.audioSpeed ?? 1) * 100);
setAudioSpeedLabel(audioSpeedInput.value);
audioPitchInput.value = Math.round(Math.max(0.5, asset.audioPitch ?? 1) * 100); audioPitchInput.value = Math.round(Math.max(0.5, asset.audioPitch ?? 1) * 100);
audioVolumeInput.value = Math.round(Math.max(0, Math.min(1, asset.audioVolume ?? 1)) * 100); audioVolumeInput.value = Math.round(Math.max(0, Math.min(1, asset.audioVolume ?? 1)) * 100);
} }
@@ -1312,16 +1425,29 @@ function updateSelectedAssetSummary(asset) {
selectedAssetName.textContent = asset ? (asset.name || `Asset ${asset.id.slice(0, 6)}`) : 'Choose an asset'; selectedAssetName.textContent = asset ? (asset.name || `Asset ${asset.id.slice(0, 6)}`) : 'Choose an asset';
} }
if (selectedAssetMeta) { if (selectedAssetMeta) {
const baseMeta = asset ? `${Math.round(asset.width)}x${Math.round(asset.height)}` : null;
const layerMeta = asset && !isAudioAsset(asset) ? ` · Layer ${asset.zIndex ?? 1}` : '';
selectedAssetMeta.textContent = asset selectedAssetMeta.textContent = asset
? `${Math.round(asset.width)}x${Math.round(asset.height)} · Layer ${asset.zIndex ?? 1}` ? `${baseMeta}${layerMeta}`
: 'Pick an asset in the list to adjust its placement and playback.'; : 'Pick an asset in the list to adjust its placement and playback.';
} }
if (selectedAssetIdLabel) {
if (asset) {
selectedAssetIdLabel.textContent = `ID: ${asset.id}`;
selectedAssetIdLabel.classList.remove('hidden');
} else {
selectedAssetIdLabel.classList.add('hidden');
selectedAssetIdLabel.textContent = '';
}
}
if (selectedAssetBadges) { if (selectedAssetBadges) {
selectedAssetBadges.innerHTML = ''; selectedAssetBadges.innerHTML = '';
if (asset) { if (asset) {
selectedAssetBadges.appendChild(createBadge(asset.hidden ? 'Hidden' : 'Visible', asset.hidden ? 'danger' : ''));
selectedAssetBadges.appendChild(createBadge(getDisplayMediaType(asset))); selectedAssetBadges.appendChild(createBadge(getDisplayMediaType(asset)));
const aspectLabel = formatAspectRatioLabel(asset); if (!isAudioAsset(asset)) {
selectedAssetBadges.appendChild(createBadge(asset.hidden ? 'Hidden' : 'Visible', asset.hidden ? 'danger' : ''));
}
const aspectLabel = !isAudioAsset(asset) ? formatAspectRatioLabel(asset) : '';
if (aspectLabel) { if (aspectLabel) {
selectedAssetBadges.appendChild(createBadge(aspectLabel, 'subtle')); selectedAssetBadges.appendChild(createBadge(aspectLabel, 'subtle'));
} }
@@ -1373,6 +1499,7 @@ function updatePlaybackFromInputs() {
const asset = getSelectedAsset(); const asset = getSelectedAsset();
if (!asset || !isVideoAsset(asset)) return; if (!asset || !isVideoAsset(asset)) return;
const percent = Math.max(0, Math.min(1000, parseFloat(speedInput?.value) || 100)); const percent = Math.max(0, Math.min(1000, parseFloat(speedInput?.value) || 100));
setSpeedLabel(percent);
asset.speed = percent / 100; asset.speed = percent / 100;
updateRenderState(asset); updateRenderState(asset);
persistTransform(asset); persistTransform(asset);
@@ -1401,7 +1528,9 @@ function updateAudioSettingsFromInputs() {
if (!asset || !isAudioAsset(asset)) return; if (!asset || !isAudioAsset(asset)) return;
asset.audioLoop = !!audioLoopInput?.checked; asset.audioLoop = !!audioLoopInput?.checked;
asset.audioDelayMillis = Math.max(0, parseInt(audioDelayInput?.value || '0', 10)); asset.audioDelayMillis = Math.max(0, parseInt(audioDelayInput?.value || '0', 10));
asset.audioSpeed = Math.max(0.25, (parseInt(audioSpeedInput?.value || '100', 10) / 100)); const nextAudioSpeedPercent = Math.max(25, parseInt(audioSpeedInput?.value || '100', 10));
setAudioSpeedLabel(nextAudioSpeedPercent);
asset.audioSpeed = Math.max(0.25, (nextAudioSpeedPercent / 100));
asset.audioPitch = Math.max(0.5, (parseInt(audioPitchInput?.value || '100', 10) / 100)); asset.audioPitch = Math.max(0.5, (parseInt(audioPitchInput?.value || '100', 10) / 100));
asset.audioVolume = Math.max(0, Math.min(1, (parseInt(audioVolumeInput?.value || '100', 10) / 100))); asset.audioVolume = Math.max(0, Math.min(1, (parseInt(audioVolumeInput?.value || '100', 10) / 100)));
const controller = ensureAudioController(asset); const controller = ensureAudioController(asset);
@@ -1499,6 +1628,9 @@ function getAssetAspectRatio(asset) {
} }
function formatAspectRatioLabel(asset) { function formatAspectRatioLabel(asset) {
if (isAudioAsset(asset)) {
return '';
}
const ratio = getAssetAspectRatio(asset); const ratio = getAssetAspectRatio(asset);
if (!ratio) { if (!ratio) {
return ''; return '';

View File

@@ -7,6 +7,8 @@ const assets = new Map();
const mediaCache = new Map(); const mediaCache = new Map();
const renderStates = new Map(); const renderStates = new Map();
const animatedCache = new Map(); const animatedCache = new Map();
const blobCache = new Map();
const animationFailures = new Map();
const audioControllers = new Map(); const audioControllers = new Map();
const pendingAudioUnlock = new Set(); const pendingAudioUnlock = new Set();
const TARGET_FPS = 60; const TARGET_FPS = 60;
@@ -283,6 +285,8 @@ function clearMedia(assetId) {
animated.decoder?.close?.(); animated.decoder?.close?.();
animatedCache.delete(assetId); animatedCache.delete(assetId);
} }
animationFailures.delete(assetId);
blobCache.delete(assetId);
const audio = audioControllers.get(assetId); const audio = audioControllers.get(assetId);
if (audio) { if (audio) {
if (audio.delayTimeout) { if (audio.delayTimeout) {
@@ -485,11 +489,17 @@ function ensureMedia(asset) {
} }
function ensureAnimatedImage(asset) { function ensureAnimatedImage(asset) {
const failedAt = animationFailures.get(asset.id);
if (failedAt && Date.now() - failedAt < 15000) {
return null;
}
const cached = animatedCache.get(asset.id); const cached = animatedCache.get(asset.id);
if (cached && cached.url === asset.url) { if (cached && cached.url === asset.url) {
return cached; return cached;
} }
animationFailures.delete(asset.id);
if (cached) { if (cached) {
clearMedia(asset.id); clearMedia(asset.id);
} }
@@ -505,8 +515,7 @@ function ensureAnimatedImage(asset) {
isAnimated: true isAnimated: true
}; };
fetch(asset.url) fetchAssetBlob(asset)
.then((r) => r.blob())
.then((blob) => new ImageDecoder({ data: blob, type: blob.type || 'image/gif' })) .then((blob) => new ImageDecoder({ data: blob, type: blob.type || 'image/gif' }))
.then((decoder) => { .then((decoder) => {
if (controller.cancelled) { if (controller.cancelled) {
@@ -519,12 +528,32 @@ function ensureAnimatedImage(asset) {
}) })
.catch(() => { .catch(() => {
animatedCache.delete(asset.id); animatedCache.delete(asset.id);
animationFailures.set(asset.id, Date.now());
}); });
animatedCache.set(asset.id, controller); animatedCache.set(asset.id, controller);
return controller; return controller;
} }
function fetchAssetBlob(asset) {
const cached = blobCache.get(asset.id);
if (cached && cached.url === asset.url && cached.blob) {
return Promise.resolve(cached.blob);
}
if (cached && cached.url === asset.url && cached.pending) {
return cached.pending;
}
const pending = fetch(asset.url)
.then((r) => r.blob())
.then((blob) => {
blobCache.set(asset.id, { url: asset.url, blob });
return blob;
});
blobCache.set(asset.id, { url: asset.url, pending });
return pending;
}
function scheduleNextFrame(controller) { function scheduleNextFrame(controller) {
if (controller.cancelled || !controller.decoder) { if (controller.cancelled || !controller.decoder) {
return; return;
@@ -559,6 +588,7 @@ function scheduleNextFrame(controller) {
}).catch(() => { }).catch(() => {
// If decoding fails, clear animated cache so static fallback is used next render // If decoding fails, clear animated cache so static fallback is used next render
animatedCache.delete(controller.id); animatedCache.delete(controller.id);
animationFailures.set(controller.id, Date.now());
}); });
} }

View File

@@ -57,6 +57,7 @@
<strong id="selected-asset-name">Choose an asset</strong> <strong id="selected-asset-name">Choose an asset</strong>
</div> </div>
<p class="meta-text" id="selected-asset-meta">Pick an asset in the list to adjust its placement and playback.</p> <p class="meta-text" id="selected-asset-meta">Pick an asset in the list to adjust its placement and playback.</p>
<p class="meta-text subtle-text hidden" id="selected-asset-id"></p>
<div class="badge-row asset-meta-badges" id="selected-asset-badges"></div> <div class="badge-row asset-meta-badges" id="selected-asset-badges"></div>
</div> </div>
<!-- <!--
@@ -116,19 +117,23 @@
<div class="panel-section" id="playback-section"> <div class="panel-section" id="playback-section">
<div class="section-header"> <div class="section-header">
<h5>Playback</h5> <h5>Playback</h5>
<p class="field-note">Video-only controls.</p>
</div> </div>
<div class="control-grid condensed"> <div class="stacked-field">
<label> <div class="label-row">
Animation speed <span>Animation speed</span>
<input id="asset-speed" class="range-input" type="range" min="0" max="1000" step="10" value="100" /> <span class="value-hint" id="asset-speed-label">100%</span>
</label> </div>
<input id="asset-speed" class="range-input" type="range" min="0" max="1000" step="10" value="100" />
<div class="range-meta"><span>0%</span><span>1000%</span></div> <div class="range-meta"><span>0%</span><span>1000%</span></div>
<label class="checkbox-inline toggle"> </div>
<div class="control-grid condensed split-row">
<label class="checkbox-inline toggle inline-toggle">
<input id="asset-muted" type="checkbox" /> <input id="asset-muted" type="checkbox" />
<span class="toggle-track" aria-hidden="true"> <span class="toggle-track" aria-hidden="true">
<span class="toggle-thumb"></span> <span class="toggle-thumb"></span>
</span> </span>
<span class="toggle-label">Mute</span> <span class="toggle-label">Mute video</span>
</label> </label>
</div> </div>
</div> </div>
@@ -137,8 +142,9 @@
<div class="section-header"> <div class="section-header">
<h5>Audio</h5> <h5>Audio</h5>
</div> </div>
<div class="control-grid condensed three-col"> <div class="control-grid condensed two-col">
<label class="checkbox-inline toggle">
<label class="checkbox-inline toggle inline-toggle">
<input id="asset-audio-loop" type="checkbox" /> <input id="asset-audio-loop" type="checkbox" />
<span class="toggle-track" aria-hidden="true"> <span class="toggle-track" aria-hidden="true">
<span class="toggle-thumb"></span> <span class="toggle-thumb"></span>
@@ -149,6 +155,16 @@
Delay (ms) Delay (ms)
<input id="asset-audio-delay" class="number-input" type="number" min="0" step="100" /> <input id="asset-audio-delay" class="number-input" type="number" min="0" step="100" />
</label> </label>
</div>
<div class="stacked-field">
<div class="label-row">
<span>Playback speed</span>
<span class="value-hint" id="asset-audio-speed-label">1.0x</span>
</div>
<input id="asset-audio-speed" class="range-input" type="range" min="25" max="400" step="5" value="100" />
<div class="range-meta"><span>0.25x</span><span>4x</span></div>
</div>
<div class="control-grid condensed two-col">
<label> <label>
Pitch (%) Pitch (%)
<input id="asset-audio-pitch" class="range-input" type="range" min="50" max="200" step="5" value="100" /> <input id="asset-audio-pitch" class="range-input" type="range" min="50" max="200" step="5" value="100" />
@@ -158,13 +174,6 @@
<input id="asset-audio-volume" class="range-input" type="range" min="0" max="100" step="1" value="100" /> <input id="asset-audio-volume" class="range-input" type="range" min="0" max="100" step="1" value="100" />
</label> </label>
</div> </div>
<div class="control-grid condensed">
<label>
Playback speed
<input id="asset-audio-speed" class="range-input" type="range" min="25" max="400" step="5" value="100" />
</label>
<div class="range-meta"><span>0.25x</span><span>4x</span></div>
</div>
</div> </div>
</div> </div>
</div> </div>