Use ffmpeg

This commit is contained in:
2026-01-22 21:53:36 +01:00
parent a8c4c97294
commit 10a7f5675d
14 changed files with 406 additions and 319 deletions

View File

@@ -9,6 +9,7 @@ RUN mvn -B package -DskipTests
FROM eclipse-temurin:17-jre FROM eclipse-temurin:17-jre
WORKDIR /app WORKDIR /app
RUN apt-get update && apt-get install -y ffmpeg && rm -rf /var/lib/apt/lists/*
COPY --from=build /app/target/imgfloat-*.jar app.jar COPY --from=build /app/target/imgfloat-*.jar app.jar
EXPOSE 8080 8443 EXPOSE 8080 8443
ENV JAVA_OPTS="" ENV JAVA_OPTS=""

12
pom.xml
View File

@@ -81,18 +81,6 @@
<artifactId>spring-boot-starter-validation</artifactId> <artifactId>spring-boot-starter-validation</artifactId>
</dependency> </dependency>
<dependency>
<groupId>org.jcodec</groupId>
<artifactId>jcodec</artifactId>
<version>0.2.5</version>
</dependency>
<dependency>
<groupId>org.jcodec</groupId>
<artifactId>jcodec-javase</artifactId>
<version>0.2.5</version>
</dependency>
<dependency> <dependency>
<groupId>org.springframework.session</groupId> <groupId>org.springframework.session</groupId>
<artifactId>spring-session-jdbc</artifactId> <artifactId>spring-session-jdbc</artifactId>

View File

@@ -1,10 +1,13 @@
{ pkgs ? import <nixpkgs> { } }: {
pkgs ? import <nixpkgs> { },
}:
pkgs.mkShell { pkgs.mkShell {
packages = [ packages = [
pkgs.jdt-language-server pkgs.jdt-language-server
pkgs.libxkbcommon pkgs.libxkbcommon
pkgs.maven pkgs.maven
pkgs.ffmpeg
pkgs.openjdk pkgs.openjdk
pkgs.git-lfs pkgs.git-lfs
]; ];

View File

@@ -3,8 +3,8 @@ package dev.kruhlmann.imgfloat.service;
import dev.kruhlmann.imgfloat.service.media.AssetContent; import dev.kruhlmann.imgfloat.service.media.AssetContent;
import java.io.IOException; import java.io.IOException;
import java.nio.file.*; import java.nio.file.*;
import dev.kruhlmann.imgfloat.service.media.MediaTypeRegistry;
import java.util.Locale; import java.util.Locale;
import java.util.Map;
import java.util.Optional; import java.util.Optional;
import java.util.Set; import java.util.Set;
import org.slf4j.Logger; import org.slf4j.Logger;
@@ -16,30 +16,7 @@ import org.springframework.stereotype.Service;
public class AssetStorageService { public class AssetStorageService {
private static final Logger logger = LoggerFactory.getLogger(AssetStorageService.class); private static final Logger logger = LoggerFactory.getLogger(AssetStorageService.class);
private static final Map<String, String> EXTENSIONS = Map.ofEntries( private static final String DEFAULT_PREVIEW_MEDIA_TYPE = "image/png";
Map.entry("image/png", ".png"),
Map.entry("image/jpeg", ".jpg"),
Map.entry("image/jpg", ".jpg"),
Map.entry("image/gif", ".gif"),
Map.entry("image/webp", ".webp"),
Map.entry("image/bmp", ".bmp"),
Map.entry("image/tiff", ".tiff"),
Map.entry("video/mp4", ".mp4"),
Map.entry("video/webm", ".webm"),
Map.entry("video/quicktime", ".mov"),
Map.entry("video/x-matroska", ".mkv"),
Map.entry("audio/mpeg", ".mp3"),
Map.entry("audio/mp3", ".mp3"),
Map.entry("audio/wav", ".wav"),
Map.entry("audio/ogg", ".ogg"),
Map.entry("audio/webm", ".webm"),
Map.entry("audio/flac", ".flac"),
Map.entry("model/gltf-binary", ".glb"),
Map.entry("model/gltf+json", ".gltf"),
Map.entry("model/obj", ".obj"),
Map.entry("application/javascript", ".js"),
Map.entry("text/javascript", ".js")
);
private final Path assetRoot; private final Path assetRoot;
private final Path previewRoot; private final Path previewRoot;
@@ -119,7 +96,7 @@ public class AssetStorageService {
if (!Files.exists(file)) return Optional.empty(); if (!Files.exists(file)) return Optional.empty();
byte[] bytes = Files.readAllBytes(file); byte[] bytes = Files.readAllBytes(file);
return Optional.of(new AssetContent(bytes, "image/png")); return Optional.of(new AssetContent(bytes, DEFAULT_PREVIEW_MEDIA_TYPE));
} catch (Exception e) { } catch (Exception e) {
logger.warn("Failed to load preview {}", assetId, e); logger.warn("Failed to load preview {}", assetId, e);
return Optional.empty(); return Optional.empty();
@@ -196,10 +173,9 @@ public class AssetStorageService {
} }
private String resolveExtension(String mediaType) throws IOException { private String resolveExtension(String mediaType) throws IOException {
if (mediaType == null || !EXTENSIONS.containsKey(mediaType)) { return MediaTypeRegistry
throw new IOException("Unsupported media type: " + mediaType); .extensionForMediaType(mediaType)
} .orElseThrow(() -> new IOException("Unsupported media type: " + mediaType));
return EXTENSIONS.get(mediaType);
} }
private Path assetPath(String broadcaster, String assetId, String mediaType) throws IOException { private Path assetPath(String broadcaster, String assetId, String mediaType) throws IOException {

View File

@@ -37,6 +37,7 @@ import dev.kruhlmann.imgfloat.service.media.AssetContent;
import dev.kruhlmann.imgfloat.service.media.MediaDetectionService; import dev.kruhlmann.imgfloat.service.media.MediaDetectionService;
import dev.kruhlmann.imgfloat.service.media.MediaOptimizationService; import dev.kruhlmann.imgfloat.service.media.MediaOptimizationService;
import dev.kruhlmann.imgfloat.service.media.OptimizedAsset; import dev.kruhlmann.imgfloat.service.media.OptimizedAsset;
import dev.kruhlmann.imgfloat.service.media.MediaTypeRegistry;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
@@ -303,11 +304,11 @@ public class ChannelDirectoryService {
byte[] bytes = file.getBytes(); byte[] bytes = file.getBytes();
String mediaType = mediaDetectionService String mediaType = mediaDetectionService
.detectAllowedMediaType(file, bytes) .detectAllowedMediaType(file, bytes)
.orElseThrow(() -> new ResponseStatusException(BAD_REQUEST, "Unsupported media type")); .orElseThrow(() -> new ResponseStatusException(BAD_REQUEST, unsupportedMediaTypeMessage()));
OptimizedAsset optimized = mediaOptimizationService.optimizeAsset(bytes, mediaType); OptimizedAsset optimized = mediaOptimizationService.optimizeAsset(bytes, mediaType);
if (optimized == null) { if (optimized == null) {
return Optional.empty(); throw new ResponseStatusException(BAD_REQUEST, mediaProcessingErrorMessage());
} }
String safeName = Optional.ofNullable(file.getOriginalFilename()) String safeName = Optional.ofNullable(file.getOriginalFilename())
@@ -499,10 +500,10 @@ public class ChannelDirectoryService {
enforceUploadLimit(bytes.length); enforceUploadLimit(bytes.length);
String mediaType = mediaDetectionService String mediaType = mediaDetectionService
.detectAllowedMediaType(file, bytes) .detectAllowedMediaType(file, bytes)
.orElseThrow(() -> new ResponseStatusException(BAD_REQUEST, "Unsupported media type")); .orElseThrow(() -> new ResponseStatusException(BAD_REQUEST, unsupportedMediaTypeMessage()));
OptimizedAsset optimized = mediaOptimizationService.optimizeAsset(bytes, mediaType); OptimizedAsset optimized = mediaOptimizationService.optimizeAsset(bytes, mediaType);
if (optimized == null) { if (optimized == null) {
return Optional.empty(); throw new ResponseStatusException(BAD_REQUEST, mediaProcessingErrorMessage());
} }
AssetType assetType = AssetType.fromMediaType(optimized.mediaType(), mediaType); AssetType assetType = AssetType.fromMediaType(optimized.mediaType(), mediaType);
if (assetType != AssetType.IMAGE) { if (assetType != AssetType.IMAGE) {
@@ -1344,11 +1345,11 @@ public class ChannelDirectoryService {
byte[] bytes = file.getBytes(); byte[] bytes = file.getBytes();
String mediaType = mediaDetectionService String mediaType = mediaDetectionService
.detectAllowedMediaType(file, bytes) .detectAllowedMediaType(file, bytes)
.orElseThrow(() -> new ResponseStatusException(BAD_REQUEST, "Unsupported media type")); .orElseThrow(() -> new ResponseStatusException(BAD_REQUEST, unsupportedMediaTypeMessage()));
OptimizedAsset optimized = mediaOptimizationService.optimizeAsset(bytes, mediaType); OptimizedAsset optimized = mediaOptimizationService.optimizeAsset(bytes, mediaType);
if (optimized == null) { if (optimized == null) {
return Optional.empty(); throw new ResponseStatusException(BAD_REQUEST, mediaProcessingErrorMessage());
} }
AssetType assetType = AssetType.fromMediaType(optimized.mediaType(), mediaType); AssetType assetType = AssetType.fromMediaType(optimized.mediaType(), mediaType);
@@ -1504,6 +1505,14 @@ public class ChannelDirectoryService {
return normalized.startsWith("application/javascript") || normalized.startsWith("text/javascript"); return normalized.startsWith("application/javascript") || normalized.startsWith("text/javascript");
} }
private String unsupportedMediaTypeMessage() {
return "Unsupported media type. Supported types: " + MediaTypeRegistry.supportedMediaTypesSummary();
}
private String mediaProcessingErrorMessage() {
return "Unable to process media. Ensure ffmpeg is installed on the server.";
}
private void validateCodeAssetSource(String source) { private void validateCodeAssetSource(String source) {
if (source == null || source.isBlank()) { if (source == null || source.isBlank()) {
throw new ResponseStatusException(BAD_REQUEST, "Script source is required"); throw new ResponseStatusException(BAD_REQUEST, "Script source is required");

View File

@@ -0,0 +1,168 @@
package dev.kruhlmann.imgfloat.service.media;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
@Service
public class FfmpegService {
private static final Logger logger = LoggerFactory.getLogger(FfmpegService.class);
public Optional<VideoDimensions> extractVideoDimensions(byte[] bytes) {
return Optional.ofNullable(withTempFile(bytes, ".bin", (input) -> {
List<String> command = List.of(
"ffprobe",
"-v",
"error",
"-select_streams",
"v:0",
"-show_entries",
"stream=width,height",
"-of",
"csv=p=0:s=x",
input.toString()
);
ProcessResult result = run(command);
if (result.exitCode() != 0) {
logger.warn("ffprobe failed: {}", result.output());
return null;
}
String output = result.output().trim();
if (output.isBlank() || !output.contains("x")) {
return null;
}
String[] parts = output.split("x", 2);
try {
int width = Integer.parseInt(parts[0].trim());
int height = Integer.parseInt(parts[1].trim());
return new VideoDimensions(width, height);
} catch (NumberFormatException e) {
logger.warn("Unable to parse ffprobe output: {}", output, e);
return null;
}
}));
}
public Optional<byte[]> extractVideoPreview(byte[] bytes) {
return Optional.ofNullable(withTempFile(bytes, ".bin", (input) -> {
Path output = Files.createTempFile("imgfloat-preview", ".png");
try {
List<String> command = List.of(
"ffmpeg",
"-y",
"-hide_banner",
"-loglevel",
"error",
"-i",
input.toString(),
"-frames:v",
"1",
"-f",
"image2",
"-vcodec",
"png",
output.toString()
);
ProcessResult result = run(command);
if (result.exitCode() != 0) {
logger.warn("ffmpeg preview failed: {}", result.output());
return null;
}
return Files.readAllBytes(output);
} finally {
Files.deleteIfExists(output);
}
}));
}
public Optional<byte[]> transcodeGifToMp4(byte[] bytes) {
return Optional.ofNullable(withTempFile(bytes, ".gif", (input) -> {
Path output = Files.createTempFile("imgfloat-transcode", ".mp4");
try {
List<String> command = List.of(
"ffmpeg",
"-y",
"-hide_banner",
"-loglevel",
"error",
"-i",
input.toString(),
"-movflags",
"+faststart",
"-pix_fmt",
"yuv420p",
"-vf",
"scale=trunc(iw/2)*2:trunc(ih/2)*2",
output.toString()
);
ProcessResult result = run(command);
if (result.exitCode() != 0) {
logger.warn("ffmpeg transcode failed: {}", result.output());
return null;
}
return Files.readAllBytes(output);
} finally {
Files.deleteIfExists(output);
}
}));
}
private <T> T withTempFile(byte[] bytes, String suffix, TempFileHandler<T> handler) {
Path input = null;
try {
input = Files.createTempFile("imgfloat-input", suffix);
Files.write(input, bytes);
return handler.handle(input);
} catch (IOException e) {
logger.warn("Unable to create temporary media file", e);
return null;
} finally {
if (input != null) {
try {
Files.deleteIfExists(input);
} catch (IOException e) {
logger.warn("Unable to delete temporary file {}", input, e);
}
}
}
}
private ProcessResult run(List<String> command) throws IOException {
ProcessBuilder builder = new ProcessBuilder(new ArrayList<>(command));
builder.redirectErrorStream(true);
Process process = builder.start();
String output;
try (InputStream stream = process.getInputStream(); ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
stream.transferTo(baos);
output = baos.toString(StandardCharsets.UTF_8);
} catch (IOException e) {
output = "";
}
int exitCode;
try {
exitCode = process.waitFor();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
exitCode = -1;
}
return new ProcessResult(exitCode, output);
}
private interface TempFileHandler<T> {
T handle(Path input) throws IOException;
}
private record ProcessResult(int exitCode, String output) {}
public record VideoDimensions(int width, int height) {}
}

View File

@@ -4,9 +4,7 @@ import java.io.ByteArrayInputStream;
import java.io.IOException; import java.io.IOException;
import java.net.URLConnection; import java.net.URLConnection;
import java.util.Locale; import java.util.Locale;
import java.util.Map;
import java.util.Optional; import java.util.Optional;
import java.util.Set;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
@@ -16,46 +14,27 @@ import org.springframework.web.multipart.MultipartFile;
public class MediaDetectionService { public class MediaDetectionService {
private static final Logger logger = LoggerFactory.getLogger(MediaDetectionService.class); private static final Logger logger = LoggerFactory.getLogger(MediaDetectionService.class);
private static final Map<String, String> EXTENSION_TYPES = Map.ofEntries(
Map.entry("png", "image/png"),
Map.entry("jpg", "image/jpeg"),
Map.entry("jpeg", "image/jpeg"),
Map.entry("gif", "image/gif"),
Map.entry("webp", "image/webp"),
Map.entry("mp4", "video/mp4"),
Map.entry("webm", "video/webm"),
Map.entry("mov", "video/quicktime"),
Map.entry("mp3", "audio/mpeg"),
Map.entry("wav", "audio/wav"),
Map.entry("ogg", "audio/ogg"),
Map.entry("glb", "model/gltf-binary"),
Map.entry("gltf", "model/gltf+json"),
Map.entry("obj", "model/obj"),
Map.entry("js", "application/javascript"),
Map.entry("mjs", "text/javascript")
);
private static final Set<String> ALLOWED_MEDIA_TYPES = Set.copyOf(EXTENSION_TYPES.values());
public Optional<String> detectAllowedMediaType(MultipartFile file, byte[] bytes) { public Optional<String> detectAllowedMediaType(MultipartFile file, byte[] bytes) {
Optional<String> detected = detectMediaType(bytes) Optional<String> detected = detectMediaType(bytes)
.map(MediaDetectionService::normalizeJavaScriptMediaType) .map(MediaTypeRegistry::normalizeJavaScriptMediaType)
.filter(MediaDetectionService::isAllowedMediaType); .filter(MediaTypeRegistry::isSupportedMediaType);
if (detected.isPresent()) { if (detected.isPresent()) {
return detected; return detected;
} }
Optional<String> declared = Optional.ofNullable(file.getContentType()) Optional<String> declared = Optional.ofNullable(file.getContentType())
.map(MediaDetectionService::normalizeJavaScriptMediaType) .map(MediaTypeRegistry::normalizeJavaScriptMediaType)
.filter(MediaDetectionService::isAllowedMediaType); .filter(MediaTypeRegistry::isSupportedMediaType);
if (declared.isPresent()) { if (declared.isPresent()) {
return declared; return declared;
} }
return Optional.ofNullable(file.getOriginalFilename()) return Optional.ofNullable(file.getOriginalFilename())
.map((name) -> name.replaceAll("^.*\\.", "").toLowerCase()) .map((name) -> name.replaceAll("^.*\\.", "").toLowerCase(Locale.ROOT))
.map(EXTENSION_TYPES::get) .flatMap(MediaTypeRegistry::mediaTypeForExtension)
.filter(MediaDetectionService::isAllowedMediaType); .filter(MediaTypeRegistry::isSupportedMediaType);
} }
private Optional<String> detectMediaType(byte[] bytes) { private Optional<String> detectMediaType(byte[] bytes) {
@@ -72,8 +51,7 @@ public class MediaDetectionService {
} }
public static boolean isAllowedMediaType(String mediaType) { public static boolean isAllowedMediaType(String mediaType) {
String normalized = normalizeJavaScriptMediaType(mediaType); return MediaTypeRegistry.isSupportedMediaType(mediaType);
return normalized != null && ALLOWED_MEDIA_TYPES.contains(normalized.toLowerCase());
} }
public static boolean isInlineDisplayType(String mediaType) { public static boolean isInlineDisplayType(String mediaType) {
@@ -82,15 +60,4 @@ public class MediaDetectionService {
(mediaType.startsWith("image/") || mediaType.startsWith("video/") || mediaType.startsWith("audio/")) (mediaType.startsWith("image/") || mediaType.startsWith("video/") || mediaType.startsWith("audio/"))
); );
} }
private static String normalizeJavaScriptMediaType(String mediaType) {
if (mediaType == null) {
return null;
}
String normalized = mediaType.toLowerCase(Locale.ROOT);
if (normalized.contains("javascript") || normalized.contains("ecmascript")) {
return "application/javascript";
}
return mediaType;
}
} }

View File

@@ -1,25 +1,9 @@
package dev.kruhlmann.imgfloat.service.media; package dev.kruhlmann.imgfloat.service.media;
import java.awt.Graphics2D;
import java.awt.image.BufferedImage; import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.util.List;
import javax.imageio.IIOImage;
import javax.imageio.ImageIO; import javax.imageio.ImageIO;
import javax.imageio.ImageWriteParam;
import javax.imageio.ImageWriter;
import javax.imageio.metadata.IIOMetadata;
import javax.imageio.stream.ImageInputStream;
import javax.imageio.stream.ImageOutputStream;
import org.jcodec.api.FrameGrab;
import org.jcodec.api.JCodecException;
import org.jcodec.common.io.ByteBufferSeekableByteChannel;
import org.jcodec.common.model.Picture;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
@@ -27,12 +11,18 @@ import org.springframework.stereotype.Service;
@Service @Service
public class MediaOptimizationService { public class MediaOptimizationService {
private static final int MIN_GIF_DELAY_MS = 20;
private static final Logger logger = LoggerFactory.getLogger(MediaOptimizationService.class); private static final Logger logger = LoggerFactory.getLogger(MediaOptimizationService.class);
private final MediaPreviewService previewService; private static final FfmpegService.VideoDimensions DEFAULT_VIDEO_DIMENSIONS = new FfmpegService.VideoDimensions(
640,
360
);
public MediaOptimizationService(MediaPreviewService previewService) { private final MediaPreviewService previewService;
private final FfmpegService ffmpegService;
public MediaOptimizationService(MediaPreviewService previewService, FfmpegService ffmpegService) {
this.previewService = previewService; this.previewService = previewService;
this.ffmpegService = ffmpegService;
} }
public OptimizedAsset optimizeAsset(byte[] bytes, String mediaType) throws IOException { public OptimizedAsset optimizeAsset(byte[] bytes, String mediaType) throws IOException {
@@ -46,15 +36,6 @@ public class MediaOptimizationService {
} }
} }
if (mediaType.startsWith("image/") && !"image/gif".equalsIgnoreCase(mediaType)) {
BufferedImage image = ImageIO.read(new ByteArrayInputStream(bytes));
if (image == null) {
return null;
}
byte[] compressed = compressPng(image);
return new OptimizedAsset(compressed, "image/png", image.getWidth(), image.getHeight(), null);
}
if (mediaType.startsWith("image/")) { if (mediaType.startsWith("image/")) {
BufferedImage image = ImageIO.read(new ByteArrayInputStream(bytes)); BufferedImage image = ImageIO.read(new ByteArrayInputStream(bytes));
if (image == null) { if (image == null) {
@@ -64,7 +45,9 @@ public class MediaOptimizationService {
} }
if (mediaType.startsWith("video/")) { if (mediaType.startsWith("video/")) {
var dimensions = extractVideoDimensions(bytes); FfmpegService.VideoDimensions dimensions = ffmpegService
.extractVideoDimensions(bytes)
.orElse(DEFAULT_VIDEO_DIMENSIONS);
byte[] preview = previewService.extractVideoPreview(bytes, mediaType); byte[] preview = previewService.extractVideoPreview(bytes, mediaType);
return new OptimizedAsset(bytes, mediaType, dimensions.width(), dimensions.height(), preview); return new OptimizedAsset(bytes, mediaType, dimensions.width(), dimensions.height(), preview);
} }
@@ -89,165 +72,18 @@ public class MediaOptimizationService {
} }
private OptimizedAsset transcodeGifToVideo(byte[] bytes) { private OptimizedAsset transcodeGifToVideo(byte[] bytes) {
try { return ffmpegService
List<GifFrame> frames = readGifFrames(bytes); .transcodeGifToMp4(bytes)
if (frames.isEmpty()) { .map((videoBytes) -> {
FfmpegService.VideoDimensions dimensions = ffmpegService
.extractVideoDimensions(videoBytes)
.orElse(DEFAULT_VIDEO_DIMENSIONS);
byte[] preview = previewService.extractVideoPreview(videoBytes, "video/mp4");
return new OptimizedAsset(videoBytes, "video/mp4", dimensions.width(), dimensions.height(), preview);
})
.orElseGet(() -> {
logger.warn("Unable to transcode GIF to video via ffmpeg");
return null; return null;
} });
int baseDelay = frames
.stream()
.mapToInt((frame) -> normalizeDelay(frame.delayMs()))
.reduce(this::greatestCommonDivisor)
.orElse(100);
int fps = Math.max(1, (int) Math.round(1000.0 / baseDelay));
File temp = File.createTempFile("gif-convert", ".mp4");
temp.deleteOnExit();
try {
var encoder = org.jcodec.api.awt.AWTSequenceEncoder.createSequenceEncoder(temp, fps);
for (GifFrame frame : frames) {
BufferedImage image = ensureEvenDimensions(frame.image());
int repeats = Math.max(1, normalizeDelay(frame.delayMs()) / baseDelay);
for (int i = 0; i < repeats; i++) {
encoder.encodeImage(image);
} }
} }
encoder.finish();
BufferedImage cover = ensureEvenDimensions(frames.get(0).image());
byte[] video = Files.readAllBytes(temp.toPath());
return new OptimizedAsset(
video,
"video/mp4",
cover.getWidth(),
cover.getHeight(),
previewService.encodePreview(cover)
);
} finally {
Files.deleteIfExists(temp.toPath());
}
} catch (IOException e) {
logger.warn("Unable to transcode GIF to video", e);
return null;
}
}
private List<GifFrame> readGifFrames(byte[] bytes) throws IOException {
try (ImageInputStream stream = ImageIO.createImageInputStream(new ByteArrayInputStream(bytes))) {
var readers = ImageIO.getImageReadersByFormatName("gif");
if (!readers.hasNext()) {
return List.of();
}
var reader = readers.next();
try {
reader.setInput(stream, false, false);
int count = reader.getNumImages(true);
var frames = new java.util.ArrayList<GifFrame>(count);
for (int i = 0; i < count; i++) {
BufferedImage image = reader.read(i);
IIOMetadata metadata = reader.getImageMetadata(i);
int delay = extractDelayMs(metadata);
frames.add(new GifFrame(image, delay));
}
return frames;
} finally {
reader.dispose();
}
}
}
private int extractDelayMs(IIOMetadata metadata) {
if (metadata == null) {
return 100;
}
try {
String format = metadata.getNativeMetadataFormatName();
var root = metadata.getAsTree(format);
var children = root.getChildNodes();
for (int i = 0; i < children.getLength(); i++) {
var node = children.item(i);
if ("GraphicControlExtension".equals(node.getNodeName()) && node.getAttributes() != null) {
var delay = node.getAttributes().getNamedItem("delayTime");
if (delay != null) {
int hundredths = Integer.parseInt(delay.getNodeValue());
return Math.max(hundredths * 10, MIN_GIF_DELAY_MS);
}
}
}
} catch (Exception e) {
logger.warn("Unable to parse GIF delay", e);
}
return 100;
}
private int normalizeDelay(int delayMs) {
return Math.max(delayMs, MIN_GIF_DELAY_MS);
}
private int greatestCommonDivisor(int a, int b) {
if (b == 0) {
return Math.max(a, 1);
}
return greatestCommonDivisor(b, a % b);
}
private byte[] compressPng(BufferedImage image) throws IOException {
var writers = ImageIO.getImageWritersByFormatName("png");
if (!writers.hasNext()) {
logger.warn("No PNG writer available; skipping compression");
try (ByteArrayOutputStream fallback = new ByteArrayOutputStream()) {
ImageIO.write(image, "png", fallback);
return fallback.toByteArray();
}
}
ImageWriter writer = writers.next();
try (
ByteArrayOutputStream baos = new ByteArrayOutputStream();
ImageOutputStream ios = ImageIO.createImageOutputStream(baos)
) {
writer.setOutput(ios);
ImageWriteParam param = writer.getDefaultWriteParam();
if (param.canWriteCompressed()) {
param.setCompressionMode(ImageWriteParam.MODE_EXPLICIT);
param.setCompressionQuality(1.0f);
}
writer.write(null, new IIOImage(image, null, null), param);
return baos.toByteArray();
} finally {
writer.dispose();
}
}
private BufferedImage ensureEvenDimensions(BufferedImage image) {
int width = image.getWidth();
int height = image.getHeight();
int evenWidth = width % 2 == 0 ? width : width + 1;
int evenHeight = height % 2 == 0 ? height : height + 1;
if (evenWidth == width && evenHeight == height) {
return image;
}
BufferedImage padded = new BufferedImage(evenWidth, evenHeight, BufferedImage.TYPE_INT_ARGB);
Graphics2D graphics = padded.createGraphics();
try {
graphics.drawImage(image, 0, 0, null);
} finally {
graphics.dispose();
}
return padded;
}
private Dimension extractVideoDimensions(byte[] bytes) {
try (var channel = new ByteBufferSeekableByteChannel(ByteBuffer.wrap(bytes), bytes.length)) {
FrameGrab grab = FrameGrab.createFrameGrab(channel);
Picture frame = grab.getNativeFrame();
if (frame != null) {
return new Dimension(frame.getWidth(), frame.getHeight());
}
} catch (IOException | JCodecException e) {
logger.warn("Unable to read video dimensions", e);
}
return new Dimension(640, 360);
}
private record GifFrame(BufferedImage image, int delayMs) {}
private record Dimension(int width, int height) {}
}

View File

@@ -1,15 +1,5 @@
package dev.kruhlmann.imgfloat.service.media; package dev.kruhlmann.imgfloat.service.media;
import java.awt.image.BufferedImage;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import javax.imageio.ImageIO;
import org.jcodec.api.FrameGrab;
import org.jcodec.api.JCodecException;
import org.jcodec.common.io.ByteBufferSeekableByteChannel;
import org.jcodec.common.model.Picture;
import org.jcodec.scale.AWTUtil;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
@@ -19,31 +9,18 @@ public class MediaPreviewService {
private static final Logger logger = LoggerFactory.getLogger(MediaPreviewService.class); private static final Logger logger = LoggerFactory.getLogger(MediaPreviewService.class);
public byte[] encodePreview(BufferedImage image) { private final FfmpegService ffmpegService;
if (image == null) {
return null; public MediaPreviewService(FfmpegService ffmpegService) {
} this.ffmpegService = ffmpegService;
try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
ImageIO.write(image, "png", baos);
return baos.toByteArray();
} catch (IOException e) {
logger.warn("Unable to encode preview image", e);
return null;
}
} }
public byte[] extractVideoPreview(byte[] bytes, String mediaType) { public byte[] extractVideoPreview(byte[] bytes, String mediaType) {
try (var channel = new ByteBufferSeekableByteChannel(ByteBuffer.wrap(bytes), bytes.length)) { return ffmpegService
FrameGrab grab = FrameGrab.createFrameGrab(channel); .extractVideoPreview(bytes)
Picture frame = grab.getNativeFrame(); .orElseGet(() -> {
if (frame == null) { logger.warn("Unable to capture video preview frame for {}", mediaType);
return null;
}
BufferedImage image = AWTUtil.toBufferedImage(frame);
return encodePreview(image);
} catch (IOException | JCodecException e) {
logger.warn("Unable to capture video preview frame for {}", mediaType, e);
return null; return null;
} });
} }
} }

View File

@@ -0,0 +1,111 @@
package dev.kruhlmann.imgfloat.service.media;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
public final class MediaTypeRegistry {
private static final Map<String, String> EXTENSION_TO_MEDIA_TYPE = buildExtensionMap();
private static final Map<String, String> MEDIA_TYPE_TO_EXTENSION = buildMediaTypeMap();
private static final Set<String> SUPPORTED_MEDIA_TYPES = Set.copyOf(MEDIA_TYPE_TO_EXTENSION.keySet());
private MediaTypeRegistry() {}
public static Optional<String> mediaTypeForExtension(String extension) {
if (extension == null) {
return Optional.empty();
}
return Optional.ofNullable(EXTENSION_TO_MEDIA_TYPE.get(extension.toLowerCase(Locale.ROOT)));
}
public static Optional<String> extensionForMediaType(String mediaType) {
if (mediaType == null) {
return Optional.empty();
}
String normalized = normalizeJavaScriptMediaType(mediaType).toLowerCase(Locale.ROOT);
return Optional.ofNullable(MEDIA_TYPE_TO_EXTENSION.get(normalized));
}
public static boolean isSupportedMediaType(String mediaType) {
if (mediaType == null || mediaType.isBlank()) {
return false;
}
String normalized = normalizeJavaScriptMediaType(mediaType).toLowerCase(Locale.ROOT);
return SUPPORTED_MEDIA_TYPES.contains(normalized);
}
public static List<String> supportedMediaTypes() {
return SUPPORTED_MEDIA_TYPES.stream().sorted().toList();
}
public static String supportedMediaTypesSummary() {
return String.join(", ", supportedMediaTypes());
}
public static String normalizeJavaScriptMediaType(String mediaType) {
if (mediaType == null) {
return null;
}
String normalized = mediaType.toLowerCase(Locale.ROOT);
if (normalized.contains("javascript") || normalized.contains("ecmascript")) {
return "application/javascript";
}
return mediaType;
}
private static Map<String, String> buildExtensionMap() {
Map<String, String> map = new LinkedHashMap<>();
map.put("png", "image/png");
map.put("jpg", "image/jpeg");
map.put("jpeg", "image/jpeg");
map.put("gif", "image/gif");
map.put("webp", "image/webp");
map.put("bmp", "image/bmp");
map.put("tiff", "image/tiff");
map.put("mp4", "video/mp4");
map.put("webm", "video/webm");
map.put("mov", "video/quicktime");
map.put("mkv", "video/x-matroska");
map.put("mp3", "audio/mpeg");
map.put("wav", "audio/wav");
map.put("ogg", "audio/ogg");
map.put("flac", "audio/flac");
map.put("glb", "model/gltf-binary");
map.put("gltf", "model/gltf+json");
map.put("obj", "model/obj");
map.put("js", "application/javascript");
map.put("mjs", "text/javascript");
return Map.copyOf(map);
}
private static Map<String, String> buildMediaTypeMap() {
Map<String, String> map = new LinkedHashMap<>();
map.put("image/png", ".png");
map.put("image/jpeg", ".jpg");
map.put("image/jpg", ".jpg");
map.put("image/gif", ".gif");
map.put("image/webp", ".webp");
map.put("image/bmp", ".bmp");
map.put("image/tiff", ".tiff");
map.put("video/mp4", ".mp4");
map.put("video/webm", ".webm");
map.put("video/quicktime", ".mov");
map.put("video/x-matroska", ".mkv");
map.put("audio/mpeg", ".mp3");
map.put("audio/mp3", ".mp3");
map.put("audio/wav", ".wav");
map.put("audio/ogg", ".ogg");
map.put("audio/webm", ".webm");
map.put("audio/flac", ".flac");
map.put("model/gltf-binary", ".glb");
map.put("model/gltf+json", ".gltf");
map.put("model/obj", ".obj");
map.put("application/javascript", ".js");
map.put("text/javascript", ".js");
return Map.copyOf(map);
}
}

View File

@@ -2406,7 +2406,9 @@ export function createAdminConsole({
}) })
.then((response) => { .then((response) => {
if (!response.ok) { if (!response.ok) {
throw new Error("Upload failed"); return extractErrorMessage(response, "Upload failed").then((message) => {
throw new Error(message);
});
} }
if (fileInput) { if (fileInput) {
fileInput.value = ""; fileInput.value = "";
@@ -2421,10 +2423,31 @@ export function createAdminConsole({
} }
console.error(e); console.error(e);
removePendingUpload(pendingId); removePendingUpload(pendingId);
showToast("Upload failed. Please try again with a supported file.", "error"); showToast(e?.message || "Upload failed. Please try again with a supported file.", "error");
}); });
} }
function extractErrorMessage(response, fallback) {
if (!response) {
return Promise.resolve(fallback);
}
return response
.json()
.then((data) => {
if (data?.message) {
return data.message;
}
if (data?.error) {
return data.error;
}
if (typeof data === "string" && data.trim()) {
return data;
}
return fallback;
})
.catch(() => response.text().then((text) => text?.trim() || fallback).catch(() => fallback));
}
function getCanvasPoint(event) { function getCanvasPoint(event) {
const rect = canvas.getBoundingClientRect(); const rect = canvas.getBoundingClientRect();
const scaleX = canvas.width / rect.width; const scaleX = canvas.width / rect.width;

View File

@@ -390,7 +390,7 @@ export function createCustomAssetModal({
} }
}) })
.catch((e) => { .catch((e) => {
showToast?.("Unable to save custom asset. Please try again.", "error"); showToast?.(e?.message || "Unable to save custom asset. Please try again.", "error");
console.error(e); console.error(e);
}) })
.finally(() => { .finally(() => {
@@ -498,7 +498,7 @@ export function createCustomAssetModal({
}) })
.catch((error) => { .catch((error) => {
console.error(error); console.error(error);
showToast?.("Unable to upload attachment. Please try again.", "error"); showToast?.(error?.message || "Unable to upload attachment. Please try again.", "error");
}) })
.finally(() => { .finally(() => {
attachmentInput.value = ""; attachmentInput.value = "";
@@ -591,12 +591,35 @@ export function createCustomAssetModal({
body: payload, body: payload,
}).then((response) => { }).then((response) => {
if (!response.ok) { if (!response.ok) {
throw new Error("Failed to upload attachment"); return extractErrorMessage(response, "Failed to upload attachment").then((message) => {
throw new Error(message);
});
} }
return response.json(); return response.json();
}); });
} }
function extractErrorMessage(response, fallback) {
if (!response) {
return Promise.resolve(fallback);
}
return response
.json()
.then((data) => {
if (data?.message) {
return data.message;
}
if (data?.error) {
return data.error;
}
if (typeof data === "string" && data.trim()) {
return data;
}
return fallback;
})
.catch(() => response.text().then((text) => text?.trim() || fallback).catch(() => fallback));
}
function removeAttachment(attachmentId) { function removeAttachment(attachmentId) {
if (!attachmentId || !currentAssetId) { if (!attachmentId || !currentAssetId) {
return; return;
@@ -694,7 +717,9 @@ export function createCustomAssetModal({
body: payload, body: payload,
}).then((response) => { }).then((response) => {
if (!response.ok) { if (!response.ok) {
throw new Error("Failed to upload logo"); return extractErrorMessage(response, "Failed to upload logo").then((message) => {
throw new Error(message);
});
} }
pendingLogoFile = null; pendingLogoFile = null;
return response.json(); return response.json();

View File

@@ -33,6 +33,7 @@ import dev.kruhlmann.imgfloat.service.SettingsService;
import dev.kruhlmann.imgfloat.service.media.MediaDetectionService; import dev.kruhlmann.imgfloat.service.media.MediaDetectionService;
import dev.kruhlmann.imgfloat.service.media.MediaOptimizationService; import dev.kruhlmann.imgfloat.service.media.MediaOptimizationService;
import dev.kruhlmann.imgfloat.service.media.MediaPreviewService; import dev.kruhlmann.imgfloat.service.media.MediaPreviewService;
import dev.kruhlmann.imgfloat.service.media.FfmpegService;
import java.awt.image.BufferedImage; import java.awt.image.BufferedImage;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import java.io.IOException; import java.io.IOException;
@@ -88,8 +89,9 @@ class ChannelDirectoryServiceTest {
Path assetRoot = Files.createTempDirectory("imgfloat-assets-test"); Path assetRoot = Files.createTempDirectory("imgfloat-assets-test");
Path previewRoot = Files.createTempDirectory("imgfloat-previews-test"); Path previewRoot = Files.createTempDirectory("imgfloat-previews-test");
AssetStorageService assetStorageService = new AssetStorageService(assetRoot.toString(), previewRoot.toString()); AssetStorageService assetStorageService = new AssetStorageService(assetRoot.toString(), previewRoot.toString());
MediaPreviewService mediaPreviewService = new MediaPreviewService(); FfmpegService ffmpegService = new FfmpegService();
MediaOptimizationService mediaOptimizationService = new MediaOptimizationService(mediaPreviewService); MediaPreviewService mediaPreviewService = new MediaPreviewService(ffmpegService);
MediaOptimizationService mediaOptimizationService = new MediaOptimizationService(mediaPreviewService, ffmpegService);
MediaDetectionService mediaDetectionService = new MediaDetectionService(); MediaDetectionService mediaDetectionService = new MediaDetectionService();
long uploadLimitBytes = 5_000_000L; long uploadLimitBytes = 5_000_000L;
Path marketplaceRoot = Files.createTempDirectory("imgfloat-marketplace-test"); Path marketplaceRoot = Files.createTempDirectory("imgfloat-marketplace-test");

View File

@@ -15,7 +15,8 @@ class MediaOptimizationServiceTest {
@BeforeEach @BeforeEach
void setUp() { void setUp() {
service = new MediaOptimizationService(new MediaPreviewService()); FfmpegService ffmpegService = new FfmpegService();
service = new MediaOptimizationService(new MediaPreviewService(ffmpegService), ffmpegService);
} }
@Test @Test