imgfloat/server
1
0
Fork 0
mirror of https://github.com/imgfloat/server.git synced 2026-02-05 03:39:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve rdd proc

Browse Source
This commit is contained in:
Andreas Krühlmann
2025-12-10 15:46:04 +01:00
parent 2bec770b4e
commit 0bca3283fa
11 changed files with 413 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/main/java/com/imgfloat/app/config/SchemaMigration.java
View File

@@ -66,6 +66,7 @@ public class SchemaMigration implements ApplicationRunner {
addColumnIfMissing("assets", columns, "audio_speed", "REAL", "1.0");
addColumnIfMissing("assets", columns, "audio_pitch", "REAL", "1.0");
addColumnIfMissing("assets", columns, "audio_volume", "REAL", "1.0");
addColumnIfMissing("assets", columns, "preview", "TEXT", "NULL");
}
private void addColumnIfMissing(String tableName, List<String> existingColumns, String columnName, String dataType, String defaultValue) {

27
src/main/java/com/imgfloat/app/controller/ChannelApiController.java
View File

@@ -231,6 +231,33 @@ public class ChannelApiController {
.orElseThrow(() -> new ResponseStatusException(FORBIDDEN, "Asset not available"));
}
@GetMapping("/assets/{assetId}/preview")
public ResponseEntity<byte[]> getAssetPreview(@PathVariable("broadcaster") String broadcaster,
@PathVariable("assetId") String assetId,
OAuth2AuthenticationToken authentication) {
boolean authorized = false;
if (authentication != null) {
String login = TwitchUser.from(authentication).login();
authorized = channelDirectoryService.isBroadcaster(broadcaster, login)
|| channelDirectoryService.isAdmin(broadcaster, login);
}
if (authorized) {
LOG.debug("Serving preview for asset {} for broadcaster {}", assetId, broadcaster);
return channelDirectoryService.getAssetPreview(broadcaster, assetId, true)
.map(content -> ResponseEntity.ok()
.contentType(MediaType.parseMediaType(content.mediaType()))
.body(content.bytes()))
.orElseThrow(() -> new ResponseStatusException(NOT_FOUND, "Preview not found"));
}
return channelDirectoryService.getAssetPreview(broadcaster, assetId, false)
.map(content -> ResponseEntity.ok()
.contentType(MediaType.parseMediaType(content.mediaType()))
.body(content.bytes()))
.orElseThrow(() -> new ResponseStatusException(FORBIDDEN, "Preview not available"));
}
@DeleteMapping("/assets/{assetId}")
public ResponseEntity<?> delete(@PathVariable("broadcaster") String broadcaster,
@PathVariable("assetId") String assetId,

10
src/main/java/com/imgfloat/app/model/Asset.java
View File

@@ -34,6 +34,8 @@ public class Asset {
private Boolean muted;
private String mediaType;
private String originalMediaType;
@Column(columnDefinition = "TEXT")
private String preview;
private Integer zIndex;
private Boolean audioLoop;
private Integer audioDelayMillis;
@@ -202,6 +204,14 @@ public class Asset {
this.originalMediaType = originalMediaType;
}
public String getPreview() {
return preview;
}
public void setPreview(String preview) {
this.preview = preview;
}
public boolean isVideo() {
return mediaType != null && mediaType.toLowerCase(Locale.ROOT).startsWith("video/");
}

4
src/main/java/com/imgfloat/app/model/AssetView.java
View File

@@ -7,6 +7,7 @@ public record AssetView(
String broadcaster,
String name,
String url,
String previewUrl,
double x,
double y,
double width,
@@ -23,6 +24,7 @@ public record AssetView(
Double audioPitch,
Double audioVolume,
boolean hidden,
boolean hasPreview,
Instant createdAt
) {
public static AssetView from(String broadcaster, Asset asset) {
@@ -31,6 +33,7 @@ public record AssetView(
asset.getBroadcaster(),
asset.getName(),
"/api/channels/" + broadcaster + "/assets/" + asset.getId() + "/content",
asset.getPreview() != null ? "/api/channels/" + broadcaster + "/assets/" + asset.getId() + "/preview" : null,
asset.getX(),
asset.getY(),
asset.getWidth(),
@@ -47,6 +50,7 @@ public record AssetView(
asset.getAudioPitch(),
asset.getAudioVolume(),
asset.isHidden(),
asset.getPreview() != null,
asset.getCreatedAt()
);
}

82
src/main/java/com/imgfloat/app/service/ChannelDirectoryService.java
View File

@@ -15,6 +15,7 @@ import org.jcodec.api.JCodecException;
import org.jcodec.api.awt.AWTSequenceEncoder;
import org.jcodec.common.io.ByteBufferSeekableByteChannel;
import org.jcodec.common.model.Picture;
import org.jcodec.scale.AWTUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.messaging.simp.SimpMessagingTemplate;
@@ -131,6 +132,7 @@ public class ChannelDirectoryService {
Asset asset = new Asset(channel.getBroadcaster(), name, dataUrl, width, height);
asset.setOriginalMediaType(mediaType);
asset.setMediaType(optimized.mediaType());
asset.setPreview(optimized.previewDataUrl());
asset.setSpeed(1.0);
asset.setMuted(optimized.mediaType().startsWith("video/"));
asset.setAudioLoop(false);
@@ -240,6 +242,24 @@ public class ChannelDirectoryService {
.flatMap(this::decodeAssetData);
}
public Optional<AssetContent> getAssetPreview(String broadcaster, String assetId, boolean includeHidden) {
String normalized = normalize(broadcaster);
return assetRepository.findById(assetId)
.filter(asset -> normalized.equals(asset.getBroadcaster()))
.filter(asset -> includeHidden || !asset.isHidden())
.map(asset -> {
Optional<AssetContent> preview = decodeDataUrl(asset.getPreview());
if (preview.isPresent()) {
return preview.get();
}
if (asset.getMediaType() != null && asset.getMediaType().startsWith("image/")) {
return decodeAssetData(asset).orElse(null);
}
return null;
})
.flatMap(Optional::ofNullable);
}
public boolean isBroadcaster(String broadcaster, String username) {
return broadcaster != null && broadcaster.equalsIgnoreCase(username);
}
@@ -279,23 +299,30 @@ public class ChannelDirectoryService {
}
private Optional<AssetContent> decodeAssetData(Asset asset) {
String url = asset.getUrl();
if (url == null || !url.startsWith("data:")) {
return decodeDataUrl(asset.getUrl())
.or(() -> {
logger.warn("Unable to decode asset data for {}", asset.getId());
return Optional.empty();
});
}
private Optional<AssetContent> decodeDataUrl(String dataUrl) {
if (dataUrl == null || !dataUrl.startsWith("data:")) {
return Optional.empty();
}
int commaIndex = url.indexOf(',');
int commaIndex = dataUrl.indexOf(',');
if (commaIndex < 0) {
return Optional.empty();
}
String metadata = url.substring(5, commaIndex);
String metadata = dataUrl.substring(5, commaIndex);
String[] parts = metadata.split(";", 2);
String mediaType = parts.length > 0 && !parts[0].isBlank() ? parts[0] : "application/octet-stream";
String encoded = url.substring(commaIndex + 1);
String encoded = dataUrl.substring(commaIndex + 1);
try {
byte[] bytes = Base64.getDecoder().decode(encoded);
return Optional.of(new AssetContent(bytes, mediaType));
} catch (IllegalArgumentException e) {
logger.warn("Unable to decode asset data for {}", asset.getId(), e);
logger.warn("Unable to decode data url", e);
return Optional.empty();
}
}
@@ -353,7 +380,7 @@ public class ChannelDirectoryService {
return null;
}
byte[] compressed = compressPng(image);
return new OptimizedAsset(compressed, "image/png", image.getWidth(), image.getHeight());
return new OptimizedAsset(compressed, "image/png", image.getWidth(), image.getHeight(), null);
}
if (mediaType.startsWith("image/")) {
@@ -361,21 +388,22 @@ public class ChannelDirectoryService {
if (image == null) {
return null;
}
return new OptimizedAsset(bytes, mediaType, image.getWidth(), image.getHeight());
return new OptimizedAsset(bytes, mediaType, image.getWidth(), image.getHeight(), null);
}
if (mediaType.startsWith("video/")) {
var dimensions = extractVideoDimensions(bytes);
return new OptimizedAsset(bytes, mediaType, dimensions.width(), dimensions.height());
String preview = extractVideoPreview(bytes, mediaType);
return new OptimizedAsset(bytes, mediaType, dimensions.width(), dimensions.height(), preview);
}
if (mediaType.startsWith("audio/")) {
return new OptimizedAsset(bytes, mediaType, 0, 0);
return new OptimizedAsset(bytes, mediaType, 0, 0, null);
}
BufferedImage image = ImageIO.read(new ByteArrayInputStream(bytes));
if (image != null) {
return new OptimizedAsset(bytes, mediaType, image.getWidth(), image.getHeight());
return new OptimizedAsset(bytes, mediaType, image.getWidth(), image.getHeight(), null);
}
return null;
}
@@ -404,7 +432,7 @@ public class ChannelDirectoryService {
encoder.finish();
BufferedImage cover = frames.get(0).image();
byte[] video = Files.readAllBytes(temp.toPath());
return new OptimizedAsset(video, "video/mp4", cover.getWidth(), cover.getHeight());
return new OptimizedAsset(video, "video/mp4", cover.getWidth(), cover.getHeight(), encodePreview(cover));
} finally {
Files.deleteIfExists(temp.toPath());
}
@@ -498,6 +526,19 @@ public class ChannelDirectoryService {
}
}
private String encodePreview(BufferedImage image) {
if (image == null) {
return null;
}
try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
ImageIO.write(image, "png", baos);
return "data:image/png;base64," + Base64.getEncoder().encodeToString(baos.toByteArray());
} catch (IOException e) {
logger.warn("Unable to encode preview image", e);
return null;
}
}
private Dimension extractVideoDimensions(byte[] bytes) {
try (var channel = new ByteBufferSeekableByteChannel(ByteBuffer.wrap(bytes), bytes.length)) {
FrameGrab grab = FrameGrab.createFrameGrab(channel);
@@ -511,9 +552,24 @@ public class ChannelDirectoryService {
return new Dimension(640, 360);
}
private String extractVideoPreview(byte[] bytes, String mediaType) {
try (var channel = new ByteBufferSeekableByteChannel(ByteBuffer.wrap(bytes), bytes.length)) {
FrameGrab grab = FrameGrab.createFrameGrab(channel);
Picture frame = grab.getNativeFrame();
if (frame == null) {
return null;
}
BufferedImage image = AWTUtil.toBufferedImage(frame);
return encodePreview(image);
} catch (IOException | JCodecException e) {
logger.warn("Unable to capture video preview frame for {}", mediaType, e);
return null;
}
}
public record AssetContent(byte[] bytes, String mediaType) { }
private record OptimizedAsset(byte[] bytes, String mediaType, int width, int height) { }
private record OptimizedAsset(byte[] bytes, String mediaType, int width, int height, String previewDataUrl) { }
private record GifFrame(BufferedImage image, int delayMs) { }