From e3580f950d5251a8dd4b5f46279b2992a465a7a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Kr=C3=BChlmann?= Date: Tue, 13 Jan 2026 23:45:27 +0100 Subject: [PATCH] Emote cache --- .../chat-overlay/source.js | 171 ++++++--- .../imgfloat/config/SecurityConfig.java | 2 + .../controller/TwitchEmoteController.java | 42 +++ .../service/TwitchAppAccessTokenService.java | 101 ++++++ .../imgfloat/service/TwitchEmoteService.java | 340 ++++++++++++++++++ src/main/resources/static/js/broadcast.js | 9 + .../resources/static/js/broadcast/renderer.js | 120 ++++++- .../static/js/broadcast/script-worker.js | 80 ++--- 8 files changed, 762 insertions(+), 103 deletions(-) create mode 100644 src/main/java/dev/kruhlmann/imgfloat/controller/TwitchEmoteController.java create mode 100644 src/main/java/dev/kruhlmann/imgfloat/service/TwitchAppAccessTokenService.java create mode 100644 src/main/java/dev/kruhlmann/imgfloat/service/TwitchEmoteService.java diff --git a/doc/marketplace-scripts/chat-overlay/source.js b/doc/marketplace-scripts/chat-overlay/source.js index 99a5e8c..de9a850 100644 --- a/doc/marketplace-scripts/chat-overlay/source.js +++ b/doc/marketplace-scripts/chat-overlay/source.js @@ -1,28 +1,78 @@ const MAX_LINES = 8; const PADDING = 16; const LINE_HEIGHT = 22; +const EMOTE_SIZE = 18; const FONT = "16px 'Helvetica Neue', Arial, sans-serif"; -function wrapLine(ctx, text, maxWidth) { - if (!text) { - return [""]; +function ensureEmoteCache(state) { + if (!state.emoteCache) { + state.emoteCache = new Map(); } - const words = text.split(" "); - const lines = []; - let current = ""; - words.forEach((word) => { - const test = current ? `${current} ${word}` : word; - if (ctx.measureText(test).width > maxWidth && current) { - lines.push(current); - current = word; - } else { - current = test; + return state.emoteCache; +} + +function getEmoteBitmap(url, state) { + if (!url) { + return null; + } + const cache = ensureEmoteCache(state); + const existing = cache.get(url); + if (existing?.bitmap) { + return existing.bitmap; + } + if (existing?.loading) { + return null; + } + const entry = { loading: true, bitmap: null }; + cache.set(url, entry); + fetch(url) + .then((response) => { + if (!response.ok) { + throw new Error("Failed to load emote"); + } + return response.blob(); + }) + .then((blob) => createImageBitmap(blob)) + .then((bitmap) => { + entry.bitmap = bitmap; + entry.loading = false; + }) + .catch(() => { + cache.delete(url); + }); + return null; +} + +function normalizeFragments(message) { + if (Array.isArray(message?.fragments) && message.fragments.length) { + return message.fragments; + } + const text = message?.message || ""; + return [{ type: "text", text }]; +} + +function tokenizeFragments(fragments) { + const tokens = []; + fragments.forEach((fragment) => { + if (fragment?.type === "emote" && fragment?.url) { + tokens.push({ + type: "emote", + url: fragment.url, + text: fragment.text || fragment.name || "", + width: EMOTE_SIZE, + }); + return; } + const text = fragment?.text || ""; + const words = text.split(" "); + words.forEach((word, index) => { + const value = index < words.length - 1 ? `${word} ` : word; + if (value) { + tokens.push({ type: "text", text: value }); + } + }); }); - if (current) { - lines.push(current); - } - return lines; + return tokens; } function formatLines(messages, ctx, width) { @@ -30,66 +80,58 @@ function formatLines(messages, ctx, width) { const lines = []; messages.forEach((message) => { const prefixText = message.displayName ? `${message.displayName}: ` : ""; - const bodyText = message.message || ""; const nameColor = message.tags?.color || "#ffffff"; - if (!prefixText) { - wrapLine(ctx, bodyText, maxWidth).forEach((line) => - lines.push({ - prefixText: "", - prefixWidth: 0, - nameColor, - text: line, - }), - ); + const prefixWidth = ctx.measureText(prefixText).width; + const tokens = tokenizeFragments(normalizeFragments(message)); + if (!tokens.length) { + lines.push({ + prefixText, + prefixWidth, + nameColor, + fragments: [], + contentWidth: 0, + }); return; } - const prefixWidth = ctx.measureText(prefixText).width; - const words = bodyText.split(" "); - let current = ""; let isFirstLine = true; - let availableWidth = Math.max(maxWidth - prefixWidth, 0); + let availableWidth = Math.max(maxWidth - (prefixText ? prefixWidth : 0), 0); + let currentFragments = []; + let currentWidth = 0; const flushLine = () => { lines.push({ prefixText: isFirstLine ? prefixText : "", prefixWidth: isFirstLine ? prefixWidth : 0, nameColor, - text: current, + fragments: currentFragments, + contentWidth: currentWidth, }); - current = ""; + currentFragments = []; + currentWidth = 0; isFirstLine = false; availableWidth = maxWidth; }; - if (!words.length || !bodyText.trim()) { - lines.push({ - prefixText, - prefixWidth, - nameColor, - text: "", - }); - return; - } - - words.forEach((word) => { - const test = current ? `${current} ${word}` : word; - if (ctx.measureText(test).width > availableWidth && current) { + tokens.forEach((token) => { + const tokenWidth = + token.type === "emote" ? token.width : ctx.measureText(token.text || "").width; + if (tokenWidth > availableWidth && currentFragments.length) { flushLine(); - current = word; - } else { - current = test; } + currentFragments.push({ ...token, width: tokenWidth }); + currentWidth += tokenWidth; + availableWidth = Math.max(availableWidth - tokenWidth, 0); }); - if (current) { + if (currentFragments.length) { flushLine(); } }); return lines.slice(-MAX_LINES); } -function tick(context) { +function tick(context, state) { const { ctx, width, height, chatMessages } = context; if (!ctx) { return; @@ -105,10 +147,7 @@ function tick(context) { const lines = formatLines(messages, ctx, width); const boxHeight = lines.length * LINE_HEIGHT + PADDING * 2; - const boxWidth = Math.max( - ...lines.map((line) => line.prefixWidth + ctx.measureText(line.text).width), - 120, - ); + const boxWidth = Math.max(...lines.map((line) => line.prefixWidth + line.contentWidth), 120); ctx.fillStyle = "rgba(0, 0, 0, 0.55)"; ctx.fillRect(PADDING, height - boxHeight - PADDING, boxWidth + PADDING * 2, boxHeight); @@ -120,7 +159,25 @@ function tick(context) { ctx.fillStyle = line.nameColor || "#ffffff"; ctx.fillText(line.prefixText, x, y); } - ctx.fillStyle = "#ffffff"; - ctx.fillText(line.text, x + line.prefixWidth, y); + let cursorX = x + line.prefixWidth; + line.fragments.forEach((fragment) => { + if (fragment.type === "emote" && fragment.url) { + const bitmap = getEmoteBitmap(fragment.url, state); + if (bitmap) { + const yOffset = y + (LINE_HEIGHT - EMOTE_SIZE) / 2; + ctx.drawImage(bitmap, cursorX, yOffset, EMOTE_SIZE, EMOTE_SIZE); + } else if (fragment.text) { + ctx.fillStyle = "#ffffff"; + ctx.fillText(fragment.text, cursorX, y); + } + cursorX += fragment.width; + return; + } + if (fragment.text) { + ctx.fillStyle = "#ffffff"; + ctx.fillText(fragment.text, cursorX, y); + } + cursorX += fragment.width; + }); }); } diff --git a/src/main/java/dev/kruhlmann/imgfloat/config/SecurityConfig.java b/src/main/java/dev/kruhlmann/imgfloat/config/SecurityConfig.java index db2cd58..e7a1aad 100644 --- a/src/main/java/dev/kruhlmann/imgfloat/config/SecurityConfig.java +++ b/src/main/java/dev/kruhlmann/imgfloat/config/SecurityConfig.java @@ -81,6 +81,8 @@ public class SecurityConfig { .permitAll() .requestMatchers(HttpMethod.GET, "/api/channels/*/assets/*/preview") .permitAll() + .requestMatchers(HttpMethod.GET, "/api/twitch/emotes/**") + .permitAll() .requestMatchers("/ws/**") .permitAll() .anyRequest() diff --git a/src/main/java/dev/kruhlmann/imgfloat/controller/TwitchEmoteController.java b/src/main/java/dev/kruhlmann/imgfloat/controller/TwitchEmoteController.java new file mode 100644 index 0000000..43301d9 --- /dev/null +++ b/src/main/java/dev/kruhlmann/imgfloat/controller/TwitchEmoteController.java @@ -0,0 +1,42 @@ +package dev.kruhlmann.imgfloat.controller; + +import dev.kruhlmann.imgfloat.service.TwitchEmoteService; +import java.util.List; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping("/api/twitch/emotes") +public class TwitchEmoteController { + + private final TwitchEmoteService twitchEmoteService; + + public TwitchEmoteController(TwitchEmoteService twitchEmoteService) { + this.twitchEmoteService = twitchEmoteService; + } + + @GetMapping + public EmoteCatalogResponse fetchEmoteCatalog(@RequestParam(value = "channel", required = false) String channel) { + List global = twitchEmoteService.getGlobalEmotes(); + List channelEmotes = twitchEmoteService.getChannelEmotes(channel); + return new EmoteCatalogResponse(global, channelEmotes); + } + + @GetMapping("/{emoteId}") + public ResponseEntity fetchEmoteAsset(@PathVariable("emoteId") String emoteId) { + return twitchEmoteService + .loadEmoteAsset(emoteId) + .map((asset) -> ResponseEntity.ok().contentType(MediaType.parseMediaType(asset.mediaType())).body(asset.bytes())) + .orElseGet(() -> ResponseEntity.notFound().build()); + } + + public record EmoteCatalogResponse( + List global, + List channel + ) {} +} diff --git a/src/main/java/dev/kruhlmann/imgfloat/service/TwitchAppAccessTokenService.java b/src/main/java/dev/kruhlmann/imgfloat/service/TwitchAppAccessTokenService.java new file mode 100644 index 0000000..f517609 --- /dev/null +++ b/src/main/java/dev/kruhlmann/imgfloat/service/TwitchAppAccessTokenService.java @@ -0,0 +1,101 @@ +package dev.kruhlmann.imgfloat.service; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.web.client.RestTemplateBuilder; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestClientException; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponentsBuilder; + +@Service +public class TwitchAppAccessTokenService { + + private static final Logger LOG = LoggerFactory.getLogger(TwitchAppAccessTokenService.class); + private final RestTemplate restTemplate; + private final String clientId; + private final String clientSecret; + private volatile AccessToken cachedToken; + + public TwitchAppAccessTokenService( + RestTemplateBuilder builder, + @Value("${spring.security.oauth2.client.registration.twitch.client-id:#{null}}") String clientId, + @Value("${spring.security.oauth2.client.registration.twitch.client-secret:#{null}}") String clientSecret + ) { + this.restTemplate = builder + .setConnectTimeout(Duration.ofSeconds(15)) + .setReadTimeout(Duration.ofSeconds(15)) + .build(); + this.clientId = clientId; + this.clientSecret = clientSecret; + } + + public Optional getAccessToken() { + if (clientId == null || clientId.isBlank() || clientSecret == null || clientSecret.isBlank()) { + return Optional.empty(); + } + AccessToken current = cachedToken; + if (current != null && !current.isExpired()) { + return Optional.of(current.token()); + } + synchronized (this) { + AccessToken refreshed = cachedToken; + if (refreshed != null && !refreshed.isExpired()) { + return Optional.of(refreshed.token()); + } + cachedToken = requestToken(); + return Optional.ofNullable(cachedToken).map(AccessToken::token); + } + } + + public Optional getClientId() { + if (clientId == null || clientId.isBlank()) { + return Optional.empty(); + } + return Optional.of(clientId); + } + + private AccessToken requestToken() { + if (clientId == null || clientSecret == null) { + return null; + } + UriComponentsBuilder builder = UriComponentsBuilder.fromHttpUrl("https://id.twitch.tv/oauth2/token") + .queryParam("client_id", clientId) + .queryParam("client_secret", clientSecret) + .queryParam("grant_type", "client_credentials"); + + try { + ResponseEntity response = restTemplate.postForEntity( + builder.build(true).toUri(), + null, + TokenResponse.class + ); + TokenResponse body = response.getBody(); + if (body == null || body.accessToken() == null || body.accessToken().isBlank()) { + LOG.warn("Unable to fetch Twitch app token: empty response"); + return null; + } + Instant expiresAt = Instant.now().plusSeconds(Math.max(0, body.expiresIn() - 60)); + return new AccessToken(body.accessToken(), expiresAt); + } catch (RestClientException ex) { + LOG.warn("Unable to fetch Twitch app token", ex); + return null; + } + } + + private record AccessToken(String token, Instant expiresAt) { + boolean isExpired() { + return expiresAt == null || expiresAt.isBefore(Instant.now()); + } + } + + @JsonIgnoreProperties(ignoreUnknown = true) + private record TokenResponse(@JsonProperty("access_token") String accessToken, @JsonProperty("expires_in") long expiresIn) {} +} diff --git a/src/main/java/dev/kruhlmann/imgfloat/service/TwitchEmoteService.java b/src/main/java/dev/kruhlmann/imgfloat/service/TwitchEmoteService.java new file mode 100644 index 0000000..2c9c44c --- /dev/null +++ b/src/main/java/dev/kruhlmann/imgfloat/service/TwitchEmoteService.java @@ -0,0 +1,340 @@ +package dev.kruhlmann.imgfloat.service; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.web.client.RestTemplateBuilder; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestClientException; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponentsBuilder; + +@Service +public class TwitchEmoteService { + + private static final Logger LOG = LoggerFactory.getLogger(TwitchEmoteService.class); + private static final String GLOBAL_EMOTE_URL = "https://api.twitch.tv/helix/chat/emotes/global"; + private static final String CHANNEL_EMOTE_URL = "https://api.twitch.tv/helix/chat/emotes"; + private static final String USERS_URL = "https://api.twitch.tv/helix/users"; + + private final RestTemplate restTemplate; + private final TwitchAppAccessTokenService tokenService; + private final Path cacheRoot; + private final Map emoteCache = new ConcurrentHashMap<>(); + private final Map> channelEmoteCache = new ConcurrentHashMap<>(); + private volatile List globalEmotes = List.of(); + + public TwitchEmoteService( + RestTemplateBuilder builder, + TwitchAppAccessTokenService tokenService, + @Value("${IMGFLOAT_TWITCH_EMOTE_CACHE_PATH:#{null}}") String cachePath + ) { + this.restTemplate = builder + .setConnectTimeout(Duration.ofSeconds(20)) + .setReadTimeout(Duration.ofSeconds(20)) + .build(); + this.tokenService = tokenService; + String root = cachePath != null + ? cachePath + : Paths.get(System.getProperty("java.io.tmpdir"), "imgfloat-emotes").toString(); + this.cacheRoot = Paths.get(root).normalize().toAbsolutePath(); + try { + Files.createDirectories(this.cacheRoot); + } catch (IOException ex) { + throw new IllegalStateException("Failed to create Twitch emote cache directory", ex); + } + warmGlobalEmotes(); + } + + public List getGlobalEmotes() { + if (globalEmotes.isEmpty()) { + warmGlobalEmotes(); + } + return globalEmotes.stream().map(CachedEmote::descriptor).toList(); + } + + public List getChannelEmotes(String channelLogin) { + if (channelLogin == null || channelLogin.isBlank()) { + return List.of(); + } + String normalized = channelLogin.toLowerCase(Locale.ROOT); + List emotes = channelEmoteCache.computeIfAbsent(normalized, this::fetchChannelEmotes); + return emotes.stream().map(CachedEmote::descriptor).toList(); + } + + public Optional loadEmoteAsset(String emoteId) { + if (emoteId == null || emoteId.isBlank()) { + return Optional.empty(); + } + CachedEmote cached = emoteCache.get(emoteId); + if (cached == null) { + cached = restoreFromDisk(emoteId).orElse(null); + } + if (cached == null) { + return Optional.empty(); + } + try { + byte[] bytes = Files.readAllBytes(cached.path()); + return Optional.of(new EmoteAsset(bytes, cached.mediaType())); + } catch (IOException ex) { + LOG.warn("Unable to read cached emote {}", emoteId, ex); + return Optional.empty(); + } + } + + private void warmGlobalEmotes() { + List data = fetchEmotes(GLOBAL_EMOTE_URL); + if (data.isEmpty()) { + return; + } + List cached = data + .stream() + .map(this::cacheEmote) + .filter(Optional::isPresent) + .map(Optional::get) + .toList(); + globalEmotes = cached; + LOG.info("Loaded {} global Twitch emotes", cached.size()); + } + + private List fetchChannelEmotes(String channelLogin) { + String broadcasterId = fetchBroadcasterId(channelLogin).orElse(null); + if (broadcasterId == null) { + return List.of(); + } + UriComponentsBuilder builder = UriComponentsBuilder.fromHttpUrl(CHANNEL_EMOTE_URL) + .queryParam("broadcaster_id", broadcasterId); + List data = fetchEmotes(builder.toUriString()); + if (data.isEmpty()) { + return List.of(); + } + List cached = data + .stream() + .map(this::cacheEmote) + .filter(Optional::isPresent) + .map(Optional::get) + .toList(); + LOG.info("Loaded {} Twitch emotes for {}", cached.size(), channelLogin); + return cached; + } + + private Optional fetchBroadcasterId(String channelLogin) { + Optional token = tokenService.getAccessToken(); + Optional clientId = tokenService.getClientId(); + if (token.isEmpty() || clientId.isEmpty()) { + return Optional.empty(); + } + UriComponentsBuilder builder = UriComponentsBuilder.fromHttpUrl(USERS_URL).queryParam("login", channelLogin); + HttpHeaders headers = new HttpHeaders(); + headers.setBearerAuth(token.get()); + headers.add("Client-ID", clientId.get()); + try { + ResponseEntity response = restTemplate.exchange( + builder.build(true).toUri(), + HttpMethod.GET, + new HttpEntity<>(headers), + TwitchUsersResponse.class + ); + TwitchUsersResponse body = response.getBody(); + if (body == null || body.data() == null || body.data().isEmpty()) { + return Optional.empty(); + } + return body.data().stream().findFirst().map(TwitchUserData::id); + } catch (RestClientException ex) { + LOG.warn("Unable to fetch Twitch broadcaster id for {}", channelLogin, ex); + return Optional.empty(); + } + } + + private List fetchEmotes(String url) { + Optional token = tokenService.getAccessToken(); + Optional clientId = tokenService.getClientId(); + if (token.isEmpty() || clientId.isEmpty()) { + return List.of(); + } + HttpHeaders headers = new HttpHeaders(); + headers.setBearerAuth(token.get()); + headers.add("Client-ID", clientId.get()); + try { + ResponseEntity response = restTemplate.exchange( + URI.create(url), + HttpMethod.GET, + new HttpEntity<>(headers), + TwitchEmoteResponse.class + ); + TwitchEmoteResponse body = response.getBody(); + if (body == null || body.data() == null) { + return List.of(); + } + return body.data(); + } catch (RestClientException ex) { + LOG.warn("Unable to fetch Twitch emotes from {}", url, ex); + return List.of(); + } + } + + private Optional cacheEmote(TwitchEmoteData emote) { + if (emote == null || emote.id() == null || emote.id().isBlank()) { + return Optional.empty(); + } + return Optional.ofNullable( + emoteCache.computeIfAbsent( + emote.id(), + (id) -> { + String imageUrl = selectImageUrl(emote); + if (imageUrl == null) { + return null; + } + return downloadEmote(id, emote.name(), imageUrl); + } + ) + ); + } + + private CachedEmote downloadEmote(String id, String name, String imageUrl) { + Path filePath = resolveEmotePath(id, imageUrl); + MediaType mediaType = null; + + if (!Files.exists(filePath)) { + try { + ResponseEntity response = restTemplate.getForEntity(URI.create(imageUrl), byte[].class); + byte[] bytes = response.getBody(); + if (bytes == null || bytes.length == 0) { + return null; + } + mediaType = response.getHeaders().getContentType(); + Files.write(filePath, bytes); + } catch (IOException | RestClientException ex) { + LOG.warn("Unable to download Twitch emote {}", id, ex); + return null; + } + } + + if (mediaType == null) { + mediaType = mediaTypeFromPath(filePath); + } + return new CachedEmote(id, name, filePath, mediaType != null ? mediaType.toString() : "image/png"); + } + + private Optional restoreFromDisk(String id) { + if (id == null || id.isBlank()) { + return Optional.empty(); + } + try { + List candidates; + try (var stream = Files.list(cacheRoot)) { + candidates = stream.filter((path) -> path.getFileName().toString().startsWith(id + ".")).toList(); + } + if (candidates.isEmpty()) { + return Optional.empty(); + } + Path path = candidates.get(0); + MediaType mediaType = mediaTypeFromPath(path); + CachedEmote cached = new CachedEmote(id, id, path, mediaType != null ? mediaType.toString() : "image/png"); + emoteCache.put(id, cached); + return Optional.of(cached); + } catch (IOException ex) { + LOG.warn("Unable to restore cached emote {}", id, ex); + return Optional.empty(); + } + } + + private Path resolveEmotePath(String id, String imageUrl) { + String extension = extensionFromUrl(imageUrl).orElse("png"); + return cacheRoot.resolve(id + "." + extension); + } + + private Optional extensionFromUrl(String imageUrl) { + if (imageUrl == null) { + return Optional.empty(); + } + try { + String path = URI.create(imageUrl).getPath(); + int dot = path.lastIndexOf('.'); + if (dot == -1) { + return Optional.empty(); + } + return Optional.of(path.substring(dot + 1)); + } catch (IllegalArgumentException ex) { + return Optional.empty(); + } + } + + private MediaType mediaTypeFromPath(Path path) { + if (path == null) { + return null; + } + String name = path.getFileName().toString().toLowerCase(Locale.ROOT); + if (name.endsWith(".gif")) { + return MediaType.IMAGE_GIF; + } + if (name.endsWith(".webp")) { + return MediaType.parseMediaType("image/webp"); + } + if (name.endsWith(".jpg") || name.endsWith(".jpeg")) { + return MediaType.IMAGE_JPEG; + } + return MediaType.IMAGE_PNG; + } + + private String selectImageUrl(TwitchEmoteData emote) { + if (emote == null || emote.images() == null) { + return null; + } + String url = emote.images().url1x(); + if (url == null || url.isBlank()) { + url = emote.images().url2x(); + } + if (url == null || url.isBlank()) { + url = emote.images().url4x(); + } + return url; + } + + public record EmoteDescriptor(String id, String name, String url) {} + + public record EmoteAsset(byte[] bytes, String mediaType) {} + + private record CachedEmote(String id, String name, Path path, String mediaType) { + EmoteDescriptor descriptor() { + return new EmoteDescriptor(id, name, "/api/twitch/emotes/" + id); + } + } + + @JsonIgnoreProperties(ignoreUnknown = true) + private record TwitchEmoteResponse(List data) {} + + @JsonIgnoreProperties(ignoreUnknown = true) + private record TwitchEmoteData(String id, String name, Images images) {} + + @JsonIgnoreProperties(ignoreUnknown = true) + private record Images( + @JsonProperty("url_1x") String url1x, + @JsonProperty("url_2x") String url2x, + @JsonProperty("url_4x") String url4x + ) {} + + @JsonIgnoreProperties(ignoreUnknown = true) + private record TwitchUsersResponse(List data) {} + + @JsonIgnoreProperties(ignoreUnknown = true) + private record TwitchUserData(String id) {} +} diff --git a/src/main/resources/static/js/broadcast.js b/src/main/resources/static/js/broadcast.js index 517629f..76fe6bb 100644 --- a/src/main/resources/static/js/broadcast.js +++ b/src/main/resources/static/js/broadcast.js @@ -7,6 +7,15 @@ const scriptLayer = document.getElementById("broadcast-script-layer"); setUpElectronWindowFrame(); const renderer = new BroadcastRenderer({ canvas, scriptLayer, broadcaster, showToast }); +fetch(`/api/twitch/emotes?channel=${encodeURIComponent(broadcaster)}`) + .then((response) => { + if (!response.ok) { + throw new Error("Failed to load Twitch emotes"); + } + return response.json(); + }) + .then((catalog) => renderer.setEmoteCatalog(catalog)) + .catch((error) => console.warn("Unable to load Twitch emotes", error)); const disconnectChat = connectTwitchChat( broadcaster, ({ channel, displayName, message, tags, prefix, raw }) => { diff --git a/src/main/resources/static/js/broadcast/renderer.js b/src/main/resources/static/js/broadcast/renderer.js index 19b89ac..5436f21 100644 --- a/src/main/resources/static/js/broadcast/renderer.js +++ b/src/main/resources/static/js/broadcast/renderer.js @@ -26,6 +26,9 @@ export class BroadcastRenderer { this.scriptAttachmentCache = new Map(); this.scriptAttachmentsByAssetId = new Map(); this.chatMessages = []; + this.emoteCatalog = []; + this.emoteCatalogById = new Map(); + this.lastChatPruneAt = 0; this.obsBrowser = !!globalThis.obsstudio; this.supportsAnimatedDecode = @@ -322,6 +325,11 @@ export class BroadcastRenderer { } renderFrame() { + const now = Date.now(); + if (now - this.lastChatPruneAt > 1000) { + this.lastChatPruneAt = now; + this.pruneChatMessages(now); + } this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height); getRenderOrder(this.state).forEach((asset) => this.drawAsset(asset)); } @@ -412,6 +420,7 @@ export class BroadcastRenderer { ); this.scriptWorkerReady = true; this.updateScriptWorkerChatMessages(); + this.updateScriptWorkerEmoteCatalog(); } updateScriptWorkerCanvas() { @@ -439,14 +448,119 @@ export class BroadcastRenderer { }); } + updateScriptWorkerEmoteCatalog() { + if (!this.scriptWorker || !this.scriptWorkerReady) { + return; + } + this.scriptWorker.postMessage({ + type: "emoteCatalog", + payload: { + emotes: this.emoteCatalog, + }, + }); + } + + setEmoteCatalog(catalog) { + const globalEmotes = Array.isArray(catalog?.global) ? catalog.global : []; + const channelEmotes = Array.isArray(catalog?.channel) ? catalog.channel : []; + this.emoteCatalog = [...globalEmotes, ...channelEmotes]; + this.emoteCatalogById = new Map( + this.emoteCatalog.map((entry) => [String(entry?.id || ""), entry]).filter(([key]) => key), + ); + if (this.chatMessages.length) { + this.chatMessages = this.chatMessages.map((message) => { + if (!Array.isArray(message.fragments)) { + return message; + } + const fragments = message.fragments.map((fragment) => { + if (fragment.type !== "emote" || fragment.url) { + return fragment; + } + const emoteInfo = this.emoteCatalogById.get(String(fragment.id)); + if (!emoteInfo) { + return fragment; + } + return { ...fragment, url: emoteInfo.url, name: emoteInfo.name || fragment.name }; + }); + return { ...message, fragments }; + }); + this.updateScriptWorkerChatMessages(); + } + this.updateScriptWorkerEmoteCatalog(); + } + + pruneChatMessages(now = Date.now()) { + const cutoff = now - 120_000; + const pruned = this.chatMessages.filter((item) => item.timestamp >= cutoff); + if (pruned.length !== this.chatMessages.length) { + this.chatMessages = pruned; + this.updateScriptWorkerChatMessages(); + } + } + + parseEmoteOffsets(rawEmotes) { + if (!rawEmotes) { + return []; + } + return rawEmotes + .split("/") + .flatMap((emoteEntry) => { + if (!emoteEntry) { + return []; + } + const [id, positions] = emoteEntry.split(":"); + if (!id || !positions) { + return []; + } + return positions.split(",").map((range) => { + const [start, end] = range.split("-").map((value) => Number.parseInt(value, 10)); + return Number.isFinite(start) && Number.isFinite(end) ? { id, start, end } : null; + }); + }) + .filter(Boolean); + } + + buildMessageFragments(message, tags) { + if (!message) { + return []; + } + const emotes = this.parseEmoteOffsets(tags?.emotes); + if (!emotes.length) { + return [{ type: "text", text: message }]; + } + const sorted = emotes.sort((a, b) => a.start - b.start); + const fragments = []; + let cursor = 0; + sorted.forEach((emote) => { + if (emote.start > cursor) { + fragments.push({ type: "text", text: message.slice(cursor, emote.start) }); + } + const emoteText = message.slice(emote.start, emote.end + 1); + const emoteInfo = this.emoteCatalogById.get(String(emote.id)); + fragments.push({ + type: "emote", + id: emote.id, + text: emoteText, + name: emoteInfo?.name || emoteText, + url: emoteInfo?.url || null, + }); + cursor = emote.end + 1; + }); + if (cursor < message.length) { + fragments.push({ type: "text", text: message.slice(cursor) }); + } + return fragments; + } + receiveChatMessage(message) { if (!message) { return; } const now = Date.now(); - const entry = { ...message, timestamp: now }; - const cutoff = now - 120_000; - this.chatMessages = [...this.chatMessages, entry].filter((item) => item.timestamp >= cutoff); + const fragments = this.buildMessageFragments(message.message || "", message.tags); + const entry = { ...message, fragments, timestamp: now }; + this.chatMessages = [...this.chatMessages, entry]; + this.pruneChatMessages(now); this.updateScriptWorkerChatMessages(); } diff --git a/src/main/resources/static/js/broadcast/script-worker.js b/src/main/resources/static/js/broadcast/script-worker.js index 22dd067..4c6a18b 100644 --- a/src/main/resources/static/js/broadcast/script-worker.js +++ b/src/main/resources/static/js/broadcast/script-worker.js @@ -10,6 +10,7 @@ const allowedImportUrls = new Set(); const nativeImportScripts = typeof self.importScripts === "function" ? self.importScripts.bind(self) : null; const sharedDependencyUrls = ["/js/vendor/three.min.js", "/js/vendor/GLTFLoader.js", "/js/vendor/OBJLoader.js"]; let chatMessages = []; +let emoteCatalog = []; function normalizeUrl(url) { try { @@ -39,48 +40,6 @@ function importAllowedScripts(...urls) { return nativeImportScripts(...resolved); } -function disableNetworkApis() { - const nativeFetch = typeof self.fetch === "function" ? self.fetch.bind(self) : null; - const blockedApis = { - fetch: (...args) => { - if (!nativeFetch) { - throw new Error("Network access is disabled in asset scripts."); - } - const request = new Request(...args); - const url = normalizeUrl(request.url); - if (!allowedFetchUrls.has(url)) { - throw new Error("Network access is disabled in asset scripts."); - } - return nativeFetch(request); - }, - XMLHttpRequest: undefined, - WebSocket: undefined, - EventSource: undefined, - importScripts: (...urls) => importAllowedScripts(...urls), - }; - - Object.entries(blockedApis).forEach(([key, value]) => { - if (!(key in self)) { - return; - } - try { - Object.defineProperty(self, key, { - value, - writable: false, - configurable: false, - }); - } catch (error) { - try { - self[key] = value; - } catch (_error) { - // ignore if the API cannot be overridden in this environment - } - } - }); -} - -disableNetworkApis(); - function loadSharedDependencies() { if (!nativeImportScripts || sharedDependencyUrls.length === 0) { return; @@ -106,6 +65,32 @@ function refreshAllowedFetchUrls() { } }); }); + if (Array.isArray(emoteCatalog)) { + emoteCatalog.forEach((emote) => { + if (emote?.url) { + const normalized = normalizeUrl(emote.url); + if (normalized) { + allowedFetchUrls.add(normalized); + } + } + }); + } + if (Array.isArray(chatMessages)) { + chatMessages.forEach((message) => { + const fragments = message?.fragments; + if (!Array.isArray(fragments)) { + return; + } + fragments.forEach((fragment) => { + if (fragment?.url) { + const normalized = normalizeUrl(fragment.url); + if (normalized) { + allowedFetchUrls.add(normalized); + } + } + }); + }); + } } function reportScriptError(id, stage, error) { @@ -139,6 +124,7 @@ function updateScriptContexts() { script.context.width = script.canvas?.width ?? 0; script.context.height = script.canvas?.height ?? 0; script.context.chatMessages = chatMessages; + script.context.emoteCatalog = emoteCatalog; }); } @@ -182,7 +168,7 @@ function stopTickLoopIfIdle() { function createScriptHandlers(source, context, state, sourceLabel = "") { const contextPrelude = - "const { canvas, ctx, channelName, width, height, now, deltaMs, elapsedMs, assets, chatMessages, playAudio } = context;"; + "const { canvas, ctx, channelName, width, height, now, deltaMs, elapsedMs, assets, chatMessages, emoteCatalog, playAudio } = context;"; const sourceUrl = sourceLabel ? `\n//# sourceURL=${sourceLabel}` : ""; const factory = new Function( "context", @@ -242,6 +228,7 @@ self.addEventListener("message", (event) => { elapsedMs: 0, assets: Array.isArray(payload.attachments) ? payload.attachments : [], chatMessages, + emoteCatalog, playAudio: (attachment) => { const attachmentId = typeof attachment === "string" ? attachment : attachment?.id; if (!attachmentId) { @@ -310,6 +297,13 @@ self.addEventListener("message", (event) => { if (type === "chatMessages") { chatMessages = Array.isArray(payload?.messages) ? payload.messages : []; + refreshAllowedFetchUrls(); + updateScriptContexts(); + } + + if (type === "emoteCatalog") { + emoteCatalog = Array.isArray(payload?.emotes) ? payload.emotes : []; + refreshAllowedFetchUrls(); updateScriptContexts(); } });