Emote cache

doc/marketplace-scripts/chat-overlay/source.js

@@ -1,28 +1,78 @@
 const MAX_LINES = 8;
 const PADDING = 16;
 const LINE_HEIGHT = 22;
+const EMOTE_SIZE = 18;
 const FONT = "16px 'Helvetica Neue', Arial, sans-serif";
 
-function wrapLine(ctx, text, maxWidth) {
+function ensureEmoteCache(state) {
-    if (!text) {
+    if (!state.emoteCache) {
-        return [""];
+        state.emoteCache = new Map();
     }
+    return state.emoteCache;
+}
+
+function getEmoteBitmap(url, state) {
+    if (!url) {
+        return null;
+    }
+    const cache = ensureEmoteCache(state);
+    const existing = cache.get(url);
+    if (existing?.bitmap) {
+        return existing.bitmap;
+    }
+    if (existing?.loading) {
+        return null;
+    }
+    const entry = { loading: true, bitmap: null };
+    cache.set(url, entry);
+    fetch(url)
+        .then((response) => {
+            if (!response.ok) {
+                throw new Error("Failed to load emote");
+            }
+            return response.blob();
+        })
+        .then((blob) => createImageBitmap(blob))
+        .then((bitmap) => {
+            entry.bitmap = bitmap;
+            entry.loading = false;
+        })
+        .catch(() => {
+            cache.delete(url);
+        });
+    return null;
+}
+
+function normalizeFragments(message) {
+    if (Array.isArray(message?.fragments) && message.fragments.length) {
+        return message.fragments;
+    }
+    const text = message?.message || "";
+    return [{ type: "text", text }];
+}
+
+function tokenizeFragments(fragments) {
+    const tokens = [];
+    fragments.forEach((fragment) => {
+        if (fragment?.type === "emote" && fragment?.url) {
+            tokens.push({
+                type: "emote",
+                url: fragment.url,
+                text: fragment.text || fragment.name || "",
+                width: EMOTE_SIZE,
+            });
+            return;
+        }
+        const text = fragment?.text || "";
         const words = text.split(" ");
-    const lines = [];
+        words.forEach((word, index) => {
-    let current = "";
+            const value = index < words.length - 1 ? `${word} ` : word;
-    words.forEach((word) => {
+            if (value) {
-        const test = current ? `${current} ${word}` : word;
+                tokens.push({ type: "text", text: value });
-        if (ctx.measureText(test).width > maxWidth && current) {
-            lines.push(current);
-            current = word;
-        } else {
-            current = test;
             }
         });
-    if (current) {
+    });
-        lines.push(current);
+    return tokens;
-    }
-    return lines;
 }
 
 function formatLines(messages, ctx, width) {
@@ -30,66 +80,58 @@ function formatLines(messages, ctx, width) {
     const lines = [];
     messages.forEach((message) => {
         const prefixText = message.displayName ? `${message.displayName}: ` : "";
-        const bodyText = message.message || "";
         const nameColor = message.tags?.color || "#ffffff";
-        if (!prefixText) {
+        const prefixWidth = ctx.measureText(prefixText).width;
-            wrapLine(ctx, bodyText, maxWidth).forEach((line) =>
+        const tokens = tokenizeFragments(normalizeFragments(message));
+        if (!tokens.length) {
             lines.push({
-                    prefixText: "",
+                prefixText,
-                    prefixWidth: 0,
+                prefixWidth,
                 nameColor,
-                    text: line,
+                fragments: [],
-                }),
+                contentWidth: 0,
-            );
+            });
             return;
         }
 
-        const prefixWidth = ctx.measureText(prefixText).width;
-        const words = bodyText.split(" ");
-        let current = "";
         let isFirstLine = true;
-        let availableWidth = Math.max(maxWidth - prefixWidth, 0);
+        let availableWidth = Math.max(maxWidth - (prefixText ? prefixWidth : 0), 0);
+        let currentFragments = [];
+        let currentWidth = 0;
 
         const flushLine = () => {
             lines.push({
                 prefixText: isFirstLine ? prefixText : "",
                 prefixWidth: isFirstLine ? prefixWidth : 0,
                 nameColor,
-                text: current,
+                fragments: currentFragments,
+                contentWidth: currentWidth,
             });
-            current = "";
+            currentFragments = [];
+            currentWidth = 0;
             isFirstLine = false;
             availableWidth = maxWidth;
         };
 
-        if (!words.length || !bodyText.trim()) {
+        tokens.forEach((token) => {
-            lines.push({
+            const tokenWidth =
-                prefixText,
+                token.type === "emote" ? token.width : ctx.measureText(token.text || "").width;
-                prefixWidth,
+            if (tokenWidth > availableWidth && currentFragments.length) {
-                nameColor,
-                text: "",
-            });
-            return;
-        }
-
-        words.forEach((word) => {
-            const test = current ? `${current} ${word}` : word;
-            if (ctx.measureText(test).width > availableWidth && current) {
                 flushLine();
-                current = word;
-            } else {
-                current = test;
             }
+            currentFragments.push({ ...token, width: tokenWidth });
+            currentWidth += tokenWidth;
+            availableWidth = Math.max(availableWidth - tokenWidth, 0);
         });
 
-        if (current) {
+        if (currentFragments.length) {
             flushLine();
         }
     });
     return lines.slice(-MAX_LINES);
 }
 
-function tick(context) {
+function tick(context, state) {
     const { ctx, width, height, chatMessages } = context;
     if (!ctx) {
         return;
@@ -105,10 +147,7 @@ function tick(context) {
 
     const lines = formatLines(messages, ctx, width);
     const boxHeight = lines.length * LINE_HEIGHT + PADDING * 2;
-    const boxWidth = Math.max(
+    const boxWidth = Math.max(...lines.map((line) => line.prefixWidth + line.contentWidth), 120);
-        ...lines.map((line) => line.prefixWidth + ctx.measureText(line.text).width),
-        120,
-    );
 
     ctx.fillStyle = "rgba(0, 0, 0, 0.55)";
     ctx.fillRect(PADDING, height - boxHeight - PADDING, boxWidth + PADDING * 2, boxHeight);
@@ -120,7 +159,25 @@ function tick(context) {
             ctx.fillStyle = line.nameColor || "#ffffff";
             ctx.fillText(line.prefixText, x, y);
         }
+        let cursorX = x + line.prefixWidth;
+        line.fragments.forEach((fragment) => {
+            if (fragment.type === "emote" && fragment.url) {
+                const bitmap = getEmoteBitmap(fragment.url, state);
+                if (bitmap) {
+                    const yOffset = y + (LINE_HEIGHT - EMOTE_SIZE) / 2;
+                    ctx.drawImage(bitmap, cursorX, yOffset, EMOTE_SIZE, EMOTE_SIZE);
+                } else if (fragment.text) {
                     ctx.fillStyle = "#ffffff";
-        ctx.fillText(line.text, x + line.prefixWidth, y);
+                    ctx.fillText(fragment.text, cursorX, y);
+                }
+                cursorX += fragment.width;
+                return;
+            }
+            if (fragment.text) {
+                ctx.fillStyle = "#ffffff";
+                ctx.fillText(fragment.text, cursorX, y);
+            }
+            cursorX += fragment.width;
+        });
     });
 }

src/main/java/dev/kruhlmann/imgfloat/config/SecurityConfig.java

@@ -81,6 +81,8 @@ public class SecurityConfig {
                     .permitAll()
                     .requestMatchers(HttpMethod.GET, "/api/channels/*/assets/*/preview")
                     .permitAll()
+                    .requestMatchers(HttpMethod.GET, "/api/twitch/emotes/**")
+                    .permitAll()
                     .requestMatchers("/ws/**")
                     .permitAll()
                     .anyRequest()

src/main/java/dev/kruhlmann/imgfloat/controller/TwitchEmoteController.java

@@ -0,0 +1,42 @@
+package dev.kruhlmann.imgfloat.controller;
+
+import dev.kruhlmann.imgfloat.service.TwitchEmoteService;
+import java.util.List;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+@RestController
+@RequestMapping("/api/twitch/emotes")
+public class TwitchEmoteController {
+
+    private final TwitchEmoteService twitchEmoteService;
+
+    public TwitchEmoteController(TwitchEmoteService twitchEmoteService) {
+        this.twitchEmoteService = twitchEmoteService;
+    }
+
+    @GetMapping
+    public EmoteCatalogResponse fetchEmoteCatalog(@RequestParam(value = "channel", required = false) String channel) {
+        List<TwitchEmoteService.EmoteDescriptor> global = twitchEmoteService.getGlobalEmotes();
+        List<TwitchEmoteService.EmoteDescriptor> channelEmotes = twitchEmoteService.getChannelEmotes(channel);
+        return new EmoteCatalogResponse(global, channelEmotes);
+    }
+
+    @GetMapping("/{emoteId}")
+    public ResponseEntity<byte[]> fetchEmoteAsset(@PathVariable("emoteId") String emoteId) {
+        return twitchEmoteService
+            .loadEmoteAsset(emoteId)
+            .map((asset) -> ResponseEntity.ok().contentType(MediaType.parseMediaType(asset.mediaType())).body(asset.bytes()))
+            .orElseGet(() -> ResponseEntity.notFound().build());
+    }
+
+    public record EmoteCatalogResponse(
+        List<TwitchEmoteService.EmoteDescriptor> global,
+        List<TwitchEmoteService.EmoteDescriptor> channel
+    ) {}
+}

src/main/java/dev/kruhlmann/imgfloat/service/TwitchAppAccessTokenService.java

@@ -0,0 +1,101 @@
+package dev.kruhlmann.imgfloat.service;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Optional;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.web.client.RestTemplateBuilder;
+import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Service;
+import org.springframework.web.client.RestClientException;
+import org.springframework.web.client.RestTemplate;
+import org.springframework.web.util.UriComponentsBuilder;
+
+@Service
+public class TwitchAppAccessTokenService {
+
+    private static final Logger LOG = LoggerFactory.getLogger(TwitchAppAccessTokenService.class);
+    private final RestTemplate restTemplate;
+    private final String clientId;
+    private final String clientSecret;
+    private volatile AccessToken cachedToken;
+
+    public TwitchAppAccessTokenService(
+        RestTemplateBuilder builder,
+        @Value("${spring.security.oauth2.client.registration.twitch.client-id:#{null}}") String clientId,
+        @Value("${spring.security.oauth2.client.registration.twitch.client-secret:#{null}}") String clientSecret
+    ) {
+        this.restTemplate = builder
+            .setConnectTimeout(Duration.ofSeconds(15))
+            .setReadTimeout(Duration.ofSeconds(15))
+            .build();
+        this.clientId = clientId;
+        this.clientSecret = clientSecret;
+    }
+
+    public Optional<String> getAccessToken() {
+        if (clientId == null || clientId.isBlank() || clientSecret == null || clientSecret.isBlank()) {
+            return Optional.empty();
+        }
+        AccessToken current = cachedToken;
+        if (current != null && !current.isExpired()) {
+            return Optional.of(current.token());
+        }
+        synchronized (this) {
+            AccessToken refreshed = cachedToken;
+            if (refreshed != null && !refreshed.isExpired()) {
+                return Optional.of(refreshed.token());
+            }
+            cachedToken = requestToken();
+            return Optional.ofNullable(cachedToken).map(AccessToken::token);
+        }
+    }
+
+    public Optional<String> getClientId() {
+        if (clientId == null || clientId.isBlank()) {
+            return Optional.empty();
+        }
+        return Optional.of(clientId);
+    }
+
+    private AccessToken requestToken() {
+        if (clientId == null || clientSecret == null) {
+            return null;
+        }
+        UriComponentsBuilder builder = UriComponentsBuilder.fromHttpUrl("https://id.twitch.tv/oauth2/token")
+            .queryParam("client_id", clientId)
+            .queryParam("client_secret", clientSecret)
+            .queryParam("grant_type", "client_credentials");
+
+        try {
+            ResponseEntity<TokenResponse> response = restTemplate.postForEntity(
+                builder.build(true).toUri(),
+                null,
+                TokenResponse.class
+            );
+            TokenResponse body = response.getBody();
+            if (body == null || body.accessToken() == null || body.accessToken().isBlank()) {
+                LOG.warn("Unable to fetch Twitch app token: empty response");
+                return null;
+            }
+            Instant expiresAt = Instant.now().plusSeconds(Math.max(0, body.expiresIn() - 60));
+            return new AccessToken(body.accessToken(), expiresAt);
+        } catch (RestClientException ex) {
+            LOG.warn("Unable to fetch Twitch app token", ex);
+            return null;
+        }
+    }
+
+    private record AccessToken(String token, Instant expiresAt) {
+        boolean isExpired() {
+            return expiresAt == null || expiresAt.isBefore(Instant.now());
+        }
+    }
+
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    private record TokenResponse(@JsonProperty("access_token") String accessToken, @JsonProperty("expires_in") long expiresIn) {}
+}

src/main/java/dev/kruhlmann/imgfloat/service/TwitchEmoteService.java

@@ -0,0 +1,340 @@
+package dev.kruhlmann.imgfloat.service;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.Duration;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.web.client.RestTemplateBuilder;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Service;
+import org.springframework.web.client.RestClientException;
+import org.springframework.web.client.RestTemplate;
+import org.springframework.web.util.UriComponentsBuilder;
+
+@Service
+public class TwitchEmoteService {
+
+    private static final Logger LOG = LoggerFactory.getLogger(TwitchEmoteService.class);
+    private static final String GLOBAL_EMOTE_URL = "https://api.twitch.tv/helix/chat/emotes/global";
+    private static final String CHANNEL_EMOTE_URL = "https://api.twitch.tv/helix/chat/emotes";
+    private static final String USERS_URL = "https://api.twitch.tv/helix/users";
+
+    private final RestTemplate restTemplate;
+    private final TwitchAppAccessTokenService tokenService;
+    private final Path cacheRoot;
+    private final Map<String, CachedEmote> emoteCache = new ConcurrentHashMap<>();
+    private final Map<String, List<CachedEmote>> channelEmoteCache = new ConcurrentHashMap<>();
+    private volatile List<CachedEmote> globalEmotes = List.of();
+
+    public TwitchEmoteService(
+        RestTemplateBuilder builder,
+        TwitchAppAccessTokenService tokenService,
+        @Value("${IMGFLOAT_TWITCH_EMOTE_CACHE_PATH:#{null}}") String cachePath
+    ) {
+        this.restTemplate = builder
+            .setConnectTimeout(Duration.ofSeconds(20))
+            .setReadTimeout(Duration.ofSeconds(20))
+            .build();
+        this.tokenService = tokenService;
+        String root = cachePath != null
+            ? cachePath
+            : Paths.get(System.getProperty("java.io.tmpdir"), "imgfloat-emotes").toString();
+        this.cacheRoot = Paths.get(root).normalize().toAbsolutePath();
+        try {
+            Files.createDirectories(this.cacheRoot);
+        } catch (IOException ex) {
+            throw new IllegalStateException("Failed to create Twitch emote cache directory", ex);
+        }
+        warmGlobalEmotes();
+    }
+
+    public List<EmoteDescriptor> getGlobalEmotes() {
+        if (globalEmotes.isEmpty()) {
+            warmGlobalEmotes();
+        }
+        return globalEmotes.stream().map(CachedEmote::descriptor).toList();
+    }
+
+    public List<EmoteDescriptor> getChannelEmotes(String channelLogin) {
+        if (channelLogin == null || channelLogin.isBlank()) {
+            return List.of();
+        }
+        String normalized = channelLogin.toLowerCase(Locale.ROOT);
+        List<CachedEmote> emotes = channelEmoteCache.computeIfAbsent(normalized, this::fetchChannelEmotes);
+        return emotes.stream().map(CachedEmote::descriptor).toList();
+    }
+
+    public Optional<EmoteAsset> loadEmoteAsset(String emoteId) {
+        if (emoteId == null || emoteId.isBlank()) {
+            return Optional.empty();
+        }
+        CachedEmote cached = emoteCache.get(emoteId);
+        if (cached == null) {
+            cached = restoreFromDisk(emoteId).orElse(null);
+        }
+        if (cached == null) {
+            return Optional.empty();
+        }
+        try {
+            byte[] bytes = Files.readAllBytes(cached.path());
+            return Optional.of(new EmoteAsset(bytes, cached.mediaType()));
+        } catch (IOException ex) {
+            LOG.warn("Unable to read cached emote {}", emoteId, ex);
+            return Optional.empty();
+        }
+    }
+
+    private void warmGlobalEmotes() {
+        List<TwitchEmoteData> data = fetchEmotes(GLOBAL_EMOTE_URL);
+        if (data.isEmpty()) {
+            return;
+        }
+        List<CachedEmote> cached = data
+            .stream()
+            .map(this::cacheEmote)
+            .filter(Optional::isPresent)
+            .map(Optional::get)
+            .toList();
+        globalEmotes = cached;
+        LOG.info("Loaded {} global Twitch emotes", cached.size());
+    }
+
+    private List<CachedEmote> fetchChannelEmotes(String channelLogin) {
+        String broadcasterId = fetchBroadcasterId(channelLogin).orElse(null);
+        if (broadcasterId == null) {
+            return List.of();
+        }
+        UriComponentsBuilder builder = UriComponentsBuilder.fromHttpUrl(CHANNEL_EMOTE_URL)
+            .queryParam("broadcaster_id", broadcasterId);
+        List<TwitchEmoteData> data = fetchEmotes(builder.toUriString());
+        if (data.isEmpty()) {
+            return List.of();
+        }
+        List<CachedEmote> cached = data
+            .stream()
+            .map(this::cacheEmote)
+            .filter(Optional::isPresent)
+            .map(Optional::get)
+            .toList();
+        LOG.info("Loaded {} Twitch emotes for {}", cached.size(), channelLogin);
+        return cached;
+    }
+
+    private Optional<String> fetchBroadcasterId(String channelLogin) {
+        Optional<String> token = tokenService.getAccessToken();
+        Optional<String> clientId = tokenService.getClientId();
+        if (token.isEmpty() || clientId.isEmpty()) {
+            return Optional.empty();
+        }
+        UriComponentsBuilder builder = UriComponentsBuilder.fromHttpUrl(USERS_URL).queryParam("login", channelLogin);
+        HttpHeaders headers = new HttpHeaders();
+        headers.setBearerAuth(token.get());
+        headers.add("Client-ID", clientId.get());
+        try {
+            ResponseEntity<TwitchUsersResponse> response = restTemplate.exchange(
+                builder.build(true).toUri(),
+                HttpMethod.GET,
+                new HttpEntity<>(headers),
+                TwitchUsersResponse.class
+            );
+            TwitchUsersResponse body = response.getBody();
+            if (body == null || body.data() == null || body.data().isEmpty()) {
+                return Optional.empty();
+            }
+            return body.data().stream().findFirst().map(TwitchUserData::id);
+        } catch (RestClientException ex) {
+            LOG.warn("Unable to fetch Twitch broadcaster id for {}", channelLogin, ex);
+            return Optional.empty();
+        }
+    }
+
+    private List<TwitchEmoteData> fetchEmotes(String url) {
+        Optional<String> token = tokenService.getAccessToken();
+        Optional<String> clientId = tokenService.getClientId();
+        if (token.isEmpty() || clientId.isEmpty()) {
+            return List.of();
+        }
+        HttpHeaders headers = new HttpHeaders();
+        headers.setBearerAuth(token.get());
+        headers.add("Client-ID", clientId.get());
+        try {
+            ResponseEntity<TwitchEmoteResponse> response = restTemplate.exchange(
+                URI.create(url),
+                HttpMethod.GET,
+                new HttpEntity<>(headers),
+                TwitchEmoteResponse.class
+            );
+            TwitchEmoteResponse body = response.getBody();
+            if (body == null || body.data() == null) {
+                return List.of();
+            }
+            return body.data();
+        } catch (RestClientException ex) {
+            LOG.warn("Unable to fetch Twitch emotes from {}", url, ex);
+            return List.of();
+        }
+    }
+
+    private Optional<CachedEmote> cacheEmote(TwitchEmoteData emote) {
+        if (emote == null || emote.id() == null || emote.id().isBlank()) {
+            return Optional.empty();
+        }
+        return Optional.ofNullable(
+            emoteCache.computeIfAbsent(
+                emote.id(),
+                (id) -> {
+                    String imageUrl = selectImageUrl(emote);
+                    if (imageUrl == null) {
+                        return null;
+                    }
+                    return downloadEmote(id, emote.name(), imageUrl);
+                }
+            )
+        );
+    }
+
+    private CachedEmote downloadEmote(String id, String name, String imageUrl) {
+        Path filePath = resolveEmotePath(id, imageUrl);
+        MediaType mediaType = null;
+
+        if (!Files.exists(filePath)) {
+            try {
+                ResponseEntity<byte[]> response = restTemplate.getForEntity(URI.create(imageUrl), byte[].class);
+                byte[] bytes = response.getBody();
+                if (bytes == null || bytes.length == 0) {
+                    return null;
+                }
+                mediaType = response.getHeaders().getContentType();
+                Files.write(filePath, bytes);
+            } catch (IOException | RestClientException ex) {
+                LOG.warn("Unable to download Twitch emote {}", id, ex);
+                return null;
+            }
+        }
+
+        if (mediaType == null) {
+            mediaType = mediaTypeFromPath(filePath);
+        }
+        return new CachedEmote(id, name, filePath, mediaType != null ? mediaType.toString() : "image/png");
+    }
+
+    private Optional<CachedEmote> restoreFromDisk(String id) {
+        if (id == null || id.isBlank()) {
+            return Optional.empty();
+        }
+        try {
+            List<Path> candidates;
+            try (var stream = Files.list(cacheRoot)) {
+                candidates = stream.filter((path) -> path.getFileName().toString().startsWith(id + ".")).toList();
+            }
+            if (candidates.isEmpty()) {
+                return Optional.empty();
+            }
+            Path path = candidates.get(0);
+            MediaType mediaType = mediaTypeFromPath(path);
+            CachedEmote cached = new CachedEmote(id, id, path, mediaType != null ? mediaType.toString() : "image/png");
+            emoteCache.put(id, cached);
+            return Optional.of(cached);
+        } catch (IOException ex) {
+            LOG.warn("Unable to restore cached emote {}", id, ex);
+            return Optional.empty();
+        }
+    }
+
+    private Path resolveEmotePath(String id, String imageUrl) {
+        String extension = extensionFromUrl(imageUrl).orElse("png");
+        return cacheRoot.resolve(id + "." + extension);
+    }
+
+    private Optional<String> extensionFromUrl(String imageUrl) {
+        if (imageUrl == null) {
+            return Optional.empty();
+        }
+        try {
+            String path = URI.create(imageUrl).getPath();
+            int dot = path.lastIndexOf('.');
+            if (dot == -1) {
+                return Optional.empty();
+            }
+            return Optional.of(path.substring(dot + 1));
+        } catch (IllegalArgumentException ex) {
+            return Optional.empty();
+        }
+    }
+
+    private MediaType mediaTypeFromPath(Path path) {
+        if (path == null) {
+            return null;
+        }
+        String name = path.getFileName().toString().toLowerCase(Locale.ROOT);
+        if (name.endsWith(".gif")) {
+            return MediaType.IMAGE_GIF;
+        }
+        if (name.endsWith(".webp")) {
+            return MediaType.parseMediaType("image/webp");
+        }
+        if (name.endsWith(".jpg") || name.endsWith(".jpeg")) {
+            return MediaType.IMAGE_JPEG;
+        }
+        return MediaType.IMAGE_PNG;
+    }
+
+    private String selectImageUrl(TwitchEmoteData emote) {
+        if (emote == null || emote.images() == null) {
+            return null;
+        }
+        String url = emote.images().url1x();
+        if (url == null || url.isBlank()) {
+            url = emote.images().url2x();
+        }
+        if (url == null || url.isBlank()) {
+            url = emote.images().url4x();
+        }
+        return url;
+    }
+
+    public record EmoteDescriptor(String id, String name, String url) {}
+
+    public record EmoteAsset(byte[] bytes, String mediaType) {}
+
+    private record CachedEmote(String id, String name, Path path, String mediaType) {
+        EmoteDescriptor descriptor() {
+            return new EmoteDescriptor(id, name, "/api/twitch/emotes/" + id);
+        }
+    }
+
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    private record TwitchEmoteResponse(List<TwitchEmoteData> data) {}
+
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    private record TwitchEmoteData(String id, String name, Images images) {}
+
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    private record Images(
+        @JsonProperty("url_1x") String url1x,
+        @JsonProperty("url_2x") String url2x,
+        @JsonProperty("url_4x") String url4x
+    ) {}
+
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    private record TwitchUsersResponse(List<TwitchUserData> data) {}
+
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    private record TwitchUserData(String id) {}
+}

src/main/resources/static/js/broadcast.js

@@ -7,6 +7,15 @@ const scriptLayer = document.getElementById("broadcast-script-layer");
 setUpElectronWindowFrame();
 
 const renderer = new BroadcastRenderer({ canvas, scriptLayer, broadcaster, showToast });
+fetch(`/api/twitch/emotes?channel=${encodeURIComponent(broadcaster)}`)
+    .then((response) => {
+        if (!response.ok) {
+            throw new Error("Failed to load Twitch emotes");
+        }
+        return response.json();
+    })
+    .then((catalog) => renderer.setEmoteCatalog(catalog))
+    .catch((error) => console.warn("Unable to load Twitch emotes", error));
 const disconnectChat = connectTwitchChat(
     broadcaster,
     ({ channel, displayName, message, tags, prefix, raw }) => {

src/main/resources/static/js/broadcast/renderer.js

@@ -26,6 +26,9 @@ export class BroadcastRenderer {
         this.scriptAttachmentCache = new Map();
         this.scriptAttachmentsByAssetId = new Map();
         this.chatMessages = [];
+        this.emoteCatalog = [];
+        this.emoteCatalogById = new Map();
+        this.lastChatPruneAt = 0;
 
         this.obsBrowser = !!globalThis.obsstudio;
         this.supportsAnimatedDecode =
@@ -322,6 +325,11 @@ export class BroadcastRenderer {
     }
 
     renderFrame() {
+        const now = Date.now();
+        if (now - this.lastChatPruneAt > 1000) {
+            this.lastChatPruneAt = now;
+            this.pruneChatMessages(now);
+        }
         this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);
         getRenderOrder(this.state).forEach((asset) => this.drawAsset(asset));
     }
@@ -412,6 +420,7 @@ export class BroadcastRenderer {
         );
         this.scriptWorkerReady = true;
         this.updateScriptWorkerChatMessages();
+        this.updateScriptWorkerEmoteCatalog();
     }
 
     updateScriptWorkerCanvas() {
@@ -439,14 +448,119 @@ export class BroadcastRenderer {
         });
     }
 
+    updateScriptWorkerEmoteCatalog() {
+        if (!this.scriptWorker || !this.scriptWorkerReady) {
+            return;
+        }
+        this.scriptWorker.postMessage({
+            type: "emoteCatalog",
+            payload: {
+                emotes: this.emoteCatalog,
+            },
+        });
+    }
+
+    setEmoteCatalog(catalog) {
+        const globalEmotes = Array.isArray(catalog?.global) ? catalog.global : [];
+        const channelEmotes = Array.isArray(catalog?.channel) ? catalog.channel : [];
+        this.emoteCatalog = [...globalEmotes, ...channelEmotes];
+        this.emoteCatalogById = new Map(
+            this.emoteCatalog.map((entry) => [String(entry?.id || ""), entry]).filter(([key]) => key),
+        );
+        if (this.chatMessages.length) {
+            this.chatMessages = this.chatMessages.map((message) => {
+                if (!Array.isArray(message.fragments)) {
+                    return message;
+                }
+                const fragments = message.fragments.map((fragment) => {
+                    if (fragment.type !== "emote" || fragment.url) {
+                        return fragment;
+                    }
+                    const emoteInfo = this.emoteCatalogById.get(String(fragment.id));
+                    if (!emoteInfo) {
+                        return fragment;
+                    }
+                    return { ...fragment, url: emoteInfo.url, name: emoteInfo.name || fragment.name };
+                });
+                return { ...message, fragments };
+            });
+            this.updateScriptWorkerChatMessages();
+        }
+        this.updateScriptWorkerEmoteCatalog();
+    }
+
+    pruneChatMessages(now = Date.now()) {
+        const cutoff = now - 120_000;
+        const pruned = this.chatMessages.filter((item) => item.timestamp >= cutoff);
+        if (pruned.length !== this.chatMessages.length) {
+            this.chatMessages = pruned;
+            this.updateScriptWorkerChatMessages();
+        }
+    }
+
+    parseEmoteOffsets(rawEmotes) {
+        if (!rawEmotes) {
+            return [];
+        }
+        return rawEmotes
+            .split("/")
+            .flatMap((emoteEntry) => {
+                if (!emoteEntry) {
+                    return [];
+                }
+                const [id, positions] = emoteEntry.split(":");
+                if (!id || !positions) {
+                    return [];
+                }
+                return positions.split(",").map((range) => {
+                    const [start, end] = range.split("-").map((value) => Number.parseInt(value, 10));
+                    return Number.isFinite(start) && Number.isFinite(end) ? { id, start, end } : null;
+                });
+            })
+            .filter(Boolean);
+    }
+
+    buildMessageFragments(message, tags) {
+        if (!message) {
+            return [];
+        }
+        const emotes = this.parseEmoteOffsets(tags?.emotes);
+        if (!emotes.length) {
+            return [{ type: "text", text: message }];
+        }
+        const sorted = emotes.sort((a, b) => a.start - b.start);
+        const fragments = [];
+        let cursor = 0;
+        sorted.forEach((emote) => {
+            if (emote.start > cursor) {
+                fragments.push({ type: "text", text: message.slice(cursor, emote.start) });
+            }
+            const emoteText = message.slice(emote.start, emote.end + 1);
+            const emoteInfo = this.emoteCatalogById.get(String(emote.id));
+            fragments.push({
+                type: "emote",
+                id: emote.id,
+                text: emoteText,
+                name: emoteInfo?.name || emoteText,
+                url: emoteInfo?.url || null,
+            });
+            cursor = emote.end + 1;
+        });
+        if (cursor < message.length) {
+            fragments.push({ type: "text", text: message.slice(cursor) });
+        }
+        return fragments;
+    }
+
     receiveChatMessage(message) {
         if (!message) {
             return;
         }
         const now = Date.now();
-        const entry = { ...message, timestamp: now };
+        const fragments = this.buildMessageFragments(message.message || "", message.tags);
-        const cutoff = now - 120_000;
+        const entry = { ...message, fragments, timestamp: now };
-        this.chatMessages = [...this.chatMessages, entry].filter((item) => item.timestamp >= cutoff);
+        this.chatMessages = [...this.chatMessages, entry];
+        this.pruneChatMessages(now);
         this.updateScriptWorkerChatMessages();
     }
 

src/main/resources/static/js/broadcast/script-worker.js

@@ -10,6 +10,7 @@ const allowedImportUrls = new Set();
 const nativeImportScripts = typeof self.importScripts === "function" ? self.importScripts.bind(self) : null;
 const sharedDependencyUrls = ["/js/vendor/three.min.js", "/js/vendor/GLTFLoader.js", "/js/vendor/OBJLoader.js"];
 let chatMessages = [];
+let emoteCatalog = [];
 
 function normalizeUrl(url) {
     try {
@@ -39,48 +40,6 @@ function importAllowedScripts(...urls) {
     return nativeImportScripts(...resolved);
 }
 
-function disableNetworkApis() {
-    const nativeFetch = typeof self.fetch === "function" ? self.fetch.bind(self) : null;
-    const blockedApis = {
-        fetch: (...args) => {
-            if (!nativeFetch) {
-                throw new Error("Network access is disabled in asset scripts.");
-            }
-            const request = new Request(...args);
-            const url = normalizeUrl(request.url);
-            if (!allowedFetchUrls.has(url)) {
-                throw new Error("Network access is disabled in asset scripts.");
-            }
-            return nativeFetch(request);
-        },
-        XMLHttpRequest: undefined,
-        WebSocket: undefined,
-        EventSource: undefined,
-        importScripts: (...urls) => importAllowedScripts(...urls),
-    };
-
-    Object.entries(blockedApis).forEach(([key, value]) => {
-        if (!(key in self)) {
-            return;
-        }
-        try {
-            Object.defineProperty(self, key, {
-                value,
-                writable: false,
-                configurable: false,
-            });
-        } catch (error) {
-            try {
-                self[key] = value;
-            } catch (_error) {
-                // ignore if the API cannot be overridden in this environment
-            }
-        }
-    });
-}
-
-disableNetworkApis();
-
 function loadSharedDependencies() {
     if (!nativeImportScripts || sharedDependencyUrls.length === 0) {
         return;
@@ -106,6 +65,32 @@ function refreshAllowedFetchUrls() {
             }
         });
     });
+    if (Array.isArray(emoteCatalog)) {
+        emoteCatalog.forEach((emote) => {
+            if (emote?.url) {
+                const normalized = normalizeUrl(emote.url);
+                if (normalized) {
+                    allowedFetchUrls.add(normalized);
+                }
+            }
+        });
+    }
+    if (Array.isArray(chatMessages)) {
+        chatMessages.forEach((message) => {
+            const fragments = message?.fragments;
+            if (!Array.isArray(fragments)) {
+                return;
+            }
+            fragments.forEach((fragment) => {
+                if (fragment?.url) {
+                    const normalized = normalizeUrl(fragment.url);
+                    if (normalized) {
+                        allowedFetchUrls.add(normalized);
+                    }
+                }
+            });
+        });
+    }
 }
 
 function reportScriptError(id, stage, error) {
@@ -139,6 +124,7 @@ function updateScriptContexts() {
         script.context.width = script.canvas?.width ?? 0;
         script.context.height = script.canvas?.height ?? 0;
         script.context.chatMessages = chatMessages;
+        script.context.emoteCatalog = emoteCatalog;
     });
 }
 
@@ -182,7 +168,7 @@ function stopTickLoopIfIdle() {
 
 function createScriptHandlers(source, context, state, sourceLabel = "") {
     const contextPrelude =
-        "const { canvas, ctx, channelName, width, height, now, deltaMs, elapsedMs, assets, chatMessages, playAudio } = context;";
+        "const { canvas, ctx, channelName, width, height, now, deltaMs, elapsedMs, assets, chatMessages, emoteCatalog, playAudio } = context;";
     const sourceUrl = sourceLabel ? `\n//# sourceURL=${sourceLabel}` : "";
     const factory = new Function(
         "context",
@@ -242,6 +228,7 @@ self.addEventListener("message", (event) => {
             elapsedMs: 0,
             assets: Array.isArray(payload.attachments) ? payload.attachments : [],
             chatMessages,
+            emoteCatalog,
             playAudio: (attachment) => {
                 const attachmentId = typeof attachment === "string" ? attachment : attachment?.id;
                 if (!attachmentId) {
@@ -310,6 +297,13 @@ self.addEventListener("message", (event) => {
 
     if (type === "chatMessages") {
         chatMessages = Array.isArray(payload?.messages) ? payload.messages : [];
+        refreshAllowedFetchUrls();
+        updateScriptContexts();
+    }
+
+    if (type === "emoteCatalog") {
+        emoteCatalog = Array.isArray(payload?.emotes) ? payload.emotes : [];
+        refreshAllowedFetchUrls();
         updateScriptContexts();
     }
 });