From 69dc6b07887e41b2d3eca7f3d84bc9905da6ddd6 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Sat, 3 Jan 2026 16:54:57 +0100 Subject: [PATCH] More attempts to get an image under 5MB (#424) --- .../coding-agent/src/utils/image-resize.ts | 175 +++++++++++++----- 1 file changed, 127 insertions(+), 48 deletions(-) diff --git a/packages/coding-agent/src/utils/image-resize.ts b/packages/coding-agent/src/utils/image-resize.ts index 3505cff9..9ab166b3 100644 --- a/packages/coding-agent/src/utils/image-resize.ts +++ b/packages/coding-agent/src/utils/image-resize.ts @@ -3,6 +3,7 @@ import type { ImageContent } from "@mariozechner/pi-ai"; export interface ImageResizeOptions { maxWidth?: number; // Default: 2000 maxHeight?: number; // Default: 2000 + maxBytes?: number; // Default: 4.5MB (below Anthropic's 5MB limit) jpegQuality?: number; // Default: 80 } @@ -16,18 +17,36 @@ export interface ResizedImage { wasResized: boolean; } +// 4.5MB - provides headroom below Anthropic's 5MB limit +const DEFAULT_MAX_BYTES = 4.5 * 1024 * 1024; + const DEFAULT_OPTIONS: Required = { maxWidth: 2000, maxHeight: 2000, + maxBytes: DEFAULT_MAX_BYTES, jpegQuality: 80, }; +/** Helper to pick the smaller of two buffers */ +function pickSmaller( + a: { buffer: Buffer; mimeType: string }, + b: { buffer: Buffer; mimeType: string }, +): { buffer: Buffer; mimeType: string } { + return a.buffer.length <= b.buffer.length ? a : b; +} + /** - * Resize an image to fit within the specified max dimensions. + * Resize an image to fit within the specified max dimensions and file size. * Returns the original image if it already fits within the limits. * * Uses sharp for image processing. If sharp is not available (e.g., in some * environments), returns the original image unchanged. + * + * Strategy for staying under maxBytes: + * 1. First resize to maxWidth/maxHeight + * 2. Try both PNG and JPEG formats, pick the smaller one + * 3. If still too large, try JPEG with decreasing quality + * 4. If still too large, progressively reduce dimensions */ export async function resizeImage(img: ImageContent, options?: ImageResizeOptions): Promise { const opts = { ...DEFAULT_OPTIONS, ...options }; @@ -53,71 +72,131 @@ export async function resizeImage(img: ImageContent, options?: ImageResizeOption const sharpImg = sharp(buffer); const metadata = await sharpImg.metadata(); - const width = metadata.width ?? 0; - const height = metadata.height ?? 0; + const originalWidth = metadata.width ?? 0; + const originalHeight = metadata.height ?? 0; const format = metadata.format ?? img.mimeType?.split("/")[1] ?? "png"; - // Check if already within limits - if (width <= opts.maxWidth && height <= opts.maxHeight) { + // Check if already within all limits (dimensions AND size) + const originalSize = buffer.length; + if (originalWidth <= opts.maxWidth && originalHeight <= opts.maxHeight && originalSize <= opts.maxBytes) { return { data: img.data, mimeType: img.mimeType ?? `image/${format}`, - originalWidth: width, - originalHeight: height, - width, - height, + originalWidth, + originalHeight, + width: originalWidth, + height: originalHeight, wasResized: false, }; } - // Calculate new dimensions maintaining aspect ratio - let newWidth = width; - let newHeight = height; + // Calculate initial dimensions respecting max limits + let targetWidth = originalWidth; + let targetHeight = originalHeight; - if (newWidth > opts.maxWidth) { - newHeight = Math.round((newHeight * opts.maxWidth) / newWidth); - newWidth = opts.maxWidth; + if (targetWidth > opts.maxWidth) { + targetHeight = Math.round((targetHeight * opts.maxWidth) / targetWidth); + targetWidth = opts.maxWidth; } - if (newHeight > opts.maxHeight) { - newWidth = Math.round((newWidth * opts.maxHeight) / newHeight); - newHeight = opts.maxHeight; + if (targetHeight > opts.maxHeight) { + targetWidth = Math.round((targetWidth * opts.maxHeight) / targetHeight); + targetHeight = opts.maxHeight; } - // Resize the image - const resized = await sharp(buffer) - .resize(newWidth, newHeight, { fit: "inside", withoutEnlargement: true }) - .toBuffer(); + // Helper to resize and encode in both formats, returning the smaller one + async function tryBothFormats( + width: number, + height: number, + jpegQuality: number, + ): Promise<{ buffer: Buffer; mimeType: string }> { + const resized = await sharp!(buffer) + .resize(width, height, { fit: "inside", withoutEnlargement: true }) + .toBuffer(); - // Determine output format - preserve original if possible, otherwise use JPEG - let outputMimeType: string; - let outputBuffer: Buffer; + const [pngBuffer, jpegBuffer] = await Promise.all([ + sharp!(resized).png({ compressionLevel: 9 }).toBuffer(), + sharp!(resized).jpeg({ quality: jpegQuality }).toBuffer(), + ]); - if (format === "jpeg" || format === "jpg") { - outputBuffer = await sharp(resized).jpeg({ quality: opts.jpegQuality }).toBuffer(); - outputMimeType = "image/jpeg"; - } else if (format === "png") { - outputBuffer = resized; - outputMimeType = "image/png"; - } else if (format === "gif") { - // GIF resize might not preserve animation; convert to PNG for quality - outputBuffer = resized; - outputMimeType = "image/png"; - } else if (format === "webp") { - outputBuffer = resized; - outputMimeType = "image/webp"; - } else { - // Default to JPEG for unknown formats - outputBuffer = await sharp(resized).jpeg({ quality: opts.jpegQuality }).toBuffer(); - outputMimeType = "image/jpeg"; + return pickSmaller({ buffer: pngBuffer, mimeType: "image/png" }, { buffer: jpegBuffer, mimeType: "image/jpeg" }); } + // Try to produce an image under maxBytes + const qualitySteps = [85, 70, 55, 40]; + const scaleSteps = [1.0, 0.75, 0.5, 0.35, 0.25]; + + let best: { buffer: Buffer; mimeType: string }; + let finalWidth = targetWidth; + let finalHeight = targetHeight; + + // First attempt: resize to target dimensions, try both formats + best = await tryBothFormats(targetWidth, targetHeight, opts.jpegQuality); + + if (best.buffer.length <= opts.maxBytes) { + return { + data: best.buffer.toString("base64"), + mimeType: best.mimeType, + originalWidth, + originalHeight, + width: finalWidth, + height: finalHeight, + wasResized: true, + }; + } + + // Still too large - try JPEG with decreasing quality (and compare to PNG each time) + for (const quality of qualitySteps) { + best = await tryBothFormats(targetWidth, targetHeight, quality); + + if (best.buffer.length <= opts.maxBytes) { + return { + data: best.buffer.toString("base64"), + mimeType: best.mimeType, + originalWidth, + originalHeight, + width: finalWidth, + height: finalHeight, + wasResized: true, + }; + } + } + + // Still too large - reduce dimensions progressively + for (const scale of scaleSteps) { + finalWidth = Math.round(targetWidth * scale); + finalHeight = Math.round(targetHeight * scale); + + // Skip if dimensions are too small + if (finalWidth < 100 || finalHeight < 100) { + break; + } + + for (const quality of qualitySteps) { + best = await tryBothFormats(finalWidth, finalHeight, quality); + + if (best.buffer.length <= opts.maxBytes) { + return { + data: best.buffer.toString("base64"), + mimeType: best.mimeType, + originalWidth, + originalHeight, + width: finalWidth, + height: finalHeight, + wasResized: true, + }; + } + } + } + + // Last resort: return smallest version we produced even if over limit + // (the API will reject it, but at least we tried everything) return { - data: outputBuffer.toString("base64"), - mimeType: outputMimeType, - originalWidth: width, - originalHeight: height, - width: newWidth, - height: newHeight, + data: best.buffer.toString("base64"), + mimeType: best.mimeType, + originalWidth, + originalHeight, + width: finalWidth, + height: finalHeight, wasResized: true, }; }