More attempts to get an image under 5MB (#424)

packages/coding-agent/src/utils/image-resize.ts

@@ -3,6 +3,7 @@ import type { ImageContent } from "@mariozechner/pi-ai";
 export interface ImageResizeOptions {
 	maxWidth?: number; // Default: 2000
 	maxHeight?: number; // Default: 2000
+	maxBytes?: number; // Default: 4.5MB (below Anthropic's 5MB limit)
 	jpegQuality?: number; // Default: 80
 }
 
@@ -16,18 +17,36 @@ export interface ResizedImage {
 	wasResized: boolean;
 }
 
+// 4.5MB - provides headroom below Anthropic's 5MB limit
+const DEFAULT_MAX_BYTES = 4.5 * 1024 * 1024;
+
 const DEFAULT_OPTIONS: Required<ImageResizeOptions> = {
 	maxWidth: 2000,
 	maxHeight: 2000,
+	maxBytes: DEFAULT_MAX_BYTES,
 	jpegQuality: 80,
 };
 
+/** Helper to pick the smaller of two buffers */
+function pickSmaller(
+	a: { buffer: Buffer; mimeType: string },
+	b: { buffer: Buffer; mimeType: string },
+): { buffer: Buffer; mimeType: string } {
+	return a.buffer.length <= b.buffer.length ? a : b;
+}
+
 /**
- * Resize an image to fit within the specified max dimensions.
+ * Resize an image to fit within the specified max dimensions and file size.
  * Returns the original image if it already fits within the limits.
  *
  * Uses sharp for image processing. If sharp is not available (e.g., in some
  * environments), returns the original image unchanged.
+ *
+ * Strategy for staying under maxBytes:
+ * 1. First resize to maxWidth/maxHeight
+ * 2. Try both PNG and JPEG formats, pick the smaller one
+ * 3. If still too large, try JPEG with decreasing quality
+ * 4. If still too large, progressively reduce dimensions
  */
 export async function resizeImage(img: ImageContent, options?: ImageResizeOptions): Promise<ResizedImage> {
 	const opts = { ...DEFAULT_OPTIONS, ...options };
@@ -53,71 +72,131 @@ export async function resizeImage(img: ImageContent, options?: ImageResizeOption
 	const sharpImg = sharp(buffer);
 	const metadata = await sharpImg.metadata();
 
-	const width = metadata.width ?? 0;
-	const height = metadata.height ?? 0;
+	const originalWidth = metadata.width ?? 0;
+	const originalHeight = metadata.height ?? 0;
 	const format = metadata.format ?? img.mimeType?.split("/")[1] ?? "png";
 
-	// Check if already within limits
-	if (width <= opts.maxWidth && height <= opts.maxHeight) {
+	// Check if already within all limits (dimensions AND size)
+	const originalSize = buffer.length;
+	if (originalWidth <= opts.maxWidth && originalHeight <= opts.maxHeight && originalSize <= opts.maxBytes) {
 		return {
 			data: img.data,
 			mimeType: img.mimeType ?? `image/${format}`,
-			originalWidth: width,
-			originalHeight: height,
-			width,
-			height,
+			originalWidth,
+			originalHeight,
+			width: originalWidth,
+			height: originalHeight,
 			wasResized: false,
 		};
 	}
 
-	// Calculate new dimensions maintaining aspect ratio
-	let newWidth = width;
-	let newHeight = height;
+	// Calculate initial dimensions respecting max limits
+	let targetWidth = originalWidth;
+	let targetHeight = originalHeight;
 
-	if (newWidth > opts.maxWidth) {
-		newHeight = Math.round((newHeight * opts.maxWidth) / newWidth);
-		newWidth = opts.maxWidth;
+	if (targetWidth > opts.maxWidth) {
+		targetHeight = Math.round((targetHeight * opts.maxWidth) / targetWidth);
+		targetWidth = opts.maxWidth;
 	}
-	if (newHeight > opts.maxHeight) {
-		newWidth = Math.round((newWidth * opts.maxHeight) / newHeight);
-		newHeight = opts.maxHeight;
+	if (targetHeight > opts.maxHeight) {
+		targetWidth = Math.round((targetWidth * opts.maxHeight) / targetHeight);
+		targetHeight = opts.maxHeight;
 	}
 
-	// Resize the image
-	const resized = await sharp(buffer)
-		.resize(newWidth, newHeight, { fit: "inside", withoutEnlargement: true })
-		.toBuffer();
+	// Helper to resize and encode in both formats, returning the smaller one
+	async function tryBothFormats(
+		width: number,
+		height: number,
+		jpegQuality: number,
+	): Promise<{ buffer: Buffer; mimeType: string }> {
+		const resized = await sharp!(buffer)
+			.resize(width, height, { fit: "inside", withoutEnlargement: true })
+			.toBuffer();
 
-	// Determine output format - preserve original if possible, otherwise use JPEG
-	let outputMimeType: string;
-	let outputBuffer: Buffer;
+		const [pngBuffer, jpegBuffer] = await Promise.all([
+			sharp!(resized).png({ compressionLevel: 9 }).toBuffer(),
+			sharp!(resized).jpeg({ quality: jpegQuality }).toBuffer(),
+		]);
 
-	if (format === "jpeg" || format === "jpg") {
-		outputBuffer = await sharp(resized).jpeg({ quality: opts.jpegQuality }).toBuffer();
-		outputMimeType = "image/jpeg";
-	} else if (format === "png") {
-		outputBuffer = resized;
-		outputMimeType = "image/png";
-	} else if (format === "gif") {
-		// GIF resize might not preserve animation; convert to PNG for quality
-		outputBuffer = resized;
-		outputMimeType = "image/png";
-	} else if (format === "webp") {
-		outputBuffer = resized;
-		outputMimeType = "image/webp";
-	} else {
-		// Default to JPEG for unknown formats
-		outputBuffer = await sharp(resized).jpeg({ quality: opts.jpegQuality }).toBuffer();
-		outputMimeType = "image/jpeg";
+		return pickSmaller({ buffer: pngBuffer, mimeType: "image/png" }, { buffer: jpegBuffer, mimeType: "image/jpeg" });
 	}
 
+	// Try to produce an image under maxBytes
+	const qualitySteps = [85, 70, 55, 40];
+	const scaleSteps = [1.0, 0.75, 0.5, 0.35, 0.25];
+
+	let best: { buffer: Buffer; mimeType: string };
+	let finalWidth = targetWidth;
+	let finalHeight = targetHeight;
+
+	// First attempt: resize to target dimensions, try both formats
+	best = await tryBothFormats(targetWidth, targetHeight, opts.jpegQuality);
+
+	if (best.buffer.length <= opts.maxBytes) {
+		return {
+			data: best.buffer.toString("base64"),
+			mimeType: best.mimeType,
+			originalWidth,
+			originalHeight,
+			width: finalWidth,
+			height: finalHeight,
+			wasResized: true,
+		};
+	}
+
+	// Still too large - try JPEG with decreasing quality (and compare to PNG each time)
+	for (const quality of qualitySteps) {
+		best = await tryBothFormats(targetWidth, targetHeight, quality);
+
+		if (best.buffer.length <= opts.maxBytes) {
+			return {
+				data: best.buffer.toString("base64"),
+				mimeType: best.mimeType,
+				originalWidth,
+				originalHeight,
+				width: finalWidth,
+				height: finalHeight,
+				wasResized: true,
+			};
+		}
+	}
+
+	// Still too large - reduce dimensions progressively
+	for (const scale of scaleSteps) {
+		finalWidth = Math.round(targetWidth * scale);
+		finalHeight = Math.round(targetHeight * scale);
+
+		// Skip if dimensions are too small
+		if (finalWidth < 100 || finalHeight < 100) {
+			break;
+		}
+
+		for (const quality of qualitySteps) {
+			best = await tryBothFormats(finalWidth, finalHeight, quality);
+
+			if (best.buffer.length <= opts.maxBytes) {
+				return {
+					data: best.buffer.toString("base64"),
+					mimeType: best.mimeType,
+					originalWidth,
+					originalHeight,
+					width: finalWidth,
+					height: finalHeight,
+					wasResized: true,
+				};
+			}
+		}
+	}
+
+	// Last resort: return smallest version we produced even if over limit
+	// (the API will reject it, but at least we tried everything)
 	return {
-		data: outputBuffer.toString("base64"),
-		mimeType: outputMimeType,
-		originalWidth: width,
-		originalHeight: height,
-		width: newWidth,
-		height: newHeight,
+		data: best.buffer.toString("base64"),
+		mimeType: best.mimeType,
+		originalWidth,
+		originalHeight,
+		width: finalWidth,
+		height: finalHeight,
 		wasResized: true,
 	};
 }