Replace sharp with wasm-vips for image processing

Fixes #696

- Replaced sharp dependency with wasm-vips (WebAssembly build of libvips)
- Eliminates native build requirements that caused installation failures
- Added vips.ts singleton wrapper for async initialization
- Updated image-resize.ts and image-convert.ts to use wasm-vips API
- Added unit tests for image processing functionality

packages/coding-agent/CHANGELOG.md

@@ -5,6 +5,7 @@
 ### Changed
 
 - Light theme colors adjusted for WCAG AA compliance (4.5:1 contrast ratio against white backgrounds)
+- Replaced `sharp` with `wasm-vips` for image processing (resize, PNG conversion). Eliminates native build requirements that caused installation failures on some systems. ([#696](https://github.com/badlogic/pi-mono/issues/696))
 
 ### Added
 

packages/coding-agent/package.json

@@ -51,7 +51,7 @@
 		"marked": "^15.0.12",
 		"minimatch": "^10.1.1",
 		"proper-lockfile": "^4.1.2",
-		"sharp": "^0.34.2"
+		"wasm-vips": "^0.0.16"
 	},
 	"devDependencies": {
 		"@types/diff": "^7.0.2",

packages/coding-agent/src/utils/image-convert.ts

@@ -1,3 +1,5 @@
+import { getVips } from "./vips.js";
+
 /**
  * Convert image to PNG format for terminal display.
  * Kitty graphics protocol requires PNG format (f=100).
@@ -11,16 +13,23 @@ export async function convertToPng(
 		return { data: base64Data, mimeType };
 	}
 
+	const vips = await getVips();
+	if (!vips) {
+		// wasm-vips not available
+		return null;
+	}
+
 	try {
-		const sharp = (await import("sharp")).default;
 		const buffer = Buffer.from(base64Data, "base64");
-		const pngBuffer = await sharp(buffer).png().toBuffer();
+		const img = vips.Image.newFromBuffer(buffer);
+		const pngBuffer = img.writeToBuffer(".png");
+		img.delete();
 		return {
-			data: pngBuffer.toString("base64"),
+			data: Buffer.from(pngBuffer).toString("base64"),
 			mimeType: "image/png",
 		};
 	} catch {
-		// Sharp not available or conversion failed
+		// Conversion failed
 		return null;
 	}
 }

packages/coding-agent/src/utils/image-resize.ts

@@ -1,4 +1,5 @@
 import type { ImageContent } from "@mariozechner/pi-ai";
+import { getVips } from "./vips.js";
 
 export interface ImageResizeOptions {
 	maxWidth?: number; // Default: 2000
@@ -29,9 +30,9 @@ const DEFAULT_OPTIONS: Required<ImageResizeOptions> = {
 
 /** Helper to pick the smaller of two buffers */
 function pickSmaller(
-	a: { buffer: Buffer; mimeType: string },
-	b: { buffer: Buffer; mimeType: string },
-): { buffer: Buffer; mimeType: string } {
+	a: { buffer: Uint8Array; mimeType: string },
+	b: { buffer: Uint8Array; mimeType: string },
+): { buffer: Uint8Array; mimeType: string } {
 	return a.buffer.length <= b.buffer.length ? a : b;
 }
 
@@ -39,7 +40,7 @@ function pickSmaller(
  * Resize an image to fit within the specified max dimensions and file size.
  * Returns the original image if it already fits within the limits.
  *
- * Uses sharp for image processing. If sharp is not available (e.g., in some
+ * Uses wasm-vips for image processing. If wasm-vips is not available (e.g., in some
  * environments), returns the original image unchanged.
  *
  * Strategy for staying under maxBytes:
@@ -52,12 +53,29 @@ export async function resizeImage(img: ImageContent, options?: ImageResizeOption
 	const opts = { ...DEFAULT_OPTIONS, ...options };
 	const buffer = Buffer.from(img.data, "base64");
 
-	let sharp: typeof import("sharp") | undefined;
+	const vipsOrNull = await getVips();
+	if (!vipsOrNull) {
+		// wasm-vips not available - return original image
+		// We can't get dimensions without vips, so return 0s
+		return {
+			data: img.data,
+			mimeType: img.mimeType,
+			originalWidth: 0,
+			originalHeight: 0,
+			width: 0,
+			height: 0,
+			wasResized: false,
+		};
+	}
+	// Capture non-null reference for use in nested functions
+	const vips = vipsOrNull;
+
+	// Load image to get metadata
+	let sourceImg: InstanceType<typeof vips.Image>;
 	try {
-		sharp = (await import("sharp")).default;
+		sourceImg = vips.Image.newFromBuffer(buffer);
 	} catch {
-		// Sharp not available - return original image
-		// We can't get dimensions without sharp, so return 0s
+		// Failed to load image
 		return {
 			data: img.data,
 			mimeType: img.mimeType,
@@ -69,16 +87,14 @@ export async function resizeImage(img: ImageContent, options?: ImageResizeOption
 		};
 	}
 
-	const sharpImg = sharp(buffer);
-	const metadata = await sharpImg.metadata();
-
-	const originalWidth = metadata.width ?? 0;
-	const originalHeight = metadata.height ?? 0;
-	const format = metadata.format ?? img.mimeType?.split("/")[1] ?? "png";
+	const originalWidth = sourceImg.width;
+	const originalHeight = sourceImg.height;
 
 	// Check if already within all limits (dimensions AND size)
 	const originalSize = buffer.length;
 	if (originalWidth <= opts.maxWidth && originalHeight <= opts.maxHeight && originalSize <= opts.maxBytes) {
+		sourceImg.delete();
+		const format = img.mimeType?.split("/")[1] ?? "png";
 		return {
 			data: img.data,
 			mimeType: img.mimeType ?? `image/${format}`,
@@ -104,37 +120,45 @@ export async function resizeImage(img: ImageContent, options?: ImageResizeOption
 	}
 
 	// Helper to resize and encode in both formats, returning the smaller one
-	async function tryBothFormats(
+	function tryBothFormats(
 		width: number,
 		height: number,
 		jpegQuality: number,
-	): Promise<{ buffer: Buffer; mimeType: string }> {
-		const resized = await sharp!(buffer)
-			.resize(width, height, { fit: "inside", withoutEnlargement: true })
-			.toBuffer();
+	): { buffer: Uint8Array; mimeType: string } {
+		// Load image fresh and resize using scale factor
+		// (Using newFromBuffer + resize instead of thumbnailBuffer to avoid lazy re-read issues)
+		const img = vips.Image.newFromBuffer(buffer);
+		const scale = Math.min(width / img.width, height / img.height);
+		const resized = scale < 1 ? img.resize(scale) : img;
 
-		const [pngBuffer, jpegBuffer] = await Promise.all([
-			sharp!(resized).png({ compressionLevel: 9 }).toBuffer(),
-			sharp!(resized).jpeg({ quality: jpegQuality }).toBuffer(),
-		]);
+		const pngBuffer = resized.writeToBuffer(".png");
+		const jpegBuffer = resized.writeToBuffer(".jpg", { Q: jpegQuality });
+
+		if (resized !== img) {
+			resized.delete();
+		}
+		img.delete();
 
 		return pickSmaller({ buffer: pngBuffer, mimeType: "image/png" }, { buffer: jpegBuffer, mimeType: "image/jpeg" });
 	}
 
+	// Clean up the source image
+	sourceImg.delete();
+
 	// Try to produce an image under maxBytes
 	const qualitySteps = [85, 70, 55, 40];
 	const scaleSteps = [1.0, 0.75, 0.5, 0.35, 0.25];
 
-	let best: { buffer: Buffer; mimeType: string };
+	let best: { buffer: Uint8Array; mimeType: string };
 	let finalWidth = targetWidth;
 	let finalHeight = targetHeight;
 
 	// First attempt: resize to target dimensions, try both formats
-	best = await tryBothFormats(targetWidth, targetHeight, opts.jpegQuality);
+	best = tryBothFormats(targetWidth, targetHeight, opts.jpegQuality);
 
 	if (best.buffer.length <= opts.maxBytes) {
 		return {
-			data: best.buffer.toString("base64"),
+			data: Buffer.from(best.buffer).toString("base64"),
 			mimeType: best.mimeType,
 			originalWidth,
 			originalHeight,
@@ -146,11 +170,11 @@ export async function resizeImage(img: ImageContent, options?: ImageResizeOption
 
 	// Still too large - try JPEG with decreasing quality (and compare to PNG each time)
 	for (const quality of qualitySteps) {
-		best = await tryBothFormats(targetWidth, targetHeight, quality);
+		best = tryBothFormats(targetWidth, targetHeight, quality);
 
 		if (best.buffer.length <= opts.maxBytes) {
 			return {
-				data: best.buffer.toString("base64"),
+				data: Buffer.from(best.buffer).toString("base64"),
 				mimeType: best.mimeType,
 				originalWidth,
 				originalHeight,
@@ -172,11 +196,11 @@ export async function resizeImage(img: ImageContent, options?: ImageResizeOption
 		}
 
 		for (const quality of qualitySteps) {
-			best = await tryBothFormats(finalWidth, finalHeight, quality);
+			best = tryBothFormats(finalWidth, finalHeight, quality);
 
 			if (best.buffer.length <= opts.maxBytes) {
 				return {
-					data: best.buffer.toString("base64"),
+					data: Buffer.from(best.buffer).toString("base64"),
 					mimeType: best.mimeType,
 					originalWidth,
 					originalHeight,
@@ -191,7 +215,7 @@ export async function resizeImage(img: ImageContent, options?: ImageResizeOption
 	// Last resort: return smallest version we produced even if over limit
 	// (the API will reject it, but at least we tried everything)
 	return {
-		data: best.buffer.toString("base64"),
+		data: Buffer.from(best.buffer).toString("base64"),
 		mimeType: best.mimeType,
 		originalWidth,
 		originalHeight,

packages/coding-agent/src/utils/vips.ts

@@ -0,0 +1,40 @@
+/**
+ * Singleton wrapper for wasm-vips initialization.
+ * wasm-vips requires async initialization, so we cache the instance.
+ */
+
+import type Vips from "wasm-vips";
+
+let vipsInstance: Awaited<ReturnType<typeof Vips>> | null = null;
+let vipsInitPromise: Promise<Awaited<ReturnType<typeof Vips>> | null> | null = null;
+
+/**
+ * Get the initialized wasm-vips instance.
+ * Returns null if wasm-vips is not available or fails to initialize.
+ */
+export async function getVips(): Promise<Awaited<ReturnType<typeof Vips>> | null> {
+	if (vipsInstance) {
+		return vipsInstance;
+	}
+
+	if (vipsInitPromise) {
+		return vipsInitPromise;
+	}
+
+	vipsInitPromise = (async () => {
+		try {
+			const VipsInit = (await import("wasm-vips")).default;
+			vipsInstance = await VipsInit();
+			return vipsInstance;
+		} catch {
+			// wasm-vips not available
+			return null;
+		}
+	})();
+
+	const result = await vipsInitPromise;
+	if (!result) {
+		vipsInitPromise = null; // Allow retry on failure
+	}
+	return result;
+}

packages/coding-agent/test/image-processing.test.ts

@@ -0,0 +1,144 @@
+/**
+ * Tests for image processing utilities using wasm-vips.
+ */
+
+import { describe, expect, it } from "vitest";
+import { convertToPng } from "../src/utils/image-convert.js";
+import { formatDimensionNote, resizeImage } from "../src/utils/image-resize.js";
+import { getVips } from "../src/utils/vips.js";
+
+// Small 2x2 red PNG image (base64)
+const TINY_PNG = "iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAIAAAD91JpzAAAADklEQVQI12P4z8DAwMAAAA0BA/m5sb9AAAAAAElFTkSuQmCC";
+
+// Small 2x2 blue JPEG image (base64)
+const TINY_JPEG =
+	"/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAACAAIDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAAAAAAAAAAAAn/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/8QAFQEBAQAAAAAAAAAAAAAAAAAAAAX/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIRAxEAPwCwAB//2Q==";
+
+// 100x100 gray PNG (generated with wasm-vips)
+const MEDIUM_PNG_100x100 =
+	"iVBORw0KGgoAAAANSUhEUgAAAGQAAABkCAIAAAD/gAIDAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAAtGVYSWZJSSoACAAAAAYAEgEDAAEAAAABAAAAGgEFAAEAAABWAAAAGwEFAAEAAABeAAAAKAEDAAEAAAACAAAAEwIDAAEAAAABAAAAaYcEAAEAAABmAAAAAAAAADhjAADoAwAAOGMAAOgDAAAGAACQBwAEAAAAMDIxMAGRBwAEAAAAAQIDAACgBwAEAAAAMDEwMAGgAwABAAAA//8AAAKgBAABAAAAZAAAAAOgBAABAAAAZAAAAAAAAAC1xMTxAAAA4klEQVR4nO3QoQEAAAiAME/3dF+QvmUSs7zNP8WswKzArMCswKzArMCswKzArMCswKzArMCswKzArMCswKzArMCswKzArMCswKzArMCswKzArMCswKzArMCswKzArMCswKzArMCswKzArMCswKzArMCswKzArMCswKzArMCswKzArMCswKzArMCswKzArMCswKzArMCswKzArMCswKzArMCswKzArMCswKzArMCswKzArMCswKzArMCswKzArMCswKzArMCswKzArMCswKzArMCswKzArMCswKzArMCswKz9zzpHfptnWvrkoQAAAABJRU5ErkJggg==";
+
+// 200x200 colored PNG (generated with wasm-vips)
+const LARGE_PNG_200x200 =
+	"iVBORw0KGgoAAAANSUhEUgAAAMgAAADICAIAAAAiOjnJAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAAtGVYSWZJSSoACAAAAAYAEgEDAAEAAAABAAAAGgEFAAEAAABWAAAAGwEFAAEAAABeAAAAKAEDAAEAAAACAAAAEwIDAAEAAAABAAAAaYcEAAEAAABmAAAAAAAAADhjAADoAwAAOGMAAOgDAAAGAACQBwAEAAAAMDIxMAGRBwAEAAAAAQIDAACgBwAEAAAAMDEwMAGgAwABAAAA//8AAAKgBAABAAAAyAAAAAOgBAABAAAAyAAAAAAAAADqHRv+AAAD8UlEQVR4nO2UAQnAQACEFtZMy/SxVmJDdggmOOUu7hMtwNsZXG3aAnxwLoVVWKewiuD85V97LN8BixSW74BFCst3wCKF5TtgkcLyHbBIYfkOWKSwfAcsUli+AxYpLN8BixSW74BFCst3wCKF5TtgkcLyHbBIYfkOWKSwfAcsUli+AxYpLN8BixSW74BFCst3wCKF5TtgkcLyHbBIYfkOWKSwfAcsUli+AxYpLN8BixSW74BFCst3wCKF5TtgkcLyHbBIYfkOWKSwfAcsUli+AxYpLN8BixSW74BFCst3wCKF5TtgkcLyHbBIYfkOWKSwfAcsUli+AxYpLN8BixSW74BFCst3wCKF5TtgkcLyHbBIYfkOWKSwfAcsUli+AxYpLN8BixSW74BFCst3wCKF5TtgkcLyHbBIYfkOWKSwfAcsUli+AxYpLN8BixSW74BFCst3wCKF5TtgkcLyHbBIYfkOWKSwfAcsUli+AxYpLN8BixSW74BFCst3wCKF5TtgkcLyHbBIYfkOWKSwfAcsUli+AxYpLN8BixSW74BFCst3wCKF5TtgkcLyHbBIYfkOWKSwfAcsUli+AxYpLN8BixSW74BFCst3wCKF5TtgkcLyHbBIYfkOWKSwfAcsUli+AxYpLN8BixSW74BFCst3wCKF5TtgkcLyHbBIYfkOWKSwfAcsUli+AxYpLN8BixSW74BFCst3wCKF5TtgkcLyHbBIYfkOWKSwfAcsUli+AxYpLN/BJIXlO2CRwvIdsEhh+Q5YpLB8ByxSWL4DFiks3wGLFJbvgEUKy3fAIoXlO2CRwvIdsEhh+Q5YpLB8ByxSWL4DFiks3wGLFJbvgEUKy3fAIoXlO2CRwvIdsEhh+Q5YpLB8ByxSWL4DFiks3wGLFJbvgEUKy3fAIoXlO2CRwvIdsEhh+Q5YpLB8ByxSWL4DFiks3wGLFJbvgEUKy3fAIoXlO2CRwvIdsEhh+Q5YpLB8ByxSWL4DFiks3wGLFJbvgEUKy3fAIoXlO2CRwvIdsEhh+Q5YpLB8ByxSWL4DFiks3wGLFJbvgEUKy3fAIoXlO2CRwvIdsEhh+Q5YpLB8ByxSWL4DFiks3wGLFJbvgEUKy3fAIoXlO2CRwvIdsEhh+Q5YpLB8ByxSWL4DFiks3wGLFJbvgEUKy3fAIoXlO2CRwvIdsEhh+Q5YpLB8ByxSWL4DFiks3wGLFJbvgEUKy3fAIoXlO2CRwvIdsEhh+Q5YpLB8ByxSWL4DFiks3wGLFJbvgEUKy3fAIoXlO2CRwvIdsEhh+Q5YpLB8ByxSWL4DFiks3wGLFJbvgEUKy3fAIoXlO2CRwvIdsEhh+Q5Y5AHNA7iPx5BmcQAAAABJRU5ErkJggg==";
+
+describe("wasm-vips initialization", () => {
+	it("should initialize wasm-vips successfully", async () => {
+		const vips = await getVips();
+		expect(vips).not.toBeNull();
+	});
+
+	it("should return cached instance on subsequent calls", async () => {
+		const vips1 = await getVips();
+		const vips2 = await getVips();
+		expect(vips1).toBe(vips2);
+	});
+});
+
+describe("convertToPng", () => {
+	it("should return original data for PNG input", async () => {
+		const result = await convertToPng(TINY_PNG, "image/png");
+		expect(result).not.toBeNull();
+		expect(result!.data).toBe(TINY_PNG);
+		expect(result!.mimeType).toBe("image/png");
+	});
+
+	it("should convert JPEG to PNG", async () => {
+		const result = await convertToPng(TINY_JPEG, "image/jpeg");
+		expect(result).not.toBeNull();
+		expect(result!.mimeType).toBe("image/png");
+		// Result should be valid base64
+		expect(() => Buffer.from(result!.data, "base64")).not.toThrow();
+		// PNG magic bytes
+		const buffer = Buffer.from(result!.data, "base64");
+		expect(buffer[0]).toBe(0x89);
+		expect(buffer[1]).toBe(0x50); // 'P'
+		expect(buffer[2]).toBe(0x4e); // 'N'
+		expect(buffer[3]).toBe(0x47); // 'G'
+	});
+});
+
+describe("resizeImage", () => {
+	it("should return original image if within limits", async () => {
+		const result = await resizeImage(
+			{ type: "image", data: TINY_PNG, mimeType: "image/png" },
+			{ maxWidth: 100, maxHeight: 100, maxBytes: 1024 * 1024 },
+		);
+
+		expect(result.wasResized).toBe(false);
+		expect(result.data).toBe(TINY_PNG);
+		expect(result.originalWidth).toBe(2);
+		expect(result.originalHeight).toBe(2);
+		expect(result.width).toBe(2);
+		expect(result.height).toBe(2);
+	});
+
+	it("should resize image exceeding dimension limits", async () => {
+		const result = await resizeImage(
+			{ type: "image", data: MEDIUM_PNG_100x100, mimeType: "image/png" },
+			{ maxWidth: 50, maxHeight: 50, maxBytes: 1024 * 1024 },
+		);
+
+		expect(result.wasResized).toBe(true);
+		expect(result.originalWidth).toBe(100);
+		expect(result.originalHeight).toBe(100);
+		expect(result.width).toBeLessThanOrEqual(50);
+		expect(result.height).toBeLessThanOrEqual(50);
+	});
+
+	it("should resize image exceeding byte limit", async () => {
+		const originalBuffer = Buffer.from(LARGE_PNG_200x200, "base64");
+		const originalSize = originalBuffer.length;
+
+		// Set maxBytes to less than the original image size
+		const result = await resizeImage(
+			{ type: "image", data: LARGE_PNG_200x200, mimeType: "image/png" },
+			{ maxWidth: 2000, maxHeight: 2000, maxBytes: Math.floor(originalSize / 2) },
+		);
+
+		// Should have tried to reduce size
+		const resultBuffer = Buffer.from(result.data, "base64");
+		expect(resultBuffer.length).toBeLessThan(originalSize);
+	});
+
+	it("should handle JPEG input", async () => {
+		const result = await resizeImage(
+			{ type: "image", data: TINY_JPEG, mimeType: "image/jpeg" },
+			{ maxWidth: 100, maxHeight: 100, maxBytes: 1024 * 1024 },
+		);
+
+		expect(result.wasResized).toBe(false);
+		expect(result.originalWidth).toBe(2);
+		expect(result.originalHeight).toBe(2);
+	});
+});
+
+describe("formatDimensionNote", () => {
+	it("should return undefined for non-resized images", () => {
+		const note = formatDimensionNote({
+			data: "",
+			mimeType: "image/png",
+			originalWidth: 100,
+			originalHeight: 100,
+			width: 100,
+			height: 100,
+			wasResized: false,
+		});
+		expect(note).toBeUndefined();
+	});
+
+	it("should return formatted note for resized images", () => {
+		const note = formatDimensionNote({
+			data: "",
+			mimeType: "image/png",
+			originalWidth: 2000,
+			originalHeight: 1000,
+			width: 1000,
+			height: 500,
+			wasResized: true,
+		});
+		expect(note).toContain("original 2000x1000");
+		expect(note).toContain("displayed at 1000x500");
+		expect(note).toContain("2.00"); // scale factor
+	});
+});