From 5c5084481bd1650863c0760724ccebe59ffec20d Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sun, 4 Jan 2026 01:05:22 +0100 Subject: [PATCH] feat(coding-agent): clipboard image paste support via Ctrl+V (fixes #419) --- package-lock.json | 145 ++++++++++++++++++ packages/coding-agent/CHANGELOG.md | 1 + packages/coding-agent/README.md | 5 + packages/coding-agent/package.json | 1 + .../interactive/components/custom-editor.ts | 9 +- .../src/modes/interactive/interactive-mode.ts | 59 +++++++ packages/tui/CHANGELOG.md | 1 + packages/tui/src/components/editor.ts | 14 +- 8 files changed, 233 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 837ca306..ef365ac3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -259,6 +259,150 @@ "url": "https://github.com/sponsors/Borewit" } }, + "node_modules/@crosscopy/clipboard": { + "version": "0.2.8", + "resolved": "https://registry.npmjs.org/@crosscopy/clipboard/-/clipboard-0.2.8.tgz", + "integrity": "sha512-0qRWscafAHzQ+DdfXX+YgPN2KDTIzWBNfN5Q6z1CgCWsRxtkwK8HfQUc00xIejfRWSGWPIxcCTg82hvg06bodg==", + "license": "MIT", + "engines": { + "node": ">= 10" + }, + "optionalDependencies": { + "@crosscopy/clipboard-darwin-arm64": "0.2.8", + "@crosscopy/clipboard-darwin-universal": "0.2.8", + "@crosscopy/clipboard-darwin-x64": "0.2.8", + "@crosscopy/clipboard-linux-arm64-gnu": "0.2.8", + "@crosscopy/clipboard-linux-riscv64-gnu": "0.2.8", + "@crosscopy/clipboard-linux-x64-gnu": "0.2.8", + "@crosscopy/clipboard-win32-arm64-msvc": "0.2.8", + "@crosscopy/clipboard-win32-x64-msvc": "0.2.8" + } + }, + "node_modules/@crosscopy/clipboard-darwin-arm64": { + "version": "0.2.8", + "resolved": "https://registry.npmjs.org/@crosscopy/clipboard-darwin-arm64/-/clipboard-darwin-arm64-0.2.8.tgz", + "integrity": "sha512-Y36ST9k5JZgtDE6SBT45bDNkPKBHd4UEIZgWnC0iC4kAWwdjPmsZ8Mn8e5W0YUKowJ/BDcO+EGm2tVTPQOQKXg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@crosscopy/clipboard-darwin-universal": { + "version": "0.2.8", + "resolved": "https://registry.npmjs.org/@crosscopy/clipboard-darwin-universal/-/clipboard-darwin-universal-0.2.8.tgz", + "integrity": "sha512-btGV1tLpJWZ4iKa66niahvpZpVRJzgQnYUE+PUX3YYZzaWD0ESuHuVtKVC8sR+b4dsXIiWW5skXbcRmLsF4rtA==", + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@crosscopy/clipboard-darwin-x64": { + "version": "0.2.8", + "resolved": "https://registry.npmjs.org/@crosscopy/clipboard-darwin-x64/-/clipboard-darwin-x64-0.2.8.tgz", + "integrity": "sha512-0QMKf0XrLZrprYYXU4lgaTuzbnYPh9wH6PvsfDB1FZvWf6rOi0syTaBZYnoghbQe700qwLPEfBRjgljJ3Tn6oA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@crosscopy/clipboard-linux-arm64-gnu": { + "version": "0.2.8", + "resolved": "https://registry.npmjs.org/@crosscopy/clipboard-linux-arm64-gnu/-/clipboard-linux-arm64-gnu-0.2.8.tgz", + "integrity": "sha512-8YrU03MRsygymqEcHkNgqCqSCQbYRmJCnMXeS4i8FYeOkAxBEeRvPbHoNmI10uppXJZNZgfIKM7Qqk9tEHiwqQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@crosscopy/clipboard-linux-riscv64-gnu": { + "version": "0.2.8", + "resolved": "https://registry.npmjs.org/@crosscopy/clipboard-linux-riscv64-gnu/-/clipboard-linux-riscv64-gnu-0.2.8.tgz", + "integrity": "sha512-/QWLhnb0QYVjEv5GOAC1q+1DaezYU8Th+IoDKUCsR5i43Cqm+g+N/I2K35yo3J+HHkK9XNbtIDZDXlFgK6tRUg==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@crosscopy/clipboard-linux-x64-gnu": { + "version": "0.2.8", + "resolved": "https://registry.npmjs.org/@crosscopy/clipboard-linux-x64-gnu/-/clipboard-linux-x64-gnu-0.2.8.tgz", + "integrity": "sha512-j17eaF/onP/6VAGGKtxA1KmmkErmdjta9gMdMV/yUmgeBYzJ9fMpWUzbk2vmaOyXfhaSzR/sk1P6VLBmvCpqHg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@crosscopy/clipboard-win32-arm64-msvc": { + "version": "0.2.8", + "resolved": "https://registry.npmjs.org/@crosscopy/clipboard-win32-arm64-msvc/-/clipboard-win32-arm64-msvc-0.2.8.tgz", + "integrity": "sha512-MVkMyuYN3y5v0s4HrijM0iA8hZVmpUhHd8X4zKG30t4nE6MbOjOt/8EabMrVmGZlsLeOL2sa0o8Wo9bvhWU+vA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@crosscopy/clipboard-win32-x64-msvc": { + "version": "0.2.8", + "resolved": "https://registry.npmjs.org/@crosscopy/clipboard-win32-x64-msvc/-/clipboard-win32-x64-msvc-0.2.8.tgz", + "integrity": "sha512-/GpiB4B3lSgg7eCLDQw9NfFjtQFjo0S88IL+EK54Hx7ZgAP4Ad/ezP/8dw0cA+N/M6iPYy0reCIjW9st82/uxw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, "node_modules/@emnapi/runtime": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.8.0.tgz", @@ -6778,6 +6922,7 @@ "version": "0.32.3", "license": "MIT", "dependencies": { + "@crosscopy/clipboard": "^0.2.8", "@mariozechner/pi-agent-core": "^0.32.3", "@mariozechner/pi-ai": "^0.32.3", "@mariozechner/pi-tui": "^0.32.3", diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 4c08bee1..760b8874 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -8,6 +8,7 @@ ### Added +- Clipboard image paste support via `Ctrl+V`. Images are saved to a temp file and attached to the message. Works on macOS, Windows, and Linux (X11). ([#419](https://github.com/badlogic/pi-mono/issues/419)) - Configurable keybindings via `~/.pi/agent/keybindings.json`. All keyboard shortcuts (editor navigation, deletion, app actions like model cycling, etc.) can now be customized. Supports multiple bindings per action. ([#405](https://github.com/badlogic/pi-mono/pull/405) by [@hjanuschka](https://github.com/hjanuschka)) - `/quit` and `/exit` slash commands to gracefully exit the application. Unlike double Ctrl+C, these properly await hook and custom tool cleanup handlers before exiting. ([#426](https://github.com/badlogic/pi-mono/pull/426) by [@ben-vargas](https://github.com/ben-vargas)) diff --git a/packages/coding-agent/README.md b/packages/coding-agent/README.md index ba0c3b15..6ac16480 100644 --- a/packages/coding-agent/README.md +++ b/packages/coding-agent/README.md @@ -266,6 +266,7 @@ Both modes are configurable via `/settings`: "one-at-a-time" delivers messages o | Ctrl+O | Toggle tool output expansion | | Ctrl+T | Toggle thinking block visibility | | Ctrl+G | Edit message in external editor (`$VISUAL` or `$EDITOR`) | +| Ctrl+V | Paste image from clipboard | ### Custom Keybindings @@ -362,6 +363,10 @@ Run multiple commands before prompting; all outputs are included together. ### Image Support +**Pasting images:** Press `Ctrl+V` to paste an image from your clipboard. + +**Dragging images:** Drag image files onto the terminal to insert their path. On macOS, you can also drag the screenshot thumbnail (after Cmd+Shift+4) directly onto the terminal. + **Attaching images:** Include image paths in your message: ``` diff --git a/packages/coding-agent/package.json b/packages/coding-agent/package.json index 1bff3ab6..274e240e 100644 --- a/packages/coding-agent/package.json +++ b/packages/coding-agent/package.json @@ -38,6 +38,7 @@ "prepublishOnly": "npm run clean && npm run build" }, "dependencies": { + "@crosscopy/clipboard": "^0.2.8", "@mariozechner/pi-agent-core": "^0.32.3", "@mariozechner/pi-ai": "^0.32.3", "@mariozechner/pi-tui": "^0.32.3", diff --git a/packages/coding-agent/src/modes/interactive/components/custom-editor.ts b/packages/coding-agent/src/modes/interactive/components/custom-editor.ts index f9e3fa4e..00afa5e7 100644 --- a/packages/coding-agent/src/modes/interactive/components/custom-editor.ts +++ b/packages/coding-agent/src/modes/interactive/components/custom-editor.ts @@ -1,4 +1,4 @@ -import { Editor, type EditorTheme } from "@mariozechner/pi-tui"; +import { Editor, type EditorTheme, matchesKey } from "@mariozechner/pi-tui"; import type { AppAction, KeybindingsManager } from "../../../core/keybindings.js"; /** @@ -11,6 +11,7 @@ export class CustomEditor extends Editor { // Special handlers that can be dynamically replaced public onEscape?: () => void; public onCtrlD?: () => void; + public onPasteImage?: () => void; constructor(theme: EditorTheme, keybindings: KeybindingsManager) { super(theme); @@ -25,6 +26,12 @@ export class CustomEditor extends Editor { } handleInput(data: string): void { + // Check for Ctrl+V to handle clipboard image paste + if (matchesKey(data, "ctrl+v")) { + this.onPasteImage?.(); + return; + } + // Check app keybindings first // Escape/interrupt - only if autocomplete is NOT active diff --git a/packages/coding-agent/src/modes/interactive/interactive-mode.ts b/packages/coding-agent/src/modes/interactive/interactive-mode.ts index 0f0b862b..82bc764b 100644 --- a/packages/coding-agent/src/modes/interactive/interactive-mode.ts +++ b/packages/coding-agent/src/modes/interactive/interactive-mode.ts @@ -3,9 +3,11 @@ * Handles TUI rendering and user interaction, delegating business logic to AgentSession. */ +import * as crypto from "node:crypto"; import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; +import Clipboard from "@crosscopy/clipboard"; import type { AgentMessage } from "@mariozechner/pi-agent-core"; import type { AssistantMessage, Message, OAuthProvider } from "@mariozechner/pi-ai"; import type { SlashCommand } from "@mariozechner/pi-tui"; @@ -141,6 +143,10 @@ export class InteractiveMode { // Custom tools for custom rendering private customTools: Map; + // Clipboard image tracking: imageId -> temp file path + private clipboardImages = new Map(); + private clipboardImageCounter = 0; + // Convenience accessors private get agent() { return this.session.agent; @@ -291,6 +297,9 @@ export class InteractiveMode { theme.fg("dim", followUp) + theme.fg("muted", " to queue follow-up") + "\n" + + theme.fg("dim", "ctrl+v") + + theme.fg("muted", " to paste image") + + "\n" + theme.fg("dim", "drop files") + theme.fg("muted", " to attach"); const header = new Text(`${logo}\n${instructions}`, 1, 0); @@ -819,6 +828,52 @@ export class InteractiveMode { this.updateEditorBorderColor(); } }; + + // Handle clipboard image paste (triggered on Ctrl+V) + this.editor.onPasteImage = () => { + this.handleClipboardImagePaste(); + }; + } + + private async handleClipboardImagePaste(): Promise { + try { + if (!Clipboard.hasImage()) { + return; + } + + const imageData = await Clipboard.getImageBinary(); + if (!imageData || imageData.length === 0) { + return; + } + + // Write to temp file + const imageId = ++this.clipboardImageCounter; + const tmpDir = os.tmpdir(); + const fileName = `pi-clipboard-${crypto.randomUUID()}.png`; + const filePath = path.join(tmpDir, fileName); + fs.writeFileSync(filePath, Buffer.from(imageData)); + + // Store mapping and insert marker + this.clipboardImages.set(imageId, filePath); + this.editor.insertTextAtCursor(`[image #${imageId}]`); + this.ui.requestRender(); + } catch { + // Silently ignore clipboard errors (may not have permission, etc.) + } + } + + /** + * Replace [image #N] markers with actual file paths and clear the image map. + */ + private replaceImageMarkers(text: string): string { + let result = text; + for (const [imageId, filePath] of this.clipboardImages) { + const marker = `[image #${imageId}]`; + result = result.replace(marker, filePath); + } + this.clipboardImages.clear(); + this.clipboardImageCounter = 0; + return result; } private setupEditorSubmitHandler(): void { @@ -948,6 +1003,9 @@ export class InteractiveMode { } // If streaming, use prompt() with steer behavior + // Replace image markers with actual file paths + text = this.replaceImageMarkers(text); + // This handles hook commands (execute immediately), slash command expansion, and queueing if (this.session.isStreaming) { this.editor.addToHistory(text); @@ -2379,6 +2437,7 @@ export class InteractiveMode { | \`${toggleThinking}\` | Toggle thinking block visibility | | \`${externalEditor}\` | Edit message in external editor | | \`${followUp}\` | Queue follow-up message | +| \`Ctrl+V\` | Paste image from clipboard | | \`/\` | Slash commands | | \`!\` | Run bash command | `; diff --git a/packages/tui/CHANGELOG.md b/packages/tui/CHANGELOG.md index b218c85b..78e452e5 100644 --- a/packages/tui/CHANGELOG.md +++ b/packages/tui/CHANGELOG.md @@ -8,6 +8,7 @@ ### Added +- `Editor.insertTextAtCursor(text)` method for programmatic text insertion ([#419](https://github.com/badlogic/pi-mono/issues/419)) - `EditorKeybindingsManager` for configurable editor keybindings. Components now use `matchesKey()` and keybindings manager instead of individual `isXxx()` functions. ([#405](https://github.com/badlogic/pi-mono/pull/405) by [@hjanuschka](https://github.com/hjanuschka)) ### Changed diff --git a/packages/tui/src/components/editor.ts b/packages/tui/src/components/editor.ts index 43622010..bee20d5a 100644 --- a/packages/tui/src/components/editor.ts +++ b/packages/tui/src/components/editor.ts @@ -408,7 +408,9 @@ export class Editor implements Component { const endIndex = this.pasteBuffer.indexOf("\x1b[201~"); if (endIndex !== -1) { const pasteContent = this.pasteBuffer.substring(0, endIndex); - this.handlePaste(pasteContent); + if (pasteContent.length > 0) { + this.handlePaste(pasteContent); + } this.isInPaste = false; const remaining = this.pasteBuffer.substring(endIndex + 6); this.pasteBuffer = ""; @@ -707,6 +709,16 @@ export class Editor implements Component { this.setTextInternal(text); } + /** + * Insert text at the current cursor position. + * Used for programmatic insertion (e.g., clipboard image markers). + */ + insertTextAtCursor(text: string): void { + for (const char of text) { + this.insertCharacter(char); + } + } + // All the editor methods from before... private insertCharacter(char: string): void { this.historyIndex = -1; // Exit history browsing mode