mirror of
https://github.com/getcompanion-ai/co-mono.git
synced 2026-04-15 12:03:49 +00:00
Merge feature/tui-inline-images: Inline image rendering for supported terminals
This commit is contained in:
Mario Zechner
commit
27e68d856e
12 changed files with 6684 additions and 23 deletions
|
|
@ -4,6 +4,8 @@
|
|||
|
||||
### Added
|
||||
|
||||
- **Inline image rendering**: Terminals supporting Kitty graphics protocol (Kitty, Ghostty, WezTerm) or iTerm2 inline images now render images inline in tool output. Aspect ratio is preserved by querying terminal cell dimensions on startup. Toggle with `/show-images` command or `terminal.showImages` setting. Falls back to text placeholder on unsupported terminals or when disabled. ([#177](https://github.com/badlogic/pi-mono/pull/177) by [@nicobailon](https://github.com/nicobailon))
|
||||
|
||||
- **Gemini 3 Pro thinking levels**: Thinking level selector now works with Gemini 3 Pro models. Minimal/low map to Google's LOW, medium/high map to Google's HIGH. ([#176](https://github.com/badlogic/pi-mono/pull/176) by [@markusylisiurunen](https://github.com/markusylisiurunen))
|
||||
|
||||
### Fixed
|
||||
|
|
|
|||
|
|
@ -165,6 +165,7 @@ The agent reads, writes, and edits files, and executes commands via bash.
|
|||
| `/compact [instructions]` | Manually compact conversation context |
|
||||
| `/autocompact` | Toggle automatic context compaction |
|
||||
| `/theme` | Select color theme |
|
||||
| `/show-images` | Toggle inline image display (supported terminals only) |
|
||||
|
||||
### Editor Features
|
||||
|
||||
|
|
@ -236,13 +237,17 @@ Run multiple commands before prompting; all outputs are included together.
|
|||
|
||||
### Image Support
|
||||
|
||||
Include image paths in your message:
|
||||
**Attaching images:** Include image paths in your message:
|
||||
|
||||
```
|
||||
You: What's in this screenshot? /path/to/image.png
|
||||
```
|
||||
|
||||
Supported: `.jpg`, `.jpeg`, `.png`, `.gif`, `.webp`
|
||||
Supported formats: `.jpg`, `.jpeg`, `.png`, `.gif`, `.webp`
|
||||
|
||||
**Inline rendering:** On terminals that support the Kitty graphics protocol (Kitty, Ghostty, WezTerm) or iTerm2 inline images, images in tool output are rendered inline. On unsupported terminals, a text placeholder is shown instead.
|
||||
|
||||
Toggle inline images with `/show-images` or set `terminal.showImages: false` in settings.
|
||||
|
||||
---
|
||||
|
||||
|
|
@ -600,6 +605,9 @@ See [Hooks Documentation](docs/hooks.md) for full API reference.
|
|||
"enabled": true,
|
||||
"maxRetries": 3,
|
||||
"baseDelayMs": 2000
|
||||
},
|
||||
"terminal": {
|
||||
"showImages": true
|
||||
}
|
||||
}
|
||||
```
|
||||
|
|
@ -609,6 +617,9 @@ See [Hooks Documentation](docs/hooks.md) for full API reference.
|
|||
- `maxRetries`: Maximum retry attempts. Default: `3`
|
||||
- `baseDelayMs`: Base delay for exponential backoff (2s, 4s, 8s). Default: `2000`
|
||||
|
||||
**Terminal settings:**
|
||||
- `showImages`: Render images inline in supported terminals. Default: `true`
|
||||
|
||||
---
|
||||
|
||||
## CLI Reference
|
||||
|
|
|
|||
|
|
@ -18,6 +18,10 @@ export interface SkillsSettings {
|
|||
enabled?: boolean; // default: true
|
||||
}
|
||||
|
||||
export interface TerminalSettings {
|
||||
showImages?: boolean; // default: true (only relevant if terminal supports images)
|
||||
}
|
||||
|
||||
export interface Settings {
|
||||
lastChangelogVersion?: string;
|
||||
defaultProvider?: string;
|
||||
|
|
@ -33,6 +37,7 @@ export interface Settings {
|
|||
hooks?: string[]; // Array of hook file paths
|
||||
hookTimeout?: number; // Timeout for hook execution in ms (default: 30000)
|
||||
skills?: SkillsSettings;
|
||||
terminal?: TerminalSettings;
|
||||
}
|
||||
|
||||
export class SettingsManager {
|
||||
|
|
@ -237,4 +242,16 @@ export class SettingsManager {
|
|||
this.settings.skills.enabled = enabled;
|
||||
this.save();
|
||||
}
|
||||
|
||||
getShowImages(): boolean {
|
||||
return this.settings.terminal?.showImages ?? true;
|
||||
}
|
||||
|
||||
setShowImages(show: boolean): void {
|
||||
if (!this.settings.terminal) {
|
||||
this.settings.terminal = {};
|
||||
}
|
||||
this.settings.terminal.showImages = show;
|
||||
this.save();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,13 @@
|
|||
import * as os from "node:os";
|
||||
import { Container, Spacer, Text } from "@mariozechner/pi-tui";
|
||||
import {
|
||||
Container,
|
||||
getCapabilities,
|
||||
getImageDimensions,
|
||||
Image,
|
||||
imageFallback,
|
||||
Spacer,
|
||||
Text,
|
||||
} from "@mariozechner/pi-tui";
|
||||
import stripAnsi from "strip-ansi";
|
||||
import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES, formatSize } from "../../../core/tools/truncate.js";
|
||||
import { theme } from "../theme/theme.js";
|
||||
|
|
@ -22,26 +30,32 @@ function replaceTabs(text: string): string {
|
|||
return text.replace(/\t/g, " ");
|
||||
}
|
||||
|
||||
export interface ToolExecutionOptions {
|
||||
showImages?: boolean; // default: true (only used if terminal supports images)
|
||||
}
|
||||
|
||||
/**
|
||||
* Component that renders a tool call with its result (updateable)
|
||||
*/
|
||||
export class ToolExecutionComponent extends Container {
|
||||
private contentText: Text;
|
||||
private imageComponents: Image[] = [];
|
||||
private toolName: string;
|
||||
private args: any;
|
||||
private expanded = false;
|
||||
private showImages: boolean;
|
||||
private result?: {
|
||||
content: Array<{ type: string; text?: string; data?: string; mimeType?: string }>;
|
||||
isError: boolean;
|
||||
details?: any;
|
||||
};
|
||||
|
||||
constructor(toolName: string, args: any) {
|
||||
constructor(toolName: string, args: any, options: ToolExecutionOptions = {}) {
|
||||
super();
|
||||
this.toolName = toolName;
|
||||
this.args = args;
|
||||
this.showImages = options.showImages ?? true;
|
||||
this.addChild(new Spacer(1));
|
||||
// Content with colored background and padding
|
||||
this.contentText = new Text("", 1, 1, (text: string) => theme.bg("toolPendingBg", text));
|
||||
this.addChild(this.contentText);
|
||||
this.updateDisplay();
|
||||
|
|
@ -66,6 +80,11 @@ export class ToolExecutionComponent extends Container {
|
|||
this.updateDisplay();
|
||||
}
|
||||
|
||||
setShowImages(show: boolean): void {
|
||||
this.showImages = show;
|
||||
this.updateDisplay();
|
||||
}
|
||||
|
||||
private updateDisplay(): void {
|
||||
const bgFn = this.result
|
||||
? this.result.isError
|
||||
|
|
@ -75,32 +94,56 @@ export class ToolExecutionComponent extends Container {
|
|||
|
||||
this.contentText.setCustomBgFn(bgFn);
|
||||
this.contentText.setText(this.formatToolExecution());
|
||||
|
||||
for (const img of this.imageComponents) {
|
||||
this.removeChild(img);
|
||||
}
|
||||
this.imageComponents = [];
|
||||
|
||||
if (this.result) {
|
||||
const imageBlocks = this.result.content?.filter((c: any) => c.type === "image") || [];
|
||||
const caps = getCapabilities();
|
||||
|
||||
for (const img of imageBlocks) {
|
||||
// Show inline image only if terminal supports it AND user setting allows it
|
||||
if (caps.images && this.showImages && img.data && img.mimeType) {
|
||||
const imageComponent = new Image(
|
||||
img.data,
|
||||
img.mimeType,
|
||||
{ fallbackColor: (s: string) => theme.fg("toolOutput", s) },
|
||||
{ maxWidthCells: 60 },
|
||||
);
|
||||
this.imageComponents.push(imageComponent);
|
||||
this.addChild(imageComponent);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private getTextOutput(): string {
|
||||
if (!this.result) return "";
|
||||
|
||||
// Extract text from content blocks
|
||||
const textBlocks = this.result.content?.filter((c: any) => c.type === "text") || [];
|
||||
const imageBlocks = this.result.content?.filter((c: any) => c.type === "image") || [];
|
||||
|
||||
// Strip ANSI codes and carriage returns from raw output
|
||||
// (bash may emit colors/formatting, and Windows may include \r)
|
||||
let output = textBlocks
|
||||
.map((c: any) => {
|
||||
let text = stripAnsi(c.text || "").replace(/\r/g, "");
|
||||
// stripAnsi misses some escape sequences like standalone ESC \ (String Terminator)
|
||||
// and leaves orphaned fragments from malformed sequences (e.g. TUI output captured to file)
|
||||
// Clean up: remove ESC + any following char, and control chars except newline/tab
|
||||
text = text.replace(/\x1b./g, "");
|
||||
text = text.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]/g, "");
|
||||
return text;
|
||||
})
|
||||
.join("\n");
|
||||
|
||||
// Add indicator for images
|
||||
if (imageBlocks.length > 0) {
|
||||
const imageIndicators = imageBlocks.map((img: any) => `[Image: ${img.mimeType}]`).join("\n");
|
||||
const caps = getCapabilities();
|
||||
// Show text fallback if terminal doesn't support images OR if user disabled inline images
|
||||
if (imageBlocks.length > 0 && (!caps.images || !this.showImages)) {
|
||||
const imageIndicators = imageBlocks
|
||||
.map((img: any) => {
|
||||
const dims = img.data ? (getImageDimensions(img.data, img.mimeType) ?? undefined) : undefined;
|
||||
return imageFallback(img.mimeType, dims);
|
||||
})
|
||||
.join("\n");
|
||||
output = output ? `${output}\n${imageIndicators}` : imageIndicators;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -12,10 +12,12 @@ import {
|
|||
CombinedAutocompleteProvider,
|
||||
type Component,
|
||||
Container,
|
||||
getCapabilities,
|
||||
Input,
|
||||
Loader,
|
||||
Markdown,
|
||||
ProcessTerminal,
|
||||
SelectList,
|
||||
Spacer,
|
||||
Text,
|
||||
TruncatedText,
|
||||
|
|
@ -52,7 +54,7 @@ import { ThinkingSelectorComponent } from "./components/thinking-selector.js";
|
|||
import { ToolExecutionComponent } from "./components/tool-execution.js";
|
||||
import { UserMessageComponent } from "./components/user-message.js";
|
||||
import { UserMessageSelectorComponent } from "./components/user-message-selector.js";
|
||||
import { getEditorTheme, getMarkdownTheme, onThemeChange, setTheme, theme } from "./theme/theme.js";
|
||||
import { getEditorTheme, getMarkdownTheme, getSelectListTheme, onThemeChange, setTheme, theme } from "./theme/theme.js";
|
||||
|
||||
export class InteractiveMode {
|
||||
private session: AgentSession;
|
||||
|
|
@ -160,6 +162,11 @@ export class InteractiveMode {
|
|||
{ name: "resume", description: "Resume a different session" },
|
||||
];
|
||||
|
||||
// Add image toggle command only if terminal supports images
|
||||
if (getCapabilities().images) {
|
||||
slashCommands.push({ name: "show-images", description: "Toggle inline image display" });
|
||||
}
|
||||
|
||||
// Load hide thinking block setting
|
||||
this.hideThinkingBlock = this.settingsManager.getHideThinkingBlock();
|
||||
|
||||
|
|
@ -585,6 +592,11 @@ export class InteractiveMode {
|
|||
this.editor.setText("");
|
||||
return;
|
||||
}
|
||||
if (text === "/show-images") {
|
||||
this.handleShowImagesCommand();
|
||||
this.editor.setText("");
|
||||
return;
|
||||
}
|
||||
if (text === "/debug") {
|
||||
this.handleDebugCommand();
|
||||
this.editor.setText("");
|
||||
|
|
@ -691,7 +703,9 @@ export class InteractiveMode {
|
|||
if (content.type === "toolCall") {
|
||||
if (!this.pendingTools.has(content.id)) {
|
||||
this.chatContainer.addChild(new Text("", 0, 0));
|
||||
const component = new ToolExecutionComponent(content.name, content.arguments);
|
||||
const component = new ToolExecutionComponent(content.name, content.arguments, {
|
||||
showImages: this.settingsManager.getShowImages(),
|
||||
});
|
||||
this.chatContainer.addChild(component);
|
||||
this.pendingTools.set(content.id, component);
|
||||
} else {
|
||||
|
|
@ -731,7 +745,9 @@ export class InteractiveMode {
|
|||
|
||||
case "tool_execution_start": {
|
||||
if (!this.pendingTools.has(event.toolCallId)) {
|
||||
const component = new ToolExecutionComponent(event.toolName, event.args);
|
||||
const component = new ToolExecutionComponent(event.toolName, event.args, {
|
||||
showImages: this.settingsManager.getShowImages(),
|
||||
});
|
||||
this.chatContainer.addChild(component);
|
||||
this.pendingTools.set(event.toolCallId, component);
|
||||
this.ui.requestRender();
|
||||
|
|
@ -958,7 +974,9 @@ export class InteractiveMode {
|
|||
|
||||
for (const content of assistantMsg.content) {
|
||||
if (content.type === "toolCall") {
|
||||
const component = new ToolExecutionComponent(content.name, content.arguments);
|
||||
const component = new ToolExecutionComponent(content.name, content.arguments, {
|
||||
showImages: this.settingsManager.getShowImages(),
|
||||
});
|
||||
this.chatContainer.addChild(component);
|
||||
|
||||
if (assistantMsg.stopReason === "aborted" || assistantMsg.stopReason === "error") {
|
||||
|
|
@ -1634,6 +1652,51 @@ export class InteractiveMode {
|
|||
this.showStatus(`Auto-compaction: ${newState ? "on" : "off"}`);
|
||||
}
|
||||
|
||||
private handleShowImagesCommand(): void {
|
||||
// Only available if terminal supports images
|
||||
const caps = getCapabilities();
|
||||
if (!caps.images) {
|
||||
this.showWarning("Your terminal does not support inline images");
|
||||
return;
|
||||
}
|
||||
|
||||
const currentValue = this.settingsManager.getShowImages();
|
||||
const items = [
|
||||
{ value: "yes", label: "Yes", description: "Show images inline in terminal" },
|
||||
{ value: "no", label: "No", description: "Show text placeholder instead" },
|
||||
];
|
||||
|
||||
const selector = new SelectList(items, 5, getSelectListTheme());
|
||||
selector.setSelectedIndex(currentValue ? 0 : 1);
|
||||
|
||||
selector.onSelect = (item) => {
|
||||
const newValue = item.value === "yes";
|
||||
this.settingsManager.setShowImages(newValue);
|
||||
this.showStatus(`Inline images: ${newValue ? "on" : "off"}`);
|
||||
this.chatContainer.removeChild(selector);
|
||||
|
||||
// Update all existing tool execution components with new setting
|
||||
for (const child of this.chatContainer.children) {
|
||||
if (child instanceof ToolExecutionComponent) {
|
||||
child.setShowImages(newValue);
|
||||
}
|
||||
}
|
||||
|
||||
this.ui.setFocus(this.editor);
|
||||
this.ui.requestRender();
|
||||
};
|
||||
|
||||
selector.onCancel = () => {
|
||||
this.chatContainer.removeChild(selector);
|
||||
this.ui.setFocus(this.editor);
|
||||
this.ui.requestRender();
|
||||
};
|
||||
|
||||
this.chatContainer.addChild(selector);
|
||||
this.ui.setFocus(selector);
|
||||
this.ui.requestRender();
|
||||
}
|
||||
|
||||
private async executeCompaction(customInstructions?: string, isAuto = false): Promise<void> {
|
||||
// Stop loading animation
|
||||
if (this.loadingAnimation) {
|
||||
|
|
|
|||
|
|
@ -8,7 +8,8 @@ Minimal terminal UI framework with differential rendering and synchronized outpu
|
|||
- **Synchronized Output**: Uses CSI 2026 for atomic screen updates (no flicker)
|
||||
- **Bracketed Paste Mode**: Handles large pastes correctly with markers for >10 line pastes
|
||||
- **Component-based**: Simple Component interface with render() method
|
||||
- **Built-in Components**: Text, Input, Editor, Markdown, Loader, SelectList, Spacer
|
||||
- **Built-in Components**: Text, Input, Editor, Markdown, Loader, SelectList, Spacer, Image
|
||||
- **Inline Images**: Renders images in terminals that support Kitty or iTerm2 graphics protocols
|
||||
- **Autocomplete Support**: File paths and slash commands
|
||||
|
||||
## Quick Start
|
||||
|
|
@ -190,6 +191,24 @@ Empty lines for vertical spacing.
|
|||
const spacer = new Spacer(2); // 2 empty lines (default: 1)
|
||||
```
|
||||
|
||||
### Image
|
||||
|
||||
Renders images inline for terminals that support the Kitty graphics protocol (Kitty, Ghostty, WezTerm) or iTerm2 inline images. Falls back to a text placeholder on unsupported terminals.
|
||||
|
||||
```typescript
|
||||
import { Image } from "@mariozechner/pi-tui";
|
||||
|
||||
const image = new Image(
|
||||
base64Data, // base64-encoded image data
|
||||
"image/png", // MIME type
|
||||
{ fallbackColor: (s) => s }, // theme for fallback text
|
||||
{ maxWidthCells: 60 } // optional: limit width
|
||||
);
|
||||
tui.addChild(image);
|
||||
```
|
||||
|
||||
Supported formats: PNG, JPEG, GIF, WebP. Dimensions are parsed from the image headers automatically.
|
||||
|
||||
## Autocomplete
|
||||
|
||||
### CombinedAutocompleteProvider
|
||||
|
|
|
|||
78
packages/tui/src/components/image.ts
Normal file
78
packages/tui/src/components/image.ts
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
import {
|
||||
getCapabilities,
|
||||
getImageDimensions,
|
||||
type ImageDimensions,
|
||||
imageFallback,
|
||||
renderImage,
|
||||
} from "../terminal-image.js";
|
||||
import type { Component } from "../tui.js";
|
||||
|
||||
export interface ImageTheme {
|
||||
fallbackColor: (str: string) => string;
|
||||
}
|
||||
|
||||
export interface ImageOptions {
|
||||
maxWidthCells?: number;
|
||||
maxHeightCells?: number;
|
||||
filename?: string;
|
||||
}
|
||||
|
||||
export class Image implements Component {
|
||||
private base64Data: string;
|
||||
private mimeType: string;
|
||||
private dimensions: ImageDimensions;
|
||||
private theme: ImageTheme;
|
||||
private options: ImageOptions;
|
||||
|
||||
private cachedLines?: string[];
|
||||
private cachedWidth?: number;
|
||||
|
||||
constructor(
|
||||
base64Data: string,
|
||||
mimeType: string,
|
||||
theme: ImageTheme,
|
||||
options: ImageOptions = {},
|
||||
dimensions?: ImageDimensions,
|
||||
) {
|
||||
this.base64Data = base64Data;
|
||||
this.mimeType = mimeType;
|
||||
this.theme = theme;
|
||||
this.options = options;
|
||||
this.dimensions = dimensions || getImageDimensions(base64Data, mimeType) || { widthPx: 800, heightPx: 600 };
|
||||
}
|
||||
|
||||
invalidate(): void {
|
||||
this.cachedLines = undefined;
|
||||
this.cachedWidth = undefined;
|
||||
}
|
||||
|
||||
render(width: number): string[] {
|
||||
if (this.cachedLines && this.cachedWidth === width) {
|
||||
return this.cachedLines;
|
||||
}
|
||||
|
||||
const maxWidth = Math.min(width - 2, this.options.maxWidthCells ?? 60);
|
||||
|
||||
const caps = getCapabilities();
|
||||
let lines: string[];
|
||||
|
||||
if (caps.images) {
|
||||
const result = renderImage(this.base64Data, this.dimensions, { maxWidthCells: maxWidth });
|
||||
|
||||
if (result) {
|
||||
lines = [result.sequence];
|
||||
} else {
|
||||
const fallback = imageFallback(this.mimeType, this.dimensions, this.options.filename);
|
||||
lines = [this.theme.fallbackColor(fallback)];
|
||||
}
|
||||
} else {
|
||||
const fallback = imageFallback(this.mimeType, this.dimensions, this.options.filename);
|
||||
lines = [this.theme.fallbackColor(fallback)];
|
||||
}
|
||||
|
||||
this.cachedLines = lines;
|
||||
this.cachedWidth = width;
|
||||
|
||||
return lines;
|
||||
}
|
||||
}
|
||||
|
|
@ -9,6 +9,7 @@ export {
|
|||
} from "./autocomplete.js";
|
||||
// Components
|
||||
export { Editor, type EditorTheme } from "./components/editor.js";
|
||||
export { Image, type ImageOptions, type ImageTheme } from "./components/image.js";
|
||||
export { Input } from "./components/input.js";
|
||||
export { Loader } from "./components/loader.js";
|
||||
export { type DefaultTextStyle, Markdown, type MarkdownTheme } from "./components/markdown.js";
|
||||
|
|
@ -18,6 +19,29 @@ export { Text } from "./components/text.js";
|
|||
export { TruncatedText } from "./components/truncated-text.js";
|
||||
// Terminal interface and implementations
|
||||
export { ProcessTerminal, type Terminal } from "./terminal.js";
|
||||
// Terminal image support
|
||||
export {
|
||||
type CellDimensions,
|
||||
calculateImageRows,
|
||||
detectCapabilities,
|
||||
encodeITerm2,
|
||||
encodeKitty,
|
||||
getCapabilities,
|
||||
getCellDimensions,
|
||||
getGifDimensions,
|
||||
getImageDimensions,
|
||||
getJpegDimensions,
|
||||
getPngDimensions,
|
||||
getWebpDimensions,
|
||||
type ImageDimensions,
|
||||
type ImageProtocol,
|
||||
type ImageRenderOptions,
|
||||
imageFallback,
|
||||
renderImage,
|
||||
resetCapabilitiesCache,
|
||||
setCellDimensions,
|
||||
type TerminalCapabilities,
|
||||
} from "./terminal-image.js";
|
||||
export { type Component, Container, TUI } from "./tui.js";
|
||||
// Utilities
|
||||
export { truncateToWidth, visibleWidth } from "./utils.js";
|
||||
|
|
|
|||
340
packages/tui/src/terminal-image.ts
Normal file
340
packages/tui/src/terminal-image.ts
Normal file
|
|
@ -0,0 +1,340 @@
|
|||
export type ImageProtocol = "kitty" | "iterm2" | null;
|
||||
|
||||
export interface TerminalCapabilities {
|
||||
images: ImageProtocol;
|
||||
trueColor: boolean;
|
||||
hyperlinks: boolean;
|
||||
}
|
||||
|
||||
export interface CellDimensions {
|
||||
widthPx: number;
|
||||
heightPx: number;
|
||||
}
|
||||
|
||||
export interface ImageDimensions {
|
||||
widthPx: number;
|
||||
heightPx: number;
|
||||
}
|
||||
|
||||
export interface ImageRenderOptions {
|
||||
maxWidthCells?: number;
|
||||
maxHeightCells?: number;
|
||||
preserveAspectRatio?: boolean;
|
||||
}
|
||||
|
||||
let cachedCapabilities: TerminalCapabilities | null = null;
|
||||
|
||||
// Default cell dimensions - updated by TUI when terminal responds to query
|
||||
let cellDimensions: CellDimensions = { widthPx: 9, heightPx: 18 };
|
||||
|
||||
export function getCellDimensions(): CellDimensions {
|
||||
return cellDimensions;
|
||||
}
|
||||
|
||||
export function setCellDimensions(dims: CellDimensions): void {
|
||||
cellDimensions = dims;
|
||||
}
|
||||
|
||||
export function detectCapabilities(): TerminalCapabilities {
|
||||
const termProgram = process.env.TERM_PROGRAM?.toLowerCase() || "";
|
||||
const term = process.env.TERM?.toLowerCase() || "";
|
||||
const colorTerm = process.env.COLORTERM?.toLowerCase() || "";
|
||||
|
||||
if (process.env.KITTY_WINDOW_ID || termProgram === "kitty") {
|
||||
return { images: "kitty", trueColor: true, hyperlinks: true };
|
||||
}
|
||||
|
||||
if (termProgram === "ghostty" || term.includes("ghostty")) {
|
||||
return { images: "kitty", trueColor: true, hyperlinks: true };
|
||||
}
|
||||
|
||||
if (process.env.WEZTERM_PANE || termProgram === "wezterm") {
|
||||
return { images: "kitty", trueColor: true, hyperlinks: true };
|
||||
}
|
||||
|
||||
if (process.env.ITERM_SESSION_ID || termProgram === "iterm.app") {
|
||||
return { images: "iterm2", trueColor: true, hyperlinks: true };
|
||||
}
|
||||
|
||||
if (termProgram === "vscode") {
|
||||
return { images: null, trueColor: true, hyperlinks: true };
|
||||
}
|
||||
|
||||
if (termProgram === "alacritty") {
|
||||
return { images: null, trueColor: true, hyperlinks: true };
|
||||
}
|
||||
|
||||
const trueColor = colorTerm === "truecolor" || colorTerm === "24bit";
|
||||
return { images: null, trueColor, hyperlinks: true };
|
||||
}
|
||||
|
||||
export function getCapabilities(): TerminalCapabilities {
|
||||
if (!cachedCapabilities) {
|
||||
cachedCapabilities = detectCapabilities();
|
||||
}
|
||||
return cachedCapabilities;
|
||||
}
|
||||
|
||||
export function resetCapabilitiesCache(): void {
|
||||
cachedCapabilities = null;
|
||||
}
|
||||
|
||||
export function encodeKitty(
|
||||
base64Data: string,
|
||||
options: {
|
||||
columns?: number;
|
||||
rows?: number;
|
||||
imageId?: number;
|
||||
} = {},
|
||||
): string {
|
||||
const CHUNK_SIZE = 4096;
|
||||
|
||||
const params: string[] = ["a=T", "f=100", "q=2"];
|
||||
|
||||
if (options.columns) params.push(`c=${options.columns}`);
|
||||
if (options.rows) params.push(`r=${options.rows}`);
|
||||
if (options.imageId) params.push(`i=${options.imageId}`);
|
||||
|
||||
if (base64Data.length <= CHUNK_SIZE) {
|
||||
return `\x1b_G${params.join(",")};${base64Data}\x1b\\`;
|
||||
}
|
||||
|
||||
const chunks: string[] = [];
|
||||
let offset = 0;
|
||||
let isFirst = true;
|
||||
|
||||
while (offset < base64Data.length) {
|
||||
const chunk = base64Data.slice(offset, offset + CHUNK_SIZE);
|
||||
const isLast = offset + CHUNK_SIZE >= base64Data.length;
|
||||
|
||||
if (isFirst) {
|
||||
chunks.push(`\x1b_G${params.join(",")},m=1;${chunk}\x1b\\`);
|
||||
isFirst = false;
|
||||
} else if (isLast) {
|
||||
chunks.push(`\x1b_Gm=0;${chunk}\x1b\\`);
|
||||
} else {
|
||||
chunks.push(`\x1b_Gm=1;${chunk}\x1b\\`);
|
||||
}
|
||||
|
||||
offset += CHUNK_SIZE;
|
||||
}
|
||||
|
||||
return chunks.join("");
|
||||
}
|
||||
|
||||
export function encodeITerm2(
|
||||
base64Data: string,
|
||||
options: {
|
||||
width?: number | string;
|
||||
height?: number | string;
|
||||
name?: string;
|
||||
preserveAspectRatio?: boolean;
|
||||
inline?: boolean;
|
||||
} = {},
|
||||
): string {
|
||||
const params: string[] = [`inline=${options.inline !== false ? 1 : 0}`];
|
||||
|
||||
if (options.width !== undefined) params.push(`width=${options.width}`);
|
||||
if (options.height !== undefined) params.push(`height=${options.height}`);
|
||||
if (options.name) {
|
||||
const nameBase64 = Buffer.from(options.name).toString("base64");
|
||||
params.push(`name=${nameBase64}`);
|
||||
}
|
||||
if (options.preserveAspectRatio === false) {
|
||||
params.push("preserveAspectRatio=0");
|
||||
}
|
||||
|
||||
return `\x1b]1337;File=${params.join(";")}:${base64Data}\x07`;
|
||||
}
|
||||
|
||||
export function calculateImageRows(
|
||||
imageDimensions: ImageDimensions,
|
||||
targetWidthCells: number,
|
||||
cellDimensions: CellDimensions = { widthPx: 9, heightPx: 18 },
|
||||
): number {
|
||||
const targetWidthPx = targetWidthCells * cellDimensions.widthPx;
|
||||
const scale = targetWidthPx / imageDimensions.widthPx;
|
||||
const scaledHeightPx = imageDimensions.heightPx * scale;
|
||||
const rows = Math.ceil(scaledHeightPx / cellDimensions.heightPx);
|
||||
return Math.max(1, rows);
|
||||
}
|
||||
|
||||
export function getPngDimensions(base64Data: string): ImageDimensions | null {
|
||||
try {
|
||||
const buffer = Buffer.from(base64Data, "base64");
|
||||
|
||||
if (buffer.length < 24) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (buffer[0] !== 0x89 || buffer[1] !== 0x50 || buffer[2] !== 0x4e || buffer[3] !== 0x47) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const width = buffer.readUInt32BE(16);
|
||||
const height = buffer.readUInt32BE(20);
|
||||
|
||||
return { widthPx: width, heightPx: height };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function getJpegDimensions(base64Data: string): ImageDimensions | null {
|
||||
try {
|
||||
const buffer = Buffer.from(base64Data, "base64");
|
||||
|
||||
if (buffer.length < 2) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (buffer[0] !== 0xff || buffer[1] !== 0xd8) {
|
||||
return null;
|
||||
}
|
||||
|
||||
let offset = 2;
|
||||
while (offset < buffer.length - 9) {
|
||||
if (buffer[offset] !== 0xff) {
|
||||
offset++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const marker = buffer[offset + 1];
|
||||
|
||||
if (marker >= 0xc0 && marker <= 0xc2) {
|
||||
const height = buffer.readUInt16BE(offset + 5);
|
||||
const width = buffer.readUInt16BE(offset + 7);
|
||||
return { widthPx: width, heightPx: height };
|
||||
}
|
||||
|
||||
if (offset + 3 >= buffer.length) {
|
||||
return null;
|
||||
}
|
||||
const length = buffer.readUInt16BE(offset + 2);
|
||||
if (length < 2) {
|
||||
return null;
|
||||
}
|
||||
offset += 2 + length;
|
||||
}
|
||||
|
||||
return null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function getGifDimensions(base64Data: string): ImageDimensions | null {
|
||||
try {
|
||||
const buffer = Buffer.from(base64Data, "base64");
|
||||
|
||||
if (buffer.length < 10) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const sig = buffer.slice(0, 6).toString("ascii");
|
||||
if (sig !== "GIF87a" && sig !== "GIF89a") {
|
||||
return null;
|
||||
}
|
||||
|
||||
const width = buffer.readUInt16LE(6);
|
||||
const height = buffer.readUInt16LE(8);
|
||||
|
||||
return { widthPx: width, heightPx: height };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function getWebpDimensions(base64Data: string): ImageDimensions | null {
|
||||
try {
|
||||
const buffer = Buffer.from(base64Data, "base64");
|
||||
|
||||
if (buffer.length < 30) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const riff = buffer.slice(0, 4).toString("ascii");
|
||||
const webp = buffer.slice(8, 12).toString("ascii");
|
||||
if (riff !== "RIFF" || webp !== "WEBP") {
|
||||
return null;
|
||||
}
|
||||
|
||||
const chunk = buffer.slice(12, 16).toString("ascii");
|
||||
if (chunk === "VP8 ") {
|
||||
if (buffer.length < 30) return null;
|
||||
const width = buffer.readUInt16LE(26) & 0x3fff;
|
||||
const height = buffer.readUInt16LE(28) & 0x3fff;
|
||||
return { widthPx: width, heightPx: height };
|
||||
} else if (chunk === "VP8L") {
|
||||
if (buffer.length < 25) return null;
|
||||
const bits = buffer.readUInt32LE(21);
|
||||
const width = (bits & 0x3fff) + 1;
|
||||
const height = ((bits >> 14) & 0x3fff) + 1;
|
||||
return { widthPx: width, heightPx: height };
|
||||
} else if (chunk === "VP8X") {
|
||||
if (buffer.length < 30) return null;
|
||||
const width = (buffer[24] | (buffer[25] << 8) | (buffer[26] << 16)) + 1;
|
||||
const height = (buffer[27] | (buffer[28] << 8) | (buffer[29] << 16)) + 1;
|
||||
return { widthPx: width, heightPx: height };
|
||||
}
|
||||
|
||||
return null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function getImageDimensions(base64Data: string, mimeType: string): ImageDimensions | null {
|
||||
if (mimeType === "image/png") {
|
||||
return getPngDimensions(base64Data);
|
||||
}
|
||||
if (mimeType === "image/jpeg") {
|
||||
return getJpegDimensions(base64Data);
|
||||
}
|
||||
if (mimeType === "image/gif") {
|
||||
return getGifDimensions(base64Data);
|
||||
}
|
||||
if (mimeType === "image/webp") {
|
||||
return getWebpDimensions(base64Data);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function renderImage(
|
||||
base64Data: string,
|
||||
imageDimensions: ImageDimensions,
|
||||
options: ImageRenderOptions = {},
|
||||
): { sequence: string; rows: number } | null {
|
||||
const caps = getCapabilities();
|
||||
|
||||
if (!caps.images) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const maxWidth = options.maxWidthCells ?? 80;
|
||||
const rows = calculateImageRows(imageDimensions, maxWidth, getCellDimensions());
|
||||
|
||||
if (caps.images === "kitty") {
|
||||
const sequence = encodeKitty(base64Data, { columns: maxWidth, rows });
|
||||
return { sequence, rows };
|
||||
}
|
||||
|
||||
if (caps.images === "iterm2") {
|
||||
const sequence = encodeITerm2(base64Data, {
|
||||
width: maxWidth,
|
||||
height: "auto",
|
||||
preserveAspectRatio: options.preserveAspectRatio ?? true,
|
||||
});
|
||||
return { sequence, rows };
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export function imageFallback(mimeType: string, dimensions?: ImageDimensions, filename?: string): string {
|
||||
const parts: string[] = [];
|
||||
if (filename) parts.push(filename);
|
||||
parts.push(`[${mimeType}]`);
|
||||
if (dimensions) parts.push(`${dimensions.widthPx}x${dimensions.heightPx}`);
|
||||
return `[Image: ${parts.join(" ")}]`;
|
||||
}
|
||||
|
|
@ -6,6 +6,7 @@ import * as fs from "node:fs";
|
|||
import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import type { Terminal } from "./terminal.js";
|
||||
import { getCapabilities, setCellDimensions } from "./terminal-image.js";
|
||||
import { visibleWidth } from "./utils.js";
|
||||
|
||||
/**
|
||||
|
|
@ -79,6 +80,8 @@ export class TUI extends Container {
|
|||
private focusedComponent: Component | null = null;
|
||||
private renderRequested = false;
|
||||
private cursorRow = 0; // Track where cursor is (0-indexed, relative to our first line)
|
||||
private inputBuffer = ""; // Buffer for parsing terminal responses
|
||||
private cellSizeQueryPending = false;
|
||||
|
||||
constructor(terminal: Terminal) {
|
||||
super();
|
||||
|
|
@ -95,9 +98,21 @@ export class TUI extends Container {
|
|||
() => this.requestRender(),
|
||||
);
|
||||
this.terminal.hideCursor();
|
||||
this.queryCellSize();
|
||||
this.requestRender();
|
||||
}
|
||||
|
||||
private queryCellSize(): void {
|
||||
// Only query if terminal supports images (cell size is only used for image rendering)
|
||||
if (!getCapabilities().images) {
|
||||
return;
|
||||
}
|
||||
// Query terminal for cell size in pixels: CSI 16 t
|
||||
// Response format: CSI 6 ; height ; width t
|
||||
this.cellSizeQueryPending = true;
|
||||
this.terminal.write("\x1b[16t");
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.terminal.showCursor();
|
||||
this.terminal.stop();
|
||||
|
|
@ -113,6 +128,14 @@ export class TUI extends Container {
|
|||
}
|
||||
|
||||
private handleInput(data: string): void {
|
||||
// If we're waiting for cell size response, buffer input and parse
|
||||
if (this.cellSizeQueryPending) {
|
||||
this.inputBuffer += data;
|
||||
const filtered = this.parseCellSizeResponse();
|
||||
if (filtered.length === 0) return;
|
||||
data = filtered;
|
||||
}
|
||||
|
||||
// Pass input to focused component (including Ctrl+C)
|
||||
// The focused component can decide how to handle Ctrl+C
|
||||
if (this.focusedComponent?.handleInput) {
|
||||
|
|
@ -121,6 +144,56 @@ export class TUI extends Container {
|
|||
}
|
||||
}
|
||||
|
||||
private parseCellSizeResponse(): string {
|
||||
// Response format: ESC [ 6 ; height ; width t
|
||||
// Match the response pattern
|
||||
const responsePattern = /\x1b\[6;(\d+);(\d+)t/;
|
||||
const match = this.inputBuffer.match(responsePattern);
|
||||
|
||||
if (match) {
|
||||
const heightPx = parseInt(match[1], 10);
|
||||
const widthPx = parseInt(match[2], 10);
|
||||
|
||||
if (heightPx > 0 && widthPx > 0) {
|
||||
setCellDimensions({ widthPx, heightPx });
|
||||
// Invalidate all components so images re-render with correct dimensions
|
||||
this.invalidate();
|
||||
this.requestRender();
|
||||
}
|
||||
|
||||
// Remove the response from buffer
|
||||
this.inputBuffer = this.inputBuffer.replace(responsePattern, "");
|
||||
this.cellSizeQueryPending = false;
|
||||
}
|
||||
|
||||
// Check if we have a partial response starting (wait for more data)
|
||||
// ESC [ 6 ; ... could be incomplete
|
||||
const partialPattern = /\x1b\[6;[\d;]*$/;
|
||||
if (partialPattern.test(this.inputBuffer)) {
|
||||
return ""; // Wait for more data
|
||||
}
|
||||
|
||||
// Check for any ESC that might be start of response
|
||||
const escIndex = this.inputBuffer.lastIndexOf("\x1b");
|
||||
if (escIndex !== -1 && escIndex > this.inputBuffer.length - 10) {
|
||||
// Might be incomplete escape sequence, wait a bit
|
||||
// But return any data before it
|
||||
const before = this.inputBuffer.substring(0, escIndex);
|
||||
this.inputBuffer = this.inputBuffer.substring(escIndex);
|
||||
return before;
|
||||
}
|
||||
|
||||
// No response found, return buffered data as user input
|
||||
const result = this.inputBuffer;
|
||||
this.inputBuffer = "";
|
||||
this.cellSizeQueryPending = false; // Give up waiting
|
||||
return result;
|
||||
}
|
||||
|
||||
private containsImage(line: string): boolean {
|
||||
return line.includes("\x1b_G") || line.includes("\x1b]1337;File=");
|
||||
}
|
||||
|
||||
private doRender(): void {
|
||||
const width = this.terminal.columns;
|
||||
const height = this.terminal.rows;
|
||||
|
|
@ -182,6 +255,38 @@ export class TUI extends Container {
|
|||
return;
|
||||
}
|
||||
|
||||
// Check if image lines changed - they require special handling to avoid duplication
|
||||
// Only force full re-render if image content actually changed, not just because images exist
|
||||
const imageLineChanged = (() => {
|
||||
for (let i = firstChanged; i < Math.max(newLines.length, this.previousLines.length); i++) {
|
||||
const prevLine = this.previousLines[i] || "";
|
||||
const newLine = newLines[i] || "";
|
||||
if (this.containsImage(prevLine) || this.containsImage(newLine)) {
|
||||
if (prevLine !== newLine) return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
})();
|
||||
|
||||
if (imageLineChanged) {
|
||||
let buffer = "\x1b[?2026h"; // Begin synchronized output
|
||||
// For Kitty protocol, delete all images before re-render
|
||||
if (getCapabilities().images === "kitty") {
|
||||
buffer += "\x1b_Ga=d\x1b\\";
|
||||
}
|
||||
buffer += "\x1b[3J\x1b[2J\x1b[H"; // Clear scrollback, screen, and home
|
||||
for (let i = 0; i < newLines.length; i++) {
|
||||
if (i > 0) buffer += "\r\n";
|
||||
buffer += newLines[i];
|
||||
}
|
||||
buffer += "\x1b[?2026l"; // End synchronized output
|
||||
this.terminal.write(buffer);
|
||||
this.cursorRow = newLines.length - 1;
|
||||
this.previousLines = newLines;
|
||||
this.previousWidth = width;
|
||||
return;
|
||||
}
|
||||
|
||||
// Check if firstChanged is outside the viewport
|
||||
// cursorRow is the line where cursor is (0-indexed)
|
||||
// Viewport shows lines from (cursorRow - height + 1) to cursorRow
|
||||
|
|
@ -222,23 +327,25 @@ export class TUI extends Container {
|
|||
for (let i = firstChanged; i < newLines.length; i++) {
|
||||
if (i > firstChanged) buffer += "\r\n";
|
||||
buffer += "\x1b[2K"; // Clear current line
|
||||
if (visibleWidth(newLines[i]) > width) {
|
||||
const line = newLines[i];
|
||||
const isImageLine = this.containsImage(line);
|
||||
if (!isImageLine && visibleWidth(line) > width) {
|
||||
// Log all lines to crash file for debugging
|
||||
const crashLogPath = path.join(os.homedir(), ".pi", "agent", "pi-crash.log");
|
||||
const crashData = [
|
||||
`Crash at ${new Date().toISOString()}`,
|
||||
`Terminal width: ${width}`,
|
||||
`Line ${i} visible width: ${visibleWidth(newLines[i])}`,
|
||||
`Line ${i} visible width: ${visibleWidth(line)}`,
|
||||
"",
|
||||
"=== All rendered lines ===",
|
||||
...newLines.map((line, idx) => `[${idx}] (w=${visibleWidth(line)}) ${line}`),
|
||||
...newLines.map((l, idx) => `[${idx}] (w=${visibleWidth(l)}) ${l}`),
|
||||
"",
|
||||
].join("\n");
|
||||
fs.mkdirSync(path.dirname(crashLogPath), { recursive: true });
|
||||
fs.writeFileSync(crashLogPath, crashData);
|
||||
throw new Error(`Rendered line ${i} exceeds terminal width. Debug log written to ${crashLogPath}`);
|
||||
}
|
||||
buffer += newLines[i];
|
||||
buffer += line;
|
||||
}
|
||||
|
||||
// If we had more lines before, clear them and move cursor back
|
||||
|
|
|
|||
56
packages/tui/test/image-test.ts
Normal file
56
packages/tui/test/image-test.ts
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
import { readFileSync } from "fs";
|
||||
import { Image } from "../src/components/image.js";
|
||||
import { Spacer } from "../src/components/spacer.js";
|
||||
import { Text } from "../src/components/text.js";
|
||||
import { ProcessTerminal } from "../src/terminal.js";
|
||||
import { getCapabilities, getImageDimensions } from "../src/terminal-image.js";
|
||||
import { TUI } from "../src/tui.js";
|
||||
|
||||
const testImagePath = process.argv[2] || "/tmp/test-image.png";
|
||||
|
||||
console.log("Terminal capabilities:", getCapabilities());
|
||||
console.log("Loading image from:", testImagePath);
|
||||
|
||||
let imageBuffer: Buffer;
|
||||
try {
|
||||
imageBuffer = readFileSync(testImagePath);
|
||||
} catch (e) {
|
||||
console.error(`Failed to load image: ${testImagePath}`);
|
||||
console.error("Usage: npx tsx test/image-test.ts [path-to-image.png]");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const base64Data = imageBuffer.toString("base64");
|
||||
const dims = getImageDimensions(base64Data, "image/png");
|
||||
|
||||
console.log("Image dimensions:", dims);
|
||||
console.log("");
|
||||
|
||||
const terminal = new ProcessTerminal();
|
||||
const tui = new TUI(terminal);
|
||||
|
||||
tui.addChild(new Text("Image Rendering Test", 1, 1));
|
||||
tui.addChild(new Spacer(1));
|
||||
|
||||
if (dims) {
|
||||
tui.addChild(
|
||||
new Image(base64Data, "image/png", { fallbackColor: (s) => `\x1b[33m${s}\x1b[0m` }, { maxWidthCells: 60 }, dims),
|
||||
);
|
||||
} else {
|
||||
tui.addChild(new Text("Could not parse image dimensions", 1, 0));
|
||||
}
|
||||
|
||||
tui.addChild(new Spacer(1));
|
||||
tui.addChild(new Text("Press Ctrl+C to exit", 1, 0));
|
||||
|
||||
const editor = {
|
||||
handleInput(data: string) {
|
||||
if (data.charCodeAt(0) === 3) {
|
||||
tui.stop();
|
||||
process.exit(0);
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
tui.setFocus(editor as any);
|
||||
tui.start();
|
||||
Loading…
Add table
Add a link
Reference in a new issue