diff --git a/packages/mom/CHANGELOG.md b/packages/mom/CHANGELOG.md index 0b7ad116..2da664dc 100644 --- a/packages/mom/CHANGELOG.md +++ b/packages/mom/CHANGELOG.md @@ -2,6 +2,31 @@ ## [Unreleased] +### Changed + +- Complete rewrite of message handling architecture + - `log.jsonl` is now the source of truth for all channel messages + - `context.jsonl` stores LLM context (messages sent to Claude) + - Sync mechanism ensures context.jsonl stays in sync with log.jsonl at run start + - Session header written immediately on new session creation (not lazily) + +- Backfill improvements + - Only backfills channels that already have a `log.jsonl` file + - Strips @mentions from backfilled messages (consistent with live messages) + - Uses largest timestamp in log for efficient incremental backfill + - Fetches DM channels in addition to public/private channels + +- Message handling improvements + - Channel chatter (messages without @mention) logged but doesn't trigger processing + - Messages sent while mom is busy are logged and synced on next run + - Pre-startup messages (replayed by Slack on reconnect) logged but not auto-processed + - Stop command executes immediately (not queued), can interrupt running tasks + - Channel @mentions no longer double-logged (was firing both app_mention and message events) + +- Usage summary now includes context window usage + - Shows current context tokens vs model's context window + - Example: `Context: 4.2k / 200k (2.1%)` + ### Fixed - Slack API errors (msg_too_long) no longer crash the process @@ -13,20 +38,12 @@ - Private channel messages not being logged - Added `message.groups` to required bot events in README - Added `groups:history` and `groups:read` to required scopes in README - - `app_mention` handler now logs messages directly instead of relying on `message` event - - Added deduplication in `ChannelStore.logMessage()` to prevent double-logging + +- Stop command now updates "Stopping..." to "Stopped" instead of posting two messages ### Added - Port truncation logic from coding-agent: bash and read tools now use consistent 2000 lines OR 50KB limits with actionable notices -- Remove redundant context history truncation (tools already provide truncation with actionable hints) - -- Message backfill on startup (#103) - - Fetches missed messages from Slack using `conversations.history` API when mom restarts - - Backfills up to 3 pages (3000 messages) per channel since last logged timestamp - - Includes mom's own responses and user messages (excludes other bots) - - Downloads attachments from backfilled messages - - Logs progress: channel count, per-channel message counts, total with duration ## [0.10.2] - 2025-11-27 diff --git a/packages/mom/README.md b/packages/mom/README.md index b893affb..97d32340 100644 --- a/packages/mom/README.md +++ b/packages/mom/README.md @@ -145,24 +145,27 @@ You provide mom with a **data directory** (e.g., `./data`) as her workspace. Whi ``` ./data/ # Your host directory ├── MEMORY.md # Global memory (shared across channels) + ├── settings.json # Global settings (compaction, retry, etc.) ├── skills/ # Global custom CLI tools mom creates ├── C123ABC/ # Each Slack channel gets a directory │ ├── MEMORY.md # Channel-specific memory - │ ├── log.jsonl # Full conversation history + │ ├── log.jsonl # Full message history (source of truth) + │ ├── context.jsonl # LLM context (synced from log.jsonl) │ ├── attachments/ # Files users shared │ ├── scratch/ # Mom's working directory │ └── skills/ # Channel-specific CLI tools - └── C456DEF/ # Another channel + └── D456DEF/ # DM channels also get directories └── ... ``` **What's stored here:** -- Conversation logs and Slack attachments. These are automatically stored by mom -- Memory files. Context mom remembers across sessions +- `log.jsonl`: All channel messages (user messages, bot responses). Source of truth. +- `context.jsonl`: Messages sent to Claude. Synced from log.jsonl at each run start. +- Memory files: Context mom remembers across sessions - Custom tools/scripts mom creates (aka "skills") - Working files, cloned repos, generated output -This is also where mom efficiently greps channel log files for conversation history, giving her essentially infinite context. +Mom efficiently greps `log.jsonl` for conversation history, giving her essentially infinite context beyond what's in `context.jsonl`. ### Memory @@ -230,33 +233,41 @@ Mom will read the `SKILL.md` file before using a skill, and reuse stored credent Update mom anytime with `npm install -g @mariozechner/pi-mom`. This only updates the Node.js app on your host. Anything mom installed inside the Docker container remains unchanged. -## Message History (log.jsonl) +## Message History -Each channel's `log.jsonl` contains the full conversation history. Every message, tool call, and result. Format: one JSON object per line with ISO 8601 timestamps: +Mom uses two files per channel to manage messages: + +### log.jsonl (Source of Truth) + +All channel messages are stored here. This includes user messages, channel chatter (messages without @mention), and bot responses. Format: one JSON object per line with ISO 8601 timestamps: ```typescript interface LoggedMessage { date: string; // ISO 8601 (e.g., "2025-11-26T10:44:00.000Z") - ts: string; // Slack timestamp or epoch ms + ts: string; // Slack timestamp (seconds.microseconds) user: string; // User ID or "bot" userName?: string; // Handle (e.g., "mario") displayName?: string; // Display name (e.g., "Mario Zechner") - text: string; // Message text - attachments: Array<{ - original: string; // Original filename - local: string; // Path relative to data dir - }>; + text: string; // Message text (@mentions stripped) + attachments: string[]; // Filenames of attachments isBot: boolean; } ``` **Example:** ```json -{"date":"2025-11-26T10:44:00.123Z","ts":"1732619040.123456","user":"U123ABC","userName":"mario","text":"@mom hello","attachments":[],"isBot":false} +{"date":"2025-11-26T10:44:00.123Z","ts":"1732619040.123456","user":"U123ABC","userName":"mario","text":"hello","attachments":[],"isBot":false} {"date":"2025-11-26T10:44:05.456Z","ts":"1732619045456","user":"bot","text":"Hi! How can I help?","attachments":[],"isBot":true} ``` -Mom knows how to query these logs efficiently (see [her system prompt](src/agent.ts)) to avoid context overflow when searching conversation history. +### context.jsonl (LLM Context) + +Messages sent to Claude are stored here. This is synced from `log.jsonl` at the start of each run to ensure: +- Backfilled messages (from Slack API on startup) are included +- Channel chatter between @mentions is included +- Messages sent while mom was busy are included + +Mom knows how to query `log.jsonl` efficiently (see [her system prompt](src/agent.ts)) for older history beyond what's in context. ## Security Considerations @@ -339,9 +350,10 @@ mom --sandbox=docker:mom-exec ./data-exec ### Code Structure -- `src/main.ts`: Entry point, CLI arg parsing, message routing -- `src/agent.ts`: Agent runner, event handling, tool execution -- `src/slack.ts`: Slack integration, context management, message posting +- `src/main.ts`: Entry point, CLI arg parsing, handler setup, SlackContext adapter +- `src/agent.ts`: Agent runner, event handling, tool execution, session management +- `src/slack.ts`: Slack integration (Socket Mode), backfill, message logging +- `src/context.ts`: Session manager (context.jsonl), log-to-context sync - `src/store.ts`: Channel data persistence, attachment downloads - `src/log.ts`: Centralized logging (console output) - `src/sandbox.ts`: Docker/host sandbox execution diff --git a/packages/mom/src/agent.ts b/packages/mom/src/agent.ts index 5992e241..b0af9387 100644 --- a/packages/mom/src/agent.ts +++ b/packages/mom/src/agent.ts @@ -617,9 +617,24 @@ function createRunner(sandboxConfig: SandboxConfig, channelId: string, channelDi } } - // Log usage summary + // Log usage summary with context info if (runState.totalUsage.cost.total > 0) { - const summary = log.logUsageSummary(runState.logCtx!, runState.totalUsage); + // Get last non-aborted assistant message for context calculation + const messages = session.messages; + const lastAssistantMessage = messages + .slice() + .reverse() + .find((m) => m.role === "assistant" && (m as any).stopReason !== "aborted") as any; + + const contextTokens = lastAssistantMessage + ? lastAssistantMessage.usage.input + + lastAssistantMessage.usage.output + + lastAssistantMessage.usage.cacheRead + + lastAssistantMessage.usage.cacheWrite + : 0; + const contextWindow = model.contextWindow || 200000; + + const summary = log.logUsageSummary(runState.logCtx!, runState.totalUsage, contextTokens, contextWindow); runState.queue.enqueue(() => ctx.respondInThread(summary), "usage summary"); await queueChain; } diff --git a/packages/mom/src/log.ts b/packages/mom/src/log.ts index ee1ac983..f8862e36 100644 --- a/packages/mom/src/log.ts +++ b/packages/mom/src/log.ts @@ -195,13 +195,26 @@ export function logUsageSummary( cacheWrite: number; cost: { input: number; output: number; cacheRead: number; cacheWrite: number; total: number }; }, + contextTokens?: number, + contextWindow?: number, ): string { + const formatTokens = (count: number): string => { + if (count < 1000) return count.toString(); + if (count < 10000) return (count / 1000).toFixed(1) + "k"; + if (count < 1000000) return Math.round(count / 1000) + "k"; + return (count / 1000000).toFixed(1) + "M"; + }; + const lines: string[] = []; lines.push("*Usage Summary*"); lines.push(`Tokens: ${usage.input.toLocaleString()} in, ${usage.output.toLocaleString()} out`); if (usage.cacheRead > 0 || usage.cacheWrite > 0) { lines.push(`Cache: ${usage.cacheRead.toLocaleString()} read, ${usage.cacheWrite.toLocaleString()} write`); } + if (contextTokens && contextWindow) { + const contextPercent = ((contextTokens / contextWindow) * 100).toFixed(1); + lines.push(`Context: ${formatTokens(contextTokens)} / ${formatTokens(contextWindow)} (${contextPercent}%)`); + } lines.push( `Cost: $${usage.cost.input.toFixed(4)} in, $${usage.cost.output.toFixed(4)} out` + (usage.cacheRead > 0 || usage.cacheWrite > 0