diff --git a/packages/ai/src/models.generated.ts b/packages/ai/src/models.generated.ts index 0d373aa4..a8757dee 100644 --- a/packages/ai/src/models.generated.ts +++ b/packages/ai/src/models.generated.ts @@ -1957,6 +1957,23 @@ export const MODELS = { } satisfies Model<"anthropic-messages">, }, openrouter: { + "prime-intellect/intellect-3": { + id: "prime-intellect/intellect-3", + name: "Prime Intellect: INTELLECT-3", + api: "openai-completions", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + reasoning: true, + input: ["text"], + cost: { + input: 0.19999999999999998, + output: 1.1, + cacheRead: 0, + cacheWrite: 0, + }, + contextWindow: 131072, + maxTokens: 131072, + } satisfies Model<"openai-completions">, "tngtech/tng-r1t-chimera:free": { id: "tngtech/tng-r1t-chimera:free", name: "TNG: R1T Chimera (free)", @@ -2238,8 +2255,8 @@ export const MODELS = { reasoning: true, input: ["text"], cost: { - input: 0.24, - output: 0.96, + input: 0.255, + output: 1.02, cacheRead: 0, cacheWrite: 0, }, @@ -4983,23 +5000,6 @@ export const MODELS = { contextWindow: 32768, maxTokens: 4096, } satisfies Model<"openai-completions">, - "cohere/command-r-08-2024": { - id: "cohere/command-r-08-2024", - name: "Cohere: Command R (08-2024)", - api: "openai-completions", - provider: "openrouter", - baseUrl: "https://openrouter.ai/api/v1", - reasoning: false, - input: ["text"], - cost: { - input: 0.15, - output: 0.6, - cacheRead: 0, - cacheWrite: 0, - }, - contextWindow: 128000, - maxTokens: 4000, - } satisfies Model<"openai-completions">, "cohere/command-r-plus-08-2024": { id: "cohere/command-r-plus-08-2024", name: "Cohere: Command R+ (08-2024)", @@ -5017,6 +5017,23 @@ export const MODELS = { contextWindow: 128000, maxTokens: 4000, } satisfies Model<"openai-completions">, + "cohere/command-r-08-2024": { + id: "cohere/command-r-08-2024", + name: "Cohere: Command R (08-2024)", + api: "openai-completions", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + reasoning: false, + input: ["text"], + cost: { + input: 0.15, + output: 0.6, + cacheRead: 0, + cacheWrite: 0, + }, + contextWindow: 128000, + maxTokens: 4000, + } satisfies Model<"openai-completions">, "sao10k/l3.1-euryale-70b": { id: "sao10k/l3.1-euryale-70b", name: "Sao10K: Llama 3.1 Euryale 70B v2.2", @@ -5153,9 +5170,9 @@ export const MODELS = { contextWindow: 131072, maxTokens: 16384, } satisfies Model<"openai-completions">, - "openai/gpt-4o-mini-2024-07-18": { - id: "openai/gpt-4o-mini-2024-07-18", - name: "OpenAI: GPT-4o-mini (2024-07-18)", + "openai/gpt-4o-mini": { + id: "openai/gpt-4o-mini", + name: "OpenAI: GPT-4o-mini", api: "openai-completions", provider: "openrouter", baseUrl: "https://openrouter.ai/api/v1", @@ -5170,9 +5187,9 @@ export const MODELS = { contextWindow: 128000, maxTokens: 16384, } satisfies Model<"openai-completions">, - "openai/gpt-4o-mini": { - id: "openai/gpt-4o-mini", - name: "OpenAI: GPT-4o-mini", + "openai/gpt-4o-mini-2024-07-18": { + id: "openai/gpt-4o-mini-2024-07-18", + name: "OpenAI: GPT-4o-mini (2024-07-18)", api: "openai-completions", provider: "openrouter", baseUrl: "https://openrouter.ai/api/v1", @@ -5272,23 +5289,6 @@ export const MODELS = { contextWindow: 128000, maxTokens: 4096, } satisfies Model<"openai-completions">, - "openai/gpt-4o-2024-05-13": { - id: "openai/gpt-4o-2024-05-13", - name: "OpenAI: GPT-4o (2024-05-13)", - api: "openai-completions", - provider: "openrouter", - baseUrl: "https://openrouter.ai/api/v1", - reasoning: false, - input: ["text", "image"], - cost: { - input: 5, - output: 15, - cacheRead: 0, - cacheWrite: 0, - }, - contextWindow: 128000, - maxTokens: 4096, - } satisfies Model<"openai-completions">, "openai/gpt-4o": { id: "openai/gpt-4o", name: "OpenAI: GPT-4o", @@ -5323,22 +5323,22 @@ export const MODELS = { contextWindow: 128000, maxTokens: 64000, } satisfies Model<"openai-completions">, - "meta-llama/llama-3-70b-instruct": { - id: "meta-llama/llama-3-70b-instruct", - name: "Meta: Llama 3 70B Instruct", + "openai/gpt-4o-2024-05-13": { + id: "openai/gpt-4o-2024-05-13", + name: "OpenAI: GPT-4o (2024-05-13)", api: "openai-completions", provider: "openrouter", baseUrl: "https://openrouter.ai/api/v1", reasoning: false, - input: ["text"], + input: ["text", "image"], cost: { - input: 0.3, - output: 0.39999999999999997, + input: 5, + output: 15, cacheRead: 0, cacheWrite: 0, }, - contextWindow: 8192, - maxTokens: 16384, + contextWindow: 128000, + maxTokens: 4096, } satisfies Model<"openai-completions">, "meta-llama/llama-3-8b-instruct": { id: "meta-llama/llama-3-8b-instruct", @@ -5357,6 +5357,23 @@ export const MODELS = { contextWindow: 8192, maxTokens: 16384, } satisfies Model<"openai-completions">, + "meta-llama/llama-3-70b-instruct": { + id: "meta-llama/llama-3-70b-instruct", + name: "Meta: Llama 3 70B Instruct", + api: "openai-completions", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + reasoning: false, + input: ["text"], + cost: { + input: 0.3, + output: 0.39999999999999997, + cacheRead: 0, + cacheWrite: 0, + }, + contextWindow: 8192, + maxTokens: 16384, + } satisfies Model<"openai-completions">, "mistralai/mixtral-8x22b-instruct": { id: "mistralai/mixtral-8x22b-instruct", name: "Mistral: Mixtral 8x22B Instruct", @@ -5442,23 +5459,6 @@ export const MODELS = { contextWindow: 128000, maxTokens: 4096, } satisfies Model<"openai-completions">, - "openai/gpt-3.5-turbo-0613": { - id: "openai/gpt-3.5-turbo-0613", - name: "OpenAI: GPT-3.5 Turbo (older v0613)", - api: "openai-completions", - provider: "openrouter", - baseUrl: "https://openrouter.ai/api/v1", - reasoning: false, - input: ["text"], - cost: { - input: 1, - output: 2, - cacheRead: 0, - cacheWrite: 0, - }, - contextWindow: 4095, - maxTokens: 4096, - } satisfies Model<"openai-completions">, "openai/gpt-4-turbo-preview": { id: "openai/gpt-4-turbo-preview", name: "OpenAI: GPT-4 Turbo Preview", @@ -5476,6 +5476,23 @@ export const MODELS = { contextWindow: 128000, maxTokens: 4096, } satisfies Model<"openai-completions">, + "openai/gpt-3.5-turbo-0613": { + id: "openai/gpt-3.5-turbo-0613", + name: "OpenAI: GPT-3.5 Turbo (older v0613)", + api: "openai-completions", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + reasoning: false, + input: ["text"], + cost: { + input: 1, + output: 2, + cacheRead: 0, + cacheWrite: 0, + }, + contextWindow: 4095, + maxTokens: 4096, + } satisfies Model<"openai-completions">, "mistralai/mistral-small": { id: "mistralai/mistral-small", name: "Mistral Small", @@ -5578,23 +5595,6 @@ export const MODELS = { contextWindow: 8191, maxTokens: 4096, } satisfies Model<"openai-completions">, - "openai/gpt-4": { - id: "openai/gpt-4", - name: "OpenAI: GPT-4", - api: "openai-completions", - provider: "openrouter", - baseUrl: "https://openrouter.ai/api/v1", - reasoning: false, - input: ["text"], - cost: { - input: 30, - output: 60, - cacheRead: 0, - cacheWrite: 0, - }, - contextWindow: 8191, - maxTokens: 4096, - } satisfies Model<"openai-completions">, "openai/gpt-3.5-turbo": { id: "openai/gpt-3.5-turbo", name: "OpenAI: GPT-3.5 Turbo", @@ -5612,6 +5612,23 @@ export const MODELS = { contextWindow: 16385, maxTokens: 4096, } satisfies Model<"openai-completions">, + "openai/gpt-4": { + id: "openai/gpt-4", + name: "OpenAI: GPT-4", + api: "openai-completions", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + reasoning: false, + input: ["text"], + cost: { + input: 30, + output: 60, + cacheRead: 0, + cacheWrite: 0, + }, + contextWindow: 8191, + maxTokens: 4096, + } satisfies Model<"openai-completions">, "openrouter/auto": { id: "openrouter/auto", name: "OpenRouter: Auto Router", diff --git a/packages/mom/CHANGELOG.md b/packages/mom/CHANGELOG.md index 6e6624e4..fade400b 100644 --- a/packages/mom/CHANGELOG.md +++ b/packages/mom/CHANGELOG.md @@ -2,6 +2,48 @@ ## [Unreleased] +### Breaking Changes + +- Timestamps now use Slack format (seconds.microseconds) and messages are sorted by `ts` field + - **Migration required**: Run `npx tsx scripts/migrate-timestamps.ts ./data` to fix existing logs + - Without migration, message context will be incorrectly ordered + +### Added + +- Channel and user ID mappings in system prompt + - Fetches all channels bot is member of and all workspace users at startup + - Mom can now reference channels by name and mention users properly +- Skills documentation in system prompt + - Explains custom CLI tools pattern with SKILL.md files + - Encourages mom to create reusable tools for recurring tasks +- Debug output: writes `last_prompt.txt` to channel directory with full context +- Bash working directory info in system prompt (/ for Docker, cwd for host) +- Token-efficient log queries that filter out tool calls/results for summaries + +### Changed + +- Turn-based message context instead of raw line count (#68) + - Groups consecutive bot messages (tool calls/results) as single turn + - "50 turns" now means ~50 conversation exchanges, not 50 log lines + - Prevents tool-heavy runs from pushing out conversation context +- Messages sorted by Slack timestamp before building context + - Fixes out-of-order issues from async attachment downloads + - Added monotonic counter for sub-millisecond ordering +- Condensed system prompt from ~5k to ~2.7k chars + - More concise workspace layout (tree format) + - Clearer log query examples (conversation-only vs full details) + - Removed redundant guidelines section +- User prompt simplified: removed duplicate "Current message" (already in history) +- Tool status labels (`_→ label_`) no longer logged to jsonl +- Thread messages and thinking no longer double-logged + +### Fixed + +- Duplicate message logging: removed redundant log from app_mention handler +- Username obfuscation in thread messages to prevent unwanted pings + - Handles @username, bare username, and <@USERID> formats + - Escapes special regex characters in usernames + ## [0.10.1] - 2025-11-27 ### Changed diff --git a/packages/mom/scripts/migrate-timestamps.ts b/packages/mom/scripts/migrate-timestamps.ts new file mode 100644 index 00000000..10604dbc --- /dev/null +++ b/packages/mom/scripts/migrate-timestamps.ts @@ -0,0 +1,121 @@ +#!/usr/bin/env npx tsx +/** + * Migrate log.jsonl timestamps from milliseconds to Slack format (seconds.microseconds) + * + * Usage: npx tsx scripts/migrate-timestamps.ts + * Example: npx tsx scripts/migrate-timestamps.ts ./data + */ + +import { readFileSync, writeFileSync, readdirSync, statSync, existsSync } from "fs"; +import { join } from "path"; + +function isMillisecondTimestamp(ts: string): boolean { + // Slack timestamps are seconds.microseconds, like "1764279530.533489" + // Millisecond timestamps are just big numbers, like "1764279320398" + // + // Key insight: + // - Slack ts from 2025: ~1.7 billion (10 digits before decimal) + // - Millisecond ts from 2025: ~1.7 trillion (13 digits) + + // If it has a decimal and the integer part is < 10^12, it's Slack format + if (ts.includes(".")) { + const intPart = parseInt(ts.split(".")[0], 10); + return intPart > 1e12; // Unlikely to have decimal AND be millis, but check anyway + } + + // No decimal - check if it's too big to be seconds + const num = parseInt(ts, 10); + return num > 1e12; // If > 1 trillion, it's milliseconds +} + +function convertToSlackTs(msTs: string): string { + const ms = parseInt(msTs, 10); + const seconds = Math.floor(ms / 1000); + const micros = (ms % 1000) * 1000; + return `${seconds}.${micros.toString().padStart(6, "0")}`; +} + +function migrateFile(filePath: string): { total: number; migrated: number } { + const content = readFileSync(filePath, "utf-8"); + const lines = content.split("\n").filter(Boolean); + + let migrated = 0; + const newLines: string[] = []; + + for (const line of lines) { + try { + const msg = JSON.parse(line); + if (msg.ts && isMillisecondTimestamp(msg.ts)) { + const oldTs = msg.ts; + msg.ts = convertToSlackTs(msg.ts); + console.log(` Converted: ${oldTs} -> ${msg.ts}`); + migrated++; + } + newLines.push(JSON.stringify(msg)); + } catch (e) { + // Keep malformed lines as-is + console.log(` Warning: Could not parse line: ${line.substring(0, 50)}...`); + newLines.push(line); + } + } + + if (migrated > 0) { + writeFileSync(filePath, newLines.join("\n") + "\n", "utf-8"); + } + + return { total: lines.length, migrated }; +} + +function findLogFiles(dir: string): string[] { + const logFiles: string[] = []; + + if (!existsSync(dir)) { + console.error(`Directory not found: ${dir}`); + return []; + } + + const entries = readdirSync(dir); + for (const entry of entries) { + const fullPath = join(dir, entry); + const stat = statSync(fullPath); + + if (stat.isDirectory()) { + // Check for log.jsonl in subdirectory + const logPath = join(fullPath, "log.jsonl"); + if (existsSync(logPath)) { + logFiles.push(logPath); + } + } + } + + return logFiles; +} + +// Main +const dataDir = process.argv[2]; +if (!dataDir) { + console.error("Usage: npx tsx scripts/migrate-timestamps.ts "); + console.error("Example: npx tsx scripts/migrate-timestamps.ts ./data"); + process.exit(1); +} + +console.log(`Scanning for log.jsonl files in: ${dataDir}\n`); + +const logFiles = findLogFiles(dataDir); +if (logFiles.length === 0) { + console.log("No log.jsonl files found."); + process.exit(0); +} + +let totalMigrated = 0; +let totalMessages = 0; + +for (const logFile of logFiles) { + console.log(`Processing: ${logFile}`); + const { total, migrated } = migrateFile(logFile); + totalMessages += total; + totalMigrated += migrated; + console.log(` ${migrated}/${total} messages migrated\n`); +} + +console.log(`Done! Migrated ${totalMigrated}/${totalMessages} total messages across ${logFiles.length} files.`); diff --git a/packages/mom/src/agent.ts b/packages/mom/src/agent.ts index 56ea7c64..5f0e667a 100644 --- a/packages/mom/src/agent.ts +++ b/packages/mom/src/agent.ts @@ -1,17 +1,39 @@ import { Agent, type AgentEvent, ProviderTransport } from "@mariozechner/pi-agent-core"; import { getModel } from "@mariozechner/pi-ai"; import { existsSync, readFileSync } from "fs"; -import { mkdir } from "fs/promises"; +import { mkdir, writeFile } from "fs/promises"; import { join } from "path"; import * as log from "./log.js"; import { createExecutor, type SandboxConfig } from "./sandbox.js"; -import type { SlackContext } from "./slack.js"; +import type { ChannelInfo, SlackContext, UserInfo } from "./slack.js"; import type { ChannelStore } from "./store.js"; import { createMomTools, setUploadFunction } from "./tools/index.js"; // Hardcoded model for now const model = getModel("anthropic", "claude-sonnet-4-5"); +/** + * Convert Date.now() to Slack timestamp format (seconds.microseconds) + * Uses a monotonic counter to ensure ordering even within the same millisecond + */ +let lastTsMs = 0; +let tsCounter = 0; + +function toSlackTs(): string { + const now = Date.now(); + if (now === lastTsMs) { + // Same millisecond - increment counter for sub-ms ordering + tsCounter++; + } else { + // New millisecond - reset counter + lastTsMs = now; + tsCounter = 0; + } + const seconds = Math.floor(now / 1000); + const micros = (now % 1000) * 1000 + tsCounter; // ms to micros + counter + return `${seconds}.${micros.toString().padStart(6, "0")}`; +} + export interface AgentRunner { run(ctx: SlackContext, channelDir: string, store: ChannelStore): Promise<{ stopReason: string }>; abort(): void; @@ -25,7 +47,17 @@ function getAnthropicApiKey(): string { return key; } -function getRecentMessages(channelDir: string, count: number): string { +interface LogMessage { + date?: string; + ts?: string; + user?: string; + userName?: string; + text?: string; + attachments?: Array<{ local: string }>; + isBot?: boolean; +} + +function getRecentMessages(channelDir: string, turnCount: number): string { const logPath = join(channelDir, "log.jsonl"); if (!existsSync(logPath)) { return "(no message history yet)"; @@ -33,23 +65,72 @@ function getRecentMessages(channelDir: string, count: number): string { const content = readFileSync(logPath, "utf-8"); const lines = content.trim().split("\n").filter(Boolean); - const recentLines = lines.slice(-count); - if (recentLines.length === 0) { + if (lines.length === 0) { return "(no message history yet)"; } - // Format as TSV for more concise system prompt - const formatted: string[] = []; - for (const line of recentLines) { + // Parse all messages and sort by Slack timestamp + // (attachment downloads can cause out-of-order logging) + const messages: LogMessage[] = []; + for (const line of lines) { try { - const msg = JSON.parse(line); + messages.push(JSON.parse(line)); + } catch {} + } + messages.sort((a, b) => { + const tsA = parseFloat(a.ts || "0"); + const tsB = parseFloat(b.ts || "0"); + return tsA - tsB; + }); + + // Group into "turns" - a turn is either: + // - A single user message (isBot: false) + // - A sequence of consecutive bot messages (isBot: true) coalesced into one turn + // We walk backwards to get the last N turns + const turns: LogMessage[][] = []; + let currentTurn: LogMessage[] = []; + let lastWasBot: boolean | null = null; + + for (let i = messages.length - 1; i >= 0; i--) { + const msg = messages[i]; + const isBot = msg.isBot === true; + + if (lastWasBot === null) { + // First message + currentTurn.unshift(msg); + lastWasBot = isBot; + } else if (isBot && lastWasBot) { + // Consecutive bot messages - same turn + currentTurn.unshift(msg); + } else { + // Transition - save current turn and start new one + turns.unshift(currentTurn); + currentTurn = [msg]; + lastWasBot = isBot; + + // Stop if we have enough turns + if (turns.length >= turnCount) { + break; + } + } + } + + // Don't forget the last turn we were building + if (currentTurn.length > 0 && turns.length < turnCount) { + turns.unshift(currentTurn); + } + + // Flatten turns back to messages and format as TSV + const formatted: string[] = []; + for (const turn of turns) { + for (const msg of turn) { const date = (msg.date || "").substring(0, 19); - const user = msg.userName || msg.user; + const user = msg.userName || msg.user || ""; const text = msg.text || ""; - const attachments = (msg.attachments || []).map((a: { local: string }) => a.local).join(","); + const attachments = (msg.attachments || []).map((a) => a.local).join(","); formatted.push(`${date}\t${user}\t${text}\t${attachments}`); - } catch (error) {} + } } return formatted.join("\n"); @@ -96,151 +177,112 @@ function buildSystemPrompt( channelId: string, memory: string, sandboxConfig: SandboxConfig, + channels: ChannelInfo[], + users: UserInfo[], ): string { const channelPath = `${workspacePath}/${channelId}`; const isDocker = sandboxConfig.type === "docker"; + // Format channel mappings + const channelMappings = + channels.length > 0 ? channels.map((c) => `${c.id}\t#${c.name}`).join("\n") : "(no channels loaded)"; + + // Format user mappings + const userMappings = + users.length > 0 ? users.map((u) => `${u.id}\t@${u.userName}\t${u.displayName}`).join("\n") : "(no users loaded)"; + const envDescription = isDocker ? `You are running inside a Docker container (Alpine Linux). +- Bash working directory: / (use cd or absolute paths) - Install tools with: apk add -- Your changes persist across sessions -- You have full control over this container` +- Your changes persist across sessions` : `You are running directly on the host machine. -- Be careful with system modifications -- Use the system's package manager if needed`; +- Bash working directory: ${process.cwd()} +- Be careful with system modifications`; const currentDate = new Date().toISOString().split("T")[0]; // YYYY-MM-DD const currentDateTime = new Date().toISOString(); // Full ISO 8601 - return `You are mom, a helpful Slack bot assistant. + return `You are mom, a Slack bot assistant. Be concise. No emojis. -## Current Date and Time -- Date: ${currentDate} -- Full timestamp: ${currentDateTime} -- Use this when working with dates or searching logs +## Context +- Date: ${currentDate} (${currentDateTime}) +- You receive the last 50 conversation turns. If you need older context, search log.jsonl. -## Communication Style -- Be concise and professional -- Do not use emojis unless the user communicates informally with you -- Get to the point quickly -- If you need clarification, ask directly -- Use Slack's mrkdwn format (NOT standard Markdown): - - Bold: *text* (single asterisks) - - Italic: _text_ - - Strikethrough: ~text~ - - Code: \`code\` - - Code block: \`\`\`code\`\`\` - - Links: - - Do NOT use **double asterisks** or [markdown](links) +## Slack Formatting (mrkdwn, NOT Markdown) +Bold: *text*, Italic: _text_, Code: \`code\`, Block: \`\`\`code\`\`\`, Links: +Do NOT use **double asterisks** or [markdown](links). -## Your Environment +## Slack IDs +Channels: ${channelMappings} + +Users: ${userMappings} + +When mentioning users, use <@username> format (e.g., <@mario>). + +## Environment ${envDescription} -## Your Workspace -Your working directory is: ${channelPath} +## Workspace Layout +${workspacePath}/ +├── MEMORY.md # Global memory (all channels) +├── skills/ # Global CLI tools you create +└── ${channelId}/ # This channel + ├── MEMORY.md # Channel-specific memory + ├── log.jsonl # Full message history + ├── attachments/ # User-shared files + ├── scratch/ # Your working directory + └── skills/ # Channel-specific tools -### Directory Structure -- ${workspacePath}/ - Root workspace (shared across all channels) - - MEMORY.md - GLOBAL memory visible to all channels (write global info here) - - ${channelId}/ - This channel's directory - - MEMORY.md - CHANNEL-SPECIFIC memory (only visible in this channel) - - scratch/ - Your working directory for files, repos, etc. - - log.jsonl - Message history in JSONL format (one JSON object per line) - - attachments/ - Files shared by users (managed by system, read-only) +## Skills (Custom CLI Tools) +You can create reusable CLI tools for recurring tasks (email, APIs, data processing, etc.). +Store in \`${workspacePath}/skills//\` or \`${channelPath}/skills//\`. +Each skill needs a \`SKILL.md\` documenting usage. Read it before using a skill. +List skills in global memory so you remember them. -### Message History Format -Each line in log.jsonl contains: -{ - "date": "2025-11-26T10:44:00.123Z", // ISO 8601 - easy to grep by date! - "ts": "1732619040.123456", // Slack timestamp or epoch ms - "user": "U123ABC", // User ID or "bot" - "userName": "mario", // User handle (optional) - "text": "message text", - "isBot": false -} +## Memory +Write to MEMORY.md files to persist context across conversations. +- Global (${workspacePath}/MEMORY.md): skills, preferences, project info +- Channel (${channelPath}/MEMORY.md): channel-specific decisions, ongoing work +Update when you learn something important or when asked to remember something. -**⚠️ CRITICAL: Efficient Log Queries (Avoid Context Overflow)** - -Log files can be VERY LARGE (100K+ lines). The problem is getting too MANY messages, not message length. -Each message can be up to 10k chars - that's fine. Use head/tail to LIMIT NUMBER OF MESSAGES (10-50 at a time). - -**Install jq first (if not already):** -\`\`\`bash -${isDocker ? "apk add jq" : "# jq should be available, or install via package manager"} -\`\`\` - -**Essential query patterns:** -\`\`\`bash -# Last N messages (compact JSON output) -tail -20 log.jsonl | jq -c '{date: .date[0:19], user: (.userName // .user), text, attachments: [(.attachments // [])[].local]}' - -# Or TSV format (easier to read) -tail -20 log.jsonl | jq -r '[.date[0:19], (.userName // .user), .text, ((.attachments // []) | map(.local) | join(","))] | @tsv' - -# Search by date (LIMIT with head/tail!) -grep '"date":"2025-11-26' log.jsonl | tail -30 | jq -c '{date: .date[0:19], user: (.userName // .user), text, attachments: [(.attachments // [])[].local]}' - -# Messages from specific user (count first, then limit) -grep '"userName":"mario"' log.jsonl | wc -l # Check count first -grep '"userName":"mario"' log.jsonl | tail -20 | jq -c '{date: .date[0:19], user: .userName, text, attachments: [(.attachments // [])[].local]}' - -# Only count (when you just need the number) -grep '"isBot":false' log.jsonl | wc -l - -# Messages with attachments only (limit!) -grep '"attachments":[{' log.jsonl | tail -10 | jq -r '[.date[0:16], (.userName // .user), .text, (.attachments | map(.local) | join(","))] | @tsv' -\`\`\` - -**KEY RULE:** Always pipe through 'head -N' or 'tail -N' to limit results BEFORE parsing with jq! -\`\`\` - -**Date filtering:** -- Today: grep '"date":"${currentDate}' log.jsonl -- Yesterday: grep '"date":"2025-11-25' log.jsonl -- Date range: grep '"date":"2025-11-(26|27|28)' log.jsonl -- Time range: grep -E '"date":"2025-11-26T(09|10|11):' log.jsonl - -### Working Memory System -You can maintain working memory across conversations by writing MEMORY.md files. - -**IMPORTANT PATH RULES:** -- Global memory (all channels): ${workspacePath}/MEMORY.md -- Channel memory (this channel only): ${channelPath}/MEMORY.md - -**What to remember:** -- Project details and architecture → Global memory -- User preferences and coding style → Global memory -- Channel-specific context → Channel memory -- Recurring tasks and patterns → Appropriate memory file -- Credentials locations (never actual secrets) → Global memory -- Decisions made and their rationale → Appropriate memory file - -**When to update:** -- After learning something important that will help in future conversations -- When user asks you to remember something -- When you discover project structure or conventions - -### Current Working Memory +### Current Memory ${memory} +## Log Queries (CRITICAL: limit output to avoid context overflow) +Format: \`{"date":"...","ts":"...","user":"...","userName":"...","text":"...","isBot":false}\` +The log contains user messages AND your tool calls/results. Filter appropriately. +${isDocker ? "Install jq: apk add jq" : ""} + +**Conversation only (excludes tool calls/results) - use for summaries:** +\`\`\`bash +# Recent conversation (no [Tool] or [Tool Result] lines) +grep -v '"text":"\\[Tool' log.jsonl | tail -30 | jq -c '{date: .date[0:19], user: (.userName // .user), text}' + +# Yesterday's conversation +grep '"date":"2025-11-26' log.jsonl | grep -v '"text":"\\[Tool' | jq -c '{date: .date[0:19], user: (.userName // .user), text}' + +# Specific user's messages +grep '"userName":"mario"' log.jsonl | grep -v '"text":"\\[Tool' | tail -20 | jq -c '{date: .date[0:19], text}' +\`\`\` + +**Full details (includes tool calls) - use when you need technical context:** +\`\`\`bash +# Raw recent entries +tail -20 log.jsonl | jq -c '{date: .date[0:19], user: (.userName // .user), text}' + +# Count all messages +wc -l log.jsonl +\`\`\` + ## Tools -You have access to: bash, read, edit, write, attach tools. -- bash: Run shell commands (this is your main tool) +- bash: Run shell commands (primary tool). Install packages as needed. - read: Read files -- edit: Edit files surgically -- write: Create/overwrite files -- attach: Share a file with the user in Slack +- write: Create/overwrite files +- edit: Surgical file edits +- attach: Share files to Slack -Each tool requires a "label" parameter - brief description shown to the user. - -## Guidelines -- Be concise and helpful -- Use bash for most operations -- If you need a tool, install it -- If you need credentials, ask the user - -## CRITICAL -- DO NOT USE EMOJIS. KEEP YOUR RESPONSES AS SHORT AS POSSIBLE. +Each tool requires a "label" parameter (shown to user). `; } @@ -324,7 +366,20 @@ export function createAgentRunner(sandboxConfig: SandboxConfig): AgentRunner { const workspacePath = executor.getWorkspacePath(channelDir.replace(`/${channelId}`, "")); const recentMessages = getRecentMessages(channelDir, 50); const memory = getMemory(channelDir); - const systemPrompt = buildSystemPrompt(workspacePath, channelId, memory, sandboxConfig); + const systemPrompt = buildSystemPrompt( + workspacePath, + channelId, + memory, + sandboxConfig, + ctx.channels, + ctx.users, + ); + + // Debug: log context sizes + log.logInfo( + `Context sizes - system: ${systemPrompt.length} chars, messages: ${recentMessages.length} chars, memory: ${memory.length} chars`, + ); + log.logInfo(`Channels: ${ctx.channels.length}, Users: ${ctx.users.length}`); // Set up file upload function for the attach tool // For Docker, we need to translate paths back to host @@ -415,10 +470,13 @@ export function createAgentRunner(sandboxConfig: SandboxConfig): AgentRunner { }); }, // Enqueue a message that may need splitting - enqueueMessage(text: string, target: "main" | "thread", errorContext: string): void { + enqueueMessage(text: string, target: "main" | "thread", errorContext: string, log = true): void { const parts = splitForSlack(text); for (const part of parts) { - this.enqueue(() => (target === "main" ? ctx.respond(part) : ctx.respondInThread(part)), errorContext); + this.enqueue( + () => (target === "main" ? ctx.respond(part, log) : ctx.respondInThread(part)), + errorContext, + ); } }, flush(): Promise { @@ -446,7 +504,7 @@ export function createAgentRunner(sandboxConfig: SandboxConfig): AgentRunner { // Log to jsonl await store.logMessage(ctx.message.channel, { date: new Date().toISOString(), - ts: Date.now().toString(), + ts: toSlackTs(), user: "bot", text: `[Tool] ${event.toolName}: ${JSON.stringify(event.args)}`, attachments: [], @@ -454,7 +512,7 @@ export function createAgentRunner(sandboxConfig: SandboxConfig): AgentRunner { }); // Show label in main message only - queue.enqueue(() => ctx.respond(`_→ ${label}_`), "tool label"); + queue.enqueue(() => ctx.respond(`_→ ${label}_`, false), "tool label"); break; } @@ -475,7 +533,7 @@ export function createAgentRunner(sandboxConfig: SandboxConfig): AgentRunner { // Log to jsonl await store.logMessage(ctx.message.channel, { date: new Date().toISOString(), - ts: Date.now().toString(), + ts: toSlackTs(), user: "bot", text: `[Tool Result] ${event.toolName}: ${event.isError ? "ERROR: " : ""}${truncate(resultStr, 1000)}`, attachments: [], @@ -500,11 +558,11 @@ export function createAgentRunner(sandboxConfig: SandboxConfig): AgentRunner { threadMessage += "*Result:*\n```\n" + resultStr + "\n```"; - queue.enqueueMessage(threadMessage, "thread", "tool result thread"); + queue.enqueueMessage(threadMessage, "thread", "tool result thread", false); // Show brief error in main message if failed if (event.isError) { - queue.enqueue(() => ctx.respond(`_Error: ${truncate(resultStr, 200)}_`), "tool error"); + queue.enqueue(() => ctx.respond(`_Error: ${truncate(resultStr, 200)}_`, false), "tool error"); } break; } @@ -560,14 +618,14 @@ export function createAgentRunner(sandboxConfig: SandboxConfig): AgentRunner { for (const thinking of thinkingParts) { log.logThinking(logCtx, thinking); queue.enqueueMessage(`_${thinking}_`, "main", "thinking main"); - queue.enqueueMessage(`_${thinking}_`, "thread", "thinking thread"); + queue.enqueueMessage(`_${thinking}_`, "thread", "thinking thread", false); } // Post text to main message and thread if (text.trim()) { log.logResponse(logCtx, text); queue.enqueueMessage(text, "main", "response main"); - queue.enqueueMessage(text, "thread", "response thread"); + queue.enqueueMessage(text, "thread", "response thread", false); } } break; @@ -576,12 +634,18 @@ export function createAgentRunner(sandboxConfig: SandboxConfig): AgentRunner { // Run the agent with user's message // Prepend recent messages to the user prompt (not system prompt) for better caching + // The current message is already the last entry in recentMessages const userPrompt = - `Recent conversation history (last 50 messages):\n` + + `Conversation history (last 50 turns). Respond to the last message.\n` + `Format: date TAB user TAB text TAB attachments\n\n` + - `${recentMessages}\n\n` + - `---\n\n` + - `Current message: ${ctx.message.text || "(attached files)"}`; + recentMessages; + // Debug: write full context to file + const toolDefs = tools.map((t) => ({ name: t.name, description: t.description, parameters: t.parameters })); + const debugPrompt = + `=== SYSTEM PROMPT (${systemPrompt.length} chars) ===\n\n${systemPrompt}\n\n` + + `=== TOOL DEFINITIONS (${JSON.stringify(toolDefs).length} chars) ===\n\n${JSON.stringify(toolDefs, null, 2)}\n\n` + + `=== USER PROMPT (${userPrompt.length} chars) ===\n\n${userPrompt}`; + await writeFile(join(channelDir, "last_prompt.txt"), debugPrompt, "utf-8"); await agent.prompt(userPrompt); diff --git a/packages/mom/src/log.ts b/packages/mom/src/log.ts index 5b84e7d5..e8d3ed86 100644 --- a/packages/mom/src/log.ts +++ b/packages/mom/src/log.ts @@ -160,6 +160,10 @@ export function logStopRequest(ctx: LogContext): void { } // System +export function logInfo(message: string): void { + console.log(chalk.blue(`${timestamp()} [system] ${message}`)); +} + export function logWarning(message: string, details?: string): void { console.log(chalk.yellow(`${timestamp()} [system] ⚠ ${message}`)); if (details) { diff --git a/packages/mom/src/slack.ts b/packages/mom/src/slack.ts index fc63796c..c5766722 100644 --- a/packages/mom/src/slack.ts +++ b/packages/mom/src/slack.ts @@ -19,8 +19,12 @@ export interface SlackContext { message: SlackMessage; channelName?: string; // channel name for logging (e.g., #dev-team) store: ChannelStore; - /** Send/update the main message (accumulates text) */ - respond(text: string): Promise; + /** All channels the bot is a member of */ + channels: ChannelInfo[]; + /** All known users in the workspace */ + users: UserInfo[]; + /** Send/update the main message (accumulates text). Set log=false to skip logging. */ + respond(text: string, log?: boolean): Promise; /** Replace the entire message text (not append) */ replaceMessage(text: string): Promise; /** Post a message in the thread under the main message (for verbose details) */ @@ -44,6 +48,17 @@ export interface MomBotConfig { workingDir: string; // directory for channel data and attachments } +export interface ChannelInfo { + id: string; + name: string; +} + +export interface UserInfo { + id: string; + userName: string; + displayName: string; +} + export class MomBot { private socketClient: SocketModeClient; private webClient: WebClient; @@ -51,6 +66,7 @@ export class MomBot { private botUserId: string | null = null; public readonly store: ChannelStore; private userCache: Map = new Map(); + private channelCache: Map = new Map(); // id -> name constructor(handler: MomHandler, config: MomBotConfig) { this.handler = handler; @@ -64,6 +80,113 @@ export class MomBot { this.setupEventHandlers(); } + /** + * Fetch all channels the bot is a member of + */ + private async fetchChannels(): Promise { + try { + let cursor: string | undefined; + do { + const result = await this.webClient.conversations.list({ + types: "public_channel,private_channel", + exclude_archived: true, + limit: 200, + cursor, + }); + + const channels = result.channels as Array<{ id?: string; name?: string; is_member?: boolean }> | undefined; + if (channels) { + for (const channel of channels) { + if (channel.id && channel.name && channel.is_member) { + this.channelCache.set(channel.id, channel.name); + } + } + } + + cursor = result.response_metadata?.next_cursor; + } while (cursor); + } catch (error) { + log.logWarning("Failed to fetch channels", String(error)); + } + } + + /** + * Fetch all workspace users + */ + private async fetchUsers(): Promise { + try { + let cursor: string | undefined; + do { + const result = await this.webClient.users.list({ + limit: 200, + cursor, + }); + + const members = result.members as + | Array<{ id?: string; name?: string; real_name?: string; deleted?: boolean }> + | undefined; + if (members) { + for (const user of members) { + if (user.id && user.name && !user.deleted) { + this.userCache.set(user.id, { + userName: user.name, + displayName: user.real_name || user.name, + }); + } + } + } + + cursor = result.response_metadata?.next_cursor; + } while (cursor); + } catch (error) { + log.logWarning("Failed to fetch users", String(error)); + } + } + + /** + * Get all known channels (id -> name) + */ + getChannels(): ChannelInfo[] { + return Array.from(this.channelCache.entries()).map(([id, name]) => ({ id, name })); + } + + /** + * Get all known users + */ + getUsers(): UserInfo[] { + return Array.from(this.userCache.entries()).map(([id, { userName, displayName }]) => ({ + id, + userName, + displayName, + })); + } + + /** + * Obfuscate usernames and user IDs in text to prevent pinging people + * e.g., "nate" -> "n_a_t_e", "@mario" -> "@m_a_r_i_o", "<@U123>" -> "<@U_1_2_3>" + */ + private obfuscateUsernames(text: string): string { + let result = text; + + // Obfuscate user IDs like <@U16LAL8LS> + result = result.replace(/<@([A-Z0-9]+)>/gi, (_match, id) => { + return `<@${id.split("").join("_")}>`; + }); + + // Obfuscate usernames + for (const { userName } of this.userCache.values()) { + // Escape special regex characters in username + const escaped = userName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + // Match @username, <@username>, or bare username (case insensitive, word boundary) + const pattern = new RegExp(`(<@|@)?(\\b${escaped}\\b)`, "gi"); + result = result.replace(pattern, (_match, prefix, name) => { + const obfuscated = name.split("").join("_"); + return (prefix || "") + obfuscated; + }); + } + return result; + } + private async getUserInfo(userId: string): Promise<{ userName: string; displayName: string }> { if (this.userCache.has(userId)) { return this.userCache.get(userId)!; @@ -85,6 +208,7 @@ export class MomBot { private setupEventHandlers(): void { // Handle @mentions in channels + // Note: We don't log here - the message event handler logs all messages this.socketClient.on("app_mention", async ({ event, ack }) => { await ack(); @@ -96,9 +220,6 @@ export class MomBot { files?: Array<{ name: string; url_private_download?: string; url_private?: string }>; }; - // Log the mention (message event may not fire for app_mention) - await this.logMessage(slackEvent); - const ctx = await this.createContext(slackEvent); await this.handler.onChannelMention(ctx); }); @@ -221,7 +342,9 @@ export class MomBot { }, channelName, store: this.store, - respond: async (responseText: string) => { + channels: this.getChannels(), + users: this.getUsers(), + respond: async (responseText: string, log = true) => { // Queue updates to avoid race conditions updatePromise = updatePromise.then(async () => { if (isThinking) { @@ -252,8 +375,10 @@ export class MomBot { messageTs = result.ts as string; } - // Log the response - await this.store.logBotResponse(event.channel, responseText, messageTs!); + // Log the response if requested + if (log) { + await this.store.logBotResponse(event.channel, responseText, messageTs!); + } }); await updatePromise; @@ -265,11 +390,13 @@ export class MomBot { // No main message yet, just skip return; } + // Obfuscate usernames to avoid pinging people in thread details + const obfuscatedText = this.obfuscateUsernames(threadText); // Post in thread under the main message await this.webClient.chat.postMessage({ channel: event.channel, thread_ts: messageTs, - text: threadText, + text: obfuscatedText, }); }); await updatePromise; @@ -343,6 +470,11 @@ export class MomBot { async start(): Promise { const auth = await this.webClient.auth.test(); this.botUserId = auth.user_id as string; + + // Fetch channels and users in parallel + await Promise.all([this.fetchChannels(), this.fetchUsers()]); + log.logInfo(`Loaded ${this.channelCache.size} channels, ${this.userCache.size} users`); + await this.socketClient.start(); log.logConnected(); }