diff --git a/packages/coding-agent/src/core/model-resolver.ts b/packages/coding-agent/src/core/model-resolver.ts index be03520f..7b352f2c 100644 --- a/packages/coding-agent/src/core/model-resolver.ts +++ b/packages/coding-agent/src/core/model-resolver.ts @@ -28,12 +28,143 @@ export interface ScopedModel { thinkingLevel: ThinkingLevel; } +/** + * Helper to check if a model ID looks like an alias (no date suffix) + * Dates are typically in format: -20241022 or -20250929 + */ +function isAlias(id: string): boolean { + // Check if ID ends with -latest + if (id.endsWith("-latest")) return true; + + // Check if ID ends with a date pattern (-YYYYMMDD) + const datePattern = /-\d{8}$/; + return !datePattern.test(id); +} + +/** + * Try to match a pattern to a model from the available models list. + * Returns the matched model or null if no match found. + */ +function tryMatchModel(modelPattern: string, availableModels: Model[]): Model | null { + // Check for provider/modelId format (provider is everything before the first /) + const slashIndex = modelPattern.indexOf("/"); + if (slashIndex !== -1) { + const provider = modelPattern.substring(0, slashIndex); + const modelId = modelPattern.substring(slashIndex + 1); + const providerMatch = availableModels.find( + (m) => m.provider.toLowerCase() === provider.toLowerCase() && m.id.toLowerCase() === modelId.toLowerCase(), + ); + if (providerMatch) { + return providerMatch; + } + // No exact provider/model match - fall through to other matching + } + + // Check for exact ID match (case-insensitive) + const exactMatch = availableModels.find((m) => m.id.toLowerCase() === modelPattern.toLowerCase()); + if (exactMatch) { + return exactMatch; + } + + // No exact match - fall back to partial matching + const matches = availableModels.filter( + (m) => + m.id.toLowerCase().includes(modelPattern.toLowerCase()) || + m.name?.toLowerCase().includes(modelPattern.toLowerCase()), + ); + + if (matches.length === 0) { + return null; + } + + // Separate into aliases and dated versions + const aliases = matches.filter((m) => isAlias(m.id)); + const datedVersions = matches.filter((m) => !isAlias(m.id)); + + if (aliases.length > 0) { + // Prefer alias - if multiple aliases, pick the one that sorts highest + aliases.sort((a, b) => b.id.localeCompare(a.id)); + return aliases[0]; + } else { + // No alias found, pick latest dated version + datedVersions.sort((a, b) => b.id.localeCompare(a.id)); + return datedVersions[0]; + } +} + +export interface ParsedModelResult { + model: Model | null; + thinkingLevel: ThinkingLevel; + warning: string | null; +} + +/** + * Parse a pattern to extract model and thinking level. + * Handles models with colons in their IDs (e.g., OpenRouter's :exacto suffix). + * + * Algorithm: + * 1. Try to match full pattern as a model + * 2. If found, return it with "off" thinking level + * 3. If not found and has colons, split on last colon: + * - If suffix is valid thinking level, use it and recurse on prefix + * - If suffix is invalid, warn and recurse on prefix with "off" + * + * @internal Exported for testing + */ +export function parseModelPattern(pattern: string, availableModels: Model[]): ParsedModelResult { + // Try exact match first + const exactMatch = tryMatchModel(pattern, availableModels); + if (exactMatch) { + return { model: exactMatch, thinkingLevel: "off", warning: null }; + } + + // No match - try splitting on last colon if present + const lastColonIndex = pattern.lastIndexOf(":"); + if (lastColonIndex === -1) { + // No colons, pattern simply doesn't match any model + return { model: null, thinkingLevel: "off", warning: null }; + } + + const prefix = pattern.substring(0, lastColonIndex); + const suffix = pattern.substring(lastColonIndex + 1); + + if (isValidThinkingLevel(suffix)) { + // Valid thinking level - recurse on prefix and use this level + const result = parseModelPattern(prefix, availableModels); + if (result.model) { + // Only use this thinking level if no warning from inner recursion + // (if there was an invalid suffix deeper, we already have "off") + return { + model: result.model, + thinkingLevel: result.warning ? "off" : suffix, + warning: result.warning, + }; + } + return result; + } else { + // Invalid suffix - recurse on prefix with "off" and warn + const result = parseModelPattern(prefix, availableModels); + if (result.model) { + return { + model: result.model, + thinkingLevel: "off", + warning: `Invalid thinking level "${suffix}" in pattern "${pattern}". Using "off" instead.`, + }; + } + return result; + } +} + /** * Resolve model patterns to actual Model objects with optional thinking levels * Format: "pattern:level" where :level is optional * For each pattern, finds all matching models and picks the best version: * 1. Prefer alias (e.g., claude-sonnet-4-5) over dated versions (claude-sonnet-4-5-20250929) * 2. If no alias, pick the latest dated version + * + * Supports models with colons in their IDs (e.g., OpenRouter's model:exacto). + * The algorithm tries to match the full pattern first, then progressively + * strips colon-suffixes to find a match. */ export async function resolveModelScope(patterns: string[]): Promise { const { models: availableModels, error } = await getAvailableModels(); @@ -46,95 +177,20 @@ export async function resolveModelScope(patterns: string[]): Promise 1) { - const level = parts[1]; - if (isValidThinkingLevel(level)) { - thinkingLevel = level; - } else { - console.warn( - chalk.yellow(`Warning: Invalid thinking level "${level}" in pattern "${pattern}". Using "off" instead.`), - ); - } + if (warning) { + console.warn(chalk.yellow(`Warning: ${warning}`)); } - // Check for provider/modelId format (provider is everything before the first /) - const slashIndex = modelPattern.indexOf("/"); - if (slashIndex !== -1) { - const provider = modelPattern.substring(0, slashIndex); - const modelId = modelPattern.substring(slashIndex + 1); - const providerMatch = availableModels.find( - (m) => m.provider.toLowerCase() === provider.toLowerCase() && m.id.toLowerCase() === modelId.toLowerCase(), - ); - if (providerMatch) { - if ( - !scopedModels.find( - (sm) => sm.model.id === providerMatch.id && sm.model.provider === providerMatch.provider, - ) - ) { - scopedModels.push({ model: providerMatch, thinkingLevel }); - } - continue; - } - // No exact provider/model match - fall through to other matching - } - - // Check for exact ID match (case-insensitive) - const exactMatch = availableModels.find((m) => m.id.toLowerCase() === modelPattern.toLowerCase()); - if (exactMatch) { - // Exact match found - use it directly - if (!scopedModels.find((sm) => sm.model.id === exactMatch.id && sm.model.provider === exactMatch.provider)) { - scopedModels.push({ model: exactMatch, thinkingLevel }); - } + if (!model) { + console.warn(chalk.yellow(`Warning: No models match pattern "${pattern}"`)); continue; } - // No exact match - fall back to partial matching - const matches = availableModels.filter( - (m) => - m.id.toLowerCase().includes(modelPattern.toLowerCase()) || - m.name?.toLowerCase().includes(modelPattern.toLowerCase()), - ); - - if (matches.length === 0) { - console.warn(chalk.yellow(`Warning: No models match pattern "${modelPattern}"`)); - continue; - } - - // Helper to check if a model ID looks like an alias (no date suffix) - // Dates are typically in format: -20241022 or -20250929 - const isAlias = (id: string): boolean => { - // Check if ID ends with -latest - if (id.endsWith("-latest")) return true; - - // Check if ID ends with a date pattern (-YYYYMMDD) - const datePattern = /-\d{8}$/; - return !datePattern.test(id); - }; - - // Separate into aliases and dated versions - const aliases = matches.filter((m) => isAlias(m.id)); - const datedVersions = matches.filter((m) => !isAlias(m.id)); - - let bestMatch: Model; - - if (aliases.length > 0) { - // Prefer alias - if multiple aliases, pick the one that sorts highest - aliases.sort((a, b) => b.id.localeCompare(a.id)); - bestMatch = aliases[0]; - } else { - // No alias found, pick latest dated version - datedVersions.sort((a, b) => b.id.localeCompare(a.id)); - bestMatch = datedVersions[0]; - } - // Avoid duplicates - if (!scopedModels.find((sm) => sm.model.id === bestMatch.id && sm.model.provider === bestMatch.provider)) { - scopedModels.push({ model: bestMatch, thinkingLevel }); + if (!scopedModels.find((sm) => sm.model.id === model.id && sm.model.provider === model.provider)) { + scopedModels.push({ model, thinkingLevel }); } } diff --git a/packages/coding-agent/test/model-resolver.test.ts b/packages/coding-agent/test/model-resolver.test.ts new file mode 100644 index 00000000..59162d6f --- /dev/null +++ b/packages/coding-agent/test/model-resolver.test.ts @@ -0,0 +1,202 @@ +import type { Model } from "@mariozechner/pi-ai"; +import { describe, expect, test } from "vitest"; +import { parseModelPattern } from "../src/core/model-resolver.js"; + +// Mock models for testing +const mockModels: Model<"anthropic-messages">[] = [ + { + id: "claude-sonnet-4-5", + name: "Claude Sonnet 4.5", + api: "anthropic-messages", + provider: "anthropic", + baseUrl: "https://api.anthropic.com", + reasoning: true, + input: ["text", "image"], + cost: { input: 3, output: 15, cacheRead: 0.3, cacheWrite: 3.75 }, + contextWindow: 200000, + maxTokens: 8192, + }, + { + id: "gpt-4o", + name: "GPT-4o", + api: "anthropic-messages", // Using same type for simplicity + provider: "openai", + baseUrl: "https://api.openai.com", + reasoning: false, + input: ["text", "image"], + cost: { input: 5, output: 15, cacheRead: 0.5, cacheWrite: 5 }, + contextWindow: 128000, + maxTokens: 4096, + }, +]; + +// Mock OpenRouter models with colons in IDs +const mockOpenRouterModels: Model<"anthropic-messages">[] = [ + { + id: "qwen/qwen3-coder:exacto", + name: "Qwen3 Coder Exacto", + api: "anthropic-messages", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + reasoning: true, + input: ["text"], + cost: { input: 1, output: 2, cacheRead: 0.1, cacheWrite: 1 }, + contextWindow: 128000, + maxTokens: 8192, + }, + { + id: "openai/gpt-4o:extended", + name: "GPT-4o Extended", + api: "anthropic-messages", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + reasoning: false, + input: ["text", "image"], + cost: { input: 5, output: 15, cacheRead: 0.5, cacheWrite: 5 }, + contextWindow: 128000, + maxTokens: 4096, + }, +]; + +const allModels = [...mockModels, ...mockOpenRouterModels]; + +describe("parseModelPattern", () => { + describe("simple patterns without colons", () => { + test("exact match returns model with off thinking level", () => { + const result = parseModelPattern("claude-sonnet-4-5", allModels); + expect(result.model?.id).toBe("claude-sonnet-4-5"); + expect(result.thinkingLevel).toBe("off"); + expect(result.warning).toBeNull(); + }); + + test("partial match returns best model", () => { + const result = parseModelPattern("sonnet", allModels); + expect(result.model?.id).toBe("claude-sonnet-4-5"); + expect(result.thinkingLevel).toBe("off"); + expect(result.warning).toBeNull(); + }); + + test("no match returns null model", () => { + const result = parseModelPattern("nonexistent", allModels); + expect(result.model).toBeNull(); + expect(result.thinkingLevel).toBe("off"); + expect(result.warning).toBeNull(); + }); + }); + + describe("patterns with valid thinking levels", () => { + test("sonnet:high returns sonnet with high thinking level", () => { + const result = parseModelPattern("sonnet:high", allModels); + expect(result.model?.id).toBe("claude-sonnet-4-5"); + expect(result.thinkingLevel).toBe("high"); + expect(result.warning).toBeNull(); + }); + + test("gpt-4o:medium returns gpt-4o with medium thinking level", () => { + const result = parseModelPattern("gpt-4o:medium", allModels); + expect(result.model?.id).toBe("gpt-4o"); + expect(result.thinkingLevel).toBe("medium"); + expect(result.warning).toBeNull(); + }); + + test("all valid thinking levels work", () => { + for (const level of ["off", "minimal", "low", "medium", "high", "xhigh"]) { + const result = parseModelPattern(`sonnet:${level}`, allModels); + expect(result.model?.id).toBe("claude-sonnet-4-5"); + expect(result.thinkingLevel).toBe(level); + expect(result.warning).toBeNull(); + } + }); + }); + + describe("patterns with invalid thinking levels", () => { + test("sonnet:random returns sonnet with off and warning", () => { + const result = parseModelPattern("sonnet:random", allModels); + expect(result.model?.id).toBe("claude-sonnet-4-5"); + expect(result.thinkingLevel).toBe("off"); + expect(result.warning).toContain("Invalid thinking level"); + expect(result.warning).toContain("random"); + }); + + test("gpt-4o:invalid returns gpt-4o with off and warning", () => { + const result = parseModelPattern("gpt-4o:invalid", allModels); + expect(result.model?.id).toBe("gpt-4o"); + expect(result.thinkingLevel).toBe("off"); + expect(result.warning).toContain("Invalid thinking level"); + }); + }); + + describe("OpenRouter models with colons in IDs", () => { + test("qwen3-coder:exacto matches the model with off", () => { + const result = parseModelPattern("qwen/qwen3-coder:exacto", allModels); + expect(result.model?.id).toBe("qwen/qwen3-coder:exacto"); + expect(result.thinkingLevel).toBe("off"); + expect(result.warning).toBeNull(); + }); + + test("openrouter/qwen/qwen3-coder:exacto matches with provider prefix", () => { + const result = parseModelPattern("openrouter/qwen/qwen3-coder:exacto", allModels); + expect(result.model?.id).toBe("qwen/qwen3-coder:exacto"); + expect(result.model?.provider).toBe("openrouter"); + expect(result.thinkingLevel).toBe("off"); + expect(result.warning).toBeNull(); + }); + + test("qwen3-coder:exacto:high matches model with high thinking level", () => { + const result = parseModelPattern("qwen/qwen3-coder:exacto:high", allModels); + expect(result.model?.id).toBe("qwen/qwen3-coder:exacto"); + expect(result.thinkingLevel).toBe("high"); + expect(result.warning).toBeNull(); + }); + + test("openrouter/qwen/qwen3-coder:exacto:high matches with provider and thinking level", () => { + const result = parseModelPattern("openrouter/qwen/qwen3-coder:exacto:high", allModels); + expect(result.model?.id).toBe("qwen/qwen3-coder:exacto"); + expect(result.model?.provider).toBe("openrouter"); + expect(result.thinkingLevel).toBe("high"); + expect(result.warning).toBeNull(); + }); + + test("gpt-4o:extended matches the extended model", () => { + const result = parseModelPattern("openai/gpt-4o:extended", allModels); + expect(result.model?.id).toBe("openai/gpt-4o:extended"); + expect(result.thinkingLevel).toBe("off"); + expect(result.warning).toBeNull(); + }); + }); + + describe("invalid thinking levels with OpenRouter models", () => { + test("qwen3-coder:exacto:random returns model with off and warning", () => { + const result = parseModelPattern("qwen/qwen3-coder:exacto:random", allModels); + expect(result.model?.id).toBe("qwen/qwen3-coder:exacto"); + expect(result.thinkingLevel).toBe("off"); + expect(result.warning).toContain("Invalid thinking level"); + expect(result.warning).toContain("random"); + }); + + test("qwen3-coder:exacto:high:random returns model with off and warning", () => { + const result = parseModelPattern("qwen/qwen3-coder:exacto:high:random", allModels); + expect(result.model?.id).toBe("qwen/qwen3-coder:exacto"); + expect(result.thinkingLevel).toBe("off"); + expect(result.warning).toContain("Invalid thinking level"); + expect(result.warning).toContain("random"); + }); + }); + + describe("edge cases", () => { + test("empty pattern matches via partial matching", () => { + // Empty string is included in all model IDs, so partial matching finds a match + const result = parseModelPattern("", allModels); + expect(result.model).not.toBeNull(); + expect(result.thinkingLevel).toBe("off"); + }); + + test("pattern ending with colon treats empty suffix as invalid", () => { + const result = parseModelPattern("sonnet:", allModels); + // Empty string after colon is not a valid thinking level + // So it tries to match "sonnet:" which won't match, then tries "sonnet" + expect(result.model?.id).toBe("claude-sonnet-4-5"); + expect(result.warning).toContain("Invalid thinking level"); + }); + }); +});