mirror of
https://github.com/getcompanion-ai/co-mono.git
synced 2026-04-16 18:03:50 +00:00
Custom provider WIP
This commit is contained in:
Mario Zechner
parent
389c80d7a8
commit
1f9a3a00cc
17 changed files with 1185 additions and 107 deletions
|
|
@ -170,6 +170,40 @@ declare module "@mariozechner/mini-lit" {
|
|||
messages: string;
|
||||
tokens: string;
|
||||
"Drop files here": string;
|
||||
// Providers & Models
|
||||
"Providers & Models": string;
|
||||
"Cloud Providers": string;
|
||||
"Cloud LLM providers with predefined models. API keys are stored locally in your browser.": string;
|
||||
"Custom Providers": string;
|
||||
"User-configured servers with auto-discovered or manually defined models.": string;
|
||||
"Add Provider": string;
|
||||
"No custom providers configured. Click 'Add Provider' to get started.": string;
|
||||
Models: string;
|
||||
"auto-discovered": string;
|
||||
Refresh: string;
|
||||
Edit: string;
|
||||
"Are you sure you want to delete this provider?": string;
|
||||
"Edit Provider": string;
|
||||
"Provider Name": string;
|
||||
"e.g., My Ollama Server": string;
|
||||
"Provider Type": string;
|
||||
"Base URL": string;
|
||||
"e.g., http://localhost:11434": string;
|
||||
"API Key (Optional)": string;
|
||||
"Leave empty if not required": string;
|
||||
"Test Connection": string;
|
||||
Discovered: string;
|
||||
models: string;
|
||||
and: string;
|
||||
more: string;
|
||||
"For manual provider types, add models after saving the provider.": string;
|
||||
"Please fill in all required fields": string;
|
||||
"Failed to save provider": string;
|
||||
"OpenAI Completions Compatible": string;
|
||||
"OpenAI Responses Compatible": string;
|
||||
"Anthropic Messages Compatible": string;
|
||||
"Checking...": string;
|
||||
Disconnected: string;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -354,6 +388,44 @@ export const translations = {
|
|||
Delete: "Delete",
|
||||
"Drop files here": "Drop files here",
|
||||
"Command failed:": "Command failed:",
|
||||
// Providers & Models
|
||||
"Providers & Models": "Providers & Models",
|
||||
"Cloud Providers": "Cloud Providers",
|
||||
"Cloud LLM providers with predefined models. API keys are stored locally in your browser.":
|
||||
"Cloud LLM providers with predefined models. API keys are stored locally in your browser.",
|
||||
"Custom Providers": "Custom Providers",
|
||||
"User-configured servers with auto-discovered or manually defined models.":
|
||||
"User-configured servers with auto-discovered or manually defined models.",
|
||||
"Add Provider": "Add Provider",
|
||||
"No custom providers configured. Click 'Add Provider' to get started.":
|
||||
"No custom providers configured. Click 'Add Provider' to get started.",
|
||||
"auto-discovered": "auto-discovered",
|
||||
Refresh: "Refresh",
|
||||
Edit: "Edit",
|
||||
"Are you sure you want to delete this provider?": "Are you sure you want to delete this provider?",
|
||||
"Edit Provider": "Edit Provider",
|
||||
"Provider Name": "Provider Name",
|
||||
"e.g., My Ollama Server": "e.g., My Ollama Server",
|
||||
"Provider Type": "Provider Type",
|
||||
"Base URL": "Base URL",
|
||||
"e.g., http://localhost:11434": "e.g., http://localhost:11434",
|
||||
"API Key (Optional)": "API Key (Optional)",
|
||||
"Leave empty if not required": "Leave empty if not required",
|
||||
"Test Connection": "Test Connection",
|
||||
Discovered: "Discovered",
|
||||
Models: "Models",
|
||||
models: "models",
|
||||
and: "and",
|
||||
more: "more",
|
||||
"For manual provider types, add models after saving the provider.":
|
||||
"For manual provider types, add models after saving the provider.",
|
||||
"Please fill in all required fields": "Please fill in all required fields",
|
||||
"Failed to save provider": "Failed to save provider",
|
||||
"OpenAI Completions Compatible": "OpenAI Completions Compatible",
|
||||
"OpenAI Responses Compatible": "OpenAI Responses Compatible",
|
||||
"Anthropic Messages Compatible": "Anthropic Messages Compatible",
|
||||
"Checking...": "Checking...",
|
||||
Disconnected: "Disconnected",
|
||||
},
|
||||
de: {
|
||||
...defaultGerman,
|
||||
|
|
@ -535,6 +607,44 @@ export const translations = {
|
|||
Delete: "Löschen",
|
||||
"Drop files here": "Dateien hier ablegen",
|
||||
"Command failed:": "Befehl fehlgeschlagen:",
|
||||
// Providers & Models
|
||||
"Providers & Models": "Anbieter & Modelle",
|
||||
"Cloud Providers": "Cloud-Anbieter",
|
||||
"Cloud LLM providers with predefined models. API keys are stored locally in your browser.":
|
||||
"Cloud-LLM-Anbieter mit vordefinierten Modellen. API-Schlüssel werden lokal in Ihrem Browser gespeichert.",
|
||||
"Custom Providers": "Benutzerdefinierte Anbieter",
|
||||
"User-configured servers with auto-discovered or manually defined models.":
|
||||
"Benutzerkonfigurierte Server mit automatisch erkannten oder manuell definierten Modellen.",
|
||||
"Add Provider": "Anbieter hinzufügen",
|
||||
"No custom providers configured. Click 'Add Provider' to get started.":
|
||||
"Keine benutzerdefinierten Anbieter konfiguriert. Klicken Sie auf 'Anbieter hinzufügen', um zu beginnen.",
|
||||
"auto-discovered": "automatisch erkannt",
|
||||
Refresh: "Aktualisieren",
|
||||
Edit: "Bearbeiten",
|
||||
"Are you sure you want to delete this provider?": "Sind Sie sicher, dass Sie diesen Anbieter löschen möchten?",
|
||||
"Edit Provider": "Anbieter bearbeiten",
|
||||
"Provider Name": "Anbietername",
|
||||
"e.g., My Ollama Server": "z.B. Mein Ollama Server",
|
||||
"Provider Type": "Anbietertyp",
|
||||
"Base URL": "Basis-URL",
|
||||
"e.g., http://localhost:11434": "z.B. http://localhost:11434",
|
||||
"API Key (Optional)": "API-Schlüssel (Optional)",
|
||||
"Leave empty if not required": "Leer lassen, falls nicht erforderlich",
|
||||
"Test Connection": "Verbindung testen",
|
||||
Discovered: "Erkannt",
|
||||
Models: "Modelle",
|
||||
models: "Modelle",
|
||||
and: "und",
|
||||
more: "mehr",
|
||||
"For manual provider types, add models after saving the provider.":
|
||||
"Für manuelle Anbietertypen fügen Sie Modelle nach dem Speichern des Anbieters hinzu.",
|
||||
"Please fill in all required fields": "Bitte füllen Sie alle erforderlichen Felder aus",
|
||||
"Failed to save provider": "Fehler beim Speichern des Anbieters",
|
||||
"OpenAI Completions Compatible": "OpenAI Completions Kompatibel",
|
||||
"OpenAI Responses Compatible": "OpenAI Responses Kompatibel",
|
||||
"Anthropic Messages Compatible": "Anthropic Messages Kompatibel",
|
||||
"Checking...": "Überprüfe...",
|
||||
Disconnected: "Getrennt",
|
||||
},
|
||||
};
|
||||
|
||||
|
|
|
|||
277
packages/web-ui/src/utils/model-discovery.ts
Normal file
277
packages/web-ui/src/utils/model-discovery.ts
Normal file
|
|
@ -0,0 +1,277 @@
|
|||
import { LMStudioClient } from "@lmstudio/sdk";
|
||||
import type { Model } from "@mariozechner/pi-ai";
|
||||
import { Ollama } from "ollama/browser";
|
||||
|
||||
/**
|
||||
* Discover models from an Ollama server.
|
||||
* @param baseUrl - Base URL of the Ollama server (e.g., "http://localhost:11434")
|
||||
* @param apiKey - Optional API key (currently unused by Ollama)
|
||||
* @returns Array of discovered models
|
||||
*/
|
||||
export async function discoverOllamaModels(baseUrl: string, apiKey?: string): Promise<Model<any>[]> {
|
||||
try {
|
||||
// Create Ollama client
|
||||
const ollama = new Ollama({ host: baseUrl });
|
||||
|
||||
// Get list of available models
|
||||
const { models } = await ollama.list();
|
||||
|
||||
// Fetch details for each model and convert to Model format
|
||||
const ollamaModelPromises: Promise<Model<any> | null>[] = models.map(async (model: any) => {
|
||||
try {
|
||||
// Get model details
|
||||
const details = await ollama.show({
|
||||
model: model.name,
|
||||
});
|
||||
|
||||
// Check capabilities - filter out models that don't support tools
|
||||
const capabilities: string[] = (details as any).capabilities || [];
|
||||
if (!capabilities.includes("tools")) {
|
||||
console.debug(`Skipping model ${model.name}: does not support tools`);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Extract model info
|
||||
const modelInfo: any = details.model_info || {};
|
||||
|
||||
// Get context window size - look for architecture-specific keys
|
||||
const architecture = modelInfo["general.architecture"] || "";
|
||||
const contextKey = `${architecture}.context_length`;
|
||||
const contextWindow = parseInt(modelInfo[contextKey] || "8192", 10);
|
||||
|
||||
// Ollama caps max tokens at 10x context length
|
||||
const maxTokens = contextWindow * 10;
|
||||
|
||||
// Ollama only supports completions API
|
||||
const ollamaModel: Model<any> = {
|
||||
id: model.name,
|
||||
name: model.name,
|
||||
api: "openai-completions" as any,
|
||||
provider: "", // Will be set by caller
|
||||
baseUrl: `${baseUrl}/v1`,
|
||||
reasoning: capabilities.includes("thinking"),
|
||||
input: ["text"],
|
||||
cost: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: contextWindow,
|
||||
maxTokens: maxTokens,
|
||||
};
|
||||
|
||||
return ollamaModel;
|
||||
} catch (err) {
|
||||
console.error(`Failed to fetch details for model ${model.name}:`, err);
|
||||
return null;
|
||||
}
|
||||
});
|
||||
|
||||
const results = await Promise.all(ollamaModelPromises);
|
||||
return results.filter((m): m is Model<any> => m !== null);
|
||||
} catch (err) {
|
||||
console.error("Failed to discover Ollama models:", err);
|
||||
throw new Error(`Ollama discovery failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Discover models from a llama.cpp server via OpenAI-compatible /v1/models endpoint.
|
||||
* @param baseUrl - Base URL of the llama.cpp server (e.g., "http://localhost:8080")
|
||||
* @param apiKey - Optional API key
|
||||
* @returns Array of discovered models
|
||||
*/
|
||||
export async function discoverLlamaCppModels(baseUrl: string, apiKey?: string): Promise<Model<any>[]> {
|
||||
try {
|
||||
const headers: HeadersInit = {
|
||||
"Content-Type": "application/json",
|
||||
};
|
||||
|
||||
if (apiKey) {
|
||||
headers["Authorization"] = `Bearer ${apiKey}`;
|
||||
}
|
||||
|
||||
const response = await fetch(`${baseUrl}/v1/models`, {
|
||||
method: "GET",
|
||||
headers,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
if (!data.data || !Array.isArray(data.data)) {
|
||||
throw new Error("Invalid response format from llama.cpp server");
|
||||
}
|
||||
|
||||
return data.data.map((model: any) => {
|
||||
// llama.cpp doesn't always provide context window info
|
||||
const contextWindow = model.context_length || 8192;
|
||||
const maxTokens = model.max_tokens || 4096;
|
||||
|
||||
const llamaModel: Model<any> = {
|
||||
id: model.id,
|
||||
name: model.id,
|
||||
api: "openai-completions" as any,
|
||||
provider: "", // Will be set by caller
|
||||
baseUrl: `${baseUrl}/v1`,
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: contextWindow,
|
||||
maxTokens: maxTokens,
|
||||
};
|
||||
|
||||
return llamaModel;
|
||||
});
|
||||
} catch (err) {
|
||||
console.error("Failed to discover llama.cpp models:", err);
|
||||
throw new Error(`llama.cpp discovery failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Discover models from a vLLM server via OpenAI-compatible /v1/models endpoint.
|
||||
* @param baseUrl - Base URL of the vLLM server (e.g., "http://localhost:8000")
|
||||
* @param apiKey - Optional API key
|
||||
* @returns Array of discovered models
|
||||
*/
|
||||
export async function discoverVLLMModels(baseUrl: string, apiKey?: string): Promise<Model<any>[]> {
|
||||
try {
|
||||
const headers: HeadersInit = {
|
||||
"Content-Type": "application/json",
|
||||
};
|
||||
|
||||
if (apiKey) {
|
||||
headers["Authorization"] = `Bearer ${apiKey}`;
|
||||
}
|
||||
|
||||
const response = await fetch(`${baseUrl}/v1/models`, {
|
||||
method: "GET",
|
||||
headers,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
if (!data.data || !Array.isArray(data.data)) {
|
||||
throw new Error("Invalid response format from vLLM server");
|
||||
}
|
||||
|
||||
return data.data.map((model: any) => {
|
||||
// vLLM provides max_model_len which is the context window
|
||||
const contextWindow = model.max_model_len || 8192;
|
||||
const maxTokens = Math.min(contextWindow, 4096); // Cap max tokens
|
||||
|
||||
const vllmModel: Model<any> = {
|
||||
id: model.id,
|
||||
name: model.id,
|
||||
api: "openai-completions" as any,
|
||||
provider: "", // Will be set by caller
|
||||
baseUrl: `${baseUrl}/v1`,
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: contextWindow,
|
||||
maxTokens: maxTokens,
|
||||
};
|
||||
|
||||
return vllmModel;
|
||||
});
|
||||
} catch (err) {
|
||||
console.error("Failed to discover vLLM models:", err);
|
||||
throw new Error(`vLLM discovery failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Discover models from an LM Studio server using the LM Studio SDK.
|
||||
* @param baseUrl - Base URL of the LM Studio server (e.g., "http://localhost:1234")
|
||||
* @param apiKey - Optional API key (unused for LM Studio SDK)
|
||||
* @returns Array of discovered models
|
||||
*/
|
||||
export async function discoverLMStudioModels(baseUrl: string, apiKey?: string): Promise<Model<any>[]> {
|
||||
try {
|
||||
// Extract host and port from baseUrl
|
||||
const url = new URL(baseUrl);
|
||||
const port = url.port ? parseInt(url.port) : 1234;
|
||||
|
||||
// Create LM Studio client
|
||||
const client = new LMStudioClient({ baseUrl: `ws://${url.hostname}:${port}` });
|
||||
|
||||
// List all downloaded models
|
||||
const models = await client.system.listDownloadedModels();
|
||||
|
||||
// Filter to only LLM models and map to our Model format
|
||||
return models
|
||||
.filter((model) => model.type === "llm")
|
||||
.map((model) => {
|
||||
const contextWindow = model.maxContextLength;
|
||||
// Use 10x context length like Ollama does
|
||||
const maxTokens = contextWindow;
|
||||
|
||||
const lmStudioModel: Model<any> = {
|
||||
id: model.path,
|
||||
name: model.displayName || model.path,
|
||||
api: "openai-completions" as any,
|
||||
provider: "", // Will be set by caller
|
||||
baseUrl: `${baseUrl}/v1`,
|
||||
reasoning: model.trainedForToolUse || false,
|
||||
input: model.vision ? ["text", "image"] : ["text"],
|
||||
cost: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: contextWindow,
|
||||
maxTokens: maxTokens,
|
||||
};
|
||||
|
||||
return lmStudioModel;
|
||||
});
|
||||
} catch (err) {
|
||||
console.error("Failed to discover LM Studio models:", err);
|
||||
throw new Error(`LM Studio discovery failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convenience function to discover models based on provider type.
|
||||
* @param type - Provider type
|
||||
* @param baseUrl - Base URL of the server
|
||||
* @param apiKey - Optional API key
|
||||
* @returns Array of discovered models
|
||||
*/
|
||||
export async function discoverModels(
|
||||
type: "ollama" | "llama.cpp" | "vllm" | "lmstudio",
|
||||
baseUrl: string,
|
||||
apiKey?: string,
|
||||
): Promise<Model<any>[]> {
|
||||
switch (type) {
|
||||
case "ollama":
|
||||
return discoverOllamaModels(baseUrl, apiKey);
|
||||
case "llama.cpp":
|
||||
return discoverLlamaCppModels(baseUrl, apiKey);
|
||||
case "vllm":
|
||||
return discoverVLLMModels(baseUrl, apiKey);
|
||||
case "lmstudio":
|
||||
return discoverLMStudioModels(baseUrl, apiKey);
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue