From 88f4319c3fbeba424bfce34e2b1115ab9e783c13 Mon Sep 17 00:00:00 2001 From: Evandro Fonseca Junior Date: Thu, 2 Apr 2026 20:42:04 -0300 Subject: [PATCH] fix: stop transcript leakage and repetition loops in Generic chat template MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backend (inference-server/compat.rs): - PreparedPrompt now carries extra_stop_sequences injected by template - Generic template injects User:/System:/Assistant:/Human: stop sequences to prevent the model from continuing the transcript after its reply - Increased repeat_penalty 1.15→1.3 and repeat_last_n 256→512 to break repetition loops more aggressively Backend (inference-server/routes.rs): - Merge template extra_stop_sequences into sampler before inference Frontend (chat.ts): - Added 'System:' and 'Human:' to CHAT_TRANSCRIPT_STOP_SEQUENCES - Extended TRANSCRIPT_LEAKAGE_PATTERN to include 'Human:' Frontend (ChatView.tsx): - buildApiMessages now strips leaked transcript from assistant messages before sending history back to the model - Reduced history window to 20 messages to limit context pollution --- BACKEND_DOCUMENTATION.md | 942 +++++++++++++++++++++++++ FRONTEND_API_DOCUMENTATION.md | 962 ++++++++++++++++++++++++++ crates/inference-server/src/compat.rs | 25 +- crates/inference-server/src/routes.rs | 13 +- tauri-app/src/components/ChatView.tsx | 11 +- tauri-app/src/lib/chat.ts | 5 +- 6 files changed, 1951 insertions(+), 7 deletions(-) create mode 100644 BACKEND_DOCUMENTATION.md create mode 100644 FRONTEND_API_DOCUMENTATION.md diff --git a/BACKEND_DOCUMENTATION.md b/BACKEND_DOCUMENTATION.md new file mode 100644 index 0000000..889f76e --- /dev/null +++ b/BACKEND_DOCUMENTATION.md @@ -0,0 +1,942 @@ +# Documentação Completa do Backend - ForgeLM + +## Sumário Executivo + +O ForgeLM é um sistema de inferência de LLMs locais composto por dois componentes principais: + +1. **HTTP API Server** (`inference-server`) - Servidor Axum que expõe endpoints compatíveis com OpenAI +2. **Tauri Desktop Backend** - Backend do aplicativo desktop que gerencia o ciclo de vida do servidor + +--- + +## Arquitetura Geral + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ ForgeLM Backend │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌──────────────────────┐ ┌─────────────────────────────────────────┐ │ +│ │ Tauri App (Frontend)│ │ HTTP API Server (Axum) │ │ +│ │ - React + TypeScript│ │ - Port: 8080 (default) │ │ +│ │ - sql.js + localStorage│ │ - OpenAI-compatible API │ │ +│ └──────────┬───────────┘ └──────────────┬──────────────────────────┘ │ +│ │ │ │ +│ │ Tauri Commands │ │ +│ ▼ │ │ +│ ┌──────────────────────┐ │ │ +│ │ Tauri Backend │◄────────────────────┘ │ +│ │ - Runtime lifecycle │ HTTP requests │ +│ │ - Model management │ │ +│ │ - GPU detection │ │ +│ └──────────┬───────────┘ │ +│ │ │ +│ │ Spawns Process │ +│ ▼ │ +│ ┌──────────────────────┐ ┌─────────────────────────────────────────┐ │ +│ │ inference-server │ │ inference-core │ │ +│ │ (Binary Executable) │──────► - Engine │ │ +│ │ │ │ - GenerationSession │ │ +│ └──────────────────────┘ └──────────────┬──────────────────────────┘ │ +│ │ │ +│ │ FFI │ +│ ▼ │ +│ ┌─────────────────┐ │ +│ │ llama-sys │ │ +│ │ (llama.dll) │ │ +│ └─────────────────┘ │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## 1. Crates e Estrutura do Workspace + +### Estrutura do Workspace + +``` +Cargo.toml (workspace root) +├── crates/ +│ ├── llama-sys/ # FFI bindings para llama.cpp +│ ├── runtime-deps/ # Gerenciamento de dependências do runtime +│ ├── inference-core/ # Lógica de inferência (Engine, Session, Sampler) +│ ├── inference-server/ # Servidor HTTP Axum +│ ├── api-contracts/ # Schemas de request/response +│ ├── metrics/ # Contadores de performance e TPS +│ └── localllm-cli/ # CLI para gerenciamento +│ +└── tauri-app/ + └── src-tauri/ + └── src/ + ├── lib.rs # Entry point Tauri + ├── runtime/ # Gerenciamento do ciclo de vida + ├── model_library/ # Biblioteca de modelos HuggingFace + ├── model_files.rs # Operações com arquivos .gguf + ├── model_alias.rs # Resolução de aliases + └── gpu.rs # Detecção de GPU +``` + +### Dependências do Workspace + +| Categoria | Crates | +|-----------|--------| +| Async Runtime | `tokio`, `axum`, `hyper`, `tower`, `tower-http` | +| Serialization | `serde`, `serde_json` | +| Error Handling | `anyhow`, `thiserror` | +| Logging | `tracing`, `tracing-subscriber` | +| FFI | `libloading`, `once_cell` | +| Utilities | `uuid`, `chrono`, `futures`, `tokio-stream` | + +--- + +## 2. HTTP API Server (inference-server) + +### Entry Point - `main.rs` + +O servidor é iniciado em `crates/inference-server/src/main.rs`. + +#### Configuração via Environment Variables + +| Variável | Descrição | Padrão | +|----------|-----------|--------| +| `LOCALLLM_MODEL` | Caminho para arquivo GGUF | (obrigatório) | +| `LOCALLLM_PROFILE` | Perfil de performance (`turbo_4b`, `balanced_9b`, `cpu_fallback`) | `turbo_4b` | +| `LOCALLLM_CTX` | Tamanho do contexto (tokens) | depende do perfil | +| `LOCALLLM_BATCH` | Tamanho do batch | 4096 (max) | +| `LOCALLLM_GPU_LAYERS` | Camadas GPU (-1 = auto) | depende do perfil | +| `LOCALLLM_THREADS` | Threads CPU | depende do perfil | +| `LOCALLLM_ADDR` | Endereço de bind | `0.0.0.0:8080` | +| `LOCALLLM_API_KEY` | Chave API para autenticação | (opcional) | +| `LOCALLLM_LOG_FORMAT` | Formato de log (`json`, `pretty`) | `pretty` | +| `LOCALLLM_MODEL_ALIASES` | Aliases para modelos (CSV) | (opcional) | +| `LOCALLLM_ENABLE_TOOL_CALLS` | Habilitar tool calls | `true` | + +### Rotas HTTP - `routes.rs` + +O roteador é construído em `crates/inference-server/src/routes.rs`. + +```rust +pub fn build_router(state: AppState) -> Router { + Router::new() + .route("/health", get(health_handler)) + .route("/v1/models", get(models_handler)) + .route("/v1/chat/completions", post(chat_completions_handler)) + .layer(ServiceBuilder::new().layer(trace_layer).layer(cors)) + .with_state(state) +} +``` + +#### Endpoints + +| Método | Endpoint | Descrição | +|--------|----------|-----------| +| `GET` | `/health` | Health check do servidor | +| `GET` | `/v1/models` | Lista modelos disponíveis | +| `POST` | `/v1/chat/completions` | Chat completions (OpenAI-compatible) | + +#### AppState + +```rust +#[derive(Clone)] +pub struct AppState { + pub engine: Arc>, // Motor de inferência + pub counters: Arc, // Contadores de performance + pub model_name: Arc, // Nome do modelo atual + pub server_config: Arc, // Configurações do servidor + pub start_time: std::time::Instant, // Timestamp de início +} +``` + +### Compatibilidade OpenAI - `compat.rs` + +Arquivo: `crates/inference-server/src/compat.rs` + +#### ServerConfig + +Gerencia configurações de autenticação, CORS, e catálogo de modelos: + +```rust +#[derive(Debug, Clone)] +pub struct ServerConfig { + pub api_key: Option, + pub auth_header: String, + pub auth_scheme: Option, + pub cors_origins: Vec, + pub default_system_prompt: String, + pub enable_tool_calls: bool, + pub model_catalog: ModelCatalog, +} +``` + +#### ModelCatalog + +Resolve aliases de modelos (case-insensitive): + +```rust +pub struct ModelCatalog { + models: Vec, + aliases: HashMap, +} + +pub struct ModelDescriptor { + pub id: String, + pub root: String, + pub parent: Option, +} +``` + +**Aliases padrão:** `local-model`, `default` + +#### Chat Templates Suportados + +O sistema detecta automaticamente o template baseado no nome do modelo: + +| Modelo | Template | Formatos | +|--------|----------|----------| +| Qwen | `<\|im_start\|>`, `<\|im_end\|>` | `system`, `user`, `assistant`, `tool` | +| Gemma | ``, `` | `model`, `user` | +| Llama | `[INST]`, `[/INST]` | com `<>` para system | +| Kimi/Moonshot | `<\|im_system\|>`, `<\|im_user\|>`, `<\|im_assistant\|>` | - | +| Genérico | `Role: content` | Padrão | + +#### Autorização + +Suporta múltiplos headers de autenticação: +- Header configurado (`LOCALLLM_AUTH_HEADER`) +- `Authorization: Bearer ` +- `X-API-Key: ` + +#### Validação de Requests + +Validações implementadas: +- Modelo existe no catálogo +- Lista de mensagens não está vazia +- `n=1` (apenas uma completion por request) +- Tool calls habilitado se tools presentes + +--- + +## 3. inference-core + +### Engine - `engine.rs` + +Local: `crates/inference-core/src/engine.rs` + +O `InferenceEngine` é o núcleo de inferência que gerencia: +- Carregamento do modelo GGUF via llama-sys FFI +- Inicialização do contexto llama.cpp +- Criação de sessões de geração + +#### EngineConfig + +```rust +pub struct EngineConfig { + pub model_path: PathBuf, // Caminho para arquivo GGUF + pub profile: RuntimeProfile, // Perfil de performance + pub n_gpu_layers: i32, // Camadas GPU (-1 = todas) + pub n_ctx: u32, // Contexto (tokens) + pub n_batch: u32, // Batch size (prefill) + pub n_ubatch: u32, // Micro-batch (geração) + pub n_threads: i32, // Threads CPU + pub n_threads_batch: i32, // Threads para batch + pub flash_attn: bool, // Flash Attention + pub use_mmap: bool, // Usar memory mapping + pub seed: u32, // Seed RNG +} +``` + +#### Runtime Profiles + +| Perfil | n_ctx | n_gpu_layers | Descrição | +|--------|-------|--------------|-----------| +| `Turbo4B` | 8192 | 999 | GPU total, otimizado para modelos até 4B | +| `Balanced9B` | 16384 | 999 | Equilibrado para modelos até 9B | +| `CpuFallback` | 4096 | 0 | CPU-only fallback | + +#### Ciclo de Vida do Engine + +```rust +// 1. Carregar engine +let mut engine = InferenceEngine::load(config)?; + +// 2. Warmup (aloca buffers CUDA) +engine.warmup()?; + +// 3. Criar sessão de geração +let session = engine.new_session(prompt, sampler_cfg)?; + +// 4. Gerar tokens +while let Some(token) = session.next_token() { + print!("{}", token); +} +``` + +**Nota sobre segurança:** O engine é `Send` mas NÃO é `Sync`. Acesso deve ser coordenado (o servidor usa `tokio::sync::Mutex`). + +### Session - `session.rs` + +```rust +pub struct GenerationSession { + // ... +} + +impl GenerationSession { + pub fn next_token(&mut self) -> Option; + pub fn prompt_tokens(&self) -> u32; +} +``` + +### Sampler - `sampler.rs` + +Configurações de sampling: + +```rust +pub struct SamplerConfig { + pub temperature: f32, // 0.0 - 2.0 + pub top_p: f32, // 0.0 - 1.0 + pub max_tokens: u32, + pub stop_sequences: Vec, + pub repeat_penalty: f32, // 1.15 (normal), 1.0 (tools) + pub repeat_last_n: usize, // 256 tokens + pub seed: u32, +} +``` + +--- + +## 4. API Contracts + +Local: `crates/api-contracts/src/` + +### Chat Completion Request (`chat.rs`) + +```rust +pub struct ChatCompletionRequest { + pub model: String, + pub messages: Vec, + pub max_tokens: u32, // default: 1024 + pub temperature: f32, // default: 0.15 + pub top_p: f32, // default: 0.95 + pub stream: bool, // default: false + pub stop: Vec, + pub n: u32, // default: 1 (apenas 1 suportado) + pub seed: Option, + pub user: Option, + pub response_format: Option, + pub tools: Vec, + pub tool_choice: Option, + pub stream_options: Option, +} +``` + +### ChatMessage + +```rust +pub struct ChatMessage { + pub role: Role, // System, Developer, User, Assistant, Tool + pub content: Option, + pub name: Option, + pub tool_calls: Option>, + pub tool_call_id: Option, +} +``` + +### ResponseFormat + +```rust +pub struct ResponseFormat { + pub format_type: String, // "json_object", "json_schema", "text" + pub json_schema: Option, +} +``` + +### ToolCalling + +```rust +pub struct ToolDefinition { + pub tool_type: ToolType, + pub function: FunctionDefinition, +} + +pub struct FunctionDefinition { + pub name: String, + pub description: Option, + pub parameters: Value, // JSON Schema +} + +pub struct ToolCall { + pub id: String, + pub tool_type: ToolType, + pub function: ToolFunctionCall, +} + +pub struct ToolFunctionCall { + pub name: String, + pub arguments: String, // JSON string +} +``` + +--- + +## 5. Tauri Desktop Backend + +### Entry Point - `lib.rs` + +Local: `tauri-app/src-tauri/src/lib.rs` + +O backend Tauri gerencia: +- Ciclo de vida do runtime (start/stop) +- Gerenciamento de modelos locais +- Downloads do HuggingFace +- Detecção de GPU +- Sistema ForgeAgent (multi-agente) + +### Commands Tauri + +```rust +.invoke_handler(tauri::generate_handler![ + query_gpu, // Detectar GPU NVIDIA + scan_models, // Escanear arquivos .gguf + pick_models_dir, // Selecionar diretório + pick_model_files, // Selecionar arquivos + import_model, // Importar modelo + search_hugging_face_models_command, // Buscar no HF + start_hugging_face_download_command, // Download do HF + cancel_hugging_face_download_command,// Cancelar download + swap_model_command, // Trocar modelo + delete_model_file, // Deletar arquivo + open_model_folder_command, // Abrir pasta + list_model_library_entries_command, // Listar biblioteca + get_model_library_state_command, // Estado da biblioteca + bootstrap_runtime_command, // Bootstrap runtime + get_runtime_snapshot_command, // Snapshot do runtime + save_runtime_config_command_handler, // Salvar config + start_runtime_command, // Iniciar runtime + stop_runtime_command, // Parar runtime + resolve_startup_model_command, // Resolver modelo inicial + get_app_data_dir, // Diretório de dados + find_server_binary, // Localizar binário + start_forgeagent_session_command, // Iniciar ForgeAgent +]) +``` + +### Runtime Types - `runtime/types.rs` + +#### ConnectionMode + +```rust +pub enum ConnectionMode { + Local, // Servidor local spawnado + Remote, // Conectar a servidor externo +} +``` + +#### RuntimeStatus + +```rust +pub enum RuntimeStatus { + Stopped, + Starting, + Loading, + Ready, + Degraded, + Stopping, + Error, +} +``` + +#### RuntimeConfig + +```rust +pub struct RuntimeConfig { + pub connection_mode: ConnectionMode, + pub start_policy: StartPolicy, // Manual, OnAppLaunch + pub server_url: String, // Para Remote mode + pub api_key: String, + pub model_path: Option, + pub models_dir: Option, + pub addr: String, // "127.0.0.1:8080" + pub gpu_layers: i32, + pub ctx_size: u32, // default: 4096 + pub extra_model_paths: Vec, + pub selected_model_id: Option, + pub remote_presets: Vec, + pub hugging_face_token: String, + pub preferred_startup_model_alias: Option, +} +``` + +#### RuntimeSnapshot + +```rust +pub struct RuntimeSnapshot { + pub status: RuntimeStatus, + pub ready: bool, + pub process_running: bool, + pub resolved_url: String, + pub binary_path: Option, + pub last_error: Option, + pub active_command: RuntimeLifecycleCommand, + pub last_command_result: RuntimeLifecycleResult, + pub config: RuntimeConfig, +} +``` + +### Model Library - `model_library/` + +#### Tipos + +```rust +pub struct ModelLibraryEntry { + pub id: String, + pub name: String, + pub repo_id: String, + pub description: String, + pub tags: Vec, + pub size_gb: f64, + pub default_file: String, + pub installed: bool, + pub installed_path: Option, +} + +pub struct HuggingFaceModelResult { + pub id: String, + pub model_id: String, + pub downloads: i64, + pub likes: i32, + pub tags: Vec, + pub gguf_files: Vec, +} +``` + +### Model Files - `model_files.rs` + +```rust +#[derive(Serialize)] +pub struct ModelFileInfo { + pub path: String, + pub file_name: String, + pub size_bytes: u64, + pub modified: Option, +} +``` + +--- + +## 6. Fluxo de Dados + +### 1. Requisição Chat Completion + +``` +┌──────────┐ ┌─────────────┐ ┌──────────────┐ ┌─────────────┐ +│ Client │───►│ Axum Router │───►│ compat.rs │───►│ routes.rs │ +└──────────┘ └─────────────┘ └──────────────┘ └──────┬──────┘ + │ + ▼ +┌──────────┐ ┌─────────────┐ ┌──────────────┐ ┌─────────────┐ +│ Client │◄───│ JSON │◄───│ Generation │◄───│ Engine │ +│ │ │ Response │ │ Session │ │ (Mutex) │ +└──────────┘ └─────────────┘ └──────────────┘ └──────┬──────┘ + │ + ▼ + ┌─────────────────┐ + │ llama-sys │ + │ (llama.dll) │ + └─────────────────┘ +``` + +### 2. Streaming vs Blocking + +**Streaming Mode:** +1. Cria stream SSE +2. Gera tokens em `spawn_blocking` +3. Envia chunks via channel +4. Client recebe tokens em tempo real + +**Blocking Mode:** +1. Gera completion completa +2. Retorna JSON único + +### 3. Ciclo de Vida do Modelo + +``` +┌─────────────┐ +│ Stopped │ +└──────┬──────┘ + │ start_runtime_command() + ▼ +┌─────────────┐ +│ Starting │ +└──────┬──────┘ + │ Processo spawnado + ▼ +┌─────────────┐ +│ Loading │ +└──────┬──────┘ + │ Modelo carregado + ▼ +┌─────────────┐ +│ Ready │◄──────────┐ +└──────┬──────┘ │ + │ stop_runtime() │ swap_model_command() + ▼ │ +┌─────────────┐ │ +│ Stopping │ │ +└─────────────┘ │ + │ +├─────────────────────────┤ +│ swap │ +│ ┌─────┐ ┌─────┐ │ +│ │Stop │──►│Start│ │ +│ └─────┘ └─────┘ │ +└─────────────────────────┘ +``` + +--- + +## 7. ForgeAgent - Sistema Multi-Agente + +### Especialistas + +| ID | Nome | Emoji | Função | +|----|------|-------|--------| +| `analyst` | Analista | 🔍 | Análise técnica e decomposição | +| `architect` | Arquiteto | 🏗️ | Arquitetura de software e padrões | +| `developer` | Desenvolvedor | 💻 | Implementação pragmática | +| `critic` | Crítico | ⚖️ | Revisão e identificação de riscos | + +### Fluxo de Eventos + +``` +start_forgeagent_session_command() + │ + ▼ +┌─────────────────────────┐ +│ forgeagent:start │──► Envia plano estruturado +└─────────────────────────┘ + │ + ▼ +┌─────────────────────────┐ +│ forgeagent:step-started │──► Início de cada passo +└─────────────────────────┘ + │ + ▼ +┌─────────────────────────┐ +│ forgeagent:step-completed│──► Passo concluído +└─────────────────────────┘ + │ + ▼ +┌───────────────────────────────┐ +│ forgeagent:specialist-thought │──► Pensamento do especialista +└───────────────────────────────┘ + │ + ▼ +┌──────────────────────────────────┐ +│ forgeagent:specialist-response │──► Resposta do especialista +└──────────────────────────────────┘ + │ + ▼ +┌─────────────────────────┐ +│ forgeagent:plan-completed│──► Resumo final +└─────────────────────────┘ +``` + +--- + +## 8. API HTTP - Exemplos + +### Health Check + +```bash +curl http://localhost:8080/health +``` + +**Response:** +```json +{ + "status": "ok", + "model_loaded": true, + "model_id": "qwen2.5-7b-instruct", + "uptime_secs": 3600 +} +``` + +### Listar Modelos + +```bash +curl http://localhost:8080/v1/models \ + -H "Authorization: Bearer sua-chave" +``` + +**Response:** +```json +{ + "object": "list", + "data": [ + { + "id": "local-model", + "object": "model", + "created": 1234567890, + "owned_by": "local", + "root": "qwen2.5-7b-instruct", + "parent": "qwen2.5-7b-instruct", + "permission": [] + } + ] +} +``` + +### Chat Completion (Non-streaming) + +```bash +curl http://localhost:8080/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sua-chave" \ + -d '{ + "model": "local-model", + "messages": [ + {"role": "user", "content": "Olá!"} + ], + "max_tokens": 1024, + "temperature": 0.7 + }' +``` + +**Response:** +```json +{ + "id": "uuid-123", + "object": "chat.completion", + "created": 1234567890, + "model": "qwen2.5-7b-instruct", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Olá! Como posso ajudar?" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 20, + "total_tokens": 30, + "tokens_per_second": 45.5 + } +} +``` + +### Chat Completion (Streaming) + +```bash +curl http://localhost:8080/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sua-chave" \ + -d '{ + "model": "local-model", + "messages": [ + {"role": "user", "content": "Conte uma piada"} + ], + "stream": true + }' +``` + +**Response (SSE):** +``` +data: {"id":"...","object":"chat.completion.chunk","choices":[{"delta":{"role":"assistant"}}]} + +data: {"id":"...","object":"chat.completion.chunk","choices":[{"delta":{"content":"Por"}}]} + +data: {"id":"...","object":"chat.completion.chunk","choices":[{"delta":{"content":" que"}}]} +... +data: {"id":"...","object":"chat.completion.chunk","choices":[{"finish_reason":"stop"}]} + +data: [DONE] +``` + +### Tool Calling + +```bash +curl http://localhost:8080/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sua-chave" \ + -d '{ + "model": "local-model", + "messages": [ + {"role": "user", "content": "Liste os arquivos"} + ], + "tools": [ + { + "type": "function", + "function": { + "name": "list_files", + "description": "Lista arquivos do projeto", + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"} + } + } + } + } + ], + "tool_choice": "auto" + }' +``` + +--- + +## 9. Configuração e Deploy + +### Compilação + +```bash +# Servidor +$env:LOCALLLM_LOG_FORMAT="json" +cargo build --release -p inference-server + +# CLI +cargo build --release -p localllm-cli + +# Tauri App +cd tauri-app +cargo tauri build +``` + +### Execução + +```bash +# Com variáveis de ambiente +$env:LOCALLLM_MODEL="D:\models\model.gguf" +$env:LOCALLLM_PROFILE="balanced_9b" +$env:LOCALLLM_API_KEY="minha-chave-secreta" +$env:LOCALLLM_ADDR="0.0.0.0:8080" + +.\target\release\inference-server.exe +``` + +### Docker (se aplicável) + +```bash +docker run -p 8080:8080 \ + -v /path/to/models:/models \ + -e LOCALLLM_MODEL=/models/model.gguf \ + -e LOCALLLM_API_KEY=secret \ + forgelm/inference-server:latest +``` + +--- + +## 10. Estrutura de Arquivos + +### Diretórios Importantes + +``` +%LOCALAPPDATA%/ForgeLM/ # Windows +~/.local/share/ForgeLM/ # Linux +~/Library/Application Support/ForgeLM/ # macOS +├── models/ # Modelos baixados +├── runtime/ # Binários do runtime +└── config.json # Configuração do runtime +``` + +### Frontend (Tauri App) + +``` +tauri-app/ +├── src/ +│ ├── lib/ +│ │ ├── db.ts # Persistência local (sql.js) +│ │ └── types.ts # Tipos TypeScript +│ ├── main.tsx # Entry React +│ └── App.tsx # App shell +└── src-tauri/ + ├── src/ + │ ├── lib.rs # Entry Tauri + │ ├── runtime/ # Gerenciamento runtime + │ ├── model_library/ # HuggingFace integration + │ ├── model_files.rs # Arquivos locais + │ ├── model_alias.rs # Resolução aliases + │ └── gpu.rs # Detecção GPU + └── taury.conf.json # Config Tauri +``` + +--- + +## 11. Glossário + +| Termo | Descrição | +|-------|-----------| +| **GGUF** | Format binary for LLMs (GPT-Generated Unified Format) | +| **KV Cache** | Key-Value cache for attention mechanism | +| **Prefill** | Initial prompt processing phase | +| **Decode** | Token generation phase | +| **Context Window** | Maximum tokens the model can process | +| **Batch** | Number of tokens processed together | +| **GPU Layers** | Number of model layers offloaded to GPU | +| **Flash Attention** | Optimized attention implementation | +| **Tool Calling** | Function calling capability for LLMs | +| **SSE** | Server-Sent Events for streaming | + +--- + +## 12. Referências Rápidas + +### Arquivos Principais + +| Função | Arquivo | +|--------|---------| +| Entry Server | `crates/inference-server/src/main.rs` | +| Rotas HTTP | `crates/inference-server/src/routes.rs` | +| Compat OpenAI | `crates/inference-server/src/compat.rs` | +| Engine | `crates/inference-core/src/engine.rs` | +| Session | `crates/inference-core/src/session.rs` | +| Sampler | `crates/inference-core/src/sampler.rs` | +| Schemas | `crates/api-contracts/src/chat.rs` | +| Entry Tauri | `tauri-app/src-tauri/src/lib.rs` | +| Runtime Types | `tauri-app/src-tauri/src/runtime/types.rs` | + +### Env Vars Completas + +```bash +# Modelo e Performance +LOCALLLM_MODEL="/path/to/model.gguf" +LOCALLLM_PROFILE="turbo_4b" # turbo_4b, balanced_9b, cpu_fallback +LOCALLLM_CTX=8192 +LOCALLLM_BATCH=4096 +LOCALLLM_GPU_LAYERS=-1 +LOCALLLM_THREADS=4 +LOCALLLM_THREADS_BATCH=4 + +# Servidor +LOCALLLM_ADDR="0.0.0.0:8080" +LOCALLLM_API_KEY="secret" +LOCALLLM_AUTH_HEADER="authorization" +LOCALLLM_AUTH_SCHEME="Bearer" +LOCALLLM_CORS_ORIGINS="*,https://example.com" + +# Features +LOCALLLM_ENABLE_TOOL_CALLS="true" +LOCALLLM_MODEL_ALIASES="alias1,alias2" +LOCALLLM_SYSTEM_PROMPT="Você é um assistente..." + +# Logging +LOCALLLM_LOG_FORMAT="json" # ou pretty +RUST_LOG="info" +``` + +--- + +## Conclusão + +Este documento cobre a arquitetura completa do backend ForgeLM, incluindo: + +1. **HTTP Server** - API compatível com OpenAI usando Axum +2. **Inference Core** - Motor de inferência via llama.cpp +3. **Tauri Backend** - Gerenciamento desktop do ciclo de vida +4. **API Contracts** - Schemas de request/response +5. **Fluxos de Dados** - Como as requisições fluem pelo sistema + +Para mais detalhes, consulte os arquivos de código fonte mencionados nas referências. diff --git a/FRONTEND_API_DOCUMENTATION.md b/FRONTEND_API_DOCUMENTATION.md new file mode 100644 index 0000000..0ef6331 --- /dev/null +++ b/FRONTEND_API_DOCUMENTATION.md @@ -0,0 +1,962 @@ +# Documentação: Frontend Consumindo o Backend - ForgeLM + +## Sumário + +Esta documentação detalha **exatamente como** o frontend React consome os dados do backend, quais rotas ele bate, como envia e recebe dados, e a arquitetura de comunicação entre as camadas. + +--- + +## Arquitetura de Comunicação + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ Frontend React + TypeScript │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────────────────┐ │ +│ │ App.tsx │ │ ChatView.tsx │ │ useRuntimeManager.ts │ │ +│ │ (Main App) │ │ (Chat UI) │ │ (Runtime State) │ │ +│ └────────┬────────┘ └────────┬────────┘ └─────────────┬───────────────┘ │ +│ │ │ │ │ +│ │ │ fetch() │ invoke() │ +│ │ │ │ │ +│ ▼ ▼ ▼ │ +│ ┌──────────────────────────────────────────────────────────────────────┐ │ +│ │ Camada de API │ │ +│ │ ┌─────────────────────┐ ┌─────────────────────────────────────────┐ │ │ +│ │ │ lib/db.ts │ │ features/runtime/api.ts │ │ │ +│ │ │ (sql.js + localStorage)│ │ (Tauri Commands) │ │ │ +│ │ └─────────────────────┘ └─────────────────────────────────────────┘ │ │ +│ └──────────────────────────────────────────────────────────────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────────────────┘ + │ + │ HTTP / Tauri Bridge + ▼ +┌─────────────────────────────────────────────────────────────────────────────┐ +│ Backend │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ ┌─────────────────────────────┐ ┌─────────────────────────────────────┐ │ +│ │ Tauri Backend (Rust) │ │ HTTP Server (Axum) │ │ +│ │ - Commands │ │ - Port: 8080 │ │ +│ │ - Events │ │ - /v1/chat/completions │ │ +│ │ - Process spawning │ │ - /v1/models │ │ +│ └─────────────────────────────┘ └─────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## 1. Camadas de Comunicação + +### 1.1 HTTP API (Chat Completions) + +**Local:** `ChatView.tsx` → HTTP Server Axum + +```typescript +// ChatView.tsx - Linha 251-263 +const res = await fetch(`${server}/v1/chat/completions`, { + method: "POST", + headers, + signal: abortRef.current.signal, + body: JSON.stringify({ + model, + messages: apiMessages, // Array de {role, content} + stream: useStream, // true/false + stream_options: useStream ? { include_usage: true } : undefined, + max_tokens: 2048, + stop: CHAT_TRANSCRIPT_STOP_SEQUENCES, // ["\nSystem:", "\nUser:", ...] + }), +}); +``` + +**Headers enviados:** +```typescript +const headers: Record = { + "Content-Type": "application/json" +}; +if (apiKey) headers["Authorization"] = `Bearer ${apiKey}`; +``` + +**Body da requisição:** +```json +{ + "model": "qwen2.5-7b-instruct", + "messages": [ + { "role": "user", "content": "Olá, como vai?" } + ], + "stream": true, + "stream_options": { "include_usage": true }, + "max_tokens": 2048, + "stop": ["\nSystem:", "\nUser:", "\nAssistant:", "\n\nSystem:", "\n\nUser:", "\n\nAssistant:"] +} +``` + +### 1.2 Tauri Commands (Gerenciamento do Runtime) + +**Local:** `features/runtime/api.ts` → Tauri Backend (Rust) + +O frontend usa `invoke` do `@tauri-apps/api/core` para chamar comandos Rust: + +```typescript +import { invoke } from "@tauri-apps/api/core"; + +// Exemplo: Iniciar runtime +export async function startRuntime(): Promise { + return invoke("start_runtime_command"); +} +``` + +--- + +## 2. Rotas HTTP Consumidas + +### 2.1 POST /v1/chat/completions + +**Arquivo:** `ChatView.tsx` (linha 251) + +**Fluxo completo:** + +``` +┌──────────────┐ ┌──────────────┐ ┌──────────────┐ +│ Usuário │────►│ ChatView │────►│ fetch() │ +│ digita │ │ handleSend │ │ POST /v1/ │ +└──────────────┘ └──────────────┘ └──────┬───────┘ + │ + ▼ +┌──────────────┐ ┌──────────────┐ ┌──────────────┐ +│ DB (local) │◄────│ parse() │◄────│ Server │ +│ sql.js │ │ response │ │ Axum │ +└──────────────┘ └──────────────┘ └──────────────┘ +``` + +**Streaming (SSE):** + +```typescript +// ChatView.tsx - Linha 274-307 +if (useStream) { + const reader = res.body!.getReader(); + const decoder = new TextDecoder('utf-8'); + let buf = ""; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + buf += decoder.decode(value, { stream: true }); + const lines = buf.split("\n"); + buf = lines.pop()!; + + for (const line of lines) { + if (!line.startsWith("data: ")) continue; + const data = line.slice(6).trim(); + if (data === "[DONE]") continue; + + const chunk = JSON.parse(data); + const delta = chunk.choices?.[0]?.delta?.content; + if (delta) { + fullContent += delta; + setStreamContent(parsed.content); + setStreamReasoning(parsed.reasoning); + } + if (chunk.usage) usage = chunk.usage; + } + } +} +``` + +**Resposta em streaming:** +``` +data: {"id":"...","object":"chat.completion.chunk","choices":[{"delta":{"role":"assistant"}}]} + +data: {"id":"...","object":"chat.completion.chunk","choices":[{"delta":{"content":"Olá"}}]} + +data: {"id":"...","object":"chat.completion.chunk","choices":[{"delta":{"content":"!"}}]} +... +data: {"id":"...","object":"chat.completion.chunk","choices":[{"finish_reason":"stop"}],"usage":{"prompt_tokens":10,"completion_tokens":20,"total_tokens":30,"tokens_per_second":45.5}} + +data: [DONE] +``` + +**Modo Blocking:** + +```typescript +// ChatView.tsx - Linha 308-318 +} else { + const data = await res.json(); + fullContent = data.choices?.[0]?.message?.content || ""; + usage = data.usage; +} +``` + +**Resposta blocking:** +```json +{ + "id": "uuid-123", + "object": "chat.completion", + "created": 1234567890, + "model": "qwen2.5-7b-instruct", + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": "Olá! Como posso ajudar?" + }, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 20, + "total_tokens": 30, + "tokens_per_second": 45.5 + } +} +``` + +### 2.2 GET /v1/models + +**Arquivo:** `hooks/useModels.ts` + +```typescript +const res = await fetch(`${serverUrl}/v1/models`, { + headers: apiKey ? { "Authorization": `Bearer ${apiKey}` } : {}, +}); +const data = await res.json(); +``` + +**Resposta:** +```json +{ + "object": "list", + "data": [ + { + "id": "local-model", + "object": "model", + "created": 1234567890, + "owned_by": "local" + } + ] +} +``` + +### 2.3 GET /health + +**Arquivo:** `hooks/useModels.ts` (status check) + +```typescript +const res = await fetch(`${serverUrl}/health`); +``` + +**Resposta:** +```json +{ + "status": "ok", + "model_loaded": true, + "model_id": "qwen2.5-7b-instruct", + "uptime_secs": 3600 +} +``` + +--- + +## 3. Comandos Tauri (Invoke) + +### 3.1 Runtime Management API + +**Arquivo:** `features/runtime/api.ts` + +| Função | Comando Tauri | Descrição | +|--------|---------------|-----------| +| `bootstrapRuntime()` | `bootstrap_runtime_command` | Inicializa o runtime | +| `getRuntimeSnapshot()` | `get_runtime_snapshot_command` | Obtém estado atual | +| `saveRuntimeConfig(config)` | `save_runtime_config_command_handler` | Salva configurações | +| `startRuntime()` | `start_runtime_command` | Inicia o servidor local | +| `stopRuntime()` | `stop_runtime_command` | Para o servidor | + +**Exemplo - Iniciar Runtime:** +```typescript +// api.ts +export async function startRuntime(): Promise { + try { + return await invoke("start_runtime_command"); + } catch { + // Fallback para modo web (sem Tauri) + const snapshot = readFallbackSnapshot(); + return writeFallbackSnapshot({ + ...snapshot, + status: "error", + last_error: { + code: "tauri_unavailable", + message: "Local runtime control requires the Tauri desktop app." + }, + }); + } +} +``` + +### 3.2 Model Management API + +| Função | Comando Tauri | Descrição | +|--------|---------------|-----------| +| `scanRuntimeModels(dir)` | `scan_models` | Escaneia diretório de modelos | +| `pickRuntimeModelsDir()` | `pick_models_dir` | Abre dialog de seleção | +| `pickRuntimeModelFiles()` | `pick_model_files` | Seleciona arquivos | +| `swapModel(path)` | `swap_model_command` | Troca modelo ativo | +| `deleteModelFile(path)` | `delete_model_file` | Deleta arquivo .gguf | +| `openModelFolder(path)` | `open_model_folder_command` | Abre pasta no Explorer | + +**Exemplo - Escaneamento:** +```typescript +export async function scanRuntimeModels(dir: string): Promise { + return invoke("scan_models", { dir }); +} +// Retorna: [{ name: "model.gguf", path: "/path/to/model.gguf", size: 4200000000 }] +``` + +### 3.3 HuggingFace Integration API + +| Função | Comando Tauri | Descrição | +|--------|---------------|-----------| +| `searchHuggingFaceModels(query, token)` | `search_hugging_face_models_command` | Busca modelos | +| `startHuggingFaceDownload(repo, file, token)` | `start_hugging_face_download_command` | Inicia download | +| `cancelHuggingFaceDownload(id)` | `cancel_hugging_face_download_command` | Cancela download | + +**Exemplo - Busca HuggingFace:** +```typescript +export async function searchHuggingFaceModels( + query: string, + token = "" +): Promise { + return invoke("search_hugging_face_models_command", { + query, + token: token.trim() || null, + }); +} +``` + +### 3.4 GPU Info API + +```typescript +export async function queryRuntimeGpu(): Promise { + return invoke("query_gpu"); +} +// Retorna: { name: "NVIDIA RTX 4090", vram_mib: 24576, recommended_layers: 999 } +``` + +### 3.5 ForgeAgent Session API + +**Arquivo:** `hooks/useForgeAgentSession.ts` + +```typescript +const response = await invoke( + "start_forgeagent_session_command", + { prompt } +); +// Retorna: { plan_id: "forgeagent-1234567890", plan: AgentPlan } +``` + +**Eventos escutados:** +```typescript +listen("forgeagent:step-started", handler); +listen("forgeagent:step-completed", handler); +listen("forgeagent:artifact-ready", handler); +listen("forgeagent:plan-completed", handler); +listen("forgeagent:specialist-thinking-started", handler); +listen("forgeagent:specialist-thinking-update", handler); +listen("forgeagent:specialist-response-completed", handler); +``` + +--- + +## 4. Persistência Local (sql.js) + +### 4.1 Estrutura do Banco + +**Arquivo:** `lib/db.ts` + +O frontend usa **sql.js** (SQLite compilado para WebAssembly) com persistência em `localStorage`: + +```typescript +// initDB() - Criação das tabelas +db.run(` + CREATE TABLE IF NOT EXISTS chats ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + title TEXT NOT NULL DEFAULT 'New Chat', + created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), + updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')) + ) +`); + +db.run(` + CREATE TABLE IF NOT EXISTS messages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + chat_id INTEGER NOT NULL, + role TEXT NOT NULL, + content TEXT NOT NULL, + reasoning_content TEXT, -- Conteúdo do ... + prompt_tokens INTEGER DEFAULT 0, + completion_tokens INTEGER DEFAULT 0, + total_tokens INTEGER DEFAULT 0, + tokens_per_second REAL, + created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')) + ) +`); + +db.run(` + CREATE TABLE IF NOT EXISTS user_memories ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + key TEXT NOT NULL UNIQUE, + value TEXT NOT NULL, + confidence REAL DEFAULT 1.0, + created_at INTEGER, + updated_at INTEGER + ) +`); +``` + +### 4.2 Persistência + +```typescript +// db.ts - persist() +persist() { + const data = this.inner.export(); // Uint8Array + // Converte para base64 em chunks + let b64 = ""; + const CHUNK = 8192; + for (let i = 0; i < data.length; i += CHUNK) { + b64 += String.fromCharCode(...data.subarray(i, i + CHUNK)); + } + const encoded = btoa(b64); + localStorage.setItem("forgelm_db_v2", encoded); +} +``` + +### 4.3 Queries Principais + +**Carregar mensagens:** +```typescript +// ChatView.tsx - loadMessages() +const rows = db.query( + "SELECT * FROM messages WHERE chat_id = ? ORDER BY created_at ASC", + [targetChatId] +); +``` + +**Inserir mensagem do usuário:** +```typescript +db.run( + "INSERT INTO messages (chat_id, role, content) VALUES (?, 'user', ?)", + [cid, text] +); +``` + +**Inserir resposta do assistente:** +```typescript +db.run( + `INSERT INTO messages ( + chat_id, role, content, reasoning_content, + prompt_tokens, completion_tokens, total_tokens, tokens_per_second + ) VALUES (?, 'assistant', ?, ?, ?, ?, ?, ?)`, + [cid, content, reasoning, usage?.prompt_tokens, usage?.completion_tokens, + usage?.total_tokens, usage?.tokens_per_second] +); +``` + +**Histórico para API:** +```typescript +// ChatView.tsx - buildApiMessages() +const history = db.query<{ role: string; content: string }>( + `SELECT role, content FROM messages + WHERE chat_id = ? AND content != '' + AND content NOT LIKE '⚠️ Error:%' + ORDER BY created_at DESC LIMIT ?`, + [cid, 100] +).reverse(); +``` + +--- + +## 5. Fluxos de Dados Detalhados + +### 5.1 Fluxo: Enviar Mensagem + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ FLUXO: ENVIAR MENSAGEM │ +└─────────────────────────────────────────────────────────────────────────────┘ + +Usuário clica "Enviar" + │ + ▼ +┌───────────────┐ +│ handleSend() │──► 1. Valida input e estado +└───────┬───────┘ + │ + ▼ +┌───────────────┐ +│ getOrCreateChat│──► 2. Cria novo chat se necessário (INSERT INTO chats) +└───────┬───────┘ + │ + ▼ +┌───────────────┐ +│ db.run(INSERT) │──► 3. Salva mensagem do usuário na DB local +└───────┬───────┘ + │ + ▼ +┌───────────────┐ +│ buildApiMessages│──► 4. Busca histórico das últimas 100 mensagens +└───────┬───────┘ + │ + ▼ +┌───────────────────────────┐ +│ fetch(POST /v1/chat/completions) │──► 5. Envia para servidor HTTP +│ │ Headers: Content-Type, Authorization +│ │ Body: {model, messages, stream, ...} +└───────────┬───────────────┘ + │ + ┌────┴────┐ + ▼ ▼ + ┌─────────┐ ┌─────────┐ + │ Stream │ │ Block │ + │ Mode │ │ Mode │ + └────┬────┘ └────┬────┘ + │ │ + ▼ ▼ + ┌───────────────────────────┐ + │ Processa chunks SSE │ + │ - setStreamContent() │ + │ - setStreamReasoning() │ + │ - parseAssistantResponse()│ + └───────────┬───────────────┘ + │ + ▼ + ┌───────────────────────────┐ + │ db.run(INSERT assistant) │──► 6. Salva resposta completa na DB + └───────────┬───────────────┘ + │ + ▼ + ┌───────────────────────────┐ + │ loadMessages() │──► 7. Recarrega mensagens do chat + └───────────────────────────┘ +``` + +### 5.2 Fluxo: Trocar Modelo + +``` +Usuário seleciona novo modelo no dropdown + │ + ▼ +┌───────────────────────────┐ +│ onModelChange() │ +│ onModelSwap() │ +└───────────┬───────────────┘ + │ + ▼ +┌───────────────────────────┐ +│ swapModel(newModelPath) │──► invoke("swap_model_command") +└───────────┬───────────────┘ + │ + ▼ +┌───────────────────────────┐ +│ Tauri Backend (Rust) │ +│ 1. stop_runtime() │ +│ 2. Atualiza model_path │ +│ 3. save_runtime_config() │ +│ 4. start_runtime() │ +└───────────┬───────────────┘ + │ + ▼ +┌───────────────────────────┐ +│ RuntimeSnapshot retornado │ +│ { status: "ready", ... } │ +└───────────┬───────────────┘ + │ + ▼ +┌───────────────────────────┐ +│ setSelectedModel(path) │──► Atualiza estado React +└───────────────────────────┘ +``` + +### 5.3 Fluxo: Download HuggingFace + +``` +Usuário busca e seleciona modelo + │ + ▼ +┌───────────────────────────────┐ +│ searchHuggingFaceModels() │──► invoke("search_hugging_face_models_command") +│ │ Args: { query, token } +└───────────────┬───────────────┘ + │ + ▼ +┌───────────────────────────────┐ +│ Lista de modelos retornada │ +│ [{ repo_id, author, │ +│ verified_files: [...] }] │ +└───────────────┬───────────────┘ + │ + ▼ +Usuário clica "Download" + │ + ▼ +┌───────────────────────────────┐ +│ startHuggingFaceDownload() │──► invoke("start_hugging_face_download_command") +│ │ Args: { repo, fileName, token } +└───────────────┬───────────────┘ + │ + ▼ +┌───────────────────────────────┐ +│ Escuta eventos de progresso │ +│ listen("hugging-face-download- │ +│ progress", handler) │ +└───────────────┬───────────────┘ + │ + ▼ +┌───────────────────────────────┐ +│ Eventos recebidos: │ +│ { phase: "progress", │ +│ progressPercentage: 45, │ +│ downloadedMB: 2048, │ +│ totalMB: 4096 } │ +└───────────────────────────────┘ +``` + +### 5.4 Fluxo: ForgeAgent + +``` +Usuário ativa modo Agentic e envia mensagem + │ + ▼ +┌───────────────────────────────┐ +│ start_forgeagent_session │──► invoke("start_forgeagent_session_command") +│ _command({ prompt }) │ +└───────────────┬───────────────┘ + │ + ▼ +┌───────────────────────────────┐ +│ Resposta inicial: │ +│ { plan_id, plan: { │ +│ title, overview, steps[] }} │ +└───────────────┬───────────────┘ + │ + ▼ +┌───────────────────────────────┐ +│ Escuta eventos Tauri: │ +├───────────────────────────────┤ +│ forgeagent:step-started │──► Atualiza UI com passo atual +│ forgeagent:step-completed │──► Marca passo como concluído +│ forgeagent:specialist- │──► Mostra especialista +│ thinking-started │ +│ forgeagent:specialist- │──► Atualiza pensamento +│ thinking-update │ +│ forgeagent:specialist- │──► Mostra resposta do especialista +│ response-completed │ +│ forgeagent:plan-completed │──► Mostra resumo final +└───────────────────────────────┘ +``` + +--- + +## 6. Hooks Principais + +### 6.1 useRuntimeManager + +**Arquivo:** `features/runtime/useRuntimeManager.ts` + +Gerencia todo o estado do runtime local: + +```typescript +export function useRuntimeManager() { + // Estados + const [draftState, setDraftState] = useState(...); + const [loading, setLoading] = useState(true); + const [busy, setBusy] = useState(false); + + // Métodos expostos + return { + snapshot: draftState.liveSnapshot, // Estado atual do runtime + connection: { mode, serverUrl, apiKey }, // Dados de conexão + startLocal, // Inicia servidor + stopLocal, // Para servidor + saveConfig, // Salva configurações + scanModels, // Escaneia modelos + // ... e mais + }; +} +``` + +**Polling:** Atualiza a cada 3 segundos +```typescript +const id = window.setInterval(() => { + refresh().catch(() => undefined); +}, 3000); +``` + +### 6.2 useModels + +**Arquivo:** `hooks/useModels.ts` + +Gerencia a lista de modelos disponíveis: + +```typescript +export function useModels(serverUrl: string, apiKey: string, enabled: boolean) { + const [models, setModels] = useState([]); + const [selectedModel, setSelectedModel] = useState(""); + const [status, setStatus] = useState("loading"); + + const refresh = useCallback(async () => { + // 1. Health check + const healthRes = await fetch(`${serverUrl}/health`); + + // 2. Lista modelos + const modelsRes = await fetch(`${serverUrl}/v1/models`, { headers }); + const data = await modelsRes.json(); + setModels(data.data); + }, [serverUrl, apiKey]); + + return { models, selectedModel, setSelectedModel, status, refresh }; +} +``` + +### 6.3 useForgeAgentSession + +**Arquivo:** `hooks/useForgeAgentSession.ts` + +```typescript +export function useForgeAgentSession() { + const [state, setState] = useState(null); + + const start = useCallback(async (prompt: string) => { + const response = await invoke( + "start_forgeagent_session_command", + { prompt } + ); + // Inicializa estado com plano recebido + const initial = createForgeAgentState(response.plan_id, response.plan); + updateState(initial); + }, []); + + // Escuta eventos em tempo real + useEffect(() => { + const unlisten = await Promise.all([ + listen("forgeagent:step-started", handler), + listen("forgeagent:step-completed", handler), + // ... mais eventos + ]); + }, []); + + return { state, start, reset }; +} +``` + +### 6.4 useDB + +**Arquivo:** `hooks/useDB.ts` + +```typescript +export function useDB() { + const [db, setDb] = useState(null); + const [ready, setReady] = useState(false); + + useEffect(() => { + initDB().then((database) => { + setDb(database); + setReady(true); + }); + }, []); + + return { db, ready }; +} +``` + +--- + +## 7. Tipos de Dados (TypeScript) + +### 7.1 Mensagens + +```typescript +// lib/types.ts +interface Message { + id: number; + chat_id: number; + role: "user" | "assistant" | "system"; + content: string; + reasoning_content?: string | null; // ... + display_mode?: "default" | "agentic_final"; + forgeagent_state?: AgentPlanState; + prompt_tokens?: number; + completion_tokens?: number; + total_tokens?: number; + tokens_per_second?: number; + created_at: number; +} +``` + +### 7.2 Runtime + +```typescript +// features/runtime/types.ts +interface RuntimeSnapshot { + status: "stopped" | "starting" | "loading" | "ready" | "degraded" | "stopping" | "error"; + ready: boolean; + process_running: boolean; + resolved_url: string; + binary_path: string | null; + last_error: { code: string; message: string } | null; + active_command: "idle" | "save" | "start" | "stop" | "save_and_start"; + last_command_result: { + command: string; + outcome: "succeeded" | "blocked" | "failed"; + message: string; + }; + config: RuntimeConfig; +} + +interface RuntimeConfig { + connection_mode: "local" | "remote"; + server_url: string; + api_key: string; + model_path: string | null; + models_dir: string | null; + addr: string; // "127.0.0.1:8080" + gpu_layers: number; + ctx_size: number; + // ... +} +``` + +### 7.3 ForgeAgent + +```typescript +// types/forgeagent.ts +interface AgentPlan { + title: string; + overview: string; + steps: AgentStep[]; +} + +interface AgentStep { + id: number; + title: string; + description: string; + status: "pending" | "running" | "completed" | "error"; + sub_steps: AgentSubStep[]; +} + +interface SpecialistResponse { + specialist_id: string; + specialist_name: string; + thinking: string; + response: string; +} +``` + +--- + +## 8. Resumo das Comunicações + +### Tabela de Endpoints + +| Origem | Destino | Método | Rota/Command | Payload | Response | +|--------|---------|--------|--------------|---------|----------| +| ChatView | HTTP Server | POST | `/v1/chat/completions` | `{model, messages, stream, max_tokens}` | SSE ou JSON | +| useModels | HTTP Server | GET | `/v1/models` | - | `{data: ModelInfo[]}` | +| useModels | HTTP Server | GET | `/health` | - | `{status, model_loaded}` | +| useRuntimeManager | Tauri | invoke | `start_runtime_command` | - | `RuntimeSnapshot` | +| useRuntimeManager | Tauri | invoke | `stop_runtime_command` | - | `RuntimeSnapshot` | +| useRuntimeManager | Tauri | invoke | `save_runtime_config_command_handler` | `{config}` | `RuntimeSnapshot` | +| useRuntimeManager | Tauri | invoke | `scan_models` | `{dir}` | `RuntimeModelFile[]` | +| useRuntimeManager | Tauri | invoke | `swap_model_command` | `{newModelPath}` | `RuntimeSnapshot` | +| API HuggingFace | Tauri | invoke | `search_hugging_face_models_command` | `{query, token}` | `HuggingFaceModelResult[]` | +| API HuggingFace | Tauri | invoke | `start_hugging_face_download_command` | `{repo, fileName, token}` | `downloadId` | +| useForgeAgentSession | Tauri | invoke | `start_forgeagent_session_command` | `{prompt}` | `{plan_id, plan}` | +| useForgeAgentSession | Tauri | listen | `forgeagent:step-started` | - | `StepStartedEvent` | +| useForgeAgentSession | Tauri | listen | `forgeagent:step-completed` | - | `StepCompletedEvent` | +| useForgeAgentSession | Tauri | listen | `forgeagent:plan-completed` | - | `PlanCompletedEvent` | + +--- + +## 9. Configurações e Estado Global + +### Estado do Runtime (Contexto Global) + +```typescript +// App.tsx +const runtime = useRuntimeManager(); + +// Acesso em componentes filhos via props + +``` + +### Persistência do Runtime + +```typescript +// localStorage: forgelm_runtime_snapshot_v1 +{ + "status": "ready", + "ready": true, + "process_running": true, + "resolved_url": "http://127.0.0.1:8080", + "config": { + "connection_mode": "local", + "model_path": "D:/models/model.gguf", + "addr": "127.0.0.1:8080" + } +} +``` + +--- + +## 10. Diagrama de Sequência - Chat Completo + +``` +Usuário ChatView DB(sql.js) HTTP Server Tauri Modelo + │ │ │ │ │ │ + │──mensagem►│ │ │ │ │ + │ │──INSERT───►│ │ │ │ + │ │◄───────────│ │ │ │ + │ │ │ │ │ │ + │ │──SELECT───►│ (histórico) │ │ │ + │ │◄───────────│ │ │ │ + │ │ │ │ │ │ + │ │────────────┼──────────────┼──────────┼──► │ + │ │ fetch() │ │ │ inference + │ │ POST /v1/ │ │ │ │ + │ │chat/completions │ │ │ + │ │ │ │ │ │ + │ │◄───────────┼──────────────┼──────────┼─────────│ + │ │ SSE │ │ │ │ + │ │ chunks │ │ │ │ + │ │ │ │ │ │ + │◄──stream─│ │ │ │ │ + │ tokens │ │ │ │ │ + │ │ │ │ │ │ + │ │──INSERT───►│ │ │ │ + │ │ assistant │ │ │ │ + │ │◄───────────│ │ │ │ + │ │ │ │ │ │ + │◄─display │ │ │ │ │ + │ completo│ │ │ │ │ +``` + +--- + +## Conclusão + +O frontend ForgeLM consome o backend através de **duas camadas** distintas: + +1. **HTTP API** - Para chat completions (streaming e blocking) via `fetch()` +2. **Tauri Commands** - Para gerenciamento do runtime, modelos e downloads via `invoke()` + +A persistência local usa **sql.js** (SQLite WASM) com backup em `localStorage`, garantindo que conversas e memórias do usuário sejam mantidas entre sessões. + +O sistema suporta **dois modos**: +- **Local**: Tauri gerencia o processo do servidor HTTP internamente +- **Remote**: Conecta a um servidor OpenAI-compatible externo + +Todos os comandos Tauri possuem **fallbacks para modo web**, permitindo que o app funcione (com funcionalidade reduzida) mesmo sem o backend Tauri. diff --git a/crates/inference-server/src/compat.rs b/crates/inference-server/src/compat.rs index 32517f9..23840a4 100644 --- a/crates/inference-server/src/compat.rs +++ b/crates/inference-server/src/compat.rs @@ -176,6 +176,8 @@ pub struct ModelDescriptor { pub struct PreparedPrompt { pub prompt: String, pub streaming_mode: StreamingMode, + /// Extra stop sequences injected by the template (e.g. Generic transcript stops) + pub extra_stop_sequences: Vec, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -370,6 +372,22 @@ pub fn prepare_prompt(req: &ChatCompletionRequest, config: &ServerConfig) -> Pre append_assistant_prefix(&mut prompt, template); } + // For Generic template, inject stop sequences to prevent transcript leakage + // (model may continue generating "User:", "System:", "Assistant:" turns) + let extra_stops: Vec = if template == ChatTemplate::Generic && req.tools.is_empty() { + vec![ + "\nUser:".to_owned(), + "\nSystem:".to_owned(), + "\nAssistant:".to_owned(), + "\n\nUser:".to_owned(), + "\n\nSystem:".to_owned(), + "\n\nAssistant:".to_owned(), + "\nHuman:".to_owned(), + ] + } else { + vec![] + }; + PreparedPrompt { prompt, streaming_mode: if req.tools.is_empty() { @@ -377,6 +395,7 @@ pub fn prepare_prompt(req: &ChatCompletionRequest, config: &ServerConfig) -> Pre } else { StreamingMode::Buffered }, + extra_stop_sequences: extra_stops, } } @@ -742,9 +761,9 @@ pub fn build_sampler(req: &ChatCompletionRequest) -> inference_core::SamplerConf sampler.top_p = req.top_p.clamp(0.0, 1.0); sampler.max_tokens = req.max_tokens.max(1); sampler.stop_sequences = req.stop.clone(); - // Higher repeat_penalty (1.15) with larger context window (256 tokens) to prevent repetition loops - sampler.repeat_penalty = if req.tools.is_empty() { 1.15 } else { 1.0 }; - sampler.repeat_last_n = 256; + // Higher repeat_penalty (1.3) with larger context window (512 tokens) to prevent repetition loops + sampler.repeat_penalty = if req.tools.is_empty() { 1.3 } else { 1.0 }; + sampler.repeat_last_n = 512; sampler.seed = req.seed.unwrap_or(0); sampler } diff --git a/crates/inference-server/src/routes.rs b/crates/inference-server/src/routes.rs index 5ce8a2d..c7505ea 100644 --- a/crates/inference-server/src/routes.rs +++ b/crates/inference-server/src/routes.rs @@ -167,10 +167,21 @@ async fn chat_completions_handler( ); } - let sampler = build_sampler(&req); + let mut sampler = build_sampler(&req); let prepared = prepare_prompt(&req, &state.server_config); tracing::debug!(prompt_len = prepared.prompt.len(), "chat_completions: prompt prepared"); + // Merge template-injected stop sequences with request-provided ones + if !prepared.extra_stop_sequences.is_empty() { + let mut stops = sampler.stop_sequences.clone(); + for s in &prepared.extra_stop_sequences { + if !stops.contains(s) { + stops.push(s.clone()); + } + } + sampler.stop_sequences = stops; + } + if req.stream { tracing::info!("chat_completions: starting stream response"); stream_response( diff --git a/tauri-app/src/components/ChatView.tsx b/tauri-app/src/components/ChatView.tsx index defb559..f806c83 100644 --- a/tauri-app/src/components/ChatView.tsx +++ b/tauri-app/src/components/ChatView.tsx @@ -205,9 +205,16 @@ export default function ChatView({ const buildApiMessages = useCallback((cid: number) => { const history = db.query<{ role: string; content: string }>( "SELECT role, content FROM messages WHERE chat_id = ? AND content != '' AND content NOT LIKE '⚠️ Error:%' AND content != 'internal error' ORDER BY created_at DESC LIMIT ?", - [cid, 100], + [cid, 20], ).reverse(); - return history; + // Strip any leaked reasoning/transcript from assistant messages before sending back to model + return history.map((msg) => { + if (msg.role !== "assistant") return msg; + // Remove content after transcript leakage markers + const leakageIdx = msg.content.search(/(?:^|\n)\s*(?:System|User|Assistant|Human):/i); + const cleaned = leakageIdx > 0 ? msg.content.slice(0, leakageIdx).trimEnd() : msg.content; + return { ...msg, content: cleaned }; + }).filter((msg) => msg.content.trim() !== ""); }, [db]); const send = useCallback(async (text: string, fromChatId?: number, replaceAssistantId?: number) => { diff --git a/tauri-app/src/lib/chat.ts b/tauri-app/src/lib/chat.ts index 82e9970..2ccf82c 100644 --- a/tauri-app/src/lib/chat.ts +++ b/tauri-app/src/lib/chat.ts @@ -7,7 +7,7 @@ export interface ParsedAssistantResponse { const THINK_OPEN = ""; const THINK_CLOSE = ""; -const TRANSCRIPT_LEAKAGE_PATTERN = /(?:^|\n)\s*(?:System|User|Assistant):/i; +const TRANSCRIPT_LEAKAGE_PATTERN = /(?:^|\n)\s*(?:System|User|Assistant|Human):/i; export const CHAT_TRANSCRIPT_STOP_SEQUENCES = [ "\nSystem:", @@ -16,6 +16,9 @@ export const CHAT_TRANSCRIPT_STOP_SEQUENCES = [ "\n\nSystem:", "\n\nUser:", "\n\nAssistant:", + "System:", + "\nHuman:", + "\n\nHuman:", ]; export function buildChatTitle(text: string): string {