wip: snapshot das alterações não commitadas (2026-10-20)

Origem: D:/Documents/Github/LocalAiApp
Arquivos omitidos (segredos, bancos de dados, logs, >50 MB): 0

local-search-service/server.ts

@@ -0,0 +1,435 @@
+import express, { Request, Response } from 'express';
+import cors from 'cors';
+import { parseHTML } from 'linkedom';
+import { Readability } from '@mozilla/readability';
+import { NodeHtmlMarkdown } from 'node-html-markdown';
+
+const app = express();
+const PORT = process.env.PORT || 3030;
+
+app.use(cors());
+app.use(express.json());
+
+// ─── Logger ────────────────────────────────────────────────────────────────────
+
+type LogLevel = 'INFO' | 'WARN' | 'ERROR' | 'DEBUG';
+
+function log(level: LogLevel, phase: string, message: string, meta?: Record<string, any>) {
+    const ts = new Date().toISOString();
+    const metaStr = meta ? ` | ${JSON.stringify(meta)}` : '';
+    const icons: Record<LogLevel, string> = { INFO: 'ℹ️', WARN: '⚠️', ERROR: '❌', DEBUG: '🔍' };
+    console.log(`[${ts}] ${icons[level]} [${phase}] ${message}${metaStr}`);
+}
+
+function logSection(title: string) {
+    const border = '═'.repeat(60);
+    console.log(`\n${border}`);
+    console.log(`  ${title}`);
+    console.log(`${border}`);
+}
+
+// ─── Helpers ───────────────────────────────────────────────────────────────────
+
+/** Rough token count estimate (~4 chars per token) */
+function estimateTokens(text: string): number {
+    return Math.ceil(text.length / 4);
+}
+
+/**
+ * Context Analysis — scores each extracted source by:
+ *   - content length (longer = more info)
+ *   - keyword overlap with the query
+ *   - penalizes sources with very short content (likely failed extractions)
+ */
+function scoreSource(
+    source: { title: string; url: string; content: string },
+    query: string,
+): number {
+    const queryWords = query
+        .toLowerCase()
+        .split(/\s+/)
+        .filter((w) => w.length > 3);
+
+    const contentLower = source.content.toLowerCase();
+    const titleLower = source.title.toLowerCase();
+
+    // Keyword relevance: count query words present in content/title
+    const keywordMatches = queryWords.filter(
+        (w) => contentLower.includes(w) || titleLower.includes(w),
+    ).length;
+    const keywordScore = queryWords.length > 0 ? keywordMatches / queryWords.length : 0;
+
+    // Length score (0–1): prefer 500–3000 char range
+    const len = source.content.length;
+    const lengthScore = len < 100 ? 0.1 : len > 500 ? Math.min(1, len / 3000) : 0.5;
+
+    // Penalize if content is just a fallback snippet (very short)
+    const snippetPenalty = len < 200 ? 0.4 : 1.0;
+
+    const total = (keywordScore * 0.6 + lengthScore * 0.4) * snippetPenalty;
+
+    log('DEBUG', 'ContextAnalysis', `Scored source: "${source.title}"`, {
+        url: source.url,
+        contentLength: len,
+        keywordMatches,
+        keywordScore: keywordScore.toFixed(2),
+        lengthScore: lengthScore.toFixed(2),
+        snippetPenalty,
+        totalScore: total.toFixed(3),
+    });
+
+    return total;
+}
+
+/**
+ * Intelligent Context Pruning:
+ *   1. Sort sources by relevance score (descending).
+ *   2. Include full sources one by one until the budget is reached.
+ *   3. If a single source already exceeds budget, truncate it at a sentence boundary.
+ *   This avoids cutting in the middle of a word / sentence.
+ */
+function buildAndPruneContext(
+    sources: Array<{ title: string; url: string; content: string }>,
+    query: string,
+    maxChars: number,
+): { markdown: string; pruned: boolean; keptSources: number; droppedSources: number } {
+    logSection('🧠 Context Analysis & Pruning');
+
+    // Score and rank sources
+    const scored = sources
+        .map((src) => ({ ...src, score: scoreSource(src, query) }))
+        .sort((a, b) => b.score - a.score);
+
+    log('INFO', 'ContextPruning', `Ranked ${scored.length} sources by relevance`, {
+        ranking: scored.map((s, i) => ({
+            rank: i + 1,
+            title: s.title.substring(0, 50),
+            score: s.score.toFixed(3),
+            chars: s.content.length,
+            estimatedTokens: estimateTokens(s.content),
+        })),
+    });
+
+    const header = `# Web Search Context for: "${query}"\n\n`;
+    let markdown = header;
+    let budgetUsed = header.length;
+    let keptSources = 0;
+    let droppedSources = 0;
+    let pruned = false;
+
+    for (const src of scored) {
+        const entry =
+            `## Source ${keptSources + 1}: [${src.title}](${src.url})\n\n` +
+            `${src.content}\n\n` +
+            `---\n\n`;
+
+        if (budgetUsed + entry.length <= maxChars) {
+            markdown += entry;
+            budgetUsed += entry.length;
+            keptSources++;
+            log('INFO', 'ContextPruning', `✅ Included source (budget OK)`, {
+                rank: keptSources,
+                title: src.title.substring(0, 50),
+                entryChars: entry.length,
+                budgetUsed,
+                budgetRemaining: maxChars - budgetUsed,
+            });
+        } else {
+            const remaining = maxChars - budgetUsed;
+
+            if (keptSources === 0 && remaining > 200) {
+                // First source is already too large — truncate at last sentence boundary
+                const entryHeader =
+                    `## Source 1: [${src.title}](${src.url})\n\n`;
+                const contentBudget = remaining - entryHeader.length - 60; // room for suffix
+
+                let truncatedContent = src.content.substring(0, contentBudget);
+                // Find last sentence boundary (. ! ?)
+                const lastSentence = Math.max(
+                    truncatedContent.lastIndexOf('. '),
+                    truncatedContent.lastIndexOf('! '),
+                    truncatedContent.lastIndexOf('? '),
+                    truncatedContent.lastIndexOf('\n'),
+                );
+                if (lastSentence > contentBudget * 0.5) {
+                    truncatedContent = truncatedContent.substring(0, lastSentence + 1);
+                }
+
+                markdown +=
+                    entryHeader +
+                    truncatedContent +
+                    '\n\n...[CONTENT TRUNCATED — token budget exceeded]...\n\n---\n\n';
+                keptSources++;
+                pruned = true;
+                droppedSources = scored.length - keptSources;
+
+                log('WARN', 'ContextPruning', `✂️ Truncated oversized first source at sentence boundary`, {
+                    original: src.content.length,
+                    truncated: truncatedContent.length,
+                    budgetRemaining: remaining,
+                });
+                break;
+            }
+
+            // Drop this source entirely — not enough budget
+            droppedSources++;
+            pruned = true;
+            log('WARN', 'ContextPruning', `🗑️ Dropped source (insufficient budget)`, {
+                title: src.title.substring(0, 50),
+                entryChars: entry.length,
+                budgetRemaining: remaining,
+                score: src.score.toFixed(3),
+            });
+        }
+    }
+
+    return { markdown, pruned, keptSources, droppedSources };
+}
+
+// ─── Main Search Endpoint ──────────────────────────────────────────────────────
+
+app.post('/search-and-extract', async (req: Request, res: Response): Promise<any> => {
+    const requestId = `req_${Date.now().toString(36)}`;
+    const startTime = Date.now();
+
+    try {
+        const {
+            query,
+            searxngUrl = 'http://127.0.0.1:8888',
+            topK = 3,
+            maxContextLength = 8000,
+        } = req.body;
+
+        logSection(`🌍 Web Search Pipeline — [${requestId}]`);
+        log('INFO', 'Request', `Received search request`, {
+            requestId,
+            query,
+            searxngUrl,
+            topK,
+            maxContextLength,
+            estimatedTokenBudget: estimateTokens(String(maxContextLength)),
+        });
+
+        if (!query) {
+            log('WARN', 'Validation', `Rejected: query is missing`, { requestId });
+            return res.status(400).json({ error: 'Query is missing' });
+        }
+
+        // ── Step 1: SearXNG Search ─────────────────────────────────────────────
+        log('INFO', 'SearXNG', `Querying SearXNG...`, { url: searxngUrl, query });
+        const searxStart = Date.now();
+
+        const searchRes = await fetch(
+            `${searxngUrl}/search?q=${encodeURIComponent(query)}&format=json`,
+        );
+
+        const searxLatency = Date.now() - searxStart;
+
+        if (!searchRes.ok) {
+            log('ERROR', 'SearXNG', `Search failed`, {
+                status: searchRes.status,
+                latencyMs: searxLatency,
+            });
+            throw new Error(
+                `SearXNG search failed. Status: ${searchRes.status}. ` +
+                `Make sure SearXNG is running and 'json' format is enabled in settings.yml.`,
+            );
+        }
+
+        const searchData = await searchRes.json();
+        const results = searchData.results || [];
+        const topResults = results.slice(0, topK).filter((r: any) => r.url);
+
+        log('INFO', 'SearXNG', `Search complete`, {
+            requestId,
+            latencyMs: searxLatency,
+            totalResultsFound: results.length,
+            processingTopK: topResults.length,
+            urls: topResults.map((r: any) => r.url),
+        });
+
+        // ── Step 2: Page Extraction (parallel) ────────────────────────────────
+        // All pages are fetched concurrently with Promise.allSettled so a single
+        // slow or failing page never blocks the others.
+        logSection('📄 Page Extraction (parallel)');
+        log('INFO', 'Extractor', `Starting parallel fetch for ${topResults.length} pages`, {
+            urls: topResults.map((r: any) => r.url),
+        });
+
+        async function extractPage(result: any): Promise<{
+            ctx: { title: string; url: string; content: string } | null;
+            stat: Record<string, any>;
+        }> {
+            const pageStart = Date.now();
+            log('INFO', 'Extractor', `Fetching page...`, { url: result.url });
+
+            try {
+                const controller = new AbortController();
+                const timeoutId = setTimeout(() => controller.abort(), 8000);
+
+                const pageRes = await fetch(result.url, {
+                    signal: controller.signal,
+                    headers: { 'User-Agent': 'Mozilla/5.0 (compatible; LocalSearchBot/1.0)' },
+                });
+                clearTimeout(timeoutId);
+
+                const pageLatency = Date.now() - pageStart;
+
+                if (!pageRes.ok) {
+                    log('WARN', 'Extractor', `Skipping page — bad HTTP status`, {
+                        url: result.url,
+                        status: pageRes.status,
+                        latencyMs: pageLatency,
+                    });
+                    return { ctx: null, stat: { url: result.url, status: 'skipped', reason: `HTTP ${pageRes.status}`, latencyMs: pageLatency } };
+                }
+
+                const html = await pageRes.text();
+                const htmlSize = html.length;
+
+                // ── Readability Extraction ──────────────────────────────────────
+                const { document } = parseHTML(html);
+                const reader = new Readability(document as any);
+                const article = reader.parse();
+
+                if (!article || !article.content) {
+                    log('WARN', 'Extractor', `Readability failed — using snippet fallback`, {
+                        url: result.url,
+                        htmlSizeChars: htmlSize,
+                        latencyMs: pageLatency,
+                    });
+                    const fallbackContent = result.content || 'Content could not be parsed.';
+                    return {
+                        ctx: { title: result.title || 'Unknown', url: result.url, content: fallbackContent },
+                        stat: { url: result.url, status: 'fallback_snippet', contentChars: fallbackContent.length, latencyMs: pageLatency },
+                    };
+                }
+
+                // ── HTML → Markdown ─────────────────────────────────────────────
+                const markdown = NodeHtmlMarkdown.translate(article.content);
+                const pageLatencyFinal = Date.now() - pageStart;
+
+                log('INFO', 'Extractor', `✅ Extracted successfully`, {
+                    url: result.url,
+                    title: (article.title || result.title).substring(0, 60),
+                    htmlSizeChars: htmlSize,
+                    markdownChars: markdown.length,
+                    estimatedTokens: estimateTokens(markdown),
+                    latencyMs: pageLatencyFinal,
+                });
+
+                return {
+                    ctx: { title: article.title || result.title, url: result.url, content: markdown },
+                    stat: { url: result.url, status: 'success', htmlChars: htmlSize, markdownChars: markdown.length, estimatedTokens: estimateTokens(markdown), latencyMs: pageLatencyFinal },
+                };
+            } catch (err: any) {
+                const pageLatency = Date.now() - pageStart;
+                const isTimeout = err.name === 'AbortError';
+                log('ERROR', 'Extractor', `Failed to process page`, {
+                    url: result.url,
+                    error: err.message,
+                    isTimeout,
+                    latencyMs: pageLatency,
+                });
+                return { ctx: null, stat: { url: result.url, status: isTimeout ? 'timeout' : 'error', error: err.message, latencyMs: pageLatency } };
+            }
+        }
+
+        // Run all extractions in parallel
+        const parallelStart = Date.now();
+        const settled = await Promise.allSettled(topResults.map((r: any) => extractPage(r)));
+        const parallelLatency = Date.now() - parallelStart;
+
+        const extractedContexts: Array<{ title: string; url: string; content: string }> = [];
+        const extractionStats: Array<Record<string, any>> = [];
+
+        for (const result of settled) {
+            if (result.status === 'fulfilled') {
+                if (result.value.ctx) extractedContexts.push(result.value.ctx);
+                extractionStats.push(result.value.stat);
+            } else {
+                // Promise itself rejected (shouldn't happen since extractPage catches internally)
+                extractionStats.push({ status: 'unexpected_rejection', error: result.reason?.message });
+            }
+        }
+
+        log('INFO', 'Extractor', `Extraction phase complete (parallel)`, {
+            requestId,
+            parallelLatencyMs: parallelLatency,
+            attempted: topResults.length,
+            succeeded: extractedContexts.length,
+            failed: topResults.length - extractedContexts.length,
+            extractionStats,
+        });
+
+        // ── Step 3: Context Analysis + Intelligent Pruning ────────────────────
+        const totalRawChars = extractedContexts.reduce((sum, c) => sum + c.content.length, 0);
+        const totalRawTokens = estimateTokens(totalRawChars.toString()) * 4; // rough
+
+        log('INFO', 'ContextAnalysis', `Pre-pruning context stats`, {
+            requestId,
+            sources: extractedContexts.length,
+            totalRawChars,
+            estimatedRawTokens: estimateTokens(extractedContexts.reduce((s, c) => s + c.content, '')),
+            maxContextLength,
+            maxContextTokensEstimate: estimateTokens('x'.repeat(maxContextLength)),
+            pruningNeeded: totalRawChars > maxContextLength,
+        });
+
+        const { markdown: combinedMarkdown, pruned, keptSources, droppedSources } =
+            buildAndPruneContext(extractedContexts, query, maxContextLength);
+
+        // ── Step 4: Final Stats & Response ────────────────────────────────────
+        const totalLatency = Date.now() - startTime;
+        const finalChars = combinedMarkdown.length;
+        const finalTokens = estimateTokens(combinedMarkdown);
+
+        logSection('📤 Pipeline Complete');
+        log('INFO', 'Response', `Sending context to LocalAiApp`, {
+            requestId,
+            query,
+            totalPipelineLatencyMs: totalLatency,
+            searxLatencyMs: searxLatency,
+            finalContextChars: finalChars,
+            finalContextEstimatedTokens: finalTokens,
+            sourcesKept: keptSources,
+            sourcesDropped: droppedSources,
+            pruned,
+            sourceTitles: extractedContexts.map((c) => c.title.substring(0, 50)),
+        });
+
+        console.log(`${'═'.repeat(60)}\n`);
+
+        return res.json({
+            success: true,
+            context: combinedMarkdown,
+            sources: extractedContexts.map((c) => ({ title: c.title, url: c.url })),
+            meta: {
+                requestId,
+                totalLatencyMs: totalLatency,
+                finalContextChars: finalChars,
+                finalContextEstimatedTokens: finalTokens,
+                sourcesKept: keptSources,
+                sourcesDropped: droppedSources,
+                pruned,
+            },
+        });
+    } catch (error: any) {
+        const totalLatency = Date.now() - startTime;
+        log('ERROR', 'Fatal', `Pipeline failed`, {
+            requestId,
+            error: error.message,
+            stack: error.stack?.split('\n').slice(0, 5),
+            totalLatencyMs: totalLatency,
+        });
+        return res.status(500).json({ error: error.message, requestId });
+    }
+});
+
+app.listen(Number(PORT), '0.0.0.0', () => {
+    log('INFO', 'Server', `Local Search Service started`, {
+        port: PORT,
+        address: `http://0.0.0.0:${PORT}`,
+        endpoints: ['/search-and-extract'],
+    });
+});