evandro
e08189f640
Origem: D:/Documents/Github/LocalAiApp Arquivos omitidos (segredos, bancos de dados, logs, >50 MB): 0
436 lines
18 KiB
TypeScript
436 lines
18 KiB
TypeScript
import express, { Request, Response } from 'express';
|
||
import cors from 'cors';
|
||
import { parseHTML } from 'linkedom';
|
||
import { Readability } from '@mozilla/readability';
|
||
import { NodeHtmlMarkdown } from 'node-html-markdown';
|
||
|
||
const app = express();
|
||
const PORT = process.env.PORT || 3030;
|
||
|
||
app.use(cors());
|
||
app.use(express.json());
|
||
|
||
// ─── Logger ────────────────────────────────────────────────────────────────────
|
||
|
||
type LogLevel = 'INFO' | 'WARN' | 'ERROR' | 'DEBUG';
|
||
|
||
function log(level: LogLevel, phase: string, message: string, meta?: Record<string, any>) {
|
||
const ts = new Date().toISOString();
|
||
const metaStr = meta ? ` | ${JSON.stringify(meta)}` : '';
|
||
const icons: Record<LogLevel, string> = { INFO: 'ℹ️', WARN: '⚠️', ERROR: '❌', DEBUG: '🔍' };
|
||
console.log(`[${ts}] ${icons[level]} [${phase}] ${message}${metaStr}`);
|
||
}
|
||
|
||
function logSection(title: string) {
|
||
const border = '═'.repeat(60);
|
||
console.log(`\n${border}`);
|
||
console.log(` ${title}`);
|
||
console.log(`${border}`);
|
||
}
|
||
|
||
// ─── Helpers ───────────────────────────────────────────────────────────────────
|
||
|
||
/** Rough token count estimate (~4 chars per token) */
|
||
function estimateTokens(text: string): number {
|
||
return Math.ceil(text.length / 4);
|
||
}
|
||
|
||
/**
|
||
* Context Analysis — scores each extracted source by:
|
||
* - content length (longer = more info)
|
||
* - keyword overlap with the query
|
||
* - penalizes sources with very short content (likely failed extractions)
|
||
*/
|
||
function scoreSource(
|
||
source: { title: string; url: string; content: string },
|
||
query: string,
|
||
): number {
|
||
const queryWords = query
|
||
.toLowerCase()
|
||
.split(/\s+/)
|
||
.filter((w) => w.length > 3);
|
||
|
||
const contentLower = source.content.toLowerCase();
|
||
const titleLower = source.title.toLowerCase();
|
||
|
||
// Keyword relevance: count query words present in content/title
|
||
const keywordMatches = queryWords.filter(
|
||
(w) => contentLower.includes(w) || titleLower.includes(w),
|
||
).length;
|
||
const keywordScore = queryWords.length > 0 ? keywordMatches / queryWords.length : 0;
|
||
|
||
// Length score (0–1): prefer 500–3000 char range
|
||
const len = source.content.length;
|
||
const lengthScore = len < 100 ? 0.1 : len > 500 ? Math.min(1, len / 3000) : 0.5;
|
||
|
||
// Penalize if content is just a fallback snippet (very short)
|
||
const snippetPenalty = len < 200 ? 0.4 : 1.0;
|
||
|
||
const total = (keywordScore * 0.6 + lengthScore * 0.4) * snippetPenalty;
|
||
|
||
log('DEBUG', 'ContextAnalysis', `Scored source: "${source.title}"`, {
|
||
url: source.url,
|
||
contentLength: len,
|
||
keywordMatches,
|
||
keywordScore: keywordScore.toFixed(2),
|
||
lengthScore: lengthScore.toFixed(2),
|
||
snippetPenalty,
|
||
totalScore: total.toFixed(3),
|
||
});
|
||
|
||
return total;
|
||
}
|
||
|
||
/**
|
||
* Intelligent Context Pruning:
|
||
* 1. Sort sources by relevance score (descending).
|
||
* 2. Include full sources one by one until the budget is reached.
|
||
* 3. If a single source already exceeds budget, truncate it at a sentence boundary.
|
||
* This avoids cutting in the middle of a word / sentence.
|
||
*/
|
||
function buildAndPruneContext(
|
||
sources: Array<{ title: string; url: string; content: string }>,
|
||
query: string,
|
||
maxChars: number,
|
||
): { markdown: string; pruned: boolean; keptSources: number; droppedSources: number } {
|
||
logSection('🧠 Context Analysis & Pruning');
|
||
|
||
// Score and rank sources
|
||
const scored = sources
|
||
.map((src) => ({ ...src, score: scoreSource(src, query) }))
|
||
.sort((a, b) => b.score - a.score);
|
||
|
||
log('INFO', 'ContextPruning', `Ranked ${scored.length} sources by relevance`, {
|
||
ranking: scored.map((s, i) => ({
|
||
rank: i + 1,
|
||
title: s.title.substring(0, 50),
|
||
score: s.score.toFixed(3),
|
||
chars: s.content.length,
|
||
estimatedTokens: estimateTokens(s.content),
|
||
})),
|
||
});
|
||
|
||
const header = `# Web Search Context for: "${query}"\n\n`;
|
||
let markdown = header;
|
||
let budgetUsed = header.length;
|
||
let keptSources = 0;
|
||
let droppedSources = 0;
|
||
let pruned = false;
|
||
|
||
for (const src of scored) {
|
||
const entry =
|
||
`## Source ${keptSources + 1}: [${src.title}](${src.url})\n\n` +
|
||
`${src.content}\n\n` +
|
||
`---\n\n`;
|
||
|
||
if (budgetUsed + entry.length <= maxChars) {
|
||
markdown += entry;
|
||
budgetUsed += entry.length;
|
||
keptSources++;
|
||
log('INFO', 'ContextPruning', `✅ Included source (budget OK)`, {
|
||
rank: keptSources,
|
||
title: src.title.substring(0, 50),
|
||
entryChars: entry.length,
|
||
budgetUsed,
|
||
budgetRemaining: maxChars - budgetUsed,
|
||
});
|
||
} else {
|
||
const remaining = maxChars - budgetUsed;
|
||
|
||
if (keptSources === 0 && remaining > 200) {
|
||
// First source is already too large — truncate at last sentence boundary
|
||
const entryHeader =
|
||
`## Source 1: [${src.title}](${src.url})\n\n`;
|
||
const contentBudget = remaining - entryHeader.length - 60; // room for suffix
|
||
|
||
let truncatedContent = src.content.substring(0, contentBudget);
|
||
// Find last sentence boundary (. ! ?)
|
||
const lastSentence = Math.max(
|
||
truncatedContent.lastIndexOf('. '),
|
||
truncatedContent.lastIndexOf('! '),
|
||
truncatedContent.lastIndexOf('? '),
|
||
truncatedContent.lastIndexOf('\n'),
|
||
);
|
||
if (lastSentence > contentBudget * 0.5) {
|
||
truncatedContent = truncatedContent.substring(0, lastSentence + 1);
|
||
}
|
||
|
||
markdown +=
|
||
entryHeader +
|
||
truncatedContent +
|
||
'\n\n...[CONTENT TRUNCATED — token budget exceeded]...\n\n---\n\n';
|
||
keptSources++;
|
||
pruned = true;
|
||
droppedSources = scored.length - keptSources;
|
||
|
||
log('WARN', 'ContextPruning', `✂️ Truncated oversized first source at sentence boundary`, {
|
||
original: src.content.length,
|
||
truncated: truncatedContent.length,
|
||
budgetRemaining: remaining,
|
||
});
|
||
break;
|
||
}
|
||
|
||
// Drop this source entirely — not enough budget
|
||
droppedSources++;
|
||
pruned = true;
|
||
log('WARN', 'ContextPruning', `🗑️ Dropped source (insufficient budget)`, {
|
||
title: src.title.substring(0, 50),
|
||
entryChars: entry.length,
|
||
budgetRemaining: remaining,
|
||
score: src.score.toFixed(3),
|
||
});
|
||
}
|
||
}
|
||
|
||
return { markdown, pruned, keptSources, droppedSources };
|
||
}
|
||
|
||
// ─── Main Search Endpoint ──────────────────────────────────────────────────────
|
||
|
||
app.post('/search-and-extract', async (req: Request, res: Response): Promise<any> => {
|
||
const requestId = `req_${Date.now().toString(36)}`;
|
||
const startTime = Date.now();
|
||
|
||
try {
|
||
const {
|
||
query,
|
||
searxngUrl = 'http://127.0.0.1:8888',
|
||
topK = 3,
|
||
maxContextLength = 8000,
|
||
} = req.body;
|
||
|
||
logSection(`🌍 Web Search Pipeline — [${requestId}]`);
|
||
log('INFO', 'Request', `Received search request`, {
|
||
requestId,
|
||
query,
|
||
searxngUrl,
|
||
topK,
|
||
maxContextLength,
|
||
estimatedTokenBudget: estimateTokens(String(maxContextLength)),
|
||
});
|
||
|
||
if (!query) {
|
||
log('WARN', 'Validation', `Rejected: query is missing`, { requestId });
|
||
return res.status(400).json({ error: 'Query is missing' });
|
||
}
|
||
|
||
// ── Step 1: SearXNG Search ─────────────────────────────────────────────
|
||
log('INFO', 'SearXNG', `Querying SearXNG...`, { url: searxngUrl, query });
|
||
const searxStart = Date.now();
|
||
|
||
const searchRes = await fetch(
|
||
`${searxngUrl}/search?q=${encodeURIComponent(query)}&format=json`,
|
||
);
|
||
|
||
const searxLatency = Date.now() - searxStart;
|
||
|
||
if (!searchRes.ok) {
|
||
log('ERROR', 'SearXNG', `Search failed`, {
|
||
status: searchRes.status,
|
||
latencyMs: searxLatency,
|
||
});
|
||
throw new Error(
|
||
`SearXNG search failed. Status: ${searchRes.status}. ` +
|
||
`Make sure SearXNG is running and 'json' format is enabled in settings.yml.`,
|
||
);
|
||
}
|
||
|
||
const searchData = await searchRes.json();
|
||
const results = searchData.results || [];
|
||
const topResults = results.slice(0, topK).filter((r: any) => r.url);
|
||
|
||
log('INFO', 'SearXNG', `Search complete`, {
|
||
requestId,
|
||
latencyMs: searxLatency,
|
||
totalResultsFound: results.length,
|
||
processingTopK: topResults.length,
|
||
urls: topResults.map((r: any) => r.url),
|
||
});
|
||
|
||
// ── Step 2: Page Extraction (parallel) ────────────────────────────────
|
||
// All pages are fetched concurrently with Promise.allSettled so a single
|
||
// slow or failing page never blocks the others.
|
||
logSection('📄 Page Extraction (parallel)');
|
||
log('INFO', 'Extractor', `Starting parallel fetch for ${topResults.length} pages`, {
|
||
urls: topResults.map((r: any) => r.url),
|
||
});
|
||
|
||
async function extractPage(result: any): Promise<{
|
||
ctx: { title: string; url: string; content: string } | null;
|
||
stat: Record<string, any>;
|
||
}> {
|
||
const pageStart = Date.now();
|
||
log('INFO', 'Extractor', `Fetching page...`, { url: result.url });
|
||
|
||
try {
|
||
const controller = new AbortController();
|
||
const timeoutId = setTimeout(() => controller.abort(), 8000);
|
||
|
||
const pageRes = await fetch(result.url, {
|
||
signal: controller.signal,
|
||
headers: { 'User-Agent': 'Mozilla/5.0 (compatible; LocalSearchBot/1.0)' },
|
||
});
|
||
clearTimeout(timeoutId);
|
||
|
||
const pageLatency = Date.now() - pageStart;
|
||
|
||
if (!pageRes.ok) {
|
||
log('WARN', 'Extractor', `Skipping page — bad HTTP status`, {
|
||
url: result.url,
|
||
status: pageRes.status,
|
||
latencyMs: pageLatency,
|
||
});
|
||
return { ctx: null, stat: { url: result.url, status: 'skipped', reason: `HTTP ${pageRes.status}`, latencyMs: pageLatency } };
|
||
}
|
||
|
||
const html = await pageRes.text();
|
||
const htmlSize = html.length;
|
||
|
||
// ── Readability Extraction ──────────────────────────────────────
|
||
const { document } = parseHTML(html);
|
||
const reader = new Readability(document as any);
|
||
const article = reader.parse();
|
||
|
||
if (!article || !article.content) {
|
||
log('WARN', 'Extractor', `Readability failed — using snippet fallback`, {
|
||
url: result.url,
|
||
htmlSizeChars: htmlSize,
|
||
latencyMs: pageLatency,
|
||
});
|
||
const fallbackContent = result.content || 'Content could not be parsed.';
|
||
return {
|
||
ctx: { title: result.title || 'Unknown', url: result.url, content: fallbackContent },
|
||
stat: { url: result.url, status: 'fallback_snippet', contentChars: fallbackContent.length, latencyMs: pageLatency },
|
||
};
|
||
}
|
||
|
||
// ── HTML → Markdown ─────────────────────────────────────────────
|
||
const markdown = NodeHtmlMarkdown.translate(article.content);
|
||
const pageLatencyFinal = Date.now() - pageStart;
|
||
|
||
log('INFO', 'Extractor', `✅ Extracted successfully`, {
|
||
url: result.url,
|
||
title: (article.title || result.title).substring(0, 60),
|
||
htmlSizeChars: htmlSize,
|
||
markdownChars: markdown.length,
|
||
estimatedTokens: estimateTokens(markdown),
|
||
latencyMs: pageLatencyFinal,
|
||
});
|
||
|
||
return {
|
||
ctx: { title: article.title || result.title, url: result.url, content: markdown },
|
||
stat: { url: result.url, status: 'success', htmlChars: htmlSize, markdownChars: markdown.length, estimatedTokens: estimateTokens(markdown), latencyMs: pageLatencyFinal },
|
||
};
|
||
} catch (err: any) {
|
||
const pageLatency = Date.now() - pageStart;
|
||
const isTimeout = err.name === 'AbortError';
|
||
log('ERROR', 'Extractor', `Failed to process page`, {
|
||
url: result.url,
|
||
error: err.message,
|
||
isTimeout,
|
||
latencyMs: pageLatency,
|
||
});
|
||
return { ctx: null, stat: { url: result.url, status: isTimeout ? 'timeout' : 'error', error: err.message, latencyMs: pageLatency } };
|
||
}
|
||
}
|
||
|
||
// Run all extractions in parallel
|
||
const parallelStart = Date.now();
|
||
const settled = await Promise.allSettled(topResults.map((r: any) => extractPage(r)));
|
||
const parallelLatency = Date.now() - parallelStart;
|
||
|
||
const extractedContexts: Array<{ title: string; url: string; content: string }> = [];
|
||
const extractionStats: Array<Record<string, any>> = [];
|
||
|
||
for (const result of settled) {
|
||
if (result.status === 'fulfilled') {
|
||
if (result.value.ctx) extractedContexts.push(result.value.ctx);
|
||
extractionStats.push(result.value.stat);
|
||
} else {
|
||
// Promise itself rejected (shouldn't happen since extractPage catches internally)
|
||
extractionStats.push({ status: 'unexpected_rejection', error: result.reason?.message });
|
||
}
|
||
}
|
||
|
||
log('INFO', 'Extractor', `Extraction phase complete (parallel)`, {
|
||
requestId,
|
||
parallelLatencyMs: parallelLatency,
|
||
attempted: topResults.length,
|
||
succeeded: extractedContexts.length,
|
||
failed: topResults.length - extractedContexts.length,
|
||
extractionStats,
|
||
});
|
||
|
||
// ── Step 3: Context Analysis + Intelligent Pruning ────────────────────
|
||
const totalRawChars = extractedContexts.reduce((sum, c) => sum + c.content.length, 0);
|
||
const totalRawTokens = estimateTokens(totalRawChars.toString()) * 4; // rough
|
||
|
||
log('INFO', 'ContextAnalysis', `Pre-pruning context stats`, {
|
||
requestId,
|
||
sources: extractedContexts.length,
|
||
totalRawChars,
|
||
estimatedRawTokens: estimateTokens(extractedContexts.reduce((s, c) => s + c.content, '')),
|
||
maxContextLength,
|
||
maxContextTokensEstimate: estimateTokens('x'.repeat(maxContextLength)),
|
||
pruningNeeded: totalRawChars > maxContextLength,
|
||
});
|
||
|
||
const { markdown: combinedMarkdown, pruned, keptSources, droppedSources } =
|
||
buildAndPruneContext(extractedContexts, query, maxContextLength);
|
||
|
||
// ── Step 4: Final Stats & Response ────────────────────────────────────
|
||
const totalLatency = Date.now() - startTime;
|
||
const finalChars = combinedMarkdown.length;
|
||
const finalTokens = estimateTokens(combinedMarkdown);
|
||
|
||
logSection('📤 Pipeline Complete');
|
||
log('INFO', 'Response', `Sending context to LocalAiApp`, {
|
||
requestId,
|
||
query,
|
||
totalPipelineLatencyMs: totalLatency,
|
||
searxLatencyMs: searxLatency,
|
||
finalContextChars: finalChars,
|
||
finalContextEstimatedTokens: finalTokens,
|
||
sourcesKept: keptSources,
|
||
sourcesDropped: droppedSources,
|
||
pruned,
|
||
sourceTitles: extractedContexts.map((c) => c.title.substring(0, 50)),
|
||
});
|
||
|
||
console.log(`${'═'.repeat(60)}\n`);
|
||
|
||
return res.json({
|
||
success: true,
|
||
context: combinedMarkdown,
|
||
sources: extractedContexts.map((c) => ({ title: c.title, url: c.url })),
|
||
meta: {
|
||
requestId,
|
||
totalLatencyMs: totalLatency,
|
||
finalContextChars: finalChars,
|
||
finalContextEstimatedTokens: finalTokens,
|
||
sourcesKept: keptSources,
|
||
sourcesDropped: droppedSources,
|
||
pruned,
|
||
},
|
||
});
|
||
} catch (error: any) {
|
||
const totalLatency = Date.now() - startTime;
|
||
log('ERROR', 'Fatal', `Pipeline failed`, {
|
||
requestId,
|
||
error: error.message,
|
||
stack: error.stack?.split('\n').slice(0, 5),
|
||
totalLatencyMs: totalLatency,
|
||
});
|
||
return res.status(500).json({ error: error.message, requestId });
|
||
}
|
||
});
|
||
|
||
app.listen(Number(PORT), '0.0.0.0', () => {
|
||
log('INFO', 'Server', `Local Search Service started`, {
|
||
port: PORT,
|
||
address: `http://0.0.0.0:${PORT}`,
|
||
endpoints: ['/search-and-extract'],
|
||
});
|
||
});
|