import express, { Request, Response } from 'express';
import cors from 'cors';
import { parseHTML } from 'linkedom';
import { Readability } from '@mozilla/readability';
import { NodeHtmlMarkdown } from 'node-html-markdown';

const app = express();
const PORT = process.env.PORT || 3030;

app.use(cors());
app.use(express.json());

// ─── Logger ────────────────────────────────────────────────────────────────────

type LogLevel = 'INFO' | 'WARN' | 'ERROR' | 'DEBUG';

function log(level: LogLevel, phase: string, message: string, meta?: Record<string, any>) {
    const ts = new Date().toISOString();
    const metaStr = meta ? ` | ${JSON.stringify(meta)}` : '';
    const icons: Record<LogLevel, string> = { INFO: 'ℹ️', WARN: '⚠️', ERROR: '❌', DEBUG: '🔍' };
    console.log(`[${ts}] ${icons[level]} [${phase}] ${message}${metaStr}`);
}

function logSection(title: string) {
    const border = '═'.repeat(60);
    console.log(`\n${border}`);
    console.log(`  ${title}`);
    console.log(`${border}`);
}

// ─── Helpers ───────────────────────────────────────────────────────────────────

/** Rough token count estimate (~4 chars per token) */
function estimateTokens(text: string): number {
    return Math.ceil(text.length / 4);
}

/**
 * Context Analysis — scores each extracted source by:
 *   - content length (longer = more info)
 *   - keyword overlap with the query
 *   - penalizes sources with very short content (likely failed extractions)
 */
function scoreSource(
    source: { title: string; url: string; content: string },
    query: string,
): number {
    const queryWords = query
        .toLowerCase()
        .split(/\s+/)
        .filter((w) => w.length > 3);

    const contentLower = source.content.toLowerCase();
    const titleLower = source.title.toLowerCase();

    // Keyword relevance: count query words present in content/title
    const keywordMatches = queryWords.filter(
        (w) => contentLower.includes(w) || titleLower.includes(w),
    ).length;
    const keywordScore = queryWords.length > 0 ? keywordMatches / queryWords.length : 0;

    // Length score (0–1): prefer 500–3000 char range
    const len = source.content.length;
    const lengthScore = len < 100 ? 0.1 : len > 500 ? Math.min(1, len / 3000) : 0.5;

    // Penalize if content is just a fallback snippet (very short)
    const snippetPenalty = len < 200 ? 0.4 : 1.0;

    const total = (keywordScore * 0.6 + lengthScore * 0.4) * snippetPenalty;

    log('DEBUG', 'ContextAnalysis', `Scored source: "${source.title}"`, {
        url: source.url,
        contentLength: len,
        keywordMatches,
        keywordScore: keywordScore.toFixed(2),
        lengthScore: lengthScore.toFixed(2),
        snippetPenalty,
        totalScore: total.toFixed(3),
    });

    return total;
}

/**
 * Intelligent Context Pruning:
 *   1. Sort sources by relevance score (descending).
 *   2. Include full sources one by one until the budget is reached.
 *   3. If a single source already exceeds budget, truncate it at a sentence boundary.
 *   This avoids cutting in the middle of a word / sentence.
 */
function buildAndPruneContext(
    sources: Array<{ title: string; url: string; content: string }>,
    query: string,
    maxChars: number,
): { markdown: string; pruned: boolean; keptSources: number; droppedSources: number } {
    logSection('🧠 Context Analysis & Pruning');

    // Score and rank sources
    const scored = sources
        .map((src) => ({ ...src, score: scoreSource(src, query) }))
        .sort((a, b) => b.score - a.score);

    log('INFO', 'ContextPruning', `Ranked ${scored.length} sources by relevance`, {
        ranking: scored.map((s, i) => ({
            rank: i + 1,
            title: s.title.substring(0, 50),
            score: s.score.toFixed(3),
            chars: s.content.length,
            estimatedTokens: estimateTokens(s.content),
        })),
    });

    const header = `# Web Search Context for: "${query}"\n\n`;
    let markdown = header;
    let budgetUsed = header.length;
    let keptSources = 0;
    let droppedSources = 0;
    let pruned = false;

    for (const src of scored) {
        const entry =
            `## Source ${keptSources + 1}: [${src.title}](${src.url})\n\n` +
            `${src.content}\n\n` +
            `---\n\n`;

        if (budgetUsed + entry.length <= maxChars) {
            markdown += entry;
            budgetUsed += entry.length;
            keptSources++;
            log('INFO', 'ContextPruning', `✅ Included source (budget OK)`, {
                rank: keptSources,
                title: src.title.substring(0, 50),
                entryChars: entry.length,
                budgetUsed,
                budgetRemaining: maxChars - budgetUsed,
            });
        } else {
            const remaining = maxChars - budgetUsed;

            if (keptSources === 0 && remaining > 200) {
                // First source is already too large — truncate at last sentence boundary
                const entryHeader =
                    `## Source 1: [${src.title}](${src.url})\n\n`;
                const contentBudget = remaining - entryHeader.length - 60; // room for suffix

                let truncatedContent = src.content.substring(0, contentBudget);
                // Find last sentence boundary (. ! ?)
                const lastSentence = Math.max(
                    truncatedContent.lastIndexOf('. '),
                    truncatedContent.lastIndexOf('! '),
                    truncatedContent.lastIndexOf('? '),
                    truncatedContent.lastIndexOf('\n'),
                );
                if (lastSentence > contentBudget * 0.5) {
                    truncatedContent = truncatedContent.substring(0, lastSentence + 1);
                }

                markdown +=
                    entryHeader +
                    truncatedContent +
                    '\n\n...[CONTENT TRUNCATED — token budget exceeded]...\n\n---\n\n';
                keptSources++;
                pruned = true;
                droppedSources = scored.length - keptSources;

                log('WARN', 'ContextPruning', `✂️ Truncated oversized first source at sentence boundary`, {
                    original: src.content.length,
                    truncated: truncatedContent.length,
                    budgetRemaining: remaining,
                });
                break;
            }

            // Drop this source entirely — not enough budget
            droppedSources++;
            pruned = true;
            log('WARN', 'ContextPruning', `🗑️ Dropped source (insufficient budget)`, {
                title: src.title.substring(0, 50),
                entryChars: entry.length,
                budgetRemaining: remaining,
                score: src.score.toFixed(3),
            });
        }
    }

    return { markdown, pruned, keptSources, droppedSources };
}

// ─── Main Search Endpoint ──────────────────────────────────────────────────────

app.post('/search-and-extract', async (req: Request, res: Response): Promise<any> => {
    const requestId = `req_${Date.now().toString(36)}`;
    const startTime = Date.now();

    try {
        const {
            query,
            searxngUrl = 'http://127.0.0.1:8888',
            topK = 3,
            maxContextLength = 8000,
        } = req.body;

        logSection(`🌍 Web Search Pipeline — [${requestId}]`);
        log('INFO', 'Request', `Received search request`, {
            requestId,
            query,
            searxngUrl,
            topK,
            maxContextLength,
            estimatedTokenBudget: estimateTokens(String(maxContextLength)),
        });

        if (!query) {
            log('WARN', 'Validation', `Rejected: query is missing`, { requestId });
            return res.status(400).json({ error: 'Query is missing' });
        }

        // ── Step 1: SearXNG Search ─────────────────────────────────────────────
        log('INFO', 'SearXNG', `Querying SearXNG...`, { url: searxngUrl, query });
        const searxStart = Date.now();

        const searchRes = await fetch(
            `${searxngUrl}/search?q=${encodeURIComponent(query)}&format=json`,
        );

        const searxLatency = Date.now() - searxStart;

        if (!searchRes.ok) {
            log('ERROR', 'SearXNG', `Search failed`, {
                status: searchRes.status,
                latencyMs: searxLatency,
            });
            throw new Error(
                `SearXNG search failed. Status: ${searchRes.status}. ` +
                `Make sure SearXNG is running and 'json' format is enabled in settings.yml.`,
            );
        }

        const searchData = await searchRes.json();
        const results = searchData.results || [];
        const topResults = results.slice(0, topK).filter((r: any) => r.url);

        log('INFO', 'SearXNG', `Search complete`, {
            requestId,
            latencyMs: searxLatency,
            totalResultsFound: results.length,
            processingTopK: topResults.length,
            urls: topResults.map((r: any) => r.url),
        });

        // ── Step 2: Page Extraction (parallel) ────────────────────────────────
        // All pages are fetched concurrently with Promise.allSettled so a single
        // slow or failing page never blocks the others.
        logSection('📄 Page Extraction (parallel)');
        log('INFO', 'Extractor', `Starting parallel fetch for ${topResults.length} pages`, {
            urls: topResults.map((r: any) => r.url),
        });

        async function extractPage(result: any): Promise<{
            ctx: { title: string; url: string; content: string } | null;
            stat: Record<string, any>;
        }> {
            const pageStart = Date.now();
            log('INFO', 'Extractor', `Fetching page...`, { url: result.url });

            try {
                const controller = new AbortController();
                const timeoutId = setTimeout(() => controller.abort(), 8000);

                const pageRes = await fetch(result.url, {
                    signal: controller.signal,
                    headers: { 'User-Agent': 'Mozilla/5.0 (compatible; LocalSearchBot/1.0)' },
                });
                clearTimeout(timeoutId);

                const pageLatency = Date.now() - pageStart;

                if (!pageRes.ok) {
                    log('WARN', 'Extractor', `Skipping page — bad HTTP status`, {
                        url: result.url,
                        status: pageRes.status,
                        latencyMs: pageLatency,
                    });
                    return { ctx: null, stat: { url: result.url, status: 'skipped', reason: `HTTP ${pageRes.status}`, latencyMs: pageLatency } };
                }

                const html = await pageRes.text();
                const htmlSize = html.length;

                // ── Readability Extraction ──────────────────────────────────────
                const { document } = parseHTML(html);
                const reader = new Readability(document as any);
                const article = reader.parse();

                if (!article || !article.content) {
                    log('WARN', 'Extractor', `Readability failed — using snippet fallback`, {
                        url: result.url,
                        htmlSizeChars: htmlSize,
                        latencyMs: pageLatency,
                    });
                    const fallbackContent = result.content || 'Content could not be parsed.';
                    return {
                        ctx: { title: result.title || 'Unknown', url: result.url, content: fallbackContent },
                        stat: { url: result.url, status: 'fallback_snippet', contentChars: fallbackContent.length, latencyMs: pageLatency },
                    };
                }

                // ── HTML → Markdown ─────────────────────────────────────────────
                const markdown = NodeHtmlMarkdown.translate(article.content);
                const pageLatencyFinal = Date.now() - pageStart;

                log('INFO', 'Extractor', `✅ Extracted successfully`, {
                    url: result.url,
                    title: (article.title || result.title).substring(0, 60),
                    htmlSizeChars: htmlSize,
                    markdownChars: markdown.length,
                    estimatedTokens: estimateTokens(markdown),
                    latencyMs: pageLatencyFinal,
                });

                return {
                    ctx: { title: article.title || result.title, url: result.url, content: markdown },
                    stat: { url: result.url, status: 'success', htmlChars: htmlSize, markdownChars: markdown.length, estimatedTokens: estimateTokens(markdown), latencyMs: pageLatencyFinal },
                };
            } catch (err: any) {
                const pageLatency = Date.now() - pageStart;
                const isTimeout = err.name === 'AbortError';
                log('ERROR', 'Extractor', `Failed to process page`, {
                    url: result.url,
                    error: err.message,
                    isTimeout,
                    latencyMs: pageLatency,
                });
                return { ctx: null, stat: { url: result.url, status: isTimeout ? 'timeout' : 'error', error: err.message, latencyMs: pageLatency } };
            }
        }

        // Run all extractions in parallel
        const parallelStart = Date.now();
        const settled = await Promise.allSettled(topResults.map((r: any) => extractPage(r)));
        const parallelLatency = Date.now() - parallelStart;

        const extractedContexts: Array<{ title: string; url: string; content: string }> = [];
        const extractionStats: Array<Record<string, any>> = [];

        for (const result of settled) {
            if (result.status === 'fulfilled') {
                if (result.value.ctx) extractedContexts.push(result.value.ctx);
                extractionStats.push(result.value.stat);
            } else {
                // Promise itself rejected (shouldn't happen since extractPage catches internally)
                extractionStats.push({ status: 'unexpected_rejection', error: result.reason?.message });
            }
        }

        log('INFO', 'Extractor', `Extraction phase complete (parallel)`, {
            requestId,
            parallelLatencyMs: parallelLatency,
            attempted: topResults.length,
            succeeded: extractedContexts.length,
            failed: topResults.length - extractedContexts.length,
            extractionStats,
        });

        // ── Step 3: Context Analysis + Intelligent Pruning ────────────────────
        const totalRawChars = extractedContexts.reduce((sum, c) => sum + c.content.length, 0);
        const totalRawTokens = estimateTokens(totalRawChars.toString()) * 4; // rough

        log('INFO', 'ContextAnalysis', `Pre-pruning context stats`, {
            requestId,
            sources: extractedContexts.length,
            totalRawChars,
            estimatedRawTokens: estimateTokens(extractedContexts.reduce((s, c) => s + c.content, '')),
            maxContextLength,
            maxContextTokensEstimate: estimateTokens('x'.repeat(maxContextLength)),
            pruningNeeded: totalRawChars > maxContextLength,
        });

        const { markdown: combinedMarkdown, pruned, keptSources, droppedSources } =
            buildAndPruneContext(extractedContexts, query, maxContextLength);

        // ── Step 4: Final Stats & Response ────────────────────────────────────
        const totalLatency = Date.now() - startTime;
        const finalChars = combinedMarkdown.length;
        const finalTokens = estimateTokens(combinedMarkdown);

        logSection('📤 Pipeline Complete');
        log('INFO', 'Response', `Sending context to LocalAiApp`, {
            requestId,
            query,
            totalPipelineLatencyMs: totalLatency,
            searxLatencyMs: searxLatency,
            finalContextChars: finalChars,
            finalContextEstimatedTokens: finalTokens,
            sourcesKept: keptSources,
            sourcesDropped: droppedSources,
            pruned,
            sourceTitles: extractedContexts.map((c) => c.title.substring(0, 50)),
        });

        console.log(`${'═'.repeat(60)}\n`);

        return res.json({
            success: true,
            context: combinedMarkdown,
            sources: extractedContexts.map((c) => ({ title: c.title, url: c.url })),
            meta: {
                requestId,
                totalLatencyMs: totalLatency,
                finalContextChars: finalChars,
                finalContextEstimatedTokens: finalTokens,
                sourcesKept: keptSources,
                sourcesDropped: droppedSources,
                pruned,
            },
        });
    } catch (error: any) {
        const totalLatency = Date.now() - startTime;
        log('ERROR', 'Fatal', `Pipeline failed`, {
            requestId,
            error: error.message,
            stack: error.stack?.split('\n').slice(0, 5),
            totalLatencyMs: totalLatency,
        });
        return res.status(500).json({ error: error.message, requestId });
    }
});

app.listen(Number(PORT), '0.0.0.0', () => {
    log('INFO', 'Server', `Local Search Service started`, {
        port: PORT,
        address: `http://0.0.0.0:${PORT}`,
        endpoints: ['/search-and-extract'],
    });
});