diff --git a/app/chat/page.tsx b/app/chat/page.tsx index 8e64194..d8f1483 100644 --- a/app/chat/page.tsx +++ b/app/chat/page.tsx @@ -154,10 +154,9 @@ export default function ChatPage() { // Passo 2: Se precisar de busca, executar if (searchQuery && searchQuery !== 'NO_SEARCH') { setThinkingStep('searching'); - // Buscar fontes usando configurações do store + // Pipeline em duas etapas: metadados → scraping de top URLs const maxResults = settings.webSearch.maxResults; - const excludedDomains = settings.webSearch.excludedDomains; - scrapedSources = await webSearch.search(searchQuery, maxResults, excludedDomains); + scrapedSources = await webSearch.smartSearchRag(searchQuery, maxResults); // Atualizar indicador enquanto lê fontes setThinkingStep('reading'); @@ -301,7 +300,10 @@ Ao responder sobre fatos atuais ou notícias, inicie mencionando explicitamente finalUserContent = `[CONTEXTO WEB OBRIGATÓRIO]\n${webContext}\n[/CONTEXTO WEB]\n\nCom base EXCLUSIVAMENTE no texto acima, responda: ${content}`; } - await sendMessage(finalUserContent, selectedModel, enhancedSystemPrompt); + // Enviar o conteúdo original para UI, mas com override no payload para incluir contexto + await sendMessage(content, selectedModel, enhancedSystemPrompt, { + payloadContentOverride: finalUserContent, + }); // Marcar como completo quando começar a streamar setTimeout(() => setThinkingStep('complete'), 100); diff --git a/hooks/use-chat.ts b/hooks/use-chat.ts index 07897ab..5d2f738 100644 --- a/hooks/use-chat.ts +++ b/hooks/use-chat.ts @@ -202,6 +202,9 @@ export function useChat() { systemPrompt?: string, options?: { onBeforeModelRequest?: (pendingMessages: Message[]) => Promise; + // Quando presente, substitui SOMENTE o conteúdo da última mensagem do usuário + // no payload enviado ao LLM, mantendo a UI com o conteúdo original. + payloadContentOverride?: string; } ) => { setIsLoading(true); @@ -257,12 +260,22 @@ export function useChat() { selectedMessages.push(lastUserMessage); totalTokens += estimateTokens(lastUserMessage.content); - const apiMessages = (systemPrompt + // Constrói mensagens para payload, aplicando override apenas à última mensagem do usuário + const baseMessages = systemPrompt ? [{ role: 'system', content: systemPrompt }, ...selectedMessages] - : selectedMessages).map(m => ({ + : selectedMessages; + + const apiMessages = baseMessages.map((m, idx) => { + // Detecta última mensagem do usuário + const isLastUser = m.role === 'user' && idx === baseMessages.length - 1; + const payloadContent = isLastUser && options?.payloadContentOverride + ? options.payloadContentOverride + : m.content; + return { role: m.role, - content: m.content // Send only content, not metadata (unless we want to inject it back) - })); + content: payloadContent, + }; + }); // [DEBUG INJECTION START] const systemMsg = apiMessages.find(m => m.role === 'system'); diff --git a/hooks/use-web-search.ts b/hooks/use-web-search.ts index 4f49ab9..791efc5 100644 --- a/hooks/use-web-search.ts +++ b/hooks/use-web-search.ts @@ -119,6 +119,62 @@ export function useWebSearch() { } }, []); + /** + * Pipeline em duas etapas: metadados → scraping das top URLs + */ + const smartSearchRag = useCallback(async (query: string, limit: number = 3): Promise => { + if (!query || !query.trim()) { + return []; + } + + setState(prev => ({ + ...prev, + status: 'searching', + currentQuery: query, + error: null, + scrapedSources: [], + })); + + try { + const settings = useSettingsStore.getState(); + const searchConfig: SearchConfig = { + maxConcurrentTabs: settings.webSearch.maxConcurrentTabs, + totalSourcesLimit: settings.webSearch.totalSourcesLimit, + categories: settings.webSearch.categories, + userCustomSites: settings.webSearch.userCustomSites, + excludedDomains: settings.webSearch.excludedDomains, + }; + + // Etapa 1: metadados + setState(prev => ({ ...prev, status: 'searching' })); + const { metadata, contents } = await webSearchService.smartSearchRag(query, limit, searchConfig); + + // Etapa 2: scraping + setState(prev => ({ ...prev, status: 'scraping' })); + + // [DEBUG] metadados coletados + console.log('🔎 Metadados coletados:', metadata); + + setState(prev => ({ + ...prev, + status: 'completed', + scrapedSources: contents, + error: null, + })); + + return contents; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Erro desconhecido no smartSearchRag'; + setState(prev => ({ + ...prev, + status: 'error', + error: errorMessage, + scrapedSources: [], + })); + throw error; + } + }, []); + /** * Extrai conteúdo de uma URL */ @@ -169,6 +225,7 @@ export function useWebSearch() { ...state, setEnabled, search, + smartSearchRag, extractUrl, reset, clearCache, diff --git a/services/webSearch.ts b/services/webSearch.ts index 48b78c1..ee1bb87 100644 --- a/services/webSearch.ts +++ b/services/webSearch.ts @@ -17,6 +17,12 @@ export interface SearchConfig { excludedDomains: string[]; } +export interface SearchResultMetadata { + title: string; + url: string; + snippet: string; +} + interface CachedResult { results: ScrapedContent[]; timestamp: number; @@ -212,6 +218,43 @@ class WebSearchService { } } + /** + * Executa busca de metadados (sem scraping) + */ + private async executeMetadataSearch( + query: string, + limit: number = 5, + searchConfig?: SearchConfig + ): Promise { + try { + let rustConfig: any = undefined; + if (searchConfig) { + rustConfig = { + max_concurrent_tabs: searchConfig.maxConcurrentTabs, + total_sources_limit: searchConfig.totalSourcesLimit, + categories: searchConfig.categories.map(cat => ({ + id: cat.id, + name: cat.name, + base_sites: cat.baseSites, + enabled: cat.enabled, + })), + user_custom_sites: searchConfig.userCustomSites, + excluded_domains: searchConfig.excludedDomains, + }; + } + + const results = await invoke('search_web_metadata', { + query: query.trim(), + limit, + search_config: rustConfig, + }); + return results || []; + } catch (error) { + console.error('Erro ao buscar metadados:', error); + throw error instanceof Error ? error : new Error('Falha ao buscar metadados'); + } + } + /** * Busca conteúdo na web com cache e rate limiting */ @@ -291,6 +334,32 @@ class WebSearchService { } } + /** + * Busca em duas etapas: metadados → scraping das melhores URLs + */ + async smartSearchRag( + query: string, + limit: number = 3, + searchConfig?: SearchConfig + ): Promise<{ metadata: SearchResultMetadata[]; contents: ScrapedContent[] }> { + if (!query || !query.trim()) { + return { metadata: [], contents: [] }; + } + + const metas = await this.executeMetadataSearch(query, Math.max(limit * 2, 5), searchConfig); + const topUrls = metas.map(m => m.url).slice(0, limit); + if (topUrls.length === 0) { + return { metadata: metas, contents: [] }; + } + try { + const contents = await invoke('scrape_urls', { urls: topUrls }); + return { metadata: metas, contents: contents || [] }; + } catch (error) { + console.error('Erro ao fazer scraping em lote:', error); + return { metadata: metas, contents: [] }; + } + } + /** * Extrai conteúdo de uma URL específica */ diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 85c993b..801988a 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -19,7 +19,18 @@ mod scheduler_loop; mod sources_config; mod system_monitor; -use web_scraper::{ScrapedContent, create_browser, search_and_scrape, search_and_scrape_with_config, scrape_url, SearchConfig}; +use web_scraper::{ + ScrapedContent, + SearchResultMetadata, + create_browser, + search_and_scrape, + search_and_scrape_with_config, + scrape_url, + SearchConfig, + search_duckduckgo_metadata, + smart_search, + scrape_urls_bulk, +}; use headless_chrome::Browser; use scheduler::{SentinelTask, SchedulerService, SchedulerState, TaskAction}; use sources_config::{SourcesConfig, load_sources_config, save_sources_config}; @@ -1232,6 +1243,56 @@ async fn extract_url_content( .map_err(|e| format!("Erro ao extrair conteúdo da URL: {}", e)) } +/// Busca metadados leves (título/URL/snippet) sem abrir páginas +#[command] +async fn search_web_metadata( + query: String, + limit: Option, + search_config: Option, +) -> Result, String> { + if query.trim().is_empty() { + return Err("Query não pode estar vazia".to_string()); + } + + let lim = limit.unwrap_or(5); + + if let Some(config) = search_config { + // Usar smart_search para obter URLs e convertê-las em metadados simples + match smart_search(&query, &config).await { + Ok(mut urls) => { + urls.truncate(lim); + let metas = urls + .into_iter() + .map(|u| SearchResultMetadata { title: u.clone(), url: u, snippet: String::new() }) + .collect::>(); + Ok(metas) + } + Err(e) => Err(format!("Erro ao executar smart_search: {}", e)), + } + } else { + search_duckduckgo_metadata(&query, lim) + .await + .map_err(|e| format!("Erro ao buscar metadados: {}", e)) + } +} + +/// Faz scraping em lote de URLs fornecidas +#[command] +async fn scrape_urls( + urls: Vec, + state: State<'_, BrowserState>, +) -> Result, String> { + if urls.is_empty() { + return Ok(Vec::new()); + } + + let browser = get_or_create_browser(state)?; + + scrape_urls_bulk(urls, browser) + .await + .map_err(|e| format!("Erro ao extrair conteúdo das URLs: {}", e)) +} + /// Reinicia o browser (útil se houver problemas) #[command] fn reset_browser(state: State<'_, BrowserState>) -> Result<(), String> { @@ -1785,6 +1846,8 @@ pub fn run() { check_mcp_server_available, search_and_extract_content, extract_url_content, + search_web_metadata, + scrape_urls, reset_browser, force_kill_browser, export_chat_sessions, diff --git a/src-tauri/src/web_scraper.rs b/src-tauri/src/web_scraper.rs index 2890f44..ad8f4b3 100644 --- a/src-tauri/src/web_scraper.rs +++ b/src-tauri/src/web_scraper.rs @@ -18,6 +18,14 @@ pub struct ScrapedContent { pub markdown: String, } +/// Metadados de resultado de busca (leve, sem abrir página) +#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] +pub struct SearchResultMetadata { + pub title: String, + pub url: String, + pub snippet: String, +} + /// Categoria de busca com sites curados #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] pub struct SearchCategory { @@ -124,6 +132,115 @@ pub async fn search_duckduckgo(query: &str, limit: usize) -> Result> Ok(links) } +/// Busca no DuckDuckGo retornando apenas metadados (título, URL, snippet) +pub async fn search_duckduckgo_metadata(query: &str, limit: usize) -> Result> { + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(10)) + .build()?; + + let url = format!("https://html.duckduckgo.com/html/?q={}", + urlencoding::encode(query)); + + let user_agent = get_random_user_agent(); + let res = client + .get(&url) + .header(USER_AGENT, user_agent) + .send() + .await? + .text() + .await?; + + let mut results: Vec = Vec::new(); + + { + let document = Html::parse_document(&res); + + // Estruturas comuns no HTML do DuckDuckGo + let container_selectors = vec![ + ".result", + ".web-result", + ".result__body", + ]; + let title_selectors = vec![ + ".result__a", + ".web-result__link", + "a.result__a", + ]; + let snippet_selectors = vec![ + ".result__snippet", + ".result__snippet.js-result-snippet", + ".web-result__snippet", + ]; + + for cont_sel in &container_selectors { + if results.len() >= limit { break; } + if let Ok(container) = Selector::parse(cont_sel) { + for node in document.select(&container) { + if results.len() >= limit { break; } + // Title + URL + let mut found_url: Option = None; + let mut found_title: Option = None; + for tsel in &title_selectors { + if let Ok(ts) = Selector::parse(tsel) { + if let Some(a) = node.select(&ts).next() { + if let Some(href) = a.value().attr("href") { + if let Some(real_url) = extract_real_url(href) { + found_url = clean_url(&real_url); + } + } + let text = a.text().collect::>().join(" ").trim().to_string(); + if !text.is_empty() { found_title = Some(text); } + } + } + if found_url.is_some() && found_title.is_some() { break; } + } + + if found_url.is_none() { continue; } + + // Snippet + let mut snippet_text: String = String::new(); + for ssel in &snippet_selectors { + if let Ok(ss) = Selector::parse(ssel) { + if let Some(s) = node.select(&ss).next() { + let t = s.text().collect::>().join(" "); + let norm = t.split_whitespace().collect::>().join(" "); + if !norm.is_empty() { snippet_text = norm; break; } + } + } + } + + let url_final = found_url.unwrap(); + if is_ad_or_tracker_url(&url_final) || url_final.is_empty() { continue; } + + results.push(SearchResultMetadata { + title: found_title.unwrap_or_else(|| url_final.clone()), + url: url_final, + snippet: snippet_text, + }); + + if results.len() >= limit { break; } + } + } + } + } + + // Se ainda vazio, tentar fallback simples: extrair todos os links conhecidos + if results.is_empty() { + let links = search_duckduckgo(query, limit).await?; + for l in links { + let url_clean = clean_url(&l).unwrap_or(l); + results.push(SearchResultMetadata { + title: url_clean.clone(), + url: url_clean, + snippet: String::new(), + }); + if results.len() >= limit { break; } + } + } + + Ok(results) +} + /// Extrai a URL real do redirecionamento do DuckDuckGo fn extract_real_url(ddg_redirect: &str) -> Option { // DuckDuckGo usa formato: /l/?kh=-1&uddg= @@ -436,6 +553,55 @@ pub async fn scrape_url( .map_err(|e| anyhow::anyhow!("Erro na task: {}", e))? } +/// Extrai conteúdo de múltiplas URLs já definidas (bulk) +pub async fn scrape_urls_bulk( + urls: Vec, + browser: Arc, +) -> Result> { + if urls.is_empty() { return Ok(Vec::new()); } + let concurrency = 5usize; + let semaphore = Arc::new(Semaphore::new(concurrency)); + let mut handles = Vec::new(); + + for url in urls { + let browser_clone = browser.clone(); + let permit = semaphore.clone().acquire_owned().await.unwrap(); + let url_clone = url.clone(); + let handle = tokio::task::spawn_blocking(move || { + let res = fetch_and_convert_sync(&browser_clone, &url_clone); + drop(permit); + res + }); + handles.push(handle); + } + + let mut results = Vec::new(); + for h in handles { + match h.await { + Ok(Ok(content)) => { + let content_len = content.content.chars().count(); + let md_len = content.markdown.chars().count(); + if content_len < 200 && md_len < 200 { + log::debug!("Descartado por conteúdo curto: {}", content.url); + } else { + results.push(content); + } + } + Ok(Err(e)) => { + let msg = format!("{}", e); + if msg.contains("Timeout") || msg.contains("ERR_HTTP") { + log::debug!("Ignorado (timeout/HTTP): {}", msg); + } else { + log::warn!("Erro ao processar URL: {}", e); + } + } + Err(e) => log::warn!("Erro na task de scraping: {}", e), + } + } + + Ok(results) +} + /// Extrai conteúdo de uma URL e converte para Markdown (versão síncrona) /// Retorna erro se timeout ou falha HTTP, mas não mata o processo fn fetch_and_convert_sync(browser: &Browser, url: &str) -> Result { @@ -534,7 +700,7 @@ fn fetch_and_convert_sync(browser: &Browser, url: &str) -> Result { - let mut markdown = html2text::from_read(product.content.as_bytes(), 80); + let markdown = html2text::from_read(product.content.as_bytes(), 80); // Se o markdown for muito curto, significa que o readability pode ter falhado if markdown.trim().chars().count() < 400 { if let Some(fallback) = extract_paragraph_fallback(url, &content) {