feat(web-search): implement two-phase smart search pipeline

Add metadata-first search approach followed by targeted scraping of top URLs. Includes:
- New `smartSearchRag` method in web search service
- Metadata-only search endpoint in Tauri backend
- Bulk URL scraping capability
- Payload content override in chat messages

app/chat/page.tsx

@@ -154,10 +154,9 @@ export default function ChatPage() {
         // Passo 2: Se precisar de busca, executar
         if (searchQuery && searchQuery !== 'NO_SEARCH') {
           setThinkingStep('searching');
-          // Buscar fontes usando configurações do store
+          // Pipeline em duas etapas: metadados → scraping de top URLs
           const maxResults = settings.webSearch.maxResults;
-          const excludedDomains = settings.webSearch.excludedDomains;
+          scrapedSources = await webSearch.smartSearchRag(searchQuery, maxResults);
-          scrapedSources = await webSearch.search(searchQuery, maxResults, excludedDomains);
           
           // Atualizar indicador enquanto lê fontes
           setThinkingStep('reading');
@@ -301,7 +300,10 @@ Ao responder sobre fatos atuais ou notícias, inicie mencionando explicitamente
       finalUserContent = `[CONTEXTO WEB OBRIGATÓRIO]\n${webContext}\n[/CONTEXTO WEB]\n\nCom base EXCLUSIVAMENTE no texto acima, responda: ${content}`;
     }
     
-    await sendMessage(finalUserContent, selectedModel, enhancedSystemPrompt);
+    // Enviar o conteúdo original para UI, mas com override no payload para incluir contexto
+    await sendMessage(content, selectedModel, enhancedSystemPrompt, {
+      payloadContentOverride: finalUserContent,
+    });
     
     // Marcar como completo quando começar a streamar
     setTimeout(() => setThinkingStep('complete'), 100);

hooks/use-chat.ts

@@ -202,6 +202,9 @@ export function useChat() {
     systemPrompt?: string,
     options?: {
       onBeforeModelRequest?: (pendingMessages: Message[]) => Promise<void>;
+      // Quando presente, substitui SOMENTE o conteúdo da última mensagem do usuário
+      // no payload enviado ao LLM, mantendo a UI com o conteúdo original.
+      payloadContentOverride?: string;
     }
   ) => {
     setIsLoading(true);
@@ -257,12 +260,22 @@ export function useChat() {
     selectedMessages.push(lastUserMessage);
     totalTokens += estimateTokens(lastUserMessage.content);
 
-    const apiMessages = (systemPrompt 
+    // Constrói mensagens para payload, aplicando override apenas à última mensagem do usuário
+    const baseMessages = systemPrompt 
       ? [{ role: 'system', content: systemPrompt }, ...selectedMessages]
-      : selectedMessages).map(m => ({
+      : selectedMessages;
+
+    const apiMessages = baseMessages.map((m, idx) => {
+      // Detecta última mensagem do usuário
+      const isLastUser = m.role === 'user' && idx === baseMessages.length - 1;
+      const payloadContent = isLastUser && options?.payloadContentOverride
+        ? options.payloadContentOverride
+        : m.content;
+      return {
         role: m.role,
-        content: m.content // Send only content, not metadata (unless we want to inject it back)
+        content: payloadContent,
-      }));
+      };
+    });
 
     // [DEBUG INJECTION START]
     const systemMsg = apiMessages.find(m => m.role === 'system');

hooks/use-web-search.ts

@@ -119,6 +119,62 @@ export function useWebSearch() {
     }
   }, []);
 
+  /**
+   * Pipeline em duas etapas: metadados → scraping das top URLs
+   */
+  const smartSearchRag = useCallback(async (query: string, limit: number = 3): Promise<ScrapedContent[]> => {
+    if (!query || !query.trim()) {
+      return [];
+    }
+
+    setState(prev => ({
+      ...prev,
+      status: 'searching',
+      currentQuery: query,
+      error: null,
+      scrapedSources: [],
+    }));
+
+    try {
+      const settings = useSettingsStore.getState();
+      const searchConfig: SearchConfig = {
+        maxConcurrentTabs: settings.webSearch.maxConcurrentTabs,
+        totalSourcesLimit: settings.webSearch.totalSourcesLimit,
+        categories: settings.webSearch.categories,
+        userCustomSites: settings.webSearch.userCustomSites,
+        excludedDomains: settings.webSearch.excludedDomains,
+      };
+
+      // Etapa 1: metadados
+      setState(prev => ({ ...prev, status: 'searching' }));
+      const { metadata, contents } = await webSearchService.smartSearchRag(query, limit, searchConfig);
+
+      // Etapa 2: scraping
+      setState(prev => ({ ...prev, status: 'scraping' }));
+
+      // [DEBUG] metadados coletados
+      console.log('🔎 Metadados coletados:', metadata);
+
+      setState(prev => ({
+        ...prev,
+        status: 'completed',
+        scrapedSources: contents,
+        error: null,
+      }));
+
+      return contents;
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : 'Erro desconhecido no smartSearchRag';
+      setState(prev => ({
+        ...prev,
+        status: 'error',
+        error: errorMessage,
+        scrapedSources: [],
+      }));
+      throw error;
+    }
+  }, []);
+
   /**
    * Extrai conteúdo de uma URL
    */
@@ -169,6 +225,7 @@ export function useWebSearch() {
     ...state,
     setEnabled,
     search,
+    smartSearchRag,
     extractUrl,
     reset,
     clearCache,

services/webSearch.ts

@@ -17,6 +17,12 @@ export interface SearchConfig {
   excludedDomains: string[];
 }
 
+export interface SearchResultMetadata {
+  title: string;
+  url: string;
+  snippet: string;
+}
+
 interface CachedResult {
   results: ScrapedContent[];
   timestamp: number;
@@ -212,6 +218,43 @@ class WebSearchService {
     }
   }
 
+  /**
+   * Executa busca de metadados (sem scraping)
+   */
+  private async executeMetadataSearch(
+    query: string,
+    limit: number = 5,
+    searchConfig?: SearchConfig
+  ): Promise<SearchResultMetadata[]> {
+    try {
+      let rustConfig: any = undefined;
+      if (searchConfig) {
+        rustConfig = {
+          max_concurrent_tabs: searchConfig.maxConcurrentTabs,
+          total_sources_limit: searchConfig.totalSourcesLimit,
+          categories: searchConfig.categories.map(cat => ({
+            id: cat.id,
+            name: cat.name,
+            base_sites: cat.baseSites,
+            enabled: cat.enabled,
+          })),
+          user_custom_sites: searchConfig.userCustomSites,
+          excluded_domains: searchConfig.excludedDomains,
+        };
+      }
+
+      const results = await invoke<SearchResultMetadata[]>('search_web_metadata', {
+        query: query.trim(),
+        limit,
+        search_config: rustConfig,
+      });
+      return results || [];
+    } catch (error) {
+      console.error('Erro ao buscar metadados:', error);
+      throw error instanceof Error ? error : new Error('Falha ao buscar metadados');
+    }
+  }
+
   /**
    * Busca conteúdo na web com cache e rate limiting
    */
@@ -291,6 +334,32 @@ class WebSearchService {
     }
   }
 
+  /**
+   * Busca em duas etapas: metadados → scraping das melhores URLs
+   */
+  async smartSearchRag(
+    query: string,
+    limit: number = 3,
+    searchConfig?: SearchConfig
+  ): Promise<{ metadata: SearchResultMetadata[]; contents: ScrapedContent[] }> {
+    if (!query || !query.trim()) {
+      return { metadata: [], contents: [] };
+    }
+
+    const metas = await this.executeMetadataSearch(query, Math.max(limit * 2, 5), searchConfig);
+    const topUrls = metas.map(m => m.url).slice(0, limit);
+    if (topUrls.length === 0) {
+      return { metadata: metas, contents: [] };
+    }
+    try {
+      const contents = await invoke<ScrapedContent[]>('scrape_urls', { urls: topUrls });
+      return { metadata: metas, contents: contents || [] };
+    } catch (error) {
+      console.error('Erro ao fazer scraping em lote:', error);
+      return { metadata: metas, contents: [] };
+    }
+  }
+
   /**
    * Extrai conteúdo de uma URL específica
    */

src-tauri/src/lib.rs

@@ -19,7 +19,18 @@ mod scheduler_loop;
 mod sources_config;
 mod system_monitor;
 
-use web_scraper::{ScrapedContent, create_browser, search_and_scrape, search_and_scrape_with_config, scrape_url, SearchConfig};
+use web_scraper::{
+    ScrapedContent,
+    SearchResultMetadata,
+    create_browser,
+    search_and_scrape,
+    search_and_scrape_with_config,
+    scrape_url,
+    SearchConfig,
+    search_duckduckgo_metadata,
+    smart_search,
+    scrape_urls_bulk,
+};
 use headless_chrome::Browser;
 use scheduler::{SentinelTask, SchedulerService, SchedulerState, TaskAction};
 use sources_config::{SourcesConfig, load_sources_config, save_sources_config};
@@ -1232,6 +1243,56 @@ async fn extract_url_content(
         .map_err(|e| format!("Erro ao extrair conteúdo da URL: {}", e))
 }
 
+/// Busca metadados leves (título/URL/snippet) sem abrir páginas
+#[command]
+async fn search_web_metadata(
+    query: String,
+    limit: Option<usize>,
+    search_config: Option<SearchConfig>,
+) -> Result<Vec<SearchResultMetadata>, String> {
+    if query.trim().is_empty() {
+        return Err("Query não pode estar vazia".to_string());
+    }
+
+    let lim = limit.unwrap_or(5);
+
+    if let Some(config) = search_config {
+        // Usar smart_search para obter URLs e convertê-las em metadados simples
+        match smart_search(&query, &config).await {
+            Ok(mut urls) => {
+                urls.truncate(lim);
+                let metas = urls
+                    .into_iter()
+                    .map(|u| SearchResultMetadata { title: u.clone(), url: u, snippet: String::new() })
+                    .collect::<Vec<_>>();
+                Ok(metas)
+            }
+            Err(e) => Err(format!("Erro ao executar smart_search: {}", e)),
+        }
+    } else {
+        search_duckduckgo_metadata(&query, lim)
+            .await
+            .map_err(|e| format!("Erro ao buscar metadados: {}", e))
+    }
+}
+
+/// Faz scraping em lote de URLs fornecidas
+#[command]
+async fn scrape_urls(
+    urls: Vec<String>,
+    state: State<'_, BrowserState>,
+) -> Result<Vec<ScrapedContent>, String> {
+    if urls.is_empty() {
+        return Ok(Vec::new());
+    }
+
+    let browser = get_or_create_browser(state)?;
+
+    scrape_urls_bulk(urls, browser)
+        .await
+        .map_err(|e| format!("Erro ao extrair conteúdo das URLs: {}", e))
+}
+
 /// Reinicia o browser (útil se houver problemas)
 #[command]
 fn reset_browser(state: State<'_, BrowserState>) -> Result<(), String> {
@@ -1785,6 +1846,8 @@ pub fn run() {
         check_mcp_server_available,
         search_and_extract_content,
         extract_url_content,
+        search_web_metadata,
+        scrape_urls,
         reset_browser,
         force_kill_browser,
         export_chat_sessions,

src-tauri/src/web_scraper.rs

@@ -18,6 +18,14 @@ pub struct ScrapedContent {
     pub markdown: String,
 }
 
+/// Metadados de resultado de busca (leve, sem abrir página)
+#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]
+pub struct SearchResultMetadata {
+    pub title: String,
+    pub url: String,
+    pub snippet: String,
+}
+
 /// Categoria de busca com sites curados
 #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]
 pub struct SearchCategory {
@@ -124,6 +132,115 @@ pub async fn search_duckduckgo(query: &str, limit: usize) -> Result<Vec<String>>
     Ok(links)
 }
 
+/// Busca no DuckDuckGo retornando apenas metadados (título, URL, snippet)
+pub async fn search_duckduckgo_metadata(query: &str, limit: usize) -> Result<Vec<SearchResultMetadata>> {
+    let client = reqwest::Client::builder()
+        .timeout(Duration::from_secs(10))
+        .build()?;
+
+    let url = format!("https://html.duckduckgo.com/html/?q={}",
+        urlencoding::encode(query));
+
+    let user_agent = get_random_user_agent();
+    let res = client
+        .get(&url)
+        .header(USER_AGENT, user_agent)
+        .send()
+        .await?
+        .text()
+        .await?;
+
+    let mut results: Vec<SearchResultMetadata> = Vec::new();
+
+    {
+        let document = Html::parse_document(&res);
+
+        // Estruturas comuns no HTML do DuckDuckGo
+        let container_selectors = vec![
+            ".result",
+            ".web-result",
+            ".result__body",
+        ];
+        let title_selectors = vec![
+            ".result__a",
+            ".web-result__link",
+            "a.result__a",
+        ];
+        let snippet_selectors = vec![
+            ".result__snippet",
+            ".result__snippet.js-result-snippet",
+            ".web-result__snippet",
+        ];
+
+        for cont_sel in &container_selectors {
+            if results.len() >= limit { break; }
+            if let Ok(container) = Selector::parse(cont_sel) {
+                for node in document.select(&container) {
+                    if results.len() >= limit { break; }
+                    // Title + URL
+                    let mut found_url: Option<String> = None;
+                    let mut found_title: Option<String> = None;
+                    for tsel in &title_selectors {
+                        if let Ok(ts) = Selector::parse(tsel) {
+                            if let Some(a) = node.select(&ts).next() {
+                                if let Some(href) = a.value().attr("href") {
+                                    if let Some(real_url) = extract_real_url(href) {
+                                        found_url = clean_url(&real_url);
+                                    }
+                                }
+                                let text = a.text().collect::<Vec<_>>().join(" ").trim().to_string();
+                                if !text.is_empty() { found_title = Some(text); }
+                            }
+                        }
+                        if found_url.is_some() && found_title.is_some() { break; }
+                    }
+
+                    if found_url.is_none() { continue; }
+
+                    // Snippet
+                    let mut snippet_text: String = String::new();
+                    for ssel in &snippet_selectors {
+                        if let Ok(ss) = Selector::parse(ssel) {
+                            if let Some(s) = node.select(&ss).next() {
+                                let t = s.text().collect::<Vec<_>>().join(" ");
+                                let norm = t.split_whitespace().collect::<Vec<_>>().join(" ");
+                                if !norm.is_empty() { snippet_text = norm; break; }
+                            }
+                        }
+                    }
+
+                    let url_final = found_url.unwrap();
+                    if is_ad_or_tracker_url(&url_final) || url_final.is_empty() { continue; }
+
+                    results.push(SearchResultMetadata {
+                        title: found_title.unwrap_or_else(|| url_final.clone()),
+                        url: url_final,
+                        snippet: snippet_text,
+                    });
+
+                    if results.len() >= limit { break; }
+                }
+            }
+        }
+    }
+
+    // Se ainda vazio, tentar fallback simples: extrair todos os links conhecidos
+    if results.is_empty() {
+        let links = search_duckduckgo(query, limit).await?;
+        for l in links {
+            let url_clean = clean_url(&l).unwrap_or(l);
+            results.push(SearchResultMetadata {
+                title: url_clean.clone(),
+                url: url_clean,
+                snippet: String::new(),
+            });
+            if results.len() >= limit { break; }
+        }
+    }
+
+    Ok(results)
+}
+
 /// Extrai a URL real do redirecionamento do DuckDuckGo
 fn extract_real_url(ddg_redirect: &str) -> Option<String> {
     // DuckDuckGo usa formato: /l/?kh=-1&uddg=<URL_ENCODED>
@@ -436,6 +553,55 @@ pub async fn scrape_url(
     .map_err(|e| anyhow::anyhow!("Erro na task: {}", e))?
 }
 
+/// Extrai conteúdo de múltiplas URLs já definidas (bulk)
+pub async fn scrape_urls_bulk(
+    urls: Vec<String>,
+    browser: Arc<Browser>,
+) -> Result<Vec<ScrapedContent>> {
+    if urls.is_empty() { return Ok(Vec::new()); }
+    let concurrency = 5usize;
+    let semaphore = Arc::new(Semaphore::new(concurrency));
+    let mut handles = Vec::new();
+
+    for url in urls {
+        let browser_clone = browser.clone();
+        let permit = semaphore.clone().acquire_owned().await.unwrap();
+        let url_clone = url.clone();
+        let handle = tokio::task::spawn_blocking(move || {
+            let res = fetch_and_convert_sync(&browser_clone, &url_clone);
+            drop(permit);
+            res
+        });
+        handles.push(handle);
+    }
+
+    let mut results = Vec::new();
+    for h in handles {
+        match h.await {
+            Ok(Ok(content)) => {
+                let content_len = content.content.chars().count();
+                let md_len = content.markdown.chars().count();
+                if content_len < 200 && md_len < 200 {
+                    log::debug!("Descartado por conteúdo curto: {}", content.url);
+                } else {
+                    results.push(content);
+                }
+            }
+            Ok(Err(e)) => {
+                let msg = format!("{}", e);
+                if msg.contains("Timeout") || msg.contains("ERR_HTTP") {
+                    log::debug!("Ignorado (timeout/HTTP): {}", msg);
+                } else {
+                    log::warn!("Erro ao processar URL: {}", e);
+                }
+            }
+            Err(e) => log::warn!("Erro na task de scraping: {}", e),
+        }
+    }
+
+    Ok(results)
+}
+
 /// Extrai conteúdo de uma URL e converte para Markdown (versão síncrona)
 /// Retorna erro se timeout ou falha HTTP, mas não mata o processo
 fn fetch_and_convert_sync(browser: &Browser, url: &str) -> Result<ScrapedContent> {
@@ -534,7 +700,7 @@ fn fetch_and_convert_sync(browser: &Browser, url: &str) -> Result<ScrapedContent
     
     match readability::extractor::extract(&mut reader, &url_obj) {
         Ok(product) => {
-            let mut markdown = html2text::from_read(product.content.as_bytes(), 80);
+            let markdown = html2text::from_read(product.content.as_bytes(), 80);
             // Se o markdown for muito curto, significa que o readability pode ter falhado
             if markdown.trim().chars().count() < 400 {
                 if let Some(fallback) = extract_paragraph_fallback(url, &content) {