ultima antes do format

2025-11-22 01:59:12 -03:00
1 parent 4006a72eec
commit 4275295e4f
@@ -316,6 +316,34 @@ Ao responder sobre fatos atuais ou notícias, inicie mencionando explicitamente
      payloadContentOverride: finalUserContent,
    });
    
+    // Persistir fontes na mensagem do assistente (metadata)
+    if (scrapedSources.length > 0) {
+      // Aguardar um pouco para garantir que o sendMessage iniciou e criou a mensagem do assistente
+      // O sendMessage é async mas retorna void enquanto a stream acontece. 
+      // O setMessages dentro dele adiciona a mensagem do assistente.
+      
+      // Nota: Como sendMessage roda em background (streaming), não podemos garantir que a mensagem 
+      // do assistente já existe aqui imediatamente se não esperarmos.
+      // Mas o sendMessage faz `setMessages` com assistant vazio logo no início.
+      
+      // Vamos atualizar o estado usando callback para garantir que pegamos o mais recente
+      setMessages(prev => {
+        const newMessages = [...prev];
+        const lastMsgIndex = newMessages.length - 1;
+        if (lastMsgIndex >= 0 && newMessages[lastMsgIndex].role === 'assistant') {
+          const lastMsg = newMessages[lastMsgIndex];
+          newMessages[lastMsgIndex] = {
+            ...lastMsg,
+            metadata: {
+              ...(lastMsg.metadata || {}),
+              sources: scrapedSources
+            }
+          };
+        }
+        return newMessages;
+      });
+    }
+    
    // Marcar como completo quando começar a streamar
    setTimeout(() => setThinkingStep('complete'), 100);
  };
@@ -483,27 +511,32 @@ Ao responder sobre fatos atuais ou notícias, inicie mencionando explicitamente
              ) : (
                <div className="flex flex-col pb-4 max-w-3xl mx-auto w-full space-y-6">
                  {messages.map((msg, i) => {
-                    // Verificar se a mensagem anterior teve busca web
-                    const hasWebSources = i > 0 && 
-                      messages[i - 1].role === 'user' && 
-                      webSearch.scrapedSources.length > 0 &&
-                      webSearch.status === 'completed';
+                    // Verificar fontes nos metadados (preferencial)
+                    const msgSources = msg.metadata?.sources || [];
+                    const hasWebSources = msgSources.length > 0;
                    
+                    // Verificar se devemos mostrar o progresso em tempo real
+                    // Apenas para a interação atual (quando isLoading é true ou acabou de terminar)
+                    const isCurrentInteraction = i === messages.length - (isLoading ? 2 : 1) || i === messages.length - 1;
+                    const showRealtimeProgress = 
+                      msg.role === 'user' &&
+                      isCurrentInteraction &&
+                      (webSearch.status === 'searching' || 
+                       webSearch.status === 'scraping' || 
+                       webSearch.status === 'completed' || 
+                       webSearch.status === 'error') &&
+                       webSearch.currentQuery; // Só mostra se tiver query
+
                    return (
                      <div key={i}>
                        <ChatMessage message={msg} />
-                        {(msg.role === 'user') && (
-                          (webSearch.status === 'searching' || 
-                           webSearch.status === 'scraping' || 
-                           webSearch.status === 'completed' || 
-                           webSearch.status === 'error') && (
+                        {(msg.role === 'user') && showRealtimeProgress && (
                            <SearchProgress
                              status={webSearch.status}
                              query={webSearch.currentQuery}
                              sources={webSearch.scrapedSources}
                              error={webSearch.error}
                            />
-                          )
                        )}
                        {/* Mostrar fontes após resposta do assistente que usou web search */}
                        {msg.role === 'assistant' && hasWebSources && (
@@ -513,7 +546,7 @@ Ao responder sobre fatos atuais ou notícias, inicie mencionando explicitamente
                                Fontes consultadas:
                              </p>
                              <div className="flex gap-2 overflow-x-auto pb-2 scrollbar-thin scrollbar-thumb-muted scrollbar-track-transparent">
-                                {webSearch.scrapedSources.map((source, idx) => (
+                                {msgSources.map((source: ScrapedContent, idx: number) => (
                                  <a
                                    key={idx}
                                    href={source.url}
@@ -0,0 +1,154 @@
+'use client';
+
+import { Card } from '@/components/ui/card';
+import { ExternalLink } from 'lucide-react';
+import { ScrapedContent } from '@/services/webSearch';
+import { useRef, useState } from 'react';
+import { cn } from '@/lib/utils';
+
+interface SourcesCarouselProps {
+  sources: ScrapedContent[];
+}
+
+export function SourcesCarousel({ sources }: SourcesCarouselProps) {
+  if (!sources || sources.length === 0) return null;
+
+  const scrollContainerRef = useRef<HTMLDivElement>(null);
+  const [isDragging, setIsDragging] = useState(false);
+  const [startX, setStartX] = useState(0);
+  const [scrollLeft, setScrollLeft] = useState(0);
+  const [hasMoved, setHasMoved] = useState(false);
+
+  // Handlers para mouse drag
+  const handleMouseDown = (e: React.MouseEvent) => {
+    if (!scrollContainerRef.current) return;
+    setIsDragging(true);
+    setStartX(e.pageX - scrollContainerRef.current.offsetLeft);
+    setScrollLeft(scrollContainerRef.current.scrollLeft);
+    setHasMoved(false);
+  };
+
+  const handleMouseMove = (e: React.MouseEvent) => {
+    if (!isDragging || !scrollContainerRef.current) return;
+    e.preventDefault();
+    const x = e.pageX - scrollContainerRef.current.offsetLeft;
+    const walk = (x - startX) * 1.5; // Velocidade do scroll
+    scrollContainerRef.current.scrollLeft = scrollLeft - walk;
+    
+    // Se moveu mais de 3px, marca como movimento (não clique)
+    if (Math.abs(walk) > 3) {
+      setHasMoved(true);
+    }
+  };
+
+  const handleMouseUp = () => {
+    setIsDragging(false);
+    // Resetar após um pequeno delay
+    setTimeout(() => setHasMoved(false), 150);
+  };
+
+  const handleMouseLeave = () => {
+    setIsDragging(false);
+    setTimeout(() => setHasMoved(false), 150);
+  };
+
+  // Handlers para touch drag (mobile)
+  const handleTouchStart = (e: React.TouchEvent) => {
+    if (!scrollContainerRef.current) return;
+    setIsDragging(true);
+    setStartX(e.touches[0].pageX - scrollContainerRef.current.offsetLeft);
+    setScrollLeft(scrollContainerRef.current.scrollLeft);
+    setHasMoved(false);
+  };
+
+  const handleTouchMove = (e: React.TouchEvent) => {
+    if (!isDragging || !scrollContainerRef.current) return;
+    const x = e.touches[0].pageX - scrollContainerRef.current.offsetLeft;
+    const walk = (x - startX) * 1.5;
+    scrollContainerRef.current.scrollLeft = scrollLeft - walk;
+    
+    if (Math.abs(walk) > 3) {
+      setHasMoved(true);
+    }
+  };
+
+  const handleTouchEnd = () => {
+    setIsDragging(false);
+    setTimeout(() => setHasMoved(false), 150);
+  };
+
+  // Prevenir navegação do link durante drag
+  const handleLinkClick = (e: React.MouseEvent<HTMLAnchorElement>) => {
+    if (hasMoved || isDragging) {
+      e.preventDefault();
+      e.stopPropagation();
+    }
+  };
+
+  return (
+    <div className="w-full max-w-3xl mx-auto mb-4 mt-2">
+      <h3 className="text-xs font-semibold text-muted-foreground uppercase tracking-wider mb-3 px-1">
+        Fontes Analisadas
+      </h3>
+      
+      <div 
+        ref={scrollContainerRef}
+        className={cn(
+          "flex gap-3 overflow-x-auto pb-4 scrollbar-hide -mx-4 px-4 sm:mx-0 sm:px-0 mask-fade-sides",
+          "cursor-grab active:cursor-grabbing select-none touch-pan-x",
+          isDragging && "cursor-grabbing"
+        )}
+        onMouseDown={handleMouseDown}
+        onMouseMove={handleMouseMove}
+        onMouseUp={handleMouseUp}
+        onMouseLeave={handleMouseLeave}
+        onTouchStart={handleTouchStart}
+        onTouchMove={handleTouchMove}
+        onTouchEnd={handleTouchEnd}
+        style={{ 
+          scrollBehavior: isDragging ? 'auto' : 'smooth',
+          WebkitOverflowScrolling: 'touch' // Smooth scrolling no iOS
+        }}
+      >
+        {sources.map((source, idx) => {
+            const hostname = new URL(source.url).hostname.replace('www.', '');
+            const faviconUrl = `https://www.google.com/s2/favicons?domain=${hostname}&sz=32`;
+            
+            return (
+                <a 
+                    key={idx} 
+                    href={source.url} 
+                    target="_blank" 
+                    rel="noopener noreferrer"
+                    onClick={handleLinkClick}
+                    className="flex-none w-40 group transition-all hover:-translate-y-1 block"
+                    style={{ pointerEvents: (isDragging || hasMoved) ? 'none' : 'auto' }}
+                >
+                    <Card className="h-24 p-3 flex flex-col justify-between hover:shadow-md hover:border-primary/20 transition-colors bg-card/50 backdrop-blur-sm cursor-pointer overflow-hidden">
+                        <div className="flex items-start justify-between min-h-0 flex-shrink-0">
+                             <img 
+                                src={faviconUrl} 
+                                alt="" 
+                                className="w-4 h-4 rounded-sm opacity-70 group-hover:opacity-100 transition-opacity flex-shrink-0" 
+                                onError={(e) => (e.target as HTMLImageElement).style.display = 'none'}
+                            />
+                            <ExternalLink className="w-3 h-3 text-muted-foreground opacity-0 group-hover:opacity-100 transition-opacity flex-shrink-0 ml-1" />
+                        </div>
+                        
+                        <div className="space-y-1 min-w-0 flex-1 overflow-hidden flex flex-col justify-end">
+                            <div className="text-xs font-medium leading-tight line-clamp-2 text-card-foreground/90 group-hover:text-primary transition-colors break-words overflow-hidden text-ellipsis">
+                                {source.title}
+                            </div>
+                            <div className="text-[10px] text-muted-foreground truncate overflow-hidden text-ellipsis">
+                                {hostname}
+                            </div>
+                        </div>
+                    </Card>
+                </a>
+            );
+        })}
+      </div>
+    </div>
+  );
+}
+
@@ -0,0 +1,97 @@
+'use client';
+
+import { usePythonBootstrap } from '@/hooks/use-python-bootstrap';
+import { Progress } from '@/components/ui/progress';
+import { Loader2, XCircle } from 'lucide-react';
+import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card';
+
+const STAGE_LABELS: Record<string, string> = {
+  checking: 'Verificando',
+  downloading: 'Baixando',
+  extracting: 'Extraindo',
+  creating_venv: 'Criando ambiente virtual',
+  installing_pip: 'Instalando pip',
+  copying_scripts: 'Copiando scripts',
+  installing_deps: 'Instalando dependências',
+  validating: 'Validando',
+  done: 'Concluído',
+};
+
+export function PythonBootstrapProgress() {
+  const bootstrap = usePythonBootstrap();
+
+  // Se estiver completo e sem erro, não mostrar nada
+  if (bootstrap.isComplete && !bootstrap.error) {
+    return null;
+  }
+
+  // Se houver erro, mostrar alerta
+  if (bootstrap.error) {
+    return (
+      <div className="fixed inset-0 z-50 flex items-center justify-center bg-background/80 backdrop-blur-sm">
+        <div className="w-full max-w-md mx-4">
+          <Card className="border-destructive">
+            <CardHeader>
+              <CardTitle className="flex items-center gap-2 text-destructive">
+                <XCircle className="h-5 w-5" />
+                Erro no Bootstrap do Python
+              </CardTitle>
+              <CardDescription>
+                O aplicativo não pode funcionar sem o runtime Python
+              </CardDescription>
+            </CardHeader>
+            <CardContent className="space-y-4">
+              <div className="rounded-md bg-destructive/10 p-3">
+                <p className="text-sm text-destructive font-medium">{bootstrap.error}</p>
+              </div>
+              <p className="text-sm text-muted-foreground">
+                Por favor, reinicie o aplicativo ou verifique os logs para mais detalhes.
+              </p>
+            </CardContent>
+          </Card>
+        </div>
+      </div>
+    );
+  }
+
+  // Mostrar progresso durante bootstrap
+  const stageLabel = STAGE_LABELS[bootstrap.stage] || bootstrap.stage;
+  const progressValue = bootstrap.progress ?? 0;
+
+  return (
+    <div className="fixed inset-0 z-50 flex items-center justify-center bg-background/80 backdrop-blur-sm">
+      <div className="w-full max-w-md mx-4">
+        <Card>
+          <CardHeader>
+            <CardTitle className="flex items-center gap-3">
+              <Loader2 className="h-5 w-5 animate-spin text-primary" />
+              Inicializando Python Runtime
+            </CardTitle>
+            <CardDescription>
+              Configurando ambiente Python para scraping web
+            </CardDescription>
+          </CardHeader>
+          <CardContent className="space-y-4">
+            <div className="space-y-2">
+              <div className="flex justify-between text-sm">
+                <span className="text-muted-foreground">{stageLabel}</span>
+                {bootstrap.progress !== null && (
+                  <span className="text-muted-foreground">{Math.round(progressValue * 100)}%</span>
+                )}
+              </div>
+              
+              {bootstrap.progress !== null && (
+                <Progress value={progressValue * 100} className="h-2" />
+              )}
+              
+              <p className="text-sm text-muted-foreground mt-2">
+                {bootstrap.message}
+              </p>
+            </div>
+          </CardContent>
+        </Card>
+      </div>
+    </div>
+  );
+}
+
@@ -0,0 +1,89 @@
+"use client";
+
+import { motion, AnimatePresence } from "framer-motion";
+import { Loader2, Search, FileText } from "lucide-react";
+import { useWebSearch } from "@/hooks/use-web-search";
+
+export function ScrapingProgressIndicator() {
+  const webSearch = useWebSearch();
+  
+  // Não mostrar se idle ou completed
+  if (webSearch.status === 'idle' || webSearch.status === 'completed') {
+    return null;
+  }
+  
+  return (
+    <AnimatePresence>
+      <motion.div 
+        initial={{ opacity: 0, y: 20, scale: 0.9 }}
+        animate={{ opacity: 1, y: 0, scale: 1 }}
+        exit={{ opacity: 0, y: 20, scale: 0.9 }}
+        transition={{ duration: 0.2, ease: "easeOut" }}
+        className="fixed bottom-24 right-6 bg-card border border-primary/20 rounded-lg p-4 shadow-lg backdrop-blur-sm z-50 min-w-[280px]"
+      >
+        <div className="flex items-start gap-3">
+          {/* Icon animado */}
+          <div className="flex-shrink-0 mt-0.5">
+            {webSearch.status === 'searching' && (
+              <Search className="w-5 h-5 text-primary animate-pulse" />
+            )}
+            {webSearch.status === 'scraping' && (
+              <FileText className="w-5 h-5 text-primary animate-pulse" />
+            )}
+            {webSearch.status === 'error' && (
+              <div className="w-5 h-5 rounded-full bg-destructive/20 flex items-center justify-center">
+                <span className="text-destructive text-xs font-bold">!</span>
+              </div>
+            )}
+          </div>
+          
+          {/* Content */}
+          <div className="flex-1 space-y-2">
+            <div className="flex items-center gap-2">
+              <Loader2 className="w-4 h-4 animate-spin text-primary" />
+              <p className="text-sm font-medium">
+                {webSearch.status === 'searching' && 'Buscando fontes...'}
+                {webSearch.status === 'scraping' && `Analisando fontes (${webSearch.scrapedSources.length})...`}
+                {webSearch.status === 'error' && 'Erro na busca'}
+              </p>
+            </div>
+            
+            {/* Mostrar query atual se disponível */}
+            {webSearch.currentQuery && webSearch.status !== 'error' && (
+              <p className="text-xs text-muted-foreground line-clamp-1">
+                {webSearch.currentQuery}
+              </p>
+            )}
+            
+            {/* Mostrar erro se houver */}
+            {webSearch.error && webSearch.status === 'error' && (
+              <p className="text-xs text-destructive line-clamp-2">
+                {webSearch.error}
+              </p>
+            )}
+            
+            {/* Barra de progresso simulada */}
+            {webSearch.status !== 'error' && (
+              <div className="w-full h-1 bg-muted rounded-full overflow-hidden">
+                <motion.div
+                  className="h-full bg-primary"
+                  initial={{ width: "0%" }}
+                  animate={{
+                    width: webSearch.status === 'searching' ? "40%" : "80%"
+                  }}
+                  transition={{
+                    duration: 2,
+                    ease: "easeInOut"
+                  }}
+                />
+              </div>
+            )}
+          </div>
+        </div>
+      </motion.div>
+    </AnimatePresence>
+  );
+}
+
+
+
@@ -0,0 +1,103 @@
+'use client';
+
+import { useState, useEffect } from 'react';
+import { listen } from '@tauri-apps/api/event';
+
+export interface PythonBootstrapProgress {
+  stage: string;
+  message: string;
+  progress: number | null;
+}
+
+export interface PythonBootstrapState {
+  stage: string;
+  message: string;
+  progress: number | null;
+  isComplete: boolean;
+  error: string | null;
+}
+
+export function usePythonBootstrap() {
+  const [state, setState] = useState<PythonBootstrapState>({
+    stage: 'checking',
+    message: 'Verificando runtime Python...',
+    progress: null,
+    isComplete: false,
+    error: null,
+  });
+
+  useEffect(() => {
+    // Verificar status inicial
+    const checkInitialStatus = async () => {
+      try {
+        const { invoke } = await import('@tauri-apps/api/core');
+        const status = await invoke<{ ready: boolean; error: string | null }>('check_python_runtime_status');
+        
+        if (status.ready) {
+          setState({
+            stage: 'done',
+            message: 'Runtime Python pronto',
+            progress: 1.0,
+            isComplete: true,
+            error: null,
+          });
+        } else if (status.error) {
+          setState(prev => ({
+            ...prev,
+            error: status.error || 'Erro desconhecido',
+            isComplete: false,
+          }));
+        }
+      } catch (error) {
+        console.error('Erro ao verificar status inicial do Python:', error);
+      }
+    };
+
+    checkInitialStatus();
+
+    // Escutar eventos de progresso
+    const progressUnlisten = listen<PythonBootstrapProgress>('python-bootstrap-progress', (event) => {
+      const payload = event.payload;
+      setState(prev => ({
+        ...prev,
+        stage: payload.stage,
+        message: payload.message,
+        progress: payload.progress ?? prev.progress,
+        error: null,
+      }));
+    });
+
+    // Escutar evento de conclusão
+    const completeUnlisten = listen('python-bootstrap-complete', () => {
+      setState(prev => ({
+        ...prev,
+        stage: 'done',
+        message: 'Runtime Python pronto',
+        progress: 1.0,
+        isComplete: true,
+        error: null,
+      }));
+    });
+
+    // Escutar evento de erro
+    const errorUnlisten = listen<string>('python-bootstrap-error', (event) => {
+      setState(prev => ({
+        ...prev,
+        error: event.payload || 'Erro desconhecido durante o bootstrap',
+        isComplete: false,
+      }));
+    });
+
+    return () => {
+      progressUnlisten.then(unlisten => unlisten());
+      completeUnlisten.then(unlisten => unlisten());
+      errorUnlisten.then(unlisten => unlisten());
+    };
+  }, []);
+
+  return state;
+}
+
+
+
+
@@ -0,0 +1,4 @@
+curl_cffi>=0.5.10
+beautifulsoup4>=4.12.0
+lxml>=4.9.0
+html2text>=2024.1.0
@@ -0,0 +1,499 @@
+#!/usr/bin/env python3
+"""
+Script Python para scraping web com curl_cffi + BeautifulSoup
+Pipeline: curl_cffi (Request HTTP) -> BeautifulSoup (Parsing) -> html2text (Formatação)
+Comunica via JSON stdin/stdout com o backend Rust
+"""
+
+import sys
+import json
+import time
+import logging
+import traceback
+from typing import List, Dict, Optional
+from urllib.parse import urljoin, urlparse, quote
+import random
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+# Configuração CRÍTICA de Encoding para Windows
+try:
+    sys.stdout.reconfigure(encoding='utf-8')
+    sys.stderr.reconfigure(encoding='utf-8')
+except (AttributeError, Exception):
+    pass
+
+# Configuração de Logging para STDERR (Não polui o STDOUT)
+logging.basicConfig(
+    level=logging.INFO,
+    format='[PYTHON LOG] %(message)s',
+    stream=sys.stderr
+)
+
+def print_json(data):
+    """Função auxiliar para garantir output limpo em JSON"""
+    try:
+        print(json.dumps(data, ensure_ascii=False))
+        sys.stdout.flush()
+    except Exception as e:
+        try:
+            print(json.dumps({"type": "error", "message": f"Failed to format output: {str(e)}"}))
+            sys.stdout.flush()
+        except:
+            pass
+
+# Tentar importar dependências
+CURL_CFFI_AVAILABLE = False
+BS4_AVAILABLE = False
+HTML2TEXT_AVAILABLE = False
+
+try:
+    from curl_cffi import requests
+    CURL_CFFI_AVAILABLE = True
+except ImportError as e:
+    logging.error(f"curl_cffi não disponível: {e}")
+
+try:
+    from bs4 import BeautifulSoup
+    BS4_AVAILABLE = True
+except ImportError as e:
+    logging.error(f"BeautifulSoup4 não disponível: {e}")
+
+try:
+    import html2text
+    HTML2TEXT_AVAILABLE = True
+except ImportError as e:
+    logging.error(f"html2text não disponível: {e}")
+
+# Pool de User Agents para parecer mais humano
+USER_AGENTS = [
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/121.0",
+]
+
+def get_random_user_agent():
+    """Retorna um user agent aleatório"""
+    return random.choice(USER_AGENTS)
+
+def clean_url(url: str) -> str:
+    """Remove fragmentos e parâmetros de rastreamento da URL"""
+    try:
+        parsed = urlparse(url)
+        return f"{parsed.scheme}://{parsed.netloc}{parsed.path}"
+    except:
+        return url
+
+def extract_domain(url: str) -> str:
+    """Extrai o domínio de uma URL"""
+    try:
+        parsed = urlparse(url)
+        return parsed.netloc
+    except:
+        return ""
+
+def is_ad_or_tracker_url(url: str) -> bool:
+    """Verifica se uma URL é de anúncio ou rastreador"""
+    trackers = ['analytics', 'google-analytics', 'doubleclick', 'facebook.com/tr', 'ads.', 'banner', 'tracking']
+    return any(tracker in url.lower() for tracker in trackers)
+
+def is_domain_blocked(url: str, excluded_domains: List[str]) -> bool:
+    """Verifica se o domínio da URL está na lista de exclusão"""
+    domain = extract_domain(url)
+    return any(excluded in domain for excluded in excluded_domains)
+
+def scrape_url(url: str, visible: bool = False) -> Dict:
+    """Função principal de scraping com curl_cffi + BeautifulSoup"""
+    
+    # Verificar dependências
+    if not CURL_CFFI_AVAILABLE:
+        return {
+            "type": "error",
+            "message": "curl_cffi não está instalado"
+        }
+    
+    if not BS4_AVAILABLE:
+        return {
+            "type": "error",
+            "message": "beautifulsoup4 não está instalado"
+        }
+    
+    if not HTML2TEXT_AVAILABLE:
+        return {
+            "type": "error",
+            "message": "html2text não está instalado"
+        }
+    
+    try:
+        logging.info(f"Iniciando requisição para: {url}")
+        
+        # Criar headers com curl_cffi (impersonate navegador)
+        headers = {
+            "User-Agent": get_random_user_agent(),
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+            "Accept-Language": "pt-BR,pt;q=0.9,en;q=0.8",
+            "Accept-Encoding": "gzip, deflate",
+            "DNT": "1",
+            "Connection": "keep-alive",
+            "Upgrade-Insecure-Requests": "1",
+        }
+        
+        # Executar requisição com timeout
+        try:
+            response = requests.get(
+                url,
+                headers=headers,
+                timeout=15,
+                impersonate="chrome120",
+                allow_redirects=True,
+                follow_redirects=True
+            )
+            response.raise_for_status()
+        except Exception as e:
+            logging.error(f"Erro na requisição HTTP: {e}")
+            return {
+                "type": "error",
+                "message": f"Failed to fetch URL: {str(e)}"
+            }
+        
+        # Verificar se recebemos conteúdo HTML válido
+        if len(response.content) < 100:
+            return {
+                "type": "error",
+                "message": "Resposta muito curta (< 100 bytes)"
+            }
+        
+        logging.info(f"Resposta recebida: {len(response.content)} bytes")
+        
+        # Parse com BeautifulSoup
+        try:
+            soup = BeautifulSoup(response.content, 'html.parser')
+        except Exception as e:
+            logging.error(f"Erro ao fazer parse HTML: {e}")
+            return {
+                "type": "error",
+                "message": f"Failed to parse HTML: {str(e)}"
+            }
+        
+        # Extrair título
+        title = "Sem título"
+        title_tag = soup.find('meta', property='og:title') or soup.find('meta', {'name': 'title'}) or soup.find('title')
+        if title_tag:
+            title = title_tag.get('content') or title_tag.string or title
+        
+        # Remover scripts e styles
+        for script in soup(["script", "style", "meta", "link", "noscript"]):
+            script.decompose()
+        
+        # Remover elementos desnecessários
+        for elem in soup.find_all(["nav", "header", "footer", "aside", "ads"]):
+            elem.decompose()
+        
+        # Extrair conteúdo principal
+        main_content = None
+        for selector in ['article', 'main', '[role="main"]', '.content', '.post', '.entry']:
+            if selector.startswith('['):
+                main_content = soup.select_one(selector)
+            else:
+                main_content = soup.find(selector)
+            if main_content and len(main_content.get_text(strip=True)) > 200:
+                break
+        
+        # Fallback: usar body
+        if not main_content or len(main_content.get_text(strip=True)) < 200:
+            main_content = soup.find('body') or soup
+        
+        # Obter HTML do conteúdo principal
+        html_content = str(main_content) if main_content else str(soup)
+        
+        # Converter para Markdown usando html2text
+        try:
+            h = html2text.HTML2Text()
+            h.ignore_links = False
+            h.ignore_images = False
+            h.body_width = 0
+            markdown_content = h.handle(html_content)
+        except Exception as e:
+            logging.warning(f"Erro ao converter para markdown, usando texto plano: {e}")
+            markdown_content = main_content.get_text(separator="\n") if main_content else response.text
+        
+        # Limpeza final
+        lines = [line.strip() for line in markdown_content.split('\n') if line.strip()]
+        final_content = '\n'.join(lines)
+        
+        # Validação de conteúdo
+        if len(final_content) < 100:
+            return {
+                "type": "error",
+                "message": f"Conteúdo insuficiente ({len(final_content)} chars)"
+            }
+        
+        logging.info(f"Sucesso! {len(final_content)} chars extraídos")
+        
+        return {
+            "type": "success",
+            "url": clean_url(url),
+            "title": title[:500],
+            "content": final_content[:30000],
+            "markdown": final_content[:30000]
+        }
+    
+    except Exception as e:
+        logging.error(f"Erro no scraping: {traceback.format_exc()}")
+        return {
+            "type": "error",
+            "message": str(e)
+        }
+
+def search_duckduckgo(query: str, limit: int = 5) -> List[str]:
+    """Realiza busca no DuckDuckGo e retorna URLs"""
+    try:
+        if not CURL_CFFI_AVAILABLE or not BS4_AVAILABLE:
+            logging.error("Dependências não disponíveis")
+            return []
+        
+        logging.info(f"Buscando no DuckDuckGo: {query}")
+        
+        headers = {
+            "User-Agent": get_random_user_agent(),
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "Accept-Language": "pt-BR,pt;q=0.9,en;q=0.8",
+        }
+        
+        search_url = f"https://duckduckgo.com/html/?q={quote(query)}"
+        response = requests.get(
+            search_url,
+            headers=headers,
+            timeout=15,
+            impersonate="chrome120",
+            allow_redirects=True
+        )
+        response.raise_for_status()
+        
+        soup = BeautifulSoup(response.content, 'html.parser')
+        
+        urls = []
+        # DuckDuckGo HTML retorna resultados em links diretos
+        for result in soup.find_all('a', {'class': 'result__a'}):
+            href = result.get('href')
+            if href and not is_ad_or_tracker_url(href):
+                urls.append(href)
+                if len(urls) >= limit:
+                    break
+        
+        logging.info(f"Encontradas {len(urls)} URLs")
+        return urls
+    
+    except Exception as e:
+        logging.error(f"Erro na busca DuckDuckGo: {e}")
+        return []
+
+def smart_search_impl(query: str, config: Dict) -> List[str]:
+    """Implementa busca inteligente com categorias e filtros"""
+    try:
+        logging.info(f"Smart search iniciado: {query}")
+        
+        total_limit = config.get('total_sources_limit', 15)
+        categories = config.get('categories', [])
+        user_custom_sites = config.get('user_custom_sites', [])
+        excluded_domains = config.get('excluded_domains', [])
+        
+        all_urls = []
+        
+        # 1. Busca geral no DuckDuckGo
+        general_urls = search_duckduckgo(query, total_limit // 2)
+        all_urls.extend(general_urls)
+        
+        # 2. Busca por categorias habilitadas
+        for category in categories:
+            if not category.get('enabled', False):
+                continue
+            
+            base_sites = category.get('base_sites', [])
+            for site in base_sites[:2]:  # Limitar a 2 sites por categoria
+                site_query = f"site:{site} {query}"
+                site_urls = search_duckduckgo(site_query, 3)
+                all_urls.extend(site_urls)
+        
+        # 3. Busca em sites personalizados
+        for custom_site in user_custom_sites[:3]:
+            custom_query = f"site:{custom_site} {query}"
+            custom_urls = search_duckduckgo(custom_query, 2)
+            all_urls.extend(custom_urls)
+        
+        # 4. Filtrar domínios excluídos e remover duplicatas
+        filtered_urls = []
+        seen = set()
+        for url in all_urls:
+            if url not in seen and not is_domain_blocked(url, excluded_domains):
+                filtered_urls.append(url)
+                seen.add(url)
+                if len(filtered_urls) >= total_limit:
+                    break
+        
+        logging.info(f"Smart search concluído: {len(filtered_urls)} URLs")
+        return filtered_urls
+    
+    except Exception as e:
+        logging.error(f"Erro no smart search: {e}")
+        return []
+
+def scrape_urls_bulk_impl(urls: List[str]) -> List[Dict]:
+    """Scraping em paralelo de múltiplas URLs"""
+    try:
+        logging.info(f"Scraping bulk iniciado: {len(urls)} URLs")
+        
+        results = []
+        
+        def scrape_with_logging(url):
+            try:
+                result = scrape_url(url, visible=False)
+                if result.get('type') == 'success':
+                    logging.info(f"✓ {url}")
+                else:
+                    logging.warning(f"✗ {url}: {result.get('message', 'unknown error')}")
+                return result
+            except Exception as e:
+                logging.error(f"✗ {url}: {e}")
+                return {"type": "error", "message": str(e), "url": url}
+        
+        # Scraping paralelo com até 5 workers
+        with ThreadPoolExecutor(max_workers=5) as executor:
+            future_to_url = {executor.submit(scrape_with_logging, url): url for url in urls}
+            for future in as_completed(future_to_url):
+                try:
+                    result = future.result()
+                    if result.get('type') == 'success':
+                        results.append(result)
+                except Exception as e:
+                    logging.error(f"Erro no future: {e}")
+        
+        logging.info(f"Scraping bulk concluído: {len(results)} sucessos")
+        return results
+    
+    except Exception as e:
+        logging.error(f"Erro no scraping bulk: {e}")
+        return []
+
+def main():
+    """Entry point do script - lê comandos JSON do stdin"""
+    try:
+        # Ler JSON do stdin
+        input_data = sys.stdin.read().strip()
+        
+        if not input_data:
+            print_json({"type": "error", "message": "No input provided"})
+            return
+        
+        # Parse JSON
+        try:
+            command_data = json.loads(input_data)
+        except json.JSONDecodeError as e:
+            logging.error(f"Erro ao fazer parse do JSON: {e}")
+            print_json({"type": "error", "message": f"Invalid JSON: {str(e)}"})
+            return
+        
+        command = command_data.get("command")
+        logging.info(f"Comando recebido: {command}")
+        
+        # Dispatch de comandos
+        if command == "scrape":
+            url = command_data.get("url")
+            visible = command_data.get("visible", False)
+            
+            if not url:
+                print_json({"type": "error", "message": "URL missing"})
+                return
+            
+            result = scrape_url(url, visible)
+            print_json(result)
+        
+        elif command == "search_duckduckgo":
+            query = command_data.get("query")
+            limit = command_data.get("limit", 5)
+            
+            if not query:
+                print_json({"type": "error", "message": "Query missing"})
+                return
+            
+            urls = search_duckduckgo(query, limit)
+            print_json({"type": "success", "urls": urls})
+        
+        elif command == "smart_search":
+            query = command_data.get("query")
+            config = command_data.get("config", {})
+            
+            if not query:
+                print_json({"type": "error", "message": "Query missing"})
+                return
+            
+            urls = smart_search_impl(query, config)
+            print_json({"type": "success", "urls": urls})
+        
+        elif command == "search_and_scrape":
+            query = command_data.get("query")
+            config = command_data.get("config", {})
+            
+            if not query:
+                print_json({"type": "error", "message": "Query missing"})
+                return
+            
+            # Primeiro faz smart_search para obter URLs
+            urls = smart_search_impl(query, config)
+            
+            if not urls:
+                print_json({"type": "success", "results": []})
+                return
+            
+            # Depois faz scraping das URLs
+            results = scrape_urls_bulk_impl(urls)
+            print_json({"type": "success", "results": results})
+        
+        elif command == "scrape_urls_bulk":
+            urls = command_data.get("urls", [])
+            
+            if not urls:
+                print_json({"type": "error", "message": "URLs missing"})
+                return
+            
+            results = scrape_urls_bulk_impl(urls)
+            print_json({"type": "success", "results": results})
+        
+        elif command == "check":
+            # Verificar dependências
+            missing = []
+            if not CURL_CFFI_AVAILABLE:
+                missing.append("curl_cffi")
+            if not BS4_AVAILABLE:
+                missing.append("beautifulsoup4")
+            if not HTML2TEXT_AVAILABLE:
+                missing.append("html2text")
+            
+            if missing:
+                print_json({"type": "error", "message": f"Missing: {', '.join(missing)}"})
+            else:
+                print_json({"type": "success", "message": "All dependencies available"})
+        
+        else:
+            print_json({"type": "error", "message": f"Unknown command: {command}"})
+    
+    except Exception as e:
+        logging.error(f"Erro no main: {traceback.format_exc()}")
+        print_json({"type": "error", "message": f"Critical error: {str(e)}"})
+
+if __name__ == "__main__":
+    try:
+        main()
+    except KeyboardInterrupt:
+        print_json({"type": "error", "message": "Script interrupted"})
+        sys.exit(1)
+    except Exception as e:
+        try:
+            logging.error(f"Unhandled exception: {traceback.format_exc()}")
+            print_json({"type": "error", "message": f"Unhandled exception: {str(e)}"})
+        except:
+            try:
+                print(json.dumps({"type": "error", "message": "Critical failure"}))
+            except:
+                pass
+        sys.exit(1)