ultima antes do format

src-tauri/scripts/requirements.txt

@@ -0,0 +1,4 @@
+curl_cffi>=0.5.10
+beautifulsoup4>=4.12.0
+lxml>=4.9.0
+html2text>=2024.1.0

src-tauri/scripts/scraper.py

@@ -0,0 +1,499 @@
+#!/usr/bin/env python3
+"""
+Script Python para scraping web com curl_cffi + BeautifulSoup
+Pipeline: curl_cffi (Request HTTP) -> BeautifulSoup (Parsing) -> html2text (Formatação)
+Comunica via JSON stdin/stdout com o backend Rust
+"""
+
+import sys
+import json
+import time
+import logging
+import traceback
+from typing import List, Dict, Optional
+from urllib.parse import urljoin, urlparse, quote
+import random
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+# Configuração CRÍTICA de Encoding para Windows
+try:
+    sys.stdout.reconfigure(encoding='utf-8')
+    sys.stderr.reconfigure(encoding='utf-8')
+except (AttributeError, Exception):
+    pass
+
+# Configuração de Logging para STDERR (Não polui o STDOUT)
+logging.basicConfig(
+    level=logging.INFO,
+    format='[PYTHON LOG] %(message)s',
+    stream=sys.stderr
+)
+
+def print_json(data):
+    """Função auxiliar para garantir output limpo em JSON"""
+    try:
+        print(json.dumps(data, ensure_ascii=False))
+        sys.stdout.flush()
+    except Exception as e:
+        try:
+            print(json.dumps({"type": "error", "message": f"Failed to format output: {str(e)}"}))
+            sys.stdout.flush()
+        except:
+            pass
+
+# Tentar importar dependências
+CURL_CFFI_AVAILABLE = False
+BS4_AVAILABLE = False
+HTML2TEXT_AVAILABLE = False
+
+try:
+    from curl_cffi import requests
+    CURL_CFFI_AVAILABLE = True
+except ImportError as e:
+    logging.error(f"curl_cffi não disponível: {e}")
+
+try:
+    from bs4 import BeautifulSoup
+    BS4_AVAILABLE = True
+except ImportError as e:
+    logging.error(f"BeautifulSoup4 não disponível: {e}")
+
+try:
+    import html2text
+    HTML2TEXT_AVAILABLE = True
+except ImportError as e:
+    logging.error(f"html2text não disponível: {e}")
+
+# Pool de User Agents para parecer mais humano
+USER_AGENTS = [
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/121.0",
+]
+
+def get_random_user_agent():
+    """Retorna um user agent aleatório"""
+    return random.choice(USER_AGENTS)
+
+def clean_url(url: str) -> str:
+    """Remove fragmentos e parâmetros de rastreamento da URL"""
+    try:
+        parsed = urlparse(url)
+        return f"{parsed.scheme}://{parsed.netloc}{parsed.path}"
+    except:
+        return url
+
+def extract_domain(url: str) -> str:
+    """Extrai o domínio de uma URL"""
+    try:
+        parsed = urlparse(url)
+        return parsed.netloc
+    except:
+        return ""
+
+def is_ad_or_tracker_url(url: str) -> bool:
+    """Verifica se uma URL é de anúncio ou rastreador"""
+    trackers = ['analytics', 'google-analytics', 'doubleclick', 'facebook.com/tr', 'ads.', 'banner', 'tracking']
+    return any(tracker in url.lower() for tracker in trackers)
+
+def is_domain_blocked(url: str, excluded_domains: List[str]) -> bool:
+    """Verifica se o domínio da URL está na lista de exclusão"""
+    domain = extract_domain(url)
+    return any(excluded in domain for excluded in excluded_domains)
+
+def scrape_url(url: str, visible: bool = False) -> Dict:
+    """Função principal de scraping com curl_cffi + BeautifulSoup"""
+    
+    # Verificar dependências
+    if not CURL_CFFI_AVAILABLE:
+        return {
+            "type": "error",
+            "message": "curl_cffi não está instalado"
+        }
+    
+    if not BS4_AVAILABLE:
+        return {
+            "type": "error",
+            "message": "beautifulsoup4 não está instalado"
+        }
+    
+    if not HTML2TEXT_AVAILABLE:
+        return {
+            "type": "error",
+            "message": "html2text não está instalado"
+        }
+    
+    try:
+        logging.info(f"Iniciando requisição para: {url}")
+        
+        # Criar headers com curl_cffi (impersonate navegador)
+        headers = {
+            "User-Agent": get_random_user_agent(),
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+            "Accept-Language": "pt-BR,pt;q=0.9,en;q=0.8",
+            "Accept-Encoding": "gzip, deflate",
+            "DNT": "1",
+            "Connection": "keep-alive",
+            "Upgrade-Insecure-Requests": "1",
+        }
+        
+        # Executar requisição com timeout
+        try:
+            response = requests.get(
+                url,
+                headers=headers,
+                timeout=15,
+                impersonate="chrome120",
+                allow_redirects=True,
+                follow_redirects=True
+            )
+            response.raise_for_status()
+        except Exception as e:
+            logging.error(f"Erro na requisição HTTP: {e}")
+            return {
+                "type": "error",
+                "message": f"Failed to fetch URL: {str(e)}"
+            }
+        
+        # Verificar se recebemos conteúdo HTML válido
+        if len(response.content) < 100:
+            return {
+                "type": "error",
+                "message": "Resposta muito curta (< 100 bytes)"
+            }
+        
+        logging.info(f"Resposta recebida: {len(response.content)} bytes")
+        
+        # Parse com BeautifulSoup
+        try:
+            soup = BeautifulSoup(response.content, 'html.parser')
+        except Exception as e:
+            logging.error(f"Erro ao fazer parse HTML: {e}")
+            return {
+                "type": "error",
+                "message": f"Failed to parse HTML: {str(e)}"
+            }
+        
+        # Extrair título
+        title = "Sem título"
+        title_tag = soup.find('meta', property='og:title') or soup.find('meta', {'name': 'title'}) or soup.find('title')
+        if title_tag:
+            title = title_tag.get('content') or title_tag.string or title
+        
+        # Remover scripts e styles
+        for script in soup(["script", "style", "meta", "link", "noscript"]):
+            script.decompose()
+        
+        # Remover elementos desnecessários
+        for elem in soup.find_all(["nav", "header", "footer", "aside", "ads"]):
+            elem.decompose()
+        
+        # Extrair conteúdo principal
+        main_content = None
+        for selector in ['article', 'main', '[role="main"]', '.content', '.post', '.entry']:
+            if selector.startswith('['):
+                main_content = soup.select_one(selector)
+            else:
+                main_content = soup.find(selector)
+            if main_content and len(main_content.get_text(strip=True)) > 200:
+                break
+        
+        # Fallback: usar body
+        if not main_content or len(main_content.get_text(strip=True)) < 200:
+            main_content = soup.find('body') or soup
+        
+        # Obter HTML do conteúdo principal
+        html_content = str(main_content) if main_content else str(soup)
+        
+        # Converter para Markdown usando html2text
+        try:
+            h = html2text.HTML2Text()
+            h.ignore_links = False
+            h.ignore_images = False
+            h.body_width = 0
+            markdown_content = h.handle(html_content)
+        except Exception as e:
+            logging.warning(f"Erro ao converter para markdown, usando texto plano: {e}")
+            markdown_content = main_content.get_text(separator="\n") if main_content else response.text
+        
+        # Limpeza final
+        lines = [line.strip() for line in markdown_content.split('\n') if line.strip()]
+        final_content = '\n'.join(lines)
+        
+        # Validação de conteúdo
+        if len(final_content) < 100:
+            return {
+                "type": "error",
+                "message": f"Conteúdo insuficiente ({len(final_content)} chars)"
+            }
+        
+        logging.info(f"Sucesso! {len(final_content)} chars extraídos")
+        
+        return {
+            "type": "success",
+            "url": clean_url(url),
+            "title": title[:500],
+            "content": final_content[:30000],
+            "markdown": final_content[:30000]
+        }
+    
+    except Exception as e:
+        logging.error(f"Erro no scraping: {traceback.format_exc()}")
+        return {
+            "type": "error",
+            "message": str(e)
+        }
+
+def search_duckduckgo(query: str, limit: int = 5) -> List[str]:
+    """Realiza busca no DuckDuckGo e retorna URLs"""
+    try:
+        if not CURL_CFFI_AVAILABLE or not BS4_AVAILABLE:
+            logging.error("Dependências não disponíveis")
+            return []
+        
+        logging.info(f"Buscando no DuckDuckGo: {query}")
+        
+        headers = {
+            "User-Agent": get_random_user_agent(),
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "Accept-Language": "pt-BR,pt;q=0.9,en;q=0.8",
+        }
+        
+        search_url = f"https://duckduckgo.com/html/?q={quote(query)}"
+        response = requests.get(
+            search_url,
+            headers=headers,
+            timeout=15,
+            impersonate="chrome120",
+            allow_redirects=True
+        )
+        response.raise_for_status()
+        
+        soup = BeautifulSoup(response.content, 'html.parser')
+        
+        urls = []
+        # DuckDuckGo HTML retorna resultados em links diretos
+        for result in soup.find_all('a', {'class': 'result__a'}):
+            href = result.get('href')
+            if href and not is_ad_or_tracker_url(href):
+                urls.append(href)
+                if len(urls) >= limit:
+                    break
+        
+        logging.info(f"Encontradas {len(urls)} URLs")
+        return urls
+    
+    except Exception as e:
+        logging.error(f"Erro na busca DuckDuckGo: {e}")
+        return []
+
+def smart_search_impl(query: str, config: Dict) -> List[str]:
+    """Implementa busca inteligente com categorias e filtros"""
+    try:
+        logging.info(f"Smart search iniciado: {query}")
+        
+        total_limit = config.get('total_sources_limit', 15)
+        categories = config.get('categories', [])
+        user_custom_sites = config.get('user_custom_sites', [])
+        excluded_domains = config.get('excluded_domains', [])
+        
+        all_urls = []
+        
+        # 1. Busca geral no DuckDuckGo
+        general_urls = search_duckduckgo(query, total_limit // 2)
+        all_urls.extend(general_urls)
+        
+        # 2. Busca por categorias habilitadas
+        for category in categories:
+            if not category.get('enabled', False):
+                continue
+            
+            base_sites = category.get('base_sites', [])
+            for site in base_sites[:2]:  # Limitar a 2 sites por categoria
+                site_query = f"site:{site} {query}"
+                site_urls = search_duckduckgo(site_query, 3)
+                all_urls.extend(site_urls)
+        
+        # 3. Busca em sites personalizados
+        for custom_site in user_custom_sites[:3]:
+            custom_query = f"site:{custom_site} {query}"
+            custom_urls = search_duckduckgo(custom_query, 2)
+            all_urls.extend(custom_urls)
+        
+        # 4. Filtrar domínios excluídos e remover duplicatas
+        filtered_urls = []
+        seen = set()
+        for url in all_urls:
+            if url not in seen and not is_domain_blocked(url, excluded_domains):
+                filtered_urls.append(url)
+                seen.add(url)
+                if len(filtered_urls) >= total_limit:
+                    break
+        
+        logging.info(f"Smart search concluído: {len(filtered_urls)} URLs")
+        return filtered_urls
+    
+    except Exception as e:
+        logging.error(f"Erro no smart search: {e}")
+        return []
+
+def scrape_urls_bulk_impl(urls: List[str]) -> List[Dict]:
+    """Scraping em paralelo de múltiplas URLs"""
+    try:
+        logging.info(f"Scraping bulk iniciado: {len(urls)} URLs")
+        
+        results = []
+        
+        def scrape_with_logging(url):
+            try:
+                result = scrape_url(url, visible=False)
+                if result.get('type') == 'success':
+                    logging.info(f"✓ {url}")
+                else:
+                    logging.warning(f"✗ {url}: {result.get('message', 'unknown error')}")
+                return result
+            except Exception as e:
+                logging.error(f"✗ {url}: {e}")
+                return {"type": "error", "message": str(e), "url": url}
+        
+        # Scraping paralelo com até 5 workers
+        with ThreadPoolExecutor(max_workers=5) as executor:
+            future_to_url = {executor.submit(scrape_with_logging, url): url for url in urls}
+            for future in as_completed(future_to_url):
+                try:
+                    result = future.result()
+                    if result.get('type') == 'success':
+                        results.append(result)
+                except Exception as e:
+                    logging.error(f"Erro no future: {e}")
+        
+        logging.info(f"Scraping bulk concluído: {len(results)} sucessos")
+        return results
+    
+    except Exception as e:
+        logging.error(f"Erro no scraping bulk: {e}")
+        return []
+
+def main():
+    """Entry point do script - lê comandos JSON do stdin"""
+    try:
+        # Ler JSON do stdin
+        input_data = sys.stdin.read().strip()
+        
+        if not input_data:
+            print_json({"type": "error", "message": "No input provided"})
+            return
+        
+        # Parse JSON
+        try:
+            command_data = json.loads(input_data)
+        except json.JSONDecodeError as e:
+            logging.error(f"Erro ao fazer parse do JSON: {e}")
+            print_json({"type": "error", "message": f"Invalid JSON: {str(e)}"})
+            return
+        
+        command = command_data.get("command")
+        logging.info(f"Comando recebido: {command}")
+        
+        # Dispatch de comandos
+        if command == "scrape":
+            url = command_data.get("url")
+            visible = command_data.get("visible", False)
+            
+            if not url:
+                print_json({"type": "error", "message": "URL missing"})
+                return
+            
+            result = scrape_url(url, visible)
+            print_json(result)
+        
+        elif command == "search_duckduckgo":
+            query = command_data.get("query")
+            limit = command_data.get("limit", 5)
+            
+            if not query:
+                print_json({"type": "error", "message": "Query missing"})
+                return
+            
+            urls = search_duckduckgo(query, limit)
+            print_json({"type": "success", "urls": urls})
+        
+        elif command == "smart_search":
+            query = command_data.get("query")
+            config = command_data.get("config", {})
+            
+            if not query:
+                print_json({"type": "error", "message": "Query missing"})
+                return
+            
+            urls = smart_search_impl(query, config)
+            print_json({"type": "success", "urls": urls})
+        
+        elif command == "search_and_scrape":
+            query = command_data.get("query")
+            config = command_data.get("config", {})
+            
+            if not query:
+                print_json({"type": "error", "message": "Query missing"})
+                return
+            
+            # Primeiro faz smart_search para obter URLs
+            urls = smart_search_impl(query, config)
+            
+            if not urls:
+                print_json({"type": "success", "results": []})
+                return
+            
+            # Depois faz scraping das URLs
+            results = scrape_urls_bulk_impl(urls)
+            print_json({"type": "success", "results": results})
+        
+        elif command == "scrape_urls_bulk":
+            urls = command_data.get("urls", [])
+            
+            if not urls:
+                print_json({"type": "error", "message": "URLs missing"})
+                return
+            
+            results = scrape_urls_bulk_impl(urls)
+            print_json({"type": "success", "results": results})
+        
+        elif command == "check":
+            # Verificar dependências
+            missing = []
+            if not CURL_CFFI_AVAILABLE:
+                missing.append("curl_cffi")
+            if not BS4_AVAILABLE:
+                missing.append("beautifulsoup4")
+            if not HTML2TEXT_AVAILABLE:
+                missing.append("html2text")
+            
+            if missing:
+                print_json({"type": "error", "message": f"Missing: {', '.join(missing)}"})
+            else:
+                print_json({"type": "success", "message": "All dependencies available"})
+        
+        else:
+            print_json({"type": "error", "message": f"Unknown command: {command}"})
+    
+    except Exception as e:
+        logging.error(f"Erro no main: {traceback.format_exc()}")
+        print_json({"type": "error", "message": f"Critical error: {str(e)}"})
+
+if __name__ == "__main__":
+    try:
+        main()
+    except KeyboardInterrupt:
+        print_json({"type": "error", "message": "Script interrupted"})
+        sys.exit(1)
+    except Exception as e:
+        try:
+            logging.error(f"Unhandled exception: {traceback.format_exc()}")
+            print_json({"type": "error", "message": f"Unhandled exception: {str(e)}"})
+        except:
+            try:
+                print(json.dumps({"type": "error", "message": "Critical failure"}))
+            except:
+                pass
+        sys.exit(1)