From 4275295e4fc072bb32866e2c6e69e6abb9ea93ac Mon Sep 17 00:00:00 2001 From: Evandro Fonseca Junior Date: Sat, 22 Nov 2025 01:59:12 -0300 Subject: [PATCH] ultima antes do format --- app/chat/page.tsx | 57 ++- components/chat/sources-carousel.tsx | 154 +++++++ components/python-bootstrap-progress.tsx | 97 ++++ components/scraping-progress-indicator.tsx | 89 ++++ hooks/use-python-bootstrap.ts | 103 +++++ src-tauri/scripts/requirements.txt | 4 + src-tauri/scripts/scraper.py | 499 +++++++++++++++++++++ 7 files changed, 991 insertions(+), 12 deletions(-) create mode 100644 components/chat/sources-carousel.tsx create mode 100644 components/python-bootstrap-progress.tsx create mode 100644 components/scraping-progress-indicator.tsx create mode 100644 hooks/use-python-bootstrap.ts create mode 100644 src-tauri/scripts/requirements.txt create mode 100644 src-tauri/scripts/scraper.py diff --git a/app/chat/page.tsx b/app/chat/page.tsx index fffa864..eac862d 100644 --- a/app/chat/page.tsx +++ b/app/chat/page.tsx @@ -316,6 +316,34 @@ Ao responder sobre fatos atuais ou notícias, inicie mencionando explicitamente payloadContentOverride: finalUserContent, }); + // Persistir fontes na mensagem do assistente (metadata) + if (scrapedSources.length > 0) { + // Aguardar um pouco para garantir que o sendMessage iniciou e criou a mensagem do assistente + // O sendMessage é async mas retorna void enquanto a stream acontece. + // O setMessages dentro dele adiciona a mensagem do assistente. + + // Nota: Como sendMessage roda em background (streaming), não podemos garantir que a mensagem + // do assistente já existe aqui imediatamente se não esperarmos. + // Mas o sendMessage faz `setMessages` com assistant vazio logo no início. + + // Vamos atualizar o estado usando callback para garantir que pegamos o mais recente + setMessages(prev => { + const newMessages = [...prev]; + const lastMsgIndex = newMessages.length - 1; + if (lastMsgIndex >= 0 && newMessages[lastMsgIndex].role === 'assistant') { + const lastMsg = newMessages[lastMsgIndex]; + newMessages[lastMsgIndex] = { + ...lastMsg, + metadata: { + ...(lastMsg.metadata || {}), + sources: scrapedSources + } + }; + } + return newMessages; + }); + } + // Marcar como completo quando começar a streamar setTimeout(() => setThinkingStep('complete'), 100); }; @@ -483,27 +511,32 @@ Ao responder sobre fatos atuais ou notícias, inicie mencionando explicitamente ) : (
{messages.map((msg, i) => { - // Verificar se a mensagem anterior teve busca web - const hasWebSources = i > 0 && - messages[i - 1].role === 'user' && - webSearch.scrapedSources.length > 0 && - webSearch.status === 'completed'; + // Verificar fontes nos metadados (preferencial) + const msgSources = msg.metadata?.sources || []; + const hasWebSources = msgSources.length > 0; + // Verificar se devemos mostrar o progresso em tempo real + // Apenas para a interação atual (quando isLoading é true ou acabou de terminar) + const isCurrentInteraction = i === messages.length - (isLoading ? 2 : 1) || i === messages.length - 1; + const showRealtimeProgress = + msg.role === 'user' && + isCurrentInteraction && + (webSearch.status === 'searching' || + webSearch.status === 'scraping' || + webSearch.status === 'completed' || + webSearch.status === 'error') && + webSearch.currentQuery; // Só mostra se tiver query + return (
- {(msg.role === 'user') && ( - (webSearch.status === 'searching' || - webSearch.status === 'scraping' || - webSearch.status === 'completed' || - webSearch.status === 'error') && ( + {(msg.role === 'user') && showRealtimeProgress && ( - ) )} {/* Mostrar fontes após resposta do assistente que usou web search */} {msg.role === 'assistant' && hasWebSources && ( @@ -513,7 +546,7 @@ Ao responder sobre fatos atuais ou notícias, inicie mencionando explicitamente Fontes consultadas:

- {webSearch.scrapedSources.map((source, idx) => ( + {msgSources.map((source: ScrapedContent, idx: number) => ( (null); + const [isDragging, setIsDragging] = useState(false); + const [startX, setStartX] = useState(0); + const [scrollLeft, setScrollLeft] = useState(0); + const [hasMoved, setHasMoved] = useState(false); + + // Handlers para mouse drag + const handleMouseDown = (e: React.MouseEvent) => { + if (!scrollContainerRef.current) return; + setIsDragging(true); + setStartX(e.pageX - scrollContainerRef.current.offsetLeft); + setScrollLeft(scrollContainerRef.current.scrollLeft); + setHasMoved(false); + }; + + const handleMouseMove = (e: React.MouseEvent) => { + if (!isDragging || !scrollContainerRef.current) return; + e.preventDefault(); + const x = e.pageX - scrollContainerRef.current.offsetLeft; + const walk = (x - startX) * 1.5; // Velocidade do scroll + scrollContainerRef.current.scrollLeft = scrollLeft - walk; + + // Se moveu mais de 3px, marca como movimento (não clique) + if (Math.abs(walk) > 3) { + setHasMoved(true); + } + }; + + const handleMouseUp = () => { + setIsDragging(false); + // Resetar após um pequeno delay + setTimeout(() => setHasMoved(false), 150); + }; + + const handleMouseLeave = () => { + setIsDragging(false); + setTimeout(() => setHasMoved(false), 150); + }; + + // Handlers para touch drag (mobile) + const handleTouchStart = (e: React.TouchEvent) => { + if (!scrollContainerRef.current) return; + setIsDragging(true); + setStartX(e.touches[0].pageX - scrollContainerRef.current.offsetLeft); + setScrollLeft(scrollContainerRef.current.scrollLeft); + setHasMoved(false); + }; + + const handleTouchMove = (e: React.TouchEvent) => { + if (!isDragging || !scrollContainerRef.current) return; + const x = e.touches[0].pageX - scrollContainerRef.current.offsetLeft; + const walk = (x - startX) * 1.5; + scrollContainerRef.current.scrollLeft = scrollLeft - walk; + + if (Math.abs(walk) > 3) { + setHasMoved(true); + } + }; + + const handleTouchEnd = () => { + setIsDragging(false); + setTimeout(() => setHasMoved(false), 150); + }; + + // Prevenir navegação do link durante drag + const handleLinkClick = (e: React.MouseEvent) => { + if (hasMoved || isDragging) { + e.preventDefault(); + e.stopPropagation(); + } + }; + + return ( + + ); +} + diff --git a/components/python-bootstrap-progress.tsx b/components/python-bootstrap-progress.tsx new file mode 100644 index 0000000..b9821c4 --- /dev/null +++ b/components/python-bootstrap-progress.tsx @@ -0,0 +1,97 @@ +'use client'; + +import { usePythonBootstrap } from '@/hooks/use-python-bootstrap'; +import { Progress } from '@/components/ui/progress'; +import { Loader2, XCircle } from 'lucide-react'; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card'; + +const STAGE_LABELS: Record = { + checking: 'Verificando', + downloading: 'Baixando', + extracting: 'Extraindo', + creating_venv: 'Criando ambiente virtual', + installing_pip: 'Instalando pip', + copying_scripts: 'Copiando scripts', + installing_deps: 'Instalando dependências', + validating: 'Validando', + done: 'Concluído', +}; + +export function PythonBootstrapProgress() { + const bootstrap = usePythonBootstrap(); + + // Se estiver completo e sem erro, não mostrar nada + if (bootstrap.isComplete && !bootstrap.error) { + return null; + } + + // Se houver erro, mostrar alerta + if (bootstrap.error) { + return ( +
+
+ + + + + Erro no Bootstrap do Python + + + O aplicativo não pode funcionar sem o runtime Python + + + +
+

{bootstrap.error}

+
+

+ Por favor, reinicie o aplicativo ou verifique os logs para mais detalhes. +

+ + +
+
+ ); + } + + // Mostrar progresso durante bootstrap + const stageLabel = STAGE_LABELS[bootstrap.stage] || bootstrap.stage; + const progressValue = bootstrap.progress ?? 0; + + return ( +
+
+ + + + + Inicializando Python Runtime + + + Configurando ambiente Python para scraping web + + + +
+
+ {stageLabel} + {bootstrap.progress !== null && ( + {Math.round(progressValue * 100)}% + )} +
+ + {bootstrap.progress !== null && ( + + )} + +

+ {bootstrap.message} +

+
+ + +
+
+ ); +} + diff --git a/components/scraping-progress-indicator.tsx b/components/scraping-progress-indicator.tsx new file mode 100644 index 0000000..58d3cad --- /dev/null +++ b/components/scraping-progress-indicator.tsx @@ -0,0 +1,89 @@ +"use client"; + +import { motion, AnimatePresence } from "framer-motion"; +import { Loader2, Search, FileText } from "lucide-react"; +import { useWebSearch } from "@/hooks/use-web-search"; + +export function ScrapingProgressIndicator() { + const webSearch = useWebSearch(); + + // Não mostrar se idle ou completed + if (webSearch.status === 'idle' || webSearch.status === 'completed') { + return null; + } + + return ( + + +
+ {/* Icon animado */} +
+ {webSearch.status === 'searching' && ( + + )} + {webSearch.status === 'scraping' && ( + + )} + {webSearch.status === 'error' && ( +
+ ! +
+ )} +
+ + {/* Content */} +
+
+ +

+ {webSearch.status === 'searching' && 'Buscando fontes...'} + {webSearch.status === 'scraping' && `Analisando fontes (${webSearch.scrapedSources.length})...`} + {webSearch.status === 'error' && 'Erro na busca'} +

+
+ + {/* Mostrar query atual se disponível */} + {webSearch.currentQuery && webSearch.status !== 'error' && ( +

+ {webSearch.currentQuery} +

+ )} + + {/* Mostrar erro se houver */} + {webSearch.error && webSearch.status === 'error' && ( +

+ {webSearch.error} +

+ )} + + {/* Barra de progresso simulada */} + {webSearch.status !== 'error' && ( +
+ +
+ )} +
+
+ + + ); +} + + + diff --git a/hooks/use-python-bootstrap.ts b/hooks/use-python-bootstrap.ts new file mode 100644 index 0000000..8ccad0a --- /dev/null +++ b/hooks/use-python-bootstrap.ts @@ -0,0 +1,103 @@ +'use client'; + +import { useState, useEffect } from 'react'; +import { listen } from '@tauri-apps/api/event'; + +export interface PythonBootstrapProgress { + stage: string; + message: string; + progress: number | null; +} + +export interface PythonBootstrapState { + stage: string; + message: string; + progress: number | null; + isComplete: boolean; + error: string | null; +} + +export function usePythonBootstrap() { + const [state, setState] = useState({ + stage: 'checking', + message: 'Verificando runtime Python...', + progress: null, + isComplete: false, + error: null, + }); + + useEffect(() => { + // Verificar status inicial + const checkInitialStatus = async () => { + try { + const { invoke } = await import('@tauri-apps/api/core'); + const status = await invoke<{ ready: boolean; error: string | null }>('check_python_runtime_status'); + + if (status.ready) { + setState({ + stage: 'done', + message: 'Runtime Python pronto', + progress: 1.0, + isComplete: true, + error: null, + }); + } else if (status.error) { + setState(prev => ({ + ...prev, + error: status.error || 'Erro desconhecido', + isComplete: false, + })); + } + } catch (error) { + console.error('Erro ao verificar status inicial do Python:', error); + } + }; + + checkInitialStatus(); + + // Escutar eventos de progresso + const progressUnlisten = listen('python-bootstrap-progress', (event) => { + const payload = event.payload; + setState(prev => ({ + ...prev, + stage: payload.stage, + message: payload.message, + progress: payload.progress ?? prev.progress, + error: null, + })); + }); + + // Escutar evento de conclusão + const completeUnlisten = listen('python-bootstrap-complete', () => { + setState(prev => ({ + ...prev, + stage: 'done', + message: 'Runtime Python pronto', + progress: 1.0, + isComplete: true, + error: null, + })); + }); + + // Escutar evento de erro + const errorUnlisten = listen('python-bootstrap-error', (event) => { + setState(prev => ({ + ...prev, + error: event.payload || 'Erro desconhecido durante o bootstrap', + isComplete: false, + })); + }); + + return () => { + progressUnlisten.then(unlisten => unlisten()); + completeUnlisten.then(unlisten => unlisten()); + errorUnlisten.then(unlisten => unlisten()); + }; + }, []); + + return state; +} + + + + diff --git a/src-tauri/scripts/requirements.txt b/src-tauri/scripts/requirements.txt new file mode 100644 index 0000000..e22dddc --- /dev/null +++ b/src-tauri/scripts/requirements.txt @@ -0,0 +1,4 @@ +curl_cffi>=0.5.10 +beautifulsoup4>=4.12.0 +lxml>=4.9.0 +html2text>=2024.1.0 diff --git a/src-tauri/scripts/scraper.py b/src-tauri/scripts/scraper.py new file mode 100644 index 0000000..837b3a2 --- /dev/null +++ b/src-tauri/scripts/scraper.py @@ -0,0 +1,499 @@ +#!/usr/bin/env python3 +""" +Script Python para scraping web com curl_cffi + BeautifulSoup +Pipeline: curl_cffi (Request HTTP) -> BeautifulSoup (Parsing) -> html2text (Formatação) +Comunica via JSON stdin/stdout com o backend Rust +""" + +import sys +import json +import time +import logging +import traceback +from typing import List, Dict, Optional +from urllib.parse import urljoin, urlparse, quote +import random +from concurrent.futures import ThreadPoolExecutor, as_completed + +# Configuração CRÍTICA de Encoding para Windows +try: + sys.stdout.reconfigure(encoding='utf-8') + sys.stderr.reconfigure(encoding='utf-8') +except (AttributeError, Exception): + pass + +# Configuração de Logging para STDERR (Não polui o STDOUT) +logging.basicConfig( + level=logging.INFO, + format='[PYTHON LOG] %(message)s', + stream=sys.stderr +) + +def print_json(data): + """Função auxiliar para garantir output limpo em JSON""" + try: + print(json.dumps(data, ensure_ascii=False)) + sys.stdout.flush() + except Exception as e: + try: + print(json.dumps({"type": "error", "message": f"Failed to format output: {str(e)}"})) + sys.stdout.flush() + except: + pass + +# Tentar importar dependências +CURL_CFFI_AVAILABLE = False +BS4_AVAILABLE = False +HTML2TEXT_AVAILABLE = False + +try: + from curl_cffi import requests + CURL_CFFI_AVAILABLE = True +except ImportError as e: + logging.error(f"curl_cffi não disponível: {e}") + +try: + from bs4 import BeautifulSoup + BS4_AVAILABLE = True +except ImportError as e: + logging.error(f"BeautifulSoup4 não disponível: {e}") + +try: + import html2text + HTML2TEXT_AVAILABLE = True +except ImportError as e: + logging.error(f"html2text não disponível: {e}") + +# Pool de User Agents para parecer mais humano +USER_AGENTS = [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/121.0", +] + +def get_random_user_agent(): + """Retorna um user agent aleatório""" + return random.choice(USER_AGENTS) + +def clean_url(url: str) -> str: + """Remove fragmentos e parâmetros de rastreamento da URL""" + try: + parsed = urlparse(url) + return f"{parsed.scheme}://{parsed.netloc}{parsed.path}" + except: + return url + +def extract_domain(url: str) -> str: + """Extrai o domínio de uma URL""" + try: + parsed = urlparse(url) + return parsed.netloc + except: + return "" + +def is_ad_or_tracker_url(url: str) -> bool: + """Verifica se uma URL é de anúncio ou rastreador""" + trackers = ['analytics', 'google-analytics', 'doubleclick', 'facebook.com/tr', 'ads.', 'banner', 'tracking'] + return any(tracker in url.lower() for tracker in trackers) + +def is_domain_blocked(url: str, excluded_domains: List[str]) -> bool: + """Verifica se o domínio da URL está na lista de exclusão""" + domain = extract_domain(url) + return any(excluded in domain for excluded in excluded_domains) + +def scrape_url(url: str, visible: bool = False) -> Dict: + """Função principal de scraping com curl_cffi + BeautifulSoup""" + + # Verificar dependências + if not CURL_CFFI_AVAILABLE: + return { + "type": "error", + "message": "curl_cffi não está instalado" + } + + if not BS4_AVAILABLE: + return { + "type": "error", + "message": "beautifulsoup4 não está instalado" + } + + if not HTML2TEXT_AVAILABLE: + return { + "type": "error", + "message": "html2text não está instalado" + } + + try: + logging.info(f"Iniciando requisição para: {url}") + + # Criar headers com curl_cffi (impersonate navegador) + headers = { + "User-Agent": get_random_user_agent(), + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Accept-Language": "pt-BR,pt;q=0.9,en;q=0.8", + "Accept-Encoding": "gzip, deflate", + "DNT": "1", + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + } + + # Executar requisição com timeout + try: + response = requests.get( + url, + headers=headers, + timeout=15, + impersonate="chrome120", + allow_redirects=True, + follow_redirects=True + ) + response.raise_for_status() + except Exception as e: + logging.error(f"Erro na requisição HTTP: {e}") + return { + "type": "error", + "message": f"Failed to fetch URL: {str(e)}" + } + + # Verificar se recebemos conteúdo HTML válido + if len(response.content) < 100: + return { + "type": "error", + "message": "Resposta muito curta (< 100 bytes)" + } + + logging.info(f"Resposta recebida: {len(response.content)} bytes") + + # Parse com BeautifulSoup + try: + soup = BeautifulSoup(response.content, 'html.parser') + except Exception as e: + logging.error(f"Erro ao fazer parse HTML: {e}") + return { + "type": "error", + "message": f"Failed to parse HTML: {str(e)}" + } + + # Extrair título + title = "Sem título" + title_tag = soup.find('meta', property='og:title') or soup.find('meta', {'name': 'title'}) or soup.find('title') + if title_tag: + title = title_tag.get('content') or title_tag.string or title + + # Remover scripts e styles + for script in soup(["script", "style", "meta", "link", "noscript"]): + script.decompose() + + # Remover elementos desnecessários + for elem in soup.find_all(["nav", "header", "footer", "aside", "ads"]): + elem.decompose() + + # Extrair conteúdo principal + main_content = None + for selector in ['article', 'main', '[role="main"]', '.content', '.post', '.entry']: + if selector.startswith('['): + main_content = soup.select_one(selector) + else: + main_content = soup.find(selector) + if main_content and len(main_content.get_text(strip=True)) > 200: + break + + # Fallback: usar body + if not main_content or len(main_content.get_text(strip=True)) < 200: + main_content = soup.find('body') or soup + + # Obter HTML do conteúdo principal + html_content = str(main_content) if main_content else str(soup) + + # Converter para Markdown usando html2text + try: + h = html2text.HTML2Text() + h.ignore_links = False + h.ignore_images = False + h.body_width = 0 + markdown_content = h.handle(html_content) + except Exception as e: + logging.warning(f"Erro ao converter para markdown, usando texto plano: {e}") + markdown_content = main_content.get_text(separator="\n") if main_content else response.text + + # Limpeza final + lines = [line.strip() for line in markdown_content.split('\n') if line.strip()] + final_content = '\n'.join(lines) + + # Validação de conteúdo + if len(final_content) < 100: + return { + "type": "error", + "message": f"Conteúdo insuficiente ({len(final_content)} chars)" + } + + logging.info(f"Sucesso! {len(final_content)} chars extraídos") + + return { + "type": "success", + "url": clean_url(url), + "title": title[:500], + "content": final_content[:30000], + "markdown": final_content[:30000] + } + + except Exception as e: + logging.error(f"Erro no scraping: {traceback.format_exc()}") + return { + "type": "error", + "message": str(e) + } + +def search_duckduckgo(query: str, limit: int = 5) -> List[str]: + """Realiza busca no DuckDuckGo e retorna URLs""" + try: + if not CURL_CFFI_AVAILABLE or not BS4_AVAILABLE: + logging.error("Dependências não disponíveis") + return [] + + logging.info(f"Buscando no DuckDuckGo: {query}") + + headers = { + "User-Agent": get_random_user_agent(), + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "pt-BR,pt;q=0.9,en;q=0.8", + } + + search_url = f"https://duckduckgo.com/html/?q={quote(query)}" + response = requests.get( + search_url, + headers=headers, + timeout=15, + impersonate="chrome120", + allow_redirects=True + ) + response.raise_for_status() + + soup = BeautifulSoup(response.content, 'html.parser') + + urls = [] + # DuckDuckGo HTML retorna resultados em links diretos + for result in soup.find_all('a', {'class': 'result__a'}): + href = result.get('href') + if href and not is_ad_or_tracker_url(href): + urls.append(href) + if len(urls) >= limit: + break + + logging.info(f"Encontradas {len(urls)} URLs") + return urls + + except Exception as e: + logging.error(f"Erro na busca DuckDuckGo: {e}") + return [] + +def smart_search_impl(query: str, config: Dict) -> List[str]: + """Implementa busca inteligente com categorias e filtros""" + try: + logging.info(f"Smart search iniciado: {query}") + + total_limit = config.get('total_sources_limit', 15) + categories = config.get('categories', []) + user_custom_sites = config.get('user_custom_sites', []) + excluded_domains = config.get('excluded_domains', []) + + all_urls = [] + + # 1. Busca geral no DuckDuckGo + general_urls = search_duckduckgo(query, total_limit // 2) + all_urls.extend(general_urls) + + # 2. Busca por categorias habilitadas + for category in categories: + if not category.get('enabled', False): + continue + + base_sites = category.get('base_sites', []) + for site in base_sites[:2]: # Limitar a 2 sites por categoria + site_query = f"site:{site} {query}" + site_urls = search_duckduckgo(site_query, 3) + all_urls.extend(site_urls) + + # 3. Busca em sites personalizados + for custom_site in user_custom_sites[:3]: + custom_query = f"site:{custom_site} {query}" + custom_urls = search_duckduckgo(custom_query, 2) + all_urls.extend(custom_urls) + + # 4. Filtrar domínios excluídos e remover duplicatas + filtered_urls = [] + seen = set() + for url in all_urls: + if url not in seen and not is_domain_blocked(url, excluded_domains): + filtered_urls.append(url) + seen.add(url) + if len(filtered_urls) >= total_limit: + break + + logging.info(f"Smart search concluído: {len(filtered_urls)} URLs") + return filtered_urls + + except Exception as e: + logging.error(f"Erro no smart search: {e}") + return [] + +def scrape_urls_bulk_impl(urls: List[str]) -> List[Dict]: + """Scraping em paralelo de múltiplas URLs""" + try: + logging.info(f"Scraping bulk iniciado: {len(urls)} URLs") + + results = [] + + def scrape_with_logging(url): + try: + result = scrape_url(url, visible=False) + if result.get('type') == 'success': + logging.info(f"✓ {url}") + else: + logging.warning(f"✗ {url}: {result.get('message', 'unknown error')}") + return result + except Exception as e: + logging.error(f"✗ {url}: {e}") + return {"type": "error", "message": str(e), "url": url} + + # Scraping paralelo com até 5 workers + with ThreadPoolExecutor(max_workers=5) as executor: + future_to_url = {executor.submit(scrape_with_logging, url): url for url in urls} + for future in as_completed(future_to_url): + try: + result = future.result() + if result.get('type') == 'success': + results.append(result) + except Exception as e: + logging.error(f"Erro no future: {e}") + + logging.info(f"Scraping bulk concluído: {len(results)} sucessos") + return results + + except Exception as e: + logging.error(f"Erro no scraping bulk: {e}") + return [] + +def main(): + """Entry point do script - lê comandos JSON do stdin""" + try: + # Ler JSON do stdin + input_data = sys.stdin.read().strip() + + if not input_data: + print_json({"type": "error", "message": "No input provided"}) + return + + # Parse JSON + try: + command_data = json.loads(input_data) + except json.JSONDecodeError as e: + logging.error(f"Erro ao fazer parse do JSON: {e}") + print_json({"type": "error", "message": f"Invalid JSON: {str(e)}"}) + return + + command = command_data.get("command") + logging.info(f"Comando recebido: {command}") + + # Dispatch de comandos + if command == "scrape": + url = command_data.get("url") + visible = command_data.get("visible", False) + + if not url: + print_json({"type": "error", "message": "URL missing"}) + return + + result = scrape_url(url, visible) + print_json(result) + + elif command == "search_duckduckgo": + query = command_data.get("query") + limit = command_data.get("limit", 5) + + if not query: + print_json({"type": "error", "message": "Query missing"}) + return + + urls = search_duckduckgo(query, limit) + print_json({"type": "success", "urls": urls}) + + elif command == "smart_search": + query = command_data.get("query") + config = command_data.get("config", {}) + + if not query: + print_json({"type": "error", "message": "Query missing"}) + return + + urls = smart_search_impl(query, config) + print_json({"type": "success", "urls": urls}) + + elif command == "search_and_scrape": + query = command_data.get("query") + config = command_data.get("config", {}) + + if not query: + print_json({"type": "error", "message": "Query missing"}) + return + + # Primeiro faz smart_search para obter URLs + urls = smart_search_impl(query, config) + + if not urls: + print_json({"type": "success", "results": []}) + return + + # Depois faz scraping das URLs + results = scrape_urls_bulk_impl(urls) + print_json({"type": "success", "results": results}) + + elif command == "scrape_urls_bulk": + urls = command_data.get("urls", []) + + if not urls: + print_json({"type": "error", "message": "URLs missing"}) + return + + results = scrape_urls_bulk_impl(urls) + print_json({"type": "success", "results": results}) + + elif command == "check": + # Verificar dependências + missing = [] + if not CURL_CFFI_AVAILABLE: + missing.append("curl_cffi") + if not BS4_AVAILABLE: + missing.append("beautifulsoup4") + if not HTML2TEXT_AVAILABLE: + missing.append("html2text") + + if missing: + print_json({"type": "error", "message": f"Missing: {', '.join(missing)}"}) + else: + print_json({"type": "success", "message": "All dependencies available"}) + + else: + print_json({"type": "error", "message": f"Unknown command: {command}"}) + + except Exception as e: + logging.error(f"Erro no main: {traceback.format_exc()}") + print_json({"type": "error", "message": f"Critical error: {str(e)}"}) + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + print_json({"type": "error", "message": "Script interrupted"}) + sys.exit(1) + except Exception as e: + try: + logging.error(f"Unhandled exception: {traceback.format_exc()}") + print_json({"type": "error", "message": f"Unhandled exception: {str(e)}"}) + except: + try: + print(json.dumps({"type": "error", "message": "Critical failure"})) + except: + pass + sys.exit(1)