chore: expurgo de blobs excedentes e realinhamento de estado

src-tauri/scripts/scraper.py

@@ -1,499 +1,367 @@
 #!/usr/bin/env python3
 """
-Script Python para scraping web com curl_cffi + BeautifulSoup
-Pipeline: curl_cffi (Request HTTP) -> BeautifulSoup (Parsing) -> html2text (Formatação)
-Comunica via JSON stdin/stdout com o backend Rust
+Micro-serviço Python para scraping e busca
+Pipeline:
+1. Scraping: curl_cffi / Playwright Stealth
+2. Busca: duckduckgo-search
 """
 
 import sys
 import json
-import time
 import logging
 import traceback
-from typing import List, Dict, Optional
-from urllib.parse import urljoin, urlparse, quote
 import random
-from concurrent.futures import ThreadPoolExecutor, as_completed
+import time
+import os
+import asyncio
+from itertools import cycle
+from typing import Dict, Optional, Union, List
+from urllib.parse import urljoin
+
+# Bibliotecas de Scraping e Parsing
+from bs4 import BeautifulSoup
+from curl_cffi import requests
+import html2text
+
+# DuckDuckGo Search
+try:
+    from duckduckgo_search import DDGS
+    HAS_DDG = True
+except ImportError:
+    HAS_DDG = False
+
+# Fallback libs
+try:
+    import requests as pyrequests
+    HAS_PYREQUESTS = True
+except ImportError:
+    HAS_PYREQUESTS = False
+
+try:
+    import httpx
+    HAS_HTTPX = True
+except ImportError:
+    HAS_HTTPX = False
+
+# Playwright & Anti-Detect
+try:
+    from playwright.async_api import async_playwright, TimeoutError as PlaywrightTimeoutError
+    HAS_PLAYWRIGHT = True
+except ImportError:
+    HAS_PLAYWRIGHT = False
+
+try:
+    from fake_useragent import UserAgent
+    ua_generator = UserAgent()
+    HAS_FAKE_UA = True
+except ImportError:
+    HAS_FAKE_UA = False
+    ua_generator = None
+
+# --- CONFIGURAÇÃO DO AMBIENTE ---
 
-# Configuração CRÍTICA de Encoding para Windows
 try:
     sys.stdout.reconfigure(encoding='utf-8')
     sys.stderr.reconfigure(encoding='utf-8')
 except (AttributeError, Exception):
     pass
 
-# Configuração de Logging para STDERR (Não polui o STDOUT)
 logging.basicConfig(
     level=logging.INFO,
-    format='[PYTHON LOG] %(message)s',
+    format='[PYTHON SCRAPER] %(levelname)s: %(message)s',
     stream=sys.stderr
 )
 
-def print_json(data):
-    """Função auxiliar para garantir output limpo em JSON"""
+# --- CONSTANTES E POOLS ---
+
+UA_POOL_FALLBACK = [
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15',
+    'Mozilla/5.0 (X11; Linux x86_64; rv:132.0) Gecko/20100101 Firefox/132.0',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0'
+]
+ua_cycle = cycle(UA_POOL_FALLBACK)
+
+REQUEST_TIMEOUT = 20
+MAX_RETRIES = 2
+
+PROXY_POOL = None
+env_proxies = os.environ.get('SCRAPER_PROXIES') or os.environ.get('SCRAPER_HTTP_PROXIES')
+if env_proxies:
+    parts = [p.strip() for p in env_proxies.split(',') if p.strip()]
+    if parts:
+        PROXY_POOL = cycle(parts)
+
+# --- FUNÇÕES AUXILIARES ---
+
+def get_random_ua() -> str:
+    if HAS_FAKE_UA:
+        try:
+            return ua_generator.random
+        except Exception:
+            pass
+    return next(ua_cycle)
+
+def generate_consistent_headers(user_agent: str) -> Dict[str, str]:
+    base_headers = {
+        'User-Agent': user_agent,
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
+        'Accept-Language': 'pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7',
+        'Accept-Encoding': 'gzip, deflate, br',
+        'DNT': '1',
+        'Connection': 'keep-alive',
+        'Upgrade-Insecure-Requests': '1',
+        'Sec-Fetch-Dest': 'document',
+        'Sec-Fetch-Mode': 'navigate',
+        'Sec-Fetch-Site': 'none',
+        'Sec-Fetch-User': '?1',
+        'Cache-Control': 'max-age=0'
+    }
+    if 'Chrome' in user_agent:
+        try:
+            version = user_agent.split('Chrome/')[1].split('.')[0]
+        except IndexError:
+            version = "120"
+        base_headers.update({
+            'sec-ch-ua': f'"Chromium";v="{version}", "Google Chrome";v="{version}", "Not-A.Brand";v="99"',
+            'sec-ch-ua-mobile': '?0',
+            'sec-ch-ua-platform': '"Windows"' if 'Windows' in user_agent else ('"macOS"' if 'Macintosh' in user_agent else '"Linux"')
+        })
+    return base_headers
+
+def get_proxy_dict() -> Optional[Dict[str, str]]:
+    if PROXY_POOL:
+        proxy = next(PROXY_POOL)
+        return {"http": proxy, "https": proxy}
+    return None
+
+def random_jitter(min_sec=0.5, max_sec=2.0):
+    time.sleep(random.uniform(min_sec, max_sec))
+
+# --- LIMPEZA E EXTRAÇÃO ---
+
+def clean_html(soup: BeautifulSoup) -> BeautifulSoup:
+    for tag in soup(["script", "style", "meta", "link", "noscript", "iframe", "object", "embed", "applet", "canvas", "svg"]):
+        tag.decompose()
+    for tag in soup.find_all(["nav", "header", "footer", "aside", "menu", "dialog"]):
+        tag.decompose()
+    ad_keywords = ["ads", "advertisement", "banner", "ad-", "popup", "modal", "cookie", "subscription", "newsletter", "share", "social", "sidebar", "promo", "sponsor"]
+    selectors_to_remove = ['.ad', '.advertisement', '[class*="ad-"]', '[id*="ad-"]', '[id*="cookie"]', '[class*="cookie"]', '.share-buttons', '.social-media', '.comments-section', '#comments']
+    for selector in selectors_to_remove:
+        for tag in soup.select(selector):
+            tag.decompose()
+    def is_ad(tag):
+        if not tag.name: return False
+        attrs = str(tag.attrs).lower()
+        return any(k in attrs for k in ad_keywords)
+    for tag in soup.find_all(is_ad):
+        tag.decompose()
+    return soup
+
+def extract_main_content(soup: BeautifulSoup) -> BeautifulSoup:
+    selectors = ['article', 'main', '[role="main"]', '.post-content', '.article-body', '#content', '.content', '.entry-content', '.page-content']
+    for selector in selectors:
+        found = soup.select_one(selector)
+        if found and len(found.get_text(strip=True)) > 200:
+            return found
+    body = soup.find('body')
+    return body if body else soup
+
+def html_to_markdown(html_content: str) -> str:
     try:
-        print(json.dumps(data, ensure_ascii=False))
+        h = html2text.HTML2Text()
+        h.ignore_links = False
+        h.ignore_images = False
+        h.images_to_alt = True
+        h.body_width = 0
+        h.protect_links = True
+        return h.handle(html_content)
+    except Exception as e:
+        logging.warning(f"Erro na conversão Markdown: {e}")
+        return html_content
+
+# --- ESTRATÉGIAS DE SCRAPING ---
+
+async def scrape_with_playwright(url: str) -> Dict:
+    if not HAS_PLAYWRIGHT:
+        return {"error": "Playwright not installed"}
+    logging.info(f"[PLAYWRIGHT] Iniciando scraping stealth para: {url}")
+    async with async_playwright() as p:
+        args = ['--disable-blink-features=AutomationControlled', '--disable-dev-shm-usage', '--no-sandbox', '--disable-setuid-sandbox', '--disable-infobars', '--window-size=1920,1080', '--start-maximized']
+        try:
+            browser = await p.chromium.launch(headless=True, args=args)
+            ua = get_random_ua()
+            context = await browser.new_context(user_agent=ua, viewport={'width': 1920, 'height': 1080}, locale='pt-BR', timezone_id='America/Sao_Paulo', java_script_enabled=True, ignore_https_errors=True)
+            await context.add_init_script("""
+                Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
+                window.chrome = { runtime: {} };
+                Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5] });
+                Object.defineProperty(navigator, 'languages', { get: () => ['pt-BR', 'pt', 'en-US', 'en'] });
+            """)
+            page = await context.new_page()
+            try:
+                await page.goto(url, wait_until='domcontentloaded', timeout=25000)
+                await page.wait_for_timeout(2000) 
+            except PlaywrightTimeoutError:
+                logging.warning(f"[PLAYWRIGHT] Timeout carregando {url}")
+            content = await page.content()
+            title = await page.title()
+            await browser.close()
+            return {"content": content, "title": title, "method": "playwright"}
+        except Exception as e:
+            logging.error(f"[PLAYWRIGHT] Erro fatal: {e}")
+            return {"error": str(e)}
+
+def scrape_with_curl_cffi(url: str) -> Dict:
+    ua = get_random_ua()
+    headers = generate_consistent_headers(ua)
+    proxies = get_proxy_dict()
+    try:
+        response = requests.get(url, headers=headers, proxies=proxies, timeout=15, impersonate="chrome120", allow_redirects=True)
+        response.raise_for_status()
+        if response.status_code in [403, 429, 503]:
+            raise requests.RequestsError(f"Blocked with status {response.status_code}")
+        return {"content": response.text, "title": "", "method": "curl_cffi", "headers": response.headers}
+    except Exception as e:
+        return {"error": str(e)}
+
+async def scrape_url_smart(url: str) -> Dict:
+    random_jitter()
+    logging.info(f"Tentativa 1: curl_cffi -> {url}")
+    result = scrape_with_curl_cffi(url)
+    html_content = None
+    title = ""
+    if "error" not in result:
+        html_content = result["content"]
+        if "challenge-platform" in html_content or "Cloudflare" in html_content and len(html_content) < 5000:
+            logging.warning("Detectado desafio Cloudflare/WAF no curl_cffi. Acionando fallback.")
+            html_content = None
+    if not html_content:
+        if HAS_PLAYWRIGHT:
+            logging.info(f"Tentativa 2: Playwright Stealth -> {url}")
+            pw_result = await scrape_with_playwright(url)
+            if "error" not in pw_result:
+                html_content = pw_result["content"]
+                title = pw_result.get("title", "")
+            else:
+                logging.error(f"Playwright falhou: {pw_result['error']}")
+        else:
+            logging.warning("Playwright não disponível para fallback.")
+    if html_content:
+        try:
+            soup = BeautifulSoup(html_content, 'html.parser')
+            if not title:
+                title = soup.title.string.strip() if soup.title else url
+            soup = clean_html(soup)
+            main_content = extract_main_content(soup)
+            cleaned_html = str(main_content)
+            markdown_text = html_to_markdown(cleaned_html)
+            final_markdown = '\n'.join([line.strip() for line in markdown_text.split('\n') if line.strip()])
+            if len(final_markdown) < 50:
+                return {"response_type": "error", "message": "Conteúdo insuficiente após processamento."}
+            return {"response_type": "success", "url": url, "title": title, "markdown": final_markdown, "content": final_markdown}
+        except Exception as e:
+            logging.error(f"Erro no processamento HTML: {traceback.format_exc()}")
+            return {"response_type": "error", "message": f"Processing error: {str(e)}"}
+    return {"response_type": "error", "message": f"Falha em todos os métodos de scraping. Erro inicial: {result.get('error', 'Unknown')}"}
+
+# --- DUCKDUCKGO SEARCH LOGIC ---
+
+async def search_duckduckgo(keywords: str, region: str = "wt-wt", safesearch: str = "moderate", max_results: int = 10) -> Dict:
+    if not HAS_DDG:
+        return {"response_type": "error", "message": "duckduckgo-search module not installed"}
+    
+    logging.info(f"Searching DuckDuckGo: {keywords} (region={region}, safe={safesearch})")
+    
+    try:
+        results_list = []
+        # DDGS is synchronous, run in thread executor to avoid blocking async loop if needed
+        # But since this script main loop is simple, direct call is fine, or wrap in to_thread
+        
+        with DDGS() as ddgs:
+            # ddgs.text() is a generator
+            ddg_gen = ddgs.text(
+                keywords=keywords,
+                region=region,
+                safesearch=safesearch,
+                max_results=max_results
+            )
+            
+            for r in ddg_gen:
+                results_list.append({
+                    "title": r.get("title", ""),
+                    "url": r.get("href", ""),
+                    "content": r.get("body", ""),
+                    "engine": "duckduckgo",
+                    "score": 1.0
+                })
+                
+        return {
+            "response_type": "success",
+            "query": keywords,
+            "results": results_list,
+            "number_of_results": len(results_list)
+        }
+        
+    except Exception as e:
+        logging.error(f"DuckDuckGo Search Error: {e}")
+        return {"response_type": "error", "message": str(e)}
+
+# --- MAIN ---
+
+def write_json_response(data: Dict):
+    try:
+        json_line = json.dumps(data, separators=(',', ':'), ensure_ascii=False)
+        print(json_line)
         sys.stdout.flush()
     except Exception as e:
-        try:
-            print(json.dumps({"type": "error", "message": f"Failed to format output: {str(e)}"}))
-            sys.stdout.flush()
-        except:
-            pass
-
-# Tentar importar dependências
-CURL_CFFI_AVAILABLE = False
-BS4_AVAILABLE = False
-HTML2TEXT_AVAILABLE = False
-
-try:
-    from curl_cffi import requests
-    CURL_CFFI_AVAILABLE = True
-except ImportError as e:
-    logging.error(f"curl_cffi não disponível: {e}")
-
-try:
-    from bs4 import BeautifulSoup
-    BS4_AVAILABLE = True
-except ImportError as e:
-    logging.error(f"BeautifulSoup4 não disponível: {e}")
-
-try:
-    import html2text
-    HTML2TEXT_AVAILABLE = True
-except ImportError as e:
-    logging.error(f"html2text não disponível: {e}")
-
-# Pool de User Agents para parecer mais humano
-USER_AGENTS = [
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/121.0",
-]
-
-def get_random_user_agent():
-    """Retorna um user agent aleatório"""
-    return random.choice(USER_AGENTS)
-
-def clean_url(url: str) -> str:
-    """Remove fragmentos e parâmetros de rastreamento da URL"""
-    try:
-        parsed = urlparse(url)
-        return f"{parsed.scheme}://{parsed.netloc}{parsed.path}"
-    except:
-        return url
-
-def extract_domain(url: str) -> str:
-    """Extrai o domínio de uma URL"""
-    try:
-        parsed = urlparse(url)
-        return parsed.netloc
-    except:
-        return ""
-
-def is_ad_or_tracker_url(url: str) -> bool:
-    """Verifica se uma URL é de anúncio ou rastreador"""
-    trackers = ['analytics', 'google-analytics', 'doubleclick', 'facebook.com/tr', 'ads.', 'banner', 'tracking']
-    return any(tracker in url.lower() for tracker in trackers)
-
-def is_domain_blocked(url: str, excluded_domains: List[str]) -> bool:
-    """Verifica se o domínio da URL está na lista de exclusão"""
-    domain = extract_domain(url)
-    return any(excluded in domain for excluded in excluded_domains)
-
-def scrape_url(url: str, visible: bool = False) -> Dict:
-    """Função principal de scraping com curl_cffi + BeautifulSoup"""
-    
-    # Verificar dependências
-    if not CURL_CFFI_AVAILABLE:
-        return {
-            "type": "error",
-            "message": "curl_cffi não está instalado"
-        }
-    
-    if not BS4_AVAILABLE:
-        return {
-            "type": "error",
-            "message": "beautifulsoup4 não está instalado"
-        }
-    
-    if not HTML2TEXT_AVAILABLE:
-        return {
-            "type": "error",
-            "message": "html2text não está instalado"
-        }
-    
-    try:
-        logging.info(f"Iniciando requisição para: {url}")
-        
-        # Criar headers com curl_cffi (impersonate navegador)
-        headers = {
-            "User-Agent": get_random_user_agent(),
-            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
-            "Accept-Language": "pt-BR,pt;q=0.9,en;q=0.8",
-            "Accept-Encoding": "gzip, deflate",
-            "DNT": "1",
-            "Connection": "keep-alive",
-            "Upgrade-Insecure-Requests": "1",
-        }
-        
-        # Executar requisição com timeout
-        try:
-            response = requests.get(
-                url,
-                headers=headers,
-                timeout=15,
-                impersonate="chrome120",
-                allow_redirects=True,
-                follow_redirects=True
-            )
-            response.raise_for_status()
-        except Exception as e:
-            logging.error(f"Erro na requisição HTTP: {e}")
-            return {
-                "type": "error",
-                "message": f"Failed to fetch URL: {str(e)}"
-            }
-        
-        # Verificar se recebemos conteúdo HTML válido
-        if len(response.content) < 100:
-            return {
-                "type": "error",
-                "message": "Resposta muito curta (< 100 bytes)"
-            }
-        
-        logging.info(f"Resposta recebida: {len(response.content)} bytes")
-        
-        # Parse com BeautifulSoup
-        try:
-            soup = BeautifulSoup(response.content, 'html.parser')
-        except Exception as e:
-            logging.error(f"Erro ao fazer parse HTML: {e}")
-            return {
-                "type": "error",
-                "message": f"Failed to parse HTML: {str(e)}"
-            }
-        
-        # Extrair título
-        title = "Sem título"
-        title_tag = soup.find('meta', property='og:title') or soup.find('meta', {'name': 'title'}) or soup.find('title')
-        if title_tag:
-            title = title_tag.get('content') or title_tag.string or title
-        
-        # Remover scripts e styles
-        for script in soup(["script", "style", "meta", "link", "noscript"]):
-            script.decompose()
-        
-        # Remover elementos desnecessários
-        for elem in soup.find_all(["nav", "header", "footer", "aside", "ads"]):
-            elem.decompose()
-        
-        # Extrair conteúdo principal
-        main_content = None
-        for selector in ['article', 'main', '[role="main"]', '.content', '.post', '.entry']:
-            if selector.startswith('['):
-                main_content = soup.select_one(selector)
-            else:
-                main_content = soup.find(selector)
-            if main_content and len(main_content.get_text(strip=True)) > 200:
-                break
-        
-        # Fallback: usar body
-        if not main_content or len(main_content.get_text(strip=True)) < 200:
-            main_content = soup.find('body') or soup
-        
-        # Obter HTML do conteúdo principal
-        html_content = str(main_content) if main_content else str(soup)
-        
-        # Converter para Markdown usando html2text
-        try:
-            h = html2text.HTML2Text()
-            h.ignore_links = False
-            h.ignore_images = False
-            h.body_width = 0
-            markdown_content = h.handle(html_content)
-        except Exception as e:
-            logging.warning(f"Erro ao converter para markdown, usando texto plano: {e}")
-            markdown_content = main_content.get_text(separator="\n") if main_content else response.text
-        
-        # Limpeza final
-        lines = [line.strip() for line in markdown_content.split('\n') if line.strip()]
-        final_content = '\n'.join(lines)
-        
-        # Validação de conteúdo
-        if len(final_content) < 100:
-            return {
-                "type": "error",
-                "message": f"Conteúdo insuficiente ({len(final_content)} chars)"
-            }
-        
-        logging.info(f"Sucesso! {len(final_content)} chars extraídos")
-        
-        return {
-            "type": "success",
-            "url": clean_url(url),
-            "title": title[:500],
-            "content": final_content[:30000],
-            "markdown": final_content[:30000]
-        }
-    
-    except Exception as e:
-        logging.error(f"Erro no scraping: {traceback.format_exc()}")
-        return {
-            "type": "error",
-            "message": str(e)
-        }
-
-def search_duckduckgo(query: str, limit: int = 5) -> List[str]:
-    """Realiza busca no DuckDuckGo e retorna URLs"""
-    try:
-        if not CURL_CFFI_AVAILABLE or not BS4_AVAILABLE:
-            logging.error("Dependências não disponíveis")
-            return []
-        
-        logging.info(f"Buscando no DuckDuckGo: {query}")
-        
-        headers = {
-            "User-Agent": get_random_user_agent(),
-            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-            "Accept-Language": "pt-BR,pt;q=0.9,en;q=0.8",
-        }
-        
-        search_url = f"https://duckduckgo.com/html/?q={quote(query)}"
-        response = requests.get(
-            search_url,
-            headers=headers,
-            timeout=15,
-            impersonate="chrome120",
-            allow_redirects=True
-        )
-        response.raise_for_status()
-        
-        soup = BeautifulSoup(response.content, 'html.parser')
-        
-        urls = []
-        # DuckDuckGo HTML retorna resultados em links diretos
-        for result in soup.find_all('a', {'class': 'result__a'}):
-            href = result.get('href')
-            if href and not is_ad_or_tracker_url(href):
-                urls.append(href)
-                if len(urls) >= limit:
-                    break
-        
-        logging.info(f"Encontradas {len(urls)} URLs")
-        return urls
-    
-    except Exception as e:
-        logging.error(f"Erro na busca DuckDuckGo: {e}")
-        return []
-
-def smart_search_impl(query: str, config: Dict) -> List[str]:
-    """Implementa busca inteligente com categorias e filtros"""
-    try:
-        logging.info(f"Smart search iniciado: {query}")
-        
-        total_limit = config.get('total_sources_limit', 15)
-        categories = config.get('categories', [])
-        user_custom_sites = config.get('user_custom_sites', [])
-        excluded_domains = config.get('excluded_domains', [])
-        
-        all_urls = []
-        
-        # 1. Busca geral no DuckDuckGo
-        general_urls = search_duckduckgo(query, total_limit // 2)
-        all_urls.extend(general_urls)
-        
-        # 2. Busca por categorias habilitadas
-        for category in categories:
-            if not category.get('enabled', False):
-                continue
-            
-            base_sites = category.get('base_sites', [])
-            for site in base_sites[:2]:  # Limitar a 2 sites por categoria
-                site_query = f"site:{site} {query}"
-                site_urls = search_duckduckgo(site_query, 3)
-                all_urls.extend(site_urls)
-        
-        # 3. Busca em sites personalizados
-        for custom_site in user_custom_sites[:3]:
-            custom_query = f"site:{custom_site} {query}"
-            custom_urls = search_duckduckgo(custom_query, 2)
-            all_urls.extend(custom_urls)
-        
-        # 4. Filtrar domínios excluídos e remover duplicatas
-        filtered_urls = []
-        seen = set()
-        for url in all_urls:
-            if url not in seen and not is_domain_blocked(url, excluded_domains):
-                filtered_urls.append(url)
-                seen.add(url)
-                if len(filtered_urls) >= total_limit:
-                    break
-        
-        logging.info(f"Smart search concluído: {len(filtered_urls)} URLs")
-        return filtered_urls
-    
-    except Exception as e:
-        logging.error(f"Erro no smart search: {e}")
-        return []
-
-def scrape_urls_bulk_impl(urls: List[str]) -> List[Dict]:
-    """Scraping em paralelo de múltiplas URLs"""
-    try:
-        logging.info(f"Scraping bulk iniciado: {len(urls)} URLs")
-        
-        results = []
-        
-        def scrape_with_logging(url):
-            try:
-                result = scrape_url(url, visible=False)
-                if result.get('type') == 'success':
-                    logging.info(f"✓ {url}")
-                else:
-                    logging.warning(f"✗ {url}: {result.get('message', 'unknown error')}")
-                return result
-            except Exception as e:
-                logging.error(f"✗ {url}: {e}")
-                return {"type": "error", "message": str(e), "url": url}
-        
-        # Scraping paralelo com até 5 workers
-        with ThreadPoolExecutor(max_workers=5) as executor:
-            future_to_url = {executor.submit(scrape_with_logging, url): url for url in urls}
-            for future in as_completed(future_to_url):
-                try:
-                    result = future.result()
-                    if result.get('type') == 'success':
-                        results.append(result)
-                except Exception as e:
-                    logging.error(f"Erro no future: {e}")
-        
-        logging.info(f"Scraping bulk concluído: {len(results)} sucessos")
-        return results
-    
-    except Exception as e:
-        logging.error(f"Erro no scraping bulk: {e}")
-        return []
+        logging.error(f"Erro enviando resposta JSON: {e}")
 
 def main():
-    """Entry point do script - lê comandos JSON do stdin"""
-    try:
-        # Ler JSON do stdin
-        input_data = sys.stdin.read().strip()
-        
-        if not input_data:
-            print_json({"type": "error", "message": "No input provided"})
-            return
-        
-        # Parse JSON
+    logging.info("Scraper/Search Service v4.0 (DuckDuckGo Edition) Initialized")
+    while True:
         try:
-            command_data = json.loads(input_data)
-        except json.JSONDecodeError as e:
-            logging.error(f"Erro ao fazer parse do JSON: {e}")
-            print_json({"type": "error", "message": f"Invalid JSON: {str(e)}"})
-            return
-        
-        command = command_data.get("command")
-        logging.info(f"Comando recebido: {command}")
-        
-        # Dispatch de comandos
-        if command == "scrape":
-            url = command_data.get("url")
-            visible = command_data.get("visible", False)
+            line = sys.stdin.readline()
+            if not line: break
+            if not line.strip(): continue
+            try:
+                request = json.loads(line)
+            except json.JSONDecodeError:
+                write_json_response({"response_type": "error", "message": "Invalid JSON"})
+                continue
             
-            if not url:
-                print_json({"type": "error", "message": "URL missing"})
-                return
+            command = request.get("command")
             
-            result = scrape_url(url, visible)
-            print_json(result)
-        
-        elif command == "search_duckduckgo":
-            query = command_data.get("query")
-            limit = command_data.get("limit", 5)
+            if command == "scrape":
+                url = request.get("url")
+                if url:
+                    result = asyncio.run(scrape_url_smart(url))
+                    write_json_response(result)
+                else:
+                    write_json_response({"response_type": "error", "message": "No URL provided"})
             
-            if not query:
-                print_json({"type": "error", "message": "Query missing"})
-                return
-            
-            urls = search_duckduckgo(query, limit)
-            print_json({"type": "success", "urls": urls})
-        
-        elif command == "smart_search":
-            query = command_data.get("query")
-            config = command_data.get("config", {})
-            
-            if not query:
-                print_json({"type": "error", "message": "Query missing"})
-                return
-            
-            urls = smart_search_impl(query, config)
-            print_json({"type": "success", "urls": urls})
-        
-        elif command == "search_and_scrape":
-            query = command_data.get("query")
-            config = command_data.get("config", {})
-            
-            if not query:
-                print_json({"type": "error", "message": "Query missing"})
-                return
-            
-            # Primeiro faz smart_search para obter URLs
-            urls = smart_search_impl(query, config)
-            
-            if not urls:
-                print_json({"type": "success", "results": []})
-                return
-            
-            # Depois faz scraping das URLs
-            results = scrape_urls_bulk_impl(urls)
-            print_json({"type": "success", "results": results})
-        
-        elif command == "scrape_urls_bulk":
-            urls = command_data.get("urls", [])
-            
-            if not urls:
-                print_json({"type": "error", "message": "URLs missing"})
-                return
-            
-            results = scrape_urls_bulk_impl(urls)
-            print_json({"type": "success", "results": results})
-        
-        elif command == "check":
-            # Verificar dependências
-            missing = []
-            if not CURL_CFFI_AVAILABLE:
-                missing.append("curl_cffi")
-            if not BS4_AVAILABLE:
-                missing.append("beautifulsoup4")
-            if not HTML2TEXT_AVAILABLE:
-                missing.append("html2text")
-            
-            if missing:
-                print_json({"type": "error", "message": f"Missing: {', '.join(missing)}"})
+            elif command == "search":
+                # Generic search command, implemented via DuckDuckGo
+                keywords = request.get("keywords") or request.get("query")
+                region = request.get("region", "wt-wt")
+                safesearch = request.get("safesearch", "moderate")
+                max_results = request.get("max_results", 10)
+                
+                if keywords:
+                    result = asyncio.run(search_duckduckgo(keywords, region, safesearch, max_results))
+                    write_json_response(result)
+                else:
+                    write_json_response({"response_type": "error", "message": "Missing keywords"})
+                    
+            elif command == "ping":
+                write_json_response({"response_type": "pong"})
             else:
-                print_json({"type": "success", "message": "All dependencies available"})
-        
-        else:
-            print_json({"type": "error", "message": f"Unknown command: {command}"})
-    
-    except Exception as e:
-        logging.error(f"Erro no main: {traceback.format_exc()}")
-        print_json({"type": "error", "message": f"Critical error: {str(e)}"})
+                write_json_response({"response_type": "error", "message": "Unknown command"})
+        except KeyboardInterrupt:
+            break
+        except Exception as e:
+            logging.critical(f"Fatal error in main loop: {e}")
+            write_json_response({"response_type": "error", "message": str(e)})
 
 if __name__ == "__main__":
-    try:
-        main()
-    except KeyboardInterrupt:
-        print_json({"type": "error", "message": "Script interrupted"})
-        sys.exit(1)
-    except Exception as e:
-        try:
-            logging.error(f"Unhandled exception: {traceback.format_exc()}")
-            print_json({"type": "error", "message": f"Unhandled exception: {str(e)}"})
-        except:
-            try:
-                print(json.dumps({"type": "error", "message": "Critical failure"}))
-            except:
-                pass
-        sys.exit(1)
+    main()