chore: expurgo de blobs excedentes e realinhamento de estado
release.yml (push) Invalid workflow file
release.yml (push) Invalid workflow file
This commit is contained in:
Evandro Fonseca Junior committed
1 parent
593efd42e0
commit
a7cbd1a3b4
114 files changed
+20716
-4258
No files matched your search
Binary file not shown.
@@ -2,3 +2,7 @@ curl_cffi>=0.5.10
|
||||
beautifulsoup4>=4.12.0
|
||||
lxml>=4.9.0
|
||||
html2text>=2024.1.0
|
||||
playwright>=1.40.0
|
||||
fake-useragent>=1.4.0
|
||||
duckduckgo-search>=4.0.0
|
||||
nodriver>=0.33
|
||||
+334
-466
@@ -1,499 +1,367 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Script Python para scraping web com curl_cffi + BeautifulSoup
|
||||
Pipeline: curl_cffi (Request HTTP) -> BeautifulSoup (Parsing) -> html2text (Formatação)
|
||||
Comunica via JSON stdin/stdout com o backend Rust
|
||||
Micro-serviço Python para scraping e busca
|
||||
Pipeline:
|
||||
1. Scraping: curl_cffi / Playwright Stealth
|
||||
2. Busca: duckduckgo-search
|
||||
"""
|
||||
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
import logging
|
||||
import traceback
|
||||
from typing import List, Dict, Optional
|
||||
from urllib.parse import urljoin, urlparse, quote
|
||||
import random
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
import time
|
||||
import os
|
||||
import asyncio
|
||||
from itertools import cycle
|
||||
from typing import Dict, Optional, Union, List
|
||||
from urllib.parse import urljoin
|
||||
|
||||
# Bibliotecas de Scraping e Parsing
|
||||
from bs4 import BeautifulSoup
|
||||
from curl_cffi import requests
|
||||
import html2text
|
||||
|
||||
# DuckDuckGo Search
|
||||
try:
|
||||
from duckduckgo_search import DDGS
|
||||
HAS_DDG = True
|
||||
except ImportError:
|
||||
HAS_DDG = False
|
||||
|
||||
# Fallback libs
|
||||
try:
|
||||
import requests as pyrequests
|
||||
HAS_PYREQUESTS = True
|
||||
except ImportError:
|
||||
HAS_PYREQUESTS = False
|
||||
|
||||
try:
|
||||
import httpx
|
||||
HAS_HTTPX = True
|
||||
except ImportError:
|
||||
HAS_HTTPX = False
|
||||
|
||||
# Playwright & Anti-Detect
|
||||
try:
|
||||
from playwright.async_api import async_playwright, TimeoutError as PlaywrightTimeoutError
|
||||
HAS_PLAYWRIGHT = True
|
||||
except ImportError:
|
||||
HAS_PLAYWRIGHT = False
|
||||
|
||||
try:
|
||||
from fake_useragent import UserAgent
|
||||
ua_generator = UserAgent()
|
||||
HAS_FAKE_UA = True
|
||||
except ImportError:
|
||||
HAS_FAKE_UA = False
|
||||
ua_generator = None
|
||||
|
||||
# --- CONFIGURAÇÃO DO AMBIENTE ---
|
||||
|
||||
# Configuração CRÍTICA de Encoding para Windows
|
||||
try:
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
sys.stderr.reconfigure(encoding='utf-8')
|
||||
except (AttributeError, Exception):
|
||||
pass
|
||||
|
||||
# Configuração de Logging para STDERR (Não polui o STDOUT)
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='[PYTHON LOG] %(message)s',
|
||||
format='[PYTHON SCRAPER] %(levelname)s: %(message)s',
|
||||
stream=sys.stderr
|
||||
)
|
||||
|
||||
def print_json(data):
|
||||
"""Função auxiliar para garantir output limpo em JSON"""
|
||||
# --- CONSTANTES E POOLS ---
|
||||
|
||||
UA_POOL_FALLBACK = [
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15',
|
||||
'Mozilla/5.0 (X11; Linux x86_64; rv:132.0) Gecko/20100101 Firefox/132.0',
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0'
|
||||
]
|
||||
ua_cycle = cycle(UA_POOL_FALLBACK)
|
||||
|
||||
REQUEST_TIMEOUT = 20
|
||||
MAX_RETRIES = 2
|
||||
|
||||
PROXY_POOL = None
|
||||
env_proxies = os.environ.get('SCRAPER_PROXIES') or os.environ.get('SCRAPER_HTTP_PROXIES')
|
||||
if env_proxies:
|
||||
parts = [p.strip() for p in env_proxies.split(',') if p.strip()]
|
||||
if parts:
|
||||
PROXY_POOL = cycle(parts)
|
||||
|
||||
# --- FUNÇÕES AUXILIARES ---
|
||||
|
||||
def get_random_ua() -> str:
|
||||
if HAS_FAKE_UA:
|
||||
try:
|
||||
return ua_generator.random
|
||||
except Exception:
|
||||
pass
|
||||
return next(ua_cycle)
|
||||
|
||||
def generate_consistent_headers(user_agent: str) -> Dict[str, str]:
|
||||
base_headers = {
|
||||
'User-Agent': user_agent,
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
|
||||
'Accept-Language': 'pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7',
|
||||
'Accept-Encoding': 'gzip, deflate, br',
|
||||
'DNT': '1',
|
||||
'Connection': 'keep-alive',
|
||||
'Upgrade-Insecure-Requests': '1',
|
||||
'Sec-Fetch-Dest': 'document',
|
||||
'Sec-Fetch-Mode': 'navigate',
|
||||
'Sec-Fetch-Site': 'none',
|
||||
'Sec-Fetch-User': '?1',
|
||||
'Cache-Control': 'max-age=0'
|
||||
}
|
||||
if 'Chrome' in user_agent:
|
||||
try:
|
||||
version = user_agent.split('Chrome/')[1].split('.')[0]
|
||||
except IndexError:
|
||||
version = "120"
|
||||
base_headers.update({
|
||||
'sec-ch-ua': f'"Chromium";v="{version}", "Google Chrome";v="{version}", "Not-A.Brand";v="99"',
|
||||
'sec-ch-ua-mobile': '?0',
|
||||
'sec-ch-ua-platform': '"Windows"' if 'Windows' in user_agent else ('"macOS"' if 'Macintosh' in user_agent else '"Linux"')
|
||||
})
|
||||
return base_headers
|
||||
|
||||
def get_proxy_dict() -> Optional[Dict[str, str]]:
|
||||
if PROXY_POOL:
|
||||
proxy = next(PROXY_POOL)
|
||||
return {"http": proxy, "https": proxy}
|
||||
return None
|
||||
|
||||
def random_jitter(min_sec=0.5, max_sec=2.0):
|
||||
time.sleep(random.uniform(min_sec, max_sec))
|
||||
|
||||
# --- LIMPEZA E EXTRAÇÃO ---
|
||||
|
||||
def clean_html(soup: BeautifulSoup) -> BeautifulSoup:
|
||||
for tag in soup(["script", "style", "meta", "link", "noscript", "iframe", "object", "embed", "applet", "canvas", "svg"]):
|
||||
tag.decompose()
|
||||
for tag in soup.find_all(["nav", "header", "footer", "aside", "menu", "dialog"]):
|
||||
tag.decompose()
|
||||
ad_keywords = ["ads", "advertisement", "banner", "ad-", "popup", "modal", "cookie", "subscription", "newsletter", "share", "social", "sidebar", "promo", "sponsor"]
|
||||
selectors_to_remove = ['.ad', '.advertisement', '[class*="ad-"]', '[id*="ad-"]', '[id*="cookie"]', '[class*="cookie"]', '.share-buttons', '.social-media', '.comments-section', '#comments']
|
||||
for selector in selectors_to_remove:
|
||||
for tag in soup.select(selector):
|
||||
tag.decompose()
|
||||
def is_ad(tag):
|
||||
if not tag.name: return False
|
||||
attrs = str(tag.attrs).lower()
|
||||
return any(k in attrs for k in ad_keywords)
|
||||
for tag in soup.find_all(is_ad):
|
||||
tag.decompose()
|
||||
return soup
|
||||
|
||||
def extract_main_content(soup: BeautifulSoup) -> BeautifulSoup:
|
||||
selectors = ['article', 'main', '[role="main"]', '.post-content', '.article-body', '#content', '.content', '.entry-content', '.page-content']
|
||||
for selector in selectors:
|
||||
found = soup.select_one(selector)
|
||||
if found and len(found.get_text(strip=True)) > 200:
|
||||
return found
|
||||
body = soup.find('body')
|
||||
return body if body else soup
|
||||
|
||||
def html_to_markdown(html_content: str) -> str:
|
||||
try:
|
||||
print(json.dumps(data, ensure_ascii=False))
|
||||
h = html2text.HTML2Text()
|
||||
h.ignore_links = False
|
||||
h.ignore_images = False
|
||||
h.images_to_alt = True
|
||||
h.body_width = 0
|
||||
h.protect_links = True
|
||||
return h.handle(html_content)
|
||||
except Exception as e:
|
||||
logging.warning(f"Erro na conversão Markdown: {e}")
|
||||
return html_content
|
||||
|
||||
# --- ESTRATÉGIAS DE SCRAPING ---
|
||||
|
||||
async def scrape_with_playwright(url: str) -> Dict:
|
||||
if not HAS_PLAYWRIGHT:
|
||||
return {"error": "Playwright not installed"}
|
||||
logging.info(f"[PLAYWRIGHT] Iniciando scraping stealth para: {url}")
|
||||
async with async_playwright() as p:
|
||||
args = ['--disable-blink-features=AutomationControlled', '--disable-dev-shm-usage', '--no-sandbox', '--disable-setuid-sandbox', '--disable-infobars', '--window-size=1920,1080', '--start-maximized']
|
||||
try:
|
||||
browser = await p.chromium.launch(headless=True, args=args)
|
||||
ua = get_random_ua()
|
||||
context = await browser.new_context(user_agent=ua, viewport={'width': 1920, 'height': 1080}, locale='pt-BR', timezone_id='America/Sao_Paulo', java_script_enabled=True, ignore_https_errors=True)
|
||||
await context.add_init_script("""
|
||||
Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
|
||||
window.chrome = { runtime: {} };
|
||||
Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5] });
|
||||
Object.defineProperty(navigator, 'languages', { get: () => ['pt-BR', 'pt', 'en-US', 'en'] });
|
||||
""")
|
||||
page = await context.new_page()
|
||||
try:
|
||||
await page.goto(url, wait_until='domcontentloaded', timeout=25000)
|
||||
await page.wait_for_timeout(2000)
|
||||
except PlaywrightTimeoutError:
|
||||
logging.warning(f"[PLAYWRIGHT] Timeout carregando {url}")
|
||||
content = await page.content()
|
||||
title = await page.title()
|
||||
await browser.close()
|
||||
return {"content": content, "title": title, "method": "playwright"}
|
||||
except Exception as e:
|
||||
logging.error(f"[PLAYWRIGHT] Erro fatal: {e}")
|
||||
return {"error": str(e)}
|
||||
|
||||
def scrape_with_curl_cffi(url: str) -> Dict:
|
||||
ua = get_random_ua()
|
||||
headers = generate_consistent_headers(ua)
|
||||
proxies = get_proxy_dict()
|
||||
try:
|
||||
response = requests.get(url, headers=headers, proxies=proxies, timeout=15, impersonate="chrome120", allow_redirects=True)
|
||||
response.raise_for_status()
|
||||
if response.status_code in [403, 429, 503]:
|
||||
raise requests.RequestsError(f"Blocked with status {response.status_code}")
|
||||
return {"content": response.text, "title": "", "method": "curl_cffi", "headers": response.headers}
|
||||
except Exception as e:
|
||||
return {"error": str(e)}
|
||||
|
||||
async def scrape_url_smart(url: str) -> Dict:
|
||||
random_jitter()
|
||||
logging.info(f"Tentativa 1: curl_cffi -> {url}")
|
||||
result = scrape_with_curl_cffi(url)
|
||||
html_content = None
|
||||
title = ""
|
||||
if "error" not in result:
|
||||
html_content = result["content"]
|
||||
if "challenge-platform" in html_content or "Cloudflare" in html_content and len(html_content) < 5000:
|
||||
logging.warning("Detectado desafio Cloudflare/WAF no curl_cffi. Acionando fallback.")
|
||||
html_content = None
|
||||
if not html_content:
|
||||
if HAS_PLAYWRIGHT:
|
||||
logging.info(f"Tentativa 2: Playwright Stealth -> {url}")
|
||||
pw_result = await scrape_with_playwright(url)
|
||||
if "error" not in pw_result:
|
||||
html_content = pw_result["content"]
|
||||
title = pw_result.get("title", "")
|
||||
else:
|
||||
logging.error(f"Playwright falhou: {pw_result['error']}")
|
||||
else:
|
||||
logging.warning("Playwright não disponível para fallback.")
|
||||
if html_content:
|
||||
try:
|
||||
soup = BeautifulSoup(html_content, 'html.parser')
|
||||
if not title:
|
||||
title = soup.title.string.strip() if soup.title else url
|
||||
soup = clean_html(soup)
|
||||
main_content = extract_main_content(soup)
|
||||
cleaned_html = str(main_content)
|
||||
markdown_text = html_to_markdown(cleaned_html)
|
||||
final_markdown = '\n'.join([line.strip() for line in markdown_text.split('\n') if line.strip()])
|
||||
if len(final_markdown) < 50:
|
||||
return {"response_type": "error", "message": "Conteúdo insuficiente após processamento."}
|
||||
return {"response_type": "success", "url": url, "title": title, "markdown": final_markdown, "content": final_markdown}
|
||||
except Exception as e:
|
||||
logging.error(f"Erro no processamento HTML: {traceback.format_exc()}")
|
||||
return {"response_type": "error", "message": f"Processing error: {str(e)}"}
|
||||
return {"response_type": "error", "message": f"Falha em todos os métodos de scraping. Erro inicial: {result.get('error', 'Unknown')}"}
|
||||
|
||||
# --- DUCKDUCKGO SEARCH LOGIC ---
|
||||
|
||||
async def search_duckduckgo(keywords: str, region: str = "wt-wt", safesearch: str = "moderate", max_results: int = 10) -> Dict:
|
||||
if not HAS_DDG:
|
||||
return {"response_type": "error", "message": "duckduckgo-search module not installed"}
|
||||
|
||||
logging.info(f"Searching DuckDuckGo: {keywords} (region={region}, safe={safesearch})")
|
||||
|
||||
try:
|
||||
results_list = []
|
||||
# DDGS is synchronous, run in thread executor to avoid blocking async loop if needed
|
||||
# But since this script main loop is simple, direct call is fine, or wrap in to_thread
|
||||
|
||||
with DDGS() as ddgs:
|
||||
# ddgs.text() is a generator
|
||||
ddg_gen = ddgs.text(
|
||||
keywords=keywords,
|
||||
region=region,
|
||||
safesearch=safesearch,
|
||||
max_results=max_results
|
||||
)
|
||||
|
||||
for r in ddg_gen:
|
||||
results_list.append({
|
||||
"title": r.get("title", ""),
|
||||
"url": r.get("href", ""),
|
||||
"content": r.get("body", ""),
|
||||
"engine": "duckduckgo",
|
||||
"score": 1.0
|
||||
})
|
||||
|
||||
return {
|
||||
"response_type": "success",
|
||||
"query": keywords,
|
||||
"results": results_list,
|
||||
"number_of_results": len(results_list)
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"DuckDuckGo Search Error: {e}")
|
||||
return {"response_type": "error", "message": str(e)}
|
||||
|
||||
# --- MAIN ---
|
||||
|
||||
def write_json_response(data: Dict):
|
||||
try:
|
||||
json_line = json.dumps(data, separators=(',', ':'), ensure_ascii=False)
|
||||
print(json_line)
|
||||
sys.stdout.flush()
|
||||
except Exception as e:
|
||||
try:
|
||||
print(json.dumps({"type": "error", "message": f"Failed to format output: {str(e)}"}))
|
||||
sys.stdout.flush()
|
||||
except:
|
||||
pass
|
||||
|
||||
# Tentar importar dependências
|
||||
CURL_CFFI_AVAILABLE = False
|
||||
BS4_AVAILABLE = False
|
||||
HTML2TEXT_AVAILABLE = False
|
||||
|
||||
try:
|
||||
from curl_cffi import requests
|
||||
CURL_CFFI_AVAILABLE = True
|
||||
except ImportError as e:
|
||||
logging.error(f"curl_cffi não disponível: {e}")
|
||||
|
||||
try:
|
||||
from bs4 import BeautifulSoup
|
||||
BS4_AVAILABLE = True
|
||||
except ImportError as e:
|
||||
logging.error(f"BeautifulSoup4 não disponível: {e}")
|
||||
|
||||
try:
|
||||
import html2text
|
||||
HTML2TEXT_AVAILABLE = True
|
||||
except ImportError as e:
|
||||
logging.error(f"html2text não disponível: {e}")
|
||||
|
||||
# Pool de User Agents para parecer mais humano
|
||||
USER_AGENTS = [
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/121.0",
|
||||
]
|
||||
|
||||
def get_random_user_agent():
|
||||
"""Retorna um user agent aleatório"""
|
||||
return random.choice(USER_AGENTS)
|
||||
|
||||
def clean_url(url: str) -> str:
|
||||
"""Remove fragmentos e parâmetros de rastreamento da URL"""
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
return f"{parsed.scheme}://{parsed.netloc}{parsed.path}"
|
||||
except:
|
||||
return url
|
||||
|
||||
def extract_domain(url: str) -> str:
|
||||
"""Extrai o domínio de uma URL"""
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
return parsed.netloc
|
||||
except:
|
||||
return ""
|
||||
|
||||
def is_ad_or_tracker_url(url: str) -> bool:
|
||||
"""Verifica se uma URL é de anúncio ou rastreador"""
|
||||
trackers = ['analytics', 'google-analytics', 'doubleclick', 'facebook.com/tr', 'ads.', 'banner', 'tracking']
|
||||
return any(tracker in url.lower() for tracker in trackers)
|
||||
|
||||
def is_domain_blocked(url: str, excluded_domains: List[str]) -> bool:
|
||||
"""Verifica se o domínio da URL está na lista de exclusão"""
|
||||
domain = extract_domain(url)
|
||||
return any(excluded in domain for excluded in excluded_domains)
|
||||
|
||||
def scrape_url(url: str, visible: bool = False) -> Dict:
|
||||
"""Função principal de scraping com curl_cffi + BeautifulSoup"""
|
||||
|
||||
# Verificar dependências
|
||||
if not CURL_CFFI_AVAILABLE:
|
||||
return {
|
||||
"type": "error",
|
||||
"message": "curl_cffi não está instalado"
|
||||
}
|
||||
|
||||
if not BS4_AVAILABLE:
|
||||
return {
|
||||
"type": "error",
|
||||
"message": "beautifulsoup4 não está instalado"
|
||||
}
|
||||
|
||||
if not HTML2TEXT_AVAILABLE:
|
||||
return {
|
||||
"type": "error",
|
||||
"message": "html2text não está instalado"
|
||||
}
|
||||
|
||||
try:
|
||||
logging.info(f"Iniciando requisição para: {url}")
|
||||
|
||||
# Criar headers com curl_cffi (impersonate navegador)
|
||||
headers = {
|
||||
"User-Agent": get_random_user_agent(),
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
||||
"Accept-Language": "pt-BR,pt;q=0.9,en;q=0.8",
|
||||
"Accept-Encoding": "gzip, deflate",
|
||||
"DNT": "1",
|
||||
"Connection": "keep-alive",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
}
|
||||
|
||||
# Executar requisição com timeout
|
||||
try:
|
||||
response = requests.get(
|
||||
url,
|
||||
headers=headers,
|
||||
timeout=15,
|
||||
impersonate="chrome120",
|
||||
allow_redirects=True,
|
||||
follow_redirects=True
|
||||
)
|
||||
response.raise_for_status()
|
||||
except Exception as e:
|
||||
logging.error(f"Erro na requisição HTTP: {e}")
|
||||
return {
|
||||
"type": "error",
|
||||
"message": f"Failed to fetch URL: {str(e)}"
|
||||
}
|
||||
|
||||
# Verificar se recebemos conteúdo HTML válido
|
||||
if len(response.content) < 100:
|
||||
return {
|
||||
"type": "error",
|
||||
"message": "Resposta muito curta (< 100 bytes)"
|
||||
}
|
||||
|
||||
logging.info(f"Resposta recebida: {len(response.content)} bytes")
|
||||
|
||||
# Parse com BeautifulSoup
|
||||
try:
|
||||
soup = BeautifulSoup(response.content, 'html.parser')
|
||||
except Exception as e:
|
||||
logging.error(f"Erro ao fazer parse HTML: {e}")
|
||||
return {
|
||||
"type": "error",
|
||||
"message": f"Failed to parse HTML: {str(e)}"
|
||||
}
|
||||
|
||||
# Extrair título
|
||||
title = "Sem título"
|
||||
title_tag = soup.find('meta', property='og:title') or soup.find('meta', {'name': 'title'}) or soup.find('title')
|
||||
if title_tag:
|
||||
title = title_tag.get('content') or title_tag.string or title
|
||||
|
||||
# Remover scripts e styles
|
||||
for script in soup(["script", "style", "meta", "link", "noscript"]):
|
||||
script.decompose()
|
||||
|
||||
# Remover elementos desnecessários
|
||||
for elem in soup.find_all(["nav", "header", "footer", "aside", "ads"]):
|
||||
elem.decompose()
|
||||
|
||||
# Extrair conteúdo principal
|
||||
main_content = None
|
||||
for selector in ['article', 'main', '[role="main"]', '.content', '.post', '.entry']:
|
||||
if selector.startswith('['):
|
||||
main_content = soup.select_one(selector)
|
||||
else:
|
||||
main_content = soup.find(selector)
|
||||
if main_content and len(main_content.get_text(strip=True)) > 200:
|
||||
break
|
||||
|
||||
# Fallback: usar body
|
||||
if not main_content or len(main_content.get_text(strip=True)) < 200:
|
||||
main_content = soup.find('body') or soup
|
||||
|
||||
# Obter HTML do conteúdo principal
|
||||
html_content = str(main_content) if main_content else str(soup)
|
||||
|
||||
# Converter para Markdown usando html2text
|
||||
try:
|
||||
h = html2text.HTML2Text()
|
||||
h.ignore_links = False
|
||||
h.ignore_images = False
|
||||
h.body_width = 0
|
||||
markdown_content = h.handle(html_content)
|
||||
except Exception as e:
|
||||
logging.warning(f"Erro ao converter para markdown, usando texto plano: {e}")
|
||||
markdown_content = main_content.get_text(separator="\n") if main_content else response.text
|
||||
|
||||
# Limpeza final
|
||||
lines = [line.strip() for line in markdown_content.split('\n') if line.strip()]
|
||||
final_content = '\n'.join(lines)
|
||||
|
||||
# Validação de conteúdo
|
||||
if len(final_content) < 100:
|
||||
return {
|
||||
"type": "error",
|
||||
"message": f"Conteúdo insuficiente ({len(final_content)} chars)"
|
||||
}
|
||||
|
||||
logging.info(f"Sucesso! {len(final_content)} chars extraídos")
|
||||
|
||||
return {
|
||||
"type": "success",
|
||||
"url": clean_url(url),
|
||||
"title": title[:500],
|
||||
"content": final_content[:30000],
|
||||
"markdown": final_content[:30000]
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Erro no scraping: {traceback.format_exc()}")
|
||||
return {
|
||||
"type": "error",
|
||||
"message": str(e)
|
||||
}
|
||||
|
||||
def search_duckduckgo(query: str, limit: int = 5) -> List[str]:
|
||||
"""Realiza busca no DuckDuckGo e retorna URLs"""
|
||||
try:
|
||||
if not CURL_CFFI_AVAILABLE or not BS4_AVAILABLE:
|
||||
logging.error("Dependências não disponíveis")
|
||||
return []
|
||||
|
||||
logging.info(f"Buscando no DuckDuckGo: {query}")
|
||||
|
||||
headers = {
|
||||
"User-Agent": get_random_user_agent(),
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "pt-BR,pt;q=0.9,en;q=0.8",
|
||||
}
|
||||
|
||||
search_url = f"https://duckduckgo.com/html/?q={quote(query)}"
|
||||
response = requests.get(
|
||||
search_url,
|
||||
headers=headers,
|
||||
timeout=15,
|
||||
impersonate="chrome120",
|
||||
allow_redirects=True
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
soup = BeautifulSoup(response.content, 'html.parser')
|
||||
|
||||
urls = []
|
||||
# DuckDuckGo HTML retorna resultados em links diretos
|
||||
for result in soup.find_all('a', {'class': 'result__a'}):
|
||||
href = result.get('href')
|
||||
if href and not is_ad_or_tracker_url(href):
|
||||
urls.append(href)
|
||||
if len(urls) >= limit:
|
||||
break
|
||||
|
||||
logging.info(f"Encontradas {len(urls)} URLs")
|
||||
return urls
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Erro na busca DuckDuckGo: {e}")
|
||||
return []
|
||||
|
||||
def smart_search_impl(query: str, config: Dict) -> List[str]:
|
||||
"""Implementa busca inteligente com categorias e filtros"""
|
||||
try:
|
||||
logging.info(f"Smart search iniciado: {query}")
|
||||
|
||||
total_limit = config.get('total_sources_limit', 15)
|
||||
categories = config.get('categories', [])
|
||||
user_custom_sites = config.get('user_custom_sites', [])
|
||||
excluded_domains = config.get('excluded_domains', [])
|
||||
|
||||
all_urls = []
|
||||
|
||||
# 1. Busca geral no DuckDuckGo
|
||||
general_urls = search_duckduckgo(query, total_limit // 2)
|
||||
all_urls.extend(general_urls)
|
||||
|
||||
# 2. Busca por categorias habilitadas
|
||||
for category in categories:
|
||||
if not category.get('enabled', False):
|
||||
continue
|
||||
|
||||
base_sites = category.get('base_sites', [])
|
||||
for site in base_sites[:2]: # Limitar a 2 sites por categoria
|
||||
site_query = f"site:{site} {query}"
|
||||
site_urls = search_duckduckgo(site_query, 3)
|
||||
all_urls.extend(site_urls)
|
||||
|
||||
# 3. Busca em sites personalizados
|
||||
for custom_site in user_custom_sites[:3]:
|
||||
custom_query = f"site:{custom_site} {query}"
|
||||
custom_urls = search_duckduckgo(custom_query, 2)
|
||||
all_urls.extend(custom_urls)
|
||||
|
||||
# 4. Filtrar domínios excluídos e remover duplicatas
|
||||
filtered_urls = []
|
||||
seen = set()
|
||||
for url in all_urls:
|
||||
if url not in seen and not is_domain_blocked(url, excluded_domains):
|
||||
filtered_urls.append(url)
|
||||
seen.add(url)
|
||||
if len(filtered_urls) >= total_limit:
|
||||
break
|
||||
|
||||
logging.info(f"Smart search concluído: {len(filtered_urls)} URLs")
|
||||
return filtered_urls
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Erro no smart search: {e}")
|
||||
return []
|
||||
|
||||
def scrape_urls_bulk_impl(urls: List[str]) -> List[Dict]:
|
||||
"""Scraping em paralelo de múltiplas URLs"""
|
||||
try:
|
||||
logging.info(f"Scraping bulk iniciado: {len(urls)} URLs")
|
||||
|
||||
results = []
|
||||
|
||||
def scrape_with_logging(url):
|
||||
try:
|
||||
result = scrape_url(url, visible=False)
|
||||
if result.get('type') == 'success':
|
||||
logging.info(f"✓ {url}")
|
||||
else:
|
||||
logging.warning(f"✗ {url}: {result.get('message', 'unknown error')}")
|
||||
return result
|
||||
except Exception as e:
|
||||
logging.error(f"✗ {url}: {e}")
|
||||
return {"type": "error", "message": str(e), "url": url}
|
||||
|
||||
# Scraping paralelo com até 5 workers
|
||||
with ThreadPoolExecutor(max_workers=5) as executor:
|
||||
future_to_url = {executor.submit(scrape_with_logging, url): url for url in urls}
|
||||
for future in as_completed(future_to_url):
|
||||
try:
|
||||
result = future.result()
|
||||
if result.get('type') == 'success':
|
||||
results.append(result)
|
||||
except Exception as e:
|
||||
logging.error(f"Erro no future: {e}")
|
||||
|
||||
logging.info(f"Scraping bulk concluído: {len(results)} sucessos")
|
||||
return results
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Erro no scraping bulk: {e}")
|
||||
return []
|
||||
logging.error(f"Erro enviando resposta JSON: {e}")
|
||||
|
||||
def main():
|
||||
"""Entry point do script - lê comandos JSON do stdin"""
|
||||
try:
|
||||
# Ler JSON do stdin
|
||||
input_data = sys.stdin.read().strip()
|
||||
|
||||
if not input_data:
|
||||
print_json({"type": "error", "message": "No input provided"})
|
||||
return
|
||||
|
||||
# Parse JSON
|
||||
logging.info("Scraper/Search Service v4.0 (DuckDuckGo Edition) Initialized")
|
||||
while True:
|
||||
try:
|
||||
command_data = json.loads(input_data)
|
||||
except json.JSONDecodeError as e:
|
||||
logging.error(f"Erro ao fazer parse do JSON: {e}")
|
||||
print_json({"type": "error", "message": f"Invalid JSON: {str(e)}"})
|
||||
return
|
||||
|
||||
command = command_data.get("command")
|
||||
logging.info(f"Comando recebido: {command}")
|
||||
|
||||
# Dispatch de comandos
|
||||
if command == "scrape":
|
||||
url = command_data.get("url")
|
||||
visible = command_data.get("visible", False)
|
||||
line = sys.stdin.readline()
|
||||
if not line: break
|
||||
if not line.strip(): continue
|
||||
try:
|
||||
request = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
write_json_response({"response_type": "error", "message": "Invalid JSON"})
|
||||
continue
|
||||
|
||||
if not url:
|
||||
print_json({"type": "error", "message": "URL missing"})
|
||||
return
|
||||
command = request.get("command")
|
||||
|
||||
result = scrape_url(url, visible)
|
||||
print_json(result)
|
||||
|
||||
elif command == "search_duckduckgo":
|
||||
query = command_data.get("query")
|
||||
limit = command_data.get("limit", 5)
|
||||
if command == "scrape":
|
||||
url = request.get("url")
|
||||
if url:
|
||||
result = asyncio.run(scrape_url_smart(url))
|
||||
write_json_response(result)
|
||||
else:
|
||||
write_json_response({"response_type": "error", "message": "No URL provided"})
|
||||
|
||||
if not query:
|
||||
print_json({"type": "error", "message": "Query missing"})
|
||||
return
|
||||
|
||||
urls = search_duckduckgo(query, limit)
|
||||
print_json({"type": "success", "urls": urls})
|
||||
|
||||
elif command == "smart_search":
|
||||
query = command_data.get("query")
|
||||
config = command_data.get("config", {})
|
||||
|
||||
if not query:
|
||||
print_json({"type": "error", "message": "Query missing"})
|
||||
return
|
||||
|
||||
urls = smart_search_impl(query, config)
|
||||
print_json({"type": "success", "urls": urls})
|
||||
|
||||
elif command == "search_and_scrape":
|
||||
query = command_data.get("query")
|
||||
config = command_data.get("config", {})
|
||||
|
||||
if not query:
|
||||
print_json({"type": "error", "message": "Query missing"})
|
||||
return
|
||||
|
||||
# Primeiro faz smart_search para obter URLs
|
||||
urls = smart_search_impl(query, config)
|
||||
|
||||
if not urls:
|
||||
print_json({"type": "success", "results": []})
|
||||
return
|
||||
|
||||
# Depois faz scraping das URLs
|
||||
results = scrape_urls_bulk_impl(urls)
|
||||
print_json({"type": "success", "results": results})
|
||||
|
||||
elif command == "scrape_urls_bulk":
|
||||
urls = command_data.get("urls", [])
|
||||
|
||||
if not urls:
|
||||
print_json({"type": "error", "message": "URLs missing"})
|
||||
return
|
||||
|
||||
results = scrape_urls_bulk_impl(urls)
|
||||
print_json({"type": "success", "results": results})
|
||||
|
||||
elif command == "check":
|
||||
# Verificar dependências
|
||||
missing = []
|
||||
if not CURL_CFFI_AVAILABLE:
|
||||
missing.append("curl_cffi")
|
||||
if not BS4_AVAILABLE:
|
||||
missing.append("beautifulsoup4")
|
||||
if not HTML2TEXT_AVAILABLE:
|
||||
missing.append("html2text")
|
||||
|
||||
if missing:
|
||||
print_json({"type": "error", "message": f"Missing: {', '.join(missing)}"})
|
||||
elif command == "search":
|
||||
# Generic search command, implemented via DuckDuckGo
|
||||
keywords = request.get("keywords") or request.get("query")
|
||||
region = request.get("region", "wt-wt")
|
||||
safesearch = request.get("safesearch", "moderate")
|
||||
max_results = request.get("max_results", 10)
|
||||
|
||||
if keywords:
|
||||
result = asyncio.run(search_duckduckgo(keywords, region, safesearch, max_results))
|
||||
write_json_response(result)
|
||||
else:
|
||||
write_json_response({"response_type": "error", "message": "Missing keywords"})
|
||||
|
||||
elif command == "ping":
|
||||
write_json_response({"response_type": "pong"})
|
||||
else:
|
||||
print_json({"type": "success", "message": "All dependencies available"})
|
||||
|
||||
else:
|
||||
print_json({"type": "error", "message": f"Unknown command: {command}"})
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Erro no main: {traceback.format_exc()}")
|
||||
print_json({"type": "error", "message": f"Critical error: {str(e)}"})
|
||||
write_json_response({"response_type": "error", "message": "Unknown command"})
|
||||
except KeyboardInterrupt:
|
||||
break
|
||||
except Exception as e:
|
||||
logging.critical(f"Fatal error in main loop: {e}")
|
||||
write_json_response({"response_type": "error", "message": str(e)})
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except KeyboardInterrupt:
|
||||
print_json({"type": "error", "message": "Script interrupted"})
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
try:
|
||||
logging.error(f"Unhandled exception: {traceback.format_exc()}")
|
||||
print_json({"type": "error", "message": f"Unhandled exception: {str(e)}"})
|
||||
except:
|
||||
try:
|
||||
print(json.dumps({"type": "error", "message": "Critical failure"}))
|
||||
except:
|
||||
pass
|
||||
sys.exit(1)
|
||||
main()
|
||||
@@ -0,0 +1,337 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Micro-serviço Python para scraping com Nodriver (anti-detecção)
|
||||
Pipeline:
|
||||
1. Scraping: Nodriver (sem CDP, indetectável)
|
||||
2. Fallback: curl_cffi se Nodriver falhar
|
||||
"""
|
||||
|
||||
import sys
|
||||
import json
|
||||
import logging
|
||||
import traceback
|
||||
import random
|
||||
import time
|
||||
import os
|
||||
import asyncio
|
||||
from typing import Dict, Optional, Union, List
|
||||
from urllib.parse import urljoin
|
||||
|
||||
# Bibliotecas de Scraping e Parsing
|
||||
from bs4 import BeautifulSoup
|
||||
import html2text
|
||||
|
||||
# Nodriver (anti-detecção)
|
||||
try:
|
||||
import nodriver as uc
|
||||
HAS_NODRIVER = True
|
||||
except ImportError:
|
||||
HAS_NODRIVER = False
|
||||
logging.warning("nodriver not installed, falling back to curl_cffi")
|
||||
|
||||
# curl_cffi como fallback
|
||||
try:
|
||||
from curl_cffi import requests
|
||||
HAS_CURL_CFFI = True
|
||||
except ImportError:
|
||||
HAS_CURL_CFFI = False
|
||||
|
||||
# Fallback libs
|
||||
try:
|
||||
import requests as pyrequests
|
||||
HAS_PYREQUESTS = True
|
||||
except ImportError:
|
||||
HAS_PYREQUESTS = False
|
||||
|
||||
# --- CONFIGURAÇÃO DO AMBIENTE ---
|
||||
|
||||
try:
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
sys.stderr.reconfigure(encoding='utf-8')
|
||||
except (AttributeError, Exception):
|
||||
pass
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='[PYTHON SCRAPER NODRIVER] %(levelname)s: %(message)s',
|
||||
stream=sys.stderr
|
||||
)
|
||||
|
||||
# --- CONSTANTES E POOLS ---
|
||||
|
||||
UA_POOL_FALLBACK = [
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15',
|
||||
'Mozilla/5.0 (X11; Linux x86_64; rv:132.0) Gecko/20100101 Firefox/132.0',
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0'
|
||||
]
|
||||
|
||||
REQUEST_TIMEOUT = 12 # Reduzido de 20 para 12s para maior velocidade
|
||||
MAX_RETRIES = 2
|
||||
NODRIVER_TIMEOUT = 10 # Timeout específico para nodriver (10s)
|
||||
|
||||
# --- FUNÇÕES AUXILIARES ---
|
||||
|
||||
def get_random_ua() -> str:
|
||||
return random.choice(UA_POOL_FALLBACK)
|
||||
|
||||
def random_jitter():
|
||||
"""Adiciona jitter aleatório para evitar padrões"""
|
||||
time.sleep(random.uniform(0.1, 0.5))
|
||||
|
||||
def clean_html(soup: BeautifulSoup) -> BeautifulSoup:
|
||||
"""Remove scripts, styles e outros elementos desnecessários"""
|
||||
for script in soup(["script", "style", "nav", "header", "footer", "aside", "noscript"]):
|
||||
script.decompose()
|
||||
return soup
|
||||
|
||||
def extract_main_content(soup: BeautifulSoup) -> BeautifulSoup:
|
||||
"""Extrai conteúdo principal usando heurísticas"""
|
||||
# Tentar encontrar main, article, ou div com classe comum
|
||||
selectors = [
|
||||
'main',
|
||||
'article',
|
||||
'[role="main"]',
|
||||
'.content',
|
||||
'.main-content',
|
||||
'#content',
|
||||
'#main',
|
||||
]
|
||||
|
||||
for selector in selectors:
|
||||
main = soup.select_one(selector)
|
||||
if main:
|
||||
return main
|
||||
|
||||
# Fallback: retornar body
|
||||
return soup.find('body') or soup
|
||||
|
||||
def html_to_markdown(html: str) -> str:
|
||||
"""Converte HTML para Markdown"""
|
||||
h = html2text.HTML2Text()
|
||||
h.ignore_links = False
|
||||
h.ignore_images = False
|
||||
h.body_width = 0
|
||||
return h.handle(html)
|
||||
|
||||
# --- SCRAPING COM NODRIVER ---
|
||||
|
||||
async def scrape_with_nodriver(url: str) -> Dict:
|
||||
"""Scraping usando Nodriver (anti-detecção) - OTIMIZADO"""
|
||||
if not HAS_NODRIVER:
|
||||
return {"error": "nodriver not installed"}
|
||||
|
||||
try:
|
||||
logging.info(f"[NODRIVER] Starting scrape for {url}")
|
||||
start_time = time.time()
|
||||
|
||||
# Iniciar browser com Nodriver (sem CDP, indetectável)
|
||||
browser = await uc.start(headless=True)
|
||||
|
||||
try:
|
||||
# Navegar para URL com timeout
|
||||
page = await asyncio.wait_for(
|
||||
browser.get(url),
|
||||
timeout=NODRIVER_TIMEOUT
|
||||
)
|
||||
|
||||
# Aguardar conteúdo carregar com wait condicional otimizado
|
||||
# Tentar múltiplas estratégias de wait para reduzir tempo total
|
||||
try:
|
||||
# Estratégia 1: Aguardar DOMContentLoaded (mais rápido)
|
||||
await asyncio.wait_for(
|
||||
page.wait_for_load_state('domcontentloaded'),
|
||||
timeout=2.0
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
# Se DOMContentLoaded demorar, usar sleep reduzido
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
# Aguardar um pouco mais para JS executar (reduzido de 2s para 0.3s)
|
||||
await asyncio.sleep(0.3)
|
||||
|
||||
# Extrair HTML
|
||||
html = await page.get_content()
|
||||
title = await page.evaluate('document.title')
|
||||
|
||||
duration = time.time() - start_time
|
||||
logging.info(f"[NODRIVER] Successfully scraped {url} ({len(html)} chars, {duration:.2f}s)")
|
||||
|
||||
return {
|
||||
"content": html,
|
||||
"title": title or "",
|
||||
"method": "nodriver"
|
||||
}
|
||||
finally:
|
||||
await browser.stop()
|
||||
|
||||
except asyncio.TimeoutError:
|
||||
logging.warning(f"[NODRIVER] Timeout after {NODRIVER_TIMEOUT}s for {url}")
|
||||
return {"error": f"Timeout after {NODRIVER_TIMEOUT}s"}
|
||||
except Exception as e:
|
||||
logging.error(f"[NODRIVER] Error scraping {url}: {e}")
|
||||
return {"error": str(e)}
|
||||
|
||||
# --- SCRAPING COM CURL_CFFI (FALLBACK) ---
|
||||
|
||||
def scrape_with_curl_cffi(url: str) -> Dict:
|
||||
"""Scraping usando curl_cffi como fallback - OTIMIZADO"""
|
||||
if not HAS_CURL_CFFI:
|
||||
return {"error": "curl_cffi not installed"}
|
||||
|
||||
ua = get_random_ua()
|
||||
headers = {
|
||||
'User-Agent': ua,
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7',
|
||||
'Accept-Encoding': 'gzip, deflate, br',
|
||||
'Connection': 'keep-alive',
|
||||
'Upgrade-Insecure-Requests': '1',
|
||||
}
|
||||
|
||||
try:
|
||||
# Timeout reduzido para fallback rápido
|
||||
response = requests.get(
|
||||
url,
|
||||
headers=headers,
|
||||
timeout=8, # Reduzido de 15 para 8s
|
||||
impersonate="chrome120",
|
||||
allow_redirects=True
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
if response.status_code in [403, 429, 503]:
|
||||
raise requests.RequestsError(f"Blocked with status {response.status_code}")
|
||||
|
||||
return {
|
||||
"content": response.text,
|
||||
"title": "",
|
||||
"method": "curl_cffi",
|
||||
"headers": dict(response.headers)
|
||||
}
|
||||
except Exception as e:
|
||||
return {"error": str(e)}
|
||||
|
||||
# --- SCRAPING INTELIGENTE ---
|
||||
|
||||
async def scrape_url_smart(url: str) -> Dict:
|
||||
"""Scraping inteligente: Nodriver (PRIMÁRIO) → curl_cffi (fallback) - OTIMIZADO"""
|
||||
# Jitter reduzido para maior velocidade
|
||||
await asyncio.sleep(random.uniform(0.05, 0.15))
|
||||
|
||||
# Tentativa 1: Nodriver (PRIMÁRIO, anti-detecção)
|
||||
if HAS_NODRIVER:
|
||||
logging.info(f"[PRIMARY] Nodriver -> {url}")
|
||||
result = await scrape_with_nodriver(url)
|
||||
|
||||
if "error" not in result:
|
||||
html_content = result["content"]
|
||||
# Verificar se não é página de bloqueio (mais permissivo para velocidade)
|
||||
if len(html_content) > 1000: # Reduzido de 5000 para 1000 para aceitar mais resultados
|
||||
if "challenge-platform" not in html_content.lower() and "cloudflare" not in html_content.lower():
|
||||
return result
|
||||
else:
|
||||
logging.warning("[NODRIVER] Detectado desafio Cloudflare/WAF. Tentando fallback.")
|
||||
elif len(html_content) > 200: # Aceitar conteúdo mínimo
|
||||
logging.info(f"[NODRIVER] Conteúdo mínimo mas aceitável ({len(html_content)} chars)")
|
||||
return result
|
||||
|
||||
# Tentativa 2: curl_cffi (fallback rápido)
|
||||
if HAS_CURL_CFFI:
|
||||
logging.info(f"[FALLBACK] curl_cffi -> {url}")
|
||||
result = scrape_with_curl_cffi(url)
|
||||
|
||||
if "error" not in result:
|
||||
html_content = result["content"]
|
||||
if len(html_content) > 1000:
|
||||
if "challenge-platform" not in html_content.lower() and "cloudflare" not in html_content.lower():
|
||||
return result
|
||||
|
||||
# Se ambos falharam, retornar erro
|
||||
return {"error": "All scraping methods failed"}
|
||||
|
||||
# --- PROCESSAMENTO HTML ---
|
||||
|
||||
async def process_scraped_content(url: str, result: Dict) -> Dict:
|
||||
"""Processa conteúdo scraped e converte para markdown"""
|
||||
if "error" in result:
|
||||
return {"response_type": "error", "message": result["error"]}
|
||||
|
||||
html_content = result.get("content", "")
|
||||
title = result.get("title", "")
|
||||
|
||||
if not html_content:
|
||||
return {"response_type": "error", "message": "No content extracted"}
|
||||
|
||||
try:
|
||||
soup = BeautifulSoup(html_content, 'html.parser')
|
||||
|
||||
if not title:
|
||||
title = soup.title.string.strip() if soup.title else url
|
||||
|
||||
soup = clean_html(soup)
|
||||
main_content = extract_main_content(soup)
|
||||
cleaned_html = str(main_content)
|
||||
markdown_text = html_to_markdown(cleaned_html)
|
||||
final_markdown = '\n'.join([line.strip() for line in markdown_text.split('\n') if line.strip()])
|
||||
|
||||
if len(final_markdown) < 50:
|
||||
return {"response_type": "error", "message": "Conteúdo insuficiente após processamento."}
|
||||
|
||||
return {
|
||||
"response_type": "success",
|
||||
"url": url,
|
||||
"title": title,
|
||||
"markdown": final_markdown,
|
||||
"content": final_markdown
|
||||
}
|
||||
except Exception as e:
|
||||
logging.error(f"Erro no processamento HTML: {traceback.format_exc()}")
|
||||
return {"response_type": "error", "message": f"Processing error: {str(e)}"}
|
||||
|
||||
# --- MAIN LOOP ---
|
||||
|
||||
def write_json_response(data: Dict):
|
||||
try:
|
||||
json_line = json.dumps(data, separators=(',', ':'), ensure_ascii=False)
|
||||
print(json_line)
|
||||
sys.stdout.flush()
|
||||
except Exception as e:
|
||||
logging.error(f"Erro enviando resposta JSON: {e}")
|
||||
|
||||
async def main():
|
||||
logging.info("Scraper/Search Service v5.0 (Nodriver Edition) Initialized")
|
||||
|
||||
while True:
|
||||
try:
|
||||
line = sys.stdin.readline()
|
||||
if not line:
|
||||
break
|
||||
if not line.strip():
|
||||
continue
|
||||
|
||||
try:
|
||||
request = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
write_json_response({"response_type": "error", "message": "Invalid JSON"})
|
||||
continue
|
||||
|
||||
command = request.get("command")
|
||||
|
||||
if command == "scrape":
|
||||
url = request.get("url")
|
||||
if url:
|
||||
result = await scrape_url_smart(url)
|
||||
processed = await process_scraped_content(url, result)
|
||||
write_json_response(processed)
|
||||
else:
|
||||
write_json_response({"response_type": "error", "message": "No URL provided"})
|
||||
else:
|
||||
write_json_response({"response_type": "error", "message": f"Unknown command: {command}"})
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Error in main loop: {traceback.format_exc()}")
|
||||
write_json_response({"response_type": "error", "message": str(e)})
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in new issue
Block a user