chore: expurgo de blobs excedentes e realinhamento de estado
release.yml (push) Invalid workflow file

This commit is contained in:
Evandro Fonseca Junior committed 2026-02-22 14:06:01 -03:00
1 parent 593efd42e0
commit a7cbd1a3b4
114 files changed
+20716 -4258

No files matched your search

+490 -317
View File
File diff suppressed because it is too large. Load diff
+35 -3
View File
@@ -16,6 +16,16 @@ crate-type = ["staticlib", "cdylib", "rlib"]
[build-dependencies]
tauri-build = { version = "2.5.1", features = [] }
zip = { version = "0.6", default-features = false, features = ["deflate"] }
reqwest = { version = "0.12", features = ["blocking", "rustls-tls"] }
sha2 = "0.10"
aes-gcm = "0.10"
base64 = "0.21"
[features]
default = [] # Por padrão, não incluir llama-cpp (requer CUDA/libclang)
llama-cpp = ["dep:llama_cpp"]
dev-no-llama = [] # Skip llama.cpp em dev builds (builds 30-45 min podem ser lentos)
[dependencies]
serde_json = "1.0"
@@ -30,10 +40,7 @@ sysinfo = "0.36.1"
uuid = { version = "1.10", features = ["v4", "serde"] }
chrono = { version = "0.4", features = ["serde"] }
tokio = { version = "1", features = ["full"] }
headless_chrome = "1.0"
scraper = "0.18"
readability = "0.3"
html2text = "0.5"
anyhow = "1.0"
url = "2.5"
urlencoding = "2.1"
@@ -48,6 +55,11 @@ tauri-plugin-updater = "2"
dirs = "5.0"
rfd = "0.14"
rusqlite = { version = "0.31", features = ["bundled"] }
<<<<<<< HEAD
aes-gcm = "0.10"
base64 = "0.21"
=======
>>>>>>> 593efd42e091a845dea82ee6646e027bce1e18c5
# ONNX Runtime para embeddings locais (ranking de relevância)
# Usando load-dynamic para evitar conflito de RuntimeLibrary (MD vs MT) no Windows
@@ -55,3 +67,23 @@ rusqlite = { version = "0.31", features = ["bundled"] }
ort = { version = "2.0.0-rc.10", features = ["load-dynamic"] }
ndarray = "0.16"
tokenizers = "0.21"
<<<<<<< HEAD
sha2 = "0.10"
# Dependências para llama.cpp e parsing GGUF
# Nota: llama_cpp 0.3.x é a versão mais recente disponível
llama_cpp = { version = "0.3", optional = true }
gguf = "0.1"
lru = "0.12"
# Features condicionais por OS (Metal/CUDA são mutuamente exclusivos)
[target.'cfg(target_os = "macos")'.dependencies]
llama_cpp = { version = "0.3", features = ["metal"], optional = true }
[target.'cfg(all(unix, not(target_os = "macos")))'.dependencies]
llama_cpp = { version = "0.3", features = ["cuda"], optional = true }
[target.'cfg(target_os = "windows")'.dependencies]
llama_cpp = { version = "0.3", features = ["cuda"], optional = true }
=======
>>>>>>> 593efd42e091a845dea82ee6646e027bce1e18c5
+444 -1
View File
@@ -1,3 +1,446 @@
use std::path::Path;
use std::process::Command;
use std::fs::File;
use std::io::Read;
use std::io::Write;
use std::time::Duration;
// Constantes para download e validação do OllamaSetup.exe
const OLLAMA_SETUP_URL: &str = "https://ollama.com/download/OllamaSetup.exe";
const MAX_RETRIES: u32 = 3;
const DOWNLOAD_TIMEOUT_SECS: u64 = 300;
// Hash SHA256 esperado do OllamaSetup.exe
// Pode ser sobrescrito via variável de ambiente OLLAMA_SETUP_EXE_SHA256
// IMPORTANTE: Atualizar este valor com o hash SHA256 oficial do instalador
const DEFAULT_EXPECTED_OLLAMA_SETUP_SHA256: &str = "AC9C74B41A8303E5566C5AD3ED6EF0E0123D58AEBC87D3109973F02A38D3FF61";
/// Calcula o hash SHA256 de um arquivo
/// Usa chunks de 8KB para leitura eficiente (padrão do projeto)
fn calculate_sha256(file_path: &Path) -> Result<String, String> {
use sha2::{Sha256, Digest};
let mut file = File::open(file_path)
.map_err(|e| format!("Failed to open file for hash calculation: {}", e))?;
let mut hasher = Sha256::new();
let mut buffer = vec![0u8; 8192]; // Buffer de 8KB para leitura em chunks
loop {
let bytes_read = file.read(&mut buffer)
.map_err(|e| format!("Failed to read file for hash: {}", e))?;
if bytes_read == 0 {
break;
}
hasher.update(&buffer[..bytes_read]);
}
let hash = hasher.finalize();
Ok(format!("{:x}", hash))
}
/// Faz download de um arquivo com retry logic e backoff exponencial
fn download_with_retry(url: &str, dest_path: &Path) -> Result<(), String> {
use reqwest::blocking::Client;
// Limpar arquivo parcial se existir antes de iniciar
if dest_path.exists() {
std::fs::remove_file(dest_path)
.map_err(|e| format!("Failed to remove existing file before download: {}", e))?;
}
let client = Client::builder()
.timeout(Duration::from_secs(DOWNLOAD_TIMEOUT_SECS))
.build()
.map_err(|e| format!("Failed to create HTTP client: {}", e))?;
let mut last_error = None;
for attempt in 1..=MAX_RETRIES {
println!("cargo:warning=Download attempt {}/{}: {}", attempt, MAX_RETRIES, url);
// Limpar arquivo parcial antes de cada tentativa (exceto na primeira, já limpo acima)
if attempt > 1 && dest_path.exists() {
std::fs::remove_file(dest_path)
.map_err(|e| format!("Failed to remove partial file before retry: {}", e))?;
}
// Garantir que o diretório pai existe
if let Some(parent) = dest_path.parent() {
std::fs::create_dir_all(parent)
.map_err(|e| format!("Failed to create parent directory: {}", e))?;
}
match client.get(url).send() {
Ok(response) => {
if !response.status().is_success() {
let err_msg = format!("HTTP error: {}", response.status());
println!("cargo:warning={}", err_msg);
last_error = Some(err_msg);
if attempt < MAX_RETRIES {
let backoff_secs = 2_u64.pow(attempt - 1); // 1s, 2s, 4s
println!("cargo:warning=Waiting {} seconds before retry...", backoff_secs);
std::thread::sleep(Duration::from_secs(backoff_secs));
}
continue;
}
// Obter bytes do response
match response.bytes() {
Ok(bytes) => {
// Criar arquivo e escrever conteúdo
match File::create(dest_path) {
Ok(mut dest_file) => {
match dest_file.write_all(&bytes) {
Ok(_) => {
drop(dest_file); // Fechar arquivo antes de validar
println!("cargo:warning=Download completed successfully");
return Ok(());
}
Err(e) => {
let err_msg = format!("Failed to write file: {}", e);
println!("cargo:warning={}", err_msg);
last_error = Some(err_msg);
// Limpar arquivo parcial
let _ = std::fs::remove_file(dest_path);
if attempt < MAX_RETRIES {
let backoff_secs = 2_u64.pow(attempt - 1);
println!("cargo:warning=Waiting {} seconds before retry...", backoff_secs);
std::thread::sleep(Duration::from_secs(backoff_secs));
}
continue;
}
}
}
Err(e) => {
let err_msg = format!("Failed to create destination file: {}", e);
println!("cargo:warning={}", err_msg);
last_error = Some(err_msg);
if attempt < MAX_RETRIES {
let backoff_secs = 2_u64.pow(attempt - 1);
println!("cargo:warning=Waiting {} seconds before retry...", backoff_secs);
std::thread::sleep(Duration::from_secs(backoff_secs));
}
continue;
}
}
}
Err(e) => {
let err_msg = format!("Failed to read response bytes: {}", e);
println!("cargo:warning={}", err_msg);
last_error = Some(err_msg);
// Limpar arquivo parcial
let _ = std::fs::remove_file(dest_path);
if attempt < MAX_RETRIES {
let backoff_secs = 2_u64.pow(attempt - 1);
println!("cargo:warning=Waiting {} seconds before retry...", backoff_secs);
std::thread::sleep(Duration::from_secs(backoff_secs));
}
continue;
}
}
}
Err(e) => {
let err_msg = format!("Download failed: {}", e);
println!("cargo:warning={}", err_msg);
last_error = Some(err_msg);
if attempt < MAX_RETRIES {
let backoff_secs = 2_u64.pow(attempt - 1);
println!("cargo:warning=Waiting {} seconds before retry...", backoff_secs);
std::thread::sleep(Duration::from_secs(backoff_secs));
}
}
}
}
Err(format!(
"Failed to download {} after {} attempts. Last error: {}",
url,
MAX_RETRIES,
last_error.unwrap_or_else(|| "Unknown error".to_string())
))
}
/// Valida o hash SHA256 de um arquivo contra o hash esperado
fn validate_sha256(file_path: &Path, expected_hash: &str) -> Result<(), String> {
println!("cargo:warning=Validating SHA256 hash...");
let calculated_hash = calculate_sha256(file_path)?;
let calculated_lower = calculated_hash.to_lowercase();
let expected_lower = expected_hash.to_lowercase();
if calculated_lower != expected_lower {
// Remover arquivo corrompido
let _ = std::fs::remove_file(file_path);
return Err(format!(
"SHA256 hash mismatch! Expected: {}, Got: {}. Corrupted file removed.",
expected_hash,
calculated_hash
));
}
println!("cargo:warning=SHA256 hash validated successfully");
Ok(())
}
/// Obtém o hash SHA256 esperado (variável de ambiente ou constante padrão)
fn get_expected_sha256() -> String {
std::env::var("OLLAMA_SETUP_EXE_SHA256")
.unwrap_or_else(|_| DEFAULT_EXPECTED_OLLAMA_SETUP_SHA256.to_string())
}
/// Baixa e valida o instalador do Ollama
fn download_and_validate_ollama_setup(ollama_path: &Path) -> Result<(), String> {
let expected_hash = get_expected_sha256();
// Se o hash esperado está vazio, falhar imediatamente
if expected_hash.is_empty() {
panic!(
"OLLAMA_SETUP_EXE_SHA256 não configurado! \
Configure a variável de ambiente OLLAMA_SETUP_EXE_SHA256 ou \
atualize DEFAULT_EXPECTED_OLLAMA_SETUP_SHA256 no build.rs"
);
}
// Verificar se arquivo já existe
if ollama_path.exists() {
println!("cargo:warning=OllamaSetup.exe already exists, validating hash...");
// Validar hash do arquivo existente
match validate_sha256(ollama_path, &expected_hash) {
Ok(_) => {
println!("cargo:warning=Existing OllamaSetup.exe is valid, skipping download");
return Ok(());
}
Err(e) => {
println!("cargo:warning=Existing file failed validation: {}", e);
println!("cargo:warning=Re-downloading...");
// Continuar para baixar novamente
}
}
}
// Fazer download com retry
println!("cargo:warning=Downloading Ollama installer...");
download_with_retry(OLLAMA_SETUP_URL, ollama_path)?;
// Validar hash após download
validate_sha256(ollama_path, &expected_hash)?;
println!("cargo:warning=Ollama installer downloaded and validated successfully");
Ok(())
}
fn main() {
tauri_build::build()
// Criar estrutura de diretórios para bundle
let bundle_dir = Path::new("bundle");
let ollama_dir = bundle_dir.join("ollama");
let embeddings_dir = bundle_dir.join("models/embeddings");
std::fs::create_dir_all(&ollama_dir).ok();
std::fs::create_dir_all(&embeddings_dir).ok();
#[cfg(target_os = "windows")]
{
println!("cargo:warning=Preparing bundle resources for Windows...");
// 1. Baixar e validar instalador do Ollama com SHA256
let ollama_path = ollama_dir.join("OllamaSetup.exe");
match download_and_validate_ollama_setup(&ollama_path) {
Ok(_) => {}
Err(e) => {
panic!("cargo:error=Failed to download and validate OllamaSetup.exe: {}", e);
}
}
// 2. Baixar modelo de embeddings ONNX (apenas se não existir)
let model_url = "https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/onnx/model.onnx";
let model_path = embeddings_dir.join("all-MiniLM-L6-v2.onnx");
if !model_path.exists() {
println!("cargo:warning=Downloading ONNX embedding model (this may take a while, ~90MB)...");
let status = Command::new("powershell")
.args(&[
"-Command",
&format!(
"if (!(Test-Path '{}')) {{ Invoke-WebRequest -Uri '{}' -OutFile '{}' -UseBasicParsing }}",
model_path.display(),
model_url,
model_path.display()
)
])
.status();
if let Ok(exit_status) = status {
if exit_status.success() {
println!("cargo:warning=ONNX model downloaded successfully");
} else {
println!("cargo:warning=Failed to download ONNX model (non-zero exit code)");
}
} else {
println!("cargo:warning=Failed to download ONNX model (command failed)");
}
} else {
println!("cargo:warning=ONNX model already exists, skipping download");
}
// 3. Baixar tokenizer
let tokenizer_url = "https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/tokenizer.json";
let tokenizer_path = embeddings_dir.join("tokenizer.json");
if !tokenizer_path.exists() {
println!("cargo:warning=Downloading tokenizer...");
let status = Command::new("powershell")
.args(&[
"-Command",
&format!(
"if (!(Test-Path '{}')) {{ Invoke-WebRequest -Uri '{}' -OutFile '{}' -UseBasicParsing }}",
tokenizer_path.display(),
tokenizer_url,
tokenizer_path.display()
)
])
.status();
if let Ok(exit_status) = status {
if exit_status.success() {
println!("cargo:warning=Tokenizer downloaded successfully");
} else {
println!("cargo:warning=Failed to download tokenizer (non-zero exit code)");
}
} else {
println!("cargo:warning=Failed to download tokenizer (command failed)");
}
} else {
println!("cargo:warning=Tokenizer already exists, skipping download");
}
// 4. Baixar e extrair ONNX Runtime DLL
let ort_url = "https://github.com/microsoft/onnxruntime/releases/download/v1.20.1/onnxruntime-win-x64-1.20.1.zip";
let ort_zip_path = embeddings_dir.join("onnxruntime.zip");
let ort_dll_path = embeddings_dir.join("onnxruntime.dll");
if !ort_dll_path.exists() {
if !ort_zip_path.exists() {
println!("cargo:warning=Downloading ONNX Runtime library (~50MB)...");
let status = Command::new("powershell")
.args(&[
"-Command",
&format!(
"if (!(Test-Path '{}')) {{ Invoke-WebRequest -Uri '{}' -OutFile '{}' -UseBasicParsing }}",
ort_zip_path.display(),
ort_url,
ort_zip_path.display()
)
])
.status();
if let Ok(exit_status) = status {
if !exit_status.success() {
println!("cargo:warning=Failed to download ONNX Runtime (non-zero exit code)");
} else {
// Aguardar um pouco para garantir que o download foi concluído
std::thread::sleep(std::time::Duration::from_secs(1));
}
} else {
println!("cargo:warning=Failed to download ONNX Runtime (command failed)");
}
}
// Extrair DLL do zip se o zip existe e está completo
if ort_zip_path.exists() && !ort_dll_path.exists() {
// Verificar se o arquivo zip não está vazio e tem tamanho mínimo razoável (~40MB)
let min_zip_size = 40 * 1024 * 1024; // 40MB
let zip_valid = if let Ok(metadata) = std::fs::metadata(&ort_zip_path) {
let size = metadata.len();
if size < 1000 {
println!("cargo:warning=ONNX Runtime zip file appears to be incomplete (too small: {} bytes), will be downloaded at runtime", size);
false
} else if size < min_zip_size {
println!("cargo:warning=ONNX Runtime zip file appears to be incomplete ({} bytes, expected at least {} bytes), will be re-downloaded", size, min_zip_size);
// Tentar remover ZIP corrompido para forçar re-download
let _ = std::fs::remove_file(&ort_zip_path);
false
} else {
true
}
} else {
println!("cargo:warning=Failed to get metadata for ONNX Runtime zip file");
false
};
if zip_valid {
println!("cargo:warning=Extracting ONNX Runtime DLL from zip...");
// Usar zip crate para extrair (mais confiável que PowerShell)
match extract_onnx_dll_from_zip(&ort_zip_path, &ort_dll_path) {
Ok(_) => {
println!("cargo:warning=ONNX Runtime DLL extracted successfully");
// Remover zip após extração bem-sucedida
let _ = std::fs::remove_file(&ort_zip_path);
}
Err(e) => {
println!("cargo:warning=Failed to extract ONNX Runtime DLL: {}. ZIP may be corrupted, will be downloaded at runtime", e);
// Remover ZIP corrompido para forçar re-download em runtime
let _ = std::fs::remove_file(&ort_zip_path);
// Não falhar o build se a extração falhar - será baixado em runtime se necessário
}
}
}
}
} else {
println!("cargo:warning=ONNX Runtime DLL already exists, skipping download");
}
}
tauri_build::build()
}
/// Extrai onnxruntime.dll do arquivo zip
fn extract_onnx_dll_from_zip(zip_path: &Path, dll_path: &Path) -> Result<(), String> {
use zip::ZipArchive;
let file = File::open(zip_path)
.map_err(|e| format!("Failed to open zip file: {}", e))?;
let mut archive = ZipArchive::new(file)
.map_err(|e| format!("Failed to read zip archive: {}", e))?;
// Procurar pela DLL no zip
for i in 0..archive.len() {
let mut file = archive.by_index(i)
.map_err(|e| format!("Failed to read zip entry {}: {}", i, e))?;
let name = file.name().to_string();
// Procurar por onnxruntime.dll (pode estar em subdiretórios)
if name.ends_with("onnxruntime.dll") {
// Criar diretório de destino se não existir
if let Some(parent) = dll_path.parent() {
std::fs::create_dir_all(parent)
.map_err(|e| format!("Failed to create destination directory: {}", e))?;
}
// Extrair arquivo
let mut outfile = File::create(dll_path)
.map_err(|e| format!("Failed to create output file: {}", e))?;
std::io::copy(&mut file, &mut outfile)
.map_err(|e| format!("Failed to copy file: {}", e))?;
return Ok(());
}
}
Err("onnxruntime.dll not found in zip archive".to_string())
}
+4
View File
@@ -2,3 +2,7 @@ curl_cffi>=0.5.10
beautifulsoup4>=4.12.0
lxml>=4.9.0
html2text>=2024.1.0
playwright>=1.40.0
fake-useragent>=1.4.0
duckduckgo-search>=4.0.0
nodriver>=0.33
+334 -466
View File
@@ -1,499 +1,367 @@
#!/usr/bin/env python3
"""
Script Python para scraping web com curl_cffi + BeautifulSoup
Pipeline: curl_cffi (Request HTTP) -> BeautifulSoup (Parsing) -> html2text (Formatação)
Comunica via JSON stdin/stdout com o backend Rust
Micro-serviço Python para scraping e busca
Pipeline:
1. Scraping: curl_cffi / Playwright Stealth
2. Busca: duckduckgo-search
"""
import sys
import json
import time
import logging
import traceback
from typing import List, Dict, Optional
from urllib.parse import urljoin, urlparse, quote
import random
from concurrent.futures import ThreadPoolExecutor, as_completed
import time
import os
import asyncio
from itertools import cycle
from typing import Dict, Optional, Union, List
from urllib.parse import urljoin
# Bibliotecas de Scraping e Parsing
from bs4 import BeautifulSoup
from curl_cffi import requests
import html2text
# DuckDuckGo Search
try:
from duckduckgo_search import DDGS
HAS_DDG = True
except ImportError:
HAS_DDG = False
# Fallback libs
try:
import requests as pyrequests
HAS_PYREQUESTS = True
except ImportError:
HAS_PYREQUESTS = False
try:
import httpx
HAS_HTTPX = True
except ImportError:
HAS_HTTPX = False
# Playwright & Anti-Detect
try:
from playwright.async_api import async_playwright, TimeoutError as PlaywrightTimeoutError
HAS_PLAYWRIGHT = True
except ImportError:
HAS_PLAYWRIGHT = False
try:
from fake_useragent import UserAgent
ua_generator = UserAgent()
HAS_FAKE_UA = True
except ImportError:
HAS_FAKE_UA = False
ua_generator = None
# --- CONFIGURAÇÃO DO AMBIENTE ---
# Configuração CRÍTICA de Encoding para Windows
try:
sys.stdout.reconfigure(encoding='utf-8')
sys.stderr.reconfigure(encoding='utf-8')
except (AttributeError, Exception):
pass
# Configuração de Logging para STDERR (Não polui o STDOUT)
logging.basicConfig(
level=logging.INFO,
format='[PYTHON LOG] %(message)s',
format='[PYTHON SCRAPER] %(levelname)s: %(message)s',
stream=sys.stderr
)
def print_json(data):
"""Função auxiliar para garantir output limpo em JSON"""
# --- CONSTANTES E POOLS ---
UA_POOL_FALLBACK = [
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15',
'Mozilla/5.0 (X11; Linux x86_64; rv:132.0) Gecko/20100101 Firefox/132.0',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0'
]
ua_cycle = cycle(UA_POOL_FALLBACK)
REQUEST_TIMEOUT = 20
MAX_RETRIES = 2
PROXY_POOL = None
env_proxies = os.environ.get('SCRAPER_PROXIES') or os.environ.get('SCRAPER_HTTP_PROXIES')
if env_proxies:
parts = [p.strip() for p in env_proxies.split(',') if p.strip()]
if parts:
PROXY_POOL = cycle(parts)
# --- FUNÇÕES AUXILIARES ---
def get_random_ua() -> str:
if HAS_FAKE_UA:
try:
return ua_generator.random
except Exception:
pass
return next(ua_cycle)
def generate_consistent_headers(user_agent: str) -> Dict[str, str]:
base_headers = {
'User-Agent': user_agent,
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
'Accept-Language': 'pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7',
'Accept-Encoding': 'gzip, deflate, br',
'DNT': '1',
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'none',
'Sec-Fetch-User': '?1',
'Cache-Control': 'max-age=0'
}
if 'Chrome' in user_agent:
try:
version = user_agent.split('Chrome/')[1].split('.')[0]
except IndexError:
version = "120"
base_headers.update({
'sec-ch-ua': f'"Chromium";v="{version}", "Google Chrome";v="{version}", "Not-A.Brand";v="99"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Windows"' if 'Windows' in user_agent else ('"macOS"' if 'Macintosh' in user_agent else '"Linux"')
})
return base_headers
def get_proxy_dict() -> Optional[Dict[str, str]]:
if PROXY_POOL:
proxy = next(PROXY_POOL)
return {"http": proxy, "https": proxy}
return None
def random_jitter(min_sec=0.5, max_sec=2.0):
time.sleep(random.uniform(min_sec, max_sec))
# --- LIMPEZA E EXTRAÇÃO ---
def clean_html(soup: BeautifulSoup) -> BeautifulSoup:
for tag in soup(["script", "style", "meta", "link", "noscript", "iframe", "object", "embed", "applet", "canvas", "svg"]):
tag.decompose()
for tag in soup.find_all(["nav", "header", "footer", "aside", "menu", "dialog"]):
tag.decompose()
ad_keywords = ["ads", "advertisement", "banner", "ad-", "popup", "modal", "cookie", "subscription", "newsletter", "share", "social", "sidebar", "promo", "sponsor"]
selectors_to_remove = ['.ad', '.advertisement', '[class*="ad-"]', '[id*="ad-"]', '[id*="cookie"]', '[class*="cookie"]', '.share-buttons', '.social-media', '.comments-section', '#comments']
for selector in selectors_to_remove:
for tag in soup.select(selector):
tag.decompose()
def is_ad(tag):
if not tag.name: return False
attrs = str(tag.attrs).lower()
return any(k in attrs for k in ad_keywords)
for tag in soup.find_all(is_ad):
tag.decompose()
return soup
def extract_main_content(soup: BeautifulSoup) -> BeautifulSoup:
selectors = ['article', 'main', '[role="main"]', '.post-content', '.article-body', '#content', '.content', '.entry-content', '.page-content']
for selector in selectors:
found = soup.select_one(selector)
if found and len(found.get_text(strip=True)) > 200:
return found
body = soup.find('body')
return body if body else soup
def html_to_markdown(html_content: str) -> str:
try:
print(json.dumps(data, ensure_ascii=False))
h = html2text.HTML2Text()
h.ignore_links = False
h.ignore_images = False
h.images_to_alt = True
h.body_width = 0
h.protect_links = True
return h.handle(html_content)
except Exception as e:
logging.warning(f"Erro na conversão Markdown: {e}")
return html_content
# --- ESTRATÉGIAS DE SCRAPING ---
async def scrape_with_playwright(url: str) -> Dict:
if not HAS_PLAYWRIGHT:
return {"error": "Playwright not installed"}
logging.info(f"[PLAYWRIGHT] Iniciando scraping stealth para: {url}")
async with async_playwright() as p:
args = ['--disable-blink-features=AutomationControlled', '--disable-dev-shm-usage', '--no-sandbox', '--disable-setuid-sandbox', '--disable-infobars', '--window-size=1920,1080', '--start-maximized']
try:
browser = await p.chromium.launch(headless=True, args=args)
ua = get_random_ua()
context = await browser.new_context(user_agent=ua, viewport={'width': 1920, 'height': 1080}, locale='pt-BR', timezone_id='America/Sao_Paulo', java_script_enabled=True, ignore_https_errors=True)
await context.add_init_script("""
Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
window.chrome = { runtime: {} };
Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5] });
Object.defineProperty(navigator, 'languages', { get: () => ['pt-BR', 'pt', 'en-US', 'en'] });
""")
page = await context.new_page()
try:
await page.goto(url, wait_until='domcontentloaded', timeout=25000)
await page.wait_for_timeout(2000)
except PlaywrightTimeoutError:
logging.warning(f"[PLAYWRIGHT] Timeout carregando {url}")
content = await page.content()
title = await page.title()
await browser.close()
return {"content": content, "title": title, "method": "playwright"}
except Exception as e:
logging.error(f"[PLAYWRIGHT] Erro fatal: {e}")
return {"error": str(e)}
def scrape_with_curl_cffi(url: str) -> Dict:
ua = get_random_ua()
headers = generate_consistent_headers(ua)
proxies = get_proxy_dict()
try:
response = requests.get(url, headers=headers, proxies=proxies, timeout=15, impersonate="chrome120", allow_redirects=True)
response.raise_for_status()
if response.status_code in [403, 429, 503]:
raise requests.RequestsError(f"Blocked with status {response.status_code}")
return {"content": response.text, "title": "", "method": "curl_cffi", "headers": response.headers}
except Exception as e:
return {"error": str(e)}
async def scrape_url_smart(url: str) -> Dict:
random_jitter()
logging.info(f"Tentativa 1: curl_cffi -> {url}")
result = scrape_with_curl_cffi(url)
html_content = None
title = ""
if "error" not in result:
html_content = result["content"]
if "challenge-platform" in html_content or "Cloudflare" in html_content and len(html_content) < 5000:
logging.warning("Detectado desafio Cloudflare/WAF no curl_cffi. Acionando fallback.")
html_content = None
if not html_content:
if HAS_PLAYWRIGHT:
logging.info(f"Tentativa 2: Playwright Stealth -> {url}")
pw_result = await scrape_with_playwright(url)
if "error" not in pw_result:
html_content = pw_result["content"]
title = pw_result.get("title", "")
else:
logging.error(f"Playwright falhou: {pw_result['error']}")
else:
logging.warning("Playwright não disponível para fallback.")
if html_content:
try:
soup = BeautifulSoup(html_content, 'html.parser')
if not title:
title = soup.title.string.strip() if soup.title else url
soup = clean_html(soup)
main_content = extract_main_content(soup)
cleaned_html = str(main_content)
markdown_text = html_to_markdown(cleaned_html)
final_markdown = '\n'.join([line.strip() for line in markdown_text.split('\n') if line.strip()])
if len(final_markdown) < 50:
return {"response_type": "error", "message": "Conteúdo insuficiente após processamento."}
return {"response_type": "success", "url": url, "title": title, "markdown": final_markdown, "content": final_markdown}
except Exception as e:
logging.error(f"Erro no processamento HTML: {traceback.format_exc()}")
return {"response_type": "error", "message": f"Processing error: {str(e)}"}
return {"response_type": "error", "message": f"Falha em todos os métodos de scraping. Erro inicial: {result.get('error', 'Unknown')}"}
# --- DUCKDUCKGO SEARCH LOGIC ---
async def search_duckduckgo(keywords: str, region: str = "wt-wt", safesearch: str = "moderate", max_results: int = 10) -> Dict:
if not HAS_DDG:
return {"response_type": "error", "message": "duckduckgo-search module not installed"}
logging.info(f"Searching DuckDuckGo: {keywords} (region={region}, safe={safesearch})")
try:
results_list = []
# DDGS is synchronous, run in thread executor to avoid blocking async loop if needed
# But since this script main loop is simple, direct call is fine, or wrap in to_thread
with DDGS() as ddgs:
# ddgs.text() is a generator
ddg_gen = ddgs.text(
keywords=keywords,
region=region,
safesearch=safesearch,
max_results=max_results
)
for r in ddg_gen:
results_list.append({
"title": r.get("title", ""),
"url": r.get("href", ""),
"content": r.get("body", ""),
"engine": "duckduckgo",
"score": 1.0
})
return {
"response_type": "success",
"query": keywords,
"results": results_list,
"number_of_results": len(results_list)
}
except Exception as e:
logging.error(f"DuckDuckGo Search Error: {e}")
return {"response_type": "error", "message": str(e)}
# --- MAIN ---
def write_json_response(data: Dict):
try:
json_line = json.dumps(data, separators=(',', ':'), ensure_ascii=False)
print(json_line)
sys.stdout.flush()
except Exception as e:
try:
print(json.dumps({"type": "error", "message": f"Failed to format output: {str(e)}"}))
sys.stdout.flush()
except:
pass
# Tentar importar dependências
CURL_CFFI_AVAILABLE = False
BS4_AVAILABLE = False
HTML2TEXT_AVAILABLE = False
try:
from curl_cffi import requests
CURL_CFFI_AVAILABLE = True
except ImportError as e:
logging.error(f"curl_cffi não disponível: {e}")
try:
from bs4 import BeautifulSoup
BS4_AVAILABLE = True
except ImportError as e:
logging.error(f"BeautifulSoup4 não disponível: {e}")
try:
import html2text
HTML2TEXT_AVAILABLE = True
except ImportError as e:
logging.error(f"html2text não disponível: {e}")
# Pool de User Agents para parecer mais humano
USER_AGENTS = [
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/121.0",
]
def get_random_user_agent():
"""Retorna um user agent aleatório"""
return random.choice(USER_AGENTS)
def clean_url(url: str) -> str:
"""Remove fragmentos e parâmetros de rastreamento da URL"""
try:
parsed = urlparse(url)
return f"{parsed.scheme}://{parsed.netloc}{parsed.path}"
except:
return url
def extract_domain(url: str) -> str:
"""Extrai o domínio de uma URL"""
try:
parsed = urlparse(url)
return parsed.netloc
except:
return ""
def is_ad_or_tracker_url(url: str) -> bool:
"""Verifica se uma URL é de anúncio ou rastreador"""
trackers = ['analytics', 'google-analytics', 'doubleclick', 'facebook.com/tr', 'ads.', 'banner', 'tracking']
return any(tracker in url.lower() for tracker in trackers)
def is_domain_blocked(url: str, excluded_domains: List[str]) -> bool:
"""Verifica se o domínio da URL está na lista de exclusão"""
domain = extract_domain(url)
return any(excluded in domain for excluded in excluded_domains)
def scrape_url(url: str, visible: bool = False) -> Dict:
"""Função principal de scraping com curl_cffi + BeautifulSoup"""
# Verificar dependências
if not CURL_CFFI_AVAILABLE:
return {
"type": "error",
"message": "curl_cffi não está instalado"
}
if not BS4_AVAILABLE:
return {
"type": "error",
"message": "beautifulsoup4 não está instalado"
}
if not HTML2TEXT_AVAILABLE:
return {
"type": "error",
"message": "html2text não está instalado"
}
try:
logging.info(f"Iniciando requisição para: {url}")
# Criar headers com curl_cffi (impersonate navegador)
headers = {
"User-Agent": get_random_user_agent(),
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Accept-Language": "pt-BR,pt;q=0.9,en;q=0.8",
"Accept-Encoding": "gzip, deflate",
"DNT": "1",
"Connection": "keep-alive",
"Upgrade-Insecure-Requests": "1",
}
# Executar requisição com timeout
try:
response = requests.get(
url,
headers=headers,
timeout=15,
impersonate="chrome120",
allow_redirects=True,
follow_redirects=True
)
response.raise_for_status()
except Exception as e:
logging.error(f"Erro na requisição HTTP: {e}")
return {
"type": "error",
"message": f"Failed to fetch URL: {str(e)}"
}
# Verificar se recebemos conteúdo HTML válido
if len(response.content) < 100:
return {
"type": "error",
"message": "Resposta muito curta (< 100 bytes)"
}
logging.info(f"Resposta recebida: {len(response.content)} bytes")
# Parse com BeautifulSoup
try:
soup = BeautifulSoup(response.content, 'html.parser')
except Exception as e:
logging.error(f"Erro ao fazer parse HTML: {e}")
return {
"type": "error",
"message": f"Failed to parse HTML: {str(e)}"
}
# Extrair título
title = "Sem título"
title_tag = soup.find('meta', property='og:title') or soup.find('meta', {'name': 'title'}) or soup.find('title')
if title_tag:
title = title_tag.get('content') or title_tag.string or title
# Remover scripts e styles
for script in soup(["script", "style", "meta", "link", "noscript"]):
script.decompose()
# Remover elementos desnecessários
for elem in soup.find_all(["nav", "header", "footer", "aside", "ads"]):
elem.decompose()
# Extrair conteúdo principal
main_content = None
for selector in ['article', 'main', '[role="main"]', '.content', '.post', '.entry']:
if selector.startswith('['):
main_content = soup.select_one(selector)
else:
main_content = soup.find(selector)
if main_content and len(main_content.get_text(strip=True)) > 200:
break
# Fallback: usar body
if not main_content or len(main_content.get_text(strip=True)) < 200:
main_content = soup.find('body') or soup
# Obter HTML do conteúdo principal
html_content = str(main_content) if main_content else str(soup)
# Converter para Markdown usando html2text
try:
h = html2text.HTML2Text()
h.ignore_links = False
h.ignore_images = False
h.body_width = 0
markdown_content = h.handle(html_content)
except Exception as e:
logging.warning(f"Erro ao converter para markdown, usando texto plano: {e}")
markdown_content = main_content.get_text(separator="\n") if main_content else response.text
# Limpeza final
lines = [line.strip() for line in markdown_content.split('\n') if line.strip()]
final_content = '\n'.join(lines)
# Validação de conteúdo
if len(final_content) < 100:
return {
"type": "error",
"message": f"Conteúdo insuficiente ({len(final_content)} chars)"
}
logging.info(f"Sucesso! {len(final_content)} chars extraídos")
return {
"type": "success",
"url": clean_url(url),
"title": title[:500],
"content": final_content[:30000],
"markdown": final_content[:30000]
}
except Exception as e:
logging.error(f"Erro no scraping: {traceback.format_exc()}")
return {
"type": "error",
"message": str(e)
}
def search_duckduckgo(query: str, limit: int = 5) -> List[str]:
"""Realiza busca no DuckDuckGo e retorna URLs"""
try:
if not CURL_CFFI_AVAILABLE or not BS4_AVAILABLE:
logging.error("Dependências não disponíveis")
return []
logging.info(f"Buscando no DuckDuckGo: {query}")
headers = {
"User-Agent": get_random_user_agent(),
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "pt-BR,pt;q=0.9,en;q=0.8",
}
search_url = f"https://duckduckgo.com/html/?q={quote(query)}"
response = requests.get(
search_url,
headers=headers,
timeout=15,
impersonate="chrome120",
allow_redirects=True
)
response.raise_for_status()
soup = BeautifulSoup(response.content, 'html.parser')
urls = []
# DuckDuckGo HTML retorna resultados em links diretos
for result in soup.find_all('a', {'class': 'result__a'}):
href = result.get('href')
if href and not is_ad_or_tracker_url(href):
urls.append(href)
if len(urls) >= limit:
break
logging.info(f"Encontradas {len(urls)} URLs")
return urls
except Exception as e:
logging.error(f"Erro na busca DuckDuckGo: {e}")
return []
def smart_search_impl(query: str, config: Dict) -> List[str]:
"""Implementa busca inteligente com categorias e filtros"""
try:
logging.info(f"Smart search iniciado: {query}")
total_limit = config.get('total_sources_limit', 15)
categories = config.get('categories', [])
user_custom_sites = config.get('user_custom_sites', [])
excluded_domains = config.get('excluded_domains', [])
all_urls = []
# 1. Busca geral no DuckDuckGo
general_urls = search_duckduckgo(query, total_limit // 2)
all_urls.extend(general_urls)
# 2. Busca por categorias habilitadas
for category in categories:
if not category.get('enabled', False):
continue
base_sites = category.get('base_sites', [])
for site in base_sites[:2]: # Limitar a 2 sites por categoria
site_query = f"site:{site} {query}"
site_urls = search_duckduckgo(site_query, 3)
all_urls.extend(site_urls)
# 3. Busca em sites personalizados
for custom_site in user_custom_sites[:3]:
custom_query = f"site:{custom_site} {query}"
custom_urls = search_duckduckgo(custom_query, 2)
all_urls.extend(custom_urls)
# 4. Filtrar domínios excluídos e remover duplicatas
filtered_urls = []
seen = set()
for url in all_urls:
if url not in seen and not is_domain_blocked(url, excluded_domains):
filtered_urls.append(url)
seen.add(url)
if len(filtered_urls) >= total_limit:
break
logging.info(f"Smart search concluído: {len(filtered_urls)} URLs")
return filtered_urls
except Exception as e:
logging.error(f"Erro no smart search: {e}")
return []
def scrape_urls_bulk_impl(urls: List[str]) -> List[Dict]:
"""Scraping em paralelo de múltiplas URLs"""
try:
logging.info(f"Scraping bulk iniciado: {len(urls)} URLs")
results = []
def scrape_with_logging(url):
try:
result = scrape_url(url, visible=False)
if result.get('type') == 'success':
logging.info(f"✓ {url}")
else:
logging.warning(f"✗ {url}: {result.get('message', 'unknown error')}")
return result
except Exception as e:
logging.error(f"✗ {url}: {e}")
return {"type": "error", "message": str(e), "url": url}
# Scraping paralelo com até 5 workers
with ThreadPoolExecutor(max_workers=5) as executor:
future_to_url = {executor.submit(scrape_with_logging, url): url for url in urls}
for future in as_completed(future_to_url):
try:
result = future.result()
if result.get('type') == 'success':
results.append(result)
except Exception as e:
logging.error(f"Erro no future: {e}")
logging.info(f"Scraping bulk concluído: {len(results)} sucessos")
return results
except Exception as e:
logging.error(f"Erro no scraping bulk: {e}")
return []
logging.error(f"Erro enviando resposta JSON: {e}")
def main():
"""Entry point do script - lê comandos JSON do stdin"""
try:
# Ler JSON do stdin
input_data = sys.stdin.read().strip()
if not input_data:
print_json({"type": "error", "message": "No input provided"})
return
# Parse JSON
logging.info("Scraper/Search Service v4.0 (DuckDuckGo Edition) Initialized")
while True:
try:
command_data = json.loads(input_data)
except json.JSONDecodeError as e:
logging.error(f"Erro ao fazer parse do JSON: {e}")
print_json({"type": "error", "message": f"Invalid JSON: {str(e)}"})
return
command = command_data.get("command")
logging.info(f"Comando recebido: {command}")
# Dispatch de comandos
if command == "scrape":
url = command_data.get("url")
visible = command_data.get("visible", False)
line = sys.stdin.readline()
if not line: break
if not line.strip(): continue
try:
request = json.loads(line)
except json.JSONDecodeError:
write_json_response({"response_type": "error", "message": "Invalid JSON"})
continue
if not url:
print_json({"type": "error", "message": "URL missing"})
return
command = request.get("command")
result = scrape_url(url, visible)
print_json(result)
elif command == "search_duckduckgo":
query = command_data.get("query")
limit = command_data.get("limit", 5)
if command == "scrape":
url = request.get("url")
if url:
result = asyncio.run(scrape_url_smart(url))
write_json_response(result)
else:
write_json_response({"response_type": "error", "message": "No URL provided"})
if not query:
print_json({"type": "error", "message": "Query missing"})
return
urls = search_duckduckgo(query, limit)
print_json({"type": "success", "urls": urls})
elif command == "smart_search":
query = command_data.get("query")
config = command_data.get("config", {})
if not query:
print_json({"type": "error", "message": "Query missing"})
return
urls = smart_search_impl(query, config)
print_json({"type": "success", "urls": urls})
elif command == "search_and_scrape":
query = command_data.get("query")
config = command_data.get("config", {})
if not query:
print_json({"type": "error", "message": "Query missing"})
return
# Primeiro faz smart_search para obter URLs
urls = smart_search_impl(query, config)
if not urls:
print_json({"type": "success", "results": []})
return
# Depois faz scraping das URLs
results = scrape_urls_bulk_impl(urls)
print_json({"type": "success", "results": results})
elif command == "scrape_urls_bulk":
urls = command_data.get("urls", [])
if not urls:
print_json({"type": "error", "message": "URLs missing"})
return
results = scrape_urls_bulk_impl(urls)
print_json({"type": "success", "results": results})
elif command == "check":
# Verificar dependências
missing = []
if not CURL_CFFI_AVAILABLE:
missing.append("curl_cffi")
if not BS4_AVAILABLE:
missing.append("beautifulsoup4")
if not HTML2TEXT_AVAILABLE:
missing.append("html2text")
if missing:
print_json({"type": "error", "message": f"Missing: {', '.join(missing)}"})
elif command == "search":
# Generic search command, implemented via DuckDuckGo
keywords = request.get("keywords") or request.get("query")
region = request.get("region", "wt-wt")
safesearch = request.get("safesearch", "moderate")
max_results = request.get("max_results", 10)
if keywords:
result = asyncio.run(search_duckduckgo(keywords, region, safesearch, max_results))
write_json_response(result)
else:
write_json_response({"response_type": "error", "message": "Missing keywords"})
elif command == "ping":
write_json_response({"response_type": "pong"})
else:
print_json({"type": "success", "message": "All dependencies available"})
else:
print_json({"type": "error", "message": f"Unknown command: {command}"})
except Exception as e:
logging.error(f"Erro no main: {traceback.format_exc()}")
print_json({"type": "error", "message": f"Critical error: {str(e)}"})
write_json_response({"response_type": "error", "message": "Unknown command"})
except KeyboardInterrupt:
break
except Exception as e:
logging.critical(f"Fatal error in main loop: {e}")
write_json_response({"response_type": "error", "message": str(e)})
if __name__ == "__main__":
try:
main()
except KeyboardInterrupt:
print_json({"type": "error", "message": "Script interrupted"})
sys.exit(1)
except Exception as e:
try:
logging.error(f"Unhandled exception: {traceback.format_exc()}")
print_json({"type": "error", "message": f"Unhandled exception: {str(e)}"})
except:
try:
print(json.dumps({"type": "error", "message": "Critical failure"}))
except:
pass
sys.exit(1)
main()
+337
View File
@@ -0,0 +1,337 @@
#!/usr/bin/env python3
"""
Micro-serviço Python para scraping com Nodriver (anti-detecção)
Pipeline:
1. Scraping: Nodriver (sem CDP, indetectável)
2. Fallback: curl_cffi se Nodriver falhar
"""
import sys
import json
import logging
import traceback
import random
import time
import os
import asyncio
from typing import Dict, Optional, Union, List
from urllib.parse import urljoin
# Bibliotecas de Scraping e Parsing
from bs4 import BeautifulSoup
import html2text
# Nodriver (anti-detecção)
try:
import nodriver as uc
HAS_NODRIVER = True
except ImportError:
HAS_NODRIVER = False
logging.warning("nodriver not installed, falling back to curl_cffi")
# curl_cffi como fallback
try:
from curl_cffi import requests
HAS_CURL_CFFI = True
except ImportError:
HAS_CURL_CFFI = False
# Fallback libs
try:
import requests as pyrequests
HAS_PYREQUESTS = True
except ImportError:
HAS_PYREQUESTS = False
# --- CONFIGURAÇÃO DO AMBIENTE ---
try:
sys.stdout.reconfigure(encoding='utf-8')
sys.stderr.reconfigure(encoding='utf-8')
except (AttributeError, Exception):
pass
logging.basicConfig(
level=logging.INFO,
format='[PYTHON SCRAPER NODRIVER] %(levelname)s: %(message)s',
stream=sys.stderr
)
# --- CONSTANTES E POOLS ---
UA_POOL_FALLBACK = [
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15',
'Mozilla/5.0 (X11; Linux x86_64; rv:132.0) Gecko/20100101 Firefox/132.0',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0'
]
REQUEST_TIMEOUT = 12 # Reduzido de 20 para 12s para maior velocidade
MAX_RETRIES = 2
NODRIVER_TIMEOUT = 10 # Timeout específico para nodriver (10s)
# --- FUNÇÕES AUXILIARES ---
def get_random_ua() -> str:
return random.choice(UA_POOL_FALLBACK)
def random_jitter():
"""Adiciona jitter aleatório para evitar padrões"""
time.sleep(random.uniform(0.1, 0.5))
def clean_html(soup: BeautifulSoup) -> BeautifulSoup:
"""Remove scripts, styles e outros elementos desnecessários"""
for script in soup(["script", "style", "nav", "header", "footer", "aside", "noscript"]):
script.decompose()
return soup
def extract_main_content(soup: BeautifulSoup) -> BeautifulSoup:
"""Extrai conteúdo principal usando heurísticas"""
# Tentar encontrar main, article, ou div com classe comum
selectors = [
'main',
'article',
'[role="main"]',
'.content',
'.main-content',
'#content',
'#main',
]
for selector in selectors:
main = soup.select_one(selector)
if main:
return main
# Fallback: retornar body
return soup.find('body') or soup
def html_to_markdown(html: str) -> str:
"""Converte HTML para Markdown"""
h = html2text.HTML2Text()
h.ignore_links = False
h.ignore_images = False
h.body_width = 0
return h.handle(html)
# --- SCRAPING COM NODRIVER ---
async def scrape_with_nodriver(url: str) -> Dict:
"""Scraping usando Nodriver (anti-detecção) - OTIMIZADO"""
if not HAS_NODRIVER:
return {"error": "nodriver not installed"}
try:
logging.info(f"[NODRIVER] Starting scrape for {url}")
start_time = time.time()
# Iniciar browser com Nodriver (sem CDP, indetectável)
browser = await uc.start(headless=True)
try:
# Navegar para URL com timeout
page = await asyncio.wait_for(
browser.get(url),
timeout=NODRIVER_TIMEOUT
)
# Aguardar conteúdo carregar com wait condicional otimizado
# Tentar múltiplas estratégias de wait para reduzir tempo total
try:
# Estratégia 1: Aguardar DOMContentLoaded (mais rápido)
await asyncio.wait_for(
page.wait_for_load_state('domcontentloaded'),
timeout=2.0
)
except asyncio.TimeoutError:
# Se DOMContentLoaded demorar, usar sleep reduzido
await asyncio.sleep(0.5)
# Aguardar um pouco mais para JS executar (reduzido de 2s para 0.3s)
await asyncio.sleep(0.3)
# Extrair HTML
html = await page.get_content()
title = await page.evaluate('document.title')
duration = time.time() - start_time
logging.info(f"[NODRIVER] Successfully scraped {url} ({len(html)} chars, {duration:.2f}s)")
return {
"content": html,
"title": title or "",
"method": "nodriver"
}
finally:
await browser.stop()
except asyncio.TimeoutError:
logging.warning(f"[NODRIVER] Timeout after {NODRIVER_TIMEOUT}s for {url}")
return {"error": f"Timeout after {NODRIVER_TIMEOUT}s"}
except Exception as e:
logging.error(f"[NODRIVER] Error scraping {url}: {e}")
return {"error": str(e)}
# --- SCRAPING COM CURL_CFFI (FALLBACK) ---
def scrape_with_curl_cffi(url: str) -> Dict:
"""Scraping usando curl_cffi como fallback - OTIMIZADO"""
if not HAS_CURL_CFFI:
return {"error": "curl_cffi not installed"}
ua = get_random_ua()
headers = {
'User-Agent': ua,
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7',
'Accept-Encoding': 'gzip, deflate, br',
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1',
}
try:
# Timeout reduzido para fallback rápido
response = requests.get(
url,
headers=headers,
timeout=8, # Reduzido de 15 para 8s
impersonate="chrome120",
allow_redirects=True
)
response.raise_for_status()
if response.status_code in [403, 429, 503]:
raise requests.RequestsError(f"Blocked with status {response.status_code}")
return {
"content": response.text,
"title": "",
"method": "curl_cffi",
"headers": dict(response.headers)
}
except Exception as e:
return {"error": str(e)}
# --- SCRAPING INTELIGENTE ---
async def scrape_url_smart(url: str) -> Dict:
"""Scraping inteligente: Nodriver (PRIMÁRIO) → curl_cffi (fallback) - OTIMIZADO"""
# Jitter reduzido para maior velocidade
await asyncio.sleep(random.uniform(0.05, 0.15))
# Tentativa 1: Nodriver (PRIMÁRIO, anti-detecção)
if HAS_NODRIVER:
logging.info(f"[PRIMARY] Nodriver -> {url}")
result = await scrape_with_nodriver(url)
if "error" not in result:
html_content = result["content"]
# Verificar se não é página de bloqueio (mais permissivo para velocidade)
if len(html_content) > 1000: # Reduzido de 5000 para 1000 para aceitar mais resultados
if "challenge-platform" not in html_content.lower() and "cloudflare" not in html_content.lower():
return result
else:
logging.warning("[NODRIVER] Detectado desafio Cloudflare/WAF. Tentando fallback.")
elif len(html_content) > 200: # Aceitar conteúdo mínimo
logging.info(f"[NODRIVER] Conteúdo mínimo mas aceitável ({len(html_content)} chars)")
return result
# Tentativa 2: curl_cffi (fallback rápido)
if HAS_CURL_CFFI:
logging.info(f"[FALLBACK] curl_cffi -> {url}")
result = scrape_with_curl_cffi(url)
if "error" not in result:
html_content = result["content"]
if len(html_content) > 1000:
if "challenge-platform" not in html_content.lower() and "cloudflare" not in html_content.lower():
return result
# Se ambos falharam, retornar erro
return {"error": "All scraping methods failed"}
# --- PROCESSAMENTO HTML ---
async def process_scraped_content(url: str, result: Dict) -> Dict:
"""Processa conteúdo scraped e converte para markdown"""
if "error" in result:
return {"response_type": "error", "message": result["error"]}
html_content = result.get("content", "")
title = result.get("title", "")
if not html_content:
return {"response_type": "error", "message": "No content extracted"}
try:
soup = BeautifulSoup(html_content, 'html.parser')
if not title:
title = soup.title.string.strip() if soup.title else url
soup = clean_html(soup)
main_content = extract_main_content(soup)
cleaned_html = str(main_content)
markdown_text = html_to_markdown(cleaned_html)
final_markdown = '\n'.join([line.strip() for line in markdown_text.split('\n') if line.strip()])
if len(final_markdown) < 50:
return {"response_type": "error", "message": "Conteúdo insuficiente após processamento."}
return {
"response_type": "success",
"url": url,
"title": title,
"markdown": final_markdown,
"content": final_markdown
}
except Exception as e:
logging.error(f"Erro no processamento HTML: {traceback.format_exc()}")
return {"response_type": "error", "message": f"Processing error: {str(e)}"}
# --- MAIN LOOP ---
def write_json_response(data: Dict):
try:
json_line = json.dumps(data, separators=(',', ':'), ensure_ascii=False)
print(json_line)
sys.stdout.flush()
except Exception as e:
logging.error(f"Erro enviando resposta JSON: {e}")
async def main():
logging.info("Scraper/Search Service v5.0 (Nodriver Edition) Initialized")
while True:
try:
line = sys.stdin.readline()
if not line:
break
if not line.strip():
continue
try:
request = json.loads(line)
except json.JSONDecodeError:
write_json_response({"response_type": "error", "message": "Invalid JSON"})
continue
command = request.get("command")
if command == "scrape":
url = request.get("url")
if url:
result = await scrape_url_smart(url)
processed = await process_scraped_content(url, result)
write_json_response(processed)
else:
write_json_response({"response_type": "error", "message": "No URL provided"})
else:
write_json_response({"response_type": "error", "message": f"Unknown command: {command}"})
except Exception as e:
logging.error(f"Error in main loop: {traceback.format_exc()}")
write_json_response({"response_type": "error", "message": str(e)})
if __name__ == "__main__":
asyncio.run(main())
+110
View File
@@ -0,0 +1,110 @@
use anyhow::Result;
use reqwest::Client;
use std::time::Duration;
use crate::web_scraper::SearchResultMetadata;
use crate::event_emitter::{SearchQueryEvent, SearchQueryCompletedEvent, emit_search_event, emit_search_completed_event};
use tauri::AppHandle;
use std::time::{SystemTime, UNIX_EPOCH};
/// Busca via Brave Search API
pub async fn brave_search(
query: &str,
limit: usize,
app_handle: Option<&AppHandle>,
) -> Result<Vec<SearchResultMetadata>> {
// Emitir evento de query iniciada
if let Some(handle) = app_handle {
let timestamp = SystemTime::now().duration_since(UNIX_EPOCH)
.unwrap_or_default().as_secs() * 1000;
emit_search_event(handle, &SearchQueryEvent {
query: query.to_string(),
source: "brave".to_string(),
round: None,
timestamp,
});
}
let api_key = std::env::var("BRAVE_API_KEY")
.map_err(|_| anyhow::anyhow!("BRAVE_API_KEY não configurada"))?;
let client = Client::builder()
.timeout(Duration::from_secs(3)) // Reduzido de 5s para 3s para maior velocidade
.build()?;
let response = client
.get("https://api.search.brave.com/res/v1/web/search")
.header("X-Subscription-Token", &api_key)
.query(&[
("q", query),
("count", &limit.to_string()),
("safesearch", "moderate"),
])
.send()
.await?;
if !response.status().is_success() {
let status = response.status();
let error_text = response.text().await.unwrap_or_default();
// Emitir evento de query completada com falha
if let Some(handle) = app_handle {
let timestamp = SystemTime::now().duration_since(UNIX_EPOCH)
.unwrap_or_default().as_secs() * 1000;
emit_search_completed_event(handle, &SearchQueryCompletedEvent {
query: query.to_string(),
source: "brave".to_string(),
results_count: 0,
success: false,
timestamp,
});
}
return Err(anyhow::anyhow!("Brave API returned status {}: {}", status, error_text));
}
#[derive(serde::Deserialize)]
struct BraveResponse {
web: BraveWebResults,
}
#[derive(serde::Deserialize)]
struct BraveWebResults {
results: Vec<BraveResult>,
}
#[derive(serde::Deserialize)]
struct BraveResult {
title: String,
url: String,
description: String,
}
let data: BraveResponse = response.json().await?;
let metadata: Vec<SearchResultMetadata> = data
.web
.results
.into_iter()
.take(limit)
.map(|r| SearchResultMetadata {
title: r.title,
url: r.url,
snippet: r.description,
})
.collect();
// Emitir evento de query completada com sucesso
if let Some(handle) = app_handle {
let timestamp = SystemTime::now().duration_since(UNIX_EPOCH)
.unwrap_or_default().as_secs() * 1000;
emit_search_completed_event(handle, &SearchQueryCompletedEvent {
query: query.to_string(),
source: "brave".to_string(),
results_count: metadata.len(),
success: true,
timestamp,
});
}
Ok(metadata)
}
+222
View File
@@ -0,0 +1,222 @@
use std::path::PathBuf;
use serde::{Serialize, Deserialize};
use sha2::{Sha256, Digest};
use tokio::fs;
use std::io;
use chrono::{DateTime, Utc};
use url::Url;
/// Estrutura do conteúdo salvo em disco (JSON)
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct CachedContent {
pub url: String,
pub domain: String,
pub access_timestamp: DateTime<Utc>,
pub expires_at: Option<DateTime<Utc>>, // TTL
pub http_status: u16,
pub title: String,
pub content_markdown: String,
pub metadata: serde_json::Value,
pub raw_html_path: Option<String>,
}
pub struct ContentStore;
impl ContentStore {
/// Retorna o diretório base do cache: app_data/scraping_cache
pub fn get_cache_dir() -> PathBuf {
// Caminho relativo ao diretório de execução ou base do projeto
// Em produção, idealmente usaria dirs::data_local_dir()
PathBuf::from("app_data").join("scraping_cache")
}
/// Gera o hash SHA256 da URL para usar como nome de arquivo
fn get_url_hash(url: &str) -> String {
let mut hasher = Sha256::new();
hasher.update(url.as_bytes());
format!("{:x}", hasher.finalize())
}
/// Extrai o domínio da URL para particionamento de pastas
fn get_domain(url: &str) -> String {
Url::parse(url)
.map(|u| u.domain().unwrap_or("unknown").to_string())
.unwrap_or_else(|_| "unknown".to_string())
}
/// Recupera conteúdo do cache se existir e não expirado
pub async fn get_cached_content(url: &str) -> Option<CachedContent> {
let domain = Self::get_domain(url);
let hash = Self::get_url_hash(url);
let path = Self::get_cache_dir().join(domain).join(format!("{}.json", hash));
if path.exists() {
// Tenta ler e deserializar
let path_clone = path.clone();
if let Ok(content) = fs::read_to_string(&path).await {
if let Ok(data) = serde_json::from_str::<CachedContent>(&content) {
// Verificar se expirou
if let Some(expires_at) = &data.expires_at {
if *expires_at < Utc::now() {
// Cache expirado, deletar arquivo
let _ = fs::remove_file(&path_clone).await;
return None;
}
}
return Some(data);
}
}
}
None
}
/// Salva conteúdo no cache com TTL
pub async fn save_content(url: &str, title: &str, content_markdown: &str, meta: Option<serde_json::Value>) -> io::Result<()> {
Self::save_content_with_ttl(url, title, content_markdown, meta, Some(168)).await // Default: 7 dias
}
/// Salva conteúdo no cache com TTL customizado (em horas)
pub async fn save_content_with_ttl(
url: &str,
title: &str,
content_markdown: &str,
meta: Option<serde_json::Value>,
ttl_hours: Option<u64>,
) -> io::Result<()> {
let domain = Self::get_domain(url);
let hash = Self::get_url_hash(url);
let dir = Self::get_cache_dir().join(&domain);
// Garante que o diretório do domínio existe
if !dir.exists() {
fs::create_dir_all(&dir).await?;
}
let path = dir.join(format!("{}.json", hash));
let now = Utc::now();
let expires_at = ttl_hours.map(|hours| now + chrono::Duration::hours(hours as i64));
let data = CachedContent {
url: url.to_string(),
domain: domain,
access_timestamp: now,
expires_at,
http_status: 200,
title: title.to_string(),
content_markdown: content_markdown.to_string(),
metadata: meta.unwrap_or(serde_json::json!({})),
raw_html_path: None,
};
let json = serde_json::to_string_pretty(&data)?;
fs::write(path, json).await?;
// Limpeza periódica: a cada 100 inserções, limpar expirados e aplicar LRU
static INSERT_COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
let count = INSERT_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
if count % 100 == 0 {
let _ = Self::cleanup_expired().await;
let _ = Self::apply_lru_limit(10000).await; // Manter apenas 10k entradas mais recentes
}
Ok(())
}
/// Limpa entradas expiradas do cache
pub async fn cleanup_expired() -> io::Result<usize> {
let cache_dir = Self::get_cache_dir();
if !cache_dir.exists() {
return Ok(0);
}
let mut cleaned = 0;
let now = Utc::now();
// Iterar sobre todos os arquivos JSON no cache
let mut entries = fs::read_dir(&cache_dir).await?;
while let Some(entry) = entries.next_entry().await? {
if entry.file_type().await?.is_dir() {
let domain_dir = entry.path();
let mut domain_files = fs::read_dir(&domain_dir).await?;
while let Some(file_entry) = domain_files.next_entry().await? {
if file_entry.path().extension().and_then(|s| s.to_str()) == Some("json") {
if let Ok(content) = fs::read_to_string(file_entry.path()).await {
if let Ok(data) = serde_json::from_str::<CachedContent>(&content) {
if let Some(expires_at) = &data.expires_at {
if *expires_at < now {
let _ = fs::remove_file(file_entry.path()).await;
cleaned += 1;
}
}
}
}
}
}
}
}
Ok(cleaned)
}
/// Aplica limite LRU: mantém apenas N entradas mais recentes
pub async fn apply_lru_limit(max_entries: usize) -> io::Result<usize> {
let cache_dir = Self::get_cache_dir();
if !cache_dir.exists() {
return Ok(0);
}
// Coletar todas as entradas com timestamp
let mut all_entries: Vec<(std::path::PathBuf, DateTime<Utc>)> = Vec::new();
let mut entries = fs::read_dir(&cache_dir).await?;
while let Some(entry) = entries.next_entry().await? {
if entry.file_type().await?.is_dir() {
let domain_dir = entry.path();
let mut domain_files = fs::read_dir(&domain_dir).await?;
while let Some(file_entry) = domain_files.next_entry().await? {
if file_entry.path().extension().and_then(|s| s.to_str()) == Some("json") {
if let Ok(content) = fs::read_to_string(file_entry.path()).await {
if let Ok(data) = serde_json::from_str::<CachedContent>(&content) {
all_entries.push((file_entry.path(), data.access_timestamp));
}
}
}
}
}
}
// Se temos mais entradas que o limite, remover as mais antigas
if all_entries.len() > max_entries {
// Ordenar por timestamp (mais antigas primeiro)
all_entries.sort_by_key(|(_, ts)| *ts);
// Remover as mais antigas
let to_remove = all_entries.len() - max_entries;
for (path, _) in all_entries.into_iter().take(to_remove) {
let _ = fs::remove_file(&path).await;
}
Ok(to_remove)
} else {
Ok(0)
}
}
/// Registra operação no log de auditoria
pub async fn log_operation(url: &str, status: &str, latency_ms: u64, bytes: usize) {
let log_entry = format!("[{}] [INFO] [{}] [{}] [{}ms] [{}bytes]\n",
Utc::now().to_rfc3339(), url, status, latency_ms, bytes);
let path = Self::get_cache_dir().join("scraping_ops.log");
// Append assíncrono
use tokio::io::AsyncWriteExt;
if let Ok(mut file) = fs::OpenOptions::new().create(true).append(true).open(path).await {
let _ = file.write_all(log_entry.as_bytes()).await;
}
}
}
+60
View File
@@ -0,0 +1,60 @@
use aes_gcm::{
aead::{Aead, AeadCore, KeyInit, OsRng},
Aes256Gcm, Nonce, Key
};
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
use sha2::{Sha256, Digest};
use std::path::Path;
use std::fs;
// Key size for AES-256 is 32 bytes
const KEY_SIZE: usize = 32;
pub fn get_or_create_key<P: AsRef<Path>>(path: P) -> Result<Key<Aes256Gcm>, String> {
if path.as_ref().exists() {
let bytes = fs::read(&path).map_err(|e| e.to_string())?;
if bytes.len() != KEY_SIZE {
return Err("Invalid key size".to_string());
}
Ok(*Key::<Aes256Gcm>::from_slice(&bytes))
} else {
let key = Aes256Gcm::generate_key(OsRng);
fs::write(&path, key.as_slice()).map_err(|e| e.to_string())?;
Ok(key)
}
}
pub fn encrypt(data: &str, key: &Key<Aes256Gcm>) -> Result<String, String> {
let cipher = Aes256Gcm::new(key);
let nonce = Aes256Gcm::generate_nonce(&mut OsRng); // 96-bits; unique per message
let ciphertext = cipher.encrypt(&nonce, data.as_bytes())
.map_err(|e| e.to_string())?;
// Return nonce + ciphertext encoded in base64
let mut combined = nonce.to_vec();
combined.extend_from_slice(&ciphertext);
Ok(BASE64.encode(combined))
}
pub fn decrypt(data: &str, key: &Key<Aes256Gcm>) -> Result<String, String> {
let combined = BASE64.decode(data).map_err(|e| e.to_string())?;
if combined.len() < 12 {
return Err("Invalid data length".to_string());
}
let (nonce_bytes, ciphertext) = combined.split_at(12);
let nonce = Nonce::from_slice(nonce_bytes);
let cipher = Aes256Gcm::new(key);
let plaintext = cipher.decrypt(nonce, ciphertext)
.map_err(|e| e.to_string())?;
String::from_utf8(plaintext).map_err(|e| e.to_string())
}
pub fn hash_identifier(id: &str) -> String {
let mut hasher = Sha256::new();
hasher.update(id.as_bytes());
format!("{:x}", hasher.finalize())
}
+253 -64
View File
@@ -22,6 +22,25 @@ pub struct ChatMessage {
pub created_at: DateTime<Utc>,
}
<<<<<<< HEAD
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct SearchLog {
pub id: String,
pub chat_id: Option<String>,
pub user_id: Option<String>,
pub timestamp: DateTime<Utc>,
pub searxng_instance: Option<String>,
pub query: String,
pub engines: Option<Vec<String>>,
pub filters: Option<String>, // JSON string
pub response_time_ms: Option<i64>,
pub total_results: Option<i64>,
pub results_per_engine: Option<String>, // JSON string
pub error: Option<String>,
}
=======
>>>>>>> 593efd42e091a845dea82ee6646e027bce1e18c5
/// Resultado de busca de sessões com contagem de matches
#[derive(Debug, Clone)]
pub struct SearchSessionResult {
@@ -116,6 +135,45 @@ impl Database {
)",
[],
)?;
// Tabela de logs de busca (SearXNG)
self.conn.execute(
"CREATE TABLE IF NOT EXISTS search_logs (
id TEXT PRIMARY KEY,
chat_id TEXT,
user_id TEXT,
timestamp TEXT NOT NULL,
searxng_instance TEXT,
query TEXT NOT NULL,
engines TEXT,
filters TEXT,
response_time_ms INTEGER,
total_results INTEGER,
results_per_engine TEXT,
error TEXT
)",
[],
)?;
// Tabela de instâncias SearXNG
self.conn.execute(
"CREATE TABLE IF NOT EXISTS searxng_instances (
url TEXT PRIMARY KEY,
network TEXT,
country_code TEXT,
response_time REAL,
status TEXT,
version TEXT,
engines TEXT,
formats TEXT,
validated BOOLEAN DEFAULT 0,
last_validated INTEGER,
failure_count INTEGER DEFAULT 0,
created_at INTEGER,
updated_at INTEGER
)",
[],
)?;
// Índices para performance
self.conn.execute(
@@ -127,6 +185,27 @@ impl Database {
"CREATE INDEX IF NOT EXISTS idx_rag_session_id ON rag_documents(session_id)",
[],
)?;
self.conn.execute(
"CREATE INDEX IF NOT EXISTS idx_search_logs_chat_id ON search_logs(chat_id)",
[],
)?;
self.conn.execute(
"CREATE INDEX IF NOT EXISTS idx_search_logs_timestamp ON search_logs(timestamp DESC)",
[],
)?;
// Índices para instâncias SearXNG
self.conn.execute(
"CREATE INDEX IF NOT EXISTS idx_status_validated ON searxng_instances(status, validated)",
[],
)?;
self.conn.execute(
"CREATE INDEX IF NOT EXISTS idx_response_time ON searxng_instances(response_time)",
[],
)?;
// Índice para ordenação por updated_at
self.conn.execute(
@@ -208,15 +287,6 @@ impl Database {
[],
)?;
// Trigger para atualizar messages_fts quando mensagem é atualizada
self.conn.execute(
"CREATE TRIGGER IF NOT EXISTS messages_fts_update AFTER UPDATE ON messages BEGIN
INSERT INTO messages_fts(messages_fts, rowid, session_id, content) VALUES ('delete', old.rowid, old.session_id, old.content);
INSERT INTO messages_fts(rowid, session_id, content) VALUES (new.rowid, new.session_id, new.content);
END",
[],
)?;
// Trigger para deletar de messages_fts quando mensagem é deletada
self.conn.execute(
"CREATE TRIGGER IF NOT EXISTS messages_fts_delete AFTER DELETE ON messages BEGIN
@@ -228,52 +298,39 @@ impl Database {
Ok(())
}
/// Popula tabelas FTS com dados existentes
/// Popula tabelas FTS se estiverem vazias e existirem dados nas tabelas principais
fn populate_fts_tables(&self) -> SqliteResult<()> {
// Verificar se sessions_fts já tem dados
let count: i64 = self.conn.query_row(
"SELECT COUNT(*) FROM sessions_fts",
[],
|row| row.get(0),
).unwrap_or(0);
// Se vazio, popular com dados existentes
if count == 0 {
self.conn.execute(
"INSERT INTO sessions_fts(rowid, id, title)
SELECT rowid, id, title FROM sessions",
[],
)?;
// Verificar se sessions_fts está vazia
let fts_count: i64 = self.conn.query_row("SELECT COUNT(*) FROM sessions_fts", [], |row| row.get(0))?;
if fts_count == 0 {
let main_count: i64 = self.conn.query_row("SELECT COUNT(*) FROM sessions", [], |row| row.get(0))?;
if main_count > 0 {
self.conn.execute(
"INSERT INTO sessions_fts(rowid, id, title) SELECT rowid, id, title FROM sessions",
[],
)?;
}
}
// Verificar se messages_fts já tem dados
let count: i64 = self.conn.query_row(
"SELECT COUNT(*) FROM messages_fts",
[],
|row| row.get(0),
).unwrap_or(0);
// Se vazio, popular com dados existentes
if count == 0 {
self.conn.execute(
"INSERT INTO messages_fts(rowid, session_id, content)
SELECT rowid, session_id, content FROM messages",
[],
)?;
// Verificar se messages_fts está vazia
let fts_msg_count: i64 = self.conn.query_row("SELECT COUNT(*) FROM messages_fts", [], |row| row.get(0))?;
if fts_msg_count == 0 {
let main_msg_count: i64 = self.conn.query_row("SELECT COUNT(*) FROM messages", [], |row| row.get(0))?;
if main_msg_count > 0 {
self.conn.execute(
"INSERT INTO messages_fts(rowid, session_id, content) SELECT rowid, session_id, content FROM messages",
[],
)?;
}
}
Ok(())
}
/// Cria uma nova sessão de chat
pub fn create_session(&self, session: &ChatSession) -> SqliteResult<()> {
self.conn.execute(
"INSERT INTO sessions (id, title, emoji, created_at, updated_at)
VALUES (?1, ?2, ?3, ?4, ?5)
ON CONFLICT(id) DO UPDATE SET
title = ?2,
emoji = ?3,
updated_at = ?5",
"INSERT INTO sessions (id, title, emoji, created_at, updated_at) VALUES (?1, ?2, ?3, ?4, ?5)",
params![
session.id,
session.title,
@@ -326,12 +383,7 @@ impl Database {
}
}
/// Salva uma sessão (create ou update)
pub fn save_session(&self, session: &ChatSession) -> SqliteResult<()> {
self.create_session(session)
}
/// Lista todas as sessões ordenadas por updated_at DESC
/// Lista todas as sessões ordenadas por atualização
pub fn list_sessions(&self) -> SqliteResult<Vec<ChatSession>> {
let mut stmt = self.conn.prepare(
"SELECT id, title, emoji, created_at, updated_at FROM sessions ORDER BY updated_at DESC"
@@ -358,14 +410,14 @@ impl Database {
Ok(sessions)
}
/// Deleta uma sessão e todas as suas mensagens
/// Deleta uma sessão e suas mensagens (cascade)
pub fn delete_session(&self, session_id: &str) -> SqliteResult<()> {
self.conn.execute("DELETE FROM sessions WHERE id = ?1", params![session_id])?;
Ok(())
}
/// Adiciona uma mensagem a uma sessão
pub fn add_message(&self, message: &ChatMessage) -> SqliteResult<i64> {
/// Salva uma mensagem
pub fn save_message(&self, message: &ChatMessage) -> SqliteResult<i64> {
self.conn.execute(
"INSERT INTO messages (session_id, role, content, metadata, created_at)
VALUES (?1, ?2, ?3, ?4, ?5)",
@@ -378,15 +430,6 @@ impl Database {
],
)?;
// Atualizar updated_at da sessão
self.conn.execute(
"UPDATE sessions SET updated_at = ?1 WHERE id = ?2",
params![
message.created_at.to_rfc3339(),
message.session_id
],
)?;
Ok(self.conn.last_insert_rowid())
}
@@ -467,6 +510,11 @@ impl Database {
Ok(messages)
}
<<<<<<< HEAD
/// Conta o total de mensagens de uma sessão
pub fn count_messages(&self, session_id: &str) -> SqliteResult<usize> {
let count: usize = self.conn.query_row(
=======
/// Busca mensagens de uma sessão com paginação (lazy loading)
///
/// Retorna as últimas `limit` mensagens a partir do `offset`.
@@ -486,21 +534,44 @@ impl Database {
) -> SqliteResult<(Vec<ChatMessage>, usize, bool)> {
// Primeiro, obter o total de mensagens
let total_count: usize = self.conn.query_row(
>>>>>>> 593efd42e091a845dea82ee6646e027bce1e18c5
"SELECT COUNT(*) FROM messages WHERE session_id = ?1",
params![session_id],
|row| row.get(0),
)?;
<<<<<<< HEAD
Ok(count)
}
/// Busca mensagens paginadas (do mais recente para o mais antigo, mas retornado em ordem cronológica)
///
/// limit: número máximo de mensagens a retornar
/// offset: quantas mensagens pular (a partir do final/mais recente)
///
/// Exemplo: Total 100 mensagens. limit=20, offset=0 -> retorna msgs 81-100
/// Exemplo: Total 100 mensagens. limit=20, offset=20 -> retorna msgs 61-80
pub fn get_messages_paginated(&self, session_id: &str, limit: usize, offset: usize) -> SqliteResult<(Vec<ChatMessage>, usize, bool)> {
// Primeiro obter contagem total
let total_count = self.count_messages(session_id)?;
// Se não houver mensagens, retornar vazio
=======
// Se não há mensagens, retornar vazio
>>>>>>> 593efd42e091a845dea82ee6646e027bce1e18c5
if total_count == 0 {
return Ok((Vec::new(), 0, false));
}
<<<<<<< HEAD
// Calcular o limite real baseado no que está disponível
=======
// Estratégia: usar subquery para pegar as últimas N mensagens ordenadas DESC,
// depois ordenar ASC para manter ordem cronológica
//
// Se offset=0 e limit=30: queremos as últimas 30 mensagens
// Se offset=30 e limit=30: queremos as 30 mensagens antes das últimas 30
>>>>>>> 593efd42e091a845dea82ee6646e027bce1e18c5
let real_limit = std::cmp::min(limit, total_count.saturating_sub(offset));
if real_limit == 0 {
@@ -712,7 +783,12 @@ impl Database {
)?;
let like_query = format!("%{}%", query);
<<<<<<< HEAD
let rows = stmt.query_map(params![&like_query, limit], |row| {
=======
let rows = stmt.query_map(params![like_query, limit], |row| {
>>>>>>> 593efd42e091a845dea82ee6646e027bce1e18c5
Ok(SearchSessionResult {
session: ChatSession {
id: row.get(0)?,
@@ -736,5 +812,118 @@ impl Database {
Ok(sessions)
}
}
/// Log a search request
pub fn log_search(&self, log: &SearchLog) -> SqliteResult<()> {
self.conn.execute(
"INSERT INTO search_logs (
id, chat_id, user_id, timestamp, searxng_instance,
query, engines, filters, response_time_ms,
total_results, results_per_engine, error
) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)",
params![
log.id,
log.chat_id,
log.user_id,
log.timestamp.to_rfc3339(),
log.searxng_instance,
log.query,
log.engines.as_ref().map(|e| serde_json::to_string(e).unwrap_or_default()),
log.filters,
log.response_time_ms,
log.total_results,
log.results_per_engine,
log.error
],
)?;
Ok(())
}
pub fn cleanup_search_logs(&self, days: i64) -> SqliteResult<usize> {
let cutoff = Utc::now() - chrono::Duration::days(days);
self.conn.execute(
"DELETE FROM search_logs WHERE timestamp < ?1",
params![cutoff.to_rfc3339()],
)
}
/// Get search logs with filters
pub fn get_search_logs(
&self,
chat_id: Option<&str>,
limit: usize,
offset: usize
) -> SqliteResult<Vec<SearchLog>> {
let mut query = String::from("SELECT id, chat_id, user_id, timestamp, searxng_instance, query, engines, filters, response_time_ms, total_results, results_per_engine, error FROM search_logs");
let mut params_vec: Vec<Box<dyn rusqlite::ToSql>> = Vec::new();
if let Some(cid) = chat_id {
query.push_str(" WHERE chat_id = ?1");
params_vec.push(Box::new(cid.to_string()));
}
query.push_str(" ORDER BY timestamp DESC LIMIT ?");
if chat_id.is_some() {
query.push_str(&format!("{}", params_vec.len() + 1));
} else {
query.push_str("1"); // ?1
}
params_vec.push(Box::new(limit));
query.push_str(" OFFSET ?");
if chat_id.is_some() {
query.push_str(&format!("{}", params_vec.len() + 1));
} else {
query.push_str("2"); // ?2
}
params_vec.push(Box::new(offset));
let mut stmt = self.conn.prepare(&query)?;
// Usar uma função auxiliar para evitar problema de tipos de closures incompatíveis
// Criar uma referência ao self para usar na closure
let logs = if let Some(cid) = chat_id {
let rows = stmt.query_map(params![cid, limit, offset], |row| {
self.map_search_log_row(row)
})?;
let mut result = Vec::new();
for row in rows {
result.push(row?);
}
result
} else {
let rows = stmt.query_map(params![limit, offset], |row| {
self.map_search_log_row(row)
})?;
let mut result = Vec::new();
for row in rows {
result.push(row?);
}
result
};
Ok(logs)
}
fn map_search_log_row(&self, row: &rusqlite::Row) -> SqliteResult<SearchLog> {
Ok(SearchLog {
id: row.get(0)?,
chat_id: row.get(1)?,
user_id: row.get(2)?,
timestamp: DateTime::parse_from_rfc3339(&row.get::<_, String>(3)?)
.map_err(|_| rusqlite::Error::InvalidColumnType(3, "TEXT".to_string(), rusqlite::types::Type::Text))?
.with_timezone(&Utc),
searxng_instance: row.get(4)?,
query: row.get(5)?,
engines: row.get::<_, Option<String>>(6)?
.map(|s| serde_json::from_str(&s).unwrap_or_default()),
filters: row.get(7)?,
response_time_ms: row.get(8)?,
total_results: row.get(9)?,
results_per_engine: row.get(10)?,
error: row.get(11)?,
})
}
}
+285
View File
@@ -0,0 +1,285 @@
use anyhow::Result;
use std::process::Command;
use tauri::{AppHandle, Manager, Emitter};
use serde::Serialize;
#[derive(Serialize, Clone)]
pub struct FixProgressEvent {
pub event_type: FixEventType,
pub message: String,
pub progress: Option<u8>,
}
#[derive(Serialize, Clone)]
#[serde(rename_all = "snake_case")]
pub enum FixEventType {
Start,
Progress,
Complete,
Error,
}
/// Corrige problemas encontrados nos requisitos do sistema
pub async fn fix_system_requirements(
app_handle: AppHandle,
os: &str,
issues: &[String],
) -> Result<()> {
emit_fix_event(&app_handle, FixEventType::Start, "Iniciando correção de problemas...", None);
match os {
"windows" => fix_windows_requirements(&app_handle, issues).await,
"linux" => fix_linux_requirements(&app_handle, issues).await,
"mac" => fix_macos_requirements(&app_handle, issues).await,
_ => Err(anyhow::anyhow!("Sistema operacional não suportado")),
}
}
async fn fix_windows_requirements(
app_handle: &AppHandle,
issues: &[String],
) -> Result<()> {
let mut fixed = 0;
let total = issues.len();
for issue in issues {
if issue.contains("VirtualMachinePlatform") {
emit_fix_event(
app_handle,
FixEventType::Progress,
"Habilitando VirtualMachinePlatform...",
Some((fixed * 100 / total) as u8),
);
let output = Command::new("powershell")
.args(&[
"-Command",
"Enable-WindowsOptionalFeature -Online -FeatureName VirtualMachinePlatform -NoRestart -All"
])
.output();
if let Ok(o) = output {
if o.status.success() {
fixed += 1;
emit_fix_event(
app_handle,
FixEventType::Progress,
"VirtualMachinePlatform habilitado",
Some((fixed * 100 / total) as u8),
);
}
}
}
if issue.contains("WSL") || issue.contains("Microsoft-Windows-Subsystem-Linux") {
emit_fix_event(
app_handle,
FixEventType::Progress,
"Habilitando WSL...",
Some((fixed * 100 / total) as u8),
);
let output = Command::new("powershell")
.args(&[
"-Command",
"Enable-WindowsOptionalFeature -Online -FeatureName Microsoft-Windows-Subsystem-Linux -NoRestart -All"
])
.output();
if let Ok(o) = output {
if o.status.success() {
fixed += 1;
emit_fix_event(
app_handle,
FixEventType::Progress,
"WSL habilitado",
Some((fixed * 100 / total) as u8),
);
}
}
}
if issue.contains("WSL 2") {
emit_fix_event(
app_handle,
FixEventType::Progress,
"Configurando WSL 2...",
Some((fixed * 100 / total) as u8),
);
let output = Command::new("wsl")
.args(&["--set-default-version", "2"])
.output();
if let Ok(o) = output {
if o.status.success() {
fixed += 1;
emit_fix_event(
app_handle,
FixEventType::Progress,
"WSL 2 configurado",
Some((fixed * 100 / total) as u8),
);
}
}
}
}
emit_fix_event(
app_handle,
FixEventType::Complete,
&format!("Correção concluída: {} de {} problemas corrigidos", fixed, total),
Some(100),
);
Ok(())
}
async fn fix_linux_requirements(
app_handle: &AppHandle,
issues: &[String],
) -> Result<()> {
let mut fixed = 0;
let total = issues.len();
for issue in issues {
if issue.contains("Pacote") || issue.contains("não instalado") {
// Extrair nome do pacote
if let Some(pkg) = issue.split("Pacote").nth(1)
.and_then(|s| s.split("não").next())
.map(|s| s.trim())
{
emit_fix_event(
app_handle,
FixEventType::Progress,
&format!("Instalando {}...", pkg),
Some((fixed * 100 / total) as u8),
);
let output = Command::new("sudo")
.args(&["apt", "install", "-y", pkg])
.output();
if let Ok(o) = output {
if o.status.success() {
fixed += 1;
emit_fix_event(
app_handle,
FixEventType::Progress,
&format!("{} instalado", pkg),
Some((fixed * 100 / total) as u8),
);
}
}
}
}
if issue.contains("Privilégios sudo") {
emit_fix_event(
app_handle,
FixEventType::Progress,
"Verificando privilégios sudo...",
Some((fixed * 100 / total) as u8),
);
// Verificar se sudo está disponível
let output = Command::new("sudo")
.args(&["-n", "true"])
.output();
if let Ok(o) = output {
if o.status.success() {
fixed += 1;
emit_fix_event(
app_handle,
FixEventType::Progress,
"Privilégios sudo disponíveis",
Some((fixed * 100 / total) as u8),
);
} else {
emit_fix_event(
app_handle,
FixEventType::Error,
"Privilégios sudo necessários. Execute com sudo ou configure sudoers.",
None,
);
}
}
}
if issue.contains("grupo docker") || issue.contains("Permissões Docker") {
emit_fix_event(
app_handle,
FixEventType::Progress,
"Adicionando usuário ao grupo docker...",
Some((fixed * 100 / total) as u8),
);
// Obter usuário atual
if let Ok(user) = std::env::var("USER") {
let output = Command::new("sudo")
.args(&["usermod", "-aG", "docker", &user])
.output();
if let Ok(o) = output {
if o.status.success() {
fixed += 1;
emit_fix_event(
app_handle,
FixEventType::Progress,
"Usuário adicionado ao grupo docker. Faça logout e login novamente.",
Some((fixed * 100 / total) as u8),
);
}
}
}
}
}
emit_fix_event(
app_handle,
FixEventType::Complete,
&format!("Correção concluída: {} de {} problemas corrigidos", fixed, total),
Some(100),
);
Ok(())
}
async fn fix_macos_requirements(
app_handle: &AppHandle,
_issues: &[String],
) -> Result<()> {
emit_fix_event(
app_handle,
FixEventType::Progress,
"No macOS, a maioria das correções requer instalação manual via Homebrew",
None,
);
emit_fix_event(
app_handle,
FixEventType::Complete,
"Verifique os avisos e instale manualmente se necessário",
Some(100),
);
Ok(())
}
fn emit_fix_event(
app_handle: &AppHandle,
event_type: FixEventType,
message: &str,
progress: Option<u8>,
) {
let event = FixProgressEvent {
event_type,
message: message.to_string(),
progress,
};
if let Some(window) = app_handle.get_webview_window("main") {
let _ = window.emit("docker-fix-progress", event);
}
}
+600
View File
@@ -0,0 +1,600 @@
use serde::Serialize;
use tauri::{AppHandle, Manager, Emitter};
use std::process::Command;
use std::process::Stdio as StdProcessStdio;
use tokio::process::Command as TokioCommand;
use tokio::io::{AsyncBufReadExt, BufReader};
use anyhow::Result;
use std::time::Duration;
use tokio::time::sleep;
#[derive(Serialize, Clone, Debug)]
pub struct InstallStep {
pub id: u8,
pub name: String,
pub weight: u8,
pub command: InstallCommand,
}
#[derive(Serialize, Clone, Debug)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum InstallCommand {
Shell { cmd: String, args: Vec<String> },
PowerShell { script: String },
WaitDockerReady,
EnableWindowsFeature { feature: String },
}
#[derive(Serialize, Clone)]
pub struct InstallProgressEvent {
pub event_type: ProgressEventType,
pub step: Option<u8>,
pub step_name: Option<String>,
pub total_steps: Option<u8>,
pub progress: u8,
pub message: Option<String>,
pub log_line: Option<String>,
}
#[derive(Serialize, Clone)]
#[serde(rename_all = "snake_case")]
pub enum ProgressEventType {
StepStart,
Log,
StepComplete,
Error,
Complete,
}
/// Instala Docker no sistema com monitoramento de progresso
pub async fn install_docker(app_handle: AppHandle, os: &str) -> Result<()> {
let steps = get_install_steps(os);
let total_weight: u16 = steps.iter().map(|s| s.weight as u16).sum();
let mut completed_weight: u16 = 0;
let total_steps = steps.len() as u8;
emit_progress_event(
&app_handle,
ProgressEventType::StepStart,
None,
None,
Some(total_steps),
0,
Some("Iniciando instalação do Docker...".to_string()),
None,
);
for (idx, step) in steps.iter().enumerate() {
let step_num = (idx + 1) as u8;
// Emitir início da etapa
emit_progress_event(
&app_handle,
ProgressEventType::StepStart,
Some(step.id),
Some(step.name.clone()),
Some(total_steps),
((completed_weight * 100) / total_weight) as u8,
Some(format!("Executando: {}", step.name)),
None,
);
// Executar comando
match execute_step(&app_handle, step, step_num).await {
Ok(()) => {
completed_weight += step.weight as u16;
let progress = ((completed_weight * 100) / total_weight) as u8;
emit_progress_event(
&app_handle,
ProgressEventType::StepComplete,
Some(step.id),
Some(step.name.clone()),
Some(total_steps),
progress,
Some(format!("✓ {} concluído", step.name)),
None,
);
}
Err(e) => {
emit_progress_event(
&app_handle,
ProgressEventType::Error,
Some(step.id),
Some(step.name.clone()),
Some(total_steps),
((completed_weight * 100) / total_weight) as u8,
Some(format!("Erro em {}: {}", step.name, e)),
None,
);
return Err(e);
}
}
// Pequeno delay entre etapas
sleep(Duration::from_millis(500)).await;
}
emit_progress_event(
&app_handle,
ProgressEventType::Complete,
None,
None,
Some(total_steps),
100,
Some("Docker instalado com sucesso!".to_string()),
None,
);
Ok(())
}
/// Executa uma etapa de instalação
async fn execute_step(app_handle: &AppHandle, step: &InstallStep, step_num: u8) -> Result<()> {
match &step.command {
InstallCommand::Shell { cmd, args } => {
execute_shell_command(app_handle, cmd, args, step_num).await
}
InstallCommand::PowerShell { script } => {
execute_powershell_script(app_handle, script, step_num).await
}
InstallCommand::WaitDockerReady => {
wait_for_docker_ready(app_handle).await
}
InstallCommand::EnableWindowsFeature { feature } => {
enable_windows_feature(app_handle, feature, step_num).await
}
}
}
/// Executa comando shell e captura saída em tempo real
async fn execute_shell_command(
app_handle: &AppHandle,
cmd: &str,
args: &[String],
step_num: u8,
) -> Result<()> {
let mut command = TokioCommand::new(cmd);
command.args(args);
command.stdout(StdProcessStdio::piped());
command.stderr(StdProcessStdio::piped());
let mut child = command.spawn()?;
// Capturar stdout
if let Some(stdout) = child.stdout.take() {
let app_handle_clone = app_handle.clone();
let step_num_clone = step_num;
tokio::spawn(async move {
let reader = BufReader::new(stdout);
let mut lines = reader.lines();
while let Ok(Some(line)) = lines.next_line().await {
emit_progress_event(
&app_handle_clone,
ProgressEventType::Log,
Some(step_num_clone),
None,
None,
0,
None,
Some(line),
);
}
});
}
// Capturar stderr
if let Some(stderr) = child.stderr.take() {
let app_handle_clone = app_handle.clone();
let step_num_clone = step_num;
tokio::spawn(async move {
let reader = BufReader::new(stderr);
let mut lines = reader.lines();
while let Ok(Some(line)) = lines.next_line().await {
emit_progress_event(
&app_handle_clone,
ProgressEventType::Log,
Some(step_num_clone),
None,
None,
0,
None,
Some(format!("[stderr] {}", line)),
);
}
});
}
let status = child.wait().await?;
if !status.success() {
return Err(anyhow::anyhow!("Comando falhou com código: {:?}", status.code()));
}
Ok(())
}
/// Executa script PowerShell
async fn execute_powershell_script(
app_handle: &AppHandle,
script: &str,
step_num: u8,
) -> Result<()> {
#[cfg(target_os = "windows")]
{
let mut command = TokioCommand::new("powershell");
command.args(&["-Command", script]);
command.stdout(StdProcessStdio::piped());
command.stderr(StdProcessStdio::piped());
let mut child = command.spawn()?;
// Capturar saída
if let Some(stdout) = child.stdout.take() {
let app_handle_clone = app_handle.clone();
let step_num_clone = step_num;
tokio::spawn(async move {
let reader = BufReader::new(stdout);
let mut lines = reader.lines();
while let Ok(Some(line)) = lines.next_line().await {
emit_progress_event(
&app_handle_clone,
ProgressEventType::Log,
Some(step_num_clone),
None,
None,
0,
None,
Some(line),
);
}
});
}
let status = child.wait().await?;
if !status.success() {
return Err(anyhow::anyhow!("PowerShell script falhou"));
}
Ok(())
}
#[cfg(not(target_os = "windows"))]
{
Err(anyhow::anyhow!("PowerShell não disponível neste sistema"))
}
}
/// Habilita feature do Windows
async fn enable_windows_feature(
app_handle: &AppHandle,
feature: &str,
step_num: u8,
) -> Result<()> {
#[cfg(target_os = "windows")]
{
let script = format!(
"Enable-WindowsOptionalFeature -Online -FeatureName {} -NoRestart -All",
feature
);
execute_powershell_script(app_handle, &script, step_num).await
}
#[cfg(not(target_os = "windows"))]
{
Err(anyhow::anyhow!("Habilitar features do Windows não disponível neste sistema"))
}
}
/// Aguarda Docker ficar pronto
async fn wait_for_docker_ready(app_handle: &AppHandle) -> Result<()> {
emit_progress_event(
app_handle,
ProgressEventType::Log,
None,
None,
None,
0,
Some("Aguardando Docker inicializar...".to_string()),
None,
);
for i in 0..30 {
// Tentar docker info
let output = Command::new("docker")
.arg("info")
.output();
if let Ok(o) = output {
if o.status.success() {
emit_progress_event(
app_handle,
ProgressEventType::Log,
None,
None,
None,
0,
Some("Docker está pronto!".to_string()),
None,
);
return Ok(());
}
}
sleep(Duration::from_secs(1)).await;
if i % 5 == 0 {
emit_progress_event(
app_handle,
ProgressEventType::Log,
None,
None,
None,
0,
Some(format!("Aguardando... ({}s)", i)),
None,
);
}
}
Err(anyhow::anyhow!("Timeout aguardando Docker inicializar"))
}
/// Obtém etapas de instalação por plataforma
fn get_install_steps(os: &str) -> Vec<InstallStep> {
match os {
"linux" => get_linux_steps(),
"windows" => get_windows_steps(),
"mac" => get_macos_steps(),
_ => vec![],
}
}
fn get_linux_steps() -> Vec<InstallStep> {
vec![
InstallStep {
id: 1,
name: "Atualizar repositórios".to_string(),
weight: 10,
command: InstallCommand::Shell {
cmd: "sudo".to_string(),
args: vec!["apt".to_string(), "update".to_string()],
},
},
InstallStep {
id: 2,
name: "Instalar dependências".to_string(),
weight: 15,
command: InstallCommand::Shell {
cmd: "sudo".to_string(),
args: vec![
"apt".to_string(),
"install".to_string(),
"-y".to_string(),
"ca-certificates".to_string(),
"curl".to_string(),
"gnupg".to_string(),
],
},
},
InstallStep {
id: 3,
name: "Adicionar chave GPG Docker".to_string(),
weight: 5,
command: InstallCommand::Shell {
cmd: "sudo".to_string(),
args: vec![
"install".to_string(),
"-m".to_string(),
"0755".to_string(),
"-d".to_string(),
"/etc/apt/keyrings".to_string(),
],
},
},
InstallStep {
id: 4,
name: "Configurar repositório Docker".to_string(),
weight: 10,
command: InstallCommand::Shell {
cmd: "bash".to_string(),
args: vec![
"-c".to_string(),
r#"curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null"#.to_string(),
],
},
},
InstallStep {
id: 5,
name: "Atualizar índice de pacotes".to_string(),
weight: 10,
command: InstallCommand::Shell {
cmd: "sudo".to_string(),
args: vec!["apt".to_string(), "update".to_string()],
},
},
InstallStep {
id: 6,
name: "Instalar Docker Engine".to_string(),
weight: 40,
command: InstallCommand::Shell {
cmd: "sudo".to_string(),
args: vec![
"apt".to_string(),
"install".to_string(),
"-y".to_string(),
"docker-ce".to_string(),
"docker-ce-cli".to_string(),
"containerd.io".to_string(),
"docker-buildx-plugin".to_string(),
"docker-compose-plugin".to_string(),
],
},
},
InstallStep {
id: 7,
name: "Verificar instalação".to_string(),
weight: 5,
command: InstallCommand::Shell {
cmd: "docker".to_string(),
args: vec!["--version".to_string()],
},
},
InstallStep {
id: 8,
name: "Testar container".to_string(),
weight: 5,
command: InstallCommand::Shell {
cmd: "sudo".to_string(),
args: vec!["docker".to_string(), "run".to_string(), "hello-world".to_string()],
},
},
]
}
fn get_windows_steps() -> Vec<InstallStep> {
vec![
InstallStep {
id: 1,
name: "Habilitar VirtualMachinePlatform".to_string(),
weight: 20,
command: InstallCommand::EnableWindowsFeature {
feature: "VirtualMachinePlatform".to_string(),
},
},
InstallStep {
id: 2,
name: "Habilitar WSL".to_string(),
weight: 20,
command: InstallCommand::EnableWindowsFeature {
feature: "Microsoft-Windows-Subsystem-Linux".to_string(),
},
},
InstallStep {
id: 3,
name: "Verificar reinício necessário".to_string(),
weight: 5,
command: InstallCommand::PowerShell {
script: r#"$restart = (Get-ComputerRestartRequired); if ($restart) { Write-Host "Reinício necessário. Por favor, reinicie o sistema e execute novamente." } else { Write-Host "Reinício não necessário." }"#.to_string(),
},
},
InstallStep {
id: 4,
name: "Configurar WSL 2".to_string(),
weight: 10,
command: InstallCommand::PowerShell {
script: "wsl --set-default-version 2".to_string(),
},
},
InstallStep {
id: 5,
name: "Instalar distribuição Ubuntu".to_string(),
weight: 15,
command: InstallCommand::PowerShell {
script: "wsl --install -d Ubuntu".to_string(),
},
},
InstallStep {
id: 6,
name: "Instalar Docker no WSL".to_string(),
weight: 25,
command: InstallCommand::PowerShell {
script: r#"wsl -d Ubuntu -e bash -c "curl -fsSL https://get.docker.com -o get-docker.sh && sudo sh get-docker.sh""#.to_string(),
},
},
InstallStep {
id: 7,
name: "Verificar instalação".to_string(),
weight: 5,
command: InstallCommand::PowerShell {
script: r#"wsl -d Ubuntu -e docker --version"#.to_string(),
},
},
]
}
fn get_macos_steps() -> Vec<InstallStep> {
vec![
InstallStep {
id: 1,
name: "Verificar Homebrew".to_string(),
weight: 5,
command: InstallCommand::Shell {
cmd: "brew".to_string(),
args: vec!["--version".to_string()],
},
},
InstallStep {
id: 2,
name: "Atualizar Homebrew".to_string(),
weight: 10,
command: InstallCommand::Shell {
cmd: "brew".to_string(),
args: vec!["update".to_string()],
},
},
InstallStep {
id: 3,
name: "Instalar Docker Desktop".to_string(),
weight: 60,
command: InstallCommand::Shell {
cmd: "brew".to_string(),
args: vec!["install".to_string(), "--cask".to_string(), "docker".to_string()],
},
},
InstallStep {
id: 4,
name: "Iniciar Docker Desktop".to_string(),
weight: 15,
command: InstallCommand::Shell {
cmd: "open".to_string(),
args: vec!["-a".to_string(), "Docker".to_string()],
},
},
InstallStep {
id: 5,
name: "Aguardar inicialização".to_string(),
weight: 5,
command: InstallCommand::WaitDockerReady,
},
InstallStep {
id: 6,
name: "Verificar instalação".to_string(),
weight: 5,
command: InstallCommand::Shell {
cmd: "docker".to_string(),
args: vec!["--version".to_string()],
},
},
]
}
/// Emite evento de progresso via Tauri
fn emit_progress_event(
app_handle: &AppHandle,
event_type: ProgressEventType,
step: Option<u8>,
step_name: Option<String>,
total_steps: Option<u8>,
progress: u8,
message: Option<String>,
log_line: Option<String>,
) {
let event = InstallProgressEvent {
event_type,
step,
step_name,
total_steps,
progress,
message,
log_line,
};
if let Some(window) = app_handle.get_webview_window("main") {
let _ = window.emit("docker-install-progress", event);
}
}
+538
View File
@@ -0,0 +1,538 @@
use serde::{Serialize, Deserialize};
use sysinfo::System;
use std::process::Command;
use anyhow::Result;
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct SystemRequirements {
pub status: RequirementStatus,
pub os: String,
pub arch: String,
pub ram_mb: u64,
pub disk_gb: u64,
pub cpu_cores: usize,
pub kernel_version: Option<String>,
pub issues: Vec<String>,
pub warnings: Vec<String>,
}
#[derive(Serialize, Deserialize, Clone, Debug)]
#[serde(rename_all = "lowercase")]
pub enum RequirementStatus {
Pass,
Fail,
Warn,
}
/// Verifica requisitos do sistema para instalação do Docker
pub fn check_system_requirements() -> Result<SystemRequirements> {
let os = get_operating_system();
match os.as_str() {
"linux" => check_linux_requirements(),
"windows" => check_windows_requirements(),
"mac" => check_macos_requirements(),
_ => {
Ok(SystemRequirements {
status: RequirementStatus::Fail,
os,
arch: "unknown".to_string(),
ram_mb: 0,
disk_gb: 0,
cpu_cores: 0,
kernel_version: None,
issues: vec!["Sistema operacional não suportado".to_string()],
warnings: vec![],
})
}
}
}
/// Verifica requisitos no Linux
fn check_linux_requirements() -> Result<SystemRequirements> {
let mut sys = System::new_all();
sys.refresh_all();
let mut status = RequirementStatus::Pass;
let mut issues = Vec::new();
let mut warnings = Vec::new();
// Verificar versão do kernel
let kernel_version = get_kernel_version();
if let Some(kernel) = &kernel_version {
if !is_kernel_version_sufficient(kernel) {
issues.push(format!("Kernel {} menor que 3.10 requerido", kernel));
status = RequirementStatus::Fail;
}
} else {
warnings.push("Não foi possível verificar versão do kernel".to_string());
status = match status {
RequirementStatus::Fail => RequirementStatus::Fail,
_ => RequirementStatus::Warn,
};
}
// Verificar RAM (mínimo 2GB)
let ram_mb = sys.total_memory() / (1024 * 1024);
if ram_mb < 2048 {
issues.push(format!("RAM insuficiente: {}MB (mínimo 2GB)", ram_mb));
status = RequirementStatus::Fail;
}
// Verificar espaço em disco (mínimo 10GB)
let disk_gb = get_disk_space_gb();
if disk_gb < 10 {
issues.push(format!("Espaço em disco insuficiente: {}GB (mínimo 10GB)", disk_gb));
status = RequirementStatus::Fail;
}
// Verificar CPU cores (mínimo 2, mas apenas warning)
let cpu_cores = sys.cpus().len();
if cpu_cores < 2 {
warnings.push(format!("CPU cores insuficientes: {} (recomendado 2+)", cpu_cores));
if matches!(status, RequirementStatus::Pass) {
status = RequirementStatus::Warn;
}
}
// Verificar privilégios sudo
if !check_sudo_available() {
issues.push("Privilégios sudo necessários para instalação".to_string());
status = RequirementStatus::Fail;
}
// Verificar dependências básicas
let missing_deps = check_linux_dependencies();
for dep in missing_deps {
issues.push(format!("Pacote {} não instalado", dep));
status = RequirementStatus::Fail;
}
Ok(SystemRequirements {
status,
os: "linux".to_string(),
arch: get_arch(),
ram_mb,
disk_gb,
cpu_cores,
kernel_version,
issues,
warnings,
})
}
/// Verifica requisitos no Windows
fn check_windows_requirements() -> Result<SystemRequirements> {
let mut sys = System::new_all();
sys.refresh_all();
let mut status = RequirementStatus::Pass;
let mut issues = Vec::new();
let mut warnings = Vec::new();
// Verificar versão do Windows (Windows 10 20H1+ ou Windows 11)
let windows_version = get_windows_version();
if let Some(version) = windows_version {
if version < 19041 {
issues.push(format!("Windows build {} menor que 19041 (20H1) requerido", version));
status = RequirementStatus::Fail;
}
} else {
warnings.push("Não foi possível verificar versão do Windows".to_string());
status = match status {
RequirementStatus::Fail => RequirementStatus::Fail,
_ => RequirementStatus::Warn,
};
}
// Verificar RAM (mínimo 4GB para Windows)
let ram_mb = sys.total_memory() / (1024 * 1024);
if ram_mb < 4096 {
issues.push(format!("RAM insuficiente: {}MB (mínimo 4GB)", ram_mb));
status = RequirementStatus::Fail;
}
// Verificar espaço em disco (mínimo 10GB)
let disk_gb = get_disk_space_gb();
if disk_gb < 10 {
issues.push(format!("Espaço em disco insuficiente: {}GB (mínimo 10GB)", disk_gb));
status = RequirementStatus::Fail;
}
// Verificar CPU cores
let cpu_cores = sys.cpus().len();
if cpu_cores < 2 {
warnings.push(format!("CPU cores insuficientes: {} (recomendado 2+)", cpu_cores));
if matches!(status, RequirementStatus::Pass) {
status = RequirementStatus::Warn;
}
}
// Verificar VirtualMachinePlatform
if !check_windows_feature("VirtualMachinePlatform") {
issues.push("VirtualMachinePlatform não habilitado".to_string());
status = RequirementStatus::Fail;
}
// Verificar WSL
if !check_windows_feature("Microsoft-Windows-Subsystem-Linux") {
issues.push("WSL não habilitado".to_string());
status = RequirementStatus::Fail;
}
// Verificar versão WSL
if !check_wsl_version() {
issues.push("WSL 2 não configurado ou não disponível".to_string());
status = RequirementStatus::Fail;
}
Ok(SystemRequirements {
status,
os: "windows".to_string(),
arch: get_arch(),
ram_mb,
disk_gb,
cpu_cores,
kernel_version: None,
issues,
warnings,
})
}
/// Verifica requisitos no macOS
fn check_macos_requirements() -> Result<SystemRequirements> {
let mut sys = System::new_all();
sys.refresh_all();
let mut status = RequirementStatus::Pass;
let mut issues = Vec::new();
let mut warnings = Vec::new();
// Verificar versão do macOS (mínimo 11)
let macos_version = get_macos_version();
if let Some(version) = &macos_version {
let major: u32 = version.split('.').next()
.and_then(|v| v.parse().ok())
.unwrap_or(0);
if major < 11 {
warnings.push(format!("macOS {} pode não ser suportado (recomendado 11+)", version));
if matches!(status, RequirementStatus::Pass) {
status = RequirementStatus::Warn;
}
}
} else {
warnings.push("Não foi possível verificar versão do macOS".to_string());
status = match status {
RequirementStatus::Fail => RequirementStatus::Fail,
_ => RequirementStatus::Warn,
};
}
// Verificar RAM (mínimo 4GB para macOS)
let ram_mb = sys.total_memory() / (1024 * 1024);
if ram_mb < 4096 {
issues.push(format!("RAM insuficiente: {}MB (mínimo 4GB)", ram_mb));
status = RequirementStatus::Fail;
}
// Verificar espaço em disco (mínimo 4GB para macOS)
let disk_gb = get_disk_space_gb();
if disk_gb < 4 {
issues.push(format!("Espaço em disco insuficiente: {}GB (mínimo 4GB)", disk_gb));
status = RequirementStatus::Fail;
}
// Verificar CPU cores
let cpu_cores = sys.cpus().len();
if cpu_cores < 2 {
warnings.push(format!("CPU cores insuficientes: {} (recomendado 2+)", cpu_cores));
if matches!(status, RequirementStatus::Pass) {
status = RequirementStatus::Warn;
}
}
// Verificar Homebrew (opcional, mas recomendado)
if !check_homebrew_available() {
warnings.push("Homebrew não instalado (recomendado para instalação)".to_string());
if matches!(status, RequirementStatus::Pass) {
status = RequirementStatus::Warn;
}
}
// Verificar Rosetta 2 para Apple Silicon
#[cfg(target_arch = "aarch64")]
{
if !check_rosetta_installed() {
warnings.push("Rosetta 2 não instalado (necessário para imagens x86)".to_string());
if matches!(status, RequirementStatus::Pass) {
status = RequirementStatus::Warn;
}
}
}
Ok(SystemRequirements {
status,
os: "mac".to_string(),
arch: get_arch(),
ram_mb,
disk_gb,
cpu_cores,
kernel_version: macos_version,
issues,
warnings,
})
}
// Funções auxiliares
fn get_operating_system() -> String {
#[cfg(target_os = "windows")]
return "windows".to_string();
#[cfg(target_os = "macos")]
return "mac".to_string();
#[cfg(target_os = "linux")]
return "linux".to_string();
#[cfg(not(any(target_os = "windows", target_os = "macos", target_os = "linux")))]
return "unknown".to_string();
}
fn get_arch() -> String {
std::env::consts::ARCH.to_string()
}
fn get_kernel_version() -> Option<String> {
#[cfg(target_os = "linux")]
{
Command::new("uname")
.arg("-r")
.output()
.ok()
.and_then(|o| String::from_utf8(o.stdout).ok())
.map(|v| v.trim().to_string())
}
#[cfg(not(target_os = "linux"))]
None
}
fn is_kernel_version_sufficient(kernel: &str) -> bool {
let parts: Vec<&str> = kernel.split('.').collect();
if parts.len() < 2 {
return false;
}
let major: u32 = parts[0].parse().unwrap_or(0);
let minor: u32 = parts[1].parse().unwrap_or(0);
major > 3 || (major == 3 && minor >= 10)
}
fn get_disk_space_gb() -> u64 {
#[cfg(target_os = "windows")]
{
// Windows: usar wmic
if let Ok(output) = Command::new("wmic")
.args(&["logicaldisk", "get", "size,freespace", "/format:list"])
.output()
{
if let Ok(text) = String::from_utf8(output.stdout) {
// Parse output para obter espaço livre
for line in text.lines() {
if line.starts_with("FreeSpace=") {
if let Some(bytes_str) = line.split('=').nth(1) {
if let Ok(bytes) = bytes_str.trim().parse::<u64>() {
return bytes / (1024 * 1024 * 1024);
}
}
}
}
}
}
}
#[cfg(target_os = "linux")]
{
// Linux: usar df
if let Ok(output) = Command::new("df")
.arg("-BG")
.arg("/")
.output()
{
if let Ok(text) = String::from_utf8(output.stdout) {
if let Some(line) = text.lines().nth(1) {
let parts: Vec<&str> = line.split_whitespace().collect();
if parts.len() >= 4 {
if let Ok(gb) = parts[3].trim_end_matches('G').parse::<u64>() {
return gb;
}
}
}
}
}
}
#[cfg(target_os = "macos")]
{
// macOS: usar df
if let Ok(output) = Command::new("df")
.arg("-g")
.arg("/")
.output()
{
if let Ok(text) = String::from_utf8(output.stdout) {
if let Some(line) = text.lines().nth(1) {
let parts: Vec<&str> = line.split_whitespace().collect();
if parts.len() >= 4 {
if let Ok(gb) = parts[3].parse::<u64>() {
return gb;
}
}
}
}
}
}
// Fallback: retornar valor alto para não bloquear
100
}
fn check_sudo_available() -> bool {
#[cfg(target_os = "linux")]
{
Command::new("sudo")
.arg("-n")
.arg("true")
.output()
.map(|o| o.status.success())
.unwrap_or(false)
}
#[cfg(not(target_os = "linux"))]
true
}
fn check_linux_dependencies() -> Vec<String> {
let mut missing = Vec::new();
let deps = vec!["curl", "ca-certificates", "gnupg"];
for dep in deps {
let output = Command::new("dpkg")
.args(&["-l", dep])
.output();
if let Ok(o) = output {
let text = String::from_utf8_lossy(&o.stdout);
if !text.contains(&format!("ii {}", dep)) {
missing.push(dep.to_string());
}
} else {
missing.push(dep.to_string());
}
}
missing
}
fn get_windows_version() -> Option<u32> {
#[cfg(target_os = "windows")]
{
if let Ok(output) = Command::new("powershell")
.args(&["-Command", "[System.Environment]::OSVersion.Version.Build"])
.output()
{
if let Ok(text) = String::from_utf8(output.stdout) {
if let Ok(build) = text.trim().parse::<u32>() {
return Some(build);
}
}
}
}
None
}
fn check_windows_feature(feature: &str) -> bool {
#[cfg(target_os = "windows")]
{
if let Ok(output) = Command::new("powershell")
.args(&[
"-Command",
&format!("(Get-WindowsOptionalFeature -Online -FeatureName {}).State", feature)
])
.output()
{
if let Ok(text) = String::from_utf8(output.stdout) {
return text.contains("Enabled");
}
}
}
false
}
fn check_wsl_version() -> bool {
#[cfg(target_os = "windows")]
{
// Verificar se WSL está disponível
if let Ok(output) = Command::new("wsl")
.arg("--version")
.output()
{
if output.status.success() {
// Verificar versão padrão
if let Ok(output) = Command::new("wsl")
.args(&["--status"])
.output()
{
let text = String::from_utf8_lossy(&output.stdout);
return text.contains("2") || text.contains("WSL 2");
}
}
}
}
false
}
fn get_macos_version() -> Option<String> {
#[cfg(target_os = "macos")]
{
Command::new("sw_vers")
.arg("-productVersion")
.output()
.ok()
.and_then(|o| String::from_utf8(o.stdout).ok())
.map(|v| v.trim().to_string())
}
#[cfg(not(target_os = "macos"))]
None
}
fn check_homebrew_available() -> bool {
#[cfg(target_os = "macos")]
{
Command::new("brew")
.arg("--version")
.output()
.map(|o| o.status.success())
.unwrap_or(false)
}
#[cfg(not(target_os = "macos"))]
false
}
fn check_rosetta_installed() -> bool {
#[cfg(target_os = "macos")]
{
Command::new("/usr/bin/pgrep")
.arg("oahd")
.output()
.map(|o| o.status.success())
.unwrap_or(false)
}
#[cfg(not(target_os = "macos"))]
false
}
+251
View File
@@ -0,0 +1,251 @@
use serde::{Serialize, Deserialize};
use std::process::Command;
use anyhow::Result;
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct ValidationResult {
pub status: ValidationStatus,
pub tests: Vec<ValidationTest>,
pub docker_version: Option<String>,
}
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct ValidationTest {
pub name: String,
pub status: TestStatus,
pub message: Option<String>,
}
#[derive(Serialize, Deserialize, Clone, Debug)]
#[serde(rename_all = "lowercase")]
pub enum ValidationStatus {
Pass,
Fail,
Partial,
}
#[derive(Serialize, Deserialize, Clone, Debug)]
#[serde(rename_all = "lowercase")]
pub enum TestStatus {
Pass,
Fail,
Warn,
}
/// Valida instalação do Docker executando testes
pub fn validate_docker_installation() -> Result<ValidationResult> {
let mut tests = Vec::new();
let mut docker_version: Option<String> = None;
// Teste 1: Verificar versão
match test_docker_version() {
Ok(Some(version)) => {
docker_version = Some(version.clone());
tests.push(ValidationTest {
name: "Versão do Docker".to_string(),
status: TestStatus::Pass,
message: Some(format!("Docker {} instalado", version)),
});
}
Ok(None) => {
tests.push(ValidationTest {
name: "Versão do Docker".to_string(),
status: TestStatus::Fail,
message: Some("Docker não encontrado no PATH".to_string()),
});
}
Err(e) => {
tests.push(ValidationTest {
name: "Versão do Docker".to_string(),
status: TestStatus::Fail,
message: Some(format!("Erro ao verificar versão: {}", e)),
});
}
}
// Teste 2: Verificar serviço
match test_docker_service() {
Ok(true) => {
tests.push(ValidationTest {
name: "Serviço Docker".to_string(),
status: TestStatus::Pass,
message: Some("Serviço Docker está rodando".to_string()),
});
}
Ok(false) => {
tests.push(ValidationTest {
name: "Serviço Docker".to_string(),
status: TestStatus::Fail,
message: Some("Serviço Docker não está rodando".to_string()),
});
}
Err(e) => {
tests.push(ValidationTest {
name: "Serviço Docker".to_string(),
status: TestStatus::Fail,
message: Some(format!("Erro ao verificar serviço: {}", e)),
});
}
}
// Teste 3: Executar container de teste
match test_docker_container() {
Ok(true) => {
tests.push(ValidationTest {
name: "Container de Teste".to_string(),
status: TestStatus::Pass,
message: Some("Container hello-world executado com sucesso".to_string()),
});
}
Ok(false) => {
tests.push(ValidationTest {
name: "Container de Teste".to_string(),
status: TestStatus::Fail,
message: Some("Falha ao executar container de teste".to_string()),
});
}
Err(e) => {
tests.push(ValidationTest {
name: "Container de Teste".to_string(),
status: TestStatus::Fail,
message: Some(format!("Erro ao testar container: {}", e)),
});
}
}
// Teste 4: Verificar permissões
match test_docker_permissions() {
Ok(true) => {
tests.push(ValidationTest {
name: "Permissões Docker".to_string(),
status: TestStatus::Pass,
message: Some("Usuário pode executar comandos Docker".to_string()),
});
}
Ok(false) => {
tests.push(ValidationTest {
name: "Permissões Docker".to_string(),
status: TestStatus::Warn,
message: Some("Pode ser necessário adicionar usuário ao grupo docker ou usar sudo".to_string()),
});
}
Err(e) => {
tests.push(ValidationTest {
name: "Permissões Docker".to_string(),
status: TestStatus::Warn,
message: Some(format!("Erro ao verificar permissões: {}", e)),
});
}
}
// Determinar status geral
let pass_count = tests.iter().filter(|t| matches!(t.status, TestStatus::Pass)).count();
let fail_count = tests.iter().filter(|t| matches!(t.status, TestStatus::Fail)).count();
let status = if fail_count == 0 {
ValidationStatus::Pass
} else if pass_count > 0 {
ValidationStatus::Partial
} else {
ValidationStatus::Fail
};
Ok(ValidationResult {
status,
tests,
docker_version,
})
}
/// Testa se Docker está instalado e retorna versão
fn test_docker_version() -> Result<Option<String>> {
let output = Command::new("docker")
.arg("--version")
.output();
match output {
Ok(o) if o.status.success() => {
let version = String::from_utf8_lossy(&o.stdout);
let version = version.trim();
// Extrair apenas número da versão (ex: "Docker version 24.0.5" -> "24.0.5")
let version = version
.split_whitespace()
.nth(2)
.map(|s| s.trim_end_matches(',').to_string())
.unwrap_or_else(|| version.to_string());
Ok(Some(version))
}
Ok(_) => Ok(None),
Err(_) => Ok(None),
}
}
/// Testa se serviço Docker está rodando
fn test_docker_service() -> Result<bool> {
// Tentar docker info (mais portável que systemctl)
let output = Command::new("docker")
.arg("info")
.output();
match output {
Ok(o) => Ok(o.status.success()),
Err(_) => Ok(false),
}
}
/// Testa executando container hello-world
fn test_docker_container() -> Result<bool> {
// Primeiro, remover container antigo se existir
let _ = Command::new("docker")
.args(&["rm", "-f", "docker-test-hello-world"])
.output();
// Executar container de teste
let output = Command::new("docker")
.args(&["run", "--rm", "--name", "docker-test-hello-world", "hello-world"])
.output();
match output {
Ok(o) => {
if o.status.success() {
Ok(true)
} else {
// Verificar se erro é por falta de imagem (normal na primeira vez)
let stderr = String::from_utf8_lossy(&o.stderr);
if stderr.contains("Unable to find image") {
// Tentar pull e executar novamente
let pull_output = Command::new("docker")
.args(&["pull", "hello-world"])
.output();
if let Ok(pull_o) = pull_output {
if pull_o.status.success() {
// Tentar executar novamente
let run_output = Command::new("docker")
.args(&["run", "--rm", "--name", "docker-test-hello-world-2", "hello-world"])
.output();
if let Ok(run_o) = run_output {
return Ok(run_o.status.success());
}
}
}
}
Ok(false)
}
}
Err(_) => Ok(false),
}
}
/// Testa se usuário tem permissões para executar docker ps
fn test_docker_permissions() -> Result<bool> {
let output = Command::new("docker")
.arg("ps")
.output();
match output {
Ok(o) => Ok(o.status.success()),
Err(_) => Ok(false),
}
}
+108
View File
@@ -0,0 +1,108 @@
use std::collections::HashMap;
use std::sync::Arc;
use tokio::sync::RwLock;
use url::Url;
/// Estratégia de circuit breaker por domínio
pub struct DomainStrategy {
// Armazena taxa de sucesso por domínio (últimas 10 tentativas)
success_rates: Arc<RwLock<HashMap<String, Vec<bool>>>>,
}
impl DomainStrategy {
pub fn new() -> Self {
Self {
success_rates: Arc::new(RwLock::new(HashMap::new())),
}
}
/// Extrai o domínio de uma URL
fn extract_domain(url: &str) -> Option<String> {
if let Ok(parsed) = Url::parse(url) {
if let Some(host) = parsed.host_str() {
return Some(host.to_string());
}
}
None
}
/// Verifica se deve fazer scraping de um domínio
pub async fn should_scrape(&self, url: &str) -> bool {
let domain = match Self::extract_domain(url) {
Some(d) => d,
None => return true, // Se não conseguir extrair domínio, permite scraping
};
let rates = self.success_rates.read().await;
if let Some(history) = rates.get(&domain) {
if history.is_empty() {
return true;
}
let success_count = history.iter().filter(|&&s| s).count();
let success_rate = success_count as f32 / history.len() as f32;
// Se taxa de sucesso < 30%, pula scraping e usa apenas snippet
if success_rate < 0.3 {
log::warn!(
"Domain {} has low success rate ({:.0}%), skipping scrape",
domain,
success_rate * 100.0
);
return false;
}
}
true
}
/// Registra resultado de scraping
pub async fn record_result(&self, url: &str, success: bool) {
let domain = match Self::extract_domain(url) {
Some(d) => d,
None => return,
};
let mut rates = self.success_rates.write().await;
let history = rates.entry(domain).or_insert_with(Vec::new);
history.push(success);
// Mantém apenas últimas 10 tentativas
if history.len() > 10 {
history.remove(0);
}
}
/// Limpa histórico de um domínio (útil para resetar após mudanças)
pub async fn clear_domain(&self, url: &str) {
let domain = match Self::extract_domain(url) {
Some(d) => d,
None => return,
};
let mut rates = self.success_rates.write().await;
rates.remove(&domain);
}
/// Limpa todo o histórico
pub async fn clear_all(&self) {
let mut rates = self.success_rates.write().await;
rates.clear();
}
}
impl Clone for DomainStrategy {
fn clone(&self) -> Self {
Self {
success_rates: Arc::clone(&self.success_rates),
}
}
}
impl Default for DomainStrategy {
fn default() -> Self {
Self::new()
}
}
+7
View File
@@ -7,7 +7,10 @@ use anyhow::{Result, anyhow};
use ndarray::Array2;
use ort::session::{Session, builder::GraphOptimizationLevel};
use ort::value::Value;
<<<<<<< HEAD
=======
use std::io::Read;
>>>>>>> 593efd42e091a845dea82ee6646e027bce1e18c5
use std::path::Path;
use std::sync::{Arc, Mutex, OnceLock};
use tokenizers::Tokenizer;
@@ -448,7 +451,11 @@ pub fn prune_context(
let mut result = Vec::new();
let mut total_tokens = 0;
<<<<<<< HEAD
for (_score, paragraph, original_idx) in scored_paragraphs {
=======
for (score, paragraph, original_idx) in scored_paragraphs {
>>>>>>> 593efd42e091a845dea82ee6646e027bce1e18c5
let paragraph_tokens = paragraph.split_whitespace().count();
if total_tokens + paragraph_tokens > max_tokens {
+73
View File
@@ -0,0 +1,73 @@
use tauri::{AppHandle, Manager, Emitter};
use serde::Serialize;
/// Evento de query de busca iniciada
#[derive(Serialize, Clone)]
pub struct SearchQueryEvent {
pub query: String,
pub source: String, // "brave", "searxng", "duckduckgo"
pub round: Option<u32>,
pub timestamp: u64,
}
/// Evento de query de busca completada
#[derive(Serialize, Clone)]
pub struct SearchQueryCompletedEvent {
pub query: String,
pub source: String,
pub results_count: usize,
pub success: bool,
pub timestamp: u64,
}
/// Evento de mudança de fonte de busca
#[derive(Serialize, Clone)]
pub struct SearchSourceSwitchEvent {
pub from: String,
pub to: String,
pub reason: String, // "timeout", "error", "quota_exceeded", "empty_results"
pub timestamp: u64,
}
/// Evento de URL sendo scraped
#[derive(Serialize, Clone)]
pub struct ScrapingUrlEvent {
pub url: String,
pub title: Option<String>,
pub status: String, // "started", "completed", "failed", "cached"
pub duration_ms: Option<u64>,
pub source: Option<String>, // "static", "nodriver", "playwright"
pub timestamp: u64,
}
/// Helper para emitir eventos de busca
pub fn emit_search_event(app_handle: &AppHandle, event: &SearchQueryEvent) {
// Emitir para todas as janelas
if let Some(window) = app_handle.get_webview_window("main") {
let _ = window.emit("search-query-started", event.clone());
}
}
/// Helper para emitir eventos de busca completada
pub fn emit_search_completed_event(app_handle: &AppHandle, event: &SearchQueryCompletedEvent) {
// Emitir para todas as janelas
if let Some(window) = app_handle.get_webview_window("main") {
let _ = window.emit("search-query-completed", event.clone());
}
}
/// Helper para emitir eventos de mudança de fonte
pub fn emit_source_switch_event(app_handle: &AppHandle, event: &SearchSourceSwitchEvent) {
// Emitir para todas as janelas
if let Some(window) = app_handle.get_webview_window("main") {
let _ = window.emit("search-source-switched", event.clone());
}
}
/// Helper para emitir eventos de scraping
pub fn emit_scraping_event(app_handle: &AppHandle, event: &ScrapingUrlEvent) {
// Emitir para todas as janelas
if let Some(window) = app_handle.get_webview_window("main") {
let _ = window.emit("scraping-url-event", event.clone());
}
}
+448
View File
@@ -0,0 +1,448 @@
use std::path::{Path, PathBuf};
use std::fs;
use std::io::Write;
use serde::{Deserialize, Serialize};
use reqwest::Client;
use sha2::{Sha256, Digest};
use regex::Regex;
use tauri::{AppHandle, WebviewWindow, Emitter};
use futures_util::StreamExt;
use crate::models_dir;
/// Modelo GGUF disponível no Hugging Face
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct HuggingFaceModel {
pub id: String,
pub filename: String,
pub size: u64,
pub quantization: String,
pub sha256: Option<String>,
pub repo_owner: String,
pub repo_name: String,
}
/// Cache de metadados do Hugging Face
#[derive(Debug, Serialize, Deserialize)]
struct HuggingFaceCache {
models: Vec<HuggingFaceModel>,
last_updated: String, // ISO8601
}
const MAX_RETRIES: u32 = 3;
const RATE_LIMIT_BACKOFF: u64 = 60; // segundos
const CACHE_TTL_HOURS: i64 = 1;
/// Extrai quantização do filename usando regex corrigido
fn extract_quantization(filename: &str) -> Option<String> {
let re = Regex::new(r"[._-](Q\d+_?[KM]?_?[MS]?|[FI]P?\d+)[._-]").ok()?;
re.captures(filename)
.and_then(|cap| cap.get(1))
.map(|m| m.as_str().to_string())
}
/// Faz requisição HTTP com retry e rate limiting
async fn fetch_with_retry(url: &str) -> Result<reqwest::Response, String> {
let client = Client::builder()
.timeout(std::time::Duration::from_secs(30))
.build()
.map_err(|e| format!("Failed to create HTTP client: {}", e))?;
for attempt in 0..MAX_RETRIES {
match client.get(url).send().await {
Ok(resp) if resp.status() == 429 => {
// Rate limited - usar Retry-After header
let retry_after = resp
.headers()
.get("Retry-After")
.and_then(|v| v.to_str().ok())
.and_then(|v| v.parse::<u64>().ok())
.unwrap_or(RATE_LIMIT_BACKOFF);
log::warn!("Rate limited, retrying after {} seconds", retry_after);
tokio::time::sleep(std::time::Duration::from_secs(retry_after)).await;
continue;
}
Ok(resp) => return Ok(resp),
Err(e) if attempt < MAX_RETRIES - 1 => {
let backoff = 2u64.pow(attempt);
log::warn!("Request failed (attempt {}), retrying in {}s: {}", attempt + 1, backoff, e);
tokio::time::sleep(std::time::Duration::from_secs(backoff)).await;
}
Err(e) => return Err(format!("Request failed after {} attempts: {}", MAX_RETRIES, e)),
}
}
Err("Max retries exceeded".to_string())
}
/// Valida SHA256 de um arquivo
fn validate_sha256(path: &Path, expected_sha256: &str) -> Result<(), String> {
use std::io::Read;
let mut file = fs::File::open(path)
.map_err(|e| format!("Failed to open file for validation: {}", e))?;
let mut hasher = Sha256::new();
let mut buffer = vec![0u8; 8192];
loop {
let bytes_read = file.read(&mut buffer)
.map_err(|e| format!("Failed to read file: {}", e))?;
if bytes_read == 0 {
break;
}
hasher.update(&buffer[..bytes_read]);
}
let hash = format!("{:x}", hasher.finalize());
if hash.to_lowercase() != expected_sha256.to_lowercase() {
return Err(format!(
"SHA256 mismatch: expected {}, got {}",
expected_sha256, hash
));
}
Ok(())
}
/// Busca SHA256 de um arquivo no Hugging Face
async fn fetch_sha256_from_hf(model_id: &str, filename: &str) -> Result<Option<String>, String> {
// API do Hugging Face para obter SHA256
// Nota: Esta API pode não estar disponível para todos os arquivos
let url = format!("https://huggingface.co/api/models/{}/tree/main", model_id);
let response = fetch_with_retry(&url).await?;
if !response.status().is_success() {
return Ok(None); // SHA256 não disponível
}
let tree: serde_json::Value = response.json().await
.map_err(|e| format!("Failed to parse response: {}", e))?;
// Procurar pelo arquivo no tree
if let Some(files) = tree.get("siblings").and_then(|s| s.as_array()) {
for file in files {
if let Some(path) = file.get("rfilename").and_then(|p| p.as_str()) {
if path == filename {
// Tentar obter SHA256 (pode não estar disponível)
if let Some(sha256) = file.get("sha256").and_then(|s| s.as_str()) {
return Ok(Some(sha256.to_string()));
}
}
}
}
}
Ok(None)
}
/// Baixa um modelo GGUF do Hugging Face com streaming de progresso
pub async fn download_gguf_model(
app_handle: AppHandle,
model_id: String,
filename: String,
window: Option<WebviewWindow>,
) -> Result<PathBuf, String> {
// Extrair repo_owner e repo_name do model_id
let parts: Vec<&str> = model_id.split('/').collect();
if parts.len() != 2 {
return Err("Invalid model_id format. Expected 'owner/repo'".to_string());
}
let repo_owner = parts[0];
let repo_name = parts[1];
// Extrair quantização do filename
let quantization = extract_quantization(&filename)
.unwrap_or_else(|| "unknown".to_string());
// Obter diretório de destino
let model_dir = models_dir::get_model_dir(&app_handle, repo_owner, repo_name, &quantization)?;
let dest_path = model_dir.join(&filename);
// Verificar se arquivo já existe
if dest_path.exists() {
log::info!("Model already exists at {}", dest_path.display());
return Ok(dest_path);
}
// URL de download do Hugging Face
let download_url = format!("https://huggingface.co/{}/resolve/main/{}", model_id, filename);
// Buscar SHA256 se disponível
let sha256 = fetch_sha256_from_hf(&model_id, &filename).await?;
// Criar arquivo temporário para download
let temp_path = dest_path.with_extension("tmp");
// Verificar se há download parcial para resumir
let mut start_pos = 0u64;
if temp_path.exists() {
start_pos = fs::metadata(&temp_path)
.map_err(|e| format!("Failed to get temp file size: {}", e))?
.len();
log::info!("Resuming download from byte {}", start_pos);
}
// Fazer download com streaming
let response = fetch_with_retry(&download_url).await?;
if !response.status().is_success() {
return Err(format!("Download failed with status: {}", response.status()));
}
let total_size = response.content_length().unwrap_or(0);
let mut file = fs::OpenOptions::new()
.create(true)
.append(true)
.open(&temp_path)
.map_err(|e| format!("Failed to open temp file: {}", e))?;
let mut stream = response.bytes_stream();
let mut downloaded: u64 = start_pos;
let mut last_update = std::time::Instant::now();
while let Some(chunk_result) = stream.next().await {
let chunk = chunk_result.map_err(|e| format!("Stream error: {}", e))?;
file.write_all(&chunk)
.map_err(|e| format!("Failed to write chunk: {}", e))?;
downloaded += chunk.len() as u64;
// Emitir progresso a cada segundo
if last_update.elapsed().as_secs() >= 1 {
let percent = if total_size > 0 {
Some(((downloaded as f64 / total_size as f64) * 100.0) as u8)
} else {
None
};
// Criar evento de progresso compatível com o formato existente
let progress_json = serde_json::json!({
"status": "downloading",
"percent": percent,
"downloaded": format_bytes(downloaded),
"total": if total_size > 0 { format_bytes(total_size) } else { "".to_string() },
"speed": null,
"raw": format!("downloaded: {} / {}", downloaded, total_size)
});
if let Some(w) = &window {
let _ = w.emit("download-progress", progress_json.to_string());
}
last_update = std::time::Instant::now();
}
}
// Mover arquivo temporário para destino final
fs::rename(&temp_path, &dest_path)
.map_err(|e| format!("Failed to move temp file: {}", e))?;
// Validar SHA256 se disponível
if let Some(expected_sha256) = &sha256 {
log::info!("Validating SHA256 for {}", filename);
validate_sha256(&dest_path, expected_sha256)?;
log::info!("SHA256 validation passed");
} else {
log::warn!("SHA256 not available for {}, skipping validation", filename);
}
// Emitir sucesso
let success_progress = serde_json::json!({
"status": "success",
"percent": 100,
"downloaded": format_bytes(downloaded),
"total": if total_size > 0 { format_bytes(total_size) } else { "".to_string() },
"speed": null,
"raw": "success"
});
if let Some(w) = &window {
let _ = w.emit("download-progress", success_progress.to_string());
}
Ok(dest_path)
}
/// Lista modelos GGUF disponíveis no Hugging Face
pub async fn list_huggingface_gguf_models(
app_handle: &AppHandle,
query: Option<String>,
quantization: Option<String>,
) -> Result<Vec<HuggingFaceModel>, String> {
// Verificar cache primeiro
let cache_path = models_dir::get_cache_dir(app_handle)?
.join("huggingface_models.json");
// Tentar carregar cache
if let Ok(cache) = load_cache(&cache_path) {
let cache_age = chrono::DateTime::parse_from_rfc3339(&cache.last_updated)
.ok()
.and_then(|dt| {
let now = chrono::Utc::now();
let cache_time = dt.with_timezone(&chrono::Utc);
Some((now - cache_time).num_hours())
});
if let Some(age) = cache_age {
if age < CACHE_TTL_HOURS {
log::info!("Using cached models (age: {} hours)", age);
return Ok(filter_models(cache.models, query, quantization));
}
}
}
// Buscar modelos do Hugging Face
let search_url = if let Some(q) = &query {
format!("https://huggingface.co/api/models?search={}&filter=gguf", q)
} else {
"https://huggingface.co/api/models?filter=gguf".to_string()
};
let response = fetch_with_retry(&search_url).await?;
if !response.status().is_success() {
return Err(format!("API request failed with status: {}", response.status()));
}
let models_json: serde_json::Value = response.json().await
.map_err(|e| format!("Failed to parse response: {}", e))?;
let mut models = Vec::new();
// Processar resultados
if let Some(items) = models_json.as_array() {
for item in items {
if let Some(model_id) = item.get("id").and_then(|i| i.as_str()) {
// Buscar arquivos GGUF no repositório
if let Ok(repo_models) = fetch_repo_gguf_files(model_id).await {
models.extend(repo_models);
}
}
}
}
// Salvar cache
let cache = HuggingFaceCache {
models: models.clone(),
last_updated: chrono::Utc::now().to_rfc3339(),
};
if let Err(e) = save_cache(&cache_path, &cache) {
log::warn!("Failed to save cache: {}", e);
}
Ok(filter_models(models, query, quantization))
}
/// Busca arquivos GGUF em um repositório específico
async fn fetch_repo_gguf_files(model_id: &str) -> Result<Vec<HuggingFaceModel>, String> {
let url = format!("https://huggingface.co/api/models/{}/tree/main", model_id);
let response = fetch_with_retry(&url).await?;
if !response.status().is_success() {
return Ok(Vec::new()); // Repositório não encontrado ou sem acesso
}
let tree: serde_json::Value = response.json().await
.map_err(|e| format!("Failed to parse response: {}", e))?;
let parts: Vec<&str> = model_id.split('/').collect();
let repo_owner = parts.get(0).unwrap_or(&"unknown").to_string();
let repo_name = parts.get(1).unwrap_or(&"unknown").to_string();
let mut models = Vec::new();
if let Some(files) = tree.get("siblings").and_then(|s| s.as_array()) {
for file in files {
if let Some(path) = file.get("rfilename").and_then(|p| p.as_str()) {
if path.ends_with(".gguf") {
let size = file.get("size").and_then(|s| s.as_u64()).unwrap_or(0);
let quantization = extract_quantization(path)
.unwrap_or_else(|| "unknown".to_string());
let sha256 = file.get("sha256").and_then(|s| s.as_str()).map(|s| s.to_string());
models.push(HuggingFaceModel {
id: model_id.to_string(),
filename: path.to_string(),
size,
quantization,
sha256,
repo_owner: repo_owner.clone(),
repo_name: repo_name.clone(),
});
}
}
}
}
Ok(models)
}
/// Filtra modelos por query e quantização
fn filter_models(
models: Vec<HuggingFaceModel>,
query: Option<String>,
quantization: Option<String>,
) -> Vec<HuggingFaceModel> {
models
.into_iter()
.filter(|m| {
if let Some(q) = &query {
m.id.to_lowercase().contains(&q.to_lowercase()) ||
m.filename.to_lowercase().contains(&q.to_lowercase())
} else {
true
}
})
.filter(|m| {
if let Some(quant) = &quantization {
m.quantization.to_lowercase() == quant.to_lowercase()
} else {
true
}
})
.collect()
}
/// Carrega cache de metadados
fn load_cache(path: &Path) -> Result<HuggingFaceCache, String> {
let content = fs::read_to_string(path)
.map_err(|e| format!("Failed to read cache: {}", e))?;
let cache: HuggingFaceCache = serde_json::from_str(&content)
.map_err(|e| format!("Failed to parse cache: {}", e))?;
Ok(cache)
}
/// Salva cache de metadados
fn save_cache(path: &Path, cache: &HuggingFaceCache) -> Result<(), String> {
let content = serde_json::to_string_pretty(cache)
.map_err(|e| format!("Failed to serialize cache: {}", e))?;
fs::write(path, content)
.map_err(|e| format!("Failed to write cache: {}", e))?;
Ok(())
}
/// Formata bytes em formato legível
fn format_bytes(bytes: u64) -> String {
const UNITS: &[&str] = &["B", "KB", "MB", "GB", "TB"];
let mut size = bytes as f64;
let mut unit_index = 0;
while size >= 1024.0 && unit_index < UNITS.len() - 1 {
size /= 1024.0;
unit_index += 1;
}
format!("{:.2} {}", size, UNITS[unit_index])
}
+1814 -278
View File
File diff suppressed because it is too large. Load diff
+108
View File
@@ -0,0 +1,108 @@
use std::path::PathBuf;
use std::sync::{Arc, Mutex};
use serde::{Deserialize, Serialize};
use lru::LruCache;
/// Parâmetros de configuração para llama.cpp
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LlamaParams {
/// Número de camadas para carregar na GPU (0 = CPU apenas)
pub n_gpu_layers: i32,
/// Tamanho do contexto
pub n_ctx: u32,
/// Número de threads (0 = auto)
pub threads: u32,
/// Temperatura para sampling
pub temperature: f32,
/// Top-p para sampling
pub top_p: f32,
/// Top-k para sampling
pub top_k: i32,
}
impl Default for LlamaParams {
fn default() -> Self {
Self {
n_gpu_layers: 0, // CPU por padrão
n_ctx: 4096,
threads: 0, // Auto
temperature: 0.7,
top_p: 0.9,
top_k: 40,
}
}
}
/// Metadados de um modelo GGUF
#[derive(Debug, Clone)]
pub struct ModelMetadata {
pub path: PathBuf,
pub architecture: Option<String>,
pub quantization: Option<String>,
pub parameter_count: Option<u64>,
pub size: u64,
}
/// Estado de um modelo carregado
#[derive(Debug, Clone)]
pub struct LlamaModel {
pub path: PathBuf,
#[cfg(feature = "llama-cpp")]
pub context: Option<()>, // Placeholder - será LlamaContext quando feature estiver ativa
pub metadata: ModelMetadata,
pub last_used: std::time::Instant,
}
/// Estado global do llama.cpp com LRU cache
pub struct LlamaCppState {
/// Cache LRU de modelos carregados (limite de 3 para prevenir memory leaks)
pub models: Arc<Mutex<LruCache<String, LlamaModel>>>,
pub default_params: LlamaParams,
}
impl LlamaCppState {
pub fn new() -> Self {
Self {
models: Arc::new(Mutex::new(LruCache::new(
std::num::NonZeroUsize::new(3).unwrap()
))),
default_params: LlamaParams::default(),
}
}
/// Obtém um modelo do cache ou retorna None
pub fn get_model(&self, model_id: &str) -> Option<LlamaModel> {
let mut cache = self.models.lock().unwrap();
cache.get(model_id).cloned()
}
/// Adiciona um modelo ao cache (LRU remove automaticamente o mais antigo se necessário)
pub fn add_model(&self, model_id: String, model: LlamaModel) {
let mut cache = self.models.lock().unwrap();
cache.put(model_id, model);
}
/// Remove um modelo do cache
pub fn remove_model(&self, model_id: &str) -> Option<LlamaModel> {
let mut cache = self.models.lock().unwrap();
cache.pop(model_id)
}
/// Limpa todos os modelos do cache
pub fn clear(&self) {
let mut cache = self.models.lock().unwrap();
cache.clear();
}
/// Lista todos os modelos no cache
pub fn list_models(&self) -> Vec<String> {
let cache = self.models.lock().unwrap();
cache.iter().map(|(k, _)| k.clone()).collect()
}
}
impl Default for LlamaCppState {
fn default() -> Self {
Self::new()
}
}
+174
View File
@@ -0,0 +1,174 @@
use std::path::Path;
use std::fs;
use walkdir::WalkDir;
use serde::{Deserialize, Serialize};
use regex::Regex;
use tauri::AppHandle;
use crate::models_dir;
/// Modelo GGUF local encontrado no sistema de arquivos
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LocalGgufModel {
pub path: String,
pub name: String,
pub size: u64,
pub quantization: Option<String>,
pub architecture: Option<String>,
pub parameter_count: Option<u64>,
}
/// Extrai quantização do filename usando regex corrigido
fn extract_quantization(filename: &str) -> Option<String> {
let re = Regex::new(r"[._-](Q\d+_?[KM]?_?[MS]?|[FI]P?\d+)[._-]").ok()?;
re.captures(filename)
.and_then(|cap| cap.get(1))
.map(|m| m.as_str().to_string())
}
/// Extrai metadados de um arquivo GGUF usando a crate gguf
/// Retorna None se o parsing falhar (não é crítico, temos fallback do filename)
fn extract_gguf_metadata(path: &Path) -> Option<GgufMetadata> {
use gguf::GGUFFile;
use std::io::Read;
// Ler apenas os primeiros bytes para parsing (mais eficiente)
let mut file = std::fs::File::open(path).ok()?;
let mut buffer = vec![0u8; 1024 * 1024]; // Ler 1MB (suficiente para header)
let bytes_read = file.read(&mut buffer).ok()?;
buffer.truncate(bytes_read);
// Tentar parsear
let _gguf_file = match GGUFFile::read(&buffer) {
Ok(Some(file)) => file,
Ok(None) | Err(_) => return None, // Dados incompletos ou erro
};
// A API do gguf 0.1 pode ser diferente, vamos tentar acessar metadados
// Se a estrutura for diferente, retornamos None e usamos fallback
// Por enquanto, retornamos None para evitar erros de compilação
// TODO: Implementar acesso correto aos metadados quando a estrutura for conhecida
None
// Placeholder para quando soubermos a estrutura exata:
// Some(GgufMetadata {
// architecture: None,
// quantization: None,
// parameter_count: None,
// })
}
#[derive(Debug)]
struct GgufMetadata {
architecture: Option<String>,
quantization: Option<String>,
parameter_count: Option<u64>,
}
/// Lista todos os modelos GGUF locais
pub fn list_local_gguf_models(app_handle: &AppHandle) -> Result<Vec<LocalGgufModel>, String> {
let base_dir = models_dir::get_models_base_dir(app_handle)?;
let mut models = Vec::new();
// Scan recursivo de arquivos .gguf
for entry in WalkDir::new(&base_dir)
.into_iter()
.filter_map(|e| e.ok())
.filter(|e| e.file_type().is_file())
{
let path = entry.path();
// Verificar se é arquivo GGUF
if let Some(ext) = path.extension() {
if ext.to_string_lossy().to_lowercase() != "gguf" {
continue;
}
} else {
// Arquivo sem extensão - verificar pelo tamanho (modelos GGUF são grandes)
let metadata = fs::metadata(path)
.map_err(|e| format!("Failed to read metadata: {}", e))?;
if metadata.len() < 50 * 1024 * 1024 {
// Menos de 50MB, provavelmente não é um modelo GGUF
continue;
}
}
// Obter tamanho do arquivo
let size = fs::metadata(path)
.map_err(|e| format!("Failed to read file size: {}", e))?
.len();
// Extrair nome do arquivo
let filename = path
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("unknown")
.to_string();
// Tentar extrair quantização do filename
let quantization_from_filename = extract_quantization(&filename);
// Tentar extrair metadados do arquivo GGUF (opcional, não crítico)
let (architecture, quantization_from_file, parameter_count) =
match extract_gguf_metadata(path) {
Some(meta) => (
meta.architecture,
meta.quantization,
meta.parameter_count,
),
None => {
// Parsing falhou ou não disponível - usar apenas filename
(None, None, None)
}
};
// Preferir quantização do arquivo, fallback para filename
let quantization = quantization_from_file.or(quantization_from_filename);
// Construir nome amigável (sem extensão)
let name = path
.file_stem()
.and_then(|s| s.to_str())
.unwrap_or(&filename)
.to_string();
models.push(LocalGgufModel {
path: path.to_string_lossy().to_string(),
name,
size,
quantization,
architecture,
parameter_count,
});
}
// Ordenar por nome
models.sort_by(|a, b| a.name.cmp(&b.name));
Ok(models)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_extract_quantization() {
assert_eq!(
extract_quantization("mistral-7b-instruct-v0.2.Q4_K_M.gguf"),
Some("Q4_K_M".to_string())
);
assert_eq!(
extract_quantization("model.Q5_0.gguf"),
Some("Q5_0".to_string())
);
assert_eq!(
extract_quantization("model-F16.gguf"),
Some("F16".to_string())
);
assert_eq!(
extract_quantization("model_FP32.gguf"),
Some("FP32".to_string())
);
}
}
+79
View File
@@ -0,0 +1,79 @@
use std::path::PathBuf;
use tauri::{AppHandle, Manager};
/// Obtém o diretório base para modelos GGUF
/// Estrutura: {app_data_dir}/models/gguf/
pub fn get_models_base_dir(app_handle: &AppHandle) -> Result<PathBuf, String> {
let app_data_dir = app_handle
.path()
.app_data_dir()
.map_err(|e| format!("Failed to get app data dir: {}", e))?;
let models_dir = app_data_dir.join("models").join("gguf");
// Criar diretório se não existir
std::fs::create_dir_all(&models_dir)
.map_err(|e| format!("Failed to create models dir: {}", e))?;
Ok(models_dir)
}
/// Obtém o diretório para um modelo específico
/// Estrutura hierárquica: {base_dir}/{repo_owner}/{repo_name}/{quantization}/
pub fn get_model_dir(
app_handle: &AppHandle,
repo_owner: &str,
repo_name: &str,
quantization: &str,
) -> Result<PathBuf, String> {
let base_dir = get_models_base_dir(app_handle)?;
let model_dir = base_dir
.join(sanitize_path_component(repo_owner))
.join(sanitize_path_component(repo_name))
.join(sanitize_path_component(quantization));
// Criar diretório se não existir
std::fs::create_dir_all(&model_dir)
.map_err(|e| format!("Failed to create model dir: {}", e))?;
Ok(model_dir)
}
/// Obtém o diretório de cache
pub fn get_cache_dir(app_handle: &AppHandle) -> Result<PathBuf, String> {
let app_data_dir = app_handle
.path()
.app_data_dir()
.map_err(|e| format!("Failed to get app data dir: {}", e))?;
let cache_dir = app_data_dir.join("cache");
// Criar diretório se não existir
std::fs::create_dir_all(&cache_dir)
.map_err(|e| format!("Failed to create cache dir: {}", e))?;
Ok(cache_dir)
}
/// Sanitiza um componente de path para evitar caracteres inválidos
fn sanitize_path_component(component: &str) -> String {
component
.chars()
.map(|c| match c {
'/' | '\\' | ':' | '*' | '?' | '"' | '<' | '>' | '|' => '_',
_ => c,
})
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_sanitize_path_component() {
assert_eq!(sanitize_path_component("test/model"), "test_model");
assert_eq!(sanitize_path_component("TheBloke/Mistral-7B"), "TheBloke_Mistral-7B");
assert_eq!(sanitize_path_component("Q4_K_M"), "Q4_K_M");
}
}
+245 -5
View File
@@ -1,10 +1,10 @@
use serde::{Deserialize, Serialize};
/// Mensagem para o Ollama API
#[derive(Debug, Serialize)]
struct OllamaMessage {
role: String,
content: String,
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct OllamaMessage {
pub role: String,
pub content: String,
}
/// Request para chat do Ollama
@@ -70,6 +70,9 @@ impl OllamaClient {
system_prompt: Option<&str>,
user_prompt: &str,
) -> Result<String, String> {
log::info!("[LLM] Headless query model={} system_prompt_present={} user_prompt_len={}", model, system_prompt.is_some(), user_prompt.len());
if let Some(sys) = system_prompt { log::info!("[LLM] System Prompt:\n{}", sys); }
log::info!("[LLM] User Prompt:\n{}", user_prompt);
// Verificar conexão primeiro
self.check_connection().await?;
@@ -96,12 +99,15 @@ impl OllamaClient {
};
let url = format!("{}/api/chat", self.base_url);
log::info!("[LLM_DEBUG] Iniciando request HTTP para o modelo (chat stream)...");
let response = self.client
.post(&url)
.json(&request)
.timeout(std::time::Duration::from_secs(30))
.send()
.await
.map_err(|e| format!("Failed to send request to Ollama: {}", e))?;
log::info!("[LLM] HTTP response received (chat stream). Status: {}", response.status());
if !response.status().is_success() {
return Err(format!("Ollama returned status: {}", response.status()));
@@ -113,9 +119,14 @@ impl OllamaClient {
use futures_util::StreamExt;
let mut logged_first_chunk = false;
while let Some(chunk) = stream.next().await {
let chunk = chunk.map_err(|e| format!("Failed to read chunk: {}", e))?;
let text = String::from_utf8_lossy(&chunk);
if !logged_first_chunk {
log::debug!("[LLM] First chunk received ({} bytes)", text.len());
logged_first_chunk = true;
}
// Processar cada linha (Ollama envia JSON por linha)
for line in text.lines() {
@@ -141,7 +152,7 @@ impl OllamaClient {
}
}
}
log::info!("[LLM] Headless Raw Output ({} chars):\n{}", full_response.len(), full_response.trim());
if full_response.is_empty() {
return Err("Empty response from Ollama".to_string());
}
@@ -252,5 +263,234 @@ impl OllamaClient {
"💬".to_string() // Default
}
}
/// Gera completion usando /api/generate (não streaming)
pub async fn generate_completion(
&self,
model: &str,
prompt: &str,
options: Option<GenerationOptions>,
) -> Result<String, String> {
log::info!("[LLM] generate_completion model={} prompt_len={}", model, prompt.len());
log::info!("[LLM] Prompt:\n{}", prompt);
self.check_connection().await?;
let mut request_body = serde_json::json!({
"model": model,
"prompt": prompt,
"stream": false,
});
// Adicionar options se fornecidas
if let Some(opts) = options.as_ref() {
let mut opts_json = serde_json::Map::new();
if let Some(temp) = opts.temperature {
// serde_json::Number não implementa From<f64>, usar json! macro ou Number::from_f64
if let Some(num) = serde_json::Number::from_f64(temp) {
opts_json.insert("temperature".to_string(), serde_json::Value::Number(num));
} else {
// Fallback: usar como string se não conseguir converter
opts_json.insert("temperature".to_string(), serde_json::Value::String(temp.to_string()));
}
}
if let Some(num_predict) = opts.num_predict {
opts_json.insert("num_predict".to_string(), serde_json::Value::Number(num_predict.into()));
}
if let Some(format) = opts.format.clone() {
opts_json.insert("format".to_string(), serde_json::Value::String(format));
}
if !opts_json.is_empty() {
request_body["options"] = serde_json::Value::Object(opts_json);
}
log::info!("[LLM] Options: temperature={:?} num_predict={:?} format={:?}", opts.temperature, opts.num_predict, opts.format.as_deref());
}
let url = format!("{}/api/generate", self.base_url);
log::info!("[LLM_DEBUG] Iniciando request HTTP para o modelo (generate)...");
let response = self.client
.post(&url)
.json(&request_body)
.send()
.await
.map_err(|e| format!("Failed to send request to Ollama: {}", e))?;
log::info!("[LLM] HTTP response received (generate). Status: {}", response.status());
if !response.status().is_success() {
return Err(format!("Ollama returned status: {}", response.status()));
}
#[derive(serde::Deserialize)]
struct GenerateResponse {
response: String,
}
let body_text = response.text()
.await
.map_err(|e| format!("Failed to read response text: {}", e))?;
log::info!("[LLM] Raw HTTP Body ({} chars)", body_text.len());
let result: GenerateResponse = match serde_json::from_str::<GenerateResponse>(&body_text) {
Ok(val) => val,
Err(e) => {
log::error!("[LLM] Failed to parse JSON body: {}", e);
log::error!("[LLM] Body content: {}", body_text);
return Err(format!("Failed to parse response: {}", e));
}
};
let mut raw = result.response.trim().to_string();
if let Ok(re) = regex::Regex::new(r"(?s)<think>.*?</think>") { raw = re.replace_all(&raw, "").to_string(); }
if let Ok(re) = regex::Regex::new(r"(?i)^\s*(here\s+is\s+the\s+query|sure|okay|final\s+query)\s*[:,-]*\s*") { raw = re.replace(&raw, "").to_string(); }
if let Ok(re) = regex::Regex::new(r"^\s*```[a-zA-Z]*\s*") { raw = re.replace(&raw, "").to_string(); }
if let Ok(re) = regex::Regex::new(r"\s*```\s*$") { raw = re.replace(&raw, "").to_string(); }
raw = raw.trim().to_string();
log::info!("[LLM] Raw Output ({} chars):\n{}", raw.len(), raw);
if let Some(fmt) = options.as_ref().and_then(|o| o.format.as_ref()) {
if fmt.to_lowercase() == "json" {
match serde_json::from_str::<serde_json::Value>(&raw) {
Ok(val) => {
let kind = if val.is_array() { "array" } else if val.is_object() { "object" } else { "other" };
log::info!("[LLM] JSON parse ok ({})", kind);
}
Err(e) => {
log::warn!("[LLM] JSON parse failed: {}", e);
}
}
}
}
Ok(raw)
}
/// Gera chat completion usando /api/chat (não streaming)
pub async fn generate_chat_completion(
&self,
model: &str,
messages: Vec<OllamaMessage>,
options: Option<GenerationOptions>,
) -> Result<String, String> {
let msgs_json = serde_json::to_string(&messages).unwrap_or_else(|_| "[]".to_string());
log::info!("[LLM] generate_chat_completion model={} messages_len={} messages_json={} options_temp={:?} options_num_predict={:?}", model, messages.len(), msgs_json, options.as_ref().and_then(|o| o.temperature), options.as_ref().and_then(|o| o.num_predict));
self.check_connection().await?;
let mut request_body = serde_json::json!({
"model": model,
"messages": messages,
"stream": false,
});
// Adicionar options se fornecidas
if let Some(opts) = options {
let mut opts_json = serde_json::Map::new();
if let Some(temp) = opts.temperature {
// serde_json::Number não implementa From<f64>, usar Number::from_f64
if let Some(num) = serde_json::Number::from_f64(temp) {
opts_json.insert("temperature".to_string(), serde_json::Value::Number(num));
} else {
// Fallback: usar como string se não conseguir converter
opts_json.insert("temperature".to_string(), serde_json::Value::String(temp.to_string()));
}
}
if let Some(num_predict) = opts.num_predict {
opts_json.insert("num_predict".to_string(), serde_json::Value::Number(num_predict.into()));
}
if !opts_json.is_empty() {
request_body["options"] = serde_json::Value::Object(opts_json);
}
}
let url = format!("{}/api/chat", self.base_url);
let response = self.client
.post(&url)
.json(&request_body)
.send()
.await
.map_err(|e| format!("Failed to send request to Ollama: {}", e))?;
if !response.status().is_success() {
return Err(format!("Ollama returned status: {}", response.status()));
}
#[derive(serde::Deserialize)]
struct ChatResponse {
message: OllamaMessageResponse,
}
let result: ChatResponse = response.json()
.await
.map_err(|e| format!("Failed to parse response: {}", e))?;
let raw = result.message.content.trim().to_string();
log::info!("[LLM] Chat Raw Output ({} chars):\n{}", raw.len(), raw);
Ok(raw)
}
/// Obtém informações de um modelo usando /api/show
pub async fn get_model_info(&self, model_name: &str) -> Result<serde_json::Value, String> {
self.check_connection().await?;
let url = format!("{}/api/show", self.base_url);
let response = self.client
.post(&url)
.json(&serde_json::json!({ "name": model_name }))
.send()
.await
.map_err(|e| format!("Failed to connect to Ollama: {}", e))?;
if !response.status().is_success() {
return Err(format!("Ollama returned status: {}", response.status()));
}
let result: serde_json::Value = response.json()
.await
.map_err(|e| format!("Failed to parse response: {}", e))?;
Ok(result)
}
/// Lista modelos disponíveis usando /api/tags
pub async fn get_tags(&self) -> Result<Vec<OllamaModel>, String> {
self.check_connection().await?;
let url = format!("{}/api/tags", self.base_url);
let response = self.client
.get(&url)
.send()
.await
.map_err(|e| format!("Failed to connect to Ollama: {}", e))?;
if !response.status().is_success() {
return Err(format!("Ollama returned status: {}", response.status()));
}
#[derive(serde::Deserialize)]
struct TagsResponse {
models: Vec<OllamaModel>,
}
let result: TagsResponse = response.json()
.await
.map_err(|e| format!("Failed to parse response: {}", e))?;
Ok(result.models)
}
}
/// Opções para geração de texto
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct GenerationOptions {
#[serde(skip_serializing_if = "Option::is_none")]
pub temperature: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub num_predict: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub format: Option<String>,
}
/// Modelo Ollama retornado por /api/tags
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct OllamaModel {
pub name: String,
#[serde(default)]
pub size: Option<u64>,
#[serde(default)]
pub modified_at: Option<String>,
#[serde(default)]
pub digest: Option<String>,
}
+545
View File
@@ -0,0 +1,545 @@
use anyhow::Result;
use std::io::{BufRead, BufReader, Write};
use std::path::PathBuf;
use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio};
use std::sync::{Arc, OnceLock};
use std::time::Duration;
use tokio::sync::Mutex;
use tokio::time::timeout;
// Estrutura para retornar conteúdo scraped do Python
// Evita dependência circular com web_scraper::ScrapedContent
pub struct PythonScrapedContent {
pub title: String,
pub url: String,
pub content: String,
pub markdown: String,
}
// Estrutura para resultados de busca via scraping
#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
pub struct PythonSearchResult {
pub url: String,
pub title: String,
pub content: String,
pub engine: String,
pub score: f64,
}
/// Timeout para operações de scraping (15 segundos - otimizado)
const SCRAPE_TIMEOUT: Duration = Duration::from_secs(15);
/// Timeout inicial mais curto para primeira tentativa (8 segundos)
const SCRAPE_TIMEOUT_INITIAL: Duration = Duration::from_secs(8);
/// Resposta JSON do script Python
#[derive(serde::Deserialize, Debug)]
struct PythonResponse {
#[serde(rename = "type")]
response_type: String,
#[serde(default)]
url: Option<String>,
#[serde(default)]
title: Option<String>,
#[serde(default)]
content: Option<String>,
#[serde(default)]
markdown: Option<String>,
#[serde(default)]
message: Option<String>,
#[serde(default)]
results: Option<Vec<PythonSearchResult>>,
#[serde(default)]
number_of_results: Option<usize>,
}
/// Gerenciador de processo Python persistente para scraping
pub struct PythonScraper {
child: Option<Child>,
stdin: Option<ChildStdin>,
stdout: Option<Arc<std::sync::Mutex<BufReader<ChildStdout>>>>,
script_path: PathBuf,
}
impl PythonScraper {
/// Cria uma nova instância do PythonScraper e spawna o processo Python
pub fn new() -> Result<Self> {
let script_path = Self::resolve_script_path()?;
log::info!("[PythonScraper] Initializing with script: {}", script_path.display());
let mut scraper = Self {
child: None,
stdin: None,
stdout: None,
script_path,
};
scraper.spawn_process()?;
Ok(scraper)
}
/// Resolve o caminho do script Python baseado no engine configurado
fn resolve_script_path() -> Result<PathBuf> {
// Verificar variável de ambiente para escolher engine
// Nodriver é mais rápido e anti-detecção, então é o padrão
let engine = std::env::var("SCRAPER_ENGINE").unwrap_or_else(|_| "nodriver".to_string());
let script_name = if engine == "nodriver" {
"scraper_nodriver.py"
} else {
"scraper.py"
};
Self::resolve_script_path_with_name(script_name)
}
/// Resolve o caminho do script Python com nome específico
fn resolve_script_path_with_name(script_name: &str) -> Result<PathBuf> {
// Estratégia 1: Tentar encontrar em relação ao executável
// Em desenvolvimento: target/debug/app.exe -> subir para raiz -> src-tauri/scripts/scraper.py
if let Ok(exe_path) = std::env::current_exe() {
let mut current_dir = exe_path.parent();
// Subir até 5 níveis para encontrar src-tauri/scripts/scraper.py
// Exemplo: target/debug/app.exe -> target/debug -> target -> root -> src-tauri/scripts
for level in 0..5 {
if let Some(dir) = current_dir {
// Tentar src-tauri/scripts/{script_name} (desenvolvimento)
let src_tauri_scripts = dir.join("src-tauri").join("scripts").join(script_name);
if src_tauri_scripts.exists() {
log::info!("[PythonScraper] Found script at level {}: {}", level, src_tauri_scripts.display());
return Ok(src_tauri_scripts);
}
// Tentar scripts/{script_name} (produção/bundle)
let scripts_path = dir.join("scripts").join(script_name);
if scripts_path.exists() {
log::info!("[PythonScraper] Found script at level {}: {}", level, scripts_path.display());
return Ok(scripts_path);
}
// Subir um nível
current_dir = dir.parent();
} else {
break;
}
}
}
// Estratégia 2: Tentar usando diretório de trabalho atual
if let Ok(cwd) = std::env::current_dir() {
// Se CWD é a raiz do projeto
let cwd_script = cwd.join("src-tauri").join("scripts").join(script_name);
if cwd_script.exists() {
log::info!("[PythonScraper] Found script at (CWD root): {}", cwd_script.display());
return Ok(cwd_script);
}
// Se CWD é src-tauri
let relative_script = cwd.join("scripts").join(script_name);
if relative_script.exists() {
log::info!("[PythonScraper] Found script at (CWD src-tauri): {}", relative_script.display());
return Ok(relative_script);
}
// Tentar subir do CWD também
let mut cwd_parent = cwd.parent();
for _ in 0..3 {
if let Some(parent) = cwd_parent {
let parent_script = parent.join("src-tauri").join("scripts").join(script_name);
if parent_script.exists() {
log::info!("[PythonScraper] Found script at (CWD parent): {}", parent_script.display());
return Ok(parent_script);
}
cwd_parent = parent.parent();
} else {
break;
}
}
}
// Estratégia 3: Tentar variável de ambiente CARGO_MANIFEST_DIR (runtime)
if let Ok(manifest_dir) = std::env::var("CARGO_MANIFEST_DIR") {
let manifest_path = PathBuf::from(&manifest_dir);
let script = manifest_path.join("scripts").join(script_name);
if script.exists() {
log::info!("[PythonScraper] Found script at (CARGO_MANIFEST_DIR): {}", script.display());
return Ok(script);
}
}
// Estratégia 4: Tentar caminho absoluto construído a partir do executável
// Se executável está em target/debug/app.exe, precisamos subir 3 níveis
if let Ok(exe_path) = std::env::current_exe() {
if let Some(exe_dir) = exe_path.parent() {
// Construir caminho absoluto: subir até encontrar src-tauri
let mut search_dir = exe_dir.to_path_buf();
for _ in 0..6 {
let test_path = search_dir.join("src-tauri").join("scripts").join(script_name);
if test_path.exists() {
log::info!("[PythonScraper] Found script at (absolute from exe): {}", test_path.display());
return Ok(test_path);
}
if let Some(parent) = search_dir.parent() {
search_dir = parent.to_path_buf();
} else {
break;
}
}
}
}
// Estratégia 5: Tentar caminho relativo simples (se executado da raiz)
let simple_path = PathBuf::from("src-tauri/scripts").join(script_name);
if simple_path.exists() {
log::info!("[PythonScraper] Found script at (simple): {}", simple_path.display());
return Ok(simple_path);
}
// Log detalhado para debug
let exe_info = std::env::current_exe()
.map(|p| format!("{}", p.display()))
.unwrap_or_else(|_| "unknown".to_string());
let cwd_info = std::env::current_dir()
.map(|p| format!("{}", p.display()))
.unwrap_or_else(|_| "unknown".to_string());
let manifest_info = std::env::var("CARGO_MANIFEST_DIR")
.unwrap_or_else(|_| "not set".to_string());
Err(anyhow::anyhow!(
"Could not find {} script.\n\
Executable: {}\n\
Current dir: {}\n\
CARGO_MANIFEST_DIR: {}\n\
Expected location: src-tauri/scripts/{}\n\
Searched: relative to executable (up to 5 levels), current working directory, \
CARGO_MANIFEST_DIR, file!() macro, and simple path.",
script_name, exe_info, cwd_info, manifest_info, script_name
))
}
/// Spawna o processo Python
fn spawn_process(&mut self) -> Result<()> {
// Limpar processo anterior se existir
self.cleanup();
log::info!("[PythonScraper] Spawning Python process: {}", self.script_path.display());
let mut cmd = Command::new("python");
cmd.arg(&self.script_path);
cmd.stdin(Stdio::piped());
cmd.stdout(Stdio::piped());
cmd.stderr(Stdio::piped());
#[cfg(target_os = "windows")]
{
use std::os::windows::process::CommandExt;
const CREATE_NO_WINDOW: u32 = 0x08000000;
cmd.creation_flags(CREATE_NO_WINDOW);
}
let mut child = cmd.spawn()
.map_err(|e| anyhow::anyhow!("Failed to spawn Python process: {}", e))?;
let stdin = child.stdin.take()
.ok_or_else(|| anyhow::anyhow!("Failed to capture stdin"))?;
let stdout = child.stdout.take()
.ok_or_else(|| anyhow::anyhow!("Failed to capture stdout"))?;
let stdout_reader = Arc::new(std::sync::Mutex::new(BufReader::new(stdout)));
self.child = Some(child);
self.stdin = Some(stdin);
self.stdout = Some(stdout_reader);
log::info!("[PythonScraper] Python process spawned successfully");
Ok(())
}
/// Garante que o processo está vivo, reiniciando se necessário
fn ensure_alive(&mut self) -> Result<()> {
if let Some(ref mut child) = self.child {
// Verificar se processo ainda está rodando (não bloqueia)
if let Ok(Some(_)) = child.try_wait() {
// Processo morreu
log::warn!("[PythonScraper] Process died, restarting...");
return self.restart();
}
} else {
// Processo nunca foi criado
return self.restart();
}
Ok(())
}
/// Reinicia o processo Python
pub fn restart(&mut self) -> Result<()> {
log::info!("[PythonScraper] Restarting Python process...");
self.spawn_process()
}
/// Limpa recursos do processo anterior
fn cleanup(&mut self) {
if let Some(mut child) = self.child.take() {
// Tentar terminar graciosamente
let _ = child.kill();
let _ = child.wait();
}
self.stdin = None;
self.stdout = None;
}
/// Realiza scraping de uma URL usando o processo Python
pub async fn scrape_url(&mut self, url: &str) -> Result<PythonScrapedContent> {
// Garantir que processo está vivo
self.ensure_alive()?;
let stdin = self.stdin.as_mut()
.ok_or_else(|| anyhow::anyhow!("Stdin not available"))?;
let stdout = self.stdout.as_ref()
.ok_or_else(|| anyhow::anyhow!("Stdout not available"))?
.clone();
// Preparar comando JSON
let command = serde_json::json!({
"command": "scrape",
"url": url,
"visible": false
});
let command_json = serde_json::to_string(&command)
.map_err(|e| anyhow::anyhow!("Failed to serialize command: {}", e))?;
log::debug!("[PythonScraper] Sending command: {}", command_json);
// Enviar comando para stdin (síncrono, mas rápido)
stdin.write_all(command_json.as_bytes())
.map_err(|e| anyhow::anyhow!("Failed to write to stdin: {}", e))?;
stdin.write_all(b"\n")
.map_err(|e| anyhow::anyhow!("Failed to write newline to stdin: {}", e))?;
stdin.flush()
.map_err(|e| anyhow::anyhow!("Failed to flush stdin: {}", e))?;
// Ler resposta do stdout com timeout progressivo (primeira tentativa mais curta)
let stdout_arc = stdout.clone();
let read_future = tokio::task::spawn_blocking(move || {
let mut stdout_guard = stdout_arc.lock()
.map_err(|e| anyhow::anyhow!("Failed to lock stdout: {}", e))?;
let mut line = String::new();
stdout_guard.read_line(&mut line)
.map_err(|e| anyhow::anyhow!("Failed to read from stdout: {}", e))?;
if line.is_empty() {
return Err(anyhow::anyhow!("Empty response from Python script"));
}
Ok(line.trim().to_string())
});
// Tentativa 1: Timeout inicial mais curto (8s) para resposta rápida
let response_line = match timeout(SCRAPE_TIMEOUT_INITIAL, read_future).await {
Ok(Ok(Ok(line))) => line,
Ok(Ok(Err(e))) => {
log::error!("[PythonScraper] Error reading response: {}", e);
return Err(e);
}
Ok(Err(e)) => {
log::error!("[PythonScraper] Task error: {}", e);
return Err(anyhow::anyhow!("Task error: {}", e));
}
Err(_) => {
// Timeout inicial, tentar novamente com timeout maior
log::debug!("[PythonScraper] Initial timeout ({}s), retrying with longer timeout...", SCRAPE_TIMEOUT_INITIAL.as_secs());
// Segunda tentativa com timeout maior
let stdout_arc2 = stdout.clone();
let read_future2 = tokio::task::spawn_blocking(move || {
let mut stdout_guard = stdout_arc2.lock()
.map_err(|e| anyhow::anyhow!("Failed to lock stdout: {}", e))?;
let mut line = String::new();
stdout_guard.read_line(&mut line)
.map_err(|e| anyhow::anyhow!("Failed to read from stdout: {}", e))?;
if line.is_empty() {
return Err(anyhow::anyhow!("Empty response from Python script"));
}
Ok(line.trim().to_string())
});
match timeout(SCRAPE_TIMEOUT, read_future2).await {
Ok(Ok(Ok(line))) => line,
Ok(Ok(Err(e))) => {
log::error!("[PythonScraper] Error reading response (retry): {}", e);
return Err(e);
}
Ok(Err(e)) => {
log::error!("[PythonScraper] Task error (retry): {}", e);
return Err(anyhow::anyhow!("Task error: {}", e));
}
Err(_) => {
log::warn!("[PythonScraper] Timeout after retry ({}s), restarting process...", SCRAPE_TIMEOUT.as_secs());
self.restart()?;
return Err(anyhow::anyhow!("Timeout waiting for Python script response"));
}
}
}
};
log::debug!("[PythonScraper] Received response: {}", response_line);
// Deserializar resposta JSON
let response: PythonResponse = serde_json::from_str(&response_line)
.map_err(|e| anyhow::anyhow!("Failed to parse JSON response: {} (response: {})", e, response_line))?;
// Processar resposta
match response.response_type.as_str() {
"success" => {
let title = response.title.unwrap_or_else(|| "Sem título".to_string());
let url = response.url.unwrap_or_else(|| url.to_string());
let content = response.content.unwrap_or_else(|| String::new());
let markdown = response.markdown.unwrap_or_else(|| content.clone());
Ok(PythonScrapedContent {
title,
url,
content,
markdown,
})
}
"error" => {
let error_msg = response.message.unwrap_or_else(|| "Unknown error".to_string());
Err(anyhow::anyhow!("Python scraper error: {}", error_msg))
}
_ => {
Err(anyhow::anyhow!("Unknown response type: {}", response.response_type))
}
}
}
/// Realiza busca no DuckDuckGo via Python
pub async fn search_duckduckgo(
&mut self,
keywords: &str,
region: &str,
safesearch: &str,
max_results: usize
) -> Result<Vec<PythonSearchResult>> {
self.ensure_alive()?;
let stdin = self.stdin.as_mut()
.ok_or_else(|| anyhow::anyhow!("Stdin not available"))?;
let stdout = self.stdout.as_ref()
.ok_or_else(|| anyhow::anyhow!("Stdout not available"))?
.clone();
let command = serde_json::json!({
"command": "search",
"keywords": keywords,
"region": region,
"safesearch": safesearch,
"max_results": max_results
});
let command_json = serde_json::to_string(&command)
.map_err(|e| anyhow::anyhow!("Failed to serialize command: {}", e))?;
log::debug!("[PythonScraper] Sending DDG search command: {}", command_json);
stdin.write_all(command_json.as_bytes())
.map_err(|e| anyhow::anyhow!("Failed to write to stdin: {}", e))?;
stdin.write_all(b"\n")
.map_err(|e| anyhow::anyhow!("Failed to write newline to stdin: {}", e))?;
stdin.flush()
.map_err(|e| anyhow::anyhow!("Failed to flush stdin: {}", e))?;
let stdout_arc = stdout.clone();
let read_future = tokio::task::spawn_blocking(move || {
let mut stdout_guard = stdout_arc.lock()
.map_err(|e| anyhow::anyhow!("Failed to lock stdout: {}", e))?;
let mut line = String::new();
stdout_guard.read_line(&mut line)
.map_err(|e| anyhow::anyhow!("Failed to read from stdout: {}", e))?;
if line.is_empty() {
return Err(anyhow::anyhow!("Empty response from Python script"));
}
Ok(line.trim().to_string())
});
// Para busca DuckDuckGo, usar timeout padrão (não precisa de progressivo)
let response_line = match timeout(SCRAPE_TIMEOUT, read_future).await {
Ok(Ok(Ok(line))) => line,
Ok(Ok(Err(e))) => {
log::error!("[PythonScraper] Error reading response: {}", e);
return Err(e);
}
Ok(Err(e)) => {
log::error!("[PythonScraper] Task error: {}", e);
return Err(anyhow::anyhow!("Task error: {}", e));
}
Err(_) => {
log::warn!("[PythonScraper] Timeout reading response, restarting process...");
self.restart()?;
return Err(anyhow::anyhow!("Timeout waiting for Python script response"));
}
};
log::debug!("[PythonScraper] Received search response: {}", response_line);
let response: PythonResponse = serde_json::from_str(&response_line)
.map_err(|e| anyhow::anyhow!("Failed to parse JSON response: {}", e))?;
match response.response_type.as_str() {
"success" => Ok(response.results.unwrap_or_default()),
"error" => {
let error_msg = response.message.unwrap_or_else(|| "Unknown error".to_string());
Err(anyhow::anyhow!("Python scraper error: {}", error_msg))
}
_ => Err(anyhow::anyhow!("Unknown response type: {}", response.response_type))
}
}
}
impl Drop for PythonScraper {
fn drop(&mut self) {
self.cleanup();
}
}
/// Instância global thread-safe do PythonScraper (lazy initialization)
static LAZY_PYTHON_SCRAPER: OnceLock<Arc<Mutex<PythonScraper>>> = OnceLock::new();
/// Obtém ou cria a instância global do PythonScraper
pub fn get_or_create_python_scraper() -> Result<Arc<Mutex<PythonScraper>>> {
if let Some(scraper) = LAZY_PYTHON_SCRAPER.get() {
return Ok(scraper.clone());
}
// Inicializar se ainda não foi inicializado
let scraper = PythonScraper::new()?;
let scraper_arc = Arc::new(Mutex::new(scraper));
// Tentar inserir (pode falhar se outra thread inseriu primeiro)
let result = LAZY_PYTHON_SCRAPER.set(scraper_arc.clone());
if result.is_err() {
// Outra thread já inicializou, usar a instância existente
return Ok(LAZY_PYTHON_SCRAPER.get().unwrap().clone());
}
Ok(scraper_arc)
}
/// Limpa a instância do PythonScraper (para liberar recursos quando não em uso)
pub async fn clear_python_scraper() {
if let Some(scraper_arc) = LAZY_PYTHON_SCRAPER.get() {
let mut scraper = scraper_arc.lock().await;
scraper.cleanup();
}
}
+4 -2
View File
@@ -1,7 +1,6 @@
use crate::scheduler::SchedulerState;
use crate::task_executor::execute_task;
use tokio_cron_scheduler::{Job, JobScheduler};
use std::sync::Arc;
use tauri::AppHandle;
use chrono::Utc;
@@ -85,6 +84,8 @@ pub async fn reload_scheduled_tasks(
return;
}
<<<<<<< HEAD
=======
// Obter browser - usando lazy initialization global
let browser_arc = {
use crate::web_scraper::get_or_create_browser;
@@ -97,8 +98,9 @@ pub async fn reload_scheduled_tasks(
}
};
>>>>>>> 593efd42e091a845dea82ee6646e027bce1e18c5
// Executar task
match execute_task(&task, app_handle.clone(), browser_arc, ollama_url).await {
match execute_task(&task, app_handle.clone(), ollama_url).await {
Ok(_) => {
// Atualizar last_run
let mut sched = scheduler.lock().await;
+115
View File
@@ -0,0 +1,115 @@
use serde::{Serialize, Deserialize};
use std::fs::OpenOptions;
use std::io::Write;
use std::sync::{Arc, Mutex};
use std::path::PathBuf;
use chrono::prelude::*;
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub enum LogLevel {
INFO,
WARNING,
ERROR,
DEBUG,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ScraperLogEntry {
pub timestamp: String,
pub level: LogLevel,
pub action: String,
pub context: Option<String>,
pub data: Option<serde_json::Value>,
pub url: Option<String>,
pub duration_ms: Option<u64>,
pub error: Option<String>,
}
pub struct ScraperLogger {
log_file_path: PathBuf,
file_mutex: Arc<Mutex<()>>, // Simple mutex to coordinate file writes
}
impl ScraperLogger {
pub fn new(app_data_dir: PathBuf) -> Self {
let log_dir = app_data_dir.join("logs");
std::fs::create_dir_all(&log_dir).unwrap_or_else(|e| {
eprintln!("Failed to create log directory: {}", e);
});
let date_str = Local::now().format("%Y-%m-%d").to_string();
let log_file_path = log_dir.join(format!("scraper_{}.jsonl", date_str));
ScraperLogger {
log_file_path,
file_mutex: Arc::new(Mutex::new(())),
}
}
fn write_entry(&self, entry: &ScraperLogEntry) {
let _guard = self.file_mutex.lock().unwrap();
let json_line = match serde_json::to_string(entry) {
Ok(json) => json,
Err(e) => {
eprintln!("Failed to serialize log entry: {}", e);
return;
}
};
let mut file = match OpenOptions::new()
.create(true)
.append(true)
.open(&self.log_file_path)
{
Ok(f) => f,
Err(e) => {
eprintln!("Failed to open log file: {}", e);
return;
}
};
if let Err(e) = writeln!(file, "{}", json_line) {
eprintln!("Failed to write to log file: {}", e);
}
}
pub fn log(&self, level: LogLevel, action: &str, context: Option<&str>, data: Option<serde_json::Value>, url: Option<&str>, duration_ms: Option<u64>, error: Option<&str>) {
let entry = ScraperLogEntry {
timestamp: Local::now().to_rfc3339(),
level,
action: action.to_string(),
context: context.map(|s| s.to_string()),
data,
url: url.map(|s| s.to_string()),
duration_ms,
error: error.map(|s| s.to_string()),
};
self.write_entry(&entry);
}
pub fn info(&self, action: &str, context: Option<&str>, data: Option<serde_json::Value>) {
self.log(LogLevel::INFO, action, context, data, None, None, None);
}
pub fn warn(&self, action: &str, context: Option<&str>, error: Option<&str>) {
self.log(LogLevel::WARNING, action, context, None, None, None, error);
}
pub fn error(&self, action: &str, context: Option<&str>, error: Option<&str>, data: Option<serde_json::Value>) {
self.log(LogLevel::ERROR, action, context, data, None, None, error);
}
pub fn log_access(&self, url: &str, action: &str) {
self.log(LogLevel::INFO, action, Some("Access"), None, Some(url), None, None);
}
pub fn log_scraping_result(&self, url: &str, success: bool, duration_ms: u64, content_len: usize, error: Option<&str>) {
let level = if success { LogLevel::INFO } else { LogLevel::ERROR };
let data = serde_json::json!({
"content_length": content_len,
"success": success
});
self.log(level, "ScrapingResult", Some("Scraper"), Some(data), Some(url), Some(duration_ms), error);
}
}
+32
View File
@@ -0,0 +1,32 @@
use std::sync::{Arc, atomic::{AtomicBool, Ordering}};
use std::path::PathBuf;
use crate::scraper_logger::ScraperLogger;
#[derive(Clone)]
pub struct ScraperState {
pub is_cancelled: Arc<AtomicBool>,
pub logger: Arc<ScraperLogger>,
}
impl ScraperState {
pub fn new(app_data_dir: PathBuf) -> Self {
ScraperState {
is_cancelled: Arc::new(AtomicBool::new(false)),
logger: Arc::new(ScraperLogger::new(app_data_dir)),
}
}
pub fn cancel(&self) {
self.is_cancelled.store(true, Ordering::SeqCst);
self.logger.info("ScraperCancelled", Some("System"), None);
}
pub fn reset(&self) {
self.is_cancelled.store(false, Ordering::SeqCst);
self.logger.info("ScraperReset", Some("System"), None);
}
pub fn check_cancelled(&self) -> bool {
self.is_cancelled.load(Ordering::SeqCst)
}
}
+117
View File
@@ -0,0 +1,117 @@
use serde::{Deserialize, Serialize};
use tauri::{AppHandle, Manager, command};
use chrono::Utc;
use uuid::Uuid;
use crate::search_config::{load_search_config, SearchConfig};
use crate::python_scraper::get_or_create_python_scraper;
use crate::db::{Database, SearchLog};
use crate::web_scraper::SearchResultMetadata;
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct SearchResult {
pub url: String,
pub title: String,
pub content: String,
pub engine: String,
pub score: f64,
}
// Helper to convert SearchResult to SearchResultMetadata
pub fn search_results_to_metadata(results: &[SearchResult]) -> Vec<SearchResultMetadata> {
results.iter().map(|result| {
SearchResultMetadata {
title: result.title.clone(),
url: result.url.clone(),
snippet: result.content.clone(),
}
}).collect()
}
// Helper to extract URLs
pub fn search_results_to_urls(results: &[SearchResult]) -> Vec<String> {
results.iter().map(|result| result.url.clone()).collect()
}
#[command]
pub async fn search_command(
app_handle: AppHandle,
query: String,
chat_id: Option<String>,
user_id: Option<String>,
) -> Result<Vec<SearchResult>, String> {
perform_search(&app_handle, query, chat_id, user_id).await
}
/// Public function to perform search (used by web_scraper)
pub async fn perform_search(
app_handle: &AppHandle,
query: String,
chat_id: Option<String>,
user_id: Option<String>,
) -> Result<Vec<SearchResult>, String> {
let config = load_search_config(app_handle)?;
if !config.enabled {
return Err("Search is disabled".to_string());
}
let python_scraper = get_or_create_python_scraper().map_err(|e| e.to_string())?;
let mut scraper = python_scraper.lock().await;
// Execute search via Python (DuckDuckGo)
let python_results = scraper.search_duckduckgo(
&query,
&config.region,
&config.safe_search,
config.max_results
).await.map_err(|e| e.to_string())?;
let results: Vec<SearchResult> = python_results.into_iter().map(|r| SearchResult {
title: r.title,
url: r.url,
content: r.content,
engine: r.engine,
score: r.score,
}).collect();
// Log to DB
if let Ok(db) = Database::new(app_handle) {
let log = SearchLog {
id: Uuid::new_v4().to_string(),
chat_id,
user_id,
timestamp: Utc::now(),
searxng_instance: Some("duckduckgo-direct".to_string()),
query: if config.log_search_content { query } else { "[REDACTED]".to_string() },
engines: Some(vec!["duckduckgo".to_string()]),
filters: Some(config.safe_search),
response_time_ms: Some(0), // TODO: measure time
total_results: Some(results.len() as i64),
results_per_engine: None,
error: None,
};
if let Err(e) = db.log_search(&log) {
log::error!("Failed to log search: {}", e);
}
}
Ok(results)
}
#[command]
pub fn cleanup_search_logs_command(app_handle: AppHandle, days: i64) -> Result<usize, String> {
let db = Database::new(&app_handle).map_err(|e| e.to_string())?;
db.cleanup_search_logs(days).map_err(|e| e.to_string())
}
#[command]
pub fn get_search_logs_command(
app_handle: AppHandle,
chat_id: Option<String>,
limit: usize,
offset: usize
) -> Result<Vec<SearchLog>, String> {
let db = Database::new(&app_handle).map_err(|e| e.to_string())?;
db.get_search_logs(chat_id.as_deref(), limit, offset).map_err(|e| e.to_string())
}
+88
View File
@@ -0,0 +1,88 @@
use anyhow::Result;
use crate::web_scraper::SearchResultMetadata;
use crate::searxng_search::searxng_search;
use crate::searxng_manager::check_searxng_status;
use crate::web_scraper::SearchConfig;
use crate::event_emitter::{SearchQueryEvent, SearchQueryCompletedEvent, emit_search_event, emit_search_completed_event};
use tauri::AppHandle;
use std::time::{SystemTime, UNIX_EPOCH, Duration};
use tokio::time::timeout;
/// Executa busca apenas com SearXNG (se habilitado)
/// Nodriver é usado para scraping das URLs encontradas (fonte primária)
pub async fn search_with_waterfall(
query: &str,
limit: usize,
config: &SearchConfig,
app_handle: Option<&AppHandle>,
) -> Result<Vec<SearchResultMetadata>> {
// Apenas SearXNG (se habilitado e disponível)
if !config.searxng.enabled {
return Err(anyhow::anyhow!("SearXNG não está habilitado. Configure SearXNG nas configurações."));
}
// Verificar status do SearXNG com timeout reduzido (2s)
match timeout(Duration::from_secs(2), check_searxng_status(&config.searxng.url)).await {
Ok(Ok(true)) => {
log::info!("[SearchOrchestrator] Using SearXNG for query: {}", query);
// Emitir evento de query iniciada
if let Some(handle) = app_handle {
let timestamp = SystemTime::now().duration_since(UNIX_EPOCH)
.unwrap_or_default().as_secs() * 1000;
emit_search_event(handle, &SearchQueryEvent {
query: query.to_string(),
source: "searxng".to_string(),
round: None,
timestamp,
});
}
// Timeout reduzido para busca SearXNG (6s total)
match timeout(Duration::from_secs(6), searxng_search(query, limit, &config.searxng.url)).await {
Ok(Ok(results)) if !results.is_empty() => {
log::info!("[SearchOrchestrator] SearXNG returned {} results", results.len());
// Emitir evento de query completada
if let Some(handle) = app_handle {
let timestamp = SystemTime::now().duration_since(UNIX_EPOCH)
.unwrap_or_default().as_secs() * 1000;
emit_search_completed_event(handle, &SearchQueryCompletedEvent {
query: query.to_string(),
source: "searxng".to_string(),
results_count: results.len(),
success: true,
timestamp,
});
}
Ok(results)
}
Ok(Ok(_)) => {
log::warn!("[SearchOrchestrator] SearXNG returned empty results");
Err(anyhow::anyhow!("SearXNG retornou resultados vazios"))
}
Ok(Err(e)) => {
log::error!("[SearchOrchestrator] SearXNG search failed: {}", e);
Err(anyhow::anyhow!("SearXNG search failed: {}", e))
}
Err(_) => {
log::error!("[SearchOrchestrator] SearXNG search timeout (6s)");
Err(anyhow::anyhow!("SearXNG search timeout"))
}
}
}
Ok(Ok(false)) => {
log::error!("[SearchOrchestrator] SearXNG is not available at {}", config.searxng.url);
Err(anyhow::anyhow!("SearXNG não está disponível. Verifique se está rodando em {}", config.searxng.url))
}
Ok(Err(e)) => {
log::error!("[SearchOrchestrator] Failed to check SearXNG status: {}", e);
Err(anyhow::anyhow!("Falha ao verificar status do SearXNG: {}", e))
}
Err(_) => {
log::error!("[SearchOrchestrator] Timeout checking SearXNG status");
Err(anyhow::anyhow!("Timeout ao verificar status do SearXNG"))
}
}
}
+284
View File
@@ -0,0 +1,284 @@
use anyhow::Result;
use std::process::Command;
use std::time::Duration;
use tokio::time::timeout;
use reqwest::Client;
use tauri::{AppHandle, Manager, Emitter};
/// Verifica se Docker está instalado no sistema
pub fn check_docker_available() -> Result<bool> {
let output = Command::new("docker")
.arg("--version")
.output();
match output {
Ok(o) => Ok(o.status.success()),
Err(_) => Ok(false),
}
}
/// Verifica se Docker está rodando
pub fn check_docker_running() -> Result<bool> {
let output = Command::new("docker")
.arg("info")
.output();
match output {
Ok(o) => Ok(o.status.success()),
Err(_) => Ok(false),
}
}
/// Instala SearXNG via Docker Compose com progresso
pub async fn install_searxng_with_progress(
app_handle: Option<&AppHandle>,
url: &str,
port: u16,
) -> Result<()> {
log::info!("Installing SearXNG at {}:{}", url, port);
// Verificar requisitos antes de instalar
if let Some(handle) = app_handle {
emit_searxng_progress(handle, "Verificando requisitos...", None);
}
// Verificar se Docker está disponível
match check_docker_available() {
Ok(true) => {
if let Some(handle) = app_handle {
emit_searxng_progress(handle, "Docker encontrado", None);
}
}
Ok(false) => {
return Err(anyhow::anyhow!("Docker não está instalado. Por favor, instale o Docker primeiro."));
}
Err(e) => {
return Err(anyhow::anyhow!("Erro ao verificar Docker: {}", e));
}
}
// Verificar se Docker está rodando
match check_docker_running() {
Ok(true) => {
if let Some(handle) = app_handle {
emit_searxng_progress(handle, "Docker está rodando", None);
}
}
Ok(false) => {
return Err(anyhow::anyhow!("Docker não está rodando. Por favor, inicie o Docker primeiro."));
}
Err(e) => {
return Err(anyhow::anyhow!("Erro ao verificar se Docker está rodando: {}", e));
}
}
// Tentar usar docker-compose.yml do projeto se existir
let project_compose = std::env::current_dir()
.ok()
.and_then(|cwd| {
let path = cwd.join("docker-compose.searxng.yml");
if path.exists() {
Some(path)
} else {
None
}
});
let compose_path = if let Some(path) = project_compose {
if let Some(handle) = app_handle {
emit_searxng_progress(handle, "Usando docker-compose do projeto", None);
}
path
} else {
// Criar docker-compose.yml temporário
if let Some(handle) = app_handle {
emit_searxng_progress(handle, "Criando configuração docker-compose...", None);
}
let compose_content = format!(
r#"version: "3.7"
services:
searxng:
image: searxng/searxng:latest
container_name: ollaHub_searxng
ports:
- "{}:8080"
volumes:
- searxng_data:/etc/searxng:rw
environment:
- SEARXNG_BASE_URL={}/
restart: unless-stopped
networks:
- searxng_network
redis:
image: redis:alpine
container_name: ollaHub_searxng_redis
command: redis-server --save "" --appendonly "no"
networks:
- searxng_network
tmpfs:
- /var/lib/redis
volumes:
searxng_data:
networks:
searxng_network:
driver: bridge
"#,
port, url
);
// Salvar docker-compose.yml temporário
let temp_path = std::env::temp_dir().join("ollahub_searxng_docker-compose.yml");
std::fs::write(&temp_path, compose_content)?;
temp_path
};
// Executar docker-compose up -d com captura de saída
if let Some(handle) = app_handle {
emit_searxng_progress(handle, "Baixando imagens Docker...", None);
}
let output = Command::new("docker")
.arg("compose")
.arg("-f")
.arg(compose_path.to_str().unwrap())
.arg("pull")
.output();
if let Ok(o) = output {
if !o.status.success() {
let error = String::from_utf8_lossy(&o.stderr);
if let Some(handle) = app_handle {
emit_searxng_progress(handle, &format!("Aviso ao baixar imagens: {}", error), None);
}
} else if let Some(handle) = app_handle {
emit_searxng_progress(handle, "Imagens baixadas com sucesso", None);
}
}
if let Some(handle) = app_handle {
emit_searxng_progress(handle, "Criando containers...", None);
}
let output = Command::new("docker")
.arg("compose")
.arg("-f")
.arg(compose_path.to_str().unwrap())
.arg("up")
.arg("-d")
.output()?;
if !output.status.success() {
let error = String::from_utf8_lossy(&output.stderr);
if let Some(handle) = app_handle {
emit_searxng_progress(handle, &format!("Erro: {}", error), Some("error"));
}
return Err(anyhow::anyhow!("Docker compose failed: {}", error));
}
if let Some(handle) = app_handle {
emit_searxng_progress(handle, "SearXNG instalado com sucesso!", Some("complete"));
}
log::info!("SearXNG installation completed");
Ok(())
}
/// Instala SearXNG via Docker Compose (versão sem progresso para compatibilidade)
pub async fn install_searxng(url: &str, port: u16) -> Result<()> {
install_searxng_with_progress(None, url, port).await
}
/// Emite evento de progresso da instalação SearXNG
fn emit_searxng_progress(app_handle: &AppHandle, message: &str, status: Option<&str>) {
#[derive(serde::Serialize, Clone)]
struct SearxngProgressEvent {
message: String,
status: Option<String>,
}
let event = SearxngProgressEvent {
message: message.to_string(),
status: status.map(|s| s.to_string()),
};
if let Some(window) = app_handle.get_webview_window("main") {
let _ = window.emit("searxng-install-progress", event);
}
}
/// Inicia o container SearXNG
pub async fn start_searxng() -> Result<()> {
log::info!("Starting SearXNG container");
let output = Command::new("docker")
.arg("start")
.arg("ollaHub_searxng")
.output()?;
if !output.status.success() {
// Se não existe, tentar criar novamente
let error = String::from_utf8_lossy(&output.stderr);
if error.contains("No such container") {
return Err(anyhow::anyhow!("Container não encontrado. Execute a instalação primeiro."));
}
return Err(anyhow::anyhow!("Failed to start container: {}", error));
}
log::info!("SearXNG container started");
Ok(())
}
/// Para o container SearXNG
pub async fn stop_searxng() -> Result<()> {
log::info!("Stopping SearXNG container");
let output = Command::new("docker")
.arg("stop")
.arg("ollaHub_searxng")
.output()?;
if !output.status.success() {
let error = String::from_utf8_lossy(&output.stderr);
return Err(anyhow::anyhow!("Failed to stop container: {}", error));
}
log::info!("SearXNG container stopped");
Ok(())
}
/// Verifica se SearXNG está respondendo
pub async fn check_searxng_status(url: &str) -> Result<bool> {
let client = Client::builder()
.timeout(Duration::from_secs(5))
.build()?;
let test_url = format!("{}/search?q=test&format=json", url.trim_end_matches('/'));
match timeout(Duration::from_secs(5), client.get(&test_url).send()).await {
Ok(Ok(response)) => Ok(response.status().is_success()),
Ok(Err(_)) => Ok(false),
Err(_) => Ok(false),
}
}
/// Obtém logs do container SearXNG
pub fn get_searxng_logs(lines: usize) -> Result<Vec<String>> {
let output = Command::new("docker")
.arg("logs")
.arg("--tail")
.arg(lines.to_string())
.arg("ollaHub_searxng")
.output()?;
if !output.status.success() {
return Ok(Vec::new());
}
let logs = String::from_utf8_lossy(&output.stdout);
Ok(logs.lines().map(|s| s.to_string()).collect())
}
+59
View File
@@ -0,0 +1,59 @@
use anyhow::Result;
use reqwest::Client;
use std::time::Duration;
use crate::web_scraper::SearchResultMetadata;
/// Busca via SearXNG self-hosted
pub async fn searxng_search(
query: &str,
limit: usize,
url: &str,
) -> Result<Vec<SearchResultMetadata>> {
let client = Client::builder()
.timeout(Duration::from_secs(5)) // Reduzido de 8s para 5s para maior velocidade
.build()?;
let search_url = format!("{}/search", url.trim_end_matches('/'));
let response = client
.get(&search_url)
.query(&[
("q", query),
("format", "json"),
("categories", "general"),
("pageno", "1"),
])
.send()
.await?;
if !response.status().is_success() {
return Err(anyhow::anyhow!("SearXNG returned status: {}", response.status()));
}
#[derive(serde::Deserialize)]
struct SearxngResponse {
results: Vec<SearxngResult>,
}
#[derive(serde::Deserialize)]
struct SearxngResult {
title: String,
url: String,
content: Option<String>,
}
let data: SearxngResponse = response.json().await?;
let metadata: Vec<SearchResultMetadata> = data
.results
.into_iter()
.take(limit)
.map(|r| SearchResultMetadata {
title: r.title,
url: r.url,
snippet: r.content.unwrap_or_default(),
})
.collect();
Ok(metadata)
}
+603
View File
@@ -0,0 +1,603 @@
use std::process::Command;
use std::path::PathBuf;
use std::time::{Duration, Instant};
use tauri::{AppHandle, Manager, Emitter};
use tokio::time::sleep;
use futures_util::StreamExt;
// Importar funções do módulo principal
use crate::{check_ollama_installed, check_ollama_running, poll_ollama_ready, check_ollama_installed_async, check_if_model_installed};
#[derive(serde::Serialize, Clone)]
pub struct SetupState {
pub ollama_installed: bool,
pub ollama_running: bool,
pub default_model_downloaded: bool,
pub default_model_name: Option<String>,
}
/// Verifica o estado atual do setup
pub async fn get_setup_state(_app_handle: &AppHandle) -> Result<SetupState, String> {
let ollama_installed = check_ollama_installed();
let ollama_running = if ollama_installed {
check_ollama_running().await
} else {
false
};
// Verificar se modelo padrão está instalado
let default_model_name = Some("llama3.2:1b".to_string());
let default_model_downloaded = if ollama_running {
check_if_model_installed(default_model_name.as_ref().unwrap().clone())
} else {
false
};
Ok(SetupState {
ollama_installed,
ollama_running,
default_model_downloaded,
default_model_name: if default_model_downloaded {
default_model_name
} else {
None
},
})
}
/// Instala o Ollama silenciosamente
/// No Windows, tenta usar Chocolatey primeiro, depois fallback para instalação direta
pub async fn install_ollama_silently(
app_handle: AppHandle,
installer_path: Option<String>,
) -> Result<(), String> {
let window = app_handle.get_webview_window("main");
// Se já está instalado, apenas verificar se está rodando
if check_ollama_installed() {
if let Some(ref w) = window {
let _ = w.emit("setup-progress", serde_json::json!({
"status": "ollama_check",
"message": "Ollama já está instalado. Verificando se está rodando..."
}));
}
// Tentar iniciar se não estiver rodando
if !check_ollama_running().await {
if let Some(ref w) = window {
let _ = w.emit("setup-progress", serde_json::json!({
"status": "ollama_starting",
"message": "Iniciando Ollama..."
}));
}
// Aguardar até 30 segundos para o Ollama iniciar
let max_attempts = 30;
for attempt in 1..=max_attempts {
if check_ollama_running().await {
if let Some(ref w) = window {
let _ = w.emit("setup-progress", serde_json::json!({
"status": "ollama_ready",
"message": "Ollama está rodando!"
}));
}
return Ok(());
}
sleep(Duration::from_secs(1)).await;
if attempt % 5 == 0 {
if let Some(ref w) = window {
let _ = w.emit("setup-progress", serde_json::json!({
"status": "ollama_starting",
"message": format!("Aguardando Ollama iniciar... ({}s)", attempt)
}));
}
}
}
return Err("Ollama instalado mas não conseguiu iniciar automaticamente".to_string());
}
if let Some(ref w) = window {
let _ = w.emit("setup-progress", serde_json::json!({
"status": "ollama_ready",
"message": "Ollama está rodando!"
}));
}
return Ok(());
}
// No Windows, tentar usar Chocolatey primeiro
#[cfg(target_os = "windows")]
{
use crate::setup_manager;
// Verificar se Chocolatey está instalado
if !setup_manager::check_chocolatey_installed() {
if let Some(ref w) = window {
let _ = w.emit("setup-progress", serde_json::json!({
"status": "installing_chocolatey",
"message": "Chocolatey não encontrado. Instalando Chocolatey..."
}));
}
// Tentar instalar Chocolatey
if let Some(ref w) = window {
if let Err(e) = setup_manager::install_chocolatey(w.clone()).await {
log::warn!("Falha ao instalar Chocolatey: {}. Tentando instalação direta...", e);
// Continuar para fallback de instalação direta
} else {
log::info!("Chocolatey instalado com sucesso");
}
} else {
return Err("Window não disponível para instalação do Chocolatey".to_string());
}
}
// Se Chocolatey está disponível, usar para instalar Ollama
if setup_manager::check_chocolatey_installed() {
if let Some(ref w) = window {
let _ = w.emit("setup-progress", serde_json::json!({
"status": "installing_ollama",
"message": "Instalando Ollama via Chocolatey..."
}));
}
if let Some(ref w) = window {
match setup_manager::install_ollama_choco(w.clone()).await {
Ok(_) => {
log::info!("Ollama instalado via Chocolatey com sucesso");
// Aguardar um pouco e verificar se está rodando
sleep(Duration::from_secs(3)).await;
if check_ollama_running().await {
if let Some(ref w) = window {
let _ = w.emit("setup-progress", serde_json::json!({
"status": "ollama_ready",
"message": "Ollama instalado e rodando!"
}));
}
return Ok(());
}
// Se não está rodando, tentar iniciar
if let Some(ref w) = window {
let _ = w.emit("setup-progress", serde_json::json!({
"status": "ollama_starting",
"message": "Iniciando Ollama..."
}));
}
// Aguardar até 30 segundos para o Ollama iniciar
let max_attempts = 30;
for attempt in 1..=max_attempts {
if check_ollama_running().await {
if let Some(ref w) = window {
let _ = w.emit("setup-progress", serde_json::json!({
"status": "ollama_ready",
"message": "Ollama está rodando!"
}));
}
return Ok(());
}
sleep(Duration::from_secs(1)).await;
if attempt % 5 == 0 {
if let Some(ref w) = window {
let _ = w.emit("setup-progress", serde_json::json!({
"status": "ollama_starting",
"message": format!("Aguardando Ollama iniciar... ({}s)", attempt)
}));
}
}
}
return Err("Ollama instalado mas não conseguiu iniciar automaticamente".to_string());
}
Err(e) => {
log::warn!("Falha ao instalar Ollama via Chocolatey: {}. Tentando instalação direta...", e);
// Continuar para fallback de instalação direta
}
}
}
}
// Fallback: instalação direta via .exe
let installer = if let Some(path) = installer_path {
PathBuf::from(path)
} else {
return Err("Caminho do instalador não fornecido e Chocolatey não disponível".to_string());
};
if !installer.exists() {
return Err(format!("Instalador não encontrado: {:?}", installer));
}
if let Some(ref w) = window {
let _ = w.emit("setup-progress", serde_json::json!({
"status": "installing_ollama",
"message": "Iniciando instalação silenciosa do Ollama..."
}));
}
use std::os::windows::process::CommandExt;
const CREATE_NO_WINDOW: u32 = 0x08000000;
let mut cmd = Command::new(&installer);
cmd.creation_flags(CREATE_NO_WINDOW);
cmd.arg("/VERYSILENT")
.arg("/NORESTART")
.arg("/SUPPRESSMSGBOXES");
log::info!("Executando instalador silencioso: {:?}", installer);
let output = cmd.output()
.map_err(|e| format!("Erro ao executar instalador: {}", e))?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
log::warn!("Instalador retornou código de erro: {}", stderr);
}
}
#[cfg(target_os = "linux")]
{
use std::os::unix::fs::PermissionsExt;
use std::process::Stdio;
let mut perms = std::fs::metadata(&installer)
.map_err(|e| format!("Erro ao obter metadados do arquivo: {}", e))?
.permissions();
perms.set_mode(0o755);
std::fs::set_permissions(&installer, perms)
.map_err(|e| format!("Erro ao definir permissões: {}", e))?;
let path_str = installer.to_string_lossy();
let mut cmd = if path_str.ends_with(".sh") {
let mut c = Command::new("sh");
c.arg(&installer).arg("--quiet");
c
} else {
Command::new(&installer)
};
// Capturar stderr para detectar falhas de permissão
cmd.stderr(Stdio::piped());
cmd.stdout(Stdio::piped());
let output = cmd.output()
.map_err(|e| format!("Erro ao executar instalador: {}", e))?;
// Verificar stderr por mensagens de erro de permissão
let stderr = String::from_utf8_lossy(&output.stderr);
let stderr_lower = stderr.to_lowercase();
if stderr_lower.contains("permission denied")
|| stderr_lower.contains("sudo required")
|| stderr_lower.contains("cannot open")
|| stderr_lower.contains("access denied") {
return Err("Instalação requer privilégios de administrador. Execute manualmente: curl https://ollama.com/install.sh | sh".to_string());
}
// Verificar se o comando falhou
if !output.status.success() {
let error_msg = if !stderr.is_empty() {
format!("Instalação falhou: {}", stderr.trim())
} else {
format!("Instalação falhou com código de saída: {:?}", output.status.code())
};
return Err(error_msg);
}
}
#[cfg(target_os = "macos")]
{
let path_str = installer.to_string_lossy();
if path_str.ends_with(".pkg") {
Command::new("installer")
.arg("-pkg")
.arg(&installer)
.arg("-target")
.arg("/")
.arg("-verboseR")
.spawn()
.map_err(|e| format!("Erro ao executar instalador: {}", e))?;
} else {
Command::new("open")
.arg(&installer)
.spawn()
.map_err(|e| format!("Erro ao abrir instalador: {}", e))?;
}
}
// Aguardar instalação e verificar
if let Some(ref w) = window {
let _ = w.emit("setup-progress", serde_json::json!({
"status": "installing_ollama",
"message": "Aguardando instalação concluir..."
}));
}
// Fazer polling para verificar quando Ollama estiver instalado e rodando
let window_clone = window.clone();
let max_attempts = 90;
let mut attempts = 0;
while attempts < max_attempts {
if check_ollama_installed_async().await {
// Aguardar um pouco para o serviço iniciar
sleep(Duration::from_secs(2)).await;
if let Ok(_) = poll_ollama_ready(5).await {
log::info!("Ollama instalado e rodando com sucesso");
if let Some(ref w) = window_clone {
let _ = w.emit("setup-progress", serde_json::json!({
"status": "ollama_ready",
"message": "Ollama instalado e rodando!"
}));
}
return Ok(());
}
}
attempts += 1;
sleep(Duration::from_secs(1)).await;
if attempts % 5 == 0 {
if let Some(ref w) = window_clone {
let _ = w.emit("setup-progress", serde_json::json!({
"status": "installing_ollama",
"message": format!("Aguardando instalação... ({}s)", attempts)
}));
}
}
}
// Verificar se pelo menos foi instalado
if check_ollama_installed_async().await {
if let Some(ref w) = window_clone {
let _ = w.emit("setup-progress", serde_json::json!({
"status": "ollama_partial",
"message": "Ollama instalado, mas não iniciou automaticamente"
}));
}
return Err("Ollama instalado mas não iniciou automaticamente".to_string());
}
Err("Timeout: Ollama não foi instalado após 90 segundos".to_string())
}
// Estruturas para processar progresso do download
#[derive(serde::Deserialize)]
struct PullProgress {
status: String,
#[serde(default)]
digest: String,
#[serde(default)]
total: u64,
#[serde(default)]
completed: u64,
}
#[derive(serde::Serialize)]
struct DownloadProgress {
status: String,
percent: Option<u8>,
downloaded: Option<String>,
total: Option<String>,
speed: Option<String>,
raw: String,
}
// Funções auxiliares para formatação
fn format_bytes(bytes: u64) -> Option<String> {
if bytes == 0 {
return None;
}
const KB: u64 = 1024;
const MB: u64 = KB * 1024;
const GB: u64 = MB * 1024;
Some(if bytes >= GB {
format!("{:.1} GB", bytes as f64 / GB as f64)
} else if bytes >= MB {
format!("{:.1} MB", bytes as f64 / MB as f64)
} else if bytes >= KB {
format!("{:.1} KB", bytes as f64 / KB as f64)
} else {
format!("{} B", bytes)
})
}
fn format_speed(bytes_per_sec: f64) -> String {
const KB: f64 = 1024.0;
const MB: f64 = KB * 1024.0;
const GB: f64 = MB * 1024.0;
if bytes_per_sec >= GB {
format!("{:.1} GB/s", bytes_per_sec / GB)
} else if bytes_per_sec >= MB {
format!("{:.1} MB/s", bytes_per_sec / MB)
} else if bytes_per_sec >= KB {
format!("{:.1} KB/s", bytes_per_sec / KB)
} else {
format!("{:.0} B/s", bytes_per_sec)
}
}
/// Baixa o modelo padrão durante o setup
pub async fn download_default_model(
app_handle: AppHandle,
model_name: String,
) -> Result<(), String> {
let window = app_handle.get_webview_window("main");
// Verificar se já está instalado
if check_if_model_installed(model_name.clone()) {
if let Some(ref w) = window {
let _ = w.emit("setup-progress", serde_json::json!({
"status": "model_ready",
"message": format!("Modelo {} já está instalado!", model_name)
}));
}
return Ok(());
}
if let Some(ref w) = window {
let _ = w.emit("setup-progress", serde_json::json!({
"status": "downloading_model",
"message": format!("Baixando modelo {}...", model_name)
}));
}
// Usar API HTTP do Ollama (mais confiável que CLI)
let client = reqwest::Client::new();
let response = client
.post("http://localhost:11434/api/pull")
.json(&serde_json::json!({ "name": model_name.clone(), "stream": true }))
.send()
.await
.map_err(|e| format!("Erro ao conectar à API do Ollama: {}", e))?;
if !response.status().is_success() {
return Err(format!("API do Ollama retornou erro: {}", response.status()));
}
let mut stream = response.bytes_stream();
let mut buffer = String::new();
let window_clone = window.clone();
let model_name_clone = model_name.clone();
let mut last_completed: u64 = 0;
let mut last_time = Instant::now();
// Processar stream NDJSON (mesmo formato que pull_model_impl_webview)
while let Some(chunk_result) = stream.next().await {
let chunk = chunk_result.map_err(|e| format!("Erro no stream: {}", e))?;
let chunk_str = String::from_utf8_lossy(&chunk);
buffer.push_str(&chunk_str);
// Processar linhas completas (separadas por \n)
while let Some(pos) = buffer.find('\n') {
let line = buffer[..pos].trim().to_string();
buffer = buffer[pos + 1..].to_string();
if line.is_empty() {
continue;
}
// Tentar deserializar como PullProgress
match serde_json::from_str::<PullProgress>(&line) {
Ok(json_progress) => {
// Calcular porcentagem se tiver total/completed
let percent = if json_progress.total > 0 {
Some(((json_progress.completed as f64 / json_progress.total as f64) * 100.0) as u8)
} else {
None
};
// Calcular velocidade
let now = Instant::now();
let delta_time = now.duration_since(last_time).as_secs_f64();
let speed = if delta_time > 0.0 && json_progress.completed > last_completed {
let delta_bytes = json_progress.completed - last_completed;
let bytes_per_sec = delta_bytes as f64 / delta_time;
Some(format_speed(bytes_per_sec))
} else {
None
};
last_completed = json_progress.completed;
last_time = now;
// Criar DownloadProgress estruturado
let progress = DownloadProgress {
status: json_progress.status.clone(),
percent,
downloaded: format_bytes(json_progress.completed),
total: format_bytes(json_progress.total),
speed,
raw: line.clone(),
};
// Emitir evento download-progress para o DownloadContext
if let Ok(json) = serde_json::to_string(&progress) {
if let Some(ref w) = window_clone {
let _ = w.emit("download-progress", json);
}
}
// Manter compatibilidade: também emitir setup-progress
if let Some(ref w) = window_clone {
let _ = w.emit("setup-progress", serde_json::json!({
"status": "downloading_model",
"message": format!("{}: {}", model_name_clone, json_progress.status)
}));
}
// Se status for "success", finalizar
if json_progress.status == "success" {
let success_progress = DownloadProgress {
status: "success".to_string(),
percent: Some(100),
downloaded: format_bytes(json_progress.completed),
total: format_bytes(json_progress.total),
speed: None,
raw: "success".to_string(),
};
if let Ok(json) = serde_json::to_string(&success_progress) {
if let Some(ref w) = window_clone {
let _ = w.emit("download-progress", json);
}
}
break;
}
}
Err(_) => {
// Se não conseguir parsear como JSON, tratar como linha raw (fallback)
let progress = DownloadProgress {
status: "downloading".to_string(),
percent: None,
downloaded: None,
total: None,
speed: None,
raw: line.clone(),
};
if let Ok(json) = serde_json::to_string(&progress) {
if let Some(ref w) = window_clone {
let _ = w.emit("download-progress", json);
}
}
}
}
}
}
// Se chegou aqui, o stream terminou - emitir sucesso final
let success_progress = DownloadProgress {
status: "success".to_string(),
percent: Some(100),
downloaded: format_bytes(last_completed),
total: None,
speed: None,
raw: "success".to_string(),
};
if let Ok(json) = serde_json::to_string(&success_progress) {
if let Some(ref w) = window {
let _ = w.emit("download-progress", json);
}
}
// Verificar se o modelo foi instalado
if check_if_model_installed(model_name.clone()) {
if let Some(ref w) = window {
let _ = w.emit("setup-progress", serde_json::json!({
"status": "model_ready",
"message": format!("Modelo {} baixado com sucesso!", model_name)
}));
}
Ok(())
} else {
Err(format!("Download do modelo {} pode ter falhado", model_name))
}
}
+401
View File
@@ -0,0 +1,401 @@
use std::process::{Command, Stdio};
use tauri::{WebviewWindow, Emitter};
use tokio::process::Command as TokioCommand;
use tokio::fs;
/// Verifica se o Chocolatey está instalado
pub fn check_chocolatey_installed() -> bool {
#[cfg(target_os = "windows")]
{
let mut cmd = Command::new("choco");
cmd.arg("--version");
cmd.stdout(Stdio::piped());
cmd.stderr(Stdio::piped());
// Ocultar janela do console no Windows
use std::os::windows::process::CommandExt;
const CREATE_NO_WINDOW: u32 = 0x08000000;
cmd.creation_flags(CREATE_NO_WINDOW);
match cmd.output() {
Ok(output) => output.status.success(),
Err(_) => false,
}
}
#[cfg(not(target_os = "windows"))]
{
false
}
}
/// Instala o Chocolatey via PowerShell com elevação UAC
/// Emite eventos 'install-log' para cada linha de stdout/stderr
/// Usa Start-Process -Verb RunAs para solicitar permissões de administrador
pub async fn install_chocolatey(window: WebviewWindow) -> Result<(), String> {
#[cfg(target_os = "windows")]
{
use std::env;
use uuid::Uuid;
// Script oficial de instalação do Chocolatey
let install_script = r#"
Set-ExecutionPolicy Bypass -Scope Process -Force;
[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072;
iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))
"#;
// Criar arquivo temporário para capturar logs do processo elevado
let temp_dir = env::temp_dir();
let log_file = temp_dir.join(format!("choco_install_{}.log", Uuid::new_v4()));
let log_file_str = log_file.to_string_lossy().replace('\\', "/");
// Criar script PowerShell que executa com elevação e redireciona saída
let ps_wrapper_script = format!(
r#"$script = @'
{}
'@
$logFile = '{}'
try {{
$proc = Start-Process powershell -ArgumentList '-NoProfile', '-ExecutionPolicy', 'Bypass', '-Command', $script -Verb RunAs -Wait -PassThru -WindowStyle Hidden
if ($proc.ExitCode -ne 0) {{
"ExitCode: $($proc.ExitCode)" | Out-File -FilePath $logFile -Append -Encoding utf8
}}
}} catch {{
"Error: $_" | Out-File -FilePath $logFile -Append -Encoding utf8
exit 1
}}"#,
install_script.trim(),
log_file_str
);
// Executar PowerShell wrapper (sem elevação, mas ele solicitará UAC internamente)
let mut cmd = TokioCommand::new("powershell");
cmd.args(&[
"-NoProfile",
"-ExecutionPolicy", "Bypass",
"-Command", &ps_wrapper_script
]);
cmd.stdout(Stdio::piped());
cmd.stderr(Stdio::piped());
// Ocultar janela do console
const CREATE_NO_WINDOW: u32 = 0x08000000;
cmd.creation_flags(CREATE_NO_WINDOW);
// Iniciar processo e monitorar arquivo de log em paralelo
let mut child = cmd.spawn()
.map_err(|e| format!("Failed to spawn PowerShell: {}", e))?;
// Aguardar conclusão do processo wrapper
let status = child.wait().await
.map_err(|e| format!("Failed to wait for PowerShell: {}", e))?;
// Aguardar um pouco para garantir que logs finais foram escritos
tokio::time::sleep(tokio::time::Duration::from_millis(1000)).await;
// Ler logs do arquivo e emitir eventos
if let Ok(contents) = fs::read_to_string(&log_file).await {
for line in contents.lines() {
let trimmed = line.trim();
if !trimmed.is_empty() && !trimmed.starts_with("ExitCode:") {
let _ = window.emit("install-log", serde_json::json!({
"line": trimmed,
"type": "stdout"
}));
}
}
} else {
// Se não conseguir ler o arquivo, emitir mensagem genérica
let _ = window.emit("install-log", serde_json::json!({
"line": "Instalando Chocolatey (solicitando permissões de administrador)...",
"type": "stdout"
}));
}
// Limpar arquivo temporário
let _ = fs::remove_file(&log_file).await;
// Verificar se Chocolatey foi instalado
if check_chocolatey_installed() {
Ok(())
} else if status.success() {
// Processo retornou sucesso mas Chocolatey não está disponível
// Pode ser que precise recarregar PATH
Err("Chocolatey installation completed but not found in PATH. Please restart the application.".to_string())
} else {
Err(format!("Chocolatey installation failed with exit code: {:?}", status.code()))
}
}
#[cfg(not(target_os = "windows"))]
{
Err("Chocolatey is only available on Windows".to_string())
}
}
/// Instala o Ollama via Chocolatey com elevação UAC
/// Usa arquivo .ps1 temporário para evitar problemas de escaping
/// Emite eventos 'install-log' para cada linha de stdout/stderr
/// Extrai progresso dos logs quando disponível
/// Verifica integridade após instalação usando verify_ollama_integrity
pub async fn install_ollama_choco(window: WebviewWindow) -> Result<(), String> {
#[cfg(target_os = "windows")]
{
use std::env;
use uuid::Uuid;
let temp_dir = env::temp_dir();
let script_id = Uuid::new_v4();
let script_file = temp_dir.join(format!("ollama_install_{}.ps1", script_id));
let log_file = temp_dir.join(format!("ollama_install_{}.log", script_id));
let log_file_str = log_file.to_string_lossy().replace('\\', "/");
let script_file_str = script_file.to_string_lossy().replace('\\', "/");
// Conteúdo do script PowerShell completo
let script_content = format!(
r#"$ErrorActionPreference = "Stop"
$logFile = '{}'
Start-Transcript -Path $logFile -Append
try {{
Write-Output "Iniciando instalação do Ollama..."
# 1. Verificar/Instalar Chocolatey
if (-not (Get-Command choco -ErrorAction SilentlyContinue)) {{
Write-Output "Instalando Chocolatey..."
Set-ExecutionPolicy Bypass -Scope Process -Force
[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072
iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))
Write-Output "Chocolatey instalado com sucesso."
}} else {{
Write-Output "Chocolatey já está instalado."
}}
# 2. Atualizar PATH para a sessão atual (Crucial)
$env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")
# 3. Instalar Ollama
Write-Output "Instalando Ollama via Choco..."
choco install ollama -y --force
Write-Output "Instalação concluída com sucesso."
exit 0
}} catch {{
Write-Error $_
"Error: $_" | Out-File -FilePath $logFile -Append -Encoding utf8
exit 1
}} finally {{
Stop-Transcript
}}"#,
log_file_str
);
// Escrever script para arquivo
fs::write(&script_file, script_content).await
.map_err(|e| format!("Failed to create PowerShell script: {}", e))?;
// Emitir evento inicial
let _ = window.emit("install-log", serde_json::json!({
"line": "Preparando instalação do Ollama via Chocolatey...",
"type": "stdout",
"progress": 0
}));
// Emitir evento inicial avisando sobre UAC
let _ = window.emit("install-log", serde_json::json!({
"line": "Solicitando permissões de administrador... (Por favor, aceite o prompt do Windows)",
"type": "stdout",
"progress": 0
}));
// Executar script com elevação usando Start-Process (janela oculta)
let mut cmd = TokioCommand::new("powershell");
cmd.args(&[
"-NoProfile",
"-ExecutionPolicy", "Bypass",
"-Command",
&format!("Start-Process powershell -ArgumentList '-NoProfile', '-ExecutionPolicy', 'Bypass', '-File', '{}' -Verb RunAs -WindowStyle Hidden -Wait", script_file_str)
]);
cmd.stdout(Stdio::piped());
cmd.stderr(Stdio::piped());
// Ocultar janela do console
const CREATE_NO_WINDOW: u32 = 0x08000000;
cmd.creation_flags(CREATE_NO_WINDOW);
// Iniciar processo
let mut child = cmd.spawn()
.map_err(|e| format!("Failed to spawn PowerShell: {}", e))?;
// Monitorar arquivo de log em tempo real enquanto o processo executa
let log_file_clone = log_file.clone();
let window_clone = window.clone();
let monitor_handle = tokio::spawn(async move {
let mut last_position = 0u64;
let mut last_read = std::time::Instant::now();
// Polling do arquivo de log enquanto o processo está rodando
loop {
if let Ok(metadata) = fs::metadata(&log_file_clone).await {
if metadata.len() > last_position {
if let Ok(mut file) = fs::File::open(&log_file_clone).await {
use tokio::io::{AsyncSeekExt, AsyncReadExt};
if file.seek(tokio::io::SeekFrom::Start(last_position)).await.is_ok() {
let bytes_to_read = (metadata.len() - last_position) as usize;
let mut buffer = vec![0u8; bytes_to_read];
if let Ok(bytes_read) = file.read(&mut buffer).await {
if bytes_read > 0 {
let new_content = String::from_utf8_lossy(&buffer[..bytes_read]);
for line in new_content.lines() {
let trimmed = line.trim();
// Filtrar linhas de controle do Transcript
if !trimmed.is_empty()
&& !trimmed.starts_with("**********************")
&& !trimmed.starts_with("Windows PowerShell transcript start")
&& !trimmed.starts_with("Windows PowerShell transcript end")
&& !trimmed.starts_with("Command start time:")
&& !trimmed.starts_with("Command end time:")
&& !trimmed.starts_with("Machine:")
&& !trimmed.starts_with("Username:")
&& !trimmed.starts_with("RunAs User:")
&& !trimmed.starts_with("Transcript started")
&& !trimmed.starts_with("Transcript stopped") {
let progress = extract_progress_from_log(trimmed);
let _ = window_clone.emit("install-log", serde_json::json!({
"line": trimmed,
"type": "stdout",
"progress": progress
}));
}
}
last_position = metadata.len();
last_read = std::time::Instant::now();
}
}
}
}
}
}
// Verificar se o processo ainda está rodando (timeout após 5 minutos de inatividade)
if last_read.elapsed().as_secs() > 300 {
break;
}
tokio::time::sleep(tokio::time::Duration::from_millis(500)).await;
}
});
// Aguardar conclusão do processo wrapper
let status = child.wait().await
.map_err(|e| format!("Failed to wait for PowerShell: {}", e))?;
// Aguardar um pouco para garantir que logs finais foram escritos
tokio::time::sleep(tokio::time::Duration::from_millis(2000)).await;
// Cancelar monitoramento
monitor_handle.abort();
// Ler logs finais do arquivo e emitir eventos (caso tenha perdido algo)
if let Ok(contents) = fs::read_to_string(&log_file).await {
for line in contents.lines() {
let trimmed = line.trim();
// Filtrar linhas de controle do Transcript
if !trimmed.is_empty()
&& !trimmed.starts_with("ExitCode:")
&& !trimmed.starts_with("**********************")
&& !trimmed.starts_with("Windows PowerShell transcript start")
&& !trimmed.starts_with("Windows PowerShell transcript end")
&& !trimmed.starts_with("Command start time:")
&& !trimmed.starts_with("Command end time:")
&& !trimmed.starts_with("Machine:")
&& !trimmed.starts_with("Username:")
&& !trimmed.starts_with("RunAs User:")
&& !trimmed.starts_with("Transcript started")
&& !trimmed.starts_with("Transcript stopped") {
let progress = extract_progress_from_log(trimmed);
let _ = window.emit("install-log", serde_json::json!({
"line": trimmed,
"type": "stdout",
"progress": progress
}));
}
}
}
// Limpar arquivos temporários
let _ = fs::remove_file(&script_file).await;
let _ = fs::remove_file(&log_file).await;
// Verificar se a instalação foi bem-sucedida
if !status.success() {
return Err(format!("Ollama installation via Chocolatey failed with exit code: {:?}", status.code()));
}
// Aguardar um pouco para o Ollama ser instalado e iniciado
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
// Verificar integridade usando verify_ollama_integrity
use crate::verify_ollama_integrity;
match verify_ollama_integrity().await {
Ok(true) => {
let _ = window.emit("install-log", serde_json::json!({
"line": "Ollama instalado e verificado com sucesso!",
"type": "stdout",
"progress": 100
}));
Ok(())
}
Ok(false) => {
// Instalado mas não funcional (serviço não está rodando)
Err("Ollama foi instalado mas o serviço não está rodando. Tente iniciar manualmente.".to_string())
}
Err(e) => {
Err(format!("Erro ao verificar instalação do Ollama: {}", e))
}
}
}
#[cfg(not(target_os = "windows"))]
{
Err("Chocolatey is only available on Windows".to_string())
}
}
/// Extrai progresso percentual de uma linha de log
/// Procura por padrões como "Progress: 50%", "50%", "Downloading...", etc.
fn extract_progress_from_log(line: &str) -> Option<u8> {
// Procurar por padrão "Progress: n%" ou "n%"
if let Some(percent_pos) = line.find('%') {
let before_percent = &line[..percent_pos];
// Procurar número antes do %
let mut num_str = String::new();
for ch in before_percent.chars().rev() {
if ch.is_ascii_digit() {
num_str.insert(0, ch);
} else if !num_str.is_empty() {
break;
}
}
if let Ok(num) = num_str.parse::<u8>() {
if num <= 100 {
return Some(num);
}
}
}
// Procurar por keywords e mapear para progresso aproximado
let line_lower = line.to_lowercase();
if line_lower.contains("downloading") {
return Some(25);
} else if line_lower.contains("installing") {
return Some(50);
} else if line_lower.contains("shim") || line_lower.contains("creating") {
return Some(75);
} else if line_lower.contains("finished") || line_lower.contains("completed") || line_lower.contains("success") {
return Some(100);
}
None
}
+14 -31
View File
@@ -2,9 +2,7 @@ use crate::scheduler::{SentinelTask, TaskAction};
use crate::ollama_client::OllamaClient;
use crate::web_scraper::search_and_scrape;
use crate::{Message, ChatSession, get_chats_dir};
use std::sync::Arc;
use std::fs;
use headless_chrome::Browser;
use chrono::Utc;
use tauri::AppHandle;
use tauri_plugin_notification::NotificationExt;
@@ -14,7 +12,6 @@ use sysinfo::System;
pub async fn execute_task(
task: &SentinelTask,
app_handle: AppHandle,
browser: Arc<Browser>,
ollama_url: Option<String>,
) -> Result<(), String> {
log::info!("Executando task: {} ({})", task.label, task.id);
@@ -29,7 +26,6 @@ pub async fn execute_task(
model,
*max_results,
&app_handle,
browser,
&client,
).await
}
@@ -55,44 +51,31 @@ async fn execute_search_and_summarize(
model: &str,
max_results: usize,
app_handle: &AppHandle,
browser: Arc<Browser>,
ollama_client: &OllamaClient,
) -> Result<(), String> {
// 1. Buscar conteúdo na web
// 1. Buscar conteúdo na web via DuckDuckGo
log::info!("Buscando conteúdo para: {}", query);
let scraped = search_and_scrape(query, max_results, browser, vec![])
let scraped = search_and_scrape(query, max_results, vec![])
.await
.map_err(|e| format!("Erro ao buscar conteúdo: {}", e))?;
if scraped.is_empty() {
return Err("Nenhum resultado encontrado na busca".to_string());
}
// 2. Combinar conteúdo em markdown
let web_context: String = scraped
.iter()
.map(|s| format!("---\nTítulo: {}\nURL: {}\n---\n\n{}", s.title, s.url, s.markdown))
.collect::<Vec<_>>()
.join("\n\n");
// 3. Criar prompt para o Ollama
let system_prompt = format!(
"Você é um assistente especializado em resumir e analisar informações da web.\n\
DATA ATUAL: {}\n\n\
Use as informações fornecidas abaixo para criar um resumo detalhado e útil.",
Utc::now().format("%d/%m/%Y %H:%M")
);
let user_prompt = format!(
"Com base nas informações abaixo sobre '{}', crie um resumo detalhado e estruturado.\n\n\
## CONTEXTO WEB\n{}\n\n\
Por favor, forneça:\n\
1. Resumo executivo (2-3 parágrafos)\n\
2. Principais pontos encontrados\n\
3. Conclusões ou insights relevantes",
query,
web_context
);
let system_prompt = if web_context.trim().is_empty() {
"Você é o OllaHub, um assistente de IA inteligente e prestativo. Responda às perguntas do usuário usando seu conhecimento interno da melhor forma possível. Seja educado, claro e prestativo.".to_string()
} else {
format!(
"Você é um assistente de pesquisa preciso e factual.\n\n### CONTEXTO RECUPERADO:\n{}\n\n### INSTRUÇÕES:\n1. Responda à pergunta do usuário BASEADO ESTRITAMENTE no contexto acima.\n2. Cite as fontes usando o formato [1] sempre que afirmar um fato do contexto.\n3. Se o contexto não contiver a resposta, diga que não encontrou nas fontes, mas ofereça seu conhecimento geral marcando claramente como 'Conhecimento Prévio'.\n4. Seja direto e evite introduções longas.",
web_context
)
};
let user_prompt = format!("Pergunta do usuário: {}", query);
// 4. Enviar para Ollama
log::info!("Enviando para Ollama (modelo: {})", model);
+373 -1246
View File
File diff suppressed because it is too large. Load diff
+17 -1
View File
@@ -1,7 +1,11 @@
{
"$schema": "../node_modules/@tauri-apps/cli/config.schema.json",
"productName": "ollahub",
<<<<<<< HEAD
"version": "0.1.1",
=======
"version": "0.1.0",
>>>>>>> 593efd42e091a845dea82ee6646e027bce1e18c5
"identifier": "dev.evandrodevbr.ollahub",
"build": {
"frontendDist": "../out",
@@ -28,7 +32,7 @@
},
"bundle": {
"active": true,
"targets": "all",
"targets": "nsis",
"icon": [
"icons/icon.ico",
"icons/32x32.ico",
@@ -36,6 +40,14 @@
"icons/256x256.ico",
"icons/512x512.ico",
"icons/icon.icns"
<<<<<<< HEAD
],
"resources": [
"bundle/ollama/OllamaSetup.exe",
"bundle/models/embeddings/**/*",
"scripts/scraper.py"
=======
>>>>>>> 593efd42e091a845dea82ee6646e027bce1e18c5
]
},
"plugins": {
@@ -45,7 +57,11 @@
"https://github.com/evandrodevbr/OllaHub/releases/latest/download/latest.json"
],
"dialog": true,
<<<<<<< HEAD
"pubkey": "~/.tauri/ollahub.key.pub"
=======
"pubkey": "dW50cnVzdGVkIGNvbW1lbnQ6IG1pbmlzaWduIHB1YmxpYyBrZXk6IDgxMjY3NThDMUQyREQ5QUYKUldTdjJTMGRqSFVtZ1k1ZWFHSEcxNExDZzgrNVh6M0Z6VXIyaTU5dDAvZnpQb3p0UVpuRW1Od3YK"
>>>>>>> 593efd42e091a845dea82ee6646e027bce1e18c5
}
}
}