import { DeepNLPService } from "./deepnlp"; import { MCPCacheRepository } from "@/database/repositories/mcp-cache"; export interface SyncProgress { current: number; total: number; page: number; totalPages: number; } export interface SyncResult { success: boolean; totalDownloaded: number; errors: string[]; duration: number; } export class MCPSyncService { private static isSyncingFlag = false; private static abortController: AbortController | null = null; private static currentProgress: SyncProgress | null = null; /** * Download completo com paginação incremental */ static async syncAll( onProgress?: (progress: SyncProgress) => void ): Promise { const startTime = Date.now(); const errors: string[] = []; let totalDownloaded = 0; try { if (this.isSyncingFlag) { throw new Error("Sync already in progress"); } this.isSyncingFlag = true; this.abortController = new AbortController(); console.log("🔄 Starting MCP marketplace sync..."); // Limpar cache antigo antes de começar MCPCacheRepository.clearCache(); const allMCPs = await this.downloadAllPages(onProgress); totalDownloaded = allMCPs.length; // Salvar todos os MCPs no banco if (allMCPs.length > 0) { MCPCacheRepository.saveBatch(allMCPs); // Atualizar metadados MCPCacheRepository.updateMetadata("last_sync", Date.now().toString()); MCPCacheRepository.updateMetadata( "total_items", allMCPs.length.toString() ); console.log(`✅ Sync completed: ${totalDownloaded} MCPs downloaded`); } const duration = Date.now() - startTime; return { success: true, totalDownloaded, errors, duration, }; } catch (error: any) { console.error("❌ Sync failed:", error); errors.push(error.message); const duration = Date.now() - startTime; return { success: false, totalDownloaded, errors, duration, }; } finally { this.isSyncingFlag = false; this.abortController = null; this.currentProgress = null; } } /** * Sincronização em background (usado automaticamente) */ static async backgroundSync(): Promise { try { if (this.isSyncingFlag) { console.log("Sync already in progress, skipping background sync"); return; } console.log("🔄 Starting background MCP sync..."); await this.syncAll((progress) => { console.log( `📥 Downloaded ${progress.current}/${progress.total} MCPs (page ${progress.page}/${progress.totalPages})` ); }); } catch (error) { console.error("Background sync failed:", error); } } /** * Verificar se sincronização está em andamento */ static isSyncing(): boolean { return this.isSyncingFlag; } /** * Cancelar sincronização atual */ static cancelSync(): void { if (this.abortController) { this.abortController.abort(); console.log("🛑 Sync cancelled by user"); } } /** * Obter progresso atual da sincronização */ static getCurrentProgress(): SyncProgress | null { return this.currentProgress; } /** * Download paginado de todas as páginas - SEM LIMITE ARTIFICIAL */ private static async downloadAllPages( onProgress?: (progress: SyncProgress) => void ): Promise { const allMCPs: any[] = []; let page = 0; let COUNT_PER_PAGE = 100; // Começar com 100 let totalHits = 0; let consecutiveEmptyPages = 0; // Contador para páginas vazias consecutivas console.log("📡 Starting unlimited paginated download..."); while (!this.abortController?.signal.aborted) { try { const params = { mode: "list" as const, page_id: page, count_per_page: COUNT_PER_PAGE, }; console.log( `📄 Downloading page ${ page + 1 } (${COUNT_PER_PAGE} items per page)...` ); const response = await DeepNLPService.searchMCPs(params); // Se não há itens na resposta, incrementar contador de páginas vazias if (!response.items || response.items.length === 0) { consecutiveEmptyPages++; console.log( `⚠️ Empty page ${ page + 1 } (${consecutiveEmptyPages} consecutive empty pages)` ); // Se temos 3 páginas vazias consecutivas, parar o download if (consecutiveEmptyPages >= 3) { console.log( "🛑 Stopping download: 3 consecutive empty pages detected" ); break; } page++; continue; } // Reset contador de páginas vazias se encontramos dados consecutiveEmptyPages = 0; allMCPs.push(...response.items); // Atualizar total hits na primeira página if (page === 0) { totalHits = response.total_hits; console.log(`📊 Total MCPs available: ${totalHits}`); } // Notificar progresso this.currentProgress = { current: allMCPs.length, total: totalHits, page: page + 1, totalPages: totalHits > 0 ? Math.ceil(totalHits / COUNT_PER_PAGE) : page + 1, }; if (onProgress) { onProgress(this.currentProgress); } // Incrementar página page++; // Salvar em batch a cada 500 itens para evitar bloqueio if (allMCPs.length % 500 === 0) { console.log(`💾 Saving batch of 500 MCPs to database...`); MCPCacheRepository.saveBatch(allMCPs.slice(-500)); } // Pequena pausa entre requisições para não sobrecarregar a API await new Promise((resolve) => setTimeout(resolve, 100)); // Aumentar tamanho da página progressivamente para otimizar if (page === 5 && COUNT_PER_PAGE === 100) { COUNT_PER_PAGE = 500; console.log("🚀 Increasing page size to 500 for faster download"); } else if (page === 20 && COUNT_PER_PAGE === 500) { COUNT_PER_PAGE = 1000; console.log("🚀 Increasing page size to 1000 for maximum efficiency"); } // Log de progresso a cada 10 páginas if (page % 10 === 0) { console.log( `📈 Progress: ${allMCPs.length} MCPs downloaded (page ${page})` ); } } catch (error: any) { console.error(`❌ Error downloading page ${page + 1}:`, error); // Se for erro de abort, parar if (this.abortController?.signal.aborted) { console.log("🛑 Download cancelled"); break; } // Se for erro de rede, tentar continuar if ( error.message.includes("fetch") || error.message.includes("network") ) { console.log("🔄 Network error, retrying in 2 seconds..."); await new Promise((resolve) => setTimeout(resolve, 2000)); continue; } // Para outros erros, parar throw error; } } // Salvar último batch se houver if (allMCPs.length > 0) { const remainingItems = allMCPs.length % 500; if (remainingItems > 0) { console.log(`💾 Saving final batch of ${remainingItems} MCPs...`); MCPCacheRepository.saveBatch(allMCPs.slice(-remainingItems)); } } console.log( `✅ Download completed: ${allMCPs.length} total MCPs downloaded` ); return allMCPs; } /** * Verificar se precisa sincronizar */ static needsSync(): boolean { return MCPCacheRepository.needsSync(); } /** * Obter estatísticas do cache */ static getCacheStats() { return MCPCacheRepository.getStats(); } /** * Forçar limpeza do cache */ static clearCache(): void { MCPCacheRepository.clearCache(); console.log("🗑️ Cache cleared"); } /** * Verificar saúde do sistema de cache */ static async healthCheck(): Promise<{ cacheExists: boolean; cacheValid: boolean; totalItems: number; lastSync: number | null; needsSync: boolean; }> { try { const cacheExists = MCPCacheRepository.tableExists( "mcp_marketplace_cache" ); const stats = MCPCacheRepository.getStats(); const needsSync = MCPCacheRepository.needsSync(); return { cacheExists, cacheValid: stats.isValid, totalItems: stats.totalItems, lastSync: stats.lastSync, needsSync, }; } catch (error) { console.error("Health check failed:", error); return { cacheExists: false, cacheValid: false, totalItems: 0, lastSync: null, needsSync: true, }; } } }