Origem: D:/Documents/Github/OllaHub Arquivos omitidos (segredos, bancos de dados, logs, >50 MB): 0
This commit is contained in:
1 parent
a7cbd1a3b4
commit
861af0bb9f
217 files changed
+39421
-1340
No files matched your search
@@ -81,16 +81,12 @@ export function condenseKnowledgeBase(
|
||||
options: CondensationOptions = {}
|
||||
): CondensationResult {
|
||||
const condensationStart = Date.now();
|
||||
|
||||
|
||||
chatLog.info(`\n[KnowledgeBase] ========== CONDENSATION START ==========`);
|
||||
chatLog.info(`[KnowledgeBase] Query: "${query}"`);
|
||||
chatLog.info(`[KnowledgeBase] Entries: ${entries.length}`);
|
||||
chatLog.info(`[KnowledgeBase] Options:`, {
|
||||
maxTokens: options.maxTokens || DEFAULT_OPTIONS.maxTokens,
|
||||
minRelevanceScore: options.minRelevanceScore || DEFAULT_OPTIONS.minRelevanceScore,
|
||||
autoSummarize: options.autoSummarize ?? DEFAULT_OPTIONS.autoSummarize,
|
||||
});
|
||||
|
||||
chatLog.info(`[KnowledgeBase] Options: maxTokens=${options.maxTokens || DEFAULT_OPTIONS.maxTokens}, minRelevanceScore=${options.minRelevanceScore || DEFAULT_OPTIONS.minRelevanceScore}, autoSummarize=${options.autoSummarize ?? DEFAULT_OPTIONS.autoSummarize}`);
|
||||
|
||||
const opts = { ...DEFAULT_OPTIONS, ...options };
|
||||
|
||||
if (entries.length === 0) {
|
||||
@@ -110,7 +106,7 @@ export function condenseKnowledgeBase(
|
||||
// Calcular tokens originais
|
||||
const originalContent = entries.map(e => e.content).join('\n\n');
|
||||
const originalTokens = estimateTokens(originalContent);
|
||||
|
||||
|
||||
chatLog.info(`[KnowledgeBase] Original content: ${originalContent.length} chars`);
|
||||
chatLog.info(`[KnowledgeBase] Original tokens (estimated): ${originalTokens}`);
|
||||
|
||||
@@ -123,14 +119,14 @@ export function condenseKnowledgeBase(
|
||||
}> = [];
|
||||
|
||||
chatLog.info(`[KnowledgeBase] Processing ${entries.length} entries into chunks...`);
|
||||
|
||||
|
||||
entries.forEach((entry, entryIndex) => {
|
||||
const entryStart = Date.now();
|
||||
chatLog.info(`\n[KnowledgeBase] --- Processing Entry ${entryIndex + 1}/${entries.length} ---`);
|
||||
chatLog.info(`[KnowledgeBase] Title: ${entry.title}`);
|
||||
chatLog.info(`[KnowledgeBase] URL: ${entry.sourceUrl}`);
|
||||
chatLog.info(`[KnowledgeBase] Content length: ${entry.content.length} chars`);
|
||||
|
||||
|
||||
const processed = processContent(entry.content, query, {
|
||||
summarizeThreshold: opts.summarizeThreshold,
|
||||
});
|
||||
@@ -145,7 +141,7 @@ export function condenseKnowledgeBase(
|
||||
|
||||
const entryDuration = Date.now() - entryStart;
|
||||
chatLog.info(`[KnowledgeBase] Generated ${chunksWithSource.length} chunks in ${entryDuration}ms`);
|
||||
|
||||
|
||||
// Log detalhes dos chunks
|
||||
if (chunksWithSource.length > 0) {
|
||||
chunksWithSource.slice(0, 3).forEach((chunk, chunkIdx) => {
|
||||
@@ -164,7 +160,7 @@ export function condenseKnowledgeBase(
|
||||
|
||||
allChunks.push(...chunksWithSource);
|
||||
});
|
||||
|
||||
|
||||
chatLog.info(`[KnowledgeBase] Total chunks generated: ${allChunks.length}`);
|
||||
|
||||
// Calcular tokens disponíveis (reservar 20% para margem)
|
||||
@@ -175,7 +171,7 @@ export function condenseKnowledgeBase(
|
||||
// Selecionar chunks mais relevantes (usando contexto se disponível)
|
||||
const selectionStart = Date.now();
|
||||
chatLog.info(`[KnowledgeBase] Selecting most relevant chunks...`);
|
||||
|
||||
|
||||
let selectedChunks = scoreAndSelectChunks(
|
||||
allChunks,
|
||||
query,
|
||||
@@ -183,10 +179,10 @@ export function condenseKnowledgeBase(
|
||||
opts.minRelevanceScore,
|
||||
opts.context
|
||||
);
|
||||
|
||||
|
||||
const selectionDuration = Date.now() - selectionStart;
|
||||
chatLog.info(`[KnowledgeBase] Selected ${selectedChunks.length} chunks from ${allChunks.length} total in ${selectionDuration}ms`);
|
||||
|
||||
|
||||
// Log top chunks selecionados
|
||||
if (selectedChunks.length > 0) {
|
||||
chatLog.info(`[KnowledgeBase] Top 5 selected chunks:`);
|
||||
@@ -208,7 +204,7 @@ export function condenseKnowledgeBase(
|
||||
} else {
|
||||
// Construir contexto a partir dos chunks selecionados
|
||||
const chunksBySource = new Map<number, Chunk[]>();
|
||||
|
||||
|
||||
selectedChunks.forEach(chunk => {
|
||||
if (!chunksBySource.has(chunk.sourceIndex)) {
|
||||
chunksBySource.set(chunk.sourceIndex, []);
|
||||
@@ -264,14 +260,14 @@ export function condenseKnowledgeBase(
|
||||
chatLog.info(`[KnowledgeBase] Compression ratio: ${(compressionRatio * 100).toFixed(1)}%`);
|
||||
chatLog.info(`[KnowledgeBase] Chunks used: ${selectedChunks.length} / ${allChunks.length}`);
|
||||
chatLog.info(`[KnowledgeBase] Final context length: ${context.length} chars`);
|
||||
|
||||
|
||||
// Log estatísticas por fonte
|
||||
chatLog.info(`[KnowledgeBase] Chunks per source:`);
|
||||
entries.forEach((entry, idx) => {
|
||||
const chunksUsed = selectedChunks.filter(c => c.sourceIndex === idx).length;
|
||||
chatLog.info(` ${entry.title}: ${chunksUsed} chunks`);
|
||||
});
|
||||
|
||||
|
||||
chatLog.info(`[KnowledgeBase] ============================================\n`);
|
||||
|
||||
return {
|
||||
@@ -327,7 +323,7 @@ export function validateCondensedContext(
|
||||
const normalizedContext = result.context.toLowerCase();
|
||||
const queryWords = normalizedQuery.split(/\s+/).filter(w => w.length > 3);
|
||||
const matchingWords = queryWords.filter(word => normalizedContext.includes(word));
|
||||
|
||||
|
||||
if (matchingWords.length < queryWords.length * 0.5) {
|
||||
warnings.push('Poucas palavras-chave da query aparecem no contexto');
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user