ChatRAG/Services/ResponseService/HierarchicalRAGService.cs
2025-06-21 14:20:07 -03:00

547 lines
25 KiB
C#

using ChatApi;
using ChatApi.Models;
using ChatRAG.Contracts.VectorSearch;
using ChatRAG.Data;
using ChatRAG.Models;
using ChatRAG.Services.Contracts;
using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.ChatCompletion;
using Microsoft.SemanticKernel.Connectors.OpenAI;
using Microsoft.SemanticKernel.Embeddings;
using System.Text.Json;
#pragma warning disable SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
namespace ChatRAG.Services.ResponseService
{
public class HierarchicalRAGService : IResponseService
{
private readonly ChatHistoryService _chatHistoryService;
private readonly Kernel _kernel;
private readonly TextFilter _textFilter;
private readonly IProjectDataRepository _projectDataRepository;
private readonly IChatCompletionService _chatCompletionService;
private readonly IVectorSearchService _vectorSearchService;
private readonly ILogger<HierarchicalRAGService> _logger;
public HierarchicalRAGService(
ChatHistoryService chatHistoryService,
Kernel kernel,
TextFilter textFilter,
IProjectDataRepository projectDataRepository,
IChatCompletionService chatCompletionService,
IVectorSearchService vectorSearchService,
ILogger<HierarchicalRAGService> logger)
{
_chatHistoryService = chatHistoryService;
_kernel = kernel;
_textFilter = textFilter;
_projectDataRepository = projectDataRepository;
_chatCompletionService = chatCompletionService;
_vectorSearchService = vectorSearchService;
_logger = logger;
}
public async Task<string> GetResponse(UserData userData, string projectId, string sessionId, string question, string language = "pt")
{
var stopWatch = new System.Diagnostics.Stopwatch();
stopWatch.Start();
try
{
// 1. Análise da query para determinar estratégia
var queryAnalysis = await AnalyzeQuery(question, language);
_logger.LogInformation("Query Analysis: {Strategy}, Complexity: {Complexity}",
queryAnalysis.Strategy, queryAnalysis.Complexity);
// 2. Execução hierárquica baseada na análise
var context = await ExecuteHierarchicalSearch(question, projectId, queryAnalysis);
// 3. Geração da resposta final
var response = await GenerateResponse(question, projectId, context, sessionId, language);
stopWatch.Stop();
return $"{response}\n\nTempo: {stopWatch.ElapsedMilliseconds / 1000}s\nEtapas: {context.Steps.Count}";
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro no RAG Hierárquico");
stopWatch.Stop();
return $"Erro: {ex.Message}\nTempo: {stopWatch.ElapsedMilliseconds / 1000}s";
}
}
private async Task<QueryAnalysis> AnalyzeQuery(string question, string language)
{
var analysisPrompt = language == "pt" ?
@"Analise esta pergunta e classifique com precisão:
PERGUNTA: ""{0}""
Responda APENAS no formato JSON:
{{
""strategy"": ""overview|specific|detailed"",
""complexity"": ""simple|medium|complex"",
""scope"": ""global|filtered|targeted"",
""concepts"": [""conceito1"", ""conceito2""],
""needs_hierarchy"": true|false
}}
DEFINIÇÕES PRECISAS:
STRATEGY:
- overview: Pergunta sobre o PROJETO COMO UM TODO. Palavras-chave: ""projeto"", ""sistema"", ""aplicação"", ""este projeto"", ""todo o"", ""geral"", ""inteiro"". NÃO menciona módulos, funcionalidades ou tecnologias específicas.
- specific: Pergunta sobre MÓDULO/FUNCIONALIDADE ESPECÍFICA. Menciona: nome de classe, controller, entidade, CRUD específico, funcionalidade particular, tecnologia específica.
- detailed: Pergunta técnica específica que precisa de CONTEXTO PROFUNDO e detalhes de implementação.
SCOPE:
- global: Busca informações de TODO o projeto (usar com overview)
- filtered: Busca com filtros específicos (usar com specific/detailed)
- targeted: Busca muito específica e direcionada
EXEMPLOS:
- ""Gere casos de teste para este projeto"" → overview/global
- ""Gere casos de teste do projeto"" → overview/global
- ""Gere casos de teste para o CRUD de usuário"" → specific/filtered
- ""Como implementar autenticação JWT neste controller"" → detailed/targeted
- ""Documente este sistema"" → overview/global
- ""Explique a classe UserService"" → specific/filtered" :
@"Analyze this question and classify precisely:
QUESTION: ""{0}""
Answer ONLY in JSON format:
{{
""strategy"": ""overview|specific|detailed"",
""complexity"": ""simple|medium|complex"",
""scope"": ""global|filtered|targeted"",
""concepts"": [""concept1"", ""concept2""],
""needs_hierarchy"": true|false
}}
PRECISE DEFINITIONS:
STRATEGY:
- overview: Question about the PROJECT AS A WHOLE. Keywords: ""project"", ""system"", ""application"", ""this project"", ""entire"", ""general"", ""whole"". Does NOT mention specific modules, functionalities or technologies.
- specific: Question about SPECIFIC MODULE/FUNCTIONALITY. Mentions: class name, controller, entity, specific CRUD, particular functionality, specific technology.
- detailed: Technical specific question needing DEEP CONTEXT and implementation details.
SCOPE:
- global: Search information from ENTIRE project (use with overview)
- filtered: Search with specific filters (use with specific/detailed)
- targeted: Very specific and directed search
EXAMPLES:
- ""Generate test cases for this project"" → overview/global
- ""Generate test cases for user CRUD"" → specific/filtered
- ""How to implement JWT authentication in this controller"" → detailed/targeted
- ""Document this system"" → overview/global
- ""Explain the UserService class"" → specific/filtered";
var prompt = string.Format(analysisPrompt, question);
var executionSettings = new OpenAIPromptExecutionSettings
{
Temperature = 0.1,
MaxTokens = 300 // Aumentei um pouco para acomodar o prompt maior
};
var response = await _chatCompletionService.GetChatMessageContentAsync(prompt, executionSettings);
try
{
var jsonResponse = response.Content?.Trim() ?? "{}";
// Extrair JSON se vier com texto extra
var startIndex = jsonResponse.IndexOf('{');
var endIndex = jsonResponse.LastIndexOf('}');
if (startIndex >= 0 && endIndex >= startIndex)
{
jsonResponse = jsonResponse.Substring(startIndex, endIndex - startIndex + 1);
}
var options = new JsonSerializerOptions
{
PropertyNameCaseInsensitive = true,
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
};
var analysis = System.Text.Json.JsonSerializer.Deserialize<QueryAnalysis>(jsonResponse, options);
// Log para debug - remover em produção
_logger.LogInformation($"Query: '{question}' → Strategy: {analysis?.Strategy}, Scope: {analysis?.Scope}");
return analysis ?? new QueryAnalysis { Strategy = "specific", Complexity = "medium" };
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Erro ao parsear análise da query, usando padrão");
return new QueryAnalysis { Strategy = "specific", Complexity = "medium" };
}
}
private async Task<HierarchicalContext> ExecuteHierarchicalSearch(string question, string projectId, QueryAnalysis analysis)
{
var context = new HierarchicalContext();
var embeddingService = _kernel.GetRequiredService<ITextEmbeddingGenerationService>();
switch (analysis.Strategy)
{
case "overview":
await ExecuteOverviewStrategy(context, question, projectId, embeddingService);
break;
case "detailed":
await ExecuteDetailedStrategy(context, question, projectId, embeddingService, analysis);
break;
default: // specific
await ExecuteSpecificStrategy(context, question, projectId, embeddingService);
break;
}
return context;
}
private async Task ExecuteOverviewStrategy(HierarchicalContext context, string question, string projectId, ITextEmbeddingGenerationService embeddingService)
{
// Etapa 1: Buscar TODOS os documentos do projeto
context.AddStep("Buscando todos os documentos do projeto");
var allProjectDocs = await _vectorSearchService.GetDocumentsByProjectAsync(projectId);
// Etapa 2: Categorizar documentos por tipo/importância
context.AddStep("Categorizando e resumindo contexto do projeto");
// Etapa 2: Categorizar documentos por tipo baseado nos seus dados reais
context.AddStep("Categorizando e resumindo contexto do projeto");
var requirementsDocs = allProjectDocs.Where(d =>
d.Title.ToLower().StartsWith("requisito") ||
d.Title.ToLower().Contains("requisito") ||
d.Content.ToLower().Contains("requisito") ||
d.Content.ToLower().Contains("funcionalidade") ||
d.Content.ToLower().Contains("aplicação deve") ||
d.Content.ToLower().Contains("sistema deve")).ToList();
var architectureDocs = allProjectDocs.Where(d =>
d.Title.ToLower().Contains("arquitetura") ||
d.Title.ToLower().Contains("estrutura") ||
d.Title.ToLower().Contains("documentação") ||
d.Title.ToLower().Contains("readme") ||
d.Content.ToLower().Contains("arquitetura") ||
d.Content.ToLower().Contains("estrutura") ||
d.Content.ToLower().Contains("tecnologia")).ToList();
// Documentos que não são requisitos nem arquitetura (códigos, outros docs)
var otherDocs = allProjectDocs
.Except(requirementsDocs)
.Except(architectureDocs)
.ToList();
// Etapa 3: Resumir cada categoria se tiver muitos documentos
var requirementsSummary = await SummarizeDocuments(requirementsDocs, "requisitos e funcionalidades do projeto");
var architectureSummary = await SummarizeDocuments(architectureDocs, "arquitetura e documentação técnica");
var otherSummary = await SummarizeDocuments(otherDocs, "outros documentos do projeto");
// Etapa 4: Busca específica para a pergunta (mantém precisão)
var questionEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(question));
var embeddingArray = questionEmbedding.ToArray().Select(e => (double)e).ToArray();
context.AddStep("Identificando documentos específicos para a pergunta");
var relevantDocs = await _vectorSearchService.SearchSimilarAsync(embeddingArray, projectId, 0.3, 8);
// Etapa 5: Combinar resumos + documentos específicos
var contextParts = new List<string>();
if (!string.IsNullOrEmpty(requirementsSummary))
contextParts.Add($"RESUMO DOS REQUISITOS E FUNCIONALIDADES:\n{requirementsSummary}");
if (!string.IsNullOrEmpty(architectureSummary))
contextParts.Add($"RESUMO DA ARQUITETURA E DOCUMENTAÇÃO:\n{architectureSummary}");
if (!string.IsNullOrEmpty(otherSummary))
contextParts.Add($"OUTROS DOCUMENTOS DO PROJETO:\n{otherSummary}");
contextParts.Add($"DOCUMENTOS MAIS RELEVANTES PARA A PERGUNTA:\n{FormatResults(relevantDocs)}");
context.CombinedContext = string.Join("\n\n", contextParts);
}
private async Task<string> SummarizeDocuments(List<VectorSearchResult> documents, string category)
{
if (!documents.Any()) return string.Empty;
// Se poucos documentos, usar todos sem resumir
if (documents.Count <= 3)
{
return FormatResults(documents);
}
// Se muitos documentos, resumir em chunks
var chunks = documents.Chunk(5).ToList(); // Grupos de 5 documentos
var tasks = new List<Task<string>>();
// Semáforo para controlar concorrência (máximo 3 chamadas simultâneas)
var semaphore = new SemaphoreSlim(3, 3);
foreach (var chunk in chunks)
{
var chunkContent = FormatResults(chunk);
tasks.Add(Task.Run(async () =>
{
await semaphore.WaitAsync();
try
{
var summaryPrompt = $@"Resuma os pontos principais destes documentos sobre {category}:
{chunkContent}
Responda apenas com uma lista concisa dos pontos mais importantes:";
var response = await _chatCompletionService.GetChatMessageContentAsync(
summaryPrompt,
new OpenAIPromptExecutionSettings
{
Temperature = 0.1,
MaxTokens = 300
});
return response.Content ?? string.Empty;
}
catch (Exception ex)
{
_logger.LogWarning(ex, $"Erro ao resumir chunk de {category}, usando conteúdo original");
return chunkContent;
}
finally
{
semaphore.Release();
}
}));
}
// Aguardar todas as tasks de resumo
var summaries = await Task.WhenAll(tasks);
var validSummaries = summaries.Where(s => !string.IsNullOrEmpty(s)).ToList();
// Se tiver múltiplos resumos, consolidar
if (validSummaries.Count > 1)
{
var consolidationPrompt = $@"Consolide estes resumos sobre {category} em um resumo final:
{string.Join("\n\n", validSummaries)}
Responda com os pontos mais importantes organizados:";
try
{
var finalResponse = await _chatCompletionService.GetChatMessageContentAsync(
consolidationPrompt,
new OpenAIPromptExecutionSettings
{
Temperature = 0.1,
MaxTokens = 400
});
return finalResponse.Content ?? string.Empty;
}
catch
{
return string.Join("\n\n", validSummaries);
}
}
return validSummaries.FirstOrDefault() ?? string.Empty;
}
private async Task ExecuteSpecificStrategy(HierarchicalContext context, string question, string projectId, ITextEmbeddingGenerationService embeddingService)
{
// Etapa 1: Busca inicial por similaridade
context.AddStep("Busca inicial por similaridade");
var questionEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(question));
var embeddingArray = questionEmbedding.ToArray().Select(e => (double)e).ToArray();
var initialResults = await _vectorSearchService.SearchSimilarAsync(embeddingArray, projectId, 0.4, 3);
if (initialResults.Any())
{
context.AddStep("Expandindo contexto com documentos relacionados");
// Etapa 2: Expandir com contexto relacionado
var expandedContext = await ExpandContext(initialResults, projectId, embeddingService);
context.CombinedContext = $"CONTEXTO PRINCIPAL:\n{FormatResults(initialResults)}\n\nCONTEXTO EXPANDIDO:\n{FormatResults(expandedContext)}";
}
else
{
context.AddStep("Fallback para busca ampla");
var fallbackResults = await _vectorSearchService.SearchSimilarAsync(embeddingArray, projectId, 0.2, 5);
context.CombinedContext = FormatResults(fallbackResults);
}
}
private async Task ExecuteDetailedStrategy(HierarchicalContext context, string question, string projectId, ITextEmbeddingGenerationService embeddingService, QueryAnalysis analysis)
{
var questionEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(question));
var embeddingArray = questionEmbedding.ToArray().Select(e => (double)e).ToArray();
// Etapa 1: Busca conceitual baseada nos conceitos identificados
context.AddStep("Busca conceitual inicial");
var conceptualResults = new List<VectorSearchResult>();
if (analysis.Concepts?.Any() == true)
{
foreach (var concept in analysis.Concepts)
{
var conceptEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(concept));
var conceptArray = conceptEmbedding.ToArray().Select(e => (double)e).ToArray();
var conceptResults = await _vectorSearchService.SearchSimilarAsync(conceptArray, projectId, 0.3, 2);
conceptualResults.AddRange(conceptResults);
}
}
// Etapa 2: Busca direta pela pergunta
context.AddStep("Busca direta pela pergunta");
var directResults = await _vectorSearchService.SearchSimilarAsync(embeddingArray, projectId, 0.3, 3);
// Etapa 3: Síntese intermediária para identificar lacunas
context.AddStep("Identificando lacunas de conhecimento");
var intermediateContext = FormatResults(conceptualResults.Concat(directResults).DistinctBy(r => r.Id));
var gaps = await IdentifyKnowledgeGaps(question, intermediateContext);
// Etapa 4: Busca complementar baseada nas lacunas
if (!string.IsNullOrEmpty(gaps))
{
context.AddStep("Preenchendo lacunas de conhecimento");
var gapEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(gaps));
var gapArray = gapEmbedding.ToArray().Select(e => (double)e).ToArray();
var gapResults = await _vectorSearchService.SearchSimilarAsync(gapArray, projectId, 0.25, 2);
context.CombinedContext = $"CONTEXTO CONCEITUAL:\n{FormatResults(conceptualResults)}\n\nCONTEXTO DIRETO:\n{FormatResults(directResults)}\n\nCONTEXTO COMPLEMENTAR:\n{FormatResults(gapResults)}";
}
else
{
context.CombinedContext = $"CONTEXTO CONCEITUAL:\n{FormatResults(conceptualResults)}\n\nCONTEXTO DIRETO:\n{FormatResults(directResults)}";
}
}
private async Task<List<VectorSearchResult>> ExpandContext(List<VectorSearchResult> initialResults, string projectId, ITextEmbeddingGenerationService embeddingService)
{
var expandedResults = new List<VectorSearchResult>();
// Para cada resultado inicial, buscar documentos relacionados
foreach (var result in initialResults.Take(2)) // Limitar para evitar explosão de contexto
{
var resultEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(result.Content));
var embeddingArray = resultEmbedding.ToArray().Select(e => (double)e).ToArray();
var relatedDocs = await _vectorSearchService.SearchSimilarAsync(embeddingArray, projectId, 0.4, 2);
expandedResults.AddRange(relatedDocs.Where(r => !initialResults.Any(ir => ir.Id == r.Id)));
}
return expandedResults.DistinctBy(r => r.Id).ToList();
}
private async Task<string> IdentifyKnowledgeGaps(string question, string currentContext)
{
var gapPrompt = @"Baseado na pergunta e contexto atual, identifique que informações ainda faltam para uma resposta completa.
PERGUNTA: {0}
CONTEXTO ATUAL: {1}
Responda APENAS com palavras-chave dos conceitos/informações que ainda faltam, separados por vírgula.
Se o contexto for suficiente, responda 'SUFICIENTE'.";
var prompt = string.Format(gapPrompt, question, currentContext.Substring(0, Math.Min(1000, currentContext.Length)));
var executionSettings = new OpenAIPromptExecutionSettings
{
Temperature = 0.2,
MaxTokens = 100
};
var response = await _chatCompletionService.GetChatMessageContentAsync(prompt, executionSettings);
var gaps = response.Content?.Trim() ?? "";
return gaps.Equals("SUFICIENTE", StringComparison.OrdinalIgnoreCase) ? "" : gaps;
}
private async Task<string> GenerateResponse(string question, string projectId, HierarchicalContext context, string sessionId, string language)
{
var projectData = await _projectDataRepository.GetAsync(projectId);
var project = $"Nome: {projectData.Nome} \n\n Descrição:{projectData.Descricao}";
var prompt = language == "pt" ?
@"Você é um especialista em análise de software e QA.
PROJETO: {0}
PERGUNTA: ""{1}""
CONTEXTO HIERÁRQUICO: {2}
ETAPAS EXECUTADAS: {3}
Responda à pergunta de forma precisa e estruturada, aproveitando todo o contexto hierárquico coletado." :
@"You are a software analysis and QA expert.
PROJECT: {0}
QUESTION: ""{1}""
HIERARCHICAL CONTEXT: {2}
EXECUTED STEPS: {3}
Answer the question precisely and structured, leveraging all the hierarchical context collected.";
var finalPrompt = string.Format(prompt, project, question, context.CombinedContext,
string.Join(" → ", context.Steps));
var history = _chatHistoryService.GetSumarizer(sessionId);
history.AddUserMessage(finalPrompt);
var executionSettings = new OpenAIPromptExecutionSettings
{
Temperature = 0.7,
TopP = 1.0,
FrequencyPenalty = 0,
PresencePenalty = 0
};
var response = await _chatCompletionService.GetChatMessageContentAsync(history, executionSettings);
history.AddMessage(response.Role, response.Content ?? "");
_chatHistoryService.UpdateHistory(sessionId, history);
return response.Content ?? "";
}
private string FormatResults(IEnumerable<VectorSearchResult> results)
{
return string.Join("\n\n", results.Select((item, index) =>
$"=== DOCUMENTO {index + 1} ===\n" +
$"Relevância: {item.Score:P1}\n" +
$"Conteúdo: {item.Content}"));
}
public Task<string> GetResponse(UserData userData, string projectId, string sessionId, string question)
{
return GetResponse(userData, projectId, sessionId, question, "pt");
}
}
// Classes de apoio para o RAG Hierárquico
public class QueryAnalysis
{
public string Strategy { get; set; } = "specific";
public string Complexity { get; set; } = "medium";
public string Scope { get; set; } = "filtered";
public string[] Concepts { get; set; } = Array.Empty<string>();
public bool Needs_Hierarchy { get; set; } = false;
}
public class HierarchicalContext
{
public List<string> Steps { get; set; } = new();
public string CombinedContext { get; set; } = "";
public Dictionary<string, object> Metadata { get; set; } = new();
public void AddStep(string step)
{
Steps.Add($"{DateTime.Now:HH:mm:ss} - {step}");
}
}
}
#pragma warning restore SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.