376 lines
17 KiB
Plaintext
376 lines
17 KiB
Plaintext
using ChatApi;
|
|
using ChatApi.Models;
|
|
using ChatRAG.Contracts.VectorSearch;
|
|
using ChatRAG.Data;
|
|
using ChatRAG.Models;
|
|
using ChatRAG.Services.Contracts;
|
|
using Microsoft.SemanticKernel;
|
|
using Microsoft.SemanticKernel.ChatCompletion;
|
|
using Microsoft.SemanticKernel.Connectors.OpenAI;
|
|
using Microsoft.SemanticKernel.Embeddings;
|
|
|
|
#pragma warning disable SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
|
|
namespace ChatRAG.Services.ResponseService
|
|
{
|
|
public class HierarchicalRAGService : IResponseService
|
|
{
|
|
private readonly ChatHistoryService _chatHistoryService;
|
|
private readonly Kernel _kernel;
|
|
private readonly TextFilter _textFilter;
|
|
private readonly ProjectDataRepository _projectDataRepository;
|
|
private readonly IChatCompletionService _chatCompletionService;
|
|
private readonly IVectorSearchService _vectorSearchService;
|
|
private readonly ILogger<HierarchicalRAGService> _logger;
|
|
|
|
public HierarchicalRAGService(
|
|
ChatHistoryService chatHistoryService,
|
|
Kernel kernel,
|
|
TextFilter textFilter,
|
|
ProjectDataRepository projectDataRepository,
|
|
IChatCompletionService chatCompletionService,
|
|
IVectorSearchService vectorSearchService,
|
|
ILogger<HierarchicalRAGService> logger)
|
|
{
|
|
_chatHistoryService = chatHistoryService;
|
|
_kernel = kernel;
|
|
_textFilter = textFilter;
|
|
_projectDataRepository = projectDataRepository;
|
|
_chatCompletionService = chatCompletionService;
|
|
_vectorSearchService = vectorSearchService;
|
|
_logger = logger;
|
|
}
|
|
|
|
public async Task<string> GetResponse(UserData userData, string projectId, string sessionId, string question, string language = "pt")
|
|
{
|
|
var stopWatch = new System.Diagnostics.Stopwatch();
|
|
stopWatch.Start();
|
|
|
|
try
|
|
{
|
|
// 1. Análise da query para determinar estratégia
|
|
var queryAnalysis = await AnalyzeQuery(question, language);
|
|
_logger.LogInformation("Query Analysis: {Strategy}, Complexity: {Complexity}",
|
|
queryAnalysis.Strategy, queryAnalysis.Complexity);
|
|
|
|
// 2. Execução hierárquica baseada na análise
|
|
var context = await ExecuteHierarchicalSearch(question, projectId, queryAnalysis);
|
|
|
|
// 3. Geração da resposta final
|
|
var response = await GenerateResponse(question, projectId, context, sessionId, language);
|
|
|
|
stopWatch.Stop();
|
|
return $"{response}\n\nTempo: {stopWatch.ElapsedMilliseconds / 1000}s\nEtapas: {context.Steps.Count}";
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex, "Erro no RAG Hierárquico");
|
|
stopWatch.Stop();
|
|
return $"Erro: {ex.Message}\nTempo: {stopWatch.ElapsedMilliseconds / 1000}s";
|
|
}
|
|
}
|
|
|
|
private async Task<QueryAnalysis> AnalyzeQuery(string question, string language)
|
|
{
|
|
var analysisPrompt = language == "pt" ?
|
|
@"Analise esta pergunta e classifique:
|
|
PERGUNTA: ""{0}""
|
|
|
|
Responda APENAS no formato JSON:
|
|
{{
|
|
""strategy"": ""overview|specific|detailed"",
|
|
""complexity"": ""simple|medium|complex"",
|
|
""scope"": ""global|filtered|targeted"",
|
|
""concepts"": [""conceito1"", ""conceito2""],
|
|
""needs_hierarchy"": true|false
|
|
}}
|
|
|
|
REGRAS:
|
|
- overview: pergunta sobre projeto inteiro
|
|
- specific: pergunta sobre módulo/funcionalidade específica
|
|
- detailed: pergunta técnica que precisa de contexto profundo
|
|
- needs_hierarchy: true se precisar de múltiplas buscas" :
|
|
|
|
@"Analyze this question and classify:
|
|
QUESTION: ""{0}""
|
|
|
|
Answer ONLY in JSON format:
|
|
{{
|
|
""strategy"": ""overview|specific|detailed"",
|
|
""complexity"": ""simple|medium|complex"",
|
|
""scope"": ""global|filtered|targeted"",
|
|
""concepts"": [""concept1"", ""concept2""],
|
|
""needs_hierarchy"": true|false
|
|
}}
|
|
|
|
RULES:
|
|
- overview: question about entire project
|
|
- specific: question about specific module/functionality
|
|
- detailed: technical question needing deep context
|
|
- needs_hierarchy: true if needs multiple searches";
|
|
|
|
var prompt = string.Format(analysisPrompt, question);
|
|
|
|
var executionSettings = new OpenAIPromptExecutionSettings
|
|
{
|
|
Temperature = 0.1,
|
|
MaxTokens = 200
|
|
};
|
|
|
|
var response = await _chatCompletionService.GetChatMessageContentAsync(prompt, executionSettings);
|
|
|
|
try
|
|
{
|
|
var jsonResponse = response.Content?.Trim() ?? "{}";
|
|
|
|
// Extrair JSON se vier com texto extra
|
|
var startIndex = jsonResponse.IndexOf('{');
|
|
var endIndex = jsonResponse.LastIndexOf('}');
|
|
if (startIndex >= 0 && endIndex >= startIndex)
|
|
{
|
|
jsonResponse = jsonResponse.Substring(startIndex, endIndex - startIndex + 1);
|
|
}
|
|
|
|
var analysis = System.Text.Json.JsonSerializer.Deserialize<QueryAnalysis>(jsonResponse);
|
|
return analysis ?? new QueryAnalysis { Strategy = "specific", Complexity = "medium" };
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogWarning(ex, "Erro ao parsear análise da query, usando padrão");
|
|
return new QueryAnalysis { Strategy = "specific", Complexity = "medium" };
|
|
}
|
|
}
|
|
|
|
private async Task<HierarchicalContext> ExecuteHierarchicalSearch(string question, string projectId, QueryAnalysis analysis)
|
|
{
|
|
var context = new HierarchicalContext();
|
|
var embeddingService = _kernel.GetRequiredService<ITextEmbeddingGenerationService>();
|
|
|
|
switch (analysis.Strategy)
|
|
{
|
|
case "overview":
|
|
await ExecuteOverviewStrategy(context, question, projectId, embeddingService);
|
|
break;
|
|
|
|
case "detailed":
|
|
await ExecuteDetailedStrategy(context, question, projectId, embeddingService, analysis);
|
|
break;
|
|
|
|
default: // specific
|
|
await ExecuteSpecificStrategy(context, question, projectId, embeddingService);
|
|
break;
|
|
}
|
|
|
|
return context;
|
|
}
|
|
|
|
private async Task ExecuteOverviewStrategy(HierarchicalContext context, string question, string projectId, ITextEmbeddingGenerationService embeddingService)
|
|
{
|
|
// Etapa 1: Buscar resumos/títulos primeiro
|
|
context.AddStep("Buscando visão geral do projeto");
|
|
var overviewResults = await _vectorSearchService.GetDocumentsByProjectAsync(projectId);
|
|
|
|
// Etapa 2: Identificar documentos principais baseado na pergunta
|
|
var questionEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(question));
|
|
var embeddingArray = questionEmbedding.ToArray().Select(e => (double)e).ToArray();
|
|
|
|
context.AddStep("Identificando documentos relevantes");
|
|
var relevantDocs = await _vectorSearchService.SearchSimilarAsync(embeddingArray, projectId, 0.3, 5);
|
|
|
|
context.CombinedContext = $"VISÃO GERAL DO PROJETO:\n{FormatResults(overviewResults.Take(3))}\n\nDOCUMENTOS RELEVANTES:\n{FormatResults(relevantDocs)}";
|
|
}
|
|
|
|
private async Task ExecuteSpecificStrategy(HierarchicalContext context, string question, string projectId, ITextEmbeddingGenerationService embeddingService)
|
|
{
|
|
// Etapa 1: Busca inicial por similaridade
|
|
context.AddStep("Busca inicial por similaridade");
|
|
var questionEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(question));
|
|
var embeddingArray = questionEmbedding.ToArray().Select(e => (double)e).ToArray();
|
|
|
|
var initialResults = await _vectorSearchService.SearchSimilarAsync(embeddingArray, projectId, 0.4, 3);
|
|
|
|
if (initialResults.Any())
|
|
{
|
|
context.AddStep("Expandindo contexto com documentos relacionados");
|
|
|
|
// Etapa 2: Expandir com contexto relacionado
|
|
var expandedContext = await ExpandContext(initialResults, projectId, embeddingService);
|
|
context.CombinedContext = $"CONTEXTO PRINCIPAL:\n{FormatResults(initialResults)}\n\nCONTEXTO EXPANDIDO:\n{FormatResults(expandedContext)}";
|
|
}
|
|
else
|
|
{
|
|
context.AddStep("Fallback para busca ampla");
|
|
var fallbackResults = await _vectorSearchService.SearchSimilarAsync(embeddingArray, projectId, 0.2, 5);
|
|
context.CombinedContext = FormatResults(fallbackResults);
|
|
}
|
|
}
|
|
|
|
private async Task ExecuteDetailedStrategy(HierarchicalContext context, string question, string projectId, ITextEmbeddingGenerationService embeddingService, QueryAnalysis analysis)
|
|
{
|
|
var questionEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(question));
|
|
var embeddingArray = questionEmbedding.ToArray().Select(e => (double)e).ToArray();
|
|
|
|
// Etapa 1: Busca conceitual baseada nos conceitos identificados
|
|
context.AddStep("Busca conceitual inicial");
|
|
var conceptualResults = new List<VectorSearchResult>();
|
|
|
|
if (analysis.Concepts?.Any() == true)
|
|
{
|
|
foreach (var concept in analysis.Concepts)
|
|
{
|
|
var conceptEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(concept));
|
|
var conceptArray = conceptEmbedding.ToArray().Select(e => (double)e).ToArray();
|
|
var conceptResults = await _vectorSearchService.SearchSimilarAsync(conceptArray, projectId, 0.3, 2);
|
|
conceptualResults.AddRange(conceptResults);
|
|
}
|
|
}
|
|
|
|
// Etapa 2: Busca direta pela pergunta
|
|
context.AddStep("Busca direta pela pergunta");
|
|
var directResults = await _vectorSearchService.SearchSimilarAsync(embeddingArray, projectId, 0.3, 3);
|
|
|
|
// Etapa 3: Síntese intermediária para identificar lacunas
|
|
context.AddStep("Identificando lacunas de conhecimento");
|
|
var intermediateContext = FormatResults(conceptualResults.Concat(directResults).DistinctBy(r => r.Id));
|
|
var gaps = await IdentifyKnowledgeGaps(question, intermediateContext);
|
|
|
|
// Etapa 4: Busca complementar baseada nas lacunas
|
|
if (!string.IsNullOrEmpty(gaps))
|
|
{
|
|
context.AddStep("Preenchendo lacunas de conhecimento");
|
|
var gapEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(gaps));
|
|
var gapArray = gapEmbedding.ToArray().Select(e => (double)e).ToArray();
|
|
var gapResults = await _vectorSearchService.SearchSimilarAsync(gapArray, projectId, 0.25, 2);
|
|
|
|
context.CombinedContext = $"CONTEXTO CONCEITUAL:\n{FormatResults(conceptualResults)}\n\nCONTEXTO DIRETO:\n{FormatResults(directResults)}\n\nCONTEXTO COMPLEMENTAR:\n{FormatResults(gapResults)}";
|
|
}
|
|
else
|
|
{
|
|
context.CombinedContext = $"CONTEXTO CONCEITUAL:\n{FormatResults(conceptualResults)}\n\nCONTEXTO DIRETO:\n{FormatResults(directResults)}";
|
|
}
|
|
}
|
|
|
|
private async Task<List<VectorSearchResult>> ExpandContext(List<VectorSearchResult> initialResults, string projectId, ITextEmbeddingGenerationService embeddingService)
|
|
{
|
|
var expandedResults = new List<VectorSearchResult>();
|
|
|
|
// Para cada resultado inicial, buscar documentos relacionados
|
|
foreach (var result in initialResults.Take(2)) // Limitar para evitar explosão de contexto
|
|
{
|
|
var resultEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(result.Content));
|
|
var embeddingArray = resultEmbedding.ToArray().Select(e => (double)e).ToArray();
|
|
|
|
var relatedDocs = await _vectorSearchService.SearchSimilarAsync(embeddingArray, projectId, 0.4, 2);
|
|
expandedResults.AddRange(relatedDocs.Where(r => !initialResults.Any(ir => ir.Id == r.Id)));
|
|
}
|
|
|
|
return expandedResults.DistinctBy(r => r.Id).ToList();
|
|
}
|
|
|
|
private async Task<string> IdentifyKnowledgeGaps(string question, string currentContext)
|
|
{
|
|
var gapPrompt = @"Baseado na pergunta e contexto atual, identifique que informações ainda faltam para uma resposta completa.
|
|
|
|
PERGUNTA: {0}
|
|
CONTEXTO ATUAL: {1}
|
|
|
|
Responda APENAS com palavras-chave dos conceitos/informações que ainda faltam, separados por vírgula.
|
|
Se o contexto for suficiente, responda 'SUFICIENTE'.";
|
|
|
|
var prompt = string.Format(gapPrompt, question, currentContext.Substring(0, Math.Min(1000, currentContext.Length)));
|
|
|
|
var executionSettings = new OpenAIPromptExecutionSettings
|
|
{
|
|
Temperature = 0.2,
|
|
MaxTokens = 100
|
|
};
|
|
|
|
var response = await _chatCompletionService.GetChatMessageContentAsync(prompt, executionSettings);
|
|
var gaps = response.Content?.Trim() ?? "";
|
|
|
|
return gaps.Equals("SUFICIENTE", StringComparison.OrdinalIgnoreCase) ? "" : gaps;
|
|
}
|
|
|
|
private async Task<string> GenerateResponse(string question, string projectId, HierarchicalContext context, string sessionId, string language)
|
|
{
|
|
var projectData = await _projectDataRepository.GetAsync(projectId);
|
|
var project = $"Nome: {projectData.Nome} \n\n Descrição:{projectData.Descricao}";
|
|
|
|
var prompt = language == "pt" ?
|
|
@"Você é um especialista em análise de software e QA.
|
|
|
|
PROJETO: {0}
|
|
PERGUNTA: ""{1}""
|
|
CONTEXTO HIERÁRQUICO: {2}
|
|
ETAPAS EXECUTADAS: {3}
|
|
|
|
Responda à pergunta de forma precisa e estruturada, aproveitando todo o contexto hierárquico coletado." :
|
|
|
|
@"You are a software analysis and QA expert.
|
|
|
|
PROJECT: {0}
|
|
QUESTION: ""{1}""
|
|
HIERARCHICAL CONTEXT: {2}
|
|
EXECUTED STEPS: {3}
|
|
|
|
Answer the question precisely and structured, leveraging all the hierarchical context collected.";
|
|
|
|
var finalPrompt = string.Format(prompt, project, question, context.CombinedContext,
|
|
string.Join(" → ", context.Steps));
|
|
|
|
var history = _chatHistoryService.GetSumarizer(sessionId);
|
|
history.AddUserMessage(finalPrompt);
|
|
|
|
var executionSettings = new OpenAIPromptExecutionSettings
|
|
{
|
|
Temperature = 0.7,
|
|
TopP = 1.0,
|
|
FrequencyPenalty = 0,
|
|
PresencePenalty = 0
|
|
};
|
|
|
|
var response = await _chatCompletionService.GetChatMessageContentAsync(history, executionSettings);
|
|
history.AddMessage(response.Role, response.Content ?? "");
|
|
_chatHistoryService.UpdateHistory(sessionId, history);
|
|
|
|
return response.Content ?? "";
|
|
}
|
|
|
|
private string FormatResults(IEnumerable<VectorSearchResult> results)
|
|
{
|
|
return string.Join("\n\n", results.Select((item, index) =>
|
|
$"=== DOCUMENTO {index + 1} ===\n" +
|
|
$"Relevância: {item.Score:P1}\n" +
|
|
$"Conteúdo: {item.Content}"));
|
|
}
|
|
|
|
public Task<string> GetResponse(UserData userData, string projectId, string sessionId, string question)
|
|
{
|
|
return GetResponse(userData, projectId, sessionId, question, "pt");
|
|
}
|
|
}
|
|
|
|
// Classes de apoio para o RAG Hierárquico
|
|
public class QueryAnalysis
|
|
{
|
|
public string Strategy { get; set; } = "specific";
|
|
public string Complexity { get; set; } = "medium";
|
|
public string Scope { get; set; } = "filtered";
|
|
public string[] Concepts { get; set; } = Array.Empty<string>();
|
|
public bool Needs_Hierarchy { get; set; } = false;
|
|
}
|
|
|
|
public class HierarchicalContext
|
|
{
|
|
public List<string> Steps { get; set; } = new();
|
|
public string CombinedContext { get; set; } = "";
|
|
public Dictionary<string, object> Metadata { get; set; } = new();
|
|
|
|
public void AddStep(string step)
|
|
{
|
|
Steps.Add($"{DateTime.Now:HH:mm:ss} - {step}");
|
|
}
|
|
}
|
|
|
|
}
|
|
#pragma warning restore SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
|