using ChatRAG.Models; using ChatRAG.Services.ResponseService; using ChatRAG.Settings; using Microsoft.Extensions.Options; namespace ChatRAG.Services.Confidence { /// /// Verifica se o RAG deve responder baseado na confiança dos resultados /// public class ConfidenceVerifier { private readonly ILogger _logger; private readonly ConfidenceSettings _settings; public ConfidenceVerifier( ILogger logger, IOptions settings) { _logger = logger; _settings = settings.Value; } /// /// Verifica se deve responder baseado na análise, resultados e contexto /// public ConfidenceResult VerifyConfidence( QueryAnalysis analysis, List results, HierarchicalContext context, bool strictMode = true, string language = "pt") { var strategy = analysis.Strategy?.ToLower() ?? "specific"; var thresholds = GetThresholds(strategy, strictMode); _logger.LogInformation("Verificando confiança - Estratégia: {Strategy}, Modo Restrito: {StrictMode}, Idioma: {Language}", strategy, strictMode, language); // Calcular métricas de confiança var metrics = CalculateConfidenceMetrics(results, context, analysis); // Verificar se deve responder baseado na estratégia var shouldRespond = ShouldRespond(strategy, metrics, thresholds); var result = new ConfidenceResult { ShouldRespond = shouldRespond, ConfidenceScore = metrics.OverallScore, Strategy = strategy, Metrics = metrics, Reason = GenerateReason(shouldRespond, strategy, metrics, thresholds, language), SuggestedResponse = shouldRespond ? null : GenerateFallbackResponse(strategy, metrics, language) }; _logger.LogInformation("Resultado da confiança: {ShouldRespond} (Score: {ConfidenceScore:P1}, Estratégia: {Strategy})", result.ShouldRespond, result.ConfidenceScore, strategy); return result; } /// /// Calcula métricas detalhadas de confiança /// private ConfidenceMetrics CalculateConfidenceMetrics( List results, HierarchicalContext context, QueryAnalysis analysis) { var relevantResults = results.Where(r => r.Score >= 0.3).ToList(); var highQualityResults = results.Where(r => r.Score >= 0.6).ToList(); var metrics = new ConfidenceMetrics { TotalDocuments = results.Count, RelevantDocuments = relevantResults.Count, HighQualityDocuments = highQualityResults.Count, AverageScore = results.Any() ? results.Average(r => r.Score) : 0, MaxScore = results.Any() ? results.Max(r => r.Score) : 0, MinScore = results.Any() ? results.Min(r => r.Score) : 0, ContextLength = context.CombinedContext?.Length ?? 0, StepsExecuted = context.Steps.Count, HasSpecificContent = HasSpecificContent(results, analysis), OverallScore = CalculateOverallScore(results, context, analysis) }; // Métricas adicionais metrics.ScoreVariance = CalculateScoreVariance(results); metrics.ContentDiversity = CalculateContentDiversity(results); metrics.ConceptCoverage = CalculateConceptCoverage(results, analysis); return metrics; } /// /// Verifica se há conteúdo específico relacionado aos conceitos da query /// private bool HasSpecificContent(List results, QueryAnalysis analysis) { if (!analysis.Concepts?.Any() == true) return results.Any(); // Se não há conceitos específicos, qualquer resultado serve var combinedContent = string.Join(" ", results.Select(r => $"{r.Title} {r.Content}")).ToLower(); var conceptsFound = analysis.Concepts.Count(concept => combinedContent.Contains(concept.ToLower())); var minConceptsRequired = Math.Max(1, Math.Min(2, analysis.Concepts.Length)); return conceptsFound >= minConceptsRequired; } /// /// Calcula score geral considerando múltiplos fatores /// private double CalculateOverallScore(List results, HierarchicalContext context, QueryAnalysis analysis) { if (!results.Any()) return 0; // Pesos para diferentes fatores const double scoreWeight = 0.35; // Qualidade dos scores const double countWeight = 0.25; // Quantidade de documentos const double contextWeight = 0.20; // Tamanho do contexto const double diversityWeight = 0.10; // Diversidade do conteúdo const double conceptWeight = 0.10; // Cobertura de conceitos // Score baseado na qualidade dos resultados var avgScore = results.Average(r => r.Score); var maxScore = results.Max(r => r.Score); var qualityScore = (avgScore * 0.7) + (maxScore * 0.3); // Média ponderada // Score baseado na quantidade (com saturação) var countScore = Math.Min(1.0, results.Count / 5.0); // Score baseado no tamanho do contexto (com saturação) var contextScore = Math.Min(1.0, (context.CombinedContext?.Length ?? 0) / 2000.0); // Score baseado na diversidade do conteúdo var diversityScore = CalculateContentDiversity(results); // Score baseado na cobertura de conceitos var conceptScore = CalculateConceptCoverage(results, analysis); var overallScore = (qualityScore * scoreWeight) + (countScore * countWeight) + (contextScore * contextWeight) + (diversityScore * diversityWeight) + (conceptScore * conceptWeight); return Math.Min(1.0, overallScore); } /// /// Calcula variância dos scores para medir consistência /// private double CalculateScoreVariance(List results) { if (results.Count < 2) return 0; var mean = results.Average(r => r.Score); var variance = results.Average(r => Math.Pow(r.Score - mean, 2)); // Normalizar para 0-1 (menor variância = melhor) return Math.Max(0, 1 - (variance * 4)); // Multiplica por 4 para normalizar } /// /// Calcula diversidade do conteúdo (documentos diferentes) /// private double CalculateContentDiversity(List results) { if (!results.Any()) return 0; // Simples heurística: títulos únicos / total var uniqueTitles = results.Select(r => r.Title?.ToLower() ?? "").Distinct().Count(); return Math.Min(1.0, (double)uniqueTitles / results.Count); } /// /// Calcula cobertura dos conceitos da query /// private double CalculateConceptCoverage(List results, QueryAnalysis analysis) { if (!analysis.Concepts?.Any() == true) return 1.0; // Se não há conceitos, assume cobertura total var combinedContent = string.Join(" ", results.Select(r => $"{r.Title} {r.Content}")).ToLower(); var conceptsFound = analysis.Concepts.Count(concept => combinedContent.Contains(concept.ToLower())); return (double)conceptsFound / analysis.Concepts.Length; } /// /// Decide se deve responder baseado na estratégia e métricas /// private bool ShouldRespond(string strategy, ConfidenceMetrics metrics, ConfidenceThresholds thresholds) { return strategy switch { "overview" => ShouldRespondOverview(metrics, thresholds), "specific" => ShouldRespondSpecific(metrics, thresholds), "detailed" => ShouldRespondDetailed(metrics, thresholds), _ => ShouldRespondSpecific(metrics, thresholds) // Default para specific }; } /// /// Lógica específica para estratégia Overview /// private bool ShouldRespondOverview(ConfidenceMetrics metrics, ConfidenceThresholds thresholds) { // Para overview, precisamos de quantidade e contexto amplo return metrics.TotalDocuments >= thresholds.MinDocuments && metrics.ContextLength >= thresholds.MinContextLength && metrics.OverallScore >= thresholds.MinOverallScore && (metrics.RelevantDocuments >= thresholds.MinRelevantDocuments || metrics.TotalDocuments >= 10); // Flexibilidade para projetos grandes } /// /// Lógica específica para estratégia Specific /// private bool ShouldRespondSpecific(ConfidenceMetrics metrics, ConfidenceThresholds thresholds) { // Para specific, precisamos de relevância e qualidade return metrics.RelevantDocuments >= thresholds.MinRelevantDocuments && metrics.MaxScore >= thresholds.MinMaxScore && metrics.OverallScore >= thresholds.MinOverallScore && metrics.HasSpecificContent; } /// /// Lógica específica para estratégia Detailed /// private bool ShouldRespondDetailed(ConfidenceMetrics metrics, ConfidenceThresholds thresholds) { // Para detailed, precisamos de alta qualidade e cobertura return metrics.HighQualityDocuments >= thresholds.MinHighQualityDocuments && metrics.AverageScore >= thresholds.MinAverageScore && metrics.HasSpecificContent && metrics.OverallScore >= thresholds.MinOverallScore && metrics.ConceptCoverage >= 0.5; // Pelo menos 50% dos conceitos cobertos } /// /// Obtém thresholds ajustados para modo restrito/relaxado /// private ConfidenceThresholds GetThresholds(string strategy, bool strictMode) { if (!_settings.Thresholds.ContainsKey(strategy)) { _logger.LogWarning("Estratégia '{Strategy}' não encontrada, usando 'specific'", strategy); strategy = "specific"; } var baseThresholds = _settings.Thresholds[strategy]; if (!strictMode) { // Modo relaxado: reduz os thresholds return new ConfidenceThresholds { MinDocuments = Math.Max(1, baseThresholds.MinDocuments - 2), MinRelevantDocuments = Math.Max(1, baseThresholds.MinRelevantDocuments - 1), MinHighQualityDocuments = Math.Max(0, baseThresholds.MinHighQualityDocuments - 1), MinContextLength = Math.Max(100, baseThresholds.MinContextLength - 500), MinOverallScore = Math.Max(0.1, baseThresholds.MinOverallScore - 0.15), MinMaxScore = Math.Max(0.2, baseThresholds.MinMaxScore - 0.15), MinAverageScore = Math.Max(0.2, baseThresholds.MinAverageScore - 0.15) }; } return baseThresholds; } /// /// Gera explicação do motivo da decisão /// private string GenerateReason(bool shouldRespond, string strategy, ConfidenceMetrics metrics, ConfidenceThresholds thresholds, string language) { if (shouldRespond) { return language == "en" ? $"Sufficient confidence for '{strategy}' strategy - Score: {metrics.OverallScore:P1}, Docs: {metrics.RelevantDocuments}/{metrics.TotalDocuments}" : $"Confiança suficiente para estratégia '{strategy}' - Score: {metrics.OverallScore:P1}, Docs: {metrics.RelevantDocuments}/{metrics.TotalDocuments}"; } var issues = new List(); if (metrics.TotalDocuments < thresholds.MinDocuments) { var msg = language == "en" ? $"few documents ({metrics.TotalDocuments} < {thresholds.MinDocuments})" : $"poucos documentos ({metrics.TotalDocuments} < {thresholds.MinDocuments})"; issues.Add(msg); } if (metrics.RelevantDocuments < thresholds.MinRelevantDocuments) { var msg = language == "en" ? $"few relevant documents ({metrics.RelevantDocuments} < {thresholds.MinRelevantDocuments})" : $"poucos documentos relevantes ({metrics.RelevantDocuments} < {thresholds.MinRelevantDocuments})"; issues.Add(msg); } if (metrics.OverallScore < thresholds.MinOverallScore) { var msg = language == "en" ? $"low overall score ({metrics.OverallScore:P1} < {thresholds.MinOverallScore:P1})" : $"score geral baixo ({metrics.OverallScore:P1} < {thresholds.MinOverallScore:P1})"; issues.Add(msg); } if (!metrics.HasSpecificContent) { var msg = language == "en" ? "no specific content found" : "conteúdo específico não encontrado"; issues.Add(msg); } var prefix = language == "en" ? "Insufficient confidence: " : "Confiança insuficiente: "; return prefix + string.Join(", ", issues); } /// /// Gera resposta de fallback apropriada /// private string GenerateFallbackResponse(string strategy, ConfidenceMetrics metrics, string language) { if (!_settings.FallbackMessages.ContainsKey(language)) { language = "pt"; // Fallback para português } var messages = _settings.FallbackMessages[language]; // Escolher mensagem baseada no problema principal if (metrics.TotalDocuments == 0) { return messages.NoDocuments; } if (metrics.RelevantDocuments == 0) { return messages.NoRelevantDocuments; } return strategy switch { "overview" => messages.InsufficientOverview, "specific" => messages.InsufficientSpecific, "detailed" => messages.InsufficientDetailed, _ => messages.Generic }; } } /// /// Resultado da verificação de confiança /// public class ConfidenceResult { public bool ShouldRespond { get; set; } public double ConfidenceScore { get; set; } public string Strategy { get; set; } = ""; public ConfidenceMetrics Metrics { get; set; } = new(); public string Reason { get; set; } = ""; public string? SuggestedResponse { get; set; } } /// /// Métricas detalhadas de confiança /// public class ConfidenceMetrics { // Métricas básicas public int TotalDocuments { get; set; } public int RelevantDocuments { get; set; } public int HighQualityDocuments { get; set; } public double AverageScore { get; set; } public double MaxScore { get; set; } public double MinScore { get; set; } public int ContextLength { get; set; } public int StepsExecuted { get; set; } public bool HasSpecificContent { get; set; } public double OverallScore { get; set; } // Métricas avançadas public double ScoreVariance { get; set; } public double ContentDiversity { get; set; } public double ConceptCoverage { get; set; } } }