390 lines
16 KiB
C#
390 lines
16 KiB
C#
using ChatRAG.Models;
|
|
using ChatRAG.Services.ResponseService;
|
|
using ChatRAG.Settings;
|
|
using Microsoft.Extensions.Options;
|
|
|
|
namespace ChatRAG.Services.Confidence
|
|
{
|
|
/// <summary>
|
|
/// Verifica se o RAG deve responder baseado na confiança dos resultados
|
|
/// </summary>
|
|
public class ConfidenceVerifier
|
|
{
|
|
private readonly ILogger<ConfidenceVerifier> _logger;
|
|
private readonly ConfidenceSettings _settings;
|
|
|
|
public ConfidenceVerifier(
|
|
ILogger<ConfidenceVerifier> logger,
|
|
IOptions<ConfidenceSettings> settings)
|
|
{
|
|
_logger = logger;
|
|
_settings = settings.Value;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Verifica se deve responder baseado na análise, resultados e contexto
|
|
/// </summary>
|
|
public ConfidenceResult VerifyConfidence(
|
|
QueryAnalysis analysis,
|
|
List<VectorSearchResult> results,
|
|
HierarchicalContext context,
|
|
bool strictMode = true,
|
|
string language = "pt")
|
|
{
|
|
var strategy = analysis.Strategy?.ToLower() ?? "specific";
|
|
var thresholds = GetThresholds(strategy, strictMode);
|
|
|
|
_logger.LogInformation("Verificando confiança - Estratégia: {Strategy}, Modo Restrito: {StrictMode}, Idioma: {Language}",
|
|
strategy, strictMode, language);
|
|
|
|
// Calcular métricas de confiança
|
|
var metrics = CalculateConfidenceMetrics(results, context, analysis);
|
|
|
|
// Verificar se deve responder baseado na estratégia
|
|
var shouldRespond = ShouldRespond(strategy, metrics, thresholds);
|
|
|
|
var result = new ConfidenceResult
|
|
{
|
|
ShouldRespond = shouldRespond,
|
|
ConfidenceScore = metrics.OverallScore,
|
|
Strategy = strategy,
|
|
Metrics = metrics,
|
|
Reason = GenerateReason(shouldRespond, strategy, metrics, thresholds, language),
|
|
SuggestedResponse = shouldRespond ? null : GenerateFallbackResponse(strategy, metrics, language)
|
|
};
|
|
|
|
_logger.LogInformation("Resultado da confiança: {ShouldRespond} (Score: {ConfidenceScore:P1}, Estratégia: {Strategy})",
|
|
result.ShouldRespond, result.ConfidenceScore, strategy);
|
|
|
|
return result;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Calcula métricas detalhadas de confiança
|
|
/// </summary>
|
|
private ConfidenceMetrics CalculateConfidenceMetrics(
|
|
List<VectorSearchResult> results,
|
|
HierarchicalContext context,
|
|
QueryAnalysis analysis)
|
|
{
|
|
var relevantResults = results.Where(r => r.Score >= 0.3).ToList();
|
|
var highQualityResults = results.Where(r => r.Score >= 0.6).ToList();
|
|
|
|
var metrics = new ConfidenceMetrics
|
|
{
|
|
TotalDocuments = results.Count,
|
|
RelevantDocuments = relevantResults.Count,
|
|
HighQualityDocuments = highQualityResults.Count,
|
|
AverageScore = results.Any() ? results.Average(r => r.Score) : 0,
|
|
MaxScore = results.Any() ? results.Max(r => r.Score) : 0,
|
|
MinScore = results.Any() ? results.Min(r => r.Score) : 0,
|
|
ContextLength = context.CombinedContext?.Length ?? 0,
|
|
StepsExecuted = context.Steps.Count,
|
|
HasSpecificContent = HasSpecificContent(results, analysis),
|
|
OverallScore = CalculateOverallScore(results, context, analysis)
|
|
};
|
|
|
|
// Métricas adicionais
|
|
metrics.ScoreVariance = CalculateScoreVariance(results);
|
|
metrics.ContentDiversity = CalculateContentDiversity(results);
|
|
metrics.ConceptCoverage = CalculateConceptCoverage(results, analysis);
|
|
|
|
return metrics;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Verifica se há conteúdo específico relacionado aos conceitos da query
|
|
/// </summary>
|
|
private bool HasSpecificContent(List<VectorSearchResult> results, QueryAnalysis analysis)
|
|
{
|
|
if (!analysis.Concepts?.Any() == true)
|
|
return results.Any(); // Se não há conceitos específicos, qualquer resultado serve
|
|
|
|
var combinedContent = string.Join(" ", results.Select(r => $"{r.Title} {r.Content}")).ToLower();
|
|
var conceptsFound = analysis.Concepts.Count(concept =>
|
|
combinedContent.Contains(concept.ToLower()));
|
|
|
|
var minConceptsRequired = Math.Max(1, Math.Min(2, analysis.Concepts.Length));
|
|
return conceptsFound >= minConceptsRequired;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Calcula score geral considerando múltiplos fatores
|
|
/// </summary>
|
|
private double CalculateOverallScore(List<VectorSearchResult> results, HierarchicalContext context, QueryAnalysis analysis)
|
|
{
|
|
if (!results.Any()) return 0;
|
|
|
|
// Pesos para diferentes fatores
|
|
const double scoreWeight = 0.35; // Qualidade dos scores
|
|
const double countWeight = 0.25; // Quantidade de documentos
|
|
const double contextWeight = 0.20; // Tamanho do contexto
|
|
const double diversityWeight = 0.10; // Diversidade do conteúdo
|
|
const double conceptWeight = 0.10; // Cobertura de conceitos
|
|
|
|
// Score baseado na qualidade dos resultados
|
|
var avgScore = results.Average(r => r.Score);
|
|
var maxScore = results.Max(r => r.Score);
|
|
var qualityScore = (avgScore * 0.7) + (maxScore * 0.3); // Média ponderada
|
|
|
|
// Score baseado na quantidade (com saturação)
|
|
var countScore = Math.Min(1.0, results.Count / 5.0);
|
|
|
|
// Score baseado no tamanho do contexto (com saturação)
|
|
var contextScore = Math.Min(1.0, (context.CombinedContext?.Length ?? 0) / 2000.0);
|
|
|
|
// Score baseado na diversidade do conteúdo
|
|
var diversityScore = CalculateContentDiversity(results);
|
|
|
|
// Score baseado na cobertura de conceitos
|
|
var conceptScore = CalculateConceptCoverage(results, analysis);
|
|
|
|
var overallScore = (qualityScore * scoreWeight) +
|
|
(countScore * countWeight) +
|
|
(contextScore * contextWeight) +
|
|
(diversityScore * diversityWeight) +
|
|
(conceptScore * conceptWeight);
|
|
|
|
return Math.Min(1.0, overallScore);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Calcula variância dos scores para medir consistência
|
|
/// </summary>
|
|
private double CalculateScoreVariance(List<VectorSearchResult> results)
|
|
{
|
|
if (results.Count < 2) return 0;
|
|
|
|
var mean = results.Average(r => r.Score);
|
|
var variance = results.Average(r => Math.Pow(r.Score - mean, 2));
|
|
|
|
// Normalizar para 0-1 (menor variância = melhor)
|
|
return Math.Max(0, 1 - (variance * 4)); // Multiplica por 4 para normalizar
|
|
}
|
|
|
|
/// <summary>
|
|
/// Calcula diversidade do conteúdo (documentos diferentes)
|
|
/// </summary>
|
|
private double CalculateContentDiversity(List<VectorSearchResult> results)
|
|
{
|
|
if (!results.Any()) return 0;
|
|
|
|
// Simples heurística: títulos únicos / total
|
|
var uniqueTitles = results.Select(r => r.Title?.ToLower() ?? "").Distinct().Count();
|
|
return Math.Min(1.0, (double)uniqueTitles / results.Count);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Calcula cobertura dos conceitos da query
|
|
/// </summary>
|
|
private double CalculateConceptCoverage(List<VectorSearchResult> results, QueryAnalysis analysis)
|
|
{
|
|
if (!analysis.Concepts?.Any() == true) return 1.0; // Se não há conceitos, assume cobertura total
|
|
|
|
var combinedContent = string.Join(" ", results.Select(r => $"{r.Title} {r.Content}")).ToLower();
|
|
var conceptsFound = analysis.Concepts.Count(concept =>
|
|
combinedContent.Contains(concept.ToLower()));
|
|
|
|
return (double)conceptsFound / analysis.Concepts.Length;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Decide se deve responder baseado na estratégia e métricas
|
|
/// </summary>
|
|
private bool ShouldRespond(string strategy, ConfidenceMetrics metrics, ConfidenceThresholds thresholds)
|
|
{
|
|
return strategy switch
|
|
{
|
|
"overview" => ShouldRespondOverview(metrics, thresholds),
|
|
"specific" => ShouldRespondSpecific(metrics, thresholds),
|
|
"detailed" => ShouldRespondDetailed(metrics, thresholds),
|
|
_ => ShouldRespondSpecific(metrics, thresholds) // Default para specific
|
|
};
|
|
}
|
|
|
|
/// <summary>
|
|
/// Lógica específica para estratégia Overview
|
|
/// </summary>
|
|
private bool ShouldRespondOverview(ConfidenceMetrics metrics, ConfidenceThresholds thresholds)
|
|
{
|
|
// Para overview, precisamos de quantidade e contexto amplo
|
|
return metrics.TotalDocuments >= thresholds.MinDocuments &&
|
|
metrics.ContextLength >= thresholds.MinContextLength &&
|
|
metrics.OverallScore >= thresholds.MinOverallScore &&
|
|
(metrics.RelevantDocuments >= thresholds.MinRelevantDocuments || metrics.TotalDocuments >= 10); // Flexibilidade para projetos grandes
|
|
}
|
|
|
|
/// <summary>
|
|
/// Lógica específica para estratégia Specific
|
|
/// </summary>
|
|
private bool ShouldRespondSpecific(ConfidenceMetrics metrics, ConfidenceThresholds thresholds)
|
|
{
|
|
// Para specific, precisamos de relevância e qualidade
|
|
return metrics.RelevantDocuments >= thresholds.MinRelevantDocuments &&
|
|
metrics.MaxScore >= thresholds.MinMaxScore &&
|
|
metrics.OverallScore >= thresholds.MinOverallScore &&
|
|
metrics.HasSpecificContent;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Lógica específica para estratégia Detailed
|
|
/// </summary>
|
|
private bool ShouldRespondDetailed(ConfidenceMetrics metrics, ConfidenceThresholds thresholds)
|
|
{
|
|
// Para detailed, precisamos de alta qualidade e cobertura
|
|
return metrics.HighQualityDocuments >= thresholds.MinHighQualityDocuments &&
|
|
metrics.AverageScore >= thresholds.MinAverageScore &&
|
|
metrics.HasSpecificContent &&
|
|
metrics.OverallScore >= thresholds.MinOverallScore &&
|
|
metrics.ConceptCoverage >= 0.5; // Pelo menos 50% dos conceitos cobertos
|
|
}
|
|
|
|
/// <summary>
|
|
/// Obtém thresholds ajustados para modo restrito/relaxado
|
|
/// </summary>
|
|
private ConfidenceThresholds GetThresholds(string strategy, bool strictMode)
|
|
{
|
|
if (!_settings.Thresholds.ContainsKey(strategy))
|
|
{
|
|
_logger.LogWarning("Estratégia '{Strategy}' não encontrada, usando 'specific'", strategy);
|
|
strategy = "specific";
|
|
}
|
|
|
|
var baseThresholds = _settings.Thresholds[strategy];
|
|
|
|
if (!strictMode)
|
|
{
|
|
// Modo relaxado: reduz os thresholds
|
|
return new ConfidenceThresholds
|
|
{
|
|
MinDocuments = Math.Max(1, baseThresholds.MinDocuments - 2),
|
|
MinRelevantDocuments = Math.Max(1, baseThresholds.MinRelevantDocuments - 1),
|
|
MinHighQualityDocuments = Math.Max(0, baseThresholds.MinHighQualityDocuments - 1),
|
|
MinContextLength = Math.Max(100, baseThresholds.MinContextLength - 500),
|
|
MinOverallScore = Math.Max(0.1, baseThresholds.MinOverallScore - 0.15),
|
|
MinMaxScore = Math.Max(0.2, baseThresholds.MinMaxScore - 0.15),
|
|
MinAverageScore = Math.Max(0.2, baseThresholds.MinAverageScore - 0.15)
|
|
};
|
|
}
|
|
|
|
return baseThresholds;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gera explicação do motivo da decisão
|
|
/// </summary>
|
|
private string GenerateReason(bool shouldRespond, string strategy, ConfidenceMetrics metrics, ConfidenceThresholds thresholds, string language)
|
|
{
|
|
if (shouldRespond)
|
|
{
|
|
return language == "en"
|
|
? $"Sufficient confidence for '{strategy}' strategy - Score: {metrics.OverallScore:P1}, Docs: {metrics.RelevantDocuments}/{metrics.TotalDocuments}"
|
|
: $"Confiança suficiente para estratégia '{strategy}' - Score: {metrics.OverallScore:P1}, Docs: {metrics.RelevantDocuments}/{metrics.TotalDocuments}";
|
|
}
|
|
|
|
var issues = new List<string>();
|
|
|
|
if (metrics.TotalDocuments < thresholds.MinDocuments)
|
|
{
|
|
var msg = language == "en"
|
|
? $"few documents ({metrics.TotalDocuments} < {thresholds.MinDocuments})"
|
|
: $"poucos documentos ({metrics.TotalDocuments} < {thresholds.MinDocuments})";
|
|
issues.Add(msg);
|
|
}
|
|
|
|
if (metrics.RelevantDocuments < thresholds.MinRelevantDocuments)
|
|
{
|
|
var msg = language == "en"
|
|
? $"few relevant documents ({metrics.RelevantDocuments} < {thresholds.MinRelevantDocuments})"
|
|
: $"poucos documentos relevantes ({metrics.RelevantDocuments} < {thresholds.MinRelevantDocuments})";
|
|
issues.Add(msg);
|
|
}
|
|
|
|
if (metrics.OverallScore < thresholds.MinOverallScore)
|
|
{
|
|
var msg = language == "en"
|
|
? $"low overall score ({metrics.OverallScore:P1} < {thresholds.MinOverallScore:P1})"
|
|
: $"score geral baixo ({metrics.OverallScore:P1} < {thresholds.MinOverallScore:P1})";
|
|
issues.Add(msg);
|
|
}
|
|
|
|
if (!metrics.HasSpecificContent)
|
|
{
|
|
var msg = language == "en" ? "no specific content found" : "conteúdo específico não encontrado";
|
|
issues.Add(msg);
|
|
}
|
|
|
|
var prefix = language == "en" ? "Insufficient confidence: " : "Confiança insuficiente: ";
|
|
return prefix + string.Join(", ", issues);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gera resposta de fallback apropriada
|
|
/// </summary>
|
|
private string GenerateFallbackResponse(string strategy, ConfidenceMetrics metrics, string language)
|
|
{
|
|
if (!_settings.FallbackMessages.ContainsKey(language))
|
|
{
|
|
language = "pt"; // Fallback para português
|
|
}
|
|
|
|
var messages = _settings.FallbackMessages[language];
|
|
|
|
// Escolher mensagem baseada no problema principal
|
|
if (metrics.TotalDocuments == 0)
|
|
{
|
|
return messages.NoDocuments;
|
|
}
|
|
|
|
if (metrics.RelevantDocuments == 0)
|
|
{
|
|
return messages.NoRelevantDocuments;
|
|
}
|
|
|
|
return strategy switch
|
|
{
|
|
"overview" => messages.InsufficientOverview,
|
|
"specific" => messages.InsufficientSpecific,
|
|
"detailed" => messages.InsufficientDetailed,
|
|
_ => messages.Generic
|
|
};
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Resultado da verificação de confiança
|
|
/// </summary>
|
|
public class ConfidenceResult
|
|
{
|
|
public bool ShouldRespond { get; set; }
|
|
public double ConfidenceScore { get; set; }
|
|
public string Strategy { get; set; } = "";
|
|
public ConfidenceMetrics Metrics { get; set; } = new();
|
|
public string Reason { get; set; } = "";
|
|
public string? SuggestedResponse { get; set; }
|
|
}
|
|
|
|
/// <summary>
|
|
/// Métricas detalhadas de confiança
|
|
/// </summary>
|
|
public class ConfidenceMetrics
|
|
{
|
|
// Métricas básicas
|
|
public int TotalDocuments { get; set; }
|
|
public int RelevantDocuments { get; set; }
|
|
public int HighQualityDocuments { get; set; }
|
|
public double AverageScore { get; set; }
|
|
public double MaxScore { get; set; }
|
|
public double MinScore { get; set; }
|
|
public int ContextLength { get; set; }
|
|
public int StepsExecuted { get; set; }
|
|
public bool HasSpecificContent { get; set; }
|
|
public double OverallScore { get; set; }
|
|
|
|
// Métricas avançadas
|
|
public double ScoreVariance { get; set; }
|
|
public double ContentDiversity { get; set; }
|
|
public double ConceptCoverage { get; set; }
|
|
}
|
|
}
|