feat:rag-hierarquico

This commit is contained in:
Ricardo Carneiro 2025-06-21 14:20:07 -03:00
parent 13083ffb5d
commit bc699abbd3
28 changed files with 5707 additions and 743 deletions

View File

@ -21,7 +21,7 @@ namespace ChatApi.Controllers
private readonly IResponseService _responseService; private readonly IResponseService _responseService;
private readonly TextFilter _textFilter; private readonly TextFilter _textFilter;
private readonly UserDataRepository _userDataRepository; private readonly UserDataRepository _userDataRepository;
private readonly ProjectDataRepository _projectDataRepository; private readonly IProjectDataRepository _projectDataRepository;
private readonly ITextDataService _textDataService; private readonly ITextDataService _textDataService;
private readonly IHttpClientFactory _httpClientFactory; private readonly IHttpClientFactory _httpClientFactory;
@ -30,7 +30,7 @@ namespace ChatApi.Controllers
IResponseService responseService, IResponseService responseService,
UserDataRepository userDataRepository, UserDataRepository userDataRepository,
ITextDataService textDataService, ITextDataService textDataService,
ProjectDataRepository projectDataRepository, IProjectDataRepository projectDataRepository,
IHttpClientFactory httpClientFactory) IHttpClientFactory httpClientFactory)
{ {
_logger = logger; _logger = logger;

View File

@ -0,0 +1,225 @@
using Microsoft.AspNetCore.Mvc;
using ChatRAG.Services.Contracts;
using ChatRAG.Services.ResponseService;
using ChatRAG.Contracts.VectorSearch;
namespace ChatApi.Controllers
{
[ApiController]
[Route("api/[controller]")]
public class RAGStrategyController : ControllerBase
{
private readonly IVectorDatabaseFactory _factory;
private readonly IServiceProvider _serviceProvider;
private readonly ILogger<RAGStrategyController> _logger;
public RAGStrategyController(
IVectorDatabaseFactory factory,
IServiceProvider serviceProvider,
ILogger<RAGStrategyController> logger)
{
_factory = factory;
_serviceProvider = serviceProvider;
_logger = logger;
}
/// <summary>
/// Lista as estratégias de RAG disponíveis
/// </summary>
[HttpGet("strategies")]
public IActionResult GetAvailableStrategies()
{
var strategies = new[]
{
new {
name = "Standard",
description = "RAG padrão com classificação automática de estratégia",
service = "ResponseRAGService",
features = new[] { "Busca por similaridade", "Filtros dinâmicos", "Classificação automática" }
},
new {
name = "Hierarchical",
description = "RAG hierárquico com múltiplas etapas de busca",
service = "HierarchicalRAGService",
features = new[] { "Análise de query", "Busca em múltiplas etapas", "Expansão de contexto", "Identificação de lacunas" }
}
};
return Ok(new
{
currentProvider = _factory.GetActiveProvider(),
availableStrategies = strategies
});
}
/// <summary>
/// Testa uma estratégia específica com uma pergunta
/// </summary>
[HttpPost("test/{strategy}")]
public async Task<IActionResult> TestStrategy(
[FromRoute] string strategy,
[FromBody] StrategyTestRequest request)
{
try
{
IResponseService responseService = strategy.ToLower() switch
{
"standard" => _serviceProvider.GetRequiredService<ResponseRAGService>(),
"hierarchical" => _serviceProvider.GetRequiredService<HierarchicalRAGService>(),
_ => throw new ArgumentException($"Estratégia não suportada: {strategy}")
};
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
// Usar dados mock se não fornecidos
var userData = request.UserData ?? new ChatApi.Models.UserData
{
Id = "test-user",
Name = "Test User"
};
var response = await responseService.GetResponse(
userData,
request.ProjectId,
request.SessionId ?? Guid.NewGuid().ToString(),
request.Question,
request.Language ?? "pt"
);
stopwatch.Stop();
return Ok(new
{
strategy,
response,
executionTime = stopwatch.ElapsedMilliseconds,
provider = _factory.GetActiveProvider()
});
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao testar estratégia {Strategy}", strategy);
return StatusCode(500, new { error = ex.Message });
}
}
/// <summary>
/// Compara resultados entre diferentes estratégias
/// </summary>
[HttpPost("compare")]
public async Task<IActionResult> CompareStrategies([FromBody] StrategyTestRequest request)
{
try
{
var userData = request.UserData ?? new ChatApi.Models.UserData
{
Id = "test-user",
Name = "Test User"
};
var sessionId = Guid.NewGuid().ToString();
var results = new Dictionary<string, object>();
// Testar estratégia padrão
try
{
var standardService = _serviceProvider.GetRequiredService<ResponseRAGService>();
var stopwatch1 = System.Diagnostics.Stopwatch.StartNew();
var standardResponse = await standardService.GetResponse(
userData, request.ProjectId, sessionId + "-standard",
request.Question, request.Language ?? "pt");
stopwatch1.Stop();
results["standard"] = new
{
response = standardResponse,
executionTime = stopwatch1.ElapsedMilliseconds,
success = true
};
}
catch (Exception ex)
{
results["standard"] = new { success = false, error = ex.Message };
}
// Testar estratégia hierárquica
try
{
var hierarchicalService = _serviceProvider.GetRequiredService<HierarchicalRAGService>();
var stopwatch2 = System.Diagnostics.Stopwatch.StartNew();
var hierarchicalResponse = await hierarchicalService.GetResponse(
userData, request.ProjectId, sessionId + "-hierarchical",
request.Question, request.Language ?? "pt");
stopwatch2.Stop();
results["hierarchical"] = new
{
response = hierarchicalResponse,
executionTime = stopwatch2.ElapsedMilliseconds,
success = true
};
}
catch (Exception ex)
{
results["hierarchical"] = new { success = false, error = ex.Message };
}
return Ok(new
{
question = request.Question,
projectId = request.ProjectId,
provider = _factory.GetActiveProvider(),
results
});
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao comparar estratégias");
return StatusCode(500, new { error = ex.Message });
}
}
/// <summary>
/// Obtém métricas de performance das estratégias
/// </summary>
[HttpGet("metrics")]
public IActionResult GetMetrics()
{
// TODO: Implementar coleta de métricas real
var metrics = new
{
standard = new
{
avgResponseTime = "1.2s",
successRate = "98%",
avgContextSize = "3 documentos",
usage = "Alto"
},
hierarchical = new
{
avgResponseTime = "2.1s",
successRate = "95%",
avgContextSize = "5-8 documentos",
usage = "Médio"
},
recommendations = new[]
{
"Use Standard para perguntas simples e rápidas",
"Use Hierarchical para análises complexas que precisam de contexto profundo",
"Hierarchical é melhor para perguntas técnicas detalhadas"
}
};
return Ok(metrics);
}
}
public class StrategyTestRequest
{
public string ProjectId { get; set; } = "";
public string Question { get; set; } = "";
public string? Language { get; set; }
public string? SessionId { get; set; }
public ChatApi.Models.UserData? UserData { get; set; }
}
}

225
Controllers/sgnjnzt5.baf~ Normal file
View File

@ -0,0 +1,225 @@
using Microsoft.AspNetCore.Mvc;
using ChatRAG.Services.Contracts;
using ChatRAG.Services.ResponseService;
using ChatRAG.Contracts.VectorSearch;
namespace ChatApi.Controllers
{
[ApiController]
[Route("api/[controller]")]
public class RAGStrategyController : ControllerBase
{
private readonly IVectorDatabaseFactory _factory;
private readonly IServiceProvider _serviceProvider;
private readonly ILogger<RAGStrategyController> _logger;
public RAGStrategyController(
IVectorDatabaseFactory factory,
IServiceProvider serviceProvider,
ILogger<RAGStrategyController> logger)
{
_factory = factory;
_serviceProvider = serviceProvider;
_logger = logger;
}
/// <summary>
/// Lista as estratégias de RAG disponíveis
/// </summary>
[HttpGet("strategies")]
public IActionResult GetAvailableStrategies()
{
var strategies = new[]
{
new {
name = "Standard",
description = "RAG padrão com classificação automática de estratégia",
service = "ResponseRAGService",
features = new[] { "Busca por similaridade", "Filtros dinâmicos", "Classificação automática" }
},
new {
name = "Hierarchical",
description = "RAG hierárquico com múltiplas etapas de busca",
service = "HierarchicalRAGService",
features = new[] { "Análise de query", "Busca em múltiplas etapas", "Expansão de contexto", "Identificação de lacunas" }
}
};
return Ok(new
{
currentProvider = _factory.GetActiveProvider(),
availableStrategies = strategies
});
}
/// <summary>
/// Testa uma estratégia específica com uma pergunta
/// </summary>
[HttpPost("test/{strategy}")]
public async Task<IActionResult> TestStrategy(
[FromRoute] string strategy,
[FromBody] StrategyTestRequest request)
{
try
{
IResponseService responseService = strategy.ToLower() switch
{
"standard" => _serviceProvider.GetRequiredService<ResponseRAGService>(),
"hierarchical" => _serviceProvider.GetRequiredService<HierarchicalRAGService>(),
_ => throw new ArgumentException($"Estratégia não suportada: {strategy}")
};
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
// Usar dados mock se não fornecidos
var userData = request.UserData ?? new ChatApi.Models.UserData
{
Id = "test-user",
Name = "Test User"
};
var response = await responseService.GetResponse(
userData,
request.ProjectId,
request.SessionId ?? Guid.NewGuid().ToString(),
request.Question,
request.Language ?? "pt"
);
stopwatch.Stop();
return Ok(new
{
strategy,
response,
executionTime = stopwatch.ElapsedMilliseconds,
provider = _factory.GetActiveProvider()
});
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao testar estratégia {Strategy}", strategy);
return StatusCode(500, new { error = ex.Message });
}
}
/// <summary>
/// Compara resultados entre diferentes estratégias
/// </summary>
[HttpPost("compare")]
public async Task<IActionResult> CompareStrategies([FromBody] StrategyTestRequest request)
{
try
{
var userData = request.UserData ?? new ChatApi.Models.UserData
{
Id = "test-user",
Name = "Test User"
};
var sessionId = Guid.NewGuid().ToString();
var results = new Dictionary<string, object>();
// Testar estratégia padrão
try
{
var standardService = _serviceProvider.GetRequiredService<ResponseRAGService>();
var stopwatch1 = System.Diagnostics.Stopwatch.StartNew();
var standardResponse = await standardService.GetResponse(
userData, request.ProjectId, sessionId + "-standard",
request.Question, request.Language ?? "pt");
stopwatch1.Stop();
results["standard"] = new
{
response = standardResponse,
executionTime = stopwatch1.ElapsedMilliseconds,
success = true
};
}
catch (Exception ex)
{
results["standard"] = new { success = false, error = ex.Message };
}
// Testar estratégia hierárquica
try
{
var hierarchicalService = _serviceProvider.GetRequiredService<HierarchicalRAGService>();
var stopwatch2 = System.Diagnostics.Stopwatch.StartNew();
var hierarchicalResponse = await hierarchicalService.GetResponse(
userData, request.ProjectId, sessionId + "-hierarchical",
request.Question, request.Language ?? "pt");
stopwatch2.Stop();
results["hierarchical"] = new
{
response = hierarchicalResponse,
executionTime = stopwatch2.ElapsedMilliseconds,
success = true
};
}
catch (Exception ex)
{
results["hierarchical"] = new { success = false, error = ex.Message };
}
return Ok(new
{
question = request.Question,
projectId = request.ProjectId,
provider = _factory.GetActiveProvider(),
results
});
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao comparar estratégias");
return StatusCode(500, new { error = ex.Message });
}
}
/// <summary>
/// Obtém métricas de performance das estratégias
/// </summary>
[HttpGet("metrics")]
public IActionResult GetMetrics()
{
// TODO: Implementar coleta de métricas real
var metrics = new
{
standard = new
{
avgResponseTime = "1.2s",
successRate = "98%",
avgContextSize = "3 documentos",
usage = "Alto"
},
hierarchical = new
{
avgResponseTime = "2.1s",
successRate = "95%",
avgContextSize = "5-8 documentos",
usage = "Médio"
},
recommendations = new[]
{
"Use Standard para perguntas simples e rápidas",
"Use Hierarchical para análises complexas que precisam de contexto profundo",
"Hierarchical é melhor para perguntas técnicas detalhadas"
}
};
return Ok(metrics);
}
}
public class StrategyTestRequest
{
public string ProjectId { get; set; } = "";
public string Question { get; set; } = "";
public string? Language { get; set; }
public string? SessionId { get; set; }
public ChatApi.Models.UserData? UserData { get; set; }
}
}

255
Data/30u0ddp1.org~ Normal file
View File

@ -0,0 +1,255 @@
using ChatRAG.Models;
using ChatRAG.Services.Contracts;
using ChatRAG.Settings.ChatRAG.Configuration;
using Microsoft.Extensions.Options;
using System.Text;
using System.Text.Json;
using Qdrant.Client;
using Qdrant.Client.Grpc;
namespace ChatRAG.Data
{
public class QdrantProjectDataRepository : IProjectDataRepository
{
private readonly HttpClient _httpClient;
private readonly string _collectionName;
private readonly ILogger<QdrantProjectDataRepository> _logger;
private readonly QdrantClient _qdrantClient;
public QdrantProjectDataRepository(
IOptions<VectorDatabaseSettings> settings,
HttpClient httpClient,
ILogger<QdrantProjectDataRepository> logger)
{
var qdrantSettings = settings.Value.Qdrant ?? throw new ArgumentNullException("Qdrant settings not configured");
_httpClient = httpClient;
_httpClient.BaseAddress = new Uri($"http://{qdrantSettings.Host}:{qdrantSettings.Port}");
_collectionName = qdrantSettings.GroupsCollectionName;
_logger = logger;
// Inicializa o QdrantClient - use GRPC (porta 6334) para melhor performance
_qdrantClient = new QdrantClient(qdrantSettings.Host, port: 6334, https: false);
InitializeAsync().GetAwaiter().GetResult();
}
private async Task InitializeAsync()
{
try
{
var exists = await _qdrantClient.CollectionExistsAsync(_collectionName);
if (!exists)
{
await CreateProjectsCollection();
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao inicializar collection de projetos no Qdrant");
}
}
public async Task<List<Project>> GetAsync()
{
try
{
var scrollRequest = new ScrollPoints
{
CollectionName = _collectionName,
Filter = new Filter(), // Filtro vazio
Limit = 1000,
WithPayload = true,
WithVectors = false
};
var result = await _qdrantClient.ScrollAsync(_collectionName, scrollRequest);
return result.Select(ConvertToProject)
.Where(p => p != null)
.ToList()!;
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao recuperar projetos do Qdrant");
return new List<Project>();
}
}
public async Task<Project?> GetAsync(string id)
{
try
{
var points = await _qdrantClient.RetrieveAsync(
_collectionName,
new[] { PointId.Parser.Parse(id) },
withPayload: true,
withVectors: false
);
var point = points.FirstOrDefault();
return point != null ? ConvertToProject(point) : null;
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao buscar projeto {Id} no Qdrant", id);
return null;
}
}
public async Task CreateAsync(Project newProject)
{
try
{
var id = string.IsNullOrEmpty(newProject.Id) ? Guid.NewGuid().ToString() : newProject.Id;
newProject.Id = id;
var point = new PointStruct
{
Id = PointId.Parser.Parse(id),
Vectors = new float[384], // Vector dummy para projetos
Payload =
{
["id"] = newProject.Id,
["nome"] = newProject.Nome,
["descricao"] = newProject.Descricao,
["created_at"] = DateTime.UtcNow.ToString("O"),
["entity_type"] = "project"
}
};
await _qdrantClient.UpsertAsync(_collectionName, new[] { point });
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao criar projeto no Qdrant");
throw;
}
}
public async Task UpdateAsync(string id, Project updatedProject)
{
try
{
updatedProject.Id = id;
var point = new PointStruct
{
Id = PointId.Parser.Parse(id),
Vectors = new float[384], // Vector dummy
Payload =
{
["id"] = updatedProject.Id,
["nome"] = updatedProject.Nome,
["descricao"] = updatedProject.Descricao,
["updated_at"] = DateTime.UtcNow.ToString("O"),
["entity_type"] = "project"
}
};
await _qdrantClient.UpsertAsync(_collectionName, new[] { point });
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao atualizar projeto {Id} no Qdrant", id);
throw;
}
}
public async Task SaveAsync(Project project)
{
try
{
if (string.IsNullOrEmpty(project.Id))
{
await CreateAsync(project);
}
else
{
var existing = await GetAsync(project.Id);
if (existing == null)
{
await CreateAsync(project);
}
else
{
await UpdateAsync(project.Id, project);
}
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao salvar projeto no Qdrant");
throw;
}
}
public async Task RemoveAsync(string id)
{
try
{
await _qdrantClient.DeleteAsync(
_collectionName,
new[] { PointId.Parser.Parse(id) }
);
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao remover projeto {Id} do Qdrant", id);
throw;
}
}
private async Task CreateProjectsCollection()
{
var vectorParams = new VectorParams
{
Size = 384,
Distance = Distance.Cosine
};
await _qdrantClient.CreateCollectionAsync(_collectionName, vectorParams);
_logger.LogInformation("Collection de projetos '{CollectionName}' criada no Qdrant", _collectionName);
}
private static Project? ConvertToProject(RetrievedPoint point)
{
try
{
if (point.Payload == null) return null;
return new Project
{
Id = point.Payload.TryGetValue("id", out var idValue) ? idValue.StringValue : point.Id.ToString(),
Nome = point.Payload.TryGetValue("nome", out var nomeValue) ? nomeValue.StringValue : "",
Descricao = point.Payload.TryGetValue("descricao", out var descValue) ? descValue.StringValue : ""
};
}
catch
{
return null;
}
}
}
public class QdrantScrollResult
{
public QdrantScrollData? result { get; set; }
}
public class QdrantScrollData
{
public QdrantPoint[]? points { get; set; }
}
public class QdrantPointResult
{
public QdrantPoint? result { get; set; }
}
public class QdrantPoint
{
public string? id { get; set; }
public Dictionary<string, object>? payload { get; set; }
}
}

255
Data/32reubjo.e20~ Normal file
View File

@ -0,0 +1,255 @@
using ChatRAG.Models;
using ChatRAG.Services.Contracts;
using ChatRAG.Settings.ChatRAG.Configuration;
using Microsoft.Extensions.Options;
using System.Text;
using System.Text.Json;
using Qdrant.Client;
using Qdrant.Client.Grpc;
namespace ChatRAG.Data
{
public class QdrantProjectDataRepository : IProjectDataRepository
{
private readonly HttpClient _httpClient;
private readonly string _collectionName;
private readonly ILogger<QdrantProjectDataRepository> _logger;
private readonly QdrantClient _qdrantClient;
public QdrantProjectDataRepository(
IOptions<VectorDatabaseSettings> settings,
HttpClient httpClient,
ILogger<QdrantProjectDataRepository> logger)
{
var qdrantSettings = settings.Value.Qdrant ?? throw new ArgumentNullException("Qdrant settings not configured");
_httpClient = httpClient;
_httpClient.BaseAddress = new Uri($"http://{qdrantSettings.Host}:{qdrantSettings.Port}");
_collectionName = qdrantSettings.GroupsCollectionName;
_logger = logger;
// Inicializa o QdrantClient - use GRPC (porta 6334) para melhor performance
_qdrantClient = new QdrantClient(qdrantSettings.Host, port: 6334, https: false);
InitializeAsync().GetAwaiter().GetResult();
}
private async Task InitializeAsync()
{
try
{
var exists = await _qdrantClient.CollectionExistsAsync(_collectionName);
if (!exists)
{
await CreateProjectsCollection();
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao inicializar collection de projetos no Qdrant");
}
}
public async Task<List<Project>> GetAsync()
{
try
{
var scrollRequest = new ScrollPoints
{
CollectionName = _collectionName,
Filter = new Filter(), // Filtro vazio
Limit = 1000,
WithPayload = true,
WithVectors = false
};
var result = await _qdrantClient.ScrollAsync(_collectionName, scrollRequest);
return result.Select(ConvertToProject)
.Where(p => p != null)
.ToList()!;
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao recuperar projetos do Qdrant");
return new List<Project>();
}
}
public async Task<Project?> GetAsync(string id)
{
try
{
var points = await _qdrantClient.RetrieveAsync(
_collectionName,
new[] { PointId.Parser.ParseFrom(id) },
withPayload: true,
withVectors: false
);
var point = points.FirstOrDefault();
return point != null ? ConvertToProject(point) : null;
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao buscar projeto {Id} no Qdrant", id);
return null;
}
}
public async Task CreateAsync(Project newProject)
{
try
{
var id = string.IsNullOrEmpty(newProject.Id) ? Guid.NewGuid().ToString() : newProject.Id;
newProject.Id = id;
var point = new PointStruct
{
Id = PointId.Parser.ParseFrom(id),
Vectors = new float[384], // Vector dummy para projetos
Payload =
{
["id"] = newProject.Id,
["nome"] = newProject.Nome,
["descricao"] = newProject.Descricao,
["created_at"] = DateTime.UtcNow.ToString("O"),
["entity_type"] = "project"
}
};
await _qdrantClient.UpsertAsync(_collectionName, new[] { point });
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao criar projeto no Qdrant");
throw;
}
}
public async Task UpdateAsync(string id, Project updatedProject)
{
try
{
updatedProject.Id = id;
var point = new PointStruct
{
Id = PointId.Parser.ParseFrom(id),
Vectors = new float[384], // Vector dummy
Payload =
{
["id"] = updatedProject.Id,
["nome"] = updatedProject.Nome,
["descricao"] = updatedProject.Descricao,
["updated_at"] = DateTime.UtcNow.ToString("O"),
["entity_type"] = "project"
}
};
await _qdrantClient.UpsertAsync(_collectionName, new[] { point });
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao atualizar projeto {Id} no Qdrant", id);
throw;
}
}
public async Task SaveAsync(Project project)
{
try
{
if (string.IsNullOrEmpty(project.Id))
{
await CreateAsync(project);
}
else
{
var existing = await GetAsync(project.Id);
if (existing == null)
{
await CreateAsync(project);
}
else
{
await UpdateAsync(project.Id, project);
}
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao salvar projeto no Qdrant");
throw;
}
}
public async Task RemoveAsync(string id)
{
try
{
await _qdrantClient.DeleteAsync(
_collectionName,
new[] { PointId.Parser.ParseFrom(id) }
);
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao remover projeto {Id} do Qdrant", id);
throw;
}
}
private async Task CreateProjectsCollection()
{
var vectorParams = new VectorParams
{
Size = 384,
Distance = Distance.Cosine
};
await _qdrantClient.CreateCollectionAsync(_collectionName, vectorParams);
_logger.LogInformation("Collection de projetos '{CollectionName}' criada no Qdrant", _collectionName);
}
private static Project? ConvertToProject(RetrievedPoint point)
{
try
{
if (point.Payload == null) return null;
return new Project
{
Id = point.Payload.TryGetValue("id", out var idValue) ? idValue.StringValue : point.Id.ToString(),
Nome = point.Payload.TryGetValue("nome", out var nomeValue) ? nomeValue.StringValue : "",
Descricao = point.Payload.TryGetValue("descricao", out var descValue) ? descValue.StringValue : ""
};
}
catch
{
return null;
}
}
}
public class QdrantScrollResult
{
public QdrantScrollData? result { get; set; }
}
public class QdrantScrollData
{
public QdrantPoint[]? points { get; set; }
}
public class QdrantPointResult
{
public QdrantPoint? result { get; set; }
}
public class QdrantPoint
{
public string? id { get; set; }
public Dictionary<string, object>? payload { get; set; }
}
}

255
Data/3qrw3lwa.v4s~ Normal file
View File

@ -0,0 +1,255 @@
using ChatRAG.Models;
using ChatRAG.Services.Contracts;
using ChatRAG.Settings.ChatRAG.Configuration;
using Microsoft.Extensions.Options;
using System.Text;
using System.Text.Json;
using Qdrant.Client;
using Qdrant.Client.Grpc;
namespace ChatRAG.Data
{
public class QdrantProjectDataRepository : IProjectDataRepository
{
private readonly HttpClient _httpClient;
private readonly string _collectionName;
private readonly ILogger<QdrantProjectDataRepository> _logger;
private readonly QdrantClient _qdrantClient;
public QdrantProjectDataRepository(
IOptions<VectorDatabaseSettings> settings,
HttpClient httpClient,
ILogger<QdrantProjectDataRepository> logger)
{
var qdrantSettings = settings.Value.Qdrant ?? throw new ArgumentNullException("Qdrant settings not configured");
_httpClient = httpClient;
_httpClient.BaseAddress = new Uri($"http://{qdrantSettings.Host}:{qdrantSettings.Port}");
_collectionName = qdrantSettings.GroupsCollectionName;
_logger = logger;
// Inicializa o QdrantClient - use GRPC (porta 6334) para melhor performance
_qdrantClient = new QdrantClient(qdrantSettings.Host, port: 6334, https: false);
InitializeAsync().GetAwaiter().GetResult();
}
private async Task InitializeAsync()
{
try
{
var exists = await _qdrantClient.CollectionExistsAsync(_collectionName);
if (!exists)
{
await CreateProjectsCollection();
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao inicializar collection de projetos no Qdrant");
}
}
public async Task<List<Project>> GetAsync()
{
try
{
var scrollRequest = new ScrollPoints
{
CollectionName = _collectionName,
Filter = new Filter(), // Filtro vazio
Limit = 1000,
WithPayload = true,
WithVectors = false
};
var result = await _qdrantClient.ScrollAsync(scrollRequest);
return result.Select(ConvertToProject)
.Where(p => p != null)
.ToList()!;
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao recuperar projetos do Qdrant");
return new List<Project>();
}
}
public async Task<Project?> GetAsync(string id)
{
try
{
var points = await _qdrantClient.RetrieveAsync(
_collectionName,
new[] { PointId.NewGuid(Guid.Parse(id)) },
withPayload: true,
withVectors: false
);
var point = points.FirstOrDefault();
return point != null ? ConvertToProject(point) : null;
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao buscar projeto {Id} no Qdrant", id);
return null;
}
}
public async Task CreateAsync(Project newProject)
{
try
{
var id = string.IsNullOrEmpty(newProject.Id) ? Guid.NewGuid().ToString() : newProject.Id;
newProject.Id = id;
var point = new PointStruct
{
Id = PointId.NewGuid(Guid.Parse(id)),
Vectors = new float[384], // Vector dummy para projetos
Payload =
{
["id"] = newProject.Id,
["nome"] = newProject.Nome,
["descricao"] = newProject.Descricao,
["created_at"] = DateTime.UtcNow.ToString("O"),
["entity_type"] = "project"
}
};
await _qdrantClient.UpsertAsync(_collectionName, new[] { point });
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao criar projeto no Qdrant");
throw;
}
}
public async Task UpdateAsync(string id, Project updatedProject)
{
try
{
updatedProject.Id = id;
var point = new PointStruct
{
Id = PointId.NewGuid(Guid.Parse(id)),
Vectors = new float[384], // Vector dummy
Payload =
{
["id"] = updatedProject.Id,
["nome"] = updatedProject.Nome,
["descricao"] = updatedProject.Descricao,
["updated_at"] = DateTime.UtcNow.ToString("O"),
["entity_type"] = "project"
}
};
await _qdrantClient.UpsertAsync(_collectionName, new[] { point });
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao atualizar projeto {Id} no Qdrant", id);
throw;
}
}
public async Task SaveAsync(Project project)
{
try
{
if (string.IsNullOrEmpty(project.Id))
{
await CreateAsync(project);
}
else
{
var existing = await GetAsync(project.Id);
if (existing == null)
{
await CreateAsync(project);
}
else
{
await UpdateAsync(project.Id, project);
}
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao salvar projeto no Qdrant");
throw;
}
}
public async Task RemoveAsync(string id)
{
try
{
await _qdrantClient.DeleteAsync(
_collectionName,
new[] { PointId.NewGuid(Guid.Parse(id)) }
);
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao remover projeto {Id} do Qdrant", id);
throw;
}
}
private async Task CreateProjectsCollection()
{
var vectorParams = new VectorParams
{
Size = 384,
Distance = Distance.Cosine
};
await _qdrantClient.CreateCollectionAsync(_collectionName, vectorParams);
_logger.LogInformation("Collection de projetos '{CollectionName}' criada no Qdrant", _collectionName);
}
private static Project? ConvertToProject(RetrievedPoint point)
{
try
{
if (point.Payload == null) return null;
return new Project
{
Id = point.Payload.TryGetValue("id", out var idValue) ? idValue.StringValue : point.Id.ToString(),
Nome = point.Payload.TryGetValue("nome", out var nomeValue) ? nomeValue.StringValue : "",
Descricao = point.Payload.TryGetValue("descricao", out var descValue) ? descValue.StringValue : ""
};
}
catch
{
return null;
}
}
}
public class QdrantScrollResult
{
public QdrantScrollData? result { get; set; }
}
public class QdrantScrollData
{
public QdrantPoint[]? points { get; set; }
}
public class QdrantPointResult
{
public QdrantPoint? result { get; set; }
}
public class QdrantPoint
{
public string? id { get; set; }
public Dictionary<string, object>? payload { get; set; }
}
}

View File

@ -0,0 +1,331 @@
using ChatRAG.Models;
using ChatRAG.Services.Contracts;
using ChatRAG.Services.SearchVectors;
using ChatRAG.Settings.ChatRAG.Configuration;
using Microsoft.Extensions.Options;
using System.Text;
using System.Text.Json;
namespace ChatRAG.Data
{
public class ChromaProjectDataRepository : IProjectDataRepository
{
private readonly HttpClient _httpClient;
private readonly string _collectionName;
private readonly ILogger<ChromaProjectDataRepository> _logger;
public ChromaProjectDataRepository(
IOptions<VectorDatabaseSettings> settings,
HttpClient httpClient,
ILogger<ChromaProjectDataRepository> logger)
{
var chromaSettings = settings.Value.Chroma ?? throw new ArgumentNullException("Chroma settings not configured");
_httpClient = httpClient;
_httpClient.BaseAddress = new Uri($"http://{chromaSettings.Host}:{chromaSettings.Port}");
_collectionName = "projects"; // Collection separada para projetos
_logger = logger;
InitializeAsync().GetAwaiter().GetResult();
}
private async Task InitializeAsync()
{
try
{
// Verificar se a collection existe, se não, criar
var collections = await GetCollectionsAsync();
if (!collections.Contains(_collectionName))
{
await CreateProjectsCollection();
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao inicializar collection de projetos no Chroma");
}
}
public async Task<List<Project>> GetAsync()
{
try
{
var query = new
{
where = new { entity_type = "project" },
include = new[] { "documents", "metadatas" }
};
var json = JsonSerializer.Serialize(query);
var content = new StringContent(json, Encoding.UTF8, "application/json");
var response = await _httpClient.PostAsync($"/api/v2/collections/{_collectionName}/get", content);
if (!response.IsSuccessStatusCode)
{
_logger.LogError("Erro ao buscar projetos no Chroma");
return new List<Project>();
}
var result = await response.Content.ReadAsStringAsync();
var getResult = JsonSerializer.Deserialize<ChromaGetResult>(result);
var projects = new List<Project>();
if (getResult?.ids?.Length > 0)
{
for (int i = 0; i < getResult.ids.Length; i++)
{
var metadata = getResult.metadatas?[i];
if (metadata != null)
{
projects.Add(new Project
{
Id = metadata.GetValueOrDefault("id")?.ToString() ?? getResult.ids[i],
Nome = metadata.GetValueOrDefault("nome")?.ToString() ?? "",
Descricao = metadata.GetValueOrDefault("descricao")?.ToString() ?? ""
});
}
}
}
return projects;
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao recuperar projetos do Chroma");
return new List<Project>();
}
}
public async Task<Project?> GetAsync(string id)
{
try
{
var query = new
{
ids = new[] { id },
include = new[] { "metadatas" }
};
var json = JsonSerializer.Serialize(query);
var content = new StringContent(json, Encoding.UTF8, "application/json");
var response = await _httpClient.PostAsync($"/api/v2/collections/{_collectionName}/get", content);
if (!response.IsSuccessStatusCode)
{
return null;
}
var result = await response.Content.ReadAsStringAsync();
var getResult = JsonSerializer.Deserialize<ChromaGetResult>(result);
if (getResult?.ids?.Length > 0 && getResult.metadatas?[0] != null)
{
var metadata = getResult.metadatas[0];
return new Project
{
Id = metadata.GetValueOrDefault("id")?.ToString() ?? id,
Nome = metadata.GetValueOrDefault("nome")?.ToString() ?? "",
Descricao = metadata.GetValueOrDefault("descricao")?.ToString() ?? ""
};
}
return null;
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao buscar projeto {Id} no Chroma", id);
return null;
}
}
public async Task CreateAsync(Project newProject)
{
try
{
var id = string.IsNullOrEmpty(newProject.Id) ? Guid.NewGuid().ToString() : newProject.Id;
newProject.Id = id;
var document = new
{
ids = new[] { id },
documents = new[] { $"Projeto: {newProject.Nome}" },
metadatas = new[] { new Dictionary<string, object>
{
["id"] = id,
["nome"] = newProject.Nome,
["descricao"] = newProject.Descricao,
["created_at"] = DateTime.UtcNow.ToString("O"),
["entity_type"] = "project"
}},
embeddings = new[] { new double[384] } // Vector dummy
};
var json = JsonSerializer.Serialize(document);
var content = new StringContent(json, Encoding.UTF8, "application/json");
var response = await _httpClient.PostAsync($"/api/v2/collections/{_collectionName}/add", content);
if (!response.IsSuccessStatusCode)
{
var error = await response.Content.ReadAsStringAsync();
throw new Exception($"Erro ao criar projeto no Chroma: {error}");
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao criar projeto no Chroma");
throw;
}
}
public async Task UpdateAsync(string id, Project updatedProject)
{
try
{
// Chroma não tem update direto, então fazemos upsert
updatedProject.Id = id;
var document = new
{
ids = new[] { id },
documents = new[] { $"Projeto: {updatedProject.Nome}" },
metadatas = new[] { new Dictionary<string, object>
{
["id"] = id,
["nome"] = updatedProject.Nome,
["descricao"] = updatedProject.Descricao,
["updated_at"] = DateTime.UtcNow.ToString("O"),
["entity_type"] = "project"
}},
embeddings = new[] { new double[384] }
};
var json = JsonSerializer.Serialize(document);
var content = new StringContent(json, Encoding.UTF8, "application/json");
var response = await _httpClient.PostAsync($"/api/v2/collections/{_collectionName}/upsert", content);
if (!response.IsSuccessStatusCode)
{
var error = await response.Content.ReadAsStringAsync();
throw new Exception($"Erro ao atualizar projeto no Chroma: {error}");
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao atualizar projeto {Id} no Chroma", id);
throw;
}
}
public async Task SaveAsync(Project project)
{
try
{
if (string.IsNullOrEmpty(project.Id))
{
await CreateAsync(project);
}
else
{
var existing = await GetAsync(project.Id);
if (existing == null)
{
await CreateAsync(project);
}
else
{
await UpdateAsync(project.Id, project);
}
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao salvar projeto no Chroma");
throw;
}
}
public async Task RemoveAsync(string id)
{
try
{
var deleteRequest = new
{
ids = new[] { id }
};
var json = JsonSerializer.Serialize(deleteRequest);
var content = new StringContent(json, Encoding.UTF8, "application/json");
var response = await _httpClient.PostAsync($"/api/v2/collections/{_collectionName}/delete", content);
if (!response.IsSuccessStatusCode)
{
var error = await response.Content.ReadAsStringAsync();
_logger.LogWarning("Erro ao remover projeto {Id} do Chroma: {Error}", id, error);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao remover projeto {Id} do Chroma", id);
throw;
}
}
private async Task<string[]> GetCollectionsAsync()
{
try
{
var response = await _httpClient.GetAsync("/api/v2/collections");
if (!response.IsSuccessStatusCode)
{
return Array.Empty<string>();
}
var content = await response.Content.ReadAsStringAsync();
var collections = JsonSerializer.Deserialize<string[]>(content);
return collections ?? Array.Empty<string>();
}
catch
{
return Array.Empty<string>();
}
}
private async Task CreateProjectsCollection()
{
var collection = new
{
name = _collectionName,
metadata = new
{
description = "Projects Collection",
created_at = DateTime.UtcNow.ToString("O")
}
};
var json = JsonSerializer.Serialize(collection);
var content = new StringContent(json, Encoding.UTF8, "application/json");
var response = await _httpClient.PostAsync("/api/v2/collections", content);
if (!response.IsSuccessStatusCode)
{
var error = await response.Content.ReadAsStringAsync();
throw new Exception($"Erro ao criar collection de projetos: {error}");
}
_logger.LogInformation("Collection de projetos '{CollectionName}' criada no Chroma", _collectionName);
}
}
public class ChromaGetResult
{
public string[] ids { get; set; } = Array.Empty<string>();
public string[] documents { get; set; } = Array.Empty<string>();
public Dictionary<string, object>[]? metadatas { get; set; }
}
}

View File

@ -1,16 +1,17 @@
using ChatApi; using ChatApi;
using ChatRAG.Models; using ChatRAG.Models;
using ChatRAG.Services.Contracts;
using ChatRAG.Settings.ChatRAG.Configuration; using ChatRAG.Settings.ChatRAG.Configuration;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
using MongoDB.Driver; using MongoDB.Driver;
namespace ChatRAG.Data namespace ChatRAG.Data
{ {
public class ProjectDataRepository public class MongoProjectDataRepository : IProjectDataRepository
{ {
private readonly IMongoCollection<Project> _textsCollection; private readonly IMongoCollection<Project> _textsCollection;
public ProjectDataRepository( public MongoProjectDataRepository(
IOptions<VectorDatabaseSettings> databaseSettings) IOptions<VectorDatabaseSettings> databaseSettings)
{ {
var mongoClient = new MongoClient( var mongoClient = new MongoClient(

View File

@ -0,0 +1,269 @@
using ChatRAG.Models;
using ChatRAG.Services.Contracts;
using ChatRAG.Settings.ChatRAG.Configuration;
using Microsoft.Extensions.Options;
using System.Text;
using System.Text.Json;
using Qdrant.Client;
using Qdrant.Client.Grpc;
using Google.Protobuf;
namespace ChatRAG.Data
{
public class QdrantProjectDataRepository : IProjectDataRepository
{
private readonly HttpClient _httpClient;
private readonly string _collectionName;
private readonly ILogger<QdrantProjectDataRepository> _logger;
private readonly QdrantClient _qdrantClient;
private volatile bool _collectionInitialized = false;
private readonly SemaphoreSlim _initializationSemaphore = new(1, 1);
public QdrantProjectDataRepository(
IOptions<VectorDatabaseSettings> settings,
HttpClient httpClient,
ILogger<QdrantProjectDataRepository> logger)
{
var qdrantSettings = settings.Value.Qdrant ?? throw new ArgumentNullException("Qdrant settings not configured");
_httpClient = httpClient;
_httpClient.BaseAddress = new Uri($"http://{qdrantSettings.Host}:{qdrantSettings.Port}");
_collectionName = qdrantSettings.GroupsCollectionName;
_logger = logger;
// Inicializa o QdrantClient - use GRPC (porta 6334) para melhor performance
_qdrantClient = new QdrantClient(qdrantSettings.Host, port: 6334, https: false);
InitializeAsync().GetAwaiter().GetResult();
}
private async Task InitializeAsync()
{
try
{
if (_collectionInitialized) return;
await _initializationSemaphore.WaitAsync();
var exists = await _qdrantClient.CollectionExistsAsync(_collectionName);
if (!exists)
{
await CreateProjectsCollection();
}
_collectionInitialized = true;
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao inicializar collection de projetos no Qdrant");
}
finally
{
_initializationSemaphore.Release();
}
}
public async Task<List<Project>> GetAsync()
{
try
{
//var scrollRequest = new ScrollPoints
//{
// CollectionName = _collectionName,
// Filter = new Filter(), // Filtro vazio
// Limit = 1000,
// WithPayload = true,
// WithVectors = false
//};
//var result = await _qdrantClient.ScrollAsync(_collectionName, scrollRequest);
var result = await _qdrantClient.ScrollAsync(_collectionName, new Filter(), 1000, null, true, false);
return result.Result.Select(ConvertToProject)
.Where(p => p != null)
.ToList()!;
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao recuperar projetos do Qdrant");
return new List<Project>();
}
}
public async Task<Project?> GetAsync(string id)
{
try
{
var points = await _qdrantClient.RetrieveAsync(
_collectionName,
new[] { new PointId { Uuid = id } },
withPayload: true,
withVectors: false
);
var point = points.FirstOrDefault();
return point != null ? ConvertToProject(point) : null;
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao buscar projeto {Id} no Qdrant", id);
return null;
}
}
public async Task CreateAsync(Project newProject)
{
try
{
var id = string.IsNullOrEmpty(newProject.Id) ? Guid.NewGuid().ToString() : newProject.Id;
newProject.Id = id;
var point = new PointStruct
{
Id = new PointId { Uuid= id },
Vectors = new float[384], // Vector dummy para projetos
Payload =
{
["id"] = newProject.Id,
["nome"] = newProject.Nome,
["descricao"] = newProject.Descricao,
["created_at"] = DateTime.UtcNow.ToString("O"),
["entity_type"] = "project"
}
};
await _qdrantClient.UpsertAsync(_collectionName, new[] { point });
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao criar projeto no Qdrant");
throw;
}
}
public async Task UpdateAsync(string id, Project updatedProject)
{
try
{
updatedProject.Id = id;
var point = new PointStruct
{
Id = new PointId { Uuid = id },
Vectors = new float[384], // Vector dummy
Payload =
{
["id"] = updatedProject.Id,
["nome"] = updatedProject.Nome,
["descricao"] = updatedProject.Descricao,
["updated_at"] = DateTime.UtcNow.ToString("O"),
["entity_type"] = "project"
}
};
await _qdrantClient.UpsertAsync(_collectionName, new[] { point });
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao atualizar projeto {Id} no Qdrant", id);
throw;
}
}
public async Task SaveAsync(Project project)
{
try
{
if (string.IsNullOrEmpty(project.Id))
{
await CreateAsync(project);
}
else
{
var existing = await GetAsync(project.Id);
if (existing == null)
{
await CreateAsync(project);
}
else
{
await UpdateAsync(project.Id, project);
}
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao salvar projeto no Qdrant");
throw;
}
}
public async Task RemoveAsync(string id)
{
try
{
await _qdrantClient.DeleteAsync(
_collectionName,
new[] { new PointId { Uuid = id }.Num }
);
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao remover projeto {Id} do Qdrant", id);
throw;
}
}
private async Task CreateProjectsCollection()
{
var vectorParams = new VectorParams
{
Size = 384,
Distance = Distance.Cosine
};
await _qdrantClient.CreateCollectionAsync(_collectionName, vectorParams);
_logger.LogInformation("Collection de projetos '{CollectionName}' criada no Qdrant", _collectionName);
}
private static Project? ConvertToProject(RetrievedPoint point)
{
try
{
if (point.Payload == null) return null;
return new Project
{
Id = point.Payload.TryGetValue("id", out var idValue) ? idValue.StringValue : point.Id.ToString(),
Nome = point.Payload.TryGetValue("nome", out var nomeValue) ? nomeValue.StringValue : "",
Descricao = point.Payload.TryGetValue("descricao", out var descValue) ? descValue.StringValue : ""
};
}
catch
{
return null;
}
}
}
public class QdrantScrollResult
{
public QdrantScrollData? result { get; set; }
}
public class QdrantScrollData
{
public QdrantPoint[]? points { get; set; }
}
public class QdrantPointResult
{
public QdrantPoint? result { get; set; }
}
public class QdrantPoint
{
public string? id { get; set; }
public Dictionary<string, object>? payload { get; set; }
}
}

263
Data/hnwhoaao.xfh~ Normal file
View File

@ -0,0 +1,263 @@
using ChatRAG.Models;
using ChatRAG.Services.Contracts;
using ChatRAG.Settings.ChatRAG.Configuration;
using Microsoft.Extensions.Options;
using System.Text;
using System.Text.Json;
using Qdrant.Client;
using Qdrant.Client.Grpc;
namespace ChatRAG.Data
{
public class QdrantProjectDataRepository : IProjectDataRepository
{
private readonly HttpClient _httpClient;
private readonly string _collectionName;
private readonly ILogger<QdrantProjectDataRepository> _logger;
private readonly QdrantClient _qdrantClient;
private volatile bool _collectionInitialized = false;
private readonly SemaphoreSlim _initializationSemaphore = new(1, 1);
public QdrantProjectDataRepository(
IOptions<VectorDatabaseSettings> settings,
HttpClient httpClient,
ILogger<QdrantProjectDataRepository> logger)
{
var qdrantSettings = settings.Value.Qdrant ?? throw new ArgumentNullException("Qdrant settings not configured");
_httpClient = httpClient;
_httpClient.BaseAddress = new Uri($"http://{qdrantSettings.Host}:{qdrantSettings.Port}");
_collectionName = qdrantSettings.GroupsCollectionName;
_logger = logger;
// Inicializa o QdrantClient - use GRPC (porta 6334) para melhor performance
_qdrantClient = new QdrantClient(qdrantSettings.Host, port: 6334, https: false);
InitializeAsync().GetAwaiter().GetResult();
}
private async Task EnsureInitializedAsync()
{
try
{
if (_collectionInitialized) return;
await _initializationSemaphore.WaitAsync();
var exists = await _qdrantClient.CollectionExistsAsync(_collectionName);
if (!exists)
{
await CreateProjectsCollection();
}
_collectionInitialized = true;
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao inicializar collection de projetos no Qdrant");
}
}
public async Task<List<Project>> GetAsync()
{
try
{
//var scrollRequest = new ScrollPoints
//{
// CollectionName = _collectionName,
// Filter = new Filter(), // Filtro vazio
// Limit = 1000,
// WithPayload = true,
// WithVectors = false
//};
//var result = await _qdrantClient.ScrollAsync(_collectionName, scrollRequest);
var result = await _qdrantClient.ScrollAsync(_collectionName, new Filter(), 1000, null, true, false);
return result.Result.Select(ConvertToProject)
.Where(p => p != null)
.ToList()!;
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao recuperar projetos do Qdrant");
return new List<Project>();
}
}
public async Task<Project?> GetAsync(string id)
{
try
{
var points = await _qdrantClient.RetrieveAsync(
_collectionName,
new[] { PointId.Parser.ParseFrom(Encoding.ASCII.GetBytes(id)) },
withPayload: true,
withVectors: false
);
var point = points.FirstOrDefault();
return point != null ? ConvertToProject(point) : null;
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao buscar projeto {Id} no Qdrant", id);
return null;
}
}
public async Task CreateAsync(Project newProject)
{
try
{
var id = string.IsNullOrEmpty(newProject.Id) ? Guid.NewGuid().ToString() : newProject.Id;
newProject.Id = id;
var point = new PointStruct
{
Id = PointId.Parser.ParseFrom(Encoding.ASCII.GetBytes(id)),
Vectors = new float[384], // Vector dummy para projetos
Payload =
{
["id"] = newProject.Id,
["nome"] = newProject.Nome,
["descricao"] = newProject.Descricao,
["created_at"] = DateTime.UtcNow.ToString("O"),
["entity_type"] = "project"
}
};
await _qdrantClient.UpsertAsync(_collectionName, new[] { point });
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao criar projeto no Qdrant");
throw;
}
}
public async Task UpdateAsync(string id, Project updatedProject)
{
try
{
updatedProject.Id = id;
var point = new PointStruct
{
Id = PointId.Parser.ParseFrom(Encoding.ASCII.GetBytes(id)),
Vectors = new float[384], // Vector dummy
Payload =
{
["id"] = updatedProject.Id,
["nome"] = updatedProject.Nome,
["descricao"] = updatedProject.Descricao,
["updated_at"] = DateTime.UtcNow.ToString("O"),
["entity_type"] = "project"
}
};
await _qdrantClient.UpsertAsync(_collectionName, new[] { point });
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao atualizar projeto {Id} no Qdrant", id);
throw;
}
}
public async Task SaveAsync(Project project)
{
try
{
if (string.IsNullOrEmpty(project.Id))
{
await CreateAsync(project);
}
else
{
var existing = await GetAsync(project.Id);
if (existing == null)
{
await CreateAsync(project);
}
else
{
await UpdateAsync(project.Id, project);
}
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao salvar projeto no Qdrant");
throw;
}
}
public async Task RemoveAsync(string id)
{
try
{
await _qdrantClient.DeleteAsync(
_collectionName,
new[] { PointId.Parser.ParseFrom(Encoding.ASCII.GetBytes(id)).Num }
);
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao remover projeto {Id} do Qdrant", id);
throw;
}
}
private async Task CreateProjectsCollection()
{
var vectorParams = new VectorParams
{
Size = 384,
Distance = Distance.Cosine
};
await _qdrantClient.CreateCollectionAsync(_collectionName, vectorParams);
_logger.LogInformation("Collection de projetos '{CollectionName}' criada no Qdrant", _collectionName);
}
private static Project? ConvertToProject(RetrievedPoint point)
{
try
{
if (point.Payload == null) return null;
return new Project
{
Id = point.Payload.TryGetValue("id", out var idValue) ? idValue.StringValue : point.Id.ToString(),
Nome = point.Payload.TryGetValue("nome", out var nomeValue) ? nomeValue.StringValue : "",
Descricao = point.Payload.TryGetValue("descricao", out var descValue) ? descValue.StringValue : ""
};
}
catch
{
return null;
}
}
}
public class QdrantScrollResult
{
public QdrantScrollData? result { get; set; }
}
public class QdrantScrollData
{
public QdrantPoint[]? points { get; set; }
}
public class QdrantPointResult
{
public QdrantPoint? result { get; set; }
}
public class QdrantPoint
{
public string? id { get; set; }
public Dictionary<string, object>? payload { get; set; }
}
}

268
Data/wgbnjwfg.nr3~ Normal file
View File

@ -0,0 +1,268 @@
using ChatRAG.Models;
using ChatRAG.Services.Contracts;
using ChatRAG.Settings.ChatRAG.Configuration;
using Microsoft.Extensions.Options;
using System.Text;
using System.Text.Json;
using Qdrant.Client;
using Qdrant.Client.Grpc;
namespace ChatRAG.Data
{
public class QdrantProjectDataRepository : IProjectDataRepository
{
private readonly HttpClient _httpClient;
private readonly string _collectionName;
private readonly ILogger<QdrantProjectDataRepository> _logger;
private readonly QdrantClient _qdrantClient;
private volatile bool _collectionInitialized = false;
private readonly SemaphoreSlim _initializationSemaphore = new(1, 1);
public QdrantProjectDataRepository(
IOptions<VectorDatabaseSettings> settings,
HttpClient httpClient,
ILogger<QdrantProjectDataRepository> logger)
{
var qdrantSettings = settings.Value.Qdrant ?? throw new ArgumentNullException("Qdrant settings not configured");
_httpClient = httpClient;
_httpClient.BaseAddress = new Uri($"http://{qdrantSettings.Host}:{qdrantSettings.Port}");
_collectionName = qdrantSettings.GroupsCollectionName;
_logger = logger;
// Inicializa o QdrantClient - use GRPC (porta 6334) para melhor performance
_qdrantClient = new QdrantClient(qdrantSettings.Host, port: 6334, https: false);
InitializeAsync().GetAwaiter().GetResult();
}
private async Task InitializeAsync()
{
try
{
if (_collectionInitialized) return;
await _initializationSemaphore.WaitAsync();
var exists = await _qdrantClient.CollectionExistsAsync(_collectionName);
if (!exists)
{
await CreateProjectsCollection();
}
_collectionInitialized = true;
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao inicializar collection de projetos no Qdrant");
}
finally
{
_initializationSemaphore.Release();
}
}
public async Task<List<Project>> GetAsync()
{
try
{
//var scrollRequest = new ScrollPoints
//{
// CollectionName = _collectionName,
// Filter = new Filter(), // Filtro vazio
// Limit = 1000,
// WithPayload = true,
// WithVectors = false
//};
//var result = await _qdrantClient.ScrollAsync(_collectionName, scrollRequest);
var result = await _qdrantClient.ScrollAsync(_collectionName, new Filter(), 1000, null, true, false);
return result.Result.Select(ConvertToProject)
.Where(p => p != null)
.ToList()!;
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao recuperar projetos do Qdrant");
return new List<Project>();
}
}
public async Task<Project?> GetAsync(string id)
{
try
{
var points = await _qdrantClient.RetrieveAsync(
_collectionName,
new[] { PointId.Parser.ParseFrom(Encoding.ASCII.GetBytes(id)) },
withPayload: true,
withVectors: false
);
var point = points.FirstOrDefault();
return point != null ? ConvertToProject(point) : null;
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao buscar projeto {Id} no Qdrant", id);
return null;
}
}
public async Task CreateAsync(Project newProject)
{
try
{
var id = string.IsNullOrEmpty(newProject.Id) ? Guid.NewGuid().ToString() : newProject.Id;
newProject.Id = id;
var point = new PointStruct
{
Id = PointId.Parser.ParseFrom(Encoding.ASCII.GetBytes(id)),
Vectors = new float[384], // Vector dummy para projetos
Payload =
{
["id"] = newProject.Id,
["nome"] = newProject.Nome,
["descricao"] = newProject.Descricao,
["created_at"] = DateTime.UtcNow.ToString("O"),
["entity_type"] = "project"
}
};
await _qdrantClient.UpsertAsync(_collectionName, new[] { point });
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao criar projeto no Qdrant");
throw;
}
}
public async Task UpdateAsync(string id, Project updatedProject)
{
try
{
updatedProject.Id = id;
var point = new PointStruct
{
Id = PointId.Parser.ParseFrom(Encoding.ASCII.GetBytes(id)),
Vectors = new float[384], // Vector dummy
Payload =
{
["id"] = updatedProject.Id,
["nome"] = updatedProject.Nome,
["descricao"] = updatedProject.Descricao,
["updated_at"] = DateTime.UtcNow.ToString("O"),
["entity_type"] = "project"
}
};
await _qdrantClient.UpsertAsync(_collectionName, new[] { point });
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao atualizar projeto {Id} no Qdrant", id);
throw;
}
}
public async Task SaveAsync(Project project)
{
try
{
if (string.IsNullOrEmpty(project.Id))
{
await CreateAsync(project);
}
else
{
var existing = await GetAsync(project.Id);
if (existing == null)
{
await CreateAsync(project);
}
else
{
await UpdateAsync(project.Id, project);
}
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao salvar projeto no Qdrant");
throw;
}
}
public async Task RemoveAsync(string id)
{
try
{
await _qdrantClient.DeleteAsync(
_collectionName,
new[] { PointId.Parser.ParseFrom(Encoding.ASCII.GetBytes(id)).Num }
);
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao remover projeto {Id} do Qdrant", id);
throw;
}
}
private async Task CreateProjectsCollection()
{
var vectorParams = new VectorParams
{
Size = 384,
Distance = Distance.Cosine
};
await _qdrantClient.CreateCollectionAsync(_collectionName, vectorParams);
_logger.LogInformation("Collection de projetos '{CollectionName}' criada no Qdrant", _collectionName);
}
private static Project? ConvertToProject(RetrievedPoint point)
{
try
{
if (point.Payload == null) return null;
return new Project
{
Id = point.Payload.TryGetValue("id", out var idValue) ? idValue.StringValue : point.Id.ToString(),
Nome = point.Payload.TryGetValue("nome", out var nomeValue) ? nomeValue.StringValue : "",
Descricao = point.Payload.TryGetValue("descricao", out var descValue) ? descValue.StringValue : ""
};
}
catch
{
return null;
}
}
}
public class QdrantScrollResult
{
public QdrantScrollData? result { get; set; }
}
public class QdrantScrollData
{
public QdrantPoint[]? points { get; set; }
}
public class QdrantPointResult
{
public QdrantPoint? result { get; set; }
}
public class QdrantPoint
{
public string? id { get; set; }
public Dictionary<string, object>? payload { get; set; }
}
}

View File

@ -10,6 +10,7 @@ using ChatRAG.Services;
using ChatRAG.Services.Contracts; using ChatRAG.Services.Contracts;
using ChatRAG.Services.ResponseService; using ChatRAG.Services.ResponseService;
using ChatRAG.Services.SearchVectors; using ChatRAG.Services.SearchVectors;
using ChatRAG.Services.TextServices;
using ChatRAG.Settings.ChatRAG.Configuration; using ChatRAG.Settings.ChatRAG.Configuration;
using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.Authentication.JwtBearer;
using Microsoft.AspNetCore.Http.Features; using Microsoft.AspNetCore.Http.Features;
@ -78,7 +79,10 @@ builder.Services.AddSwaggerGen(c =>
builder.Services.Configure<ChatRHSettings>( builder.Services.Configure<ChatRHSettings>(
builder.Configuration.GetSection("ChatRHSettings")); builder.Configuration.GetSection("ChatRHSettings"));
builder.Services.AddScoped<IVectorSearchService, MongoVectorSearchService>(); //builder.Services.AddScoped<IVectorSearchService, MongoVectorSearchService>();
builder.Services.AddScoped<QdrantVectorSearchService>();
builder.Services.AddScoped<MongoVectorSearchService>();
builder.Services.AddScoped<ChromaVectorSearchService>();
builder.Services.AddVectorDatabase(builder.Configuration); builder.Services.AddVectorDatabase(builder.Configuration);
@ -89,12 +93,76 @@ builder.Services.AddScoped<IVectorSearchService>(provider =>
return factory.CreateVectorSearchService(); return factory.CreateVectorSearchService();
}); });
builder.Services.AddScoped<QdrantProjectDataRepository>();
builder.Services.AddScoped<MongoProjectDataRepository>();
builder.Services.AddScoped<ChromaProjectDataRepository>();
builder.Services.AddScoped<IProjectDataRepository>(provider =>
{
var database = builder.Configuration["VectorDatabase:Provider"];
if (string.IsNullOrEmpty(database))
{
throw new InvalidOperationException("VectorDatabase:Provider is not configured.");
}
else if (database.Equals("Qdrant", StringComparison.OrdinalIgnoreCase))
{
return provider.GetRequiredService<QdrantProjectDataRepository>();
}
else if (database.Equals("MongoDB", StringComparison.OrdinalIgnoreCase))
{
return provider.GetRequiredService<MongoProjectDataRepository>();
}
else if (database.Equals("Chroma", StringComparison.OrdinalIgnoreCase))
{
return provider.GetRequiredService<ChromaProjectDataRepository>();
}
return provider.GetRequiredService<MongoProjectDataRepository>();
});
builder.Services.AddScoped<QdrantTextDataService>();
builder.Services.AddScoped<MongoTextDataService>();
builder.Services.AddScoped<ChromaTextDataService>();
builder.Services.AddScoped<ITextDataService>(provider =>
{
var database = builder.Configuration["VectorDatabase:Provider"];
if (string.IsNullOrEmpty(database))
{
throw new InvalidOperationException("VectorDatabase:Provider is not configured.");
}
else if (database.Equals("Qdrant", StringComparison.OrdinalIgnoreCase))
{
return provider.GetRequiredService<QdrantTextDataService>();
}
else if (database.Equals("MongoDB", StringComparison.OrdinalIgnoreCase))
{
return provider.GetRequiredService<MongoTextDataService>();
}
else if (database.Equals("Chroma", StringComparison.OrdinalIgnoreCase))
{
return provider.GetRequiredService<ChromaTextDataService>();
}
return provider.GetRequiredService<MongoTextDataService>();
});
builder.Services.AddSingleton<ChatHistoryService>(); builder.Services.AddSingleton<ChatHistoryService>();
builder.Services.AddScoped<TextDataRepository>(); builder.Services.AddScoped<TextDataRepository>();
builder.Services.AddScoped<ProjectDataRepository>();
builder.Services.AddSingleton<TextFilter>(); builder.Services.AddSingleton<TextFilter>();
builder.Services.AddScoped<IResponseService, ResponseRAGService>(); //builder.Services.AddScoped<IResponseService, ResponseRAGService>();
builder.Services.AddScoped<ResponseRAGService>();
builder.Services.AddScoped<HierarchicalRAGService>();
builder.Services.AddScoped<IResponseService>(provider =>
{
var configuration = provider.GetService<IConfiguration>();
var useHierarchical = configuration?.GetValue<bool>("Features:UseHierarchicalRAG") ?? false;
return useHierarchical
? provider.GetRequiredService<HierarchicalRAGService>()
: provider.GetRequiredService<ResponseRAGService>();
});
builder.Services.AddTransient<UserDataRepository>(); builder.Services.AddTransient<UserDataRepository>();
builder.Services.AddTransient<TextData>(); builder.Services.AddTransient<TextData>();
builder.Services.AddSingleton<CryptUtil>(); builder.Services.AddSingleton<CryptUtil>();
@ -107,11 +175,11 @@ builder.Services.AddSingleton<CryptUtil>();
//Desktop //Desktop
//builder.Services.AddOllamaChatCompletion("llama3.2", new Uri("http://localhost:11434")); //builder.Services.AddOllamaChatCompletion("llama3.2", new Uri("http://localhost:11434"));
//Notebook //Notebook
var model = "meta-llama/Llama-3.2-3B-Instruct"; //var model = "meta-llama/Llama-3.2-3B-Instruct";
var url = "https://api.deepinfra.com/v1/openai"; // Adicione o /v1/openai //var url = "https://api.deepinfra.com/v1/openai"; // Adicione o /v1/openai
builder.Services.AddOpenAIChatCompletion(model, new Uri(url), "HedaR4yPrp9N2XSHfwdZjpZvPIxejPFK"); //builder.Services.AddOpenAIChatCompletion(model, new Uri(url), "HedaR4yPrp9N2XSHfwdZjpZvPIxejPFK");
//builder.Services.AddOllamaChatCompletion("llama3.2:3b", new Uri("http://localhost:11435")); builder.Services.AddOllamaChatCompletion("llama3.2:3b", new Uri("http://localhost:11435"));
//builder.Services.AddOllamaChatCompletion("llama3.2:1b", new Uri("http://localhost:11435")); //builder.Services.AddOllamaChatCompletion("llama3.2:1b", new Uri("http://localhost:11435"));

View File

@ -0,0 +1,14 @@
using ChatRAG.Models;
namespace ChatRAG.Services.Contracts
{
public interface IProjectDataRepository
{
Task<List<Project>> GetAsync();
Task<Project?> GetAsync(string id);
Task CreateAsync(Project newProject);
Task UpdateAsync(string id, Project updatedProject);
Task SaveAsync(Project project);
Task RemoveAsync(string id);
}
}

View File

@ -0,0 +1,375 @@
using ChatApi;
using ChatApi.Models;
using ChatRAG.Contracts.VectorSearch;
using ChatRAG.Data;
using ChatRAG.Models;
using ChatRAG.Services.Contracts;
using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.ChatCompletion;
using Microsoft.SemanticKernel.Connectors.OpenAI;
using Microsoft.SemanticKernel.Embeddings;
#pragma warning disable SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
namespace ChatRAG.Services.ResponseService
{
public class HierarchicalRAGService : IResponseService
{
private readonly ChatHistoryService _chatHistoryService;
private readonly Kernel _kernel;
private readonly TextFilter _textFilter;
private readonly ProjectDataRepository _projectDataRepository;
private readonly IChatCompletionService _chatCompletionService;
private readonly IVectorSearchService _vectorSearchService;
private readonly ILogger<HierarchicalRAGService> _logger;
public HierarchicalRAGService(
ChatHistoryService chatHistoryService,
Kernel kernel,
TextFilter textFilter,
ProjectDataRepository projectDataRepository,
IChatCompletionService chatCompletionService,
IVectorSearchService vectorSearchService,
ILogger<HierarchicalRAGService> logger)
{
_chatHistoryService = chatHistoryService;
_kernel = kernel;
_textFilter = textFilter;
_projectDataRepository = projectDataRepository;
_chatCompletionService = chatCompletionService;
_vectorSearchService = vectorSearchService;
_logger = logger;
}
public async Task<string> GetResponse(UserData userData, string projectId, string sessionId, string question, string language = "pt")
{
var stopWatch = new System.Diagnostics.Stopwatch();
stopWatch.Start();
try
{
// 1. Análise da query para determinar estratégia
var queryAnalysis = await AnalyzeQuery(question, language);
_logger.LogInformation("Query Analysis: {Strategy}, Complexity: {Complexity}",
queryAnalysis.Strategy, queryAnalysis.Complexity);
// 2. Execução hierárquica baseada na análise
var context = await ExecuteHierarchicalSearch(question, projectId, queryAnalysis);
// 3. Geração da resposta final
var response = await GenerateResponse(question, projectId, context, sessionId, language);
stopWatch.Stop();
return $"{response}\n\nTempo: {stopWatch.ElapsedMilliseconds / 1000}s\nEtapas: {context.Steps.Count}";
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro no RAG Hierárquico");
stopWatch.Stop();
return $"Erro: {ex.Message}\nTempo: {stopWatch.ElapsedMilliseconds / 1000}s";
}
}
private async Task<QueryAnalysis> AnalyzeQuery(string question, string language)
{
var analysisPrompt = language == "pt" ?
@"Analise esta pergunta e classifique:
PERGUNTA: ""{0}""
Responda APENAS no formato JSON:
{{
""strategy"": ""overview|specific|detailed"",
""complexity"": ""simple|medium|complex"",
""scope"": ""global|filtered|targeted"",
""concepts"": [""conceito1"", ""conceito2""],
""needs_hierarchy"": true|false
}}
REGRAS:
- overview: pergunta sobre projeto inteiro
- specific: pergunta sobre módulo/funcionalidade específica
- detailed: pergunta técnica que precisa de contexto profundo
- needs_hierarchy: true se precisar de múltiplas buscas" :
@"Analyze this question and classify:
QUESTION: ""{0}""
Answer ONLY in JSON format:
{{
""strategy"": ""overview|specific|detailed"",
""complexity"": ""simple|medium|complex"",
""scope"": ""global|filtered|targeted"",
""concepts"": [""concept1"", ""concept2""],
""needs_hierarchy"": true|false
}}
RULES:
- overview: question about entire project
- specific: question about specific module/functionality
- detailed: technical question needing deep context
- needs_hierarchy: true if needs multiple searches";
var prompt = string.Format(analysisPrompt, question);
var executionSettings = new OpenAIPromptExecutionSettings
{
Temperature = 0.1,
MaxTokens = 200
};
var response = await _chatCompletionService.GetChatMessageContentAsync(prompt, executionSettings);
try
{
var jsonResponse = response.Content?.Trim() ?? "{}";
// Extrair JSON se vier com texto extra
var startIndex = jsonResponse.IndexOf('{');
var endIndex = jsonResponse.LastIndexOf('}');
if (startIndex >= 0 && endIndex >= startIndex)
{
jsonResponse = jsonResponse.Substring(startIndex, endIndex - startIndex + 1);
}
var analysis = System.Text.Json.JsonSerializer.Deserialize<QueryAnalysis>(jsonResponse);
return analysis ?? new QueryAnalysis { Strategy = "specific", Complexity = "medium" };
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Erro ao parsear análise da query, usando padrão");
return new QueryAnalysis { Strategy = "specific", Complexity = "medium" };
}
}
private async Task<HierarchicalContext> ExecuteHierarchicalSearch(string question, string projectId, QueryAnalysis analysis)
{
var context = new HierarchicalContext();
var embeddingService = _kernel.GetRequiredService<ITextEmbeddingGenerationService>();
switch (analysis.Strategy)
{
case "overview":
await ExecuteOverviewStrategy(context, question, projectId, embeddingService);
break;
case "detailed":
await ExecuteDetailedStrategy(context, question, projectId, embeddingService, analysis);
break;
default: // specific
await ExecuteSpecificStrategy(context, question, projectId, embeddingService);
break;
}
return context;
}
private async Task ExecuteOverviewStrategy(HierarchicalContext context, string question, string projectId, ITextEmbeddingGenerationService embeddingService)
{
// Etapa 1: Buscar resumos/títulos primeiro
context.AddStep("Buscando visão geral do projeto");
var overviewResults = await _vectorSearchService.GetDocumentsByProjectAsync(projectId);
// Etapa 2: Identificar documentos principais baseado na pergunta
var questionEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(question));
var embeddingArray = questionEmbedding.ToArray().Select(e => (double)e).ToArray();
context.AddStep("Identificando documentos relevantes");
var relevantDocs = await _vectorSearchService.SearchSimilarAsync(embeddingArray, projectId, 0.3, 5);
context.CombinedContext = $"VISÃO GERAL DO PROJETO:\n{FormatResults(overviewResults.Take(3))}\n\nDOCUMENTOS RELEVANTES:\n{FormatResults(relevantDocs)}";
}
private async Task ExecuteSpecificStrategy(HierarchicalContext context, string question, string projectId, ITextEmbeddingGenerationService embeddingService)
{
// Etapa 1: Busca inicial por similaridade
context.AddStep("Busca inicial por similaridade");
var questionEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(question));
var embeddingArray = questionEmbedding.ToArray().Select(e => (double)e).ToArray();
var initialResults = await _vectorSearchService.SearchSimilarAsync(embeddingArray, projectId, 0.4, 3);
if (initialResults.Any())
{
context.AddStep("Expandindo contexto com documentos relacionados");
// Etapa 2: Expandir com contexto relacionado
var expandedContext = await ExpandContext(initialResults, projectId, embeddingService);
context.CombinedContext = $"CONTEXTO PRINCIPAL:\n{FormatResults(initialResults)}\n\nCONTEXTO EXPANDIDO:\n{FormatResults(expandedContext)}";
}
else
{
context.AddStep("Fallback para busca ampla");
var fallbackResults = await _vectorSearchService.SearchSimilarAsync(embeddingArray, projectId, 0.2, 5);
context.CombinedContext = FormatResults(fallbackResults);
}
}
private async Task ExecuteDetailedStrategy(HierarchicalContext context, string question, string projectId, ITextEmbeddingGenerationService embeddingService, QueryAnalysis analysis)
{
var questionEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(question));
var embeddingArray = questionEmbedding.ToArray().Select(e => (double)e).ToArray();
// Etapa 1: Busca conceitual baseada nos conceitos identificados
context.AddStep("Busca conceitual inicial");
var conceptualResults = new List<VectorSearchResult>();
if (analysis.Concepts?.Any() == true)
{
foreach (var concept in analysis.Concepts)
{
var conceptEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(concept));
var conceptArray = conceptEmbedding.ToArray().Select(e => (double)e).ToArray();
var conceptResults = await _vectorSearchService.SearchSimilarAsync(conceptArray, projectId, 0.3, 2);
conceptualResults.AddRange(conceptResults);
}
}
// Etapa 2: Busca direta pela pergunta
context.AddStep("Busca direta pela pergunta");
var directResults = await _vectorSearchService.SearchSimilarAsync(embeddingArray, projectId, 0.3, 3);
// Etapa 3: Síntese intermediária para identificar lacunas
context.AddStep("Identificando lacunas de conhecimento");
var intermediateContext = FormatResults(conceptualResults.Concat(directResults).DistinctBy(r => r.Id));
var gaps = await IdentifyKnowledgeGaps(question, intermediateContext);
// Etapa 4: Busca complementar baseada nas lacunas
if (!string.IsNullOrEmpty(gaps))
{
context.AddStep("Preenchendo lacunas de conhecimento");
var gapEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(gaps));
var gapArray = gapEmbedding.ToArray().Select(e => (double)e).ToArray();
var gapResults = await _vectorSearchService.SearchSimilarAsync(gapArray, projectId, 0.25, 2);
context.CombinedContext = $"CONTEXTO CONCEITUAL:\n{FormatResults(conceptualResults)}\n\nCONTEXTO DIRETO:\n{FormatResults(directResults)}\n\nCONTEXTO COMPLEMENTAR:\n{FormatResults(gapResults)}";
}
else
{
context.CombinedContext = $"CONTEXTO CONCEITUAL:\n{FormatResults(conceptualResults)}\n\nCONTEXTO DIRETO:\n{FormatResults(directResults)}";
}
}
private async Task<List<VectorSearchResult>> ExpandContext(List<VectorSearchResult> initialResults, string projectId, ITextEmbeddingGenerationService embeddingService)
{
var expandedResults = new List<VectorSearchResult>();
// Para cada resultado inicial, buscar documentos relacionados
foreach (var result in initialResults.Take(2)) // Limitar para evitar explosão de contexto
{
var resultEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(result.Content));
var embeddingArray = resultEmbedding.ToArray().Select(e => (double)e).ToArray();
var relatedDocs = await _vectorSearchService.SearchSimilarAsync(embeddingArray, projectId, 0.4, 2);
expandedResults.AddRange(relatedDocs.Where(r => !initialResults.Any(ir => ir.Id == r.Id)));
}
return expandedResults.DistinctBy(r => r.Id).ToList();
}
private async Task<string> IdentifyKnowledgeGaps(string question, string currentContext)
{
var gapPrompt = @"Baseado na pergunta e contexto atual, identifique que informações ainda faltam para uma resposta completa.
PERGUNTA: {0}
CONTEXTO ATUAL: {1}
Responda APENAS com palavras-chave dos conceitos/informações que ainda faltam, separados por vírgula.
Se o contexto for suficiente, responda 'SUFICIENTE'.";
var prompt = string.Format(gapPrompt, question, currentContext.Substring(0, Math.Min(1000, currentContext.Length)));
var executionSettings = new OpenAIPromptExecutionSettings
{
Temperature = 0.2,
MaxTokens = 100
};
var response = await _chatCompletionService.GetChatMessageContentAsync(prompt, executionSettings);
var gaps = response.Content?.Trim() ?? "";
return gaps.Equals("SUFICIENTE", StringComparison.OrdinalIgnoreCase) ? "" : gaps;
}
private async Task<string> GenerateResponse(string question, string projectId, HierarchicalContext context, string sessionId, string language)
{
var projectData = await _projectDataRepository.GetAsync(projectId);
var project = $"Nome: {projectData.Nome} \n\n Descrição:{projectData.Descricao}";
var prompt = language == "pt" ?
@"Você é um especialista em análise de software e QA.
PROJETO: {0}
PERGUNTA: ""{1}""
CONTEXTO HIERÁRQUICO: {2}
ETAPAS EXECUTADAS: {3}
Responda à pergunta de forma precisa e estruturada, aproveitando todo o contexto hierárquico coletado." :
@"You are a software analysis and QA expert.
PROJECT: {0}
QUESTION: ""{1}""
HIERARCHICAL CONTEXT: {2}
EXECUTED STEPS: {3}
Answer the question precisely and structured, leveraging all the hierarchical context collected.";
var finalPrompt = string.Format(prompt, project, question, context.CombinedContext,
string.Join(" → ", context.Steps));
var history = _chatHistoryService.GetSumarizer(sessionId);
history.AddUserMessage(finalPrompt);
var executionSettings = new OpenAIPromptExecutionSettings
{
Temperature = 0.7,
TopP = 1.0,
FrequencyPenalty = 0,
PresencePenalty = 0
};
var response = await _chatCompletionService.GetChatMessageContentAsync(history, executionSettings);
history.AddMessage(response.Role, response.Content ?? "");
_chatHistoryService.UpdateHistory(sessionId, history);
return response.Content ?? "";
}
private string FormatResults(IEnumerable<VectorSearchResult> results)
{
return string.Join("\n\n", results.Select((item, index) =>
$"=== DOCUMENTO {index + 1} ===\n" +
$"Relevância: {item.Score:P1}\n" +
$"Conteúdo: {item.Content}"));
}
public Task<string> GetResponse(UserData userData, string projectId, string sessionId, string question)
{
return GetResponse(userData, projectId, sessionId, question, "pt");
}
}
// Classes de apoio para o RAG Hierárquico
public class QueryAnalysis
{
public string Strategy { get; set; } = "specific";
public string Complexity { get; set; } = "medium";
public string Scope { get; set; } = "filtered";
public string[] Concepts { get; set; } = Array.Empty<string>();
public bool Needs_Hierarchy { get; set; } = false;
}
public class HierarchicalContext
{
public List<string> Steps { get; set; } = new();
public string CombinedContext { get; set; } = "";
public Dictionary<string, object> Metadata { get; set; } = new();
public void AddStep(string step)
{
Steps.Add($"{DateTime.Now:HH:mm:ss} - {step}");
}
}
}
#pragma warning restore SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.

View File

@ -0,0 +1,546 @@
using ChatApi;
using ChatApi.Models;
using ChatRAG.Contracts.VectorSearch;
using ChatRAG.Data;
using ChatRAG.Models;
using ChatRAG.Services.Contracts;
using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.ChatCompletion;
using Microsoft.SemanticKernel.Connectors.OpenAI;
using Microsoft.SemanticKernel.Embeddings;
using System.Text.Json;
#pragma warning disable SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
namespace ChatRAG.Services.ResponseService
{
public class HierarchicalRAGService : IResponseService
{
private readonly ChatHistoryService _chatHistoryService;
private readonly Kernel _kernel;
private readonly TextFilter _textFilter;
private readonly IProjectDataRepository _projectDataRepository;
private readonly IChatCompletionService _chatCompletionService;
private readonly IVectorSearchService _vectorSearchService;
private readonly ILogger<HierarchicalRAGService> _logger;
public HierarchicalRAGService(
ChatHistoryService chatHistoryService,
Kernel kernel,
TextFilter textFilter,
IProjectDataRepository projectDataRepository,
IChatCompletionService chatCompletionService,
IVectorSearchService vectorSearchService,
ILogger<HierarchicalRAGService> logger)
{
_chatHistoryService = chatHistoryService;
_kernel = kernel;
_textFilter = textFilter;
_projectDataRepository = projectDataRepository;
_chatCompletionService = chatCompletionService;
_vectorSearchService = vectorSearchService;
_logger = logger;
}
public async Task<string> GetResponse(UserData userData, string projectId, string sessionId, string question, string language = "pt")
{
var stopWatch = new System.Diagnostics.Stopwatch();
stopWatch.Start();
try
{
// 1. Análise da query para determinar estratégia
var queryAnalysis = await AnalyzeQuery(question, language);
_logger.LogInformation("Query Analysis: {Strategy}, Complexity: {Complexity}",
queryAnalysis.Strategy, queryAnalysis.Complexity);
// 2. Execução hierárquica baseada na análise
var context = await ExecuteHierarchicalSearch(question, projectId, queryAnalysis);
// 3. Geração da resposta final
var response = await GenerateResponse(question, projectId, context, sessionId, language);
stopWatch.Stop();
return $"{response}\n\nTempo: {stopWatch.ElapsedMilliseconds / 1000}s\nEtapas: {context.Steps.Count}";
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro no RAG Hierárquico");
stopWatch.Stop();
return $"Erro: {ex.Message}\nTempo: {stopWatch.ElapsedMilliseconds / 1000}s";
}
}
private async Task<QueryAnalysis> AnalyzeQuery(string question, string language)
{
var analysisPrompt = language == "pt" ?
@"Analise esta pergunta e classifique com precisão:
PERGUNTA: ""{0}""
Responda APENAS no formato JSON:
{{
""strategy"": ""overview|specific|detailed"",
""complexity"": ""simple|medium|complex"",
""scope"": ""global|filtered|targeted"",
""concepts"": [""conceito1"", ""conceito2""],
""needs_hierarchy"": true|false
}}
DEFINIÇÕES PRECISAS:
STRATEGY:
- overview: Pergunta sobre o PROJETO COMO UM TODO. Palavras-chave: ""projeto"", ""sistema"", ""aplicação"", ""este projeto"", ""todo o"", ""geral"", ""inteiro"". NÃO menciona módulos, funcionalidades ou tecnologias específicas.
- specific: Pergunta sobre MÓDULO/FUNCIONALIDADE ESPECÍFICA. Menciona: nome de classe, controller, entidade, CRUD específico, funcionalidade particular, tecnologia específica.
- detailed: Pergunta técnica específica que precisa de CONTEXTO PROFUNDO e detalhes de implementação.
SCOPE:
- global: Busca informações de TODO o projeto (usar com overview)
- filtered: Busca com filtros específicos (usar com specific/detailed)
- targeted: Busca muito específica e direcionada
EXEMPLOS:
- ""Gere casos de teste para este projeto"" overview/global
- ""Gere casos de teste do projeto"" overview/global
- ""Gere casos de teste para o CRUD de usuário"" specific/filtered
- ""Como implementar autenticação JWT neste controller"" detailed/targeted
- ""Documente este sistema"" overview/global
- ""Explique a classe UserService"" specific/filtered" :
@"Analyze this question and classify precisely:
QUESTION: ""{0}""
Answer ONLY in JSON format:
{{
""strategy"": ""overview|specific|detailed"",
""complexity"": ""simple|medium|complex"",
""scope"": ""global|filtered|targeted"",
""concepts"": [""concept1"", ""concept2""],
""needs_hierarchy"": true|false
}}
PRECISE DEFINITIONS:
STRATEGY:
- overview: Question about the PROJECT AS A WHOLE. Keywords: ""project"", ""system"", ""application"", ""this project"", ""entire"", ""general"", ""whole"". Does NOT mention specific modules, functionalities or technologies.
- specific: Question about SPECIFIC MODULE/FUNCTIONALITY. Mentions: class name, controller, entity, specific CRUD, particular functionality, specific technology.
- detailed: Technical specific question needing DEEP CONTEXT and implementation details.
SCOPE:
- global: Search information from ENTIRE project (use with overview)
- filtered: Search with specific filters (use with specific/detailed)
- targeted: Very specific and directed search
EXAMPLES:
- ""Generate test cases for this project"" overview/global
- ""Generate test cases for user CRUD"" specific/filtered
- ""How to implement JWT authentication in this controller"" detailed/targeted
- ""Document this system"" overview/global
- ""Explain the UserService class"" specific/filtered";
var prompt = string.Format(analysisPrompt, question);
var executionSettings = new OpenAIPromptExecutionSettings
{
Temperature = 0.1,
MaxTokens = 300 // Aumentei um pouco para acomodar o prompt maior
};
var response = await _chatCompletionService.GetChatMessageContentAsync(prompt, executionSettings);
try
{
var jsonResponse = response.Content?.Trim() ?? "{}";
// Extrair JSON se vier com texto extra
var startIndex = jsonResponse.IndexOf('{');
var endIndex = jsonResponse.LastIndexOf('}');
if (startIndex >= 0 && endIndex >= startIndex)
{
jsonResponse = jsonResponse.Substring(startIndex, endIndex - startIndex + 1);
}
var options = new JsonSerializerOptions
{
PropertyNameCaseInsensitive = true,
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
};
var analysis = System.Text.Json.JsonSerializer.Deserialize<QueryAnalysis>(jsonResponse, options);
// Log para debug - remover em produção
_logger.LogInformation($"Query: '{question}' → Strategy: {analysis?.Strategy}, Scope: {analysis?.Scope}");
return analysis ?? new QueryAnalysis { Strategy = "specific", Complexity = "medium" };
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Erro ao parsear análise da query, usando padrão");
return new QueryAnalysis { Strategy = "specific", Complexity = "medium" };
}
}
private async Task<HierarchicalContext> ExecuteHierarchicalSearch(string question, string projectId, QueryAnalysis analysis)
{
var context = new HierarchicalContext();
var embeddingService = _kernel.GetRequiredService<ITextEmbeddingGenerationService>();
switch (analysis.Strategy)
{
case "overview":
await ExecuteOverviewStrategy(context, question, projectId, embeddingService);
break;
case "detailed":
await ExecuteDetailedStrategy(context, question, projectId, embeddingService, analysis);
break;
default: // specific
await ExecuteSpecificStrategy(context, question, projectId, embeddingService);
break;
}
return context;
}
private async Task ExecuteOverviewStrategy(HierarchicalContext context, string question, string projectId, ITextEmbeddingGenerationService embeddingService)
{
// Etapa 1: Buscar TODOS os documentos do projeto
context.AddStep("Buscando todos os documentos do projeto");
var allProjectDocs = await _vectorSearchService.GetDocumentsByProjectAsync(projectId);
// Etapa 2: Categorizar documentos por tipo/importância
context.AddStep("Categorizando e resumindo contexto do projeto");
// Etapa 2: Categorizar documentos por tipo baseado nos seus dados reais
context.AddStep("Categorizando e resumindo contexto do projeto");
var requirementsDocs = allProjectDocs.Where(d =>
d.Title.ToLower().StartsWith("requisito") ||
d.Title.ToLower().Contains("requisito") ||
d.Content.ToLower().Contains("requisito") ||
d.Content.ToLower().Contains("funcionalidade") ||
d.Content.ToLower().Contains("aplicação deve") ||
d.Content.ToLower().Contains("sistema deve")).ToList();
var architectureDocs = allProjectDocs.Where(d =>
d.Title.ToLower().Contains("arquitetura") ||
d.Title.ToLower().Contains("estrutura") ||
d.Title.ToLower().Contains("documentação") ||
d.Title.ToLower().Contains("readme") ||
d.Content.ToLower().Contains("arquitetura") ||
d.Content.ToLower().Contains("estrutura") ||
d.Content.ToLower().Contains("tecnologia")).ToList();
// Documentos que não são requisitos nem arquitetura (códigos, outros docs)
var otherDocs = allProjectDocs
.Except(requirementsDocs)
.Except(architectureDocs)
.ToList();
// Etapa 3: Resumir cada categoria se tiver muitos documentos
var requirementsSummary = await SummarizeDocuments(requirementsDocs, "requisitos e funcionalidades do projeto");
var architectureSummary = await SummarizeDocuments(architectureDocs, "arquitetura e documentação técnica");
var otherSummary = await SummarizeDocuments(otherDocs, "outros documentos do projeto");
// Etapa 4: Busca específica para a pergunta (mantém precisão)
var questionEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(question));
var embeddingArray = questionEmbedding.ToArray().Select(e => (double)e).ToArray();
context.AddStep("Identificando documentos específicos para a pergunta");
var relevantDocs = await _vectorSearchService.SearchSimilarAsync(embeddingArray, projectId, 0.3, 8);
// Etapa 5: Combinar resumos + documentos específicos
var contextParts = new List<string>();
if (!string.IsNullOrEmpty(requirementsSummary))
contextParts.Add($"RESUMO DOS REQUISITOS E FUNCIONALIDADES:\n{requirementsSummary}");
if (!string.IsNullOrEmpty(architectureSummary))
contextParts.Add($"RESUMO DA ARQUITETURA E DOCUMENTAÇÃO:\n{architectureSummary}");
if (!string.IsNullOrEmpty(otherSummary))
contextParts.Add($"OUTROS DOCUMENTOS DO PROJETO:\n{otherSummary}");
contextParts.Add($"DOCUMENTOS MAIS RELEVANTES PARA A PERGUNTA:\n{FormatResults(relevantDocs)}");
context.CombinedContext = string.Join("\n\n", contextParts);
}
private async Task<string> SummarizeDocuments(List<VectorSearchResult> documents, string category)
{
if (!documents.Any()) return string.Empty;
// Se poucos documentos, usar todos sem resumir
if (documents.Count <= 3)
{
return FormatResults(documents);
}
// Se muitos documentos, resumir em chunks
var chunks = documents.Chunk(5).ToList(); // Grupos de 5 documentos
var tasks = new List<Task<string>>();
// Semáforo para controlar concorrência (máximo 3 chamadas simultâneas)
var semaphore = new SemaphoreSlim(3, 3);
foreach (var chunk in chunks)
{
var chunkContent = FormatResults(chunk);
tasks.Add(Task.Run(async () =>
{
await semaphore.WaitAsync();
try
{
var summaryPrompt = $@"Resuma os pontos principais destes documentos sobre {category}:
{chunkContent}
Responda apenas com uma lista concisa dos pontos mais importantes:";
var response = await _chatCompletionService.GetChatMessageContentAsync(
summaryPrompt,
new OpenAIPromptExecutionSettings
{
Temperature = 0.1,
MaxTokens = 300
});
return response.Content ?? string.Empty;
}
catch (Exception ex)
{
_logger.LogWarning(ex, $"Erro ao resumir chunk de {category}, usando conteúdo original");
return chunkContent;
}
finally
{
semaphore.Release();
}
}));
}
// Aguardar todas as tasks de resumo
var summaries = await Task.WhenAll(tasks);
var validSummaries = summaries.Where(s => !string.IsNullOrEmpty(s)).ToList();
// Se tiver múltiplos resumos, consolidar
if (validSummaries.Count > 1)
{
var consolidationPrompt = $@"Consolide estes resumos sobre {category} em um resumo final:
{string.Join("\n\n", validSummaries)}
Responda com os pontos mais importantes organizados:";
try
{
var finalResponse = await _chatCompletionService.GetChatMessageContentAsync(
consolidationPrompt,
new OpenAIPromptExecutionSettings
{
Temperature = 0.1,
MaxTokens = 400
});
return finalResponse.Content ?? string.Empty;
}
catch
{
return string.Join("\n\n", validSummaries);
}
}
return validSummaries.FirstOrDefault() ?? string.Empty;
}
private async Task ExecuteSpecificStrategy(HierarchicalContext context, string question, string projectId, ITextEmbeddingGenerationService embeddingService)
{
// Etapa 1: Busca inicial por similaridade
context.AddStep("Busca inicial por similaridade");
var questionEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(question));
var embeddingArray = questionEmbedding.ToArray().Select(e => (double)e).ToArray();
var initialResults = await _vectorSearchService.SearchSimilarAsync(embeddingArray, projectId, 0.4, 3);
if (initialResults.Any())
{
context.AddStep("Expandindo contexto com documentos relacionados");
// Etapa 2: Expandir com contexto relacionado
var expandedContext = await ExpandContext(initialResults, projectId, embeddingService);
context.CombinedContext = $"CONTEXTO PRINCIPAL:\n{FormatResults(initialResults)}\n\nCONTEXTO EXPANDIDO:\n{FormatResults(expandedContext)}";
}
else
{
context.AddStep("Fallback para busca ampla");
var fallbackResults = await _vectorSearchService.SearchSimilarAsync(embeddingArray, projectId, 0.2, 5);
context.CombinedContext = FormatResults(fallbackResults);
}
}
private async Task ExecuteDetailedStrategy(HierarchicalContext context, string question, string projectId, ITextEmbeddingGenerationService embeddingService, QueryAnalysis analysis)
{
var questionEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(question));
var embeddingArray = questionEmbedding.ToArray().Select(e => (double)e).ToArray();
// Etapa 1: Busca conceitual baseada nos conceitos identificados
context.AddStep("Busca conceitual inicial");
var conceptualResults = new List<VectorSearchResult>();
if (analysis.Concepts?.Any() == true)
{
foreach (var concept in analysis.Concepts)
{
var conceptEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(concept));
var conceptArray = conceptEmbedding.ToArray().Select(e => (double)e).ToArray();
var conceptResults = await _vectorSearchService.SearchSimilarAsync(conceptArray, projectId, 0.3, 2);
conceptualResults.AddRange(conceptResults);
}
}
// Etapa 2: Busca direta pela pergunta
context.AddStep("Busca direta pela pergunta");
var directResults = await _vectorSearchService.SearchSimilarAsync(embeddingArray, projectId, 0.3, 3);
// Etapa 3: Síntese intermediária para identificar lacunas
context.AddStep("Identificando lacunas de conhecimento");
var intermediateContext = FormatResults(conceptualResults.Concat(directResults).DistinctBy(r => r.Id));
var gaps = await IdentifyKnowledgeGaps(question, intermediateContext);
// Etapa 4: Busca complementar baseada nas lacunas
if (!string.IsNullOrEmpty(gaps))
{
context.AddStep("Preenchendo lacunas de conhecimento");
var gapEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(gaps));
var gapArray = gapEmbedding.ToArray().Select(e => (double)e).ToArray();
var gapResults = await _vectorSearchService.SearchSimilarAsync(gapArray, projectId, 0.25, 2);
context.CombinedContext = $"CONTEXTO CONCEITUAL:\n{FormatResults(conceptualResults)}\n\nCONTEXTO DIRETO:\n{FormatResults(directResults)}\n\nCONTEXTO COMPLEMENTAR:\n{FormatResults(gapResults)}";
}
else
{
context.CombinedContext = $"CONTEXTO CONCEITUAL:\n{FormatResults(conceptualResults)}\n\nCONTEXTO DIRETO:\n{FormatResults(directResults)}";
}
}
private async Task<List<VectorSearchResult>> ExpandContext(List<VectorSearchResult> initialResults, string projectId, ITextEmbeddingGenerationService embeddingService)
{
var expandedResults = new List<VectorSearchResult>();
// Para cada resultado inicial, buscar documentos relacionados
foreach (var result in initialResults.Take(2)) // Limitar para evitar explosão de contexto
{
var resultEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(result.Content));
var embeddingArray = resultEmbedding.ToArray().Select(e => (double)e).ToArray();
var relatedDocs = await _vectorSearchService.SearchSimilarAsync(embeddingArray, projectId, 0.4, 2);
expandedResults.AddRange(relatedDocs.Where(r => !initialResults.Any(ir => ir.Id == r.Id)));
}
return expandedResults.DistinctBy(r => r.Id).ToList();
}
private async Task<string> IdentifyKnowledgeGaps(string question, string currentContext)
{
var gapPrompt = @"Baseado na pergunta e contexto atual, identifique que informações ainda faltam para uma resposta completa.
PERGUNTA: {0}
CONTEXTO ATUAL: {1}
Responda APENAS com palavras-chave dos conceitos/informações que ainda faltam, separados por vírgula.
Se o contexto for suficiente, responda 'SUFICIENTE'.";
var prompt = string.Format(gapPrompt, question, currentContext.Substring(0, Math.Min(1000, currentContext.Length)));
var executionSettings = new OpenAIPromptExecutionSettings
{
Temperature = 0.2,
MaxTokens = 100
};
var response = await _chatCompletionService.GetChatMessageContentAsync(prompt, executionSettings);
var gaps = response.Content?.Trim() ?? "";
return gaps.Equals("SUFICIENTE", StringComparison.OrdinalIgnoreCase) ? "" : gaps;
}
private async Task<string> GenerateResponse(string question, string projectId, HierarchicalContext context, string sessionId, string language)
{
var projectData = await _projectDataRepository.GetAsync(projectId);
var project = $"Nome: {projectData.Nome} \n\n Descrição:{projectData.Descricao}";
var prompt = language == "pt" ?
@"Você é um especialista em análise de software e QA.
PROJETO: {0}
PERGUNTA: ""{1}""
CONTEXTO HIERÁRQUICO: {2}
ETAPAS EXECUTADAS: {3}
Responda à pergunta de forma precisa e estruturada, aproveitando todo o contexto hierárquico coletado." :
@"You are a software analysis and QA expert.
PROJECT: {0}
QUESTION: ""{1}""
HIERARCHICAL CONTEXT: {2}
EXECUTED STEPS: {3}
Answer the question precisely and structured, leveraging all the hierarchical context collected.";
var finalPrompt = string.Format(prompt, project, question, context.CombinedContext,
string.Join(" → ", context.Steps));
var history = _chatHistoryService.GetSumarizer(sessionId);
history.AddUserMessage(finalPrompt);
var executionSettings = new OpenAIPromptExecutionSettings
{
Temperature = 0.7,
TopP = 1.0,
FrequencyPenalty = 0,
PresencePenalty = 0
};
var response = await _chatCompletionService.GetChatMessageContentAsync(history, executionSettings);
history.AddMessage(response.Role, response.Content ?? "");
_chatHistoryService.UpdateHistory(sessionId, history);
return response.Content ?? "";
}
private string FormatResults(IEnumerable<VectorSearchResult> results)
{
return string.Join("\n\n", results.Select((item, index) =>
$"=== DOCUMENTO {index + 1} ===\n" +
$"Relevância: {item.Score:P1}\n" +
$"Conteúdo: {item.Content}"));
}
public Task<string> GetResponse(UserData userData, string projectId, string sessionId, string question)
{
return GetResponse(userData, projectId, sessionId, question, "pt");
}
}
// Classes de apoio para o RAG Hierárquico
public class QueryAnalysis
{
public string Strategy { get; set; } = "specific";
public string Complexity { get; set; } = "medium";
public string Scope { get; set; } = "filtered";
public string[] Concepts { get; set; } = Array.Empty<string>();
public bool Needs_Hierarchy { get; set; } = false;
}
public class HierarchicalContext
{
public List<string> Steps { get; set; } = new();
public string CombinedContext { get; set; } = "";
public Dictionary<string, object> Metadata { get; set; } = new();
public void AddStep(string step)
{
Steps.Add($"{DateTime.Now:HH:mm:ss} - {step}");
}
}
}
#pragma warning restore SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.

View File

@ -20,7 +20,7 @@ namespace ChatRAG.Services.ResponseService
private readonly Kernel _kernel; private readonly Kernel _kernel;
private readonly TextFilter _textFilter; private readonly TextFilter _textFilter;
private readonly TextDataRepository _textDataRepository; private readonly TextDataRepository _textDataRepository;
private readonly ProjectDataRepository _projectDataRepository; private readonly IProjectDataRepository _projectDataRepository;
private readonly IChatCompletionService _chatCompletionService; private readonly IChatCompletionService _chatCompletionService;
private readonly IVectorSearchService _vectorSearchService; private readonly IVectorSearchService _vectorSearchService;
@ -29,7 +29,7 @@ namespace ChatRAG.Services.ResponseService
Kernel kernel, Kernel kernel,
TextFilter textFilter, TextFilter textFilter,
TextDataRepository textDataRepository, TextDataRepository textDataRepository,
ProjectDataRepository projectDataRepository, IProjectDataRepository projectDataRepository,
IChatCompletionService chatCompletionService, IChatCompletionService chatCompletionService,
IVectorSearchService vectorSearchService, IVectorSearchService vectorSearchService,
ITextDataService textDataService) ITextDataService textDataService)

View File

@ -0,0 +1,651 @@
using ChatRAG.Contracts.VectorSearch;
using ChatRAG.Models;
using Microsoft.Extensions.Options;
using System.Text;
using System.Text.Json;
using Microsoft.SemanticKernel.Embeddings;
using ChatRAG.Settings.ChatRAG.Configuration;
namespace ChatRAG.Services.SearchVectors
{
public class ChromaVectorSearchService : IVectorSearchService
{
private readonly HttpClient _httpClient;
private readonly ILogger<ChromaVectorSearchService> _logger;
private readonly ChromaSettings _settings;
private readonly string _collectionName;
public ChromaVectorSearchService(
IOptions<VectorDatabaseSettings> settings,
ILogger<ChromaVectorSearchService> logger,
HttpClient httpClient)
{
_settings = settings.Value.Chroma ?? throw new ArgumentNullException("Chroma settings not configured");
_logger = logger;
_httpClient = httpClient;
_httpClient.BaseAddress = new Uri($"http://{_settings.Host}:{_settings.Port}");
_collectionName = _settings.CollectionName;
InitializeAsync().GetAwaiter().GetResult();
}
private async Task InitializeAsync()
{
try
{
// Verificar se a collection existe, se não, criar
var collections = await GetCollectionsAsync();
if (!collections.Contains(_collectionName))
{
await CreateCollectionAsync();
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao inicializar Chroma");
throw;
}
}
// ========================================
// BUSCA VETORIAL
// ========================================
public async Task<List<VectorSearchResult>> SearchSimilarAsync(
double[] queryEmbedding,
string? projectId = null,
double threshold = 0.3,
int limit = 5,
Dictionary<string, object>? filters = null)
{
try
{
// Construir filtros WHERE
var whereClause = BuildWhereClause(projectId, filters);
var query = new
{
query_embeddings = new[] { queryEmbedding },
n_results = limit,
where = whereClause,
include = new[] { "documents", "metadatas", "distances" }
};
var json = JsonSerializer.Serialize(query);
var content = new StringContent(json, Encoding.UTF8, "application/json");
var response = await _httpClient.PostAsync($"/api/v1/collections/{_collectionName}/query", content);
if (!response.IsSuccessStatusCode)
{
var error = await response.Content.ReadAsStringAsync();
_logger.LogError("Erro na busca Chroma: {Error}", error);
return new List<VectorSearchResult>();
}
var result = await response.Content.ReadAsStringAsync();
var queryResult = JsonSerializer.Deserialize<ChromaQueryResult>(result);
return ParseQueryResults(queryResult, threshold);
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao buscar similares no Chroma");
return new List<VectorSearchResult>();
}
}
public async Task<List<VectorSearchResult>> SearchSimilarDynamicAsync(
double[] queryEmbedding,
string projectId,
double minThreshold = 0.5,
int limit = 5)
{
// Estratégia 1: Busca com threshold alto
var results = await SearchSimilarAsync(queryEmbedding, projectId, minThreshold, limit);
if (results.Count >= limit)
{
return results.Take(limit).ToList();
}
// Estratégia 2: Relaxar threshold se não conseguiu o suficiente
if (results.Count < limit && minThreshold > 0.35)
{
var mediumResults = await SearchSimilarAsync(queryEmbedding, projectId, 0.35, limit * 2);
if (mediumResults.Count >= limit)
{
return mediumResults.Take(limit).ToList();
}
results = mediumResults;
}
// Estratégia 3: Threshold baixo como último recurso
if (results.Count < limit && minThreshold > 0.2)
{
var lowResults = await SearchSimilarAsync(queryEmbedding, projectId, 0.2, limit * 3);
results = lowResults;
}
return results.Take(limit).ToList();
}
// ========================================
// CRUD DE DOCUMENTOS
// ========================================
public async Task<string> AddDocumentAsync(
string title,
string content,
string projectId,
double[] embedding,
Dictionary<string, object>? metadata = null)
{
try
{
var documentId = Guid.NewGuid().ToString();
var combinedMetadata = new Dictionary<string, object>
{
["title"] = title,
["project_id"] = projectId,
["created_at"] = DateTime.UtcNow.ToString("O")
};
if (metadata != null)
{
foreach (var kvp in metadata)
{
combinedMetadata[kvp.Key] = kvp.Value;
}
}
var document = new
{
ids = new[] { documentId },
documents = new[] { content },
metadatas = new[] { combinedMetadata },
embeddings = new[] { embedding }
};
var json = JsonSerializer.Serialize(document);
var requestContent = new StringContent(json, Encoding.UTF8, "application/json");
var response = await _httpClient.PostAsync($"/api/v1/collections/{_collectionName}/add", requestContent);
if (!response.IsSuccessStatusCode)
{
var error = await response.Content.ReadAsStringAsync();
throw new Exception($"Erro ao adicionar documento: {error}");
}
return documentId;
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao adicionar documento no Chroma");
throw;
}
}
public async Task UpdateDocumentAsync(
string id,
string title,
string content,
string projectId,
double[] embedding,
Dictionary<string, object>? metadata = null)
{
try
{
// Chroma não tem update direto, então fazemos delete + add
await DeleteDocumentAsync(id);
var combinedMetadata = new Dictionary<string, object>
{
["title"] = title,
["project_id"] = projectId,
["updated_at"] = DateTime.UtcNow.ToString("O")
};
if (metadata != null)
{
foreach (var kvp in metadata)
{
combinedMetadata[kvp.Key] = kvp.Value;
}
}
var document = new
{
ids = new[] { id },
documents = new[] { content },
metadatas = new[] { combinedMetadata },
embeddings = new[] { embedding }
};
var json = JsonSerializer.Serialize(document);
var requestContent = new StringContent(json, Encoding.UTF8, "application/json");
var response = await _httpClient.PostAsync($"/api/v1/collections/{_collectionName}/upsert", requestContent);
if (!response.IsSuccessStatusCode)
{
var error = await response.Content.ReadAsStringAsync();
throw new Exception($"Erro ao atualizar documento: {error}");
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao atualizar documento no Chroma");
throw;
}
}
public async Task DeleteDocumentAsync(string id)
{
try
{
var deleteRequest = new
{
ids = new[] { id }
};
var json = JsonSerializer.Serialize(deleteRequest);
var content = new StringContent(json, Encoding.UTF8, "application/json");
var response = await _httpClient.PostAsync($"/api/v1/collections/{_collectionName}/delete", content);
if (!response.IsSuccessStatusCode)
{
var error = await response.Content.ReadAsStringAsync();
_logger.LogWarning("Erro ao deletar documento {Id}: {Error}", id, error);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao deletar documento {Id} no Chroma", id);
throw;
}
}
// ========================================
// CONSULTAS AUXILIARES
// ========================================
public async Task<bool> DocumentExistsAsync(string id)
{
try
{
var doc = await GetDocumentAsync(id);
return doc != null;
}
catch
{
return false;
}
}
public async Task<VectorSearchResult?> GetDocumentAsync(string id)
{
try
{
var query = new
{
ids = new[] { id },
include = new[] { "documents", "metadatas" }
};
var json = JsonSerializer.Serialize(query);
var content = new StringContent(json, Encoding.UTF8, "application/json");
var response = await _httpClient.PostAsync($"/api/v1/collections/{_collectionName}/get", content);
if (!response.IsSuccessStatusCode)
{
return null;
}
var result = await response.Content.ReadAsStringAsync();
var getResult = JsonSerializer.Deserialize<ChromaGetResult>(result);
if (getResult?.ids?.Length > 0)
{
return new VectorSearchResult
{
Id = getResult.ids[0],
Content = getResult.documents?[0] ?? "",
Score = 1.0,
Metadata = getResult.metadatas?[0]
};
}
return null;
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao buscar documento {Id} no Chroma", id);
return null;
}
}
public async Task<List<VectorSearchResult>> GetDocumentsByProjectAsync(string projectId)
{
try
{
var query = new
{
where = new { project_id = projectId },
include = new[] { "documents", "metadatas" }
};
var json = JsonSerializer.Serialize(query);
var content = new StringContent(json, Encoding.UTF8, "application/json");
var response = await _httpClient.PostAsync($"/api/v1/collections/{_collectionName}/get", content);
if (!response.IsSuccessStatusCode)
{
var error = await response.Content.ReadAsStringAsync();
_logger.LogError("Erro ao buscar documentos do projeto {ProjectId}: {Error}", projectId, error);
return new List<VectorSearchResult>();
}
var result = await response.Content.ReadAsStringAsync();
var getResult = JsonSerializer.Deserialize<ChromaGetResult>(result);
var results = new List<VectorSearchResult>();
if (getResult?.documents?.Length > 0)
{
for (int i = 0; i < getResult.documents.Length; i++)
{
results.Add(new VectorSearchResult
{
Id = getResult.ids[i],
Content = getResult.documents[i],
Score = 1.0, // Todos os documentos do projeto
Metadata = getResult.metadatas?[i]
});
}
}
return results;
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao buscar documentos do projeto {ProjectId} no Chroma", projectId);
return new List<VectorSearchResult>();
}
}
public async Task<int> GetDocumentCountAsync(string? projectId = null)
{
try
{
var query = new
{
where = projectId != null ? new { project_id = projectId } : null
};
var json = JsonSerializer.Serialize(query);
var content = new StringContent(json, Encoding.UTF8, "application/json");
var response = await _httpClient.PostAsync($"/api/v1/collections/{_collectionName}/count", content);
if (!response.IsSuccessStatusCode)
{
_logger.LogWarning("Erro ao contar documentos no Chroma");
return 0;
}
var result = await response.Content.ReadAsStringAsync();
var countResult = JsonSerializer.Deserialize<ChromaCountResult>(result);
return countResult?.count ?? 0;
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao contar documentos no Chroma");
return 0;
}
}
// ========================================
// HEALTH CHECK E MÉTRICAS
// ========================================
public async Task<bool> IsHealthyAsync()
{
try
{
var response = await _httpClient.GetAsync("/api/v1/heartbeat");
return response.IsSuccessStatusCode;
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro no health check do Chroma");
return false;
}
}
public async Task<Dictionary<string, object>> GetStatsAsync()
{
try
{
var stats = new Dictionary<string, object>
{
["provider"] = "Chroma",
["collection"] = _collectionName,
["host"] = _settings.Host,
["port"] = _settings.Port
};
// Tentar obter informações da collection
var response = await _httpClient.GetAsync($"/api/v1/collections/{_collectionName}");
if (response.IsSuccessStatusCode)
{
var content = await response.Content.ReadAsStringAsync();
var collectionInfo = JsonSerializer.Deserialize<Dictionary<string, object>>(content);
if (collectionInfo != null)
{
stats["collection_info"] = collectionInfo;
}
}
// Contar documentos totais
stats["total_documents"] = await GetDocumentCountAsync();
return stats;
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao obter stats do Chroma");
return new Dictionary<string, object>
{
["provider"] = "Chroma",
["error"] = ex.Message,
["status"] = "error"
};
}
}
// ========================================
// MÉTODOS AUXILIARES PRIVADOS
// ========================================
private async Task<string[]> GetCollectionsAsync()
{
try
{
var response = await _httpClient.GetAsync("/api/v1/collections");
if (!response.IsSuccessStatusCode)
{
_logger.LogWarning("Erro ao obter collections: {StatusCode}", response.StatusCode);
return Array.Empty<string>();
}
var content = await response.Content.ReadAsStringAsync();
// Tentar desserializar como array de strings (versão simples)
try
{
var collections = JsonSerializer.Deserialize<string[]>(content);
return collections ?? Array.Empty<string>();
}
catch
{
// Tentar desserializar como array de objetos (versão mais nova)
try
{
var collectionsObj = JsonSerializer.Deserialize<CollectionInfo[]>(content);
return collectionsObj?.Select(c => c.name).ToArray() ?? Array.Empty<string>();
}
catch
{
_logger.LogWarning("Não foi possível parsear lista de collections");
return Array.Empty<string>();
}
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao buscar collections");
return Array.Empty<string>();
}
}
// Classe auxiliar para desserialização
private class CollectionInfo
{
public string name { get; set; } = "";
public Dictionary<string, object>? metadata { get; set; }
}
private async Task CreateCollectionAsync()
{
var collection = new
{
name = _collectionName,
metadata = new
{
description = "RAG Collection",
created_at = DateTime.UtcNow.ToString("O")
}
};
var json = JsonSerializer.Serialize(collection);
var content = new StringContent(json, Encoding.UTF8, "application/json");
// Tentar primeira abordagem (versão mais nova)
var response = await _httpClient.PostAsync("/api/v1/collections", content);
// Se falhar, tentar segunda abordagem (criar collection via get_or_create)
if (!response.IsSuccessStatusCode)
{
_logger.LogWarning("Método POST falhou, tentando abordagem alternativa");
// Criar usando get_or_create approach
var createPayload = new
{
name = _collectionName,
metadata = new
{
description = "RAG Collection",
created_at = DateTime.UtcNow.ToString("O")
},
get_or_create = true
};
var createJson = JsonSerializer.Serialize(createPayload);
var createContent = new StringContent(createJson, Encoding.UTF8, "application/json");
var createResponse = await _httpClient.PostAsync("/api/v1/collections", createContent);
if (!createResponse.IsSuccessStatusCode)
{
var error = await createResponse.Content.ReadAsStringAsync();
_logger.LogError("Erro ao criar collection: {Error}", error);
// Última tentativa: assumir que collection já existe
_logger.LogWarning("Assumindo que collection {CollectionName} já existe", _collectionName);
return;
}
}
_logger.LogInformation("Collection {CollectionName} criada/verificada com sucesso", _collectionName);
}
private object? BuildWhereClause(string? projectId, Dictionary<string, object>? filters)
{
var where = new Dictionary<string, object>();
if (!string.IsNullOrEmpty(projectId))
{
where["project_id"] = projectId;
}
if (filters != null)
{
foreach (var filter in filters)
{
where[filter.Key] = filter.Value;
}
}
return where.Any() ? where : null;
}
private List<VectorSearchResult> ParseQueryResults(ChromaQueryResult? queryResult, double threshold)
{
var results = new List<VectorSearchResult>();
if (queryResult?.documents?.Length > 0 && queryResult.documents[0].Length > 0)
{
for (int i = 0; i < queryResult.documents[0].Length; i++)
{
var distance = queryResult.distances?[0][i] ?? 1.0;
// Chroma retorna distâncias, converter para similaridade (1 - distance)
var similarity = 1.0 - distance;
if (similarity >= threshold)
{
results.Add(new VectorSearchResult
{
Id = queryResult.ids[0][i],
Content = queryResult.documents[0][i],
Score = similarity,
Metadata = queryResult.metadatas?[0][i]
});
}
}
}
return results.OrderByDescending(r => r.Score).ToList();
}
}
// ========================================
// DTOs PARA CHROMA API
// ========================================
public class ChromaQueryResult
{
public string[][] ids { get; set; } = Array.Empty<string[]>();
public string[][] documents { get; set; } = Array.Empty<string[]>();
public double[][]? distances { get; set; }
public Dictionary<string, object>[][]? metadatas { get; set; }
}
public class ChromaGetResult
{
public string[] ids { get; set; } = Array.Empty<string>();
public string[] documents { get; set; } = Array.Empty<string>();
public Dictionary<string, object>[]? metadatas { get; set; }
}
public class ChromaCountResult
{
public int count { get; set; }
}
}

View File

@ -7,10 +7,10 @@ using ChatRAG.Services.Contracts;
using Qdrant.Client; using Qdrant.Client;
using static Qdrant.Client.Grpc.Conditions; using static Qdrant.Client.Grpc.Conditions;
using System.Drawing; using System.Drawing;
using System.Collections.Concurrent;
#pragma warning disable SKEXP0001 #pragma warning disable SKEXP0001
namespace ChatRAG.Services.SearchVectors namespace ChatRAG.Services.SearchVectors
{ {
public class QdrantVectorSearchService : IVectorSearchService public class QdrantVectorSearchService : IVectorSearchService
@ -18,7 +18,9 @@ namespace ChatRAG.Services.SearchVectors
private readonly QdrantClient _client; private readonly QdrantClient _client;
private readonly QdrantSettings _settings; private readonly QdrantSettings _settings;
private readonly ILogger<QdrantVectorSearchService> _logger; private readonly ILogger<QdrantVectorSearchService> _logger;
private bool _collectionInitialized = false; private volatile bool _collectionInitialized = false;
private readonly SemaphoreSlim _initializationSemaphore = new(1, 1);
private readonly ConcurrentDictionary<string, bool> _collectionCache = new();
public QdrantVectorSearchService( public QdrantVectorSearchService(
IOptions<VectorDatabaseSettings> settings, IOptions<VectorDatabaseSettings> settings,
@ -37,9 +39,20 @@ namespace ChatRAG.Services.SearchVectors
{ {
if (_collectionInitialized) return; if (_collectionInitialized) return;
await _initializationSemaphore.WaitAsync();
try try
{ {
if (_collectionInitialized) return;
// Verifica cache primeiro
if (_collectionCache.TryGetValue(_settings.CollectionName, out bool exists) && exists)
{
_collectionInitialized = true;
return;
}
var collectionExists = await _client.CollectionExistsAsync(_settings.CollectionName); var collectionExists = await _client.CollectionExistsAsync(_settings.CollectionName);
_collectionCache.TryAdd(_settings.CollectionName, collectionExists);
if (!collectionExists) if (!collectionExists)
{ {
@ -58,7 +71,7 @@ namespace ChatRAG.Services.SearchVectors
} }
}; };
// Configurações HNSW opcionais // Configurações HNSW otimizadas
if (_settings.HnswM > 0) if (_settings.HnswM > 0)
{ {
vectorsConfig.HnswConfig = new HnswConfigDiff vectorsConfig.HnswConfig = new HnswConfigDiff
@ -74,15 +87,15 @@ namespace ChatRAG.Services.SearchVectors
vectorsConfig: vectorsConfig vectorsConfig: vectorsConfig
); );
_collectionCache.TryAdd(_settings.CollectionName, true);
_logger.LogInformation("✅ Collection {CollectionName} criada", _settings.CollectionName); _logger.LogInformation("✅ Collection {CollectionName} criada", _settings.CollectionName);
} }
_collectionInitialized = true; _collectionInitialized = true;
} }
catch (Exception ex) finally
{ {
_logger.LogError(ex, "Erro ao inicializar collection {CollectionName}", _settings.CollectionName); _initializationSemaphore.Release();
throw;
} }
} }
@ -131,21 +144,10 @@ namespace ChatRAG.Services.SearchVectors
limit: (ulong)limit, limit: (ulong)limit,
scoreThreshold: (float)threshold, scoreThreshold: (float)threshold,
payloadSelector: true, payloadSelector: true,
vectorsSelector: true vectorsSelector: false // Otimização: não buscar vetores desnecessariamente
); );
return searchResult.Select(point => new VectorSearchResult return searchResult.Select(ConvertToVectorSearchResult).ToList();
{
Id = point.Id.Uuid ?? point.Id.Num.ToString(),
Title = GetStringFromPayload(point.Payload, "title"),
Content = GetStringFromPayload(point.Payload, "content"),
ProjectId = GetStringFromPayload(point.Payload, "project_id"),
Score = point.Score,
Provider = "Qdrant",
CreatedAt = GetDateTimeFromPayload(point.Payload, "created_at"),
UpdatedAt = GetDateTimeFromPayload(point.Payload, "updated_at"),
Metadata = ConvertPayloadToMetadata(point.Payload)
}).ToList();
} }
catch (Exception ex) catch (Exception ex)
{ {
@ -189,27 +191,12 @@ namespace ChatRAG.Services.SearchVectors
var id = Guid.NewGuid().ToString(); var id = Guid.NewGuid().ToString();
var vector = embedding.Select(x => (float)x).ToArray(); var vector = embedding.Select(x => (float)x).ToArray();
var payload = new Dictionary<string, Value> var payload = CreatePayload(title, content, projectId, metadata, isUpdate: false);
{
["title"] = title,
["content"] = content,
["project_id"] = projectId,
["created_at"] = DateTime.UtcNow.ToString("O"),
["updated_at"] = DateTime.UtcNow.ToString("O")
};
if (metadata?.Any() == true)
{
foreach (var kvp in metadata)
{
payload[$"meta_{kvp.Key}"] = ConvertToValue(kvp.Value);
}
}
var point = new PointStruct var point = new PointStruct
{ {
Id = new PointId { Uuid = id }, Id = new PointId { Uuid = id },
Vectors = vector, Vectors = vector,
Payload = { payload } Payload = { payload }
}; };
@ -241,22 +228,7 @@ namespace ChatRAG.Services.SearchVectors
try try
{ {
var vector = embedding.Select(x => (float)x).ToArray(); var vector = embedding.Select(x => (float)x).ToArray();
var payload = CreatePayload(title, content, projectId, metadata, isUpdate: true);
var payload = new Dictionary<string, Value>
{
["title"] = title,
["content"] = content,
["project_id"] = projectId,
["updated_at"] = DateTime.UtcNow.ToString("O")
};
if (metadata?.Any() == true)
{
foreach (var kvp in metadata)
{
payload[$"meta_{kvp.Key}"] = ConvertToValue(kvp.Value);
}
}
var point = new PointStruct var point = new PointStruct
{ {
@ -285,11 +257,11 @@ namespace ChatRAG.Services.SearchVectors
try try
{ {
var pointId = new PointId { Uuid = id } ; var pointId = new PointId { Uuid = id };
await _client.DeleteAsync( await _client.DeleteAsync(
collectionName: _settings.CollectionName, collectionName: _settings.CollectionName,
ids: new ulong[] { pointId.Num } ids: new ulong[] { pointId.Num }
); );
_logger.LogDebug("Documento {Id} removido do Qdrant", id); _logger.LogDebug("Documento {Id} removido do Qdrant", id);
@ -305,8 +277,18 @@ namespace ChatRAG.Services.SearchVectors
{ {
try try
{ {
var result = await GetDocumentAsync(id); await EnsureCollectionExistsAsync();
return result != null;
var pointId = new PointId { Uuid = id };
var results = await _client.RetrieveAsync(
collectionName: _settings.CollectionName,
ids: new PointId[] { pointId },
withPayload: false, // Otimização: só queremos saber se existe
withVectors: false
);
return results.Any();
} }
catch catch
{ {
@ -330,20 +312,7 @@ namespace ChatRAG.Services.SearchVectors
); );
var point = results.FirstOrDefault(); var point = results.FirstOrDefault();
if (point == null) return null; return point != null ? ConvertToVectorSearchResult(point) : null;
return new VectorSearchResult
{
Id = point.Id.Uuid ?? point.Id.Num.ToString(),
Title = GetStringFromPayload(point.Payload, "title"),
Content = GetStringFromPayload(point.Payload, "content"),
ProjectId = GetStringFromPayload(point.Payload, "project_id"),
Score = 1.0,
Provider = "Qdrant",
CreatedAt = GetDateTimeFromPayload(point.Payload, "created_at"),
UpdatedAt = GetDateTimeFromPayload(point.Payload, "updated_at"),
Metadata = ConvertPayloadToMetadata(point.Payload)
};
} }
catch (Exception ex) catch (Exception ex)
{ {
@ -361,26 +330,18 @@ namespace ChatRAG.Services.SearchVectors
var filter = new Filter(); var filter = new Filter();
filter.Must.Add(MatchKeyword("project_id", projectId)); filter.Must.Add(MatchKeyword("project_id", projectId));
var results = await _client.ScrollAsync( var scrollRequest = new ScrollPoints
collectionName: _settings.CollectionName,
filter: filter,
limit: 10000,
payloadSelector: true,
vectorsSelector: true
);
return results.Result.Select(point => new VectorSearchResult
{ {
Id = point.Id.Uuid ?? point.Id.Num.ToString(), CollectionName = _settings.CollectionName,
Title = GetStringFromPayload(point.Payload, "title"), Filter = filter,
Content = GetStringFromPayload(point.Payload, "content"), Limit = 10000,
ProjectId = GetStringFromPayload(point.Payload, "project_id"), WithPayload = true,
Score = 1.0, WithVectors = false // Otimização: não buscar vetores
Provider = "Qdrant", };
CreatedAt = GetDateTimeFromPayload(point.Payload, "created_at"),
UpdatedAt = GetDateTimeFromPayload(point.Payload, "updated_at"), var results = await _client.ScrollAsync(_settings.CollectionName, filter, 10000, null, true, false);
Metadata = ConvertPayloadToMetadata(point.Payload)
}).ToList(); return results.Result.Select(ConvertToVectorSearchResult).ToList();
} }
catch (Exception ex) catch (Exception ex)
{ {
@ -459,16 +420,84 @@ namespace ChatRAG.Services.SearchVectors
} }
} }
// Métodos auxiliares otimizados
private static Dictionary<string, Value> CreatePayload(
string title,
string content,
string projectId,
Dictionary<string, object>? metadata,
bool isUpdate)
{
var payload = new Dictionary<string, Value>
{
["title"] = title,
["content"] = content,
["project_id"] = projectId
};
if (isUpdate)
{
payload["updated_at"] = DateTime.UtcNow.ToString("O");
}
else
{
payload["created_at"] = DateTime.UtcNow.ToString("O");
payload["updated_at"] = DateTime.UtcNow.ToString("O");
}
if (metadata?.Any() == true)
{
foreach (var kvp in metadata)
{
payload[$"meta_{kvp.Key}"] = ConvertToValue(kvp.Value);
}
}
return payload;
}
private static VectorSearchResult ConvertToVectorSearchResult(ScoredPoint point)
{
return new VectorSearchResult
{
Id = point.Id.Uuid ?? point.Id.Num.ToString(),
Title = GetStringFromPayload(point.Payload, "title"),
Content = GetStringFromPayload(point.Payload, "content"),
ProjectId = GetStringFromPayload(point.Payload, "project_id"),
Score = point.Score,
Provider = "Qdrant",
CreatedAt = GetDateTimeFromPayload(point.Payload, "created_at"),
UpdatedAt = GetDateTimeFromPayload(point.Payload, "updated_at"),
Metadata = ConvertPayloadToMetadata(point.Payload)
};
}
private static VectorSearchResult ConvertToVectorSearchResult(RetrievedPoint point)
{
return new VectorSearchResult
{
Id = point.Id.Uuid ?? point.Id.Num.ToString(),
Title = GetStringFromPayload(point.Payload, "title"),
Content = GetStringFromPayload(point.Payload, "content"),
ProjectId = GetStringFromPayload(point.Payload, "project_id"),
Score = 1.0,
Provider = "Qdrant",
CreatedAt = GetDateTimeFromPayload(point.Payload, "created_at"),
UpdatedAt = GetDateTimeFromPayload(point.Payload, "updated_at"),
Metadata = ConvertPayloadToMetadata(point.Payload)
};
}
private static Value ConvertToValue(object value) private static Value ConvertToValue(object value)
{ {
return value switch return value switch
{ {
string s => s, string s => s,
int i => i, int i => i,
long l => l, long l => l,
double d => d, double d => d,
float f => f, float f => f,
bool b => b, bool b => b,
DateTime dt => dt.ToString("O"), DateTime dt => dt.ToString("O"),
_ => value?.ToString() ?? "" _ => value?.ToString() ?? ""
}; };
@ -519,6 +548,7 @@ namespace ChatRAG.Services.SearchVectors
public void Dispose() public void Dispose()
{ {
_initializationSemaphore?.Dispose();
_client?.Dispose(); _client?.Dispose();
} }
} }

View File

@ -1,5 +1,6 @@
using ChatApi.Data; using ChatApi.Data;
using ChatRAG.Contracts.VectorSearch; using ChatRAG.Contracts.VectorSearch;
using ChatRAG.Data;
using ChatRAG.Services.Contracts; using ChatRAG.Services.Contracts;
using ChatRAG.Services.ResponseService; using ChatRAG.Services.ResponseService;
using ChatRAG.Services.TextServices; using ChatRAG.Services.TextServices;
@ -8,33 +9,31 @@ using Microsoft.Extensions.Options;
namespace ChatRAG.Services.SearchVectors namespace ChatRAG.Services.SearchVectors
{ {
/// <summary>
/// Factory principal que cria implementações baseadas na configuração
/// </summary>
/// <summary>
/// Factory principal que cria implementações baseadas na configuração
/// </summary>
public class VectorDatabaseFactory : IVectorDatabaseFactory public class VectorDatabaseFactory : IVectorDatabaseFactory
{ {
private readonly VectorDatabaseSettings _settings;
private readonly IServiceProvider _serviceProvider; private readonly IServiceProvider _serviceProvider;
private readonly VectorDatabaseSettings _settings;
private readonly ILogger<VectorDatabaseFactory> _logger; private readonly ILogger<VectorDatabaseFactory> _logger;
public VectorDatabaseFactory( public VectorDatabaseFactory(
IOptions<VectorDatabaseSettings> settings,
IServiceProvider serviceProvider, IServiceProvider serviceProvider,
IOptions<VectorDatabaseSettings> settings,
ILogger<VectorDatabaseFactory> logger) ILogger<VectorDatabaseFactory> logger)
{ {
_settings = settings.Value;
_serviceProvider = serviceProvider; _serviceProvider = serviceProvider;
_settings = settings.Value;
_logger = logger; _logger = logger;
// Valida configurações na inicialização
ValidateSettings();
} }
public string GetActiveProvider() => _settings.Provider; public string GetActiveProvider()
public VectorDatabaseSettings GetSettings() => _settings; {
return _settings.Provider;
}
public VectorDatabaseSettings GetSettings()
{
return _settings;
}
public IVectorSearchService CreateVectorSearchService() public IVectorSearchService CreateVectorSearchService()
{ {
@ -42,8 +41,9 @@ namespace ChatRAG.Services.SearchVectors
return _settings.Provider.ToLower() switch return _settings.Provider.ToLower() switch
{ {
"qdrant" => GetService<ChatRAG.Services.SearchVectors.QdrantVectorSearchService>(), "qdrant" => GetService<QdrantVectorSearchService>(),
"mongodb" => GetService<ChatRAG.Services.SearchVectors.MongoVectorSearchService>(), "mongodb" => GetService<MongoVectorSearchService>(),
"chroma" => GetService<ChromaVectorSearchService>(),
_ => throw new ArgumentException($"Provider de VectorSearch não suportado: {_settings.Provider}") _ => throw new ArgumentException($"Provider de VectorSearch não suportado: {_settings.Provider}")
}; };
} }
@ -54,10 +54,10 @@ namespace ChatRAG.Services.SearchVectors
return _settings.Provider.ToLower() switch return _settings.Provider.ToLower() switch
{ {
// ✅ CORRIGIDO: Usa os namespaces corretos "qdrant" => GetService<QdrantTextDataService>(),
"qdrant" => GetService<ChatRAG.Services.TextServices.QdrantTextDataService>(), "mongodb" => GetService<MongoTextDataService>(),
"mongodb" => GetService<ChatApi.Data.TextData>(), // Sua classe atual! "chroma" => GetService<ChromaTextDataService>(),
_ => throw new ArgumentException($"Provider de TextData não suportado: {_settings.Provider}") _ => throw new ArgumentException($"Provider de TextDataService não suportado: {_settings.Provider}")
}; };
} }
@ -65,50 +65,36 @@ namespace ChatRAG.Services.SearchVectors
{ {
_logger.LogInformation("Criando ResponseService para provider: {Provider}", _settings.Provider); _logger.LogInformation("Criando ResponseService para provider: {Provider}", _settings.Provider);
return _settings.Provider.ToLower() switch // Verificar se deve usar RAG Hierárquico
var configuration = _serviceProvider.GetService<IConfiguration>();
var useHierarchical = configuration?.GetValue<bool>("Features:UseHierarchicalRAG") ?? false;
if (useHierarchical)
{ {
// ✅ CORRIGIDO: Usa os namespaces corretos _logger.LogInformation("Usando HierarchicalRAGService");
"qdrant" => GetService<ChatRAG.Services.ResponseService.QdrantResponseService>(), return GetService<HierarchicalRAGService>();
"mongodb" => GetService<ChatRAG.Services.ResponseService.ResponseRAGService>(), // Sua classe atual! }
_ => throw new ArgumentException($"Provider de Response não suportado: {_settings.Provider}")
// Usar estratégia baseada no provider ou configuração
var ragStrategy = configuration?.GetValue<string>("Features:RAGStrategy");
return ragStrategy?.ToLower() switch
{
"hierarchical" => GetService<HierarchicalRAGService>(),
"standard" => GetService<ResponseRAGService>(),
_ => GetService<ResponseRAGService>() // Padrão
}; };
} }
// ========================================
// MÉTODOS AUXILIARES
// ========================================
private T GetService<T>() where T : class private T GetService<T>() where T : class
{ {
try var service = _serviceProvider.GetService<T>();
if (service == null)
{ {
var service = _serviceProvider.GetRequiredService<T>(); throw new InvalidOperationException($"Serviço {typeof(T).Name} não está registrado no DI container. " +
_logger.LogDebug("Serviço {ServiceType} criado com sucesso", typeof(T).Name); $"Verifique se o serviço foi registrado para o provider '{_settings.Provider}'.");
return service;
} }
catch (InvalidOperationException ex) return service;
{
_logger.LogError(ex, "Erro ao criar serviço {ServiceType} para provider {Provider}",
typeof(T).Name, _settings.Provider);
throw new InvalidOperationException(
$"Serviço {typeof(T).Name} não está registrado para provider {_settings.Provider}. " +
$"Certifique-se de chamar services.Add{_settings.Provider}Provider() no DI.", ex);
}
}
private void ValidateSettings()
{
if (!_settings.IsValid())
{
var errors = _settings.GetValidationErrors();
var errorMessage = $"Configurações inválidas para VectorDatabase: {string.Join(", ", errors)}";
_logger.LogError(errorMessage);
throw new InvalidOperationException(errorMessage);
}
_logger.LogInformation("Configurações validadas com sucesso para provider: {Provider}", _settings.Provider);
} }
} }
} }

View File

@ -0,0 +1,55 @@
using ChatApi.Data;
using ChatRAG.Contracts.VectorSearch;
using ChatRAG.Services.Contracts;
using ChatRAG.Services.ResponseService;
using ChatRAG.Services.TextServices;
using ChatRAG.Settings.ChatRAG.Configuration;
using Microsoft.Extensions.Options;
namespace ChatRAG.Services.SearchVectors
{
public class VectorDatabaseFactory : IVectorDatabaseFactory
{
private readonly IServiceProvider _serviceProvider;
private readonly VectorDatabaseSettings _settings;
private readonly ILogger<VectorDatabaseFactory> _logger;
public VectorDatabaseFactory(
IServiceProvider serviceProvider,
IOptions<VectorDatabaseSettings> settings,
ILogger<VectorDatabaseFactory> logger)
{
_serviceProvider = serviceProvider;
_settings = settings.Value;
_logger = logger;
}
public string GetActiveProvider()
{
return _settings.Provider;
}
public IVectorSearchService CreateVectorSearchService()
{
_logger.LogInformation("Criando VectorSearchService para provider: {Provider}", _settings.Provider);
return _settings.Provider.ToLower() switch
{
"qdrant" => GetService<QdrantVectorSearchService>(),
"mongodb" => GetService<MongoVectorSearchService>(),
"chroma" => GetService<ChromaVectorSearchService>(),
_ => throw new ArgumentException($"Provider de VectorSearch não suportado: {_settings.Provider}")
};
}
private T GetService<T>() where T : class
{
var service = _serviceProvider.GetService<T>();
if (service == null)
{
throw new InvalidOperationException($"Serviço {typeof(T).Name} não está registrado no DI container");
}
return service;
}
}
}

View File

@ -0,0 +1,537 @@
using ChatRAG.Contracts.VectorSearch;
using ChatRAG.Models;
using ChatRAG.Services.Contracts;
using ChatRAG.Data;
using Microsoft.SemanticKernel.Embeddings;
using System.Text;
#pragma warning disable SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
namespace ChatRAG.Services
{
// ========================================
// CHROMA TEXT DATA SERVICE - IMPLEMENTAÇÃO COMPLETA
// ========================================
public class ChromaTextDataService : ITextDataService
{
private readonly IVectorSearchService _vectorSearchService;
private readonly ITextEmbeddingGenerationService _embeddingService;
private readonly ILogger<ChromaTextDataService> _logger;
public ChromaTextDataService(
IVectorSearchService vectorSearchService,
ITextEmbeddingGenerationService embeddingService,
ILogger<ChromaTextDataService> logger)
{
_vectorSearchService = vectorSearchService;
_embeddingService = embeddingService;
_logger = logger;
}
public string ProviderName => "Chroma";
// ========================================
// MÉTODOS ORIGINAIS (compatibilidade com MongoDB)
// ========================================
public async Task SalvarNoMongoDB(string titulo, string texto, string projectId)
{
await SalvarNoMongoDB(null, titulo, texto, projectId);
}
public async Task SalvarNoMongoDB(string? id, string titulo, string texto, string projectId)
{
try
{
var conteudo = $"**{titulo}** \n\n {texto}";
// Gera embedding
var embedding = await _embeddingService.GenerateEmbeddingAsync(conteudo);
var embeddingArray = embedding.ToArray().Select(e => (double)e).ToArray();
if (string.IsNullOrEmpty(id))
{
// Cria novo documento
await _vectorSearchService.AddDocumentAsync(titulo, texto, projectId, embeddingArray);
_logger.LogDebug("Documento '{Title}' criado no Chroma", titulo);
}
else
{
// Atualiza documento existente
await _vectorSearchService.UpdateDocumentAsync(id, titulo, texto, projectId, embeddingArray);
_logger.LogDebug("Documento '{Id}' atualizado no Chroma", id);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao salvar documento '{Title}' no Chroma", titulo);
throw;
}
}
public async Task SalvarTextoComEmbeddingNoMongoDB(string textoCompleto, string projectId)
{
try
{
var textoArray = new List<string>();
string[] textolinhas = textoCompleto.Split(
new string[] { "\n" },
StringSplitOptions.None
);
var title = textolinhas[0];
var builder = new StringBuilder();
foreach (string line in textolinhas)
{
if (line.StartsWith("**") || line.StartsWith("\r**"))
{
if (builder.Length > 0)
{
textoArray.Add(title.Replace("**", "").Replace("\r", "") + ": " + Environment.NewLine + builder.ToString());
builder = new StringBuilder();
title = line;
}
}
else
{
builder.AppendLine(line);
}
}
// Adiciona último bloco se houver
if (builder.Length > 0)
{
textoArray.Add(title.Replace("**", "").Replace("\r", "") + ": " + Environment.NewLine + builder.ToString());
}
// Processa cada seção
foreach (var item in textoArray)
{
await SalvarNoMongoDB(title.Replace("**", "").Replace("\r", ""), item, projectId);
}
_logger.LogInformation("Texto completo processado: {SectionCount} seções salvas no Chroma", textoArray.Count);
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao processar texto completo no Chroma");
throw;
}
}
public async Task<IEnumerable<TextoComEmbedding>> GetAll()
{
try
{
// Busca todos os projetos e depois todos os documentos
var allDocuments = new List<VectorSearchResult>();
// Como Chroma não tem um "GetAll" direto, vamos usar scroll
// Isso é uma limitação vs MongoDB, mas é mais eficiente
var projects = await GetAllProjectIds();
foreach (var projectId in projects)
{
var projectDocs = await _vectorSearchService.GetDocumentsByProjectAsync(projectId);
allDocuments.AddRange(projectDocs);
}
return allDocuments.Select(ConvertToTextoComEmbedding);
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao recuperar todos os documentos do Chroma");
throw;
}
}
public async Task<IEnumerable<TextoComEmbedding>> GetByPorjectId(string projectId)
{
try
{
var documents = await _vectorSearchService.GetDocumentsByProjectAsync(projectId);
return documents.Select(ConvertToTextoComEmbedding);
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao recuperar documentos do projeto {ProjectId} no Chroma", projectId);
throw;
}
}
public async Task<TextoComEmbedding> GetById(string id)
{
try
{
var document = await _vectorSearchService.GetDocumentAsync(id);
if (document == null)
{
throw new ArgumentException($"Documento {id} não encontrado no Chroma");
}
return ConvertToTextoComEmbedding(document);
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao recuperar documento {Id} do Chroma", id);
throw;
}
}
// ========================================
// MÉTODOS NOVOS DA INTERFACE
// ========================================
public async Task<string> SaveDocumentAsync(DocumentInput document)
{
try
{
var conteudo = $"**{document.Title}** \n\n {document.Content}";
var embedding = await _embeddingService.GenerateEmbeddingAsync(conteudo);
var embeddingArray = embedding.ToArray().Select(e => (double)e).ToArray();
string id;
if (!string.IsNullOrEmpty(document.Id))
{
// Atualizar documento existente
await _vectorSearchService.UpdateDocumentAsync(
document.Id,
document.Title,
document.Content,
document.ProjectId,
embeddingArray,
document.Metadata);
id = document.Id;
}
else
{
// Criar novo documento
id = await _vectorSearchService.AddDocumentAsync(
document.Title,
document.Content,
document.ProjectId,
embeddingArray,
document.Metadata);
}
_logger.LogDebug("Documento {Id} salvo no Chroma via SaveDocumentAsync", id);
return id;
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao salvar documento no Chroma");
throw;
}
}
public async Task UpdateDocumentAsync(string id, DocumentInput document)
{
try
{
var conteudo = $"**{document.Title}** \n\n {document.Content}";
var embedding = await _embeddingService.GenerateEmbeddingAsync(conteudo);
var embeddingArray = embedding.ToArray().Select(e => (double)e).ToArray();
await _vectorSearchService.UpdateDocumentAsync(
id,
document.Title,
document.Content,
document.ProjectId,
embeddingArray,
document.Metadata);
_logger.LogDebug("Documento {Id} atualizado no Chroma", id);
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao atualizar documento {Id} no Chroma", id);
throw;
}
}
public async Task DeleteDocumentAsync(string id)
{
try
{
await _vectorSearchService.DeleteDocumentAsync(id);
_logger.LogDebug("Documento {Id} deletado do Chroma", id);
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao deletar documento {Id} do Chroma", id);
throw;
}
}
public async Task<bool> DocumentExistsAsync(string id)
{
try
{
return await _vectorSearchService.DocumentExistsAsync(id);
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao verificar existência do documento {Id} no Chroma", id);
return false;
}
}
public async Task<DocumentOutput?> GetDocumentAsync(string id)
{
try
{
var result = await _vectorSearchService.GetDocumentAsync(id);
if (result == null) return null;
return new DocumentOutput
{
Id = result.Id,
Title = result.Metadata?.GetValueOrDefault("title")?.ToString() ?? "",
Content = result.Content,
ProjectId = result.Metadata?.GetValueOrDefault("project_id")?.ToString() ?? "",
Embedding = Array.Empty<double>(), // Chroma não retorna embedding na busca
CreatedAt = ParseDateTime(result.Metadata?.GetValueOrDefault("created_at")?.ToString()).Value,
UpdatedAt = ParseDateTime(result.Metadata?.GetValueOrDefault("updated_at")?.ToString()).Value,
Metadata = result.Metadata
};
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao recuperar documento {Id} do Chroma", id);
return null;
}
}
public async Task<List<DocumentOutput>> GetDocumentsByProjectAsync(string projectId)
{
try
{
var results = await _vectorSearchService.GetDocumentsByProjectAsync(projectId);
return results.Select(result => new DocumentOutput
{
Id = result.Id,
Title = result.Metadata?.GetValueOrDefault("title")?.ToString() ?? "",
Content = result.Content,
ProjectId = projectId,
Embedding = Array.Empty<double>(),
CreatedAt = ParseDateTime(result.Metadata?.GetValueOrDefault("created_at")?.ToString()).Value,
UpdatedAt = ParseDateTime(result.Metadata?.GetValueOrDefault("updated_at")?.ToString()).Value,
Metadata = result.Metadata
}).ToList();
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao recuperar documentos do projeto {ProjectId} do Chroma", projectId);
throw;
}
}
public async Task<int> GetDocumentCountAsync(string? projectId = null)
{
try
{
return await _vectorSearchService.GetDocumentCountAsync(projectId);
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao contar documentos no Chroma");
return 0;
}
}
// ========================================
// OPERAÇÕES EM LOTE
// ========================================
public async Task<List<string>> SaveDocumentsBatchAsync(List<DocumentInput> documents)
{
var ids = new List<string>();
var errors = new List<Exception>();
// Processa em lotes menores para performance
var batchSize = 10;
for (int i = 0; i < documents.Count; i += batchSize)
{
var batch = documents.Skip(i).Take(batchSize);
var tasks = batch.Select(async doc =>
{
try
{
var id = await SaveDocumentAsync(doc);
return id;
}
catch (Exception ex)
{
errors.Add(ex);
_logger.LogError(ex, "Erro ao salvar documento '{Title}' em lote", doc.Title);
return null;
}
});
var batchResults = await Task.WhenAll(tasks);
ids.AddRange(batchResults.Where(id => id != null)!);
}
if (errors.Any())
{
_logger.LogWarning("Batch save completado com {ErrorCount} erros de {TotalCount} documentos",
errors.Count, documents.Count);
}
_logger.LogInformation("Batch save: {SuccessCount}/{TotalCount} documentos salvos no Chroma",
ids.Count, documents.Count);
return ids;
}
public async Task DeleteDocumentsBatchAsync(List<string> ids)
{
var errors = new List<Exception>();
// Processa em lotes para não sobrecarregar
var batchSize = 20;
for (int i = 0; i < ids.Count; i += batchSize)
{
var batch = ids.Skip(i).Take(batchSize);
var tasks = batch.Select(async id =>
{
try
{
await DeleteDocumentAsync(id);
return true;
}
catch (Exception ex)
{
errors.Add(ex);
_logger.LogError(ex, "Erro ao deletar documento {Id} em lote", id);
return false;
}
});
await Task.WhenAll(tasks);
}
if (errors.Any())
{
_logger.LogWarning("Batch delete completado com {ErrorCount} erros de {TotalCount} documentos",
errors.Count, ids.Count);
}
else
{
_logger.LogInformation("Batch delete: {TotalCount} documentos removidos do Chroma", ids.Count);
}
}
// ========================================
// ESTATÍSTICAS DO PROVIDER
// ========================================
public async Task<Dictionary<string, object>> GetProviderStatsAsync()
{
try
{
var baseStats = await _vectorSearchService.GetStatsAsync();
var totalDocs = await GetDocumentCountAsync();
// Adiciona estatísticas específicas do TextData
var projectIds = await GetAllProjectIds();
var projectStats = new Dictionary<string, int>();
foreach (var projectId in projectIds)
{
var count = await GetDocumentCountAsync(projectId);
projectStats[projectId] = count;
}
var enhancedStats = new Dictionary<string, object>(baseStats)
{
["text_service_provider"] = "Chroma",
["total_documents_via_text_service"] = totalDocs,
["projects_count"] = projectIds.Count,
["documents_by_project"] = projectStats,
["supports_batch_operations"] = true,
["supports_metadata"] = true,
["embedding_auto_generation"] = true
};
return enhancedStats;
}
catch (Exception ex)
{
return new Dictionary<string, object>
{
["provider"] = "Chroma",
["text_service_provider"] = "Chroma",
["health"] = "error",
["error"] = ex.Message,
["last_check"] = DateTime.UtcNow
};
}
}
// ========================================
// MÉTODOS AUXILIARES PRIVADOS
// ========================================
private static TextoComEmbedding ConvertToTextoComEmbedding(VectorSearchResult result)
{
return new TextoComEmbedding
{
Id = result.Id,
Titulo = result.Metadata?.GetValueOrDefault("title")?.ToString() ?? "",
Conteudo = result.Content,
ProjetoId = result.Metadata?.GetValueOrDefault("project_id")?.ToString() ?? "",
Embedding = Array.Empty<double>(), // Chroma não retorna embedding na busca
// Campos que podem não existir no Chroma
ProjetoNome = result.Metadata?.GetValueOrDefault("project_name")?.ToString() ?? "",
TipoDocumento = result.Metadata?.GetValueOrDefault("document_type")?.ToString() ?? "",
Categoria = result.Metadata?.GetValueOrDefault("category")?.ToString() ?? "",
Tags = result.Metadata?.GetValueOrDefault("tags") as string[] ?? Array.Empty<string>()
};
}
private async Task<List<string>> GetAllProjectIds()
{
try
{
// Esta é uma operação custosa no Chroma
// Em produção, seria melhor manter um cache de project IDs
// ou usar uma estrutura de dados separada
// Por agora, vamos usar uma busca com um vetor dummy para pegar todos os documentos
var dummyVector = new double[384]; // Assumindo embeddings padrão
var allResults = await _vectorSearchService.SearchSimilarAsync(
dummyVector,
projectId: null,
threshold: 0.0,
limit: 10000);
return allResults
.Select(r => r.Metadata?.GetValueOrDefault("project_id")?.ToString())
.Where(pid => !string.IsNullOrEmpty(pid))
.Distinct()
.ToList()!;
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao recuperar IDs de projetos do Chroma");
return new List<string>();
}
}
private static DateTime? ParseDateTime(string? dateString)
{
if (string.IsNullOrEmpty(dateString))
return null;
if (DateTime.TryParse(dateString, out var date))
return date;
return null;
}
}
}
#pragma warning restore SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.

View File

@ -6,6 +6,7 @@ using ChatRAG.Models;
using ChatRAG.Services.Contracts; using ChatRAG.Services.Contracts;
using Microsoft.SemanticKernel.Embeddings; using Microsoft.SemanticKernel.Embeddings;
using System.Text; using System.Text;
using System.Collections.Concurrent;
namespace ChatRAG.Services.TextServices namespace ChatRAG.Services.TextServices
{ {
@ -15,6 +16,10 @@ namespace ChatRAG.Services.TextServices
private readonly ITextEmbeddingGenerationService _embeddingService; private readonly ITextEmbeddingGenerationService _embeddingService;
private readonly ILogger<QdrantTextDataService> _logger; private readonly ILogger<QdrantTextDataService> _logger;
// Cache para project IDs para evitar buscas custosas
private readonly ConcurrentDictionary<string, DateTime> _projectIdCache = new();
private readonly TimeSpan _cacheTimeout = TimeSpan.FromMinutes(5);
public QdrantTextDataService( public QdrantTextDataService(
IVectorSearchService vectorSearchService, IVectorSearchService vectorSearchService,
ITextEmbeddingGenerationService embeddingService, ITextEmbeddingGenerationService embeddingService,
@ -42,20 +47,23 @@ namespace ChatRAG.Services.TextServices
{ {
var conteudo = $"**{titulo}** \n\n {texto}"; var conteudo = $"**{titulo}** \n\n {texto}";
// Gera embedding // Gera embedding uma única vez
var embedding = await _embeddingService.GenerateEmbeddingAsync(conteudo); var embedding = await GenerateEmbeddingOptimized(conteudo);
var embeddingArray = embedding.ToArray().Select(e => (double)e).ToArray();
if (string.IsNullOrEmpty(id)) if (string.IsNullOrEmpty(id))
{ {
// Cria novo documento // Cria novo documento
await _vectorSearchService.AddDocumentAsync(titulo, texto, projectId, embeddingArray); var newId = await _vectorSearchService.AddDocumentAsync(titulo, texto, projectId, embedding);
_logger.LogDebug("Documento '{Title}' criado no Qdrant", titulo);
// Atualiza cache de project IDs
_projectIdCache.TryAdd(projectId, DateTime.UtcNow);
_logger.LogDebug("Documento '{Title}' criado no Qdrant com ID {Id}", titulo, newId);
} }
else else
{ {
// Atualiza documento existente // Atualiza documento existente
await _vectorSearchService.UpdateDocumentAsync(id, titulo, texto, projectId, embeddingArray); await _vectorSearchService.UpdateDocumentAsync(id, titulo, texto, projectId, embedding);
_logger.LogDebug("Documento '{Id}' atualizado no Qdrant", id); _logger.LogDebug("Documento '{Id}' atualizado no Qdrant", id);
} }
} }
@ -70,43 +78,29 @@ namespace ChatRAG.Services.TextServices
{ {
try try
{ {
var textoArray = new List<string>(); var textoArray = ParseTextIntoSections(textoCompleto);
string[] textolinhas = textoCompleto.Split(
new string[] { "\n" },
StringSplitOptions.None
);
var title = textolinhas[0]; // Processa seções em paralelo com limite de concorrência
var builder = new StringBuilder(); var semaphore = new SemaphoreSlim(5, 5); // Máximo 5 operações simultâneas
var tasks = textoArray.Select(async item =>
foreach (string line in textolinhas)
{ {
if (line.StartsWith("**") || line.StartsWith("\r**")) await semaphore.WaitAsync();
try
{ {
if (builder.Length > 0) var lines = item.Split('\n', 2);
{ var title = lines[0].Replace("**", "").Replace("\r", "").Trim();
textoArray.Add(title.Replace("**", "").Replace("\r", "") + ": " + Environment.NewLine + builder.ToString()); var content = lines.Length > 1 ? lines[1] : "";
builder = new StringBuilder();
title = line; await SalvarNoMongoDB(title, content, projectId);
}
} }
else finally
{ {
builder.AppendLine(line); semaphore.Release();
} }
} });
// Adiciona último bloco se houver await Task.WhenAll(tasks);
if (builder.Length > 0) semaphore.Dispose();
{
textoArray.Add(title.Replace("**", "").Replace("\r", "") + ": " + Environment.NewLine + builder.ToString());
}
// Processa cada seção
foreach (var item in textoArray)
{
await SalvarNoMongoDB(title.Replace("**", "").Replace("\r", ""), item, projectId);
}
_logger.LogInformation("Texto completo processado: {SectionCount} seções salvas no Qdrant", textoArray.Count); _logger.LogInformation("Texto completo processado: {SectionCount} seções salvas no Qdrant", textoArray.Count);
} }
@ -121,16 +115,41 @@ namespace ChatRAG.Services.TextServices
{ {
try try
{ {
// Busca todos os projetos e depois todos os documentos // Usa cache de project IDs quando possível
var projectIds = await GetAllProjectIdsOptimized();
if (!projectIds.Any())
{
return Enumerable.Empty<TextoComEmbedding>();
}
var allDocuments = new List<VectorSearchResult>(); var allDocuments = new List<VectorSearchResult>();
// Como Qdrant não tem um "GetAll" direto, vamos usar scroll // Busca documentos em paralelo por projeto
// Isso é uma limitação vs MongoDB, mas é mais eficiente var semaphore = new SemaphoreSlim(3, 3); // Máximo 3 projetos simultâneos
var projects = await GetAllProjectIds(); var tasks = projectIds.Select(async projectId =>
{
foreach (var projectId in projects) await semaphore.WaitAsync();
try
{
return await _vectorSearchService.GetDocumentsByProjectAsync(projectId);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Erro ao buscar documentos do projeto {ProjectId}", projectId);
return new List<VectorSearchResult>();
}
finally
{
semaphore.Release();
}
});
var results = await Task.WhenAll(tasks);
semaphore.Dispose();
foreach (var projectDocs in results)
{ {
var projectDocs = await _vectorSearchService.GetDocumentsByProjectAsync(projectId);
allDocuments.AddRange(projectDocs); allDocuments.AddRange(projectDocs);
} }
@ -184,9 +203,7 @@ namespace ChatRAG.Services.TextServices
{ {
try try
{ {
var conteudo = $"**{document.Title}** \n\n {document.Content}"; var embedding = await GenerateEmbeddingOptimized($"**{document.Title}** \n\n {document.Content}");
var embedding = await _embeddingService.GenerateEmbeddingAsync(conteudo);
var embeddingArray = embedding.ToArray().Select(e => (double)e).ToArray();
string id; string id;
if (!string.IsNullOrEmpty(document.Id)) if (!string.IsNullOrEmpty(document.Id))
@ -197,7 +214,7 @@ namespace ChatRAG.Services.TextServices
document.Title, document.Title,
document.Content, document.Content,
document.ProjectId, document.ProjectId,
embeddingArray, embedding,
document.Metadata); document.Metadata);
id = document.Id; id = document.Id;
} }
@ -208,10 +225,13 @@ namespace ChatRAG.Services.TextServices
document.Title, document.Title,
document.Content, document.Content,
document.ProjectId, document.ProjectId,
embeddingArray, embedding,
document.Metadata); document.Metadata);
} }
// Atualiza cache de project IDs
_projectIdCache.TryAdd(document.ProjectId, DateTime.UtcNow);
_logger.LogDebug("Documento {Id} salvo no Qdrant via SaveDocumentAsync", id); _logger.LogDebug("Documento {Id} salvo no Qdrant via SaveDocumentAsync", id);
return id; return id;
} }
@ -226,16 +246,14 @@ namespace ChatRAG.Services.TextServices
{ {
try try
{ {
var conteudo = $"**{document.Title}** \n\n {document.Content}"; var embedding = await GenerateEmbeddingOptimized($"**{document.Title}** \n\n {document.Content}");
var embedding = await _embeddingService.GenerateEmbeddingAsync(conteudo);
var embeddingArray = embedding.ToArray().Select(e => (double)e).ToArray();
await _vectorSearchService.UpdateDocumentAsync( await _vectorSearchService.UpdateDocumentAsync(
id, id,
document.Title, document.Title,
document.Content, document.Content,
document.ProjectId, document.ProjectId,
embeddingArray, embedding,
document.Metadata); document.Metadata);
_logger.LogDebug("Documento {Id} atualizado no Qdrant", id); _logger.LogDebug("Documento {Id} atualizado no Qdrant", id);
@ -339,7 +357,7 @@ namespace ChatRAG.Services.TextServices
} }
// ======================================== // ========================================
// OPERAÇÕES EM LOTE // OPERAÇÕES EM LOTE OTIMIZADAS
// ======================================== // ========================================
public async Task<List<string>> SaveDocumentsBatchAsync(List<DocumentInput> documents) public async Task<List<string>> SaveDocumentsBatchAsync(List<DocumentInput> documents)
@ -347,28 +365,83 @@ namespace ChatRAG.Services.TextServices
var ids = new List<string>(); var ids = new List<string>();
var errors = new List<Exception>(); var errors = new List<Exception>();
// Processa em lotes menores para performance // Agrupa documentos por projeto para otimizar embeddings
var batchSize = 10; var documentsByProject = documents.GroupBy(d => d.ProjectId).ToList();
for (int i = 0; i < documents.Count; i += batchSize)
{
var batch = documents.Skip(i).Take(batchSize);
var tasks = batch.Select(async doc =>
{
try
{
var id = await SaveDocumentAsync(doc);
return id;
}
catch (Exception ex)
{
errors.Add(ex);
_logger.LogError(ex, "Erro ao salvar documento '{Title}' em lote", doc.Title);
return null;
}
});
var batchResults = await Task.WhenAll(tasks); foreach (var projectGroup in documentsByProject)
ids.AddRange(batchResults.Where(id => id != null)!); {
var projectDocs = projectGroup.ToList();
// Processa em lotes menores dentro do projeto
var batchSize = 5; // Reduzido para evitar timeout
for (int i = 0; i < projectDocs.Count; i += batchSize)
{
var batch = projectDocs.Skip(i).Take(batchSize);
// Gera embeddings em paralelo para o lote
var embeddingTasks = batch.Select(async doc =>
{
try
{
var embedding = await GenerateEmbeddingOptimized($"**{doc.Title}** \n\n {doc.Content}");
return new { Document = doc, Embedding = embedding, Error = (Exception?)null };
}
catch (Exception ex)
{
return new { Document = doc, Embedding = (double[]?)null, Error = ex };
}
});
var embeddingResults = await Task.WhenAll(embeddingTasks);
// Salva documentos com embeddings gerados
var saveTasks = embeddingResults.Select(async result =>
{
if (result.Error != null)
{
errors.Add(result.Error);
return null;
}
try
{
string id;
if (!string.IsNullOrEmpty(result.Document.Id))
{
await _vectorSearchService.UpdateDocumentAsync(
result.Document.Id,
result.Document.Title,
result.Document.Content,
result.Document.ProjectId,
result.Embedding!,
result.Document.Metadata);
id = result.Document.Id;
}
else
{
id = await _vectorSearchService.AddDocumentAsync(
result.Document.Title,
result.Document.Content,
result.Document.ProjectId,
result.Embedding!,
result.Document.Metadata);
}
return id;
}
catch (Exception ex)
{
errors.Add(ex);
_logger.LogError(ex, "Erro ao salvar documento '{Title}' em lote", result.Document.Title);
return null;
}
});
var batchResults = await Task.WhenAll(saveTasks);
ids.AddRange(batchResults.Where(id => id != null)!);
}
// Atualiza cache para o projeto
_projectIdCache.TryAdd(projectGroup.Key, DateTime.UtcNow);
} }
if (errors.Any()) if (errors.Any())
@ -387,8 +460,8 @@ namespace ChatRAG.Services.TextServices
{ {
var errors = new List<Exception>(); var errors = new List<Exception>();
// Processa em lotes para não sobrecarregar // Processa em lotes pequenos para não sobrecarregar
var batchSize = 20; var batchSize = 10; // Reduzido para melhor estabilidade
for (int i = 0; i < ids.Count; i += batchSize) for (int i = 0; i < ids.Count; i += batchSize)
{ {
var batch = ids.Skip(i).Take(batchSize); var batch = ids.Skip(i).Take(batchSize);
@ -396,7 +469,7 @@ namespace ChatRAG.Services.TextServices
{ {
try try
{ {
await DeleteDocumentAsync(id); await _vectorSearchService.DeleteDocumentAsync(id);
return true; return true;
} }
catch (Exception ex) catch (Exception ex)
@ -432,14 +505,28 @@ namespace ChatRAG.Services.TextServices
var baseStats = await _vectorSearchService.GetStatsAsync(); var baseStats = await _vectorSearchService.GetStatsAsync();
var totalDocs = await GetDocumentCountAsync(); var totalDocs = await GetDocumentCountAsync();
// Adiciona estatísticas específicas do TextData // Usa cache para project IDs
var projectIds = await GetAllProjectIds(); var projectIds = await GetAllProjectIdsOptimized();
var projectStats = new Dictionary<string, int>(); var projectStats = new Dictionary<string, int>();
foreach (var projectId in projectIds) // Busca contadores em paralelo
var countTasks = projectIds.Select(async projectId =>
{ {
var count = await GetDocumentCountAsync(projectId); try
projectStats[projectId] = count; {
var count = await GetDocumentCountAsync(projectId);
return new { ProjectId = projectId, Count = count };
}
catch
{
return new { ProjectId = projectId, Count = 0 };
}
});
var countResults = await Task.WhenAll(countTasks);
foreach (var result in countResults)
{
projectStats[result.ProjectId] = result.Count;
} }
var enhancedStats = new Dictionary<string, object>(baseStats) var enhancedStats = new Dictionary<string, object>(baseStats)
@ -450,7 +537,9 @@ namespace ChatRAG.Services.TextServices
["documents_by_project"] = projectStats, ["documents_by_project"] = projectStats,
["supports_batch_operations"] = true, ["supports_batch_operations"] = true,
["supports_metadata"] = true, ["supports_metadata"] = true,
["embedding_auto_generation"] = true ["embedding_auto_generation"] = true,
["cache_enabled"] = true,
["cached_project_ids"] = _projectIdCache.Count
}; };
return enhancedStats; return enhancedStats;
@ -469,9 +558,100 @@ namespace ChatRAG.Services.TextServices
} }
// ======================================== // ========================================
// MÉTODOS AUXILIARES PRIVADOS // MÉTODOS AUXILIARES PRIVADOS OTIMIZADOS
// ======================================== // ========================================
private async Task<double[]> GenerateEmbeddingOptimized(string content)
{
var embedding = await _embeddingService.GenerateEmbeddingAsync(content);
return embedding.ToArray().Select(e => (double)e).ToArray();
}
private static List<string> ParseTextIntoSections(string textoCompleto)
{
var textoArray = new List<string>();
string[] textolinhas = textoCompleto.Split(new string[] { "\n" }, StringSplitOptions.None);
var title = textolinhas[0];
var builder = new StringBuilder();
foreach (string line in textolinhas)
{
if (line.StartsWith("**") || line.StartsWith("\r**"))
{
if (builder.Length > 0)
{
textoArray.Add(title.Replace("**", "").Replace("\r", "") + ": " + Environment.NewLine + builder.ToString());
builder = new StringBuilder();
title = line;
}
}
else
{
builder.AppendLine(line);
}
}
// Adiciona último bloco se houver
if (builder.Length > 0)
{
textoArray.Add(title.Replace("**", "").Replace("\r", "") + ": " + Environment.NewLine + builder.ToString());
}
return textoArray;
}
private async Task<List<string>> GetAllProjectIdsOptimized()
{
// Remove entradas expiradas do cache
var now = DateTime.UtcNow;
var expiredKeys = _projectIdCache
.Where(kvp => now - kvp.Value > _cacheTimeout)
.Select(kvp => kvp.Key)
.ToList();
foreach (var key in expiredKeys)
{
_projectIdCache.TryRemove(key, out _);
}
// Se temos dados no cache e não estão muito antigos, usa o cache
if (_projectIdCache.Any())
{
return _projectIdCache.Keys.ToList();
}
// Caso contrário, busca no Qdrant
try
{
// Esta busca é custosa, mas só será executada quando o cache estiver vazio
var allResults = await _vectorSearchService.SearchSimilarAsync(
new double[384], // Vector dummy menor
projectId: null,
threshold: 0.0,
limit: 1000); // Limit menor para melhor performance
var projectIds = allResults
.Select(r => r.ProjectId)
.Where(pid => !string.IsNullOrEmpty(pid))
.Distinct()
.ToList();
// Atualiza cache
foreach (var projectId in projectIds)
{
_projectIdCache.TryAdd(projectId, now);
}
return projectIds;
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao recuperar IDs de projetos do Qdrant");
return new List<string>();
}
}
private static TextoComEmbedding ConvertToTextoComEmbedding(VectorSearchResult result) private static TextoComEmbedding ConvertToTextoComEmbedding(VectorSearchResult result)
{ {
return new TextoComEmbedding return new TextoComEmbedding
@ -488,35 +668,6 @@ namespace ChatRAG.Services.TextServices
Tags = result.Metadata?.GetValueOrDefault("tags") as string[] ?? Array.Empty<string>() Tags = result.Metadata?.GetValueOrDefault("tags") as string[] ?? Array.Empty<string>()
}; };
} }
private async Task<List<string>> GetAllProjectIds()
{
try
{
// Esta é uma operação custosa no Qdrant
// Em produção, seria melhor manter um cache de project IDs
// ou usar uma estrutura de dados separada
// Por agora, vamos usar uma busca com um vetor dummy para pegar todos os documentos
var dummyVector = new double[1536]; // Assumindo embeddings OpenAI
var allResults = await _vectorSearchService.SearchSimilarAsync(
dummyVector,
projectId: null,
threshold: 0.0,
limit: 10000);
return allResults
.Select(r => r.ProjectId)
.Where(pid => !string.IsNullOrEmpty(pid))
.Distinct()
.ToList();
}
catch (Exception ex)
{
_logger.LogError(ex, "Erro ao recuperar IDs de projetos do Qdrant");
return new List<string>();
}
}
} }
} }

View File

@ -1,156 +1,59 @@
namespace ChatRAG.Settings using Microsoft.Extensions.AI;
using Qdrant.Client.Grpc;
namespace ChatRAG.Settings.ChatRAG.Configuration
{ {
// ============================================================================ public class VectorDatabaseSettings
// 📁 Configuration/VectorDatabaseSettings.cs
// Settings unificados para todos os providers (MongoDB, Qdrant, etc.)
// ============================================================================
namespace ChatRAG.Configuration
{ {
public string Provider { get; set; } = "Qdrant";
public MongoDBSettings? MongoDB { get; set; }
public QdrantSettings? Qdrant { get; set; }
public ChromaSettings? Chroma { get; set; }
public EmbeddingSettings Embedding { get; set; } = new();
/// <summary> /// <summary>
/// Configurações principais do sistema de Vector Database /// Retorna erros de validação
/// </summary> /// </summary>
public class VectorDatabaseSettings public List<string> GetValidationErrors()
{ {
/// <summary> var errors = new List<string>();
/// Provider ativo (MongoDB, Qdrant, Pinecone, etc.)
/// </summary>
public string Provider { get; set; } = "MongoDB";
/// <summary> if (string.IsNullOrWhiteSpace(Provider))
/// Configurações específicas do MongoDB errors.Add("Provider é obrigatório");
/// </summary>
public MongoDbSettings MongoDB { get; set; } = new();
/// <summary> switch (Provider.ToLower())
/// Configurações específicas do Qdrant
/// </summary>
public QdrantSettings Qdrant { get; set; } = new();
/// <summary>
/// Configurações globais de embedding
/// </summary>
public EmbeddingSettings Embedding { get; set; } = new();
/// <summary>
/// Configurações de performance e cache
/// </summary>
public PerformanceSettings Performance { get; set; } = new();
/// <summary>
/// Configurações de logging e monitoramento
/// </summary>
public MonitoringSettings Monitoring { get; set; } = new();
/// <summary>
/// Valida se as configurações estão corretas
/// </summary>
public bool IsValid()
{ {
if (string.IsNullOrWhiteSpace(Provider)) case "mongodb":
return false; errors.AddRange(MongoDB.GetValidationErrors());
break;
return Provider.ToLower() switch case "qdrant":
{ errors.AddRange(Qdrant.GetValidationErrors());
"mongodb" => MongoDB.IsValid(), break;
"qdrant" => Qdrant.IsValid(), case "chroma":
_ => false errors.AddRange(Chroma.GetValidationErrors());
}; break;
default:
errors.Add($"Provider '{Provider}' não é suportado");
break;
} }
/// <summary> errors.AddRange(Embedding.GetValidationErrors());
/// Retorna erros de validação
/// </summary>
public List<string> GetValidationErrors()
{
var errors = new List<string>();
if (string.IsNullOrWhiteSpace(Provider)) return errors;
errors.Add("Provider é obrigatório");
switch (Provider.ToLower())
{
case "mongodb":
errors.AddRange(MongoDB.GetValidationErrors());
break;
case "qdrant":
errors.AddRange(Qdrant.GetValidationErrors());
break;
default:
errors.Add($"Provider '{Provider}' não é suportado");
break;
}
errors.AddRange(Embedding.GetValidationErrors());
return errors;
}
} }
}
/// <summary> public class MongoDBSettings
/// Configurações específicas do MongoDB {
/// </summary> public string ConnectionString { get; set; } = "";
public class MongoDbSettings public string DatabaseName { get; set; } = "";
public string TextCollectionName { get; set; } = "Texts";
public string ProjectCollectionName { get; set; } = "Groups";
public string UserDataName { get; set; } = "UserData";
public int ConnectionTimeoutSeconds { get; set; } = 30;
public List<string> GetValidationErrors()
{ {
/// <summary>
/// String de conexão do MongoDB
/// </summary>
public string ConnectionString { get; set; } = string.Empty;
/// <summary>
/// Nome do banco de dados
/// </summary>
public string DatabaseName { get; set; } = string.Empty;
/// <summary>
/// Nome da coleção de textos/documentos
/// </summary>
public string TextCollectionName { get; set; } = "texts";
/// <summary>
/// Nome da coleção de projetos
/// </summary>
public string ProjectCollectionName { get; set; } = "projects";
/// <summary>
/// Nome da coleção de dados de usuário
/// </summary>
public string UserDataName { get; set; } = "users";
/// <summary>
/// Timeout de conexão em segundos
/// </summary>
public int ConnectionTimeoutSeconds { get; set; } = 30;
/// <summary>
/// Timeout de operação em segundos
/// </summary>
public int OperationTimeoutSeconds { get; set; } = 60;
/// <summary>
/// Tamanho máximo do pool de conexões
/// </summary>
public int MaxConnectionPoolSize { get; set; } = 100;
/// <summary>
/// Habilitar índices de busca vetorial
/// </summary>
public bool EnableVectorSearch { get; set; } = true;
/// <summary>
/// Configurações específicas do Atlas Search
/// </summary>
public MongoAtlasSearchSettings AtlasSearch { get; set; } = new();
public bool IsValid()
{
return !string.IsNullOrWhiteSpace(ConnectionString) &&
!string.IsNullOrWhiteSpace(DatabaseName) &&
!string.IsNullOrWhiteSpace(TextCollectionName);
}
public List<string> GetValidationErrors()
{
var errors = new List<string>(); var errors = new List<string>();
if (string.IsNullOrWhiteSpace(ConnectionString)) if (string.IsNullOrWhiteSpace(ConnectionString))
@ -166,329 +69,110 @@
errors.Add("MongoDB ConnectionTimeoutSeconds deve ser maior que 0"); errors.Add("MongoDB ConnectionTimeoutSeconds deve ser maior que 0");
return errors; return errors;
}
}
/// <summary>
/// Configurações do MongoDB Atlas Search
/// </summary>
public class MongoAtlasSearchSettings
{
/// <summary>
/// Nome do índice de busca vetorial
/// </summary>
public string VectorIndexName { get; set; } = "vector_index";
/// <summary>
/// Número de candidatos para busca aproximada
/// </summary>
public int NumCandidates { get; set; } = 200;
/// <summary>
/// Limite de resultados do Atlas Search
/// </summary>
public int SearchLimit { get; set; } = 100;
}
/// <summary>
/// Configurações específicas do Qdrant
/// </summary>
public class QdrantSettings
{
/// <summary>
/// Host do servidor Qdrant
/// </summary>
public string Host { get; set; } = "localhost";
/// <summary>
/// Porta do servidor Qdrant (REST API)
/// </summary>
public int Port { get; set; } = 6333;
/// <summary>
/// Porta gRPC (opcional, para performance)
/// </summary>
public int? GrpcPort { get; set; } = 6334;
/// <summary>
/// Chave de API (se autenticação estiver habilitada)
/// </summary>
public string? ApiKey { get; set; }
/// <summary>
/// Usar TLS/SSL
/// </summary>
public bool UseTls { get; set; } = false;
/// <summary>
/// Nome da coleção principal
/// </summary>
public string CollectionName { get; set; } = "documents";
/// <summary>
/// Tamanho do vetor de embedding
/// </summary>
public int VectorSize { get; set; } = 1536; // OpenAI embedding size
/// <summary>
/// Métrica de distância (Cosine, Euclid, Dot, Manhattan)
/// </summary>
public string Distance { get; set; } = "Cosine";
// ========================================
// CONFIGURAÇÕES DE PERFORMANCE
// ========================================
/// <summary>
/// Parâmetro M do algoritmo HNSW (conectividade)
/// Valores típicos: 16-48, maior = melhor recall, mais memória
/// </summary>
public int HnswM { get; set; } = 16;
/// <summary>
/// Parâmetro ef_construct do HNSW (construção do índice)
/// Valores típicos: 100-800, maior = melhor qualidade, mais lento
/// </summary>
public int HnswEfConstruct { get; set; } = 200;
/// <summary>
/// Parâmetro ef do HNSW (busca)
/// Valores típicos: igual ou maior que o número de resultados desejados
/// </summary>
public int HnswEf { get; set; } = 128;
/// <summary>
/// Armazenar vetores em disco (economiza RAM)
/// </summary>
public bool OnDisk { get; set; } = false;
/// <summary>
/// Fator de replicação (para clusters)
/// </summary>
public int ReplicationFactor { get; set; } = 1;
/// <summary>
/// Número de shards (para distribuição)
/// </summary>
public int ShardNumber { get; set; } = 1;
/// <summary>
/// Usar quantização para reduzir uso de memória
/// </summary>
public bool UseQuantization { get; set; } = false;
/// <summary>
/// Configurações de quantização
/// </summary>
public QuantizationSettings Quantization { get; set; } = new();
/// <summary>
/// Timeout de conexão em segundos
/// </summary>
public int ConnectionTimeoutSeconds { get; set; } = 30;
/// <summary>
/// Timeout de operação em segundos
/// </summary>
public int OperationTimeoutSeconds { get; set; } = 60;
/// <summary>
/// URL completa calculada
/// </summary>
public string GetConnectionUrl()
{
var protocol = UseTls ? "https" : "http";
return $"{protocol}://{Host}:{Port}";
}
public bool IsValid()
{
return !string.IsNullOrWhiteSpace(Host) &&
Port > 0 &&
!string.IsNullOrWhiteSpace(CollectionName) &&
VectorSize > 0;
}
public List<string> GetValidationErrors()
{
var errors = new List<string>();
if (string.IsNullOrWhiteSpace(Host))
errors.Add("Qdrant Host é obrigatório");
if (Port <= 0)
errors.Add("Qdrant Port deve ser maior que 0");
if (string.IsNullOrWhiteSpace(CollectionName))
errors.Add("Qdrant CollectionName é obrigatório");
if (VectorSize <= 0)
errors.Add("Qdrant VectorSize deve ser maior que 0");
if (HnswM <= 0)
errors.Add("Qdrant HnswM deve ser maior que 0");
if (HnswEfConstruct <= 0)
errors.Add("Qdrant HnswEfConstruct deve ser maior que 0");
var validDistances = new[] { "Cosine", "Euclid", "Dot", "Manhattan" };
if (!validDistances.Contains(Distance))
errors.Add($"Qdrant Distance deve ser um de: {string.Join(", ", validDistances)}");
return errors;
}
}
/// <summary>
/// Configurações de quantização para Qdrant
/// </summary>
public class QuantizationSettings
{
/// <summary>
/// Tipo de quantização (scalar, product, binary)
/// </summary>
public string Type { get; set; } = "scalar";
/// <summary>
/// Quantil para quantização scalar
/// </summary>
public double Quantile { get; set; } = 0.99;
/// <summary>
/// Sempre usar RAM para quantização
/// </summary>
public bool AlwaysRam { get; set; } = false;
}
/// <summary>
/// Configurações globais de embedding
/// </summary>
public class EmbeddingSettings
{
/// <summary>
/// Provider de embedding (OpenAI, Ollama, Azure, etc.)
/// </summary>
public string Provider { get; set; } = "OpenAI";
/// <summary>
/// Modelo de embedding
/// </summary>
public string Model { get; set; } = "text-embedding-ada-002";
/// <summary>
/// Tamanho esperado do embedding
/// </summary>
public int ExpectedSize { get; set; } = 1536;
/// <summary>
/// Batch size para processamento em lote
/// </summary>
public int BatchSize { get; set; } = 100;
/// <summary>
/// Cache de embeddings em memória
/// </summary>
public bool EnableCache { get; set; } = true;
/// <summary>
/// TTL do cache em minutos
/// </summary>
public int CacheTtlMinutes { get; set; } = 60;
public List<string> GetValidationErrors()
{
var errors = new List<string>();
if (ExpectedSize <= 0)
errors.Add("Embedding ExpectedSize deve ser maior que 0");
if (BatchSize <= 0)
errors.Add("Embedding BatchSize deve ser maior que 0");
return errors;
}
}
/// <summary>
/// Configurações de performance e otimização
/// </summary>
public class PerformanceSettings
{
/// <summary>
/// Habilitar paralelização em operações de lote
/// </summary>
public bool EnableParallelization { get; set; } = true;
/// <summary>
/// Número máximo de threads paralelas
/// </summary>
public int MaxParallelism { get; set; } = Environment.ProcessorCount;
/// <summary>
/// Tamanho do batch para operações em lote
/// </summary>
public int BatchSize { get; set; } = 100;
/// <summary>
/// Timeout padrão para operações em segundos
/// </summary>
public int DefaultTimeoutSeconds { get; set; } = 30;
/// <summary>
/// Habilitar retry automático
/// </summary>
public bool EnableRetry { get; set; } = true;
/// <summary>
/// Número máximo de tentativas
/// </summary>
public int MaxRetryAttempts { get; set; } = 3;
/// <summary>
/// Delay entre tentativas em segundos
/// </summary>
public int RetryDelaySeconds { get; set; } = 2;
}
/// <summary>
/// Configurações de monitoramento e logging
/// </summary>
public class MonitoringSettings
{
/// <summary>
/// Habilitar logging detalhado
/// </summary>
public bool EnableDetailedLogging { get; set; } = false;
/// <summary>
/// Logar tempos de operação
/// </summary>
public bool LogOperationTimes { get; set; } = true;
/// <summary>
/// Threshold para log de operações lentas (ms)
/// </summary>
public int SlowOperationThresholdMs { get; set; } = 1000;
/// <summary>
/// Habilitar métricas de performance
/// </summary>
public bool EnableMetrics { get; set; } = true;
/// <summary>
/// Intervalo de coleta de métricas em segundos
/// </summary>
public int MetricsIntervalSeconds { get; set; } = 60;
/// <summary>
/// Habilitar health checks
/// </summary>
public bool EnableHealthChecks { get; set; } = true;
/// <summary>
/// Intervalo de health check em segundos
/// </summary>
public int HealthCheckIntervalSeconds { get; set; } = 30;
} }
} }
}
public class QdrantSettings
{
public string Host { get; set; } = "localhost";
public int Port { get; set; } = 6334;
public string CollectionName { get; set; } = "texts";
public string GroupsCollectionName { get; set; } = "projects";
public int VectorSize { get; set; } = 384;
public string Distance { get; set; } = "Cosine";
public int HnswM { get; set; } = 16;
public int HnswEfConstruct { get; set; } = 200;
public bool OnDisk { get; set; } = false;
public bool UseTls { get; set; } = false;
public List<string> GetValidationErrors()
{
var errors = new List<string>();
if (string.IsNullOrWhiteSpace(Host))
errors.Add("Qdrant Host é obrigatório");
if (Port <= 0)
errors.Add("Qdrant Port deve ser maior que 0");
if (string.IsNullOrWhiteSpace(CollectionName))
errors.Add("Qdrant CollectionName é obrigatório");
if (VectorSize <= 0)
errors.Add("Qdrant VectorSize deve ser maior que 0");
if (HnswM <= 0)
errors.Add("Qdrant HnswM deve ser maior que 0");
if (HnswEfConstruct <= 0)
errors.Add("Qdrant HnswEfConstruct deve ser maior que 0");
var validDistances = new[] { "Cosine", "Euclid", "Dot", "Manhattan" };
if (!validDistances.Contains(Distance))
errors.Add($"Qdrant Distance deve ser um de: {string.Join(", ", validDistances)}");
return errors;
}
}
public class ChromaSettings
{
public string Host { get; set; } = "localhost";
public int Port { get; set; } = 8000;
public string CollectionName { get; set; } = "rag_documents";
public string ApiVersion { get; set; } = "v1";
public List<string> GetValidationErrors()
{
var errors = new List<string>();
return errors;
}
}
public class EmbeddingSettings
{
/// <summary>
/// Provider de embedding (OpenAI, Ollama, Azure, etc.)
/// </summary>
public string Provider { get; set; } = "OpenAI";
/// <summary>
/// Modelo de embedding
/// </summary>
public string Model { get; set; } = "text-embedding-ada-002";
/// <summary>
/// Tamanho esperado do embedding
/// </summary>
public int ExpectedSize { get; set; } = 1536;
/// <summary>
/// Batch size para processamento em lote
/// </summary>
public int BatchSize { get; set; } = 100;
/// <summary>
/// Cache de embeddings em memória
/// </summary>
public bool EnableCache { get; set; } = true;
/// <summary>
/// TTL do cache em minutos
/// </summary>
public int CacheTtlMinutes { get; set; } = 60;
public List<string> GetValidationErrors()
{
var errors = new List<string>();
if (ExpectedSize <= 0)
errors.Add("Embedding ExpectedSize deve ser maior que 0");
if (BatchSize <= 0)
errors.Add("Embedding BatchSize deve ser maior que 0");
return errors;
}
}
}

128
Settings/gckijn3t.ir5~ Normal file
View File

@ -0,0 +1,128 @@
using Microsoft.Extensions.AI;
using Qdrant.Client.Grpc;
namespace ChatRAG.Settings.ChatRAG.Configuration
{
public class VectorDatabaseSettings
{
public string Provider { get; set; } = "Qdrant";
public MongoDBSettings? MongoDB { get; set; }
public QdrantSettings? Qdrant { get; set; }
public ChromaSettings? Chroma { get; set; }
/// <summary>
/// Retorna erros de validação
/// </summary>
public List<string> GetValidationErrors()
{
var errors = new List<string>();
if (string.IsNullOrWhiteSpace(Provider))
errors.Add("Provider é obrigatório");
switch (Provider.ToLower())
{
case "mongodb":
errors.AddRange(MongoDB.GetValidationErrors());
break;
case "qdrant":
errors.AddRange(Qdrant.GetValidationErrors());
break;
default:
errors.Add($"Provider '{Provider}' não é suportado");
break;
}
errors.AddRange(Embedding.GetValidationErrors());
return errors;
}
}
}
public class MongoDBSettings
{
public string ConnectionString { get; set; } = "";
public string DatabaseName { get; set; } = "";
public string TextCollectionName { get; set; } = "Texts";
public string ProjectCollectionName { get; set; } = "Groups";
public string UserDataName { get; set; } = "UserData";
public int ConnectionTimeoutSeconds { get; set; } = 30;
public List<string> GetValidationErrors()
{
var errors = new List<string>();
if (string.IsNullOrWhiteSpace(ConnectionString))
errors.Add("MongoDB ConnectionString é obrigatória");
if (string.IsNullOrWhiteSpace(DatabaseName))
errors.Add("MongoDB DatabaseName é obrigatório");
if (string.IsNullOrWhiteSpace(TextCollectionName))
errors.Add("MongoDB TextCollectionName é obrigatório");
if (ConnectionTimeoutSeconds <= 0)
errors.Add("MongoDB ConnectionTimeoutSeconds deve ser maior que 0");
return errors;
}
}
public class QdrantSettings
{
public string Host { get; set; } = "localhost";
public int Port { get; set; } = 6334;
public string CollectionName { get; set; } = "texts";
public int VectorSize { get; set; } = 384;
public string Distance { get; set; } = "Cosine";
public int HnswM { get; set; } = 16;
public int HnswEfConstruct { get; set; } = 200;
public bool OnDisk { get; set; } = false;
public bool UseTls { get; set; } = false;
public List<string> GetValidationErrors()
{
var errors = new List<string>();
if (string.IsNullOrWhiteSpace(Host))
errors.Add("Qdrant Host é obrigatório");
if (Port <= 0)
errors.Add("Qdrant Port deve ser maior que 0");
if (string.IsNullOrWhiteSpace(CollectionName))
errors.Add("Qdrant CollectionName é obrigatório");
if (VectorSize <= 0)
errors.Add("Qdrant VectorSize deve ser maior que 0");
if (HnswM <= 0)
errors.Add("Qdrant HnswM deve ser maior que 0");
if (HnswEfConstruct <= 0)
errors.Add("Qdrant HnswEfConstruct deve ser maior que 0");
var validDistances = new[] { "Cosine", "Euclid", "Dot", "Manhattan" };
if (!validDistances.Contains(Distance))
errors.Add($"Qdrant Distance deve ser um de: {string.Join(", ", validDistances)}");
return errors;
}
}
public class ChromaSettings
{
public string Host { get; set; } = "localhost";
public int Port { get; set; } = 8000;
public string CollectionName { get; set; } = "rag_documents";
public string ApiVersion { get; set; } = "v1";
public List<string> GetValidationErrors()
{
var errors = new List<string>();
return errors;
}
}
}

120
Settings/ghcutjxi.wn3~ Normal file
View File

@ -0,0 +1,120 @@
using Microsoft.Extensions.AI;
namespace ChatRAG.Settings.ChatRAG.Configuration
{
public class VectorDatabaseSettings
{
public string Provider { get; set; } = "Qdrant";
public MongoDBSettings? MongoDB { get; set; }
public QdrantSettings? Qdrant { get; set; }
public ChromaSettings? Chroma { get; set; }
/// <summary>
/// Retorna erros de validação
/// </summary>
public List<string> GetValidationErrors()
{
var errors = new List<string>();
if (string.IsNullOrWhiteSpace(Provider))
errors.Add("Provider é obrigatório");
switch (Provider.ToLower())
{
case "mongodb":
errors.AddRange(MongoDB.GetValidationErrors());
break;
case "qdrant":
errors.AddRange(Qdrant.GetValidationErrors());
break;
default:
errors.Add($"Provider '{Provider}' não é suportado");
break;
}
errors.AddRange(Embedding.GetValidationErrors());
return errors;
}
}
}
public class MongoDBSettings
{
public string ConnectionString { get; set; } = "";
public string DatabaseName { get; set; } = "";
public string TextCollectionName { get; set; } = "Texts";
public string ProjectCollectionName { get; set; } = "Groups";
public string UserDataName { get; set; } = "UserData";
public int ConnectionTimeoutSeconds { get; set; } = 30;
public List<string> GetValidationErrors()
{
var errors = new List<string>();
if (string.IsNullOrWhiteSpace(ConnectionString))
errors.Add("MongoDB ConnectionString é obrigatória");
if (string.IsNullOrWhiteSpace(DatabaseName))
errors.Add("MongoDB DatabaseName é obrigatório");
if (string.IsNullOrWhiteSpace(TextCollectionName))
errors.Add("MongoDB TextCollectionName é obrigatório");
if (ConnectionTimeoutSeconds <= 0)
errors.Add("MongoDB ConnectionTimeoutSeconds deve ser maior que 0");
return errors;
}
}
public class QdrantSettings
{
public string Host { get; set; } = "localhost";
public int Port { get; set; } = 6334;
public string CollectionName { get; set; } = "texts";
public int VectorSize { get; set; } = 384;
public string Distance { get; set; } = "Cosine";
public int HnswM { get; set; } = 16;
public int HnswEfConstruct { get; set; } = 200;
public bool OnDisk { get; set; } = false;
public bool UseTls { get; set; } = false;
public List<string> GetValidationErrors()
{
var errors = new List<string>();
if (string.IsNullOrWhiteSpace(Host))
errors.Add("Qdrant Host é obrigatório");
if (Port <= 0)
errors.Add("Qdrant Port deve ser maior que 0");
if (string.IsNullOrWhiteSpace(CollectionName))
errors.Add("Qdrant CollectionName é obrigatório");
if (VectorSize <= 0)
errors.Add("Qdrant VectorSize deve ser maior que 0");
if (HnswM <= 0)
errors.Add("Qdrant HnswM deve ser maior que 0");
if (HnswEfConstruct <= 0)
errors.Add("Qdrant HnswEfConstruct deve ser maior que 0");
var validDistances = new[] { "Cosine", "Euclid", "Dot", "Manhattan" };
if (!validDistances.Contains(Distance))
errors.Add($"Qdrant Distance deve ser um de: {string.Join(", ", validDistances)}");
return errors;
}
}
public class ChromaSettings
{
public string Host { get; set; } = "localhost";
public int Port { get; set; } = 8000;
public string CollectionName { get; set; } = "rag_documents";
public string ApiVersion { get; set; } = "v1";
}
}

91
Settings/ixb5gark.btp~ Normal file
View File

@ -0,0 +1,91 @@
using Microsoft.Extensions.AI;
namespace ChatRAG.Settings.ChatRAG.Configuration
{
public class VectorDatabaseSettings
{
public string Provider { get; set; } = "Qdrant";
public MongoDBSettings? MongoDB { get; set; }
public QdrantSettings? Qdrant { get; set; }
public ChromaSettings? Chroma { get; set; }
/// <summary>
/// Retorna erros de validação
/// </summary>
public List<string> GetValidationErrors()
{
var errors = new List<string>();
if (string.IsNullOrWhiteSpace(Provider))
errors.Add("Provider é obrigatório");
switch (Provider.ToLower())
{
case "mongodb":
errors.AddRange(MongoDB.GetValidationErrors());
break;
case "qdrant":
errors.AddRange(Qdrant.GetValidationErrors());
break;
default:
errors.Add($"Provider '{Provider}' não é suportado");
break;
}
errors.AddRange(Embedding.GetValidationErrors());
return errors;
}
}
}
public class MongoDBSettings
{
public string ConnectionString { get; set; } = "";
public string DatabaseName { get; set; } = "";
public string TextCollectionName { get; set; } = "Texts";
public string ProjectCollectionName { get; set; } = "Groups";
public string UserDataName { get; set; } = "UserData";
public int ConnectionTimeoutSeconds { get; set; } = 30;
public List<string> GetValidationErrors()
{
var errors = new List<string>();
if (string.IsNullOrWhiteSpace(ConnectionString))
errors.Add("MongoDB ConnectionString é obrigatória");
if (string.IsNullOrWhiteSpace(DatabaseName))
errors.Add("MongoDB DatabaseName é obrigatório");
if (string.IsNullOrWhiteSpace(TextCollectionName))
errors.Add("MongoDB TextCollectionName é obrigatório");
if (ConnectionTimeoutSeconds <= 0)
errors.Add("MongoDB ConnectionTimeoutSeconds deve ser maior que 0");
return errors;
}
}
public class QdrantSettings
{
public string Host { get; set; } = "localhost";
public int Port { get; set; } = 6334;
public string CollectionName { get; set; } = "texts";
public int VectorSize { get; set; } = 384;
public string Distance { get; set; } = "Cosine";
public int HnswM { get; set; } = 16;
public int HnswEfConstruct { get; set; } = 200;
public bool OnDisk { get; set; } = false;
public bool UseTls { get; set; } = false;
}
public class ChromaSettings
{
public string Host { get; set; } = "localhost";
public int Port { get; set; } = 8000;
public string CollectionName { get; set; } = "rag_documents";
public string ApiVersion { get; set; } = "v1";
}
}

174
Settings/vwuy0ebd.cjy~ Normal file
View File

@ -0,0 +1,174 @@
using Microsoft.Extensions.AI;
using Qdrant.Client.Grpc;
namespace ChatRAG.Settings.ChatRAG.Configuration
{
public class VectorDatabaseSettings
{
public string Provider { get; set; } = "Qdrant";
public MongoDBSettings? MongoDB { get; set; }
public QdrantSettings? Qdrant { get; set; }
public ChromaSettings? Chroma { get; set; }
public EmbeddingSettings Embedding { get; set; } = new();
/// <summary>
/// Retorna erros de validação
/// </summary>
public List<string> GetValidationErrors()
{
var errors = new List<string>();
if (string.IsNullOrWhiteSpace(Provider))
errors.Add("Provider é obrigatório");
switch (Provider.ToLower())
{
case "mongodb":
errors.AddRange(MongoDB.GetValidationErrors());
break;
case "qdrant":
errors.AddRange(Qdrant.GetValidationErrors());
break;
default:
errors.Add($"Provider '{Provider}' não é suportado");
break;
}
errors.AddRange(Embedding.GetValidationErrors());
return errors;
}
}
public class MongoDBSettings
{
public string ConnectionString { get; set; } = "";
public string DatabaseName { get; set; } = "";
public string TextCollectionName { get; set; } = "Texts";
public string ProjectCollectionName { get; set; } = "Groups";
public string UserDataName { get; set; } = "UserData";
public int ConnectionTimeoutSeconds { get; set; } = 30;
public List<string> GetValidationErrors()
{
var errors = new List<string>();
if (string.IsNullOrWhiteSpace(ConnectionString))
errors.Add("MongoDB ConnectionString é obrigatória");
if (string.IsNullOrWhiteSpace(DatabaseName))
errors.Add("MongoDB DatabaseName é obrigatório");
if (string.IsNullOrWhiteSpace(TextCollectionName))
errors.Add("MongoDB TextCollectionName é obrigatório");
if (ConnectionTimeoutSeconds <= 0)
errors.Add("MongoDB ConnectionTimeoutSeconds deve ser maior que 0");
return errors;
}
}
public class QdrantSettings
{
public string Host { get; set; } = "localhost";
public int Port { get; set; } = 6334;
public string CollectionName { get; set; } = "texts";
public int VectorSize { get; set; } = 384;
public string Distance { get; set; } = "Cosine";
public int HnswM { get; set; } = 16;
public int HnswEfConstruct { get; set; } = 200;
public bool OnDisk { get; set; } = false;
public bool UseTls { get; set; } = false;
public List<string> GetValidationErrors()
{
var errors = new List<string>();
if (string.IsNullOrWhiteSpace(Host))
errors.Add("Qdrant Host é obrigatório");
if (Port <= 0)
errors.Add("Qdrant Port deve ser maior que 0");
if (string.IsNullOrWhiteSpace(CollectionName))
errors.Add("Qdrant CollectionName é obrigatório");
if (VectorSize <= 0)
errors.Add("Qdrant VectorSize deve ser maior que 0");
if (HnswM <= 0)
errors.Add("Qdrant HnswM deve ser maior que 0");
if (HnswEfConstruct <= 0)
errors.Add("Qdrant HnswEfConstruct deve ser maior que 0");
var validDistances = new[] { "Cosine", "Euclid", "Dot", "Manhattan" };
if (!validDistances.Contains(Distance))
errors.Add($"Qdrant Distance deve ser um de: {string.Join(", ", validDistances)}");
return errors;
}
}
public class ChromaSettings
{
public string Host { get; set; } = "localhost";
public int Port { get; set; } = 8000;
public string CollectionName { get; set; } = "rag_documents";
public string ApiVersion { get; set; } = "v1";
public List<string> GetValidationErrors()
{
var errors = new List<string>();
return errors;
}
}
public class EmbeddingSettings
{
/// <summary>
/// Provider de embedding (OpenAI, Ollama, Azure, etc.)
/// </summary>
public string Provider { get; set; } = "OpenAI";
/// <summary>
/// Modelo de embedding
/// </summary>
public string Model { get; set; } = "text-embedding-ada-002";
/// <summary>
/// Tamanho esperado do embedding
/// </summary>
public int ExpectedSize { get; set; } = 1536;
/// <summary>
/// Batch size para processamento em lote
/// </summary>
public int BatchSize { get; set; } = 100;
/// <summary>
/// Cache de embeddings em memória
/// </summary>
public bool EnableCache { get; set; } = true;
/// <summary>
/// TTL do cache em minutos
/// </summary>
public int CacheTtlMinutes { get; set; } = 60;
public List<string> GetValidationErrors()
{
var errors = new List<string>();
if (ExpectedSize <= 0)
errors.Add("Embedding ExpectedSize deve ser maior que 0");
if (BatchSize <= 0)
errors.Add("Embedding BatchSize deve ser maior que 0");
return errors;
}
}
}

View File

@ -7,7 +7,7 @@
} }
}, },
"VectorDatabase": { "VectorDatabase": {
"Provider": "Qdrant", "Provider": "Qdrant",
"MongoDB": { "MongoDB": {
"ConnectionString": "mongodb://admin:c4rn31r0@k3sw2:27017,k3ss1:27017/?authSource=admin", "ConnectionString": "mongodb://admin:c4rn31r0@k3sw2:27017,k3ss1:27017/?authSource=admin",
"DatabaseName": "RAGProjects-dev-pt", "DatabaseName": "RAGProjects-dev-pt",
@ -19,15 +19,22 @@
"Host": "localhost", "Host": "localhost",
"Port": 6334, "Port": 6334,
"CollectionName": "texts", "CollectionName": "texts",
"GroupsCollectionName": "projects",
"VectorSize": 384, "VectorSize": 384,
"Distance": "Cosine", "Distance": "Cosine",
"HnswM": 16, "HnswM": 16,
"HnswEfConstruct": 200, "HnswEfConstruct": 200,
"OnDisk": false "OnDisk": false
},
"Chroma": {
"Host": "localhost",
"Port": 8000,
"CollectionName": "rag_documents"
} }
}, },
"Features": { "Features": {
"UseQdrant": true "UseQdrant": true,
"UseHierarchicalRAG": true
}, },
"AllowedHosts": "*", "AllowedHosts": "*",
"AppTenantId": "20190830-5fd4-4a72-b8fd-1c1cb35b25bc", "AppTenantId": "20190830-5fd4-4a72-b8fd-1c1cb35b25bc",