diff --git a/Controllers/ChatController.cs b/Controllers/ChatController.cs index c6aa742..ebe38ab 100644 --- a/Controllers/ChatController.cs +++ b/Controllers/ChatController.cs @@ -21,7 +21,7 @@ namespace ChatApi.Controllers private readonly IResponseService _responseService; private readonly TextFilter _textFilter; private readonly UserDataRepository _userDataRepository; - private readonly ProjectDataRepository _projectDataRepository; + private readonly IProjectDataRepository _projectDataRepository; private readonly ITextDataService _textDataService; private readonly IHttpClientFactory _httpClientFactory; @@ -30,7 +30,7 @@ namespace ChatApi.Controllers IResponseService responseService, UserDataRepository userDataRepository, ITextDataService textDataService, - ProjectDataRepository projectDataRepository, + IProjectDataRepository projectDataRepository, IHttpClientFactory httpClientFactory) { _logger = logger; diff --git a/Controllers/RAGStrategyController.cs b/Controllers/RAGStrategyController.cs new file mode 100644 index 0000000..de0b99d --- /dev/null +++ b/Controllers/RAGStrategyController.cs @@ -0,0 +1,225 @@ +using Microsoft.AspNetCore.Mvc; +using ChatRAG.Services.Contracts; +using ChatRAG.Services.ResponseService; +using ChatRAG.Contracts.VectorSearch; + +namespace ChatApi.Controllers +{ + [ApiController] + [Route("api/[controller]")] + public class RAGStrategyController : ControllerBase + { + private readonly IVectorDatabaseFactory _factory; + private readonly IServiceProvider _serviceProvider; + private readonly ILogger _logger; + + public RAGStrategyController( + IVectorDatabaseFactory factory, + IServiceProvider serviceProvider, + ILogger logger) + { + _factory = factory; + _serviceProvider = serviceProvider; + _logger = logger; + } + + /// + /// Lista as estratégias de RAG disponíveis + /// + [HttpGet("strategies")] + public IActionResult GetAvailableStrategies() + { + var strategies = new[] + { + new { + name = "Standard", + description = "RAG padrão com classificação automática de estratégia", + service = "ResponseRAGService", + features = new[] { "Busca por similaridade", "Filtros dinâmicos", "Classificação automática" } + }, + new { + name = "Hierarchical", + description = "RAG hierárquico com múltiplas etapas de busca", + service = "HierarchicalRAGService", + features = new[] { "Análise de query", "Busca em múltiplas etapas", "Expansão de contexto", "Identificação de lacunas" } + } + }; + + return Ok(new + { + currentProvider = _factory.GetActiveProvider(), + availableStrategies = strategies + }); + } + + /// + /// Testa uma estratégia específica com uma pergunta + /// + [HttpPost("test/{strategy}")] + public async Task TestStrategy( + [FromRoute] string strategy, + [FromBody] StrategyTestRequest request) + { + try + { + IResponseService responseService = strategy.ToLower() switch + { + "standard" => _serviceProvider.GetRequiredService(), + "hierarchical" => _serviceProvider.GetRequiredService(), + _ => throw new ArgumentException($"Estratégia não suportada: {strategy}") + }; + + var stopwatch = System.Diagnostics.Stopwatch.StartNew(); + + // Usar dados mock se não fornecidos + var userData = request.UserData ?? new ChatApi.Models.UserData + { + Id = "test-user", + Name = "Test User" + }; + + var response = await responseService.GetResponse( + userData, + request.ProjectId, + request.SessionId ?? Guid.NewGuid().ToString(), + request.Question, + request.Language ?? "pt" + ); + + stopwatch.Stop(); + + return Ok(new + { + strategy, + response, + executionTime = stopwatch.ElapsedMilliseconds, + provider = _factory.GetActiveProvider() + }); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao testar estratégia {Strategy}", strategy); + return StatusCode(500, new { error = ex.Message }); + } + } + + /// + /// Compara resultados entre diferentes estratégias + /// + [HttpPost("compare")] + public async Task CompareStrategies([FromBody] StrategyTestRequest request) + { + try + { + var userData = request.UserData ?? new ChatApi.Models.UserData + { + Id = "test-user", + Name = "Test User" + }; + + var sessionId = Guid.NewGuid().ToString(); + var results = new Dictionary(); + + // Testar estratégia padrão + try + { + var standardService = _serviceProvider.GetRequiredService(); + var stopwatch1 = System.Diagnostics.Stopwatch.StartNew(); + var standardResponse = await standardService.GetResponse( + userData, request.ProjectId, sessionId + "-standard", + request.Question, request.Language ?? "pt"); + stopwatch1.Stop(); + + results["standard"] = new + { + response = standardResponse, + executionTime = stopwatch1.ElapsedMilliseconds, + success = true + }; + } + catch (Exception ex) + { + results["standard"] = new { success = false, error = ex.Message }; + } + + // Testar estratégia hierárquica + try + { + var hierarchicalService = _serviceProvider.GetRequiredService(); + var stopwatch2 = System.Diagnostics.Stopwatch.StartNew(); + var hierarchicalResponse = await hierarchicalService.GetResponse( + userData, request.ProjectId, sessionId + "-hierarchical", + request.Question, request.Language ?? "pt"); + stopwatch2.Stop(); + + results["hierarchical"] = new + { + response = hierarchicalResponse, + executionTime = stopwatch2.ElapsedMilliseconds, + success = true + }; + } + catch (Exception ex) + { + results["hierarchical"] = new { success = false, error = ex.Message }; + } + + return Ok(new + { + question = request.Question, + projectId = request.ProjectId, + provider = _factory.GetActiveProvider(), + results + }); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao comparar estratégias"); + return StatusCode(500, new { error = ex.Message }); + } + } + + /// + /// Obtém métricas de performance das estratégias + /// + [HttpGet("metrics")] + public IActionResult GetMetrics() + { + // TODO: Implementar coleta de métricas real + var metrics = new + { + standard = new + { + avgResponseTime = "1.2s", + successRate = "98%", + avgContextSize = "3 documentos", + usage = "Alto" + }, + hierarchical = new + { + avgResponseTime = "2.1s", + successRate = "95%", + avgContextSize = "5-8 documentos", + usage = "Médio" + }, + recommendations = new[] + { + "Use Standard para perguntas simples e rápidas", + "Use Hierarchical para análises complexas que precisam de contexto profundo", + "Hierarchical é melhor para perguntas técnicas detalhadas" + } + }; + + return Ok(metrics); + } + } + + public class StrategyTestRequest + { + public string ProjectId { get; set; } = ""; + public string Question { get; set; } = ""; + public string? Language { get; set; } + public string? SessionId { get; set; } + public ChatApi.Models.UserData? UserData { get; set; } + } +} \ No newline at end of file diff --git a/Controllers/sgnjnzt5.baf~ b/Controllers/sgnjnzt5.baf~ new file mode 100644 index 0000000..de0b99d --- /dev/null +++ b/Controllers/sgnjnzt5.baf~ @@ -0,0 +1,225 @@ +using Microsoft.AspNetCore.Mvc; +using ChatRAG.Services.Contracts; +using ChatRAG.Services.ResponseService; +using ChatRAG.Contracts.VectorSearch; + +namespace ChatApi.Controllers +{ + [ApiController] + [Route("api/[controller]")] + public class RAGStrategyController : ControllerBase + { + private readonly IVectorDatabaseFactory _factory; + private readonly IServiceProvider _serviceProvider; + private readonly ILogger _logger; + + public RAGStrategyController( + IVectorDatabaseFactory factory, + IServiceProvider serviceProvider, + ILogger logger) + { + _factory = factory; + _serviceProvider = serviceProvider; + _logger = logger; + } + + /// + /// Lista as estratégias de RAG disponíveis + /// + [HttpGet("strategies")] + public IActionResult GetAvailableStrategies() + { + var strategies = new[] + { + new { + name = "Standard", + description = "RAG padrão com classificação automática de estratégia", + service = "ResponseRAGService", + features = new[] { "Busca por similaridade", "Filtros dinâmicos", "Classificação automática" } + }, + new { + name = "Hierarchical", + description = "RAG hierárquico com múltiplas etapas de busca", + service = "HierarchicalRAGService", + features = new[] { "Análise de query", "Busca em múltiplas etapas", "Expansão de contexto", "Identificação de lacunas" } + } + }; + + return Ok(new + { + currentProvider = _factory.GetActiveProvider(), + availableStrategies = strategies + }); + } + + /// + /// Testa uma estratégia específica com uma pergunta + /// + [HttpPost("test/{strategy}")] + public async Task TestStrategy( + [FromRoute] string strategy, + [FromBody] StrategyTestRequest request) + { + try + { + IResponseService responseService = strategy.ToLower() switch + { + "standard" => _serviceProvider.GetRequiredService(), + "hierarchical" => _serviceProvider.GetRequiredService(), + _ => throw new ArgumentException($"Estratégia não suportada: {strategy}") + }; + + var stopwatch = System.Diagnostics.Stopwatch.StartNew(); + + // Usar dados mock se não fornecidos + var userData = request.UserData ?? new ChatApi.Models.UserData + { + Id = "test-user", + Name = "Test User" + }; + + var response = await responseService.GetResponse( + userData, + request.ProjectId, + request.SessionId ?? Guid.NewGuid().ToString(), + request.Question, + request.Language ?? "pt" + ); + + stopwatch.Stop(); + + return Ok(new + { + strategy, + response, + executionTime = stopwatch.ElapsedMilliseconds, + provider = _factory.GetActiveProvider() + }); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao testar estratégia {Strategy}", strategy); + return StatusCode(500, new { error = ex.Message }); + } + } + + /// + /// Compara resultados entre diferentes estratégias + /// + [HttpPost("compare")] + public async Task CompareStrategies([FromBody] StrategyTestRequest request) + { + try + { + var userData = request.UserData ?? new ChatApi.Models.UserData + { + Id = "test-user", + Name = "Test User" + }; + + var sessionId = Guid.NewGuid().ToString(); + var results = new Dictionary(); + + // Testar estratégia padrão + try + { + var standardService = _serviceProvider.GetRequiredService(); + var stopwatch1 = System.Diagnostics.Stopwatch.StartNew(); + var standardResponse = await standardService.GetResponse( + userData, request.ProjectId, sessionId + "-standard", + request.Question, request.Language ?? "pt"); + stopwatch1.Stop(); + + results["standard"] = new + { + response = standardResponse, + executionTime = stopwatch1.ElapsedMilliseconds, + success = true + }; + } + catch (Exception ex) + { + results["standard"] = new { success = false, error = ex.Message }; + } + + // Testar estratégia hierárquica + try + { + var hierarchicalService = _serviceProvider.GetRequiredService(); + var stopwatch2 = System.Diagnostics.Stopwatch.StartNew(); + var hierarchicalResponse = await hierarchicalService.GetResponse( + userData, request.ProjectId, sessionId + "-hierarchical", + request.Question, request.Language ?? "pt"); + stopwatch2.Stop(); + + results["hierarchical"] = new + { + response = hierarchicalResponse, + executionTime = stopwatch2.ElapsedMilliseconds, + success = true + }; + } + catch (Exception ex) + { + results["hierarchical"] = new { success = false, error = ex.Message }; + } + + return Ok(new + { + question = request.Question, + projectId = request.ProjectId, + provider = _factory.GetActiveProvider(), + results + }); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao comparar estratégias"); + return StatusCode(500, new { error = ex.Message }); + } + } + + /// + /// Obtém métricas de performance das estratégias + /// + [HttpGet("metrics")] + public IActionResult GetMetrics() + { + // TODO: Implementar coleta de métricas real + var metrics = new + { + standard = new + { + avgResponseTime = "1.2s", + successRate = "98%", + avgContextSize = "3 documentos", + usage = "Alto" + }, + hierarchical = new + { + avgResponseTime = "2.1s", + successRate = "95%", + avgContextSize = "5-8 documentos", + usage = "Médio" + }, + recommendations = new[] + { + "Use Standard para perguntas simples e rápidas", + "Use Hierarchical para análises complexas que precisam de contexto profundo", + "Hierarchical é melhor para perguntas técnicas detalhadas" + } + }; + + return Ok(metrics); + } + } + + public class StrategyTestRequest + { + public string ProjectId { get; set; } = ""; + public string Question { get; set; } = ""; + public string? Language { get; set; } + public string? SessionId { get; set; } + public ChatApi.Models.UserData? UserData { get; set; } + } +} \ No newline at end of file diff --git a/Data/30u0ddp1.org~ b/Data/30u0ddp1.org~ new file mode 100644 index 0000000..53750ec --- /dev/null +++ b/Data/30u0ddp1.org~ @@ -0,0 +1,255 @@ +using ChatRAG.Models; +using ChatRAG.Services.Contracts; +using ChatRAG.Settings.ChatRAG.Configuration; +using Microsoft.Extensions.Options; +using System.Text; +using System.Text.Json; +using Qdrant.Client; +using Qdrant.Client.Grpc; + +namespace ChatRAG.Data +{ + public class QdrantProjectDataRepository : IProjectDataRepository + { + private readonly HttpClient _httpClient; + private readonly string _collectionName; + private readonly ILogger _logger; + private readonly QdrantClient _qdrantClient; + + public QdrantProjectDataRepository( + IOptions settings, + HttpClient httpClient, + ILogger logger) + { + var qdrantSettings = settings.Value.Qdrant ?? throw new ArgumentNullException("Qdrant settings not configured"); + _httpClient = httpClient; + _httpClient.BaseAddress = new Uri($"http://{qdrantSettings.Host}:{qdrantSettings.Port}"); + _collectionName = qdrantSettings.GroupsCollectionName; + _logger = logger; + + // Inicializa o QdrantClient - use GRPC (porta 6334) para melhor performance + _qdrantClient = new QdrantClient(qdrantSettings.Host, port: 6334, https: false); + + InitializeAsync().GetAwaiter().GetResult(); + } + + private async Task InitializeAsync() + { + try + { + var exists = await _qdrantClient.CollectionExistsAsync(_collectionName); + if (!exists) + { + await CreateProjectsCollection(); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao inicializar collection de projetos no Qdrant"); + } + } + + public async Task> GetAsync() + { + try + { + var scrollRequest = new ScrollPoints + { + CollectionName = _collectionName, + Filter = new Filter(), // Filtro vazio + Limit = 1000, + WithPayload = true, + WithVectors = false + }; + + var result = await _qdrantClient.ScrollAsync(_collectionName, scrollRequest); + + return result.Select(ConvertToProject) + .Where(p => p != null) + .ToList()!; + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao recuperar projetos do Qdrant"); + return new List(); + } + } + + public async Task GetAsync(string id) + { + try + { + var points = await _qdrantClient.RetrieveAsync( + _collectionName, + new[] { PointId.Parser.Parse(id) }, + withPayload: true, + withVectors: false + ); + + var point = points.FirstOrDefault(); + return point != null ? ConvertToProject(point) : null; + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao buscar projeto {Id} no Qdrant", id); + return null; + } + } + + public async Task CreateAsync(Project newProject) + { + try + { + var id = string.IsNullOrEmpty(newProject.Id) ? Guid.NewGuid().ToString() : newProject.Id; + newProject.Id = id; + + var point = new PointStruct + { + Id = PointId.Parser.Parse(id), + Vectors = new float[384], // Vector dummy para projetos + Payload = + { + ["id"] = newProject.Id, + ["nome"] = newProject.Nome, + ["descricao"] = newProject.Descricao, + ["created_at"] = DateTime.UtcNow.ToString("O"), + ["entity_type"] = "project" + } + }; + + await _qdrantClient.UpsertAsync(_collectionName, new[] { point }); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao criar projeto no Qdrant"); + throw; + } + } + + public async Task UpdateAsync(string id, Project updatedProject) + { + try + { + updatedProject.Id = id; + + var point = new PointStruct + { + Id = PointId.Parser.Parse(id), + Vectors = new float[384], // Vector dummy + Payload = + { + ["id"] = updatedProject.Id, + ["nome"] = updatedProject.Nome, + ["descricao"] = updatedProject.Descricao, + ["updated_at"] = DateTime.UtcNow.ToString("O"), + ["entity_type"] = "project" + } + }; + + await _qdrantClient.UpsertAsync(_collectionName, new[] { point }); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao atualizar projeto {Id} no Qdrant", id); + throw; + } + } + + public async Task SaveAsync(Project project) + { + try + { + if (string.IsNullOrEmpty(project.Id)) + { + await CreateAsync(project); + } + else + { + var existing = await GetAsync(project.Id); + if (existing == null) + { + await CreateAsync(project); + } + else + { + await UpdateAsync(project.Id, project); + } + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao salvar projeto no Qdrant"); + throw; + } + } + + public async Task RemoveAsync(string id) + { + try + { + await _qdrantClient.DeleteAsync( + _collectionName, + new[] { PointId.Parser.Parse(id) } + ); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao remover projeto {Id} do Qdrant", id); + throw; + } + } + + private async Task CreateProjectsCollection() + { + var vectorParams = new VectorParams + { + Size = 384, + Distance = Distance.Cosine + }; + + await _qdrantClient.CreateCollectionAsync(_collectionName, vectorParams); + + _logger.LogInformation("Collection de projetos '{CollectionName}' criada no Qdrant", _collectionName); + } + + private static Project? ConvertToProject(RetrievedPoint point) + { + try + { + if (point.Payload == null) return null; + + return new Project + { + Id = point.Payload.TryGetValue("id", out var idValue) ? idValue.StringValue : point.Id.ToString(), + Nome = point.Payload.TryGetValue("nome", out var nomeValue) ? nomeValue.StringValue : "", + Descricao = point.Payload.TryGetValue("descricao", out var descValue) ? descValue.StringValue : "" + }; + } + catch + { + return null; + } + } + } + + public class QdrantScrollResult + { + public QdrantScrollData? result { get; set; } + } + + public class QdrantScrollData + { + public QdrantPoint[]? points { get; set; } + } + + public class QdrantPointResult + { + public QdrantPoint? result { get; set; } + } + + public class QdrantPoint + { + public string? id { get; set; } + public Dictionary? payload { get; set; } + } +} \ No newline at end of file diff --git a/Data/32reubjo.e20~ b/Data/32reubjo.e20~ new file mode 100644 index 0000000..962c274 --- /dev/null +++ b/Data/32reubjo.e20~ @@ -0,0 +1,255 @@ +using ChatRAG.Models; +using ChatRAG.Services.Contracts; +using ChatRAG.Settings.ChatRAG.Configuration; +using Microsoft.Extensions.Options; +using System.Text; +using System.Text.Json; +using Qdrant.Client; +using Qdrant.Client.Grpc; + +namespace ChatRAG.Data +{ + public class QdrantProjectDataRepository : IProjectDataRepository + { + private readonly HttpClient _httpClient; + private readonly string _collectionName; + private readonly ILogger _logger; + private readonly QdrantClient _qdrantClient; + + public QdrantProjectDataRepository( + IOptions settings, + HttpClient httpClient, + ILogger logger) + { + var qdrantSettings = settings.Value.Qdrant ?? throw new ArgumentNullException("Qdrant settings not configured"); + _httpClient = httpClient; + _httpClient.BaseAddress = new Uri($"http://{qdrantSettings.Host}:{qdrantSettings.Port}"); + _collectionName = qdrantSettings.GroupsCollectionName; + _logger = logger; + + // Inicializa o QdrantClient - use GRPC (porta 6334) para melhor performance + _qdrantClient = new QdrantClient(qdrantSettings.Host, port: 6334, https: false); + + InitializeAsync().GetAwaiter().GetResult(); + } + + private async Task InitializeAsync() + { + try + { + var exists = await _qdrantClient.CollectionExistsAsync(_collectionName); + if (!exists) + { + await CreateProjectsCollection(); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao inicializar collection de projetos no Qdrant"); + } + } + + public async Task> GetAsync() + { + try + { + var scrollRequest = new ScrollPoints + { + CollectionName = _collectionName, + Filter = new Filter(), // Filtro vazio + Limit = 1000, + WithPayload = true, + WithVectors = false + }; + + var result = await _qdrantClient.ScrollAsync(_collectionName, scrollRequest); + + return result.Select(ConvertToProject) + .Where(p => p != null) + .ToList()!; + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao recuperar projetos do Qdrant"); + return new List(); + } + } + + public async Task GetAsync(string id) + { + try + { + var points = await _qdrantClient.RetrieveAsync( + _collectionName, + new[] { PointId.Parser.ParseFrom(id) }, + withPayload: true, + withVectors: false + ); + + var point = points.FirstOrDefault(); + return point != null ? ConvertToProject(point) : null; + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao buscar projeto {Id} no Qdrant", id); + return null; + } + } + + public async Task CreateAsync(Project newProject) + { + try + { + var id = string.IsNullOrEmpty(newProject.Id) ? Guid.NewGuid().ToString() : newProject.Id; + newProject.Id = id; + + var point = new PointStruct + { + Id = PointId.Parser.ParseFrom(id), + Vectors = new float[384], // Vector dummy para projetos + Payload = + { + ["id"] = newProject.Id, + ["nome"] = newProject.Nome, + ["descricao"] = newProject.Descricao, + ["created_at"] = DateTime.UtcNow.ToString("O"), + ["entity_type"] = "project" + } + }; + + await _qdrantClient.UpsertAsync(_collectionName, new[] { point }); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao criar projeto no Qdrant"); + throw; + } + } + + public async Task UpdateAsync(string id, Project updatedProject) + { + try + { + updatedProject.Id = id; + + var point = new PointStruct + { + Id = PointId.Parser.ParseFrom(id), + Vectors = new float[384], // Vector dummy + Payload = + { + ["id"] = updatedProject.Id, + ["nome"] = updatedProject.Nome, + ["descricao"] = updatedProject.Descricao, + ["updated_at"] = DateTime.UtcNow.ToString("O"), + ["entity_type"] = "project" + } + }; + + await _qdrantClient.UpsertAsync(_collectionName, new[] { point }); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao atualizar projeto {Id} no Qdrant", id); + throw; + } + } + + public async Task SaveAsync(Project project) + { + try + { + if (string.IsNullOrEmpty(project.Id)) + { + await CreateAsync(project); + } + else + { + var existing = await GetAsync(project.Id); + if (existing == null) + { + await CreateAsync(project); + } + else + { + await UpdateAsync(project.Id, project); + } + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao salvar projeto no Qdrant"); + throw; + } + } + + public async Task RemoveAsync(string id) + { + try + { + await _qdrantClient.DeleteAsync( + _collectionName, + new[] { PointId.Parser.ParseFrom(id) } + ); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao remover projeto {Id} do Qdrant", id); + throw; + } + } + + private async Task CreateProjectsCollection() + { + var vectorParams = new VectorParams + { + Size = 384, + Distance = Distance.Cosine + }; + + await _qdrantClient.CreateCollectionAsync(_collectionName, vectorParams); + + _logger.LogInformation("Collection de projetos '{CollectionName}' criada no Qdrant", _collectionName); + } + + private static Project? ConvertToProject(RetrievedPoint point) + { + try + { + if (point.Payload == null) return null; + + return new Project + { + Id = point.Payload.TryGetValue("id", out var idValue) ? idValue.StringValue : point.Id.ToString(), + Nome = point.Payload.TryGetValue("nome", out var nomeValue) ? nomeValue.StringValue : "", + Descricao = point.Payload.TryGetValue("descricao", out var descValue) ? descValue.StringValue : "" + }; + } + catch + { + return null; + } + } + } + + public class QdrantScrollResult + { + public QdrantScrollData? result { get; set; } + } + + public class QdrantScrollData + { + public QdrantPoint[]? points { get; set; } + } + + public class QdrantPointResult + { + public QdrantPoint? result { get; set; } + } + + public class QdrantPoint + { + public string? id { get; set; } + public Dictionary? payload { get; set; } + } +} \ No newline at end of file diff --git a/Data/3qrw3lwa.v4s~ b/Data/3qrw3lwa.v4s~ new file mode 100644 index 0000000..b624596 --- /dev/null +++ b/Data/3qrw3lwa.v4s~ @@ -0,0 +1,255 @@ +using ChatRAG.Models; +using ChatRAG.Services.Contracts; +using ChatRAG.Settings.ChatRAG.Configuration; +using Microsoft.Extensions.Options; +using System.Text; +using System.Text.Json; +using Qdrant.Client; +using Qdrant.Client.Grpc; + +namespace ChatRAG.Data +{ + public class QdrantProjectDataRepository : IProjectDataRepository + { + private readonly HttpClient _httpClient; + private readonly string _collectionName; + private readonly ILogger _logger; + private readonly QdrantClient _qdrantClient; + + public QdrantProjectDataRepository( + IOptions settings, + HttpClient httpClient, + ILogger logger) + { + var qdrantSettings = settings.Value.Qdrant ?? throw new ArgumentNullException("Qdrant settings not configured"); + _httpClient = httpClient; + _httpClient.BaseAddress = new Uri($"http://{qdrantSettings.Host}:{qdrantSettings.Port}"); + _collectionName = qdrantSettings.GroupsCollectionName; + _logger = logger; + + // Inicializa o QdrantClient - use GRPC (porta 6334) para melhor performance + _qdrantClient = new QdrantClient(qdrantSettings.Host, port: 6334, https: false); + + InitializeAsync().GetAwaiter().GetResult(); + } + + private async Task InitializeAsync() + { + try + { + var exists = await _qdrantClient.CollectionExistsAsync(_collectionName); + if (!exists) + { + await CreateProjectsCollection(); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao inicializar collection de projetos no Qdrant"); + } + } + + public async Task> GetAsync() + { + try + { + var scrollRequest = new ScrollPoints + { + CollectionName = _collectionName, + Filter = new Filter(), // Filtro vazio + Limit = 1000, + WithPayload = true, + WithVectors = false + }; + + var result = await _qdrantClient.ScrollAsync(scrollRequest); + + return result.Select(ConvertToProject) + .Where(p => p != null) + .ToList()!; + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao recuperar projetos do Qdrant"); + return new List(); + } + } + + public async Task GetAsync(string id) + { + try + { + var points = await _qdrantClient.RetrieveAsync( + _collectionName, + new[] { PointId.NewGuid(Guid.Parse(id)) }, + withPayload: true, + withVectors: false + ); + + var point = points.FirstOrDefault(); + return point != null ? ConvertToProject(point) : null; + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao buscar projeto {Id} no Qdrant", id); + return null; + } + } + + public async Task CreateAsync(Project newProject) + { + try + { + var id = string.IsNullOrEmpty(newProject.Id) ? Guid.NewGuid().ToString() : newProject.Id; + newProject.Id = id; + + var point = new PointStruct + { + Id = PointId.NewGuid(Guid.Parse(id)), + Vectors = new float[384], // Vector dummy para projetos + Payload = + { + ["id"] = newProject.Id, + ["nome"] = newProject.Nome, + ["descricao"] = newProject.Descricao, + ["created_at"] = DateTime.UtcNow.ToString("O"), + ["entity_type"] = "project" + } + }; + + await _qdrantClient.UpsertAsync(_collectionName, new[] { point }); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao criar projeto no Qdrant"); + throw; + } + } + + public async Task UpdateAsync(string id, Project updatedProject) + { + try + { + updatedProject.Id = id; + + var point = new PointStruct + { + Id = PointId.NewGuid(Guid.Parse(id)), + Vectors = new float[384], // Vector dummy + Payload = + { + ["id"] = updatedProject.Id, + ["nome"] = updatedProject.Nome, + ["descricao"] = updatedProject.Descricao, + ["updated_at"] = DateTime.UtcNow.ToString("O"), + ["entity_type"] = "project" + } + }; + + await _qdrantClient.UpsertAsync(_collectionName, new[] { point }); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao atualizar projeto {Id} no Qdrant", id); + throw; + } + } + + public async Task SaveAsync(Project project) + { + try + { + if (string.IsNullOrEmpty(project.Id)) + { + await CreateAsync(project); + } + else + { + var existing = await GetAsync(project.Id); + if (existing == null) + { + await CreateAsync(project); + } + else + { + await UpdateAsync(project.Id, project); + } + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao salvar projeto no Qdrant"); + throw; + } + } + + public async Task RemoveAsync(string id) + { + try + { + await _qdrantClient.DeleteAsync( + _collectionName, + new[] { PointId.NewGuid(Guid.Parse(id)) } + ); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao remover projeto {Id} do Qdrant", id); + throw; + } + } + + private async Task CreateProjectsCollection() + { + var vectorParams = new VectorParams + { + Size = 384, + Distance = Distance.Cosine + }; + + await _qdrantClient.CreateCollectionAsync(_collectionName, vectorParams); + + _logger.LogInformation("Collection de projetos '{CollectionName}' criada no Qdrant", _collectionName); + } + + private static Project? ConvertToProject(RetrievedPoint point) + { + try + { + if (point.Payload == null) return null; + + return new Project + { + Id = point.Payload.TryGetValue("id", out var idValue) ? idValue.StringValue : point.Id.ToString(), + Nome = point.Payload.TryGetValue("nome", out var nomeValue) ? nomeValue.StringValue : "", + Descricao = point.Payload.TryGetValue("descricao", out var descValue) ? descValue.StringValue : "" + }; + } + catch + { + return null; + } + } + } + + public class QdrantScrollResult + { + public QdrantScrollData? result { get; set; } + } + + public class QdrantScrollData + { + public QdrantPoint[]? points { get; set; } + } + + public class QdrantPointResult + { + public QdrantPoint? result { get; set; } + } + + public class QdrantPoint + { + public string? id { get; set; } + public Dictionary? payload { get; set; } + } +} \ No newline at end of file diff --git a/Data/ChromaProjectDataRepository.cs b/Data/ChromaProjectDataRepository.cs new file mode 100644 index 0000000..a3f43bd --- /dev/null +++ b/Data/ChromaProjectDataRepository.cs @@ -0,0 +1,331 @@ +using ChatRAG.Models; +using ChatRAG.Services.Contracts; +using ChatRAG.Services.SearchVectors; +using ChatRAG.Settings.ChatRAG.Configuration; +using Microsoft.Extensions.Options; +using System.Text; +using System.Text.Json; + +namespace ChatRAG.Data +{ + + public class ChromaProjectDataRepository : IProjectDataRepository + { + private readonly HttpClient _httpClient; + private readonly string _collectionName; + private readonly ILogger _logger; + + public ChromaProjectDataRepository( + IOptions settings, + HttpClient httpClient, + ILogger logger) + { + var chromaSettings = settings.Value.Chroma ?? throw new ArgumentNullException("Chroma settings not configured"); + _httpClient = httpClient; + _httpClient.BaseAddress = new Uri($"http://{chromaSettings.Host}:{chromaSettings.Port}"); + _collectionName = "projects"; // Collection separada para projetos + _logger = logger; + + InitializeAsync().GetAwaiter().GetResult(); + } + + private async Task InitializeAsync() + { + try + { + // Verificar se a collection existe, se não, criar + var collections = await GetCollectionsAsync(); + if (!collections.Contains(_collectionName)) + { + await CreateProjectsCollection(); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao inicializar collection de projetos no Chroma"); + } + } + + public async Task> GetAsync() + { + try + { + var query = new + { + where = new { entity_type = "project" }, + include = new[] { "documents", "metadatas" } + }; + + var json = JsonSerializer.Serialize(query); + var content = new StringContent(json, Encoding.UTF8, "application/json"); + + var response = await _httpClient.PostAsync($"/api/v2/collections/{_collectionName}/get", content); + + if (!response.IsSuccessStatusCode) + { + _logger.LogError("Erro ao buscar projetos no Chroma"); + return new List(); + } + + var result = await response.Content.ReadAsStringAsync(); + var getResult = JsonSerializer.Deserialize(result); + + var projects = new List(); + if (getResult?.ids?.Length > 0) + { + for (int i = 0; i < getResult.ids.Length; i++) + { + var metadata = getResult.metadatas?[i]; + if (metadata != null) + { + projects.Add(new Project + { + Id = metadata.GetValueOrDefault("id")?.ToString() ?? getResult.ids[i], + Nome = metadata.GetValueOrDefault("nome")?.ToString() ?? "", + Descricao = metadata.GetValueOrDefault("descricao")?.ToString() ?? "" + }); + } + } + } + + return projects; + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao recuperar projetos do Chroma"); + return new List(); + } + } + + public async Task GetAsync(string id) + { + try + { + var query = new + { + ids = new[] { id }, + include = new[] { "metadatas" } + }; + + var json = JsonSerializer.Serialize(query); + var content = new StringContent(json, Encoding.UTF8, "application/json"); + + var response = await _httpClient.PostAsync($"/api/v2/collections/{_collectionName}/get", content); + + if (!response.IsSuccessStatusCode) + { + return null; + } + + var result = await response.Content.ReadAsStringAsync(); + var getResult = JsonSerializer.Deserialize(result); + + if (getResult?.ids?.Length > 0 && getResult.metadatas?[0] != null) + { + var metadata = getResult.metadatas[0]; + return new Project + { + Id = metadata.GetValueOrDefault("id")?.ToString() ?? id, + Nome = metadata.GetValueOrDefault("nome")?.ToString() ?? "", + Descricao = metadata.GetValueOrDefault("descricao")?.ToString() ?? "" + }; + } + + return null; + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao buscar projeto {Id} no Chroma", id); + return null; + } + } + + public async Task CreateAsync(Project newProject) + { + try + { + var id = string.IsNullOrEmpty(newProject.Id) ? Guid.NewGuid().ToString() : newProject.Id; + newProject.Id = id; + + var document = new + { + ids = new[] { id }, + documents = new[] { $"Projeto: {newProject.Nome}" }, + metadatas = new[] { new Dictionary + { + ["id"] = id, + ["nome"] = newProject.Nome, + ["descricao"] = newProject.Descricao, + ["created_at"] = DateTime.UtcNow.ToString("O"), + ["entity_type"] = "project" + }}, + embeddings = new[] { new double[384] } // Vector dummy + }; + + var json = JsonSerializer.Serialize(document); + var content = new StringContent(json, Encoding.UTF8, "application/json"); + + var response = await _httpClient.PostAsync($"/api/v2/collections/{_collectionName}/add", content); + + if (!response.IsSuccessStatusCode) + { + var error = await response.Content.ReadAsStringAsync(); + throw new Exception($"Erro ao criar projeto no Chroma: {error}"); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao criar projeto no Chroma"); + throw; + } + } + + public async Task UpdateAsync(string id, Project updatedProject) + { + try + { + // Chroma não tem update direto, então fazemos upsert + updatedProject.Id = id; + + var document = new + { + ids = new[] { id }, + documents = new[] { $"Projeto: {updatedProject.Nome}" }, + metadatas = new[] { new Dictionary + { + ["id"] = id, + ["nome"] = updatedProject.Nome, + ["descricao"] = updatedProject.Descricao, + ["updated_at"] = DateTime.UtcNow.ToString("O"), + ["entity_type"] = "project" + }}, + embeddings = new[] { new double[384] } + }; + + var json = JsonSerializer.Serialize(document); + var content = new StringContent(json, Encoding.UTF8, "application/json"); + + var response = await _httpClient.PostAsync($"/api/v2/collections/{_collectionName}/upsert", content); + + if (!response.IsSuccessStatusCode) + { + var error = await response.Content.ReadAsStringAsync(); + throw new Exception($"Erro ao atualizar projeto no Chroma: {error}"); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao atualizar projeto {Id} no Chroma", id); + throw; + } + } + + public async Task SaveAsync(Project project) + { + try + { + if (string.IsNullOrEmpty(project.Id)) + { + await CreateAsync(project); + } + else + { + var existing = await GetAsync(project.Id); + if (existing == null) + { + await CreateAsync(project); + } + else + { + await UpdateAsync(project.Id, project); + } + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao salvar projeto no Chroma"); + throw; + } + } + + public async Task RemoveAsync(string id) + { + try + { + var deleteRequest = new + { + ids = new[] { id } + }; + + var json = JsonSerializer.Serialize(deleteRequest); + var content = new StringContent(json, Encoding.UTF8, "application/json"); + + var response = await _httpClient.PostAsync($"/api/v2/collections/{_collectionName}/delete", content); + + if (!response.IsSuccessStatusCode) + { + var error = await response.Content.ReadAsStringAsync(); + _logger.LogWarning("Erro ao remover projeto {Id} do Chroma: {Error}", id, error); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao remover projeto {Id} do Chroma", id); + throw; + } + } + + private async Task GetCollectionsAsync() + { + try + { + var response = await _httpClient.GetAsync("/api/v2/collections"); + if (!response.IsSuccessStatusCode) + { + return Array.Empty(); + } + + var content = await response.Content.ReadAsStringAsync(); + var collections = JsonSerializer.Deserialize(content); + return collections ?? Array.Empty(); + } + catch + { + return Array.Empty(); + } + } + + private async Task CreateProjectsCollection() + { + var collection = new + { + name = _collectionName, + metadata = new + { + description = "Projects Collection", + created_at = DateTime.UtcNow.ToString("O") + } + }; + + var json = JsonSerializer.Serialize(collection); + var content = new StringContent(json, Encoding.UTF8, "application/json"); + + var response = await _httpClient.PostAsync("/api/v2/collections", content); + + if (!response.IsSuccessStatusCode) + { + var error = await response.Content.ReadAsStringAsync(); + throw new Exception($"Erro ao criar collection de projetos: {error}"); + } + + _logger.LogInformation("Collection de projetos '{CollectionName}' criada no Chroma", _collectionName); + } + } + + public class ChromaGetResult + { + public string[] ids { get; set; } = Array.Empty(); + public string[] documents { get; set; } = Array.Empty(); + public Dictionary[]? metadatas { get; set; } + } +} diff --git a/Data/ProjectDataRepository.cs b/Data/MongoProjectDataRepository.cs similarity index 92% rename from Data/ProjectDataRepository.cs rename to Data/MongoProjectDataRepository.cs index 936ec18..cc2fe18 100644 --- a/Data/ProjectDataRepository.cs +++ b/Data/MongoProjectDataRepository.cs @@ -1,16 +1,17 @@ using ChatApi; using ChatRAG.Models; +using ChatRAG.Services.Contracts; using ChatRAG.Settings.ChatRAG.Configuration; using Microsoft.Extensions.Options; using MongoDB.Driver; namespace ChatRAG.Data { - public class ProjectDataRepository + public class MongoProjectDataRepository : IProjectDataRepository { private readonly IMongoCollection _textsCollection; - public ProjectDataRepository( + public MongoProjectDataRepository( IOptions databaseSettings) { var mongoClient = new MongoClient( diff --git a/Data/QdrantProjectDataRepository.cs b/Data/QdrantProjectDataRepository.cs new file mode 100644 index 0000000..814b9e8 --- /dev/null +++ b/Data/QdrantProjectDataRepository.cs @@ -0,0 +1,269 @@ +using ChatRAG.Models; +using ChatRAG.Services.Contracts; +using ChatRAG.Settings.ChatRAG.Configuration; +using Microsoft.Extensions.Options; +using System.Text; +using System.Text.Json; +using Qdrant.Client; +using Qdrant.Client.Grpc; +using Google.Protobuf; + +namespace ChatRAG.Data +{ + public class QdrantProjectDataRepository : IProjectDataRepository + { + private readonly HttpClient _httpClient; + private readonly string _collectionName; + private readonly ILogger _logger; + private readonly QdrantClient _qdrantClient; + private volatile bool _collectionInitialized = false; + private readonly SemaphoreSlim _initializationSemaphore = new(1, 1); + + public QdrantProjectDataRepository( + IOptions settings, + HttpClient httpClient, + ILogger logger) + { + var qdrantSettings = settings.Value.Qdrant ?? throw new ArgumentNullException("Qdrant settings not configured"); + _httpClient = httpClient; + _httpClient.BaseAddress = new Uri($"http://{qdrantSettings.Host}:{qdrantSettings.Port}"); + _collectionName = qdrantSettings.GroupsCollectionName; + _logger = logger; + + // Inicializa o QdrantClient - use GRPC (porta 6334) para melhor performance + _qdrantClient = new QdrantClient(qdrantSettings.Host, port: 6334, https: false); + + InitializeAsync().GetAwaiter().GetResult(); + } + + private async Task InitializeAsync() + { + try + { + if (_collectionInitialized) return; + + await _initializationSemaphore.WaitAsync(); + + var exists = await _qdrantClient.CollectionExistsAsync(_collectionName); + if (!exists) + { + await CreateProjectsCollection(); + } + _collectionInitialized = true; + + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao inicializar collection de projetos no Qdrant"); + } + finally + { + _initializationSemaphore.Release(); + } + } + + public async Task> GetAsync() + { + try + { + //var scrollRequest = new ScrollPoints + //{ + // CollectionName = _collectionName, + // Filter = new Filter(), // Filtro vazio + // Limit = 1000, + // WithPayload = true, + // WithVectors = false + //}; + + //var result = await _qdrantClient.ScrollAsync(_collectionName, scrollRequest); + var result = await _qdrantClient.ScrollAsync(_collectionName, new Filter(), 1000, null, true, false); + + return result.Result.Select(ConvertToProject) + .Where(p => p != null) + .ToList()!; + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao recuperar projetos do Qdrant"); + return new List(); + } + } + + public async Task GetAsync(string id) + { + try + { + var points = await _qdrantClient.RetrieveAsync( + _collectionName, + new[] { new PointId { Uuid = id } }, + withPayload: true, + withVectors: false + ); + + var point = points.FirstOrDefault(); + return point != null ? ConvertToProject(point) : null; + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao buscar projeto {Id} no Qdrant", id); + return null; + } + } + + public async Task CreateAsync(Project newProject) + { + try + { + var id = string.IsNullOrEmpty(newProject.Id) ? Guid.NewGuid().ToString() : newProject.Id; + newProject.Id = id; + + var point = new PointStruct + { + Id = new PointId { Uuid= id }, + Vectors = new float[384], // Vector dummy para projetos + Payload = + { + ["id"] = newProject.Id, + ["nome"] = newProject.Nome, + ["descricao"] = newProject.Descricao, + ["created_at"] = DateTime.UtcNow.ToString("O"), + ["entity_type"] = "project" + } + }; + + await _qdrantClient.UpsertAsync(_collectionName, new[] { point }); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao criar projeto no Qdrant"); + throw; + } + } + + public async Task UpdateAsync(string id, Project updatedProject) + { + try + { + updatedProject.Id = id; + + var point = new PointStruct + { + Id = new PointId { Uuid = id }, + Vectors = new float[384], // Vector dummy + Payload = + { + ["id"] = updatedProject.Id, + ["nome"] = updatedProject.Nome, + ["descricao"] = updatedProject.Descricao, + ["updated_at"] = DateTime.UtcNow.ToString("O"), + ["entity_type"] = "project" + } + }; + + await _qdrantClient.UpsertAsync(_collectionName, new[] { point }); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao atualizar projeto {Id} no Qdrant", id); + throw; + } + } + + public async Task SaveAsync(Project project) + { + try + { + if (string.IsNullOrEmpty(project.Id)) + { + await CreateAsync(project); + } + else + { + var existing = await GetAsync(project.Id); + if (existing == null) + { + await CreateAsync(project); + } + else + { + await UpdateAsync(project.Id, project); + } + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao salvar projeto no Qdrant"); + throw; + } + } + + public async Task RemoveAsync(string id) + { + try + { + await _qdrantClient.DeleteAsync( + _collectionName, + new[] { new PointId { Uuid = id }.Num } + ); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao remover projeto {Id} do Qdrant", id); + throw; + } + } + + private async Task CreateProjectsCollection() + { + var vectorParams = new VectorParams + { + Size = 384, + Distance = Distance.Cosine + }; + + await _qdrantClient.CreateCollectionAsync(_collectionName, vectorParams); + + _logger.LogInformation("Collection de projetos '{CollectionName}' criada no Qdrant", _collectionName); + } + + private static Project? ConvertToProject(RetrievedPoint point) + { + try + { + if (point.Payload == null) return null; + + return new Project + { + Id = point.Payload.TryGetValue("id", out var idValue) ? idValue.StringValue : point.Id.ToString(), + Nome = point.Payload.TryGetValue("nome", out var nomeValue) ? nomeValue.StringValue : "", + Descricao = point.Payload.TryGetValue("descricao", out var descValue) ? descValue.StringValue : "" + }; + } + catch + { + return null; + } + } + } + + public class QdrantScrollResult + { + public QdrantScrollData? result { get; set; } + } + + public class QdrantScrollData + { + public QdrantPoint[]? points { get; set; } + } + + public class QdrantPointResult + { + public QdrantPoint? result { get; set; } + } + + public class QdrantPoint + { + public string? id { get; set; } + public Dictionary? payload { get; set; } + } +} \ No newline at end of file diff --git a/Data/hnwhoaao.xfh~ b/Data/hnwhoaao.xfh~ new file mode 100644 index 0000000..b1d44d7 --- /dev/null +++ b/Data/hnwhoaao.xfh~ @@ -0,0 +1,263 @@ +using ChatRAG.Models; +using ChatRAG.Services.Contracts; +using ChatRAG.Settings.ChatRAG.Configuration; +using Microsoft.Extensions.Options; +using System.Text; +using System.Text.Json; +using Qdrant.Client; +using Qdrant.Client.Grpc; + +namespace ChatRAG.Data +{ + public class QdrantProjectDataRepository : IProjectDataRepository + { + private readonly HttpClient _httpClient; + private readonly string _collectionName; + private readonly ILogger _logger; + private readonly QdrantClient _qdrantClient; + private volatile bool _collectionInitialized = false; + private readonly SemaphoreSlim _initializationSemaphore = new(1, 1); + + public QdrantProjectDataRepository( + IOptions settings, + HttpClient httpClient, + ILogger logger) + { + var qdrantSettings = settings.Value.Qdrant ?? throw new ArgumentNullException("Qdrant settings not configured"); + _httpClient = httpClient; + _httpClient.BaseAddress = new Uri($"http://{qdrantSettings.Host}:{qdrantSettings.Port}"); + _collectionName = qdrantSettings.GroupsCollectionName; + _logger = logger; + + // Inicializa o QdrantClient - use GRPC (porta 6334) para melhor performance + _qdrantClient = new QdrantClient(qdrantSettings.Host, port: 6334, https: false); + + InitializeAsync().GetAwaiter().GetResult(); + } + + private async Task EnsureInitializedAsync() + { + try + { + if (_collectionInitialized) return; + + await _initializationSemaphore.WaitAsync(); + + var exists = await _qdrantClient.CollectionExistsAsync(_collectionName); + if (!exists) + { + await CreateProjectsCollection(); + } + _collectionInitialized = true; + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao inicializar collection de projetos no Qdrant"); + } + } + + public async Task> GetAsync() + { + try + { + //var scrollRequest = new ScrollPoints + //{ + // CollectionName = _collectionName, + // Filter = new Filter(), // Filtro vazio + // Limit = 1000, + // WithPayload = true, + // WithVectors = false + //}; + + //var result = await _qdrantClient.ScrollAsync(_collectionName, scrollRequest); + var result = await _qdrantClient.ScrollAsync(_collectionName, new Filter(), 1000, null, true, false); + + return result.Result.Select(ConvertToProject) + .Where(p => p != null) + .ToList()!; + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao recuperar projetos do Qdrant"); + return new List(); + } + } + + public async Task GetAsync(string id) + { + try + { + var points = await _qdrantClient.RetrieveAsync( + _collectionName, + new[] { PointId.Parser.ParseFrom(Encoding.ASCII.GetBytes(id)) }, + withPayload: true, + withVectors: false + ); + + var point = points.FirstOrDefault(); + return point != null ? ConvertToProject(point) : null; + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao buscar projeto {Id} no Qdrant", id); + return null; + } + } + + public async Task CreateAsync(Project newProject) + { + try + { + var id = string.IsNullOrEmpty(newProject.Id) ? Guid.NewGuid().ToString() : newProject.Id; + newProject.Id = id; + + var point = new PointStruct + { + Id = PointId.Parser.ParseFrom(Encoding.ASCII.GetBytes(id)), + Vectors = new float[384], // Vector dummy para projetos + Payload = + { + ["id"] = newProject.Id, + ["nome"] = newProject.Nome, + ["descricao"] = newProject.Descricao, + ["created_at"] = DateTime.UtcNow.ToString("O"), + ["entity_type"] = "project" + } + }; + + await _qdrantClient.UpsertAsync(_collectionName, new[] { point }); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao criar projeto no Qdrant"); + throw; + } + } + + public async Task UpdateAsync(string id, Project updatedProject) + { + try + { + updatedProject.Id = id; + + var point = new PointStruct + { + Id = PointId.Parser.ParseFrom(Encoding.ASCII.GetBytes(id)), + Vectors = new float[384], // Vector dummy + Payload = + { + ["id"] = updatedProject.Id, + ["nome"] = updatedProject.Nome, + ["descricao"] = updatedProject.Descricao, + ["updated_at"] = DateTime.UtcNow.ToString("O"), + ["entity_type"] = "project" + } + }; + + await _qdrantClient.UpsertAsync(_collectionName, new[] { point }); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao atualizar projeto {Id} no Qdrant", id); + throw; + } + } + + public async Task SaveAsync(Project project) + { + try + { + if (string.IsNullOrEmpty(project.Id)) + { + await CreateAsync(project); + } + else + { + var existing = await GetAsync(project.Id); + if (existing == null) + { + await CreateAsync(project); + } + else + { + await UpdateAsync(project.Id, project); + } + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao salvar projeto no Qdrant"); + throw; + } + } + + public async Task RemoveAsync(string id) + { + try + { + await _qdrantClient.DeleteAsync( + _collectionName, + new[] { PointId.Parser.ParseFrom(Encoding.ASCII.GetBytes(id)).Num } + ); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao remover projeto {Id} do Qdrant", id); + throw; + } + } + + private async Task CreateProjectsCollection() + { + var vectorParams = new VectorParams + { + Size = 384, + Distance = Distance.Cosine + }; + + await _qdrantClient.CreateCollectionAsync(_collectionName, vectorParams); + + _logger.LogInformation("Collection de projetos '{CollectionName}' criada no Qdrant", _collectionName); + } + + private static Project? ConvertToProject(RetrievedPoint point) + { + try + { + if (point.Payload == null) return null; + + return new Project + { + Id = point.Payload.TryGetValue("id", out var idValue) ? idValue.StringValue : point.Id.ToString(), + Nome = point.Payload.TryGetValue("nome", out var nomeValue) ? nomeValue.StringValue : "", + Descricao = point.Payload.TryGetValue("descricao", out var descValue) ? descValue.StringValue : "" + }; + } + catch + { + return null; + } + } + } + + public class QdrantScrollResult + { + public QdrantScrollData? result { get; set; } + } + + public class QdrantScrollData + { + public QdrantPoint[]? points { get; set; } + } + + public class QdrantPointResult + { + public QdrantPoint? result { get; set; } + } + + public class QdrantPoint + { + public string? id { get; set; } + public Dictionary? payload { get; set; } + } +} \ No newline at end of file diff --git a/Data/wgbnjwfg.nr3~ b/Data/wgbnjwfg.nr3~ new file mode 100644 index 0000000..66bbb45 --- /dev/null +++ b/Data/wgbnjwfg.nr3~ @@ -0,0 +1,268 @@ +using ChatRAG.Models; +using ChatRAG.Services.Contracts; +using ChatRAG.Settings.ChatRAG.Configuration; +using Microsoft.Extensions.Options; +using System.Text; +using System.Text.Json; +using Qdrant.Client; +using Qdrant.Client.Grpc; + +namespace ChatRAG.Data +{ + public class QdrantProjectDataRepository : IProjectDataRepository + { + private readonly HttpClient _httpClient; + private readonly string _collectionName; + private readonly ILogger _logger; + private readonly QdrantClient _qdrantClient; + private volatile bool _collectionInitialized = false; + private readonly SemaphoreSlim _initializationSemaphore = new(1, 1); + + public QdrantProjectDataRepository( + IOptions settings, + HttpClient httpClient, + ILogger logger) + { + var qdrantSettings = settings.Value.Qdrant ?? throw new ArgumentNullException("Qdrant settings not configured"); + _httpClient = httpClient; + _httpClient.BaseAddress = new Uri($"http://{qdrantSettings.Host}:{qdrantSettings.Port}"); + _collectionName = qdrantSettings.GroupsCollectionName; + _logger = logger; + + // Inicializa o QdrantClient - use GRPC (porta 6334) para melhor performance + _qdrantClient = new QdrantClient(qdrantSettings.Host, port: 6334, https: false); + + InitializeAsync().GetAwaiter().GetResult(); + } + + private async Task InitializeAsync() + { + try + { + if (_collectionInitialized) return; + + await _initializationSemaphore.WaitAsync(); + + var exists = await _qdrantClient.CollectionExistsAsync(_collectionName); + if (!exists) + { + await CreateProjectsCollection(); + } + _collectionInitialized = true; + + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao inicializar collection de projetos no Qdrant"); + } + finally + { + _initializationSemaphore.Release(); + } + } + + public async Task> GetAsync() + { + try + { + //var scrollRequest = new ScrollPoints + //{ + // CollectionName = _collectionName, + // Filter = new Filter(), // Filtro vazio + // Limit = 1000, + // WithPayload = true, + // WithVectors = false + //}; + + //var result = await _qdrantClient.ScrollAsync(_collectionName, scrollRequest); + var result = await _qdrantClient.ScrollAsync(_collectionName, new Filter(), 1000, null, true, false); + + return result.Result.Select(ConvertToProject) + .Where(p => p != null) + .ToList()!; + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao recuperar projetos do Qdrant"); + return new List(); + } + } + + public async Task GetAsync(string id) + { + try + { + var points = await _qdrantClient.RetrieveAsync( + _collectionName, + new[] { PointId.Parser.ParseFrom(Encoding.ASCII.GetBytes(id)) }, + withPayload: true, + withVectors: false + ); + + var point = points.FirstOrDefault(); + return point != null ? ConvertToProject(point) : null; + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao buscar projeto {Id} no Qdrant", id); + return null; + } + } + + public async Task CreateAsync(Project newProject) + { + try + { + var id = string.IsNullOrEmpty(newProject.Id) ? Guid.NewGuid().ToString() : newProject.Id; + newProject.Id = id; + + var point = new PointStruct + { + Id = PointId.Parser.ParseFrom(Encoding.ASCII.GetBytes(id)), + Vectors = new float[384], // Vector dummy para projetos + Payload = + { + ["id"] = newProject.Id, + ["nome"] = newProject.Nome, + ["descricao"] = newProject.Descricao, + ["created_at"] = DateTime.UtcNow.ToString("O"), + ["entity_type"] = "project" + } + }; + + await _qdrantClient.UpsertAsync(_collectionName, new[] { point }); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao criar projeto no Qdrant"); + throw; + } + } + + public async Task UpdateAsync(string id, Project updatedProject) + { + try + { + updatedProject.Id = id; + + var point = new PointStruct + { + Id = PointId.Parser.ParseFrom(Encoding.ASCII.GetBytes(id)), + Vectors = new float[384], // Vector dummy + Payload = + { + ["id"] = updatedProject.Id, + ["nome"] = updatedProject.Nome, + ["descricao"] = updatedProject.Descricao, + ["updated_at"] = DateTime.UtcNow.ToString("O"), + ["entity_type"] = "project" + } + }; + + await _qdrantClient.UpsertAsync(_collectionName, new[] { point }); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao atualizar projeto {Id} no Qdrant", id); + throw; + } + } + + public async Task SaveAsync(Project project) + { + try + { + if (string.IsNullOrEmpty(project.Id)) + { + await CreateAsync(project); + } + else + { + var existing = await GetAsync(project.Id); + if (existing == null) + { + await CreateAsync(project); + } + else + { + await UpdateAsync(project.Id, project); + } + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao salvar projeto no Qdrant"); + throw; + } + } + + public async Task RemoveAsync(string id) + { + try + { + await _qdrantClient.DeleteAsync( + _collectionName, + new[] { PointId.Parser.ParseFrom(Encoding.ASCII.GetBytes(id)).Num } + ); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao remover projeto {Id} do Qdrant", id); + throw; + } + } + + private async Task CreateProjectsCollection() + { + var vectorParams = new VectorParams + { + Size = 384, + Distance = Distance.Cosine + }; + + await _qdrantClient.CreateCollectionAsync(_collectionName, vectorParams); + + _logger.LogInformation("Collection de projetos '{CollectionName}' criada no Qdrant", _collectionName); + } + + private static Project? ConvertToProject(RetrievedPoint point) + { + try + { + if (point.Payload == null) return null; + + return new Project + { + Id = point.Payload.TryGetValue("id", out var idValue) ? idValue.StringValue : point.Id.ToString(), + Nome = point.Payload.TryGetValue("nome", out var nomeValue) ? nomeValue.StringValue : "", + Descricao = point.Payload.TryGetValue("descricao", out var descValue) ? descValue.StringValue : "" + }; + } + catch + { + return null; + } + } + } + + public class QdrantScrollResult + { + public QdrantScrollData? result { get; set; } + } + + public class QdrantScrollData + { + public QdrantPoint[]? points { get; set; } + } + + public class QdrantPointResult + { + public QdrantPoint? result { get; set; } + } + + public class QdrantPoint + { + public string? id { get; set; } + public Dictionary? payload { get; set; } + } +} \ No newline at end of file diff --git a/Program.cs b/Program.cs index f061ca4..c2a99e1 100644 --- a/Program.cs +++ b/Program.cs @@ -10,6 +10,7 @@ using ChatRAG.Services; using ChatRAG.Services.Contracts; using ChatRAG.Services.ResponseService; using ChatRAG.Services.SearchVectors; +using ChatRAG.Services.TextServices; using ChatRAG.Settings.ChatRAG.Configuration; using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.Http.Features; @@ -78,7 +79,10 @@ builder.Services.AddSwaggerGen(c => builder.Services.Configure( builder.Configuration.GetSection("ChatRHSettings")); -builder.Services.AddScoped(); +//builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); builder.Services.AddVectorDatabase(builder.Configuration); @@ -89,12 +93,76 @@ builder.Services.AddScoped(provider => return factory.CreateVectorSearchService(); }); +builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); + +builder.Services.AddScoped(provider => +{ + var database = builder.Configuration["VectorDatabase:Provider"]; + if (string.IsNullOrEmpty(database)) + { + throw new InvalidOperationException("VectorDatabase:Provider is not configured."); + } + else if (database.Equals("Qdrant", StringComparison.OrdinalIgnoreCase)) + { + return provider.GetRequiredService(); + } + else if (database.Equals("MongoDB", StringComparison.OrdinalIgnoreCase)) + { + return provider.GetRequiredService(); + } + else if (database.Equals("Chroma", StringComparison.OrdinalIgnoreCase)) + { + return provider.GetRequiredService(); + } + return provider.GetRequiredService(); +}); + +builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); + +builder.Services.AddScoped(provider => +{ + var database = builder.Configuration["VectorDatabase:Provider"]; + if (string.IsNullOrEmpty(database)) + { + throw new InvalidOperationException("VectorDatabase:Provider is not configured."); + } + else if (database.Equals("Qdrant", StringComparison.OrdinalIgnoreCase)) + { + return provider.GetRequiredService(); + } + else if (database.Equals("MongoDB", StringComparison.OrdinalIgnoreCase)) + { + return provider.GetRequiredService(); + } + else if (database.Equals("Chroma", StringComparison.OrdinalIgnoreCase)) + { + return provider.GetRequiredService(); + } + return provider.GetRequiredService(); +}); + builder.Services.AddSingleton(); builder.Services.AddScoped(); -builder.Services.AddScoped(); builder.Services.AddSingleton(); -builder.Services.AddScoped(); +//builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); + +builder.Services.AddScoped(provider => +{ + var configuration = provider.GetService(); + var useHierarchical = configuration?.GetValue("Features:UseHierarchicalRAG") ?? false; + + return useHierarchical + ? provider.GetRequiredService() + : provider.GetRequiredService(); +}); + builder.Services.AddTransient(); builder.Services.AddTransient(); builder.Services.AddSingleton(); @@ -107,11 +175,11 @@ builder.Services.AddSingleton(); //Desktop //builder.Services.AddOllamaChatCompletion("llama3.2", new Uri("http://localhost:11434")); //Notebook -var model = "meta-llama/Llama-3.2-3B-Instruct"; -var url = "https://api.deepinfra.com/v1/openai"; // Adicione o /v1/openai -builder.Services.AddOpenAIChatCompletion(model, new Uri(url), "HedaR4yPrp9N2XSHfwdZjpZvPIxejPFK"); +//var model = "meta-llama/Llama-3.2-3B-Instruct"; +//var url = "https://api.deepinfra.com/v1/openai"; // Adicione o /v1/openai +//builder.Services.AddOpenAIChatCompletion(model, new Uri(url), "HedaR4yPrp9N2XSHfwdZjpZvPIxejPFK"); -//builder.Services.AddOllamaChatCompletion("llama3.2:3b", new Uri("http://localhost:11435")); +builder.Services.AddOllamaChatCompletion("llama3.2:3b", new Uri("http://localhost:11435")); //builder.Services.AddOllamaChatCompletion("llama3.2:1b", new Uri("http://localhost:11435")); diff --git a/Services/Contracts/IProjectDataRepository.cs b/Services/Contracts/IProjectDataRepository.cs new file mode 100644 index 0000000..3ba22fc --- /dev/null +++ b/Services/Contracts/IProjectDataRepository.cs @@ -0,0 +1,14 @@ +using ChatRAG.Models; + +namespace ChatRAG.Services.Contracts +{ + public interface IProjectDataRepository + { + Task> GetAsync(); + Task GetAsync(string id); + Task CreateAsync(Project newProject); + Task UpdateAsync(string id, Project updatedProject); + Task SaveAsync(Project project); + Task RemoveAsync(string id); + } +} diff --git a/Services/ResponseService/0vwtdxh2.jyi~ b/Services/ResponseService/0vwtdxh2.jyi~ new file mode 100644 index 0000000..4995fd4 --- /dev/null +++ b/Services/ResponseService/0vwtdxh2.jyi~ @@ -0,0 +1,375 @@ +using ChatApi; +using ChatApi.Models; +using ChatRAG.Contracts.VectorSearch; +using ChatRAG.Data; +using ChatRAG.Models; +using ChatRAG.Services.Contracts; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.ChatCompletion; +using Microsoft.SemanticKernel.Connectors.OpenAI; +using Microsoft.SemanticKernel.Embeddings; + +#pragma warning disable SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. +namespace ChatRAG.Services.ResponseService +{ + public class HierarchicalRAGService : IResponseService + { + private readonly ChatHistoryService _chatHistoryService; + private readonly Kernel _kernel; + private readonly TextFilter _textFilter; + private readonly ProjectDataRepository _projectDataRepository; + private readonly IChatCompletionService _chatCompletionService; + private readonly IVectorSearchService _vectorSearchService; + private readonly ILogger _logger; + + public HierarchicalRAGService( + ChatHistoryService chatHistoryService, + Kernel kernel, + TextFilter textFilter, + ProjectDataRepository projectDataRepository, + IChatCompletionService chatCompletionService, + IVectorSearchService vectorSearchService, + ILogger logger) + { + _chatHistoryService = chatHistoryService; + _kernel = kernel; + _textFilter = textFilter; + _projectDataRepository = projectDataRepository; + _chatCompletionService = chatCompletionService; + _vectorSearchService = vectorSearchService; + _logger = logger; + } + + public async Task GetResponse(UserData userData, string projectId, string sessionId, string question, string language = "pt") + { + var stopWatch = new System.Diagnostics.Stopwatch(); + stopWatch.Start(); + + try + { + // 1. Análise da query para determinar estratégia + var queryAnalysis = await AnalyzeQuery(question, language); + _logger.LogInformation("Query Analysis: {Strategy}, Complexity: {Complexity}", + queryAnalysis.Strategy, queryAnalysis.Complexity); + + // 2. Execução hierárquica baseada na análise + var context = await ExecuteHierarchicalSearch(question, projectId, queryAnalysis); + + // 3. Geração da resposta final + var response = await GenerateResponse(question, projectId, context, sessionId, language); + + stopWatch.Stop(); + return $"{response}\n\nTempo: {stopWatch.ElapsedMilliseconds / 1000}s\nEtapas: {context.Steps.Count}"; + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro no RAG Hierárquico"); + stopWatch.Stop(); + return $"Erro: {ex.Message}\nTempo: {stopWatch.ElapsedMilliseconds / 1000}s"; + } + } + + private async Task AnalyzeQuery(string question, string language) + { + var analysisPrompt = language == "pt" ? + @"Analise esta pergunta e classifique: + PERGUNTA: ""{0}"" + + Responda APENAS no formato JSON: + {{ + ""strategy"": ""overview|specific|detailed"", + ""complexity"": ""simple|medium|complex"", + ""scope"": ""global|filtered|targeted"", + ""concepts"": [""conceito1"", ""conceito2""], + ""needs_hierarchy"": true|false + }} + + REGRAS: + - overview: pergunta sobre projeto inteiro + - specific: pergunta sobre módulo/funcionalidade específica + - detailed: pergunta técnica que precisa de contexto profundo + - needs_hierarchy: true se precisar de múltiplas buscas" : + + @"Analyze this question and classify: + QUESTION: ""{0}"" + + Answer ONLY in JSON format: + {{ + ""strategy"": ""overview|specific|detailed"", + ""complexity"": ""simple|medium|complex"", + ""scope"": ""global|filtered|targeted"", + ""concepts"": [""concept1"", ""concept2""], + ""needs_hierarchy"": true|false + }} + + RULES: + - overview: question about entire project + - specific: question about specific module/functionality + - detailed: technical question needing deep context + - needs_hierarchy: true if needs multiple searches"; + + var prompt = string.Format(analysisPrompt, question); + + var executionSettings = new OpenAIPromptExecutionSettings + { + Temperature = 0.1, + MaxTokens = 200 + }; + + var response = await _chatCompletionService.GetChatMessageContentAsync(prompt, executionSettings); + + try + { + var jsonResponse = response.Content?.Trim() ?? "{}"; + + // Extrair JSON se vier com texto extra + var startIndex = jsonResponse.IndexOf('{'); + var endIndex = jsonResponse.LastIndexOf('}'); + if (startIndex >= 0 && endIndex >= startIndex) + { + jsonResponse = jsonResponse.Substring(startIndex, endIndex - startIndex + 1); + } + + var analysis = System.Text.Json.JsonSerializer.Deserialize(jsonResponse); + return analysis ?? new QueryAnalysis { Strategy = "specific", Complexity = "medium" }; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Erro ao parsear análise da query, usando padrão"); + return new QueryAnalysis { Strategy = "specific", Complexity = "medium" }; + } + } + + private async Task ExecuteHierarchicalSearch(string question, string projectId, QueryAnalysis analysis) + { + var context = new HierarchicalContext(); + var embeddingService = _kernel.GetRequiredService(); + + switch (analysis.Strategy) + { + case "overview": + await ExecuteOverviewStrategy(context, question, projectId, embeddingService); + break; + + case "detailed": + await ExecuteDetailedStrategy(context, question, projectId, embeddingService, analysis); + break; + + default: // specific + await ExecuteSpecificStrategy(context, question, projectId, embeddingService); + break; + } + + return context; + } + + private async Task ExecuteOverviewStrategy(HierarchicalContext context, string question, string projectId, ITextEmbeddingGenerationService embeddingService) + { + // Etapa 1: Buscar resumos/títulos primeiro + context.AddStep("Buscando visão geral do projeto"); + var overviewResults = await _vectorSearchService.GetDocumentsByProjectAsync(projectId); + + // Etapa 2: Identificar documentos principais baseado na pergunta + var questionEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(question)); + var embeddingArray = questionEmbedding.ToArray().Select(e => (double)e).ToArray(); + + context.AddStep("Identificando documentos relevantes"); + var relevantDocs = await _vectorSearchService.SearchSimilarAsync(embeddingArray, projectId, 0.3, 5); + + context.CombinedContext = $"VISÃO GERAL DO PROJETO:\n{FormatResults(overviewResults.Take(3))}\n\nDOCUMENTOS RELEVANTES:\n{FormatResults(relevantDocs)}"; + } + + private async Task ExecuteSpecificStrategy(HierarchicalContext context, string question, string projectId, ITextEmbeddingGenerationService embeddingService) + { + // Etapa 1: Busca inicial por similaridade + context.AddStep("Busca inicial por similaridade"); + var questionEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(question)); + var embeddingArray = questionEmbedding.ToArray().Select(e => (double)e).ToArray(); + + var initialResults = await _vectorSearchService.SearchSimilarAsync(embeddingArray, projectId, 0.4, 3); + + if (initialResults.Any()) + { + context.AddStep("Expandindo contexto com documentos relacionados"); + + // Etapa 2: Expandir com contexto relacionado + var expandedContext = await ExpandContext(initialResults, projectId, embeddingService); + context.CombinedContext = $"CONTEXTO PRINCIPAL:\n{FormatResults(initialResults)}\n\nCONTEXTO EXPANDIDO:\n{FormatResults(expandedContext)}"; + } + else + { + context.AddStep("Fallback para busca ampla"); + var fallbackResults = await _vectorSearchService.SearchSimilarAsync(embeddingArray, projectId, 0.2, 5); + context.CombinedContext = FormatResults(fallbackResults); + } + } + + private async Task ExecuteDetailedStrategy(HierarchicalContext context, string question, string projectId, ITextEmbeddingGenerationService embeddingService, QueryAnalysis analysis) + { + var questionEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(question)); + var embeddingArray = questionEmbedding.ToArray().Select(e => (double)e).ToArray(); + + // Etapa 1: Busca conceitual baseada nos conceitos identificados + context.AddStep("Busca conceitual inicial"); + var conceptualResults = new List(); + + if (analysis.Concepts?.Any() == true) + { + foreach (var concept in analysis.Concepts) + { + var conceptEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(concept)); + var conceptArray = conceptEmbedding.ToArray().Select(e => (double)e).ToArray(); + var conceptResults = await _vectorSearchService.SearchSimilarAsync(conceptArray, projectId, 0.3, 2); + conceptualResults.AddRange(conceptResults); + } + } + + // Etapa 2: Busca direta pela pergunta + context.AddStep("Busca direta pela pergunta"); + var directResults = await _vectorSearchService.SearchSimilarAsync(embeddingArray, projectId, 0.3, 3); + + // Etapa 3: Síntese intermediária para identificar lacunas + context.AddStep("Identificando lacunas de conhecimento"); + var intermediateContext = FormatResults(conceptualResults.Concat(directResults).DistinctBy(r => r.Id)); + var gaps = await IdentifyKnowledgeGaps(question, intermediateContext); + + // Etapa 4: Busca complementar baseada nas lacunas + if (!string.IsNullOrEmpty(gaps)) + { + context.AddStep("Preenchendo lacunas de conhecimento"); + var gapEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(gaps)); + var gapArray = gapEmbedding.ToArray().Select(e => (double)e).ToArray(); + var gapResults = await _vectorSearchService.SearchSimilarAsync(gapArray, projectId, 0.25, 2); + + context.CombinedContext = $"CONTEXTO CONCEITUAL:\n{FormatResults(conceptualResults)}\n\nCONTEXTO DIRETO:\n{FormatResults(directResults)}\n\nCONTEXTO COMPLEMENTAR:\n{FormatResults(gapResults)}"; + } + else + { + context.CombinedContext = $"CONTEXTO CONCEITUAL:\n{FormatResults(conceptualResults)}\n\nCONTEXTO DIRETO:\n{FormatResults(directResults)}"; + } + } + + private async Task> ExpandContext(List initialResults, string projectId, ITextEmbeddingGenerationService embeddingService) + { + var expandedResults = new List(); + + // Para cada resultado inicial, buscar documentos relacionados + foreach (var result in initialResults.Take(2)) // Limitar para evitar explosão de contexto + { + var resultEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(result.Content)); + var embeddingArray = resultEmbedding.ToArray().Select(e => (double)e).ToArray(); + + var relatedDocs = await _vectorSearchService.SearchSimilarAsync(embeddingArray, projectId, 0.4, 2); + expandedResults.AddRange(relatedDocs.Where(r => !initialResults.Any(ir => ir.Id == r.Id))); + } + + return expandedResults.DistinctBy(r => r.Id).ToList(); + } + + private async Task IdentifyKnowledgeGaps(string question, string currentContext) + { + var gapPrompt = @"Baseado na pergunta e contexto atual, identifique que informações ainda faltam para uma resposta completa. + + PERGUNTA: {0} + CONTEXTO ATUAL: {1} + + Responda APENAS com palavras-chave dos conceitos/informações que ainda faltam, separados por vírgula. + Se o contexto for suficiente, responda 'SUFICIENTE'."; + + var prompt = string.Format(gapPrompt, question, currentContext.Substring(0, Math.Min(1000, currentContext.Length))); + + var executionSettings = new OpenAIPromptExecutionSettings + { + Temperature = 0.2, + MaxTokens = 100 + }; + + var response = await _chatCompletionService.GetChatMessageContentAsync(prompt, executionSettings); + var gaps = response.Content?.Trim() ?? ""; + + return gaps.Equals("SUFICIENTE", StringComparison.OrdinalIgnoreCase) ? "" : gaps; + } + + private async Task GenerateResponse(string question, string projectId, HierarchicalContext context, string sessionId, string language) + { + var projectData = await _projectDataRepository.GetAsync(projectId); + var project = $"Nome: {projectData.Nome} \n\n Descrição:{projectData.Descricao}"; + + var prompt = language == "pt" ? + @"Você é um especialista em análise de software e QA. + + PROJETO: {0} + PERGUNTA: ""{1}"" + CONTEXTO HIERÁRQUICO: {2} + ETAPAS EXECUTADAS: {3} + + Responda à pergunta de forma precisa e estruturada, aproveitando todo o contexto hierárquico coletado." : + + @"You are a software analysis and QA expert. + + PROJECT: {0} + QUESTION: ""{1}"" + HIERARCHICAL CONTEXT: {2} + EXECUTED STEPS: {3} + + Answer the question precisely and structured, leveraging all the hierarchical context collected."; + + var finalPrompt = string.Format(prompt, project, question, context.CombinedContext, + string.Join(" → ", context.Steps)); + + var history = _chatHistoryService.GetSumarizer(sessionId); + history.AddUserMessage(finalPrompt); + + var executionSettings = new OpenAIPromptExecutionSettings + { + Temperature = 0.7, + TopP = 1.0, + FrequencyPenalty = 0, + PresencePenalty = 0 + }; + + var response = await _chatCompletionService.GetChatMessageContentAsync(history, executionSettings); + history.AddMessage(response.Role, response.Content ?? ""); + _chatHistoryService.UpdateHistory(sessionId, history); + + return response.Content ?? ""; + } + + private string FormatResults(IEnumerable results) + { + return string.Join("\n\n", results.Select((item, index) => + $"=== DOCUMENTO {index + 1} ===\n" + + $"Relevância: {item.Score:P1}\n" + + $"Conteúdo: {item.Content}")); + } + + public Task GetResponse(UserData userData, string projectId, string sessionId, string question) + { + return GetResponse(userData, projectId, sessionId, question, "pt"); + } + } + + // Classes de apoio para o RAG Hierárquico + public class QueryAnalysis + { + public string Strategy { get; set; } = "specific"; + public string Complexity { get; set; } = "medium"; + public string Scope { get; set; } = "filtered"; + public string[] Concepts { get; set; } = Array.Empty(); + public bool Needs_Hierarchy { get; set; } = false; + } + + public class HierarchicalContext + { + public List Steps { get; set; } = new(); + public string CombinedContext { get; set; } = ""; + public Dictionary Metadata { get; set; } = new(); + + public void AddStep(string step) + { + Steps.Add($"{DateTime.Now:HH:mm:ss} - {step}"); + } + } + +} +#pragma warning restore SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. diff --git a/Services/ResponseService/HierarchicalRAGService.cs b/Services/ResponseService/HierarchicalRAGService.cs new file mode 100644 index 0000000..b073233 --- /dev/null +++ b/Services/ResponseService/HierarchicalRAGService.cs @@ -0,0 +1,546 @@ +using ChatApi; +using ChatApi.Models; +using ChatRAG.Contracts.VectorSearch; +using ChatRAG.Data; +using ChatRAG.Models; +using ChatRAG.Services.Contracts; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.ChatCompletion; +using Microsoft.SemanticKernel.Connectors.OpenAI; +using Microsoft.SemanticKernel.Embeddings; +using System.Text.Json; + +#pragma warning disable SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. +namespace ChatRAG.Services.ResponseService +{ + public class HierarchicalRAGService : IResponseService + { + private readonly ChatHistoryService _chatHistoryService; + private readonly Kernel _kernel; + private readonly TextFilter _textFilter; + private readonly IProjectDataRepository _projectDataRepository; + private readonly IChatCompletionService _chatCompletionService; + private readonly IVectorSearchService _vectorSearchService; + private readonly ILogger _logger; + + public HierarchicalRAGService( + ChatHistoryService chatHistoryService, + Kernel kernel, + TextFilter textFilter, + IProjectDataRepository projectDataRepository, + IChatCompletionService chatCompletionService, + IVectorSearchService vectorSearchService, + ILogger logger) + { + _chatHistoryService = chatHistoryService; + _kernel = kernel; + _textFilter = textFilter; + _projectDataRepository = projectDataRepository; + _chatCompletionService = chatCompletionService; + _vectorSearchService = vectorSearchService; + _logger = logger; + } + + public async Task GetResponse(UserData userData, string projectId, string sessionId, string question, string language = "pt") + { + var stopWatch = new System.Diagnostics.Stopwatch(); + stopWatch.Start(); + + try + { + // 1. Análise da query para determinar estratégia + var queryAnalysis = await AnalyzeQuery(question, language); + _logger.LogInformation("Query Analysis: {Strategy}, Complexity: {Complexity}", + queryAnalysis.Strategy, queryAnalysis.Complexity); + + // 2. Execução hierárquica baseada na análise + var context = await ExecuteHierarchicalSearch(question, projectId, queryAnalysis); + + // 3. Geração da resposta final + var response = await GenerateResponse(question, projectId, context, sessionId, language); + + stopWatch.Stop(); + return $"{response}\n\nTempo: {stopWatch.ElapsedMilliseconds / 1000}s\nEtapas: {context.Steps.Count}"; + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro no RAG Hierárquico"); + stopWatch.Stop(); + return $"Erro: {ex.Message}\nTempo: {stopWatch.ElapsedMilliseconds / 1000}s"; + } + } + + private async Task AnalyzeQuery(string question, string language) + { + var analysisPrompt = language == "pt" ? + @"Analise esta pergunta e classifique com precisão: + PERGUNTA: ""{0}"" + + Responda APENAS no formato JSON: + {{ + ""strategy"": ""overview|specific|detailed"", + ""complexity"": ""simple|medium|complex"", + ""scope"": ""global|filtered|targeted"", + ""concepts"": [""conceito1"", ""conceito2""], + ""needs_hierarchy"": true|false + }} + + DEFINIÇÕES PRECISAS: + + STRATEGY: + - overview: Pergunta sobre o PROJETO COMO UM TODO. Palavras-chave: ""projeto"", ""sistema"", ""aplicação"", ""este projeto"", ""todo o"", ""geral"", ""inteiro"". NÃO menciona módulos, funcionalidades ou tecnologias específicas. + - specific: Pergunta sobre MÓDULO/FUNCIONALIDADE ESPECÍFICA. Menciona: nome de classe, controller, entidade, CRUD específico, funcionalidade particular, tecnologia específica. + - detailed: Pergunta técnica específica que precisa de CONTEXTO PROFUNDO e detalhes de implementação. + + SCOPE: + - global: Busca informações de TODO o projeto (usar com overview) + - filtered: Busca com filtros específicos (usar com specific/detailed) + - targeted: Busca muito específica e direcionada + + EXEMPLOS: + - ""Gere casos de teste para este projeto"" → overview/global + - ""Gere casos de teste do projeto"" → overview/global + - ""Gere casos de teste para o CRUD de usuário"" → specific/filtered + - ""Como implementar autenticação JWT neste controller"" → detailed/targeted + - ""Documente este sistema"" → overview/global + - ""Explique a classe UserService"" → specific/filtered" : + + @"Analyze this question and classify precisely: + QUESTION: ""{0}"" + + Answer ONLY in JSON format: + {{ + ""strategy"": ""overview|specific|detailed"", + ""complexity"": ""simple|medium|complex"", + ""scope"": ""global|filtered|targeted"", + ""concepts"": [""concept1"", ""concept2""], + ""needs_hierarchy"": true|false + }} + + PRECISE DEFINITIONS: + + STRATEGY: + - overview: Question about the PROJECT AS A WHOLE. Keywords: ""project"", ""system"", ""application"", ""this project"", ""entire"", ""general"", ""whole"". Does NOT mention specific modules, functionalities or technologies. + - specific: Question about SPECIFIC MODULE/FUNCTIONALITY. Mentions: class name, controller, entity, specific CRUD, particular functionality, specific technology. + - detailed: Technical specific question needing DEEP CONTEXT and implementation details. + + SCOPE: + - global: Search information from ENTIRE project (use with overview) + - filtered: Search with specific filters (use with specific/detailed) + - targeted: Very specific and directed search + + EXAMPLES: + - ""Generate test cases for this project"" → overview/global + - ""Generate test cases for user CRUD"" → specific/filtered + - ""How to implement JWT authentication in this controller"" → detailed/targeted + - ""Document this system"" → overview/global + - ""Explain the UserService class"" → specific/filtered"; + + var prompt = string.Format(analysisPrompt, question); + var executionSettings = new OpenAIPromptExecutionSettings + { + Temperature = 0.1, + MaxTokens = 300 // Aumentei um pouco para acomodar o prompt maior + }; + var response = await _chatCompletionService.GetChatMessageContentAsync(prompt, executionSettings); + + try + { + var jsonResponse = response.Content?.Trim() ?? "{}"; + // Extrair JSON se vier com texto extra + var startIndex = jsonResponse.IndexOf('{'); + var endIndex = jsonResponse.LastIndexOf('}'); + if (startIndex >= 0 && endIndex >= startIndex) + { + jsonResponse = jsonResponse.Substring(startIndex, endIndex - startIndex + 1); + } + + var options = new JsonSerializerOptions + { + PropertyNameCaseInsensitive = true, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase + }; + + var analysis = System.Text.Json.JsonSerializer.Deserialize(jsonResponse, options); + + // Log para debug - remover em produção + _logger.LogInformation($"Query: '{question}' → Strategy: {analysis?.Strategy}, Scope: {analysis?.Scope}"); + + return analysis ?? new QueryAnalysis { Strategy = "specific", Complexity = "medium" }; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Erro ao parsear análise da query, usando padrão"); + return new QueryAnalysis { Strategy = "specific", Complexity = "medium" }; + } + } + + private async Task ExecuteHierarchicalSearch(string question, string projectId, QueryAnalysis analysis) + { + var context = new HierarchicalContext(); + var embeddingService = _kernel.GetRequiredService(); + + switch (analysis.Strategy) + { + case "overview": + await ExecuteOverviewStrategy(context, question, projectId, embeddingService); + break; + + case "detailed": + await ExecuteDetailedStrategy(context, question, projectId, embeddingService, analysis); + break; + + default: // specific + await ExecuteSpecificStrategy(context, question, projectId, embeddingService); + break; + } + + return context; + } + + private async Task ExecuteOverviewStrategy(HierarchicalContext context, string question, string projectId, ITextEmbeddingGenerationService embeddingService) + { + // Etapa 1: Buscar TODOS os documentos do projeto + context.AddStep("Buscando todos os documentos do projeto"); + var allProjectDocs = await _vectorSearchService.GetDocumentsByProjectAsync(projectId); + + // Etapa 2: Categorizar documentos por tipo/importância + context.AddStep("Categorizando e resumindo contexto do projeto"); + + // Etapa 2: Categorizar documentos por tipo baseado nos seus dados reais + context.AddStep("Categorizando e resumindo contexto do projeto"); + + var requirementsDocs = allProjectDocs.Where(d => + d.Title.ToLower().StartsWith("requisito") || + d.Title.ToLower().Contains("requisito") || + d.Content.ToLower().Contains("requisito") || + d.Content.ToLower().Contains("funcionalidade") || + d.Content.ToLower().Contains("aplicação deve") || + d.Content.ToLower().Contains("sistema deve")).ToList(); + + var architectureDocs = allProjectDocs.Where(d => + d.Title.ToLower().Contains("arquitetura") || + d.Title.ToLower().Contains("estrutura") || + d.Title.ToLower().Contains("documentação") || + d.Title.ToLower().Contains("readme") || + d.Content.ToLower().Contains("arquitetura") || + d.Content.ToLower().Contains("estrutura") || + d.Content.ToLower().Contains("tecnologia")).ToList(); + + // Documentos que não são requisitos nem arquitetura (códigos, outros docs) + var otherDocs = allProjectDocs + .Except(requirementsDocs) + .Except(architectureDocs) + .ToList(); + + // Etapa 3: Resumir cada categoria se tiver muitos documentos + var requirementsSummary = await SummarizeDocuments(requirementsDocs, "requisitos e funcionalidades do projeto"); + var architectureSummary = await SummarizeDocuments(architectureDocs, "arquitetura e documentação técnica"); + var otherSummary = await SummarizeDocuments(otherDocs, "outros documentos do projeto"); + + // Etapa 4: Busca específica para a pergunta (mantém precisão) + var questionEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(question)); + var embeddingArray = questionEmbedding.ToArray().Select(e => (double)e).ToArray(); + + context.AddStep("Identificando documentos específicos para a pergunta"); + var relevantDocs = await _vectorSearchService.SearchSimilarAsync(embeddingArray, projectId, 0.3, 8); + + // Etapa 5: Combinar resumos + documentos específicos + var contextParts = new List(); + + if (!string.IsNullOrEmpty(requirementsSummary)) + contextParts.Add($"RESUMO DOS REQUISITOS E FUNCIONALIDADES:\n{requirementsSummary}"); + + if (!string.IsNullOrEmpty(architectureSummary)) + contextParts.Add($"RESUMO DA ARQUITETURA E DOCUMENTAÇÃO:\n{architectureSummary}"); + + if (!string.IsNullOrEmpty(otherSummary)) + contextParts.Add($"OUTROS DOCUMENTOS DO PROJETO:\n{otherSummary}"); + + contextParts.Add($"DOCUMENTOS MAIS RELEVANTES PARA A PERGUNTA:\n{FormatResults(relevantDocs)}"); + + context.CombinedContext = string.Join("\n\n", contextParts); + } + + private async Task SummarizeDocuments(List documents, string category) + { + if (!documents.Any()) return string.Empty; + + // Se poucos documentos, usar todos sem resumir + if (documents.Count <= 3) + { + return FormatResults(documents); + } + + // Se muitos documentos, resumir em chunks + var chunks = documents.Chunk(5).ToList(); // Grupos de 5 documentos + var tasks = new List>(); + + // Semáforo para controlar concorrência (máximo 3 chamadas simultâneas) + var semaphore = new SemaphoreSlim(3, 3); + + foreach (var chunk in chunks) + { + var chunkContent = FormatResults(chunk); + + tasks.Add(Task.Run(async () => + { + await semaphore.WaitAsync(); + try + { + var summaryPrompt = $@"Resuma os pontos principais destes documentos sobre {category}: + + {chunkContent} + + Responda apenas com uma lista concisa dos pontos mais importantes:"; + + var response = await _chatCompletionService.GetChatMessageContentAsync( + summaryPrompt, + new OpenAIPromptExecutionSettings + { + Temperature = 0.1, + MaxTokens = 300 + }); + + return response.Content ?? string.Empty; + } + catch (Exception ex) + { + _logger.LogWarning(ex, $"Erro ao resumir chunk de {category}, usando conteúdo original"); + return chunkContent; + } + finally + { + semaphore.Release(); + } + })); + } + + // Aguardar todas as tasks de resumo + var summaries = await Task.WhenAll(tasks); + var validSummaries = summaries.Where(s => !string.IsNullOrEmpty(s)).ToList(); + + // Se tiver múltiplos resumos, consolidar + if (validSummaries.Count > 1) + { + var consolidationPrompt = $@"Consolide estes resumos sobre {category} em um resumo final: + + {string.Join("\n\n", validSummaries)} + + Responda com os pontos mais importantes organizados:"; + + try + { + var finalResponse = await _chatCompletionService.GetChatMessageContentAsync( + consolidationPrompt, + new OpenAIPromptExecutionSettings + { + Temperature = 0.1, + MaxTokens = 400 + }); + + return finalResponse.Content ?? string.Empty; + } + catch + { + return string.Join("\n\n", validSummaries); + } + } + + return validSummaries.FirstOrDefault() ?? string.Empty; + } + + private async Task ExecuteSpecificStrategy(HierarchicalContext context, string question, string projectId, ITextEmbeddingGenerationService embeddingService) + { + // Etapa 1: Busca inicial por similaridade + context.AddStep("Busca inicial por similaridade"); + var questionEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(question)); + var embeddingArray = questionEmbedding.ToArray().Select(e => (double)e).ToArray(); + + var initialResults = await _vectorSearchService.SearchSimilarAsync(embeddingArray, projectId, 0.4, 3); + + if (initialResults.Any()) + { + context.AddStep("Expandindo contexto com documentos relacionados"); + + // Etapa 2: Expandir com contexto relacionado + var expandedContext = await ExpandContext(initialResults, projectId, embeddingService); + context.CombinedContext = $"CONTEXTO PRINCIPAL:\n{FormatResults(initialResults)}\n\nCONTEXTO EXPANDIDO:\n{FormatResults(expandedContext)}"; + } + else + { + context.AddStep("Fallback para busca ampla"); + var fallbackResults = await _vectorSearchService.SearchSimilarAsync(embeddingArray, projectId, 0.2, 5); + context.CombinedContext = FormatResults(fallbackResults); + } + } + + private async Task ExecuteDetailedStrategy(HierarchicalContext context, string question, string projectId, ITextEmbeddingGenerationService embeddingService, QueryAnalysis analysis) + { + var questionEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(question)); + var embeddingArray = questionEmbedding.ToArray().Select(e => (double)e).ToArray(); + + // Etapa 1: Busca conceitual baseada nos conceitos identificados + context.AddStep("Busca conceitual inicial"); + var conceptualResults = new List(); + + if (analysis.Concepts?.Any() == true) + { + foreach (var concept in analysis.Concepts) + { + var conceptEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(concept)); + var conceptArray = conceptEmbedding.ToArray().Select(e => (double)e).ToArray(); + var conceptResults = await _vectorSearchService.SearchSimilarAsync(conceptArray, projectId, 0.3, 2); + conceptualResults.AddRange(conceptResults); + } + } + + // Etapa 2: Busca direta pela pergunta + context.AddStep("Busca direta pela pergunta"); + var directResults = await _vectorSearchService.SearchSimilarAsync(embeddingArray, projectId, 0.3, 3); + + // Etapa 3: Síntese intermediária para identificar lacunas + context.AddStep("Identificando lacunas de conhecimento"); + var intermediateContext = FormatResults(conceptualResults.Concat(directResults).DistinctBy(r => r.Id)); + var gaps = await IdentifyKnowledgeGaps(question, intermediateContext); + + // Etapa 4: Busca complementar baseada nas lacunas + if (!string.IsNullOrEmpty(gaps)) + { + context.AddStep("Preenchendo lacunas de conhecimento"); + var gapEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(gaps)); + var gapArray = gapEmbedding.ToArray().Select(e => (double)e).ToArray(); + var gapResults = await _vectorSearchService.SearchSimilarAsync(gapArray, projectId, 0.25, 2); + + context.CombinedContext = $"CONTEXTO CONCEITUAL:\n{FormatResults(conceptualResults)}\n\nCONTEXTO DIRETO:\n{FormatResults(directResults)}\n\nCONTEXTO COMPLEMENTAR:\n{FormatResults(gapResults)}"; + } + else + { + context.CombinedContext = $"CONTEXTO CONCEITUAL:\n{FormatResults(conceptualResults)}\n\nCONTEXTO DIRETO:\n{FormatResults(directResults)}"; + } + } + + private async Task> ExpandContext(List initialResults, string projectId, ITextEmbeddingGenerationService embeddingService) + { + var expandedResults = new List(); + + // Para cada resultado inicial, buscar documentos relacionados + foreach (var result in initialResults.Take(2)) // Limitar para evitar explosão de contexto + { + var resultEmbedding = await embeddingService.GenerateEmbeddingAsync(_textFilter.ToLowerAndWithoutAccents(result.Content)); + var embeddingArray = resultEmbedding.ToArray().Select(e => (double)e).ToArray(); + + var relatedDocs = await _vectorSearchService.SearchSimilarAsync(embeddingArray, projectId, 0.4, 2); + expandedResults.AddRange(relatedDocs.Where(r => !initialResults.Any(ir => ir.Id == r.Id))); + } + + return expandedResults.DistinctBy(r => r.Id).ToList(); + } + + private async Task IdentifyKnowledgeGaps(string question, string currentContext) + { + var gapPrompt = @"Baseado na pergunta e contexto atual, identifique que informações ainda faltam para uma resposta completa. + + PERGUNTA: {0} + CONTEXTO ATUAL: {1} + + Responda APENAS com palavras-chave dos conceitos/informações que ainda faltam, separados por vírgula. + Se o contexto for suficiente, responda 'SUFICIENTE'."; + + var prompt = string.Format(gapPrompt, question, currentContext.Substring(0, Math.Min(1000, currentContext.Length))); + + var executionSettings = new OpenAIPromptExecutionSettings + { + Temperature = 0.2, + MaxTokens = 100 + }; + + var response = await _chatCompletionService.GetChatMessageContentAsync(prompt, executionSettings); + var gaps = response.Content?.Trim() ?? ""; + + return gaps.Equals("SUFICIENTE", StringComparison.OrdinalIgnoreCase) ? "" : gaps; + } + + private async Task GenerateResponse(string question, string projectId, HierarchicalContext context, string sessionId, string language) + { + var projectData = await _projectDataRepository.GetAsync(projectId); + var project = $"Nome: {projectData.Nome} \n\n Descrição:{projectData.Descricao}"; + + var prompt = language == "pt" ? + @"Você é um especialista em análise de software e QA. + + PROJETO: {0} + PERGUNTA: ""{1}"" + CONTEXTO HIERÁRQUICO: {2} + ETAPAS EXECUTADAS: {3} + + Responda à pergunta de forma precisa e estruturada, aproveitando todo o contexto hierárquico coletado." : + + @"You are a software analysis and QA expert. + + PROJECT: {0} + QUESTION: ""{1}"" + HIERARCHICAL CONTEXT: {2} + EXECUTED STEPS: {3} + + Answer the question precisely and structured, leveraging all the hierarchical context collected."; + + var finalPrompt = string.Format(prompt, project, question, context.CombinedContext, + string.Join(" → ", context.Steps)); + + var history = _chatHistoryService.GetSumarizer(sessionId); + history.AddUserMessage(finalPrompt); + + var executionSettings = new OpenAIPromptExecutionSettings + { + Temperature = 0.7, + TopP = 1.0, + FrequencyPenalty = 0, + PresencePenalty = 0 + }; + + var response = await _chatCompletionService.GetChatMessageContentAsync(history, executionSettings); + history.AddMessage(response.Role, response.Content ?? ""); + _chatHistoryService.UpdateHistory(sessionId, history); + + return response.Content ?? ""; + } + + private string FormatResults(IEnumerable results) + { + return string.Join("\n\n", results.Select((item, index) => + $"=== DOCUMENTO {index + 1} ===\n" + + $"Relevância: {item.Score:P1}\n" + + $"Conteúdo: {item.Content}")); + } + + public Task GetResponse(UserData userData, string projectId, string sessionId, string question) + { + return GetResponse(userData, projectId, sessionId, question, "pt"); + } + } + + // Classes de apoio para o RAG Hierárquico + public class QueryAnalysis + { + public string Strategy { get; set; } = "specific"; + public string Complexity { get; set; } = "medium"; + public string Scope { get; set; } = "filtered"; + public string[] Concepts { get; set; } = Array.Empty(); + public bool Needs_Hierarchy { get; set; } = false; + } + + public class HierarchicalContext + { + public List Steps { get; set; } = new(); + public string CombinedContext { get; set; } = ""; + public Dictionary Metadata { get; set; } = new(); + + public void AddStep(string step) + { + Steps.Add($"{DateTime.Now:HH:mm:ss} - {step}"); + } + } + +} +#pragma warning restore SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. diff --git a/Services/ResponseService/ResponseCompanyService.cs b/Services/ResponseService/ResponseCompanyService.cs index 943c3e4..6a22cb7 100644 --- a/Services/ResponseService/ResponseCompanyService.cs +++ b/Services/ResponseService/ResponseCompanyService.cs @@ -20,7 +20,7 @@ namespace ChatRAG.Services.ResponseService private readonly Kernel _kernel; private readonly TextFilter _textFilter; private readonly TextDataRepository _textDataRepository; - private readonly ProjectDataRepository _projectDataRepository; + private readonly IProjectDataRepository _projectDataRepository; private readonly IChatCompletionService _chatCompletionService; private readonly IVectorSearchService _vectorSearchService; @@ -29,7 +29,7 @@ namespace ChatRAG.Services.ResponseService Kernel kernel, TextFilter textFilter, TextDataRepository textDataRepository, - ProjectDataRepository projectDataRepository, + IProjectDataRepository projectDataRepository, IChatCompletionService chatCompletionService, IVectorSearchService vectorSearchService, ITextDataService textDataService) diff --git a/Services/SearchVectors/ChromaVectorSearchService.cs b/Services/SearchVectors/ChromaVectorSearchService.cs new file mode 100644 index 0000000..a7ef322 --- /dev/null +++ b/Services/SearchVectors/ChromaVectorSearchService.cs @@ -0,0 +1,651 @@ +using ChatRAG.Contracts.VectorSearch; +using ChatRAG.Models; +using Microsoft.Extensions.Options; +using System.Text; +using System.Text.Json; +using Microsoft.SemanticKernel.Embeddings; +using ChatRAG.Settings.ChatRAG.Configuration; + +namespace ChatRAG.Services.SearchVectors +{ + public class ChromaVectorSearchService : IVectorSearchService + { + private readonly HttpClient _httpClient; + private readonly ILogger _logger; + private readonly ChromaSettings _settings; + private readonly string _collectionName; + + public ChromaVectorSearchService( + IOptions settings, + ILogger logger, + HttpClient httpClient) + { + _settings = settings.Value.Chroma ?? throw new ArgumentNullException("Chroma settings not configured"); + _logger = logger; + _httpClient = httpClient; + _httpClient.BaseAddress = new Uri($"http://{_settings.Host}:{_settings.Port}"); + _collectionName = _settings.CollectionName; + + InitializeAsync().GetAwaiter().GetResult(); + } + + private async Task InitializeAsync() + { + try + { + // Verificar se a collection existe, se não, criar + var collections = await GetCollectionsAsync(); + if (!collections.Contains(_collectionName)) + { + await CreateCollectionAsync(); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao inicializar Chroma"); + throw; + } + } + + // ======================================== + // BUSCA VETORIAL + // ======================================== + + public async Task> SearchSimilarAsync( + double[] queryEmbedding, + string? projectId = null, + double threshold = 0.3, + int limit = 5, + Dictionary? filters = null) + { + try + { + // Construir filtros WHERE + var whereClause = BuildWhereClause(projectId, filters); + + var query = new + { + query_embeddings = new[] { queryEmbedding }, + n_results = limit, + where = whereClause, + include = new[] { "documents", "metadatas", "distances" } + }; + + var json = JsonSerializer.Serialize(query); + var content = new StringContent(json, Encoding.UTF8, "application/json"); + + var response = await _httpClient.PostAsync($"/api/v1/collections/{_collectionName}/query", content); + + if (!response.IsSuccessStatusCode) + { + var error = await response.Content.ReadAsStringAsync(); + _logger.LogError("Erro na busca Chroma: {Error}", error); + return new List(); + } + + var result = await response.Content.ReadAsStringAsync(); + var queryResult = JsonSerializer.Deserialize(result); + + return ParseQueryResults(queryResult, threshold); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao buscar similares no Chroma"); + return new List(); + } + } + + public async Task> SearchSimilarDynamicAsync( + double[] queryEmbedding, + string projectId, + double minThreshold = 0.5, + int limit = 5) + { + // Estratégia 1: Busca com threshold alto + var results = await SearchSimilarAsync(queryEmbedding, projectId, minThreshold, limit); + + if (results.Count >= limit) + { + return results.Take(limit).ToList(); + } + + // Estratégia 2: Relaxar threshold se não conseguiu o suficiente + if (results.Count < limit && minThreshold > 0.35) + { + var mediumResults = await SearchSimilarAsync(queryEmbedding, projectId, 0.35, limit * 2); + if (mediumResults.Count >= limit) + { + return mediumResults.Take(limit).ToList(); + } + results = mediumResults; + } + + // Estratégia 3: Threshold baixo como último recurso + if (results.Count < limit && minThreshold > 0.2) + { + var lowResults = await SearchSimilarAsync(queryEmbedding, projectId, 0.2, limit * 3); + results = lowResults; + } + + return results.Take(limit).ToList(); + } + + // ======================================== + // CRUD DE DOCUMENTOS + // ======================================== + + public async Task AddDocumentAsync( + string title, + string content, + string projectId, + double[] embedding, + Dictionary? metadata = null) + { + try + { + var documentId = Guid.NewGuid().ToString(); + + var combinedMetadata = new Dictionary + { + ["title"] = title, + ["project_id"] = projectId, + ["created_at"] = DateTime.UtcNow.ToString("O") + }; + + if (metadata != null) + { + foreach (var kvp in metadata) + { + combinedMetadata[kvp.Key] = kvp.Value; + } + } + + var document = new + { + ids = new[] { documentId }, + documents = new[] { content }, + metadatas = new[] { combinedMetadata }, + embeddings = new[] { embedding } + }; + + var json = JsonSerializer.Serialize(document); + var requestContent = new StringContent(json, Encoding.UTF8, "application/json"); + + var response = await _httpClient.PostAsync($"/api/v1/collections/{_collectionName}/add", requestContent); + + if (!response.IsSuccessStatusCode) + { + var error = await response.Content.ReadAsStringAsync(); + throw new Exception($"Erro ao adicionar documento: {error}"); + } + + return documentId; + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao adicionar documento no Chroma"); + throw; + } + } + + public async Task UpdateDocumentAsync( + string id, + string title, + string content, + string projectId, + double[] embedding, + Dictionary? metadata = null) + { + try + { + // Chroma não tem update direto, então fazemos delete + add + await DeleteDocumentAsync(id); + + var combinedMetadata = new Dictionary + { + ["title"] = title, + ["project_id"] = projectId, + ["updated_at"] = DateTime.UtcNow.ToString("O") + }; + + if (metadata != null) + { + foreach (var kvp in metadata) + { + combinedMetadata[kvp.Key] = kvp.Value; + } + } + + var document = new + { + ids = new[] { id }, + documents = new[] { content }, + metadatas = new[] { combinedMetadata }, + embeddings = new[] { embedding } + }; + + var json = JsonSerializer.Serialize(document); + var requestContent = new StringContent(json, Encoding.UTF8, "application/json"); + + var response = await _httpClient.PostAsync($"/api/v1/collections/{_collectionName}/upsert", requestContent); + + if (!response.IsSuccessStatusCode) + { + var error = await response.Content.ReadAsStringAsync(); + throw new Exception($"Erro ao atualizar documento: {error}"); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao atualizar documento no Chroma"); + throw; + } + } + + public async Task DeleteDocumentAsync(string id) + { + try + { + var deleteRequest = new + { + ids = new[] { id } + }; + + var json = JsonSerializer.Serialize(deleteRequest); + var content = new StringContent(json, Encoding.UTF8, "application/json"); + + var response = await _httpClient.PostAsync($"/api/v1/collections/{_collectionName}/delete", content); + + if (!response.IsSuccessStatusCode) + { + var error = await response.Content.ReadAsStringAsync(); + _logger.LogWarning("Erro ao deletar documento {Id}: {Error}", id, error); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao deletar documento {Id} no Chroma", id); + throw; + } + } + + // ======================================== + // CONSULTAS AUXILIARES + // ======================================== + + public async Task DocumentExistsAsync(string id) + { + try + { + var doc = await GetDocumentAsync(id); + return doc != null; + } + catch + { + return false; + } + } + + public async Task GetDocumentAsync(string id) + { + try + { + var query = new + { + ids = new[] { id }, + include = new[] { "documents", "metadatas" } + }; + + var json = JsonSerializer.Serialize(query); + var content = new StringContent(json, Encoding.UTF8, "application/json"); + + var response = await _httpClient.PostAsync($"/api/v1/collections/{_collectionName}/get", content); + + if (!response.IsSuccessStatusCode) + { + return null; + } + + var result = await response.Content.ReadAsStringAsync(); + var getResult = JsonSerializer.Deserialize(result); + + if (getResult?.ids?.Length > 0) + { + return new VectorSearchResult + { + Id = getResult.ids[0], + Content = getResult.documents?[0] ?? "", + Score = 1.0, + Metadata = getResult.metadatas?[0] + }; + } + + return null; + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao buscar documento {Id} no Chroma", id); + return null; + } + } + + public async Task> GetDocumentsByProjectAsync(string projectId) + { + try + { + var query = new + { + where = new { project_id = projectId }, + include = new[] { "documents", "metadatas" } + }; + + var json = JsonSerializer.Serialize(query); + var content = new StringContent(json, Encoding.UTF8, "application/json"); + + var response = await _httpClient.PostAsync($"/api/v1/collections/{_collectionName}/get", content); + + if (!response.IsSuccessStatusCode) + { + var error = await response.Content.ReadAsStringAsync(); + _logger.LogError("Erro ao buscar documentos do projeto {ProjectId}: {Error}", projectId, error); + return new List(); + } + + var result = await response.Content.ReadAsStringAsync(); + var getResult = JsonSerializer.Deserialize(result); + + var results = new List(); + + if (getResult?.documents?.Length > 0) + { + for (int i = 0; i < getResult.documents.Length; i++) + { + results.Add(new VectorSearchResult + { + Id = getResult.ids[i], + Content = getResult.documents[i], + Score = 1.0, // Todos os documentos do projeto + Metadata = getResult.metadatas?[i] + }); + } + } + + return results; + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao buscar documentos do projeto {ProjectId} no Chroma", projectId); + return new List(); + } + } + + public async Task GetDocumentCountAsync(string? projectId = null) + { + try + { + var query = new + { + where = projectId != null ? new { project_id = projectId } : null + }; + + var json = JsonSerializer.Serialize(query); + var content = new StringContent(json, Encoding.UTF8, "application/json"); + + var response = await _httpClient.PostAsync($"/api/v1/collections/{_collectionName}/count", content); + + if (!response.IsSuccessStatusCode) + { + _logger.LogWarning("Erro ao contar documentos no Chroma"); + return 0; + } + + var result = await response.Content.ReadAsStringAsync(); + var countResult = JsonSerializer.Deserialize(result); + return countResult?.count ?? 0; + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao contar documentos no Chroma"); + return 0; + } + } + + // ======================================== + // HEALTH CHECK E MÉTRICAS + // ======================================== + + public async Task IsHealthyAsync() + { + try + { + var response = await _httpClient.GetAsync("/api/v1/heartbeat"); + return response.IsSuccessStatusCode; + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro no health check do Chroma"); + return false; + } + } + + public async Task> GetStatsAsync() + { + try + { + var stats = new Dictionary + { + ["provider"] = "Chroma", + ["collection"] = _collectionName, + ["host"] = _settings.Host, + ["port"] = _settings.Port + }; + + // Tentar obter informações da collection + var response = await _httpClient.GetAsync($"/api/v1/collections/{_collectionName}"); + if (response.IsSuccessStatusCode) + { + var content = await response.Content.ReadAsStringAsync(); + var collectionInfo = JsonSerializer.Deserialize>(content); + if (collectionInfo != null) + { + stats["collection_info"] = collectionInfo; + } + } + + // Contar documentos totais + stats["total_documents"] = await GetDocumentCountAsync(); + + return stats; + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao obter stats do Chroma"); + return new Dictionary + { + ["provider"] = "Chroma", + ["error"] = ex.Message, + ["status"] = "error" + }; + } + } + + // ======================================== + // MÉTODOS AUXILIARES PRIVADOS + // ======================================== + + private async Task GetCollectionsAsync() + { + try + { + var response = await _httpClient.GetAsync("/api/v1/collections"); + if (!response.IsSuccessStatusCode) + { + _logger.LogWarning("Erro ao obter collections: {StatusCode}", response.StatusCode); + return Array.Empty(); + } + + var content = await response.Content.ReadAsStringAsync(); + + // Tentar desserializar como array de strings (versão simples) + try + { + var collections = JsonSerializer.Deserialize(content); + return collections ?? Array.Empty(); + } + catch + { + // Tentar desserializar como array de objetos (versão mais nova) + try + { + var collectionsObj = JsonSerializer.Deserialize(content); + return collectionsObj?.Select(c => c.name).ToArray() ?? Array.Empty(); + } + catch + { + _logger.LogWarning("Não foi possível parsear lista de collections"); + return Array.Empty(); + } + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao buscar collections"); + return Array.Empty(); + } + } + + // Classe auxiliar para desserialização + private class CollectionInfo + { + public string name { get; set; } = ""; + public Dictionary? metadata { get; set; } + } + + private async Task CreateCollectionAsync() + { + var collection = new + { + name = _collectionName, + metadata = new + { + description = "RAG Collection", + created_at = DateTime.UtcNow.ToString("O") + } + }; + + var json = JsonSerializer.Serialize(collection); + var content = new StringContent(json, Encoding.UTF8, "application/json"); + + // Tentar primeira abordagem (versão mais nova) + var response = await _httpClient.PostAsync("/api/v1/collections", content); + + // Se falhar, tentar segunda abordagem (criar collection via get_or_create) + if (!response.IsSuccessStatusCode) + { + _logger.LogWarning("Método POST falhou, tentando abordagem alternativa"); + + // Criar usando get_or_create approach + var createPayload = new + { + name = _collectionName, + metadata = new + { + description = "RAG Collection", + created_at = DateTime.UtcNow.ToString("O") + }, + get_or_create = true + }; + + var createJson = JsonSerializer.Serialize(createPayload); + var createContent = new StringContent(createJson, Encoding.UTF8, "application/json"); + + var createResponse = await _httpClient.PostAsync("/api/v1/collections", createContent); + + if (!createResponse.IsSuccessStatusCode) + { + var error = await createResponse.Content.ReadAsStringAsync(); + _logger.LogError("Erro ao criar collection: {Error}", error); + + // Última tentativa: assumir que collection já existe + _logger.LogWarning("Assumindo que collection {CollectionName} já existe", _collectionName); + return; + } + } + + _logger.LogInformation("Collection {CollectionName} criada/verificada com sucesso", _collectionName); + } + + private object? BuildWhereClause(string? projectId, Dictionary? filters) + { + var where = new Dictionary(); + + if (!string.IsNullOrEmpty(projectId)) + { + where["project_id"] = projectId; + } + + if (filters != null) + { + foreach (var filter in filters) + { + where[filter.Key] = filter.Value; + } + } + + return where.Any() ? where : null; + } + + private List ParseQueryResults(ChromaQueryResult? queryResult, double threshold) + { + var results = new List(); + + if (queryResult?.documents?.Length > 0 && queryResult.documents[0].Length > 0) + { + for (int i = 0; i < queryResult.documents[0].Length; i++) + { + var distance = queryResult.distances?[0][i] ?? 1.0; + + // Chroma retorna distâncias, converter para similaridade (1 - distance) + var similarity = 1.0 - distance; + + if (similarity >= threshold) + { + results.Add(new VectorSearchResult + { + Id = queryResult.ids[0][i], + Content = queryResult.documents[0][i], + Score = similarity, + Metadata = queryResult.metadatas?[0][i] + }); + } + } + } + + return results.OrderByDescending(r => r.Score).ToList(); + } + } + + // ======================================== + // DTOs PARA CHROMA API + // ======================================== + + public class ChromaQueryResult + { + public string[][] ids { get; set; } = Array.Empty(); + public string[][] documents { get; set; } = Array.Empty(); + public double[][]? distances { get; set; } + public Dictionary[][]? metadatas { get; set; } + } + + public class ChromaGetResult + { + public string[] ids { get; set; } = Array.Empty(); + public string[] documents { get; set; } = Array.Empty(); + public Dictionary[]? metadatas { get; set; } + } + + public class ChromaCountResult + { + public int count { get; set; } + } +} \ No newline at end of file diff --git a/Services/SearchVectors/QdrantVectorSearchService.cs b/Services/SearchVectors/QdrantVectorSearchService.cs index f59adbd..f748450 100644 --- a/Services/SearchVectors/QdrantVectorSearchService.cs +++ b/Services/SearchVectors/QdrantVectorSearchService.cs @@ -7,10 +7,10 @@ using ChatRAG.Services.Contracts; using Qdrant.Client; using static Qdrant.Client.Grpc.Conditions; using System.Drawing; +using System.Collections.Concurrent; #pragma warning disable SKEXP0001 - namespace ChatRAG.Services.SearchVectors { public class QdrantVectorSearchService : IVectorSearchService @@ -18,7 +18,9 @@ namespace ChatRAG.Services.SearchVectors private readonly QdrantClient _client; private readonly QdrantSettings _settings; private readonly ILogger _logger; - private bool _collectionInitialized = false; + private volatile bool _collectionInitialized = false; + private readonly SemaphoreSlim _initializationSemaphore = new(1, 1); + private readonly ConcurrentDictionary _collectionCache = new(); public QdrantVectorSearchService( IOptions settings, @@ -37,9 +39,20 @@ namespace ChatRAG.Services.SearchVectors { if (_collectionInitialized) return; + await _initializationSemaphore.WaitAsync(); try { + if (_collectionInitialized) return; + + // Verifica cache primeiro + if (_collectionCache.TryGetValue(_settings.CollectionName, out bool exists) && exists) + { + _collectionInitialized = true; + return; + } + var collectionExists = await _client.CollectionExistsAsync(_settings.CollectionName); + _collectionCache.TryAdd(_settings.CollectionName, collectionExists); if (!collectionExists) { @@ -58,7 +71,7 @@ namespace ChatRAG.Services.SearchVectors } }; - // Configurações HNSW opcionais + // Configurações HNSW otimizadas if (_settings.HnswM > 0) { vectorsConfig.HnswConfig = new HnswConfigDiff @@ -74,15 +87,15 @@ namespace ChatRAG.Services.SearchVectors vectorsConfig: vectorsConfig ); + _collectionCache.TryAdd(_settings.CollectionName, true); _logger.LogInformation("✅ Collection {CollectionName} criada", _settings.CollectionName); } _collectionInitialized = true; } - catch (Exception ex) + finally { - _logger.LogError(ex, "Erro ao inicializar collection {CollectionName}", _settings.CollectionName); - throw; + _initializationSemaphore.Release(); } } @@ -131,21 +144,10 @@ namespace ChatRAG.Services.SearchVectors limit: (ulong)limit, scoreThreshold: (float)threshold, payloadSelector: true, - vectorsSelector: true + vectorsSelector: false // Otimização: não buscar vetores desnecessariamente ); - return searchResult.Select(point => new VectorSearchResult - { - Id = point.Id.Uuid ?? point.Id.Num.ToString(), - Title = GetStringFromPayload(point.Payload, "title"), - Content = GetStringFromPayload(point.Payload, "content"), - ProjectId = GetStringFromPayload(point.Payload, "project_id"), - Score = point.Score, - Provider = "Qdrant", - CreatedAt = GetDateTimeFromPayload(point.Payload, "created_at"), - UpdatedAt = GetDateTimeFromPayload(point.Payload, "updated_at"), - Metadata = ConvertPayloadToMetadata(point.Payload) - }).ToList(); + return searchResult.Select(ConvertToVectorSearchResult).ToList(); } catch (Exception ex) { @@ -189,27 +191,12 @@ namespace ChatRAG.Services.SearchVectors var id = Guid.NewGuid().ToString(); var vector = embedding.Select(x => (float)x).ToArray(); - var payload = new Dictionary - { - ["title"] = title, - ["content"] = content, - ["project_id"] = projectId, - ["created_at"] = DateTime.UtcNow.ToString("O"), - ["updated_at"] = DateTime.UtcNow.ToString("O") - }; - - if (metadata?.Any() == true) - { - foreach (var kvp in metadata) - { - payload[$"meta_{kvp.Key}"] = ConvertToValue(kvp.Value); - } - } + var payload = CreatePayload(title, content, projectId, metadata, isUpdate: false); var point = new PointStruct { - Id = new PointId { Uuid = id }, - Vectors = vector, + Id = new PointId { Uuid = id }, + Vectors = vector, Payload = { payload } }; @@ -241,22 +228,7 @@ namespace ChatRAG.Services.SearchVectors try { var vector = embedding.Select(x => (float)x).ToArray(); - - var payload = new Dictionary - { - ["title"] = title, - ["content"] = content, - ["project_id"] = projectId, - ["updated_at"] = DateTime.UtcNow.ToString("O") - }; - - if (metadata?.Any() == true) - { - foreach (var kvp in metadata) - { - payload[$"meta_{kvp.Key}"] = ConvertToValue(kvp.Value); - } - } + var payload = CreatePayload(title, content, projectId, metadata, isUpdate: true); var point = new PointStruct { @@ -285,11 +257,11 @@ namespace ChatRAG.Services.SearchVectors try { - var pointId = new PointId { Uuid = id } ; + var pointId = new PointId { Uuid = id }; await _client.DeleteAsync( collectionName: _settings.CollectionName, - ids: new ulong[] { pointId.Num } + ids: new ulong[] { pointId.Num } ); _logger.LogDebug("Documento {Id} removido do Qdrant", id); @@ -305,8 +277,18 @@ namespace ChatRAG.Services.SearchVectors { try { - var result = await GetDocumentAsync(id); - return result != null; + await EnsureCollectionExistsAsync(); + + var pointId = new PointId { Uuid = id }; + + var results = await _client.RetrieveAsync( + collectionName: _settings.CollectionName, + ids: new PointId[] { pointId }, + withPayload: false, // Otimização: só queremos saber se existe + withVectors: false + ); + + return results.Any(); } catch { @@ -330,20 +312,7 @@ namespace ChatRAG.Services.SearchVectors ); var point = results.FirstOrDefault(); - if (point == null) return null; - - return new VectorSearchResult - { - Id = point.Id.Uuid ?? point.Id.Num.ToString(), - Title = GetStringFromPayload(point.Payload, "title"), - Content = GetStringFromPayload(point.Payload, "content"), - ProjectId = GetStringFromPayload(point.Payload, "project_id"), - Score = 1.0, - Provider = "Qdrant", - CreatedAt = GetDateTimeFromPayload(point.Payload, "created_at"), - UpdatedAt = GetDateTimeFromPayload(point.Payload, "updated_at"), - Metadata = ConvertPayloadToMetadata(point.Payload) - }; + return point != null ? ConvertToVectorSearchResult(point) : null; } catch (Exception ex) { @@ -361,26 +330,18 @@ namespace ChatRAG.Services.SearchVectors var filter = new Filter(); filter.Must.Add(MatchKeyword("project_id", projectId)); - var results = await _client.ScrollAsync( - collectionName: _settings.CollectionName, - filter: filter, - limit: 10000, - payloadSelector: true, - vectorsSelector: true - ); - - return results.Result.Select(point => new VectorSearchResult + var scrollRequest = new ScrollPoints { - Id = point.Id.Uuid ?? point.Id.Num.ToString(), - Title = GetStringFromPayload(point.Payload, "title"), - Content = GetStringFromPayload(point.Payload, "content"), - ProjectId = GetStringFromPayload(point.Payload, "project_id"), - Score = 1.0, - Provider = "Qdrant", - CreatedAt = GetDateTimeFromPayload(point.Payload, "created_at"), - UpdatedAt = GetDateTimeFromPayload(point.Payload, "updated_at"), - Metadata = ConvertPayloadToMetadata(point.Payload) - }).ToList(); + CollectionName = _settings.CollectionName, + Filter = filter, + Limit = 10000, + WithPayload = true, + WithVectors = false // Otimização: não buscar vetores + }; + + var results = await _client.ScrollAsync(_settings.CollectionName, filter, 10000, null, true, false); + + return results.Result.Select(ConvertToVectorSearchResult).ToList(); } catch (Exception ex) { @@ -459,16 +420,84 @@ namespace ChatRAG.Services.SearchVectors } } + // Métodos auxiliares otimizados + private static Dictionary CreatePayload( + string title, + string content, + string projectId, + Dictionary? metadata, + bool isUpdate) + { + var payload = new Dictionary + { + ["title"] = title, + ["content"] = content, + ["project_id"] = projectId + }; + + if (isUpdate) + { + payload["updated_at"] = DateTime.UtcNow.ToString("O"); + } + else + { + payload["created_at"] = DateTime.UtcNow.ToString("O"); + payload["updated_at"] = DateTime.UtcNow.ToString("O"); + } + + if (metadata?.Any() == true) + { + foreach (var kvp in metadata) + { + payload[$"meta_{kvp.Key}"] = ConvertToValue(kvp.Value); + } + } + + return payload; + } + + private static VectorSearchResult ConvertToVectorSearchResult(ScoredPoint point) + { + return new VectorSearchResult + { + Id = point.Id.Uuid ?? point.Id.Num.ToString(), + Title = GetStringFromPayload(point.Payload, "title"), + Content = GetStringFromPayload(point.Payload, "content"), + ProjectId = GetStringFromPayload(point.Payload, "project_id"), + Score = point.Score, + Provider = "Qdrant", + CreatedAt = GetDateTimeFromPayload(point.Payload, "created_at"), + UpdatedAt = GetDateTimeFromPayload(point.Payload, "updated_at"), + Metadata = ConvertPayloadToMetadata(point.Payload) + }; + } + + private static VectorSearchResult ConvertToVectorSearchResult(RetrievedPoint point) + { + return new VectorSearchResult + { + Id = point.Id.Uuid ?? point.Id.Num.ToString(), + Title = GetStringFromPayload(point.Payload, "title"), + Content = GetStringFromPayload(point.Payload, "content"), + ProjectId = GetStringFromPayload(point.Payload, "project_id"), + Score = 1.0, + Provider = "Qdrant", + CreatedAt = GetDateTimeFromPayload(point.Payload, "created_at"), + UpdatedAt = GetDateTimeFromPayload(point.Payload, "updated_at"), + Metadata = ConvertPayloadToMetadata(point.Payload) + }; + } + private static Value ConvertToValue(object value) { return value switch { - string s => s, - int i => i, - long l => l, - double d => d, - float f => f, - bool b => b, + string s => s, + int i => i, + long l => l, + double d => d, + float f => f, + bool b => b, DateTime dt => dt.ToString("O"), _ => value?.ToString() ?? "" }; @@ -519,6 +548,7 @@ namespace ChatRAG.Services.SearchVectors public void Dispose() { + _initializationSemaphore?.Dispose(); _client?.Dispose(); } } diff --git a/Services/SearchVectors/VectorDatabaseFactory.cs b/Services/SearchVectors/VectorDatabaseFactory.cs index 64316c7..7a83e11 100644 --- a/Services/SearchVectors/VectorDatabaseFactory.cs +++ b/Services/SearchVectors/VectorDatabaseFactory.cs @@ -1,5 +1,6 @@ using ChatApi.Data; using ChatRAG.Contracts.VectorSearch; +using ChatRAG.Data; using ChatRAG.Services.Contracts; using ChatRAG.Services.ResponseService; using ChatRAG.Services.TextServices; @@ -8,33 +9,31 @@ using Microsoft.Extensions.Options; namespace ChatRAG.Services.SearchVectors { - /// - /// Factory principal que cria implementações baseadas na configuração - /// - /// - /// Factory principal que cria implementações baseadas na configuração - /// public class VectorDatabaseFactory : IVectorDatabaseFactory { - private readonly VectorDatabaseSettings _settings; private readonly IServiceProvider _serviceProvider; + private readonly VectorDatabaseSettings _settings; private readonly ILogger _logger; public VectorDatabaseFactory( - IOptions settings, IServiceProvider serviceProvider, + IOptions settings, ILogger logger) { - _settings = settings.Value; _serviceProvider = serviceProvider; + _settings = settings.Value; _logger = logger; - - // Valida configurações na inicialização - ValidateSettings(); } - public string GetActiveProvider() => _settings.Provider; - public VectorDatabaseSettings GetSettings() => _settings; + public string GetActiveProvider() + { + return _settings.Provider; + } + + public VectorDatabaseSettings GetSettings() + { + return _settings; + } public IVectorSearchService CreateVectorSearchService() { @@ -42,8 +41,9 @@ namespace ChatRAG.Services.SearchVectors return _settings.Provider.ToLower() switch { - "qdrant" => GetService(), - "mongodb" => GetService(), + "qdrant" => GetService(), + "mongodb" => GetService(), + "chroma" => GetService(), _ => throw new ArgumentException($"Provider de VectorSearch não suportado: {_settings.Provider}") }; } @@ -54,10 +54,10 @@ namespace ChatRAG.Services.SearchVectors return _settings.Provider.ToLower() switch { - // ✅ CORRIGIDO: Usa os namespaces corretos - "qdrant" => GetService(), - "mongodb" => GetService(), // Sua classe atual! - _ => throw new ArgumentException($"Provider de TextData não suportado: {_settings.Provider}") + "qdrant" => GetService(), + "mongodb" => GetService(), + "chroma" => GetService(), + _ => throw new ArgumentException($"Provider de TextDataService não suportado: {_settings.Provider}") }; } @@ -65,50 +65,36 @@ namespace ChatRAG.Services.SearchVectors { _logger.LogInformation("Criando ResponseService para provider: {Provider}", _settings.Provider); - return _settings.Provider.ToLower() switch + // Verificar se deve usar RAG Hierárquico + var configuration = _serviceProvider.GetService(); + var useHierarchical = configuration?.GetValue("Features:UseHierarchicalRAG") ?? false; + + if (useHierarchical) { - // ✅ CORRIGIDO: Usa os namespaces corretos - "qdrant" => GetService(), - "mongodb" => GetService(), // Sua classe atual! - _ => throw new ArgumentException($"Provider de Response não suportado: {_settings.Provider}") + _logger.LogInformation("Usando HierarchicalRAGService"); + return GetService(); + } + + // Usar estratégia baseada no provider ou configuração + var ragStrategy = configuration?.GetValue("Features:RAGStrategy"); + + return ragStrategy?.ToLower() switch + { + "hierarchical" => GetService(), + "standard" => GetService(), + _ => GetService() // Padrão }; } - // ======================================== - // MÉTODOS AUXILIARES - // ======================================== - private T GetService() where T : class { - try + var service = _serviceProvider.GetService(); + if (service == null) { - var service = _serviceProvider.GetRequiredService(); - _logger.LogDebug("Serviço {ServiceType} criado com sucesso", typeof(T).Name); - return service; + throw new InvalidOperationException($"Serviço {typeof(T).Name} não está registrado no DI container. " + + $"Verifique se o serviço foi registrado para o provider '{_settings.Provider}'."); } - catch (InvalidOperationException ex) - { - _logger.LogError(ex, "Erro ao criar serviço {ServiceType} para provider {Provider}", - typeof(T).Name, _settings.Provider); - - throw new InvalidOperationException( - $"Serviço {typeof(T).Name} não está registrado para provider {_settings.Provider}. " + - $"Certifique-se de chamar services.Add{_settings.Provider}Provider() no DI.", ex); - } - } - - private void ValidateSettings() - { - if (!_settings.IsValid()) - { - var errors = _settings.GetValidationErrors(); - var errorMessage = $"Configurações inválidas para VectorDatabase: {string.Join(", ", errors)}"; - - _logger.LogError(errorMessage); - throw new InvalidOperationException(errorMessage); - } - - _logger.LogInformation("Configurações validadas com sucesso para provider: {Provider}", _settings.Provider); + return service; } } } diff --git a/Services/SearchVectors/anv10eil.vdi~ b/Services/SearchVectors/anv10eil.vdi~ new file mode 100644 index 0000000..e09cb32 --- /dev/null +++ b/Services/SearchVectors/anv10eil.vdi~ @@ -0,0 +1,55 @@ +using ChatApi.Data; +using ChatRAG.Contracts.VectorSearch; +using ChatRAG.Services.Contracts; +using ChatRAG.Services.ResponseService; +using ChatRAG.Services.TextServices; +using ChatRAG.Settings.ChatRAG.Configuration; +using Microsoft.Extensions.Options; + +namespace ChatRAG.Services.SearchVectors +{ + public class VectorDatabaseFactory : IVectorDatabaseFactory + { + private readonly IServiceProvider _serviceProvider; + private readonly VectorDatabaseSettings _settings; + private readonly ILogger _logger; + + public VectorDatabaseFactory( + IServiceProvider serviceProvider, + IOptions settings, + ILogger logger) + { + _serviceProvider = serviceProvider; + _settings = settings.Value; + _logger = logger; + } + + public string GetActiveProvider() + { + return _settings.Provider; + } + + public IVectorSearchService CreateVectorSearchService() + { + _logger.LogInformation("Criando VectorSearchService para provider: {Provider}", _settings.Provider); + + return _settings.Provider.ToLower() switch + { + "qdrant" => GetService(), + "mongodb" => GetService(), + "chroma" => GetService(), + _ => throw new ArgumentException($"Provider de VectorSearch não suportado: {_settings.Provider}") + }; + } + + private T GetService() where T : class + { + var service = _serviceProvider.GetService(); + if (service == null) + { + throw new InvalidOperationException($"Serviço {typeof(T).Name} não está registrado no DI container"); + } + return service; + } + } +} diff --git a/Services/TextServices/ChromaTextDataService.cs b/Services/TextServices/ChromaTextDataService.cs new file mode 100644 index 0000000..5c205cd --- /dev/null +++ b/Services/TextServices/ChromaTextDataService.cs @@ -0,0 +1,537 @@ +using ChatRAG.Contracts.VectorSearch; +using ChatRAG.Models; +using ChatRAG.Services.Contracts; +using ChatRAG.Data; +using Microsoft.SemanticKernel.Embeddings; +using System.Text; + +#pragma warning disable SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. +namespace ChatRAG.Services +{ + // ======================================== + // CHROMA TEXT DATA SERVICE - IMPLEMENTAÇÃO COMPLETA + // ======================================== + + public class ChromaTextDataService : ITextDataService + { + private readonly IVectorSearchService _vectorSearchService; + private readonly ITextEmbeddingGenerationService _embeddingService; + private readonly ILogger _logger; + + public ChromaTextDataService( + IVectorSearchService vectorSearchService, + ITextEmbeddingGenerationService embeddingService, + ILogger logger) + { + _vectorSearchService = vectorSearchService; + _embeddingService = embeddingService; + _logger = logger; + } + + public string ProviderName => "Chroma"; + + // ======================================== + // MÉTODOS ORIGINAIS (compatibilidade com MongoDB) + // ======================================== + + public async Task SalvarNoMongoDB(string titulo, string texto, string projectId) + { + await SalvarNoMongoDB(null, titulo, texto, projectId); + } + + public async Task SalvarNoMongoDB(string? id, string titulo, string texto, string projectId) + { + try + { + var conteudo = $"**{titulo}** \n\n {texto}"; + + // Gera embedding + var embedding = await _embeddingService.GenerateEmbeddingAsync(conteudo); + var embeddingArray = embedding.ToArray().Select(e => (double)e).ToArray(); + + if (string.IsNullOrEmpty(id)) + { + // Cria novo documento + await _vectorSearchService.AddDocumentAsync(titulo, texto, projectId, embeddingArray); + _logger.LogDebug("Documento '{Title}' criado no Chroma", titulo); + } + else + { + // Atualiza documento existente + await _vectorSearchService.UpdateDocumentAsync(id, titulo, texto, projectId, embeddingArray); + _logger.LogDebug("Documento '{Id}' atualizado no Chroma", id); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao salvar documento '{Title}' no Chroma", titulo); + throw; + } + } + + public async Task SalvarTextoComEmbeddingNoMongoDB(string textoCompleto, string projectId) + { + try + { + var textoArray = new List(); + string[] textolinhas = textoCompleto.Split( + new string[] { "\n" }, + StringSplitOptions.None + ); + + var title = textolinhas[0]; + var builder = new StringBuilder(); + + foreach (string line in textolinhas) + { + if (line.StartsWith("**") || line.StartsWith("\r**")) + { + if (builder.Length > 0) + { + textoArray.Add(title.Replace("**", "").Replace("\r", "") + ": " + Environment.NewLine + builder.ToString()); + builder = new StringBuilder(); + title = line; + } + } + else + { + builder.AppendLine(line); + } + } + + // Adiciona último bloco se houver + if (builder.Length > 0) + { + textoArray.Add(title.Replace("**", "").Replace("\r", "") + ": " + Environment.NewLine + builder.ToString()); + } + + // Processa cada seção + foreach (var item in textoArray) + { + await SalvarNoMongoDB(title.Replace("**", "").Replace("\r", ""), item, projectId); + } + + _logger.LogInformation("Texto completo processado: {SectionCount} seções salvas no Chroma", textoArray.Count); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao processar texto completo no Chroma"); + throw; + } + } + + public async Task> GetAll() + { + try + { + // Busca todos os projetos e depois todos os documentos + var allDocuments = new List(); + + // Como Chroma não tem um "GetAll" direto, vamos usar scroll + // Isso é uma limitação vs MongoDB, mas é mais eficiente + var projects = await GetAllProjectIds(); + + foreach (var projectId in projects) + { + var projectDocs = await _vectorSearchService.GetDocumentsByProjectAsync(projectId); + allDocuments.AddRange(projectDocs); + } + + return allDocuments.Select(ConvertToTextoComEmbedding); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao recuperar todos os documentos do Chroma"); + throw; + } + } + + public async Task> GetByPorjectId(string projectId) + { + try + { + var documents = await _vectorSearchService.GetDocumentsByProjectAsync(projectId); + return documents.Select(ConvertToTextoComEmbedding); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao recuperar documentos do projeto {ProjectId} no Chroma", projectId); + throw; + } + } + + public async Task GetById(string id) + { + try + { + var document = await _vectorSearchService.GetDocumentAsync(id); + if (document == null) + { + throw new ArgumentException($"Documento {id} não encontrado no Chroma"); + } + + return ConvertToTextoComEmbedding(document); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao recuperar documento {Id} do Chroma", id); + throw; + } + } + + // ======================================== + // MÉTODOS NOVOS DA INTERFACE + // ======================================== + + public async Task SaveDocumentAsync(DocumentInput document) + { + try + { + var conteudo = $"**{document.Title}** \n\n {document.Content}"; + var embedding = await _embeddingService.GenerateEmbeddingAsync(conteudo); + var embeddingArray = embedding.ToArray().Select(e => (double)e).ToArray(); + + string id; + if (!string.IsNullOrEmpty(document.Id)) + { + // Atualizar documento existente + await _vectorSearchService.UpdateDocumentAsync( + document.Id, + document.Title, + document.Content, + document.ProjectId, + embeddingArray, + document.Metadata); + id = document.Id; + } + else + { + // Criar novo documento + id = await _vectorSearchService.AddDocumentAsync( + document.Title, + document.Content, + document.ProjectId, + embeddingArray, + document.Metadata); + } + + _logger.LogDebug("Documento {Id} salvo no Chroma via SaveDocumentAsync", id); + return id; + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao salvar documento no Chroma"); + throw; + } + } + + public async Task UpdateDocumentAsync(string id, DocumentInput document) + { + try + { + var conteudo = $"**{document.Title}** \n\n {document.Content}"; + var embedding = await _embeddingService.GenerateEmbeddingAsync(conteudo); + var embeddingArray = embedding.ToArray().Select(e => (double)e).ToArray(); + + await _vectorSearchService.UpdateDocumentAsync( + id, + document.Title, + document.Content, + document.ProjectId, + embeddingArray, + document.Metadata); + + _logger.LogDebug("Documento {Id} atualizado no Chroma", id); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao atualizar documento {Id} no Chroma", id); + throw; + } + } + + public async Task DeleteDocumentAsync(string id) + { + try + { + await _vectorSearchService.DeleteDocumentAsync(id); + _logger.LogDebug("Documento {Id} deletado do Chroma", id); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao deletar documento {Id} do Chroma", id); + throw; + } + } + + public async Task DocumentExistsAsync(string id) + { + try + { + return await _vectorSearchService.DocumentExistsAsync(id); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao verificar existência do documento {Id} no Chroma", id); + return false; + } + } + + public async Task GetDocumentAsync(string id) + { + try + { + var result = await _vectorSearchService.GetDocumentAsync(id); + if (result == null) return null; + + return new DocumentOutput + { + Id = result.Id, + Title = result.Metadata?.GetValueOrDefault("title")?.ToString() ?? "", + Content = result.Content, + ProjectId = result.Metadata?.GetValueOrDefault("project_id")?.ToString() ?? "", + Embedding = Array.Empty(), // Chroma não retorna embedding na busca + CreatedAt = ParseDateTime(result.Metadata?.GetValueOrDefault("created_at")?.ToString()).Value, + UpdatedAt = ParseDateTime(result.Metadata?.GetValueOrDefault("updated_at")?.ToString()).Value, + Metadata = result.Metadata + }; + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao recuperar documento {Id} do Chroma", id); + return null; + } + } + + public async Task> GetDocumentsByProjectAsync(string projectId) + { + try + { + var results = await _vectorSearchService.GetDocumentsByProjectAsync(projectId); + + return results.Select(result => new DocumentOutput + { + Id = result.Id, + Title = result.Metadata?.GetValueOrDefault("title")?.ToString() ?? "", + Content = result.Content, + ProjectId = projectId, + Embedding = Array.Empty(), + CreatedAt = ParseDateTime(result.Metadata?.GetValueOrDefault("created_at")?.ToString()).Value, + UpdatedAt = ParseDateTime(result.Metadata?.GetValueOrDefault("updated_at")?.ToString()).Value, + Metadata = result.Metadata + }).ToList(); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao recuperar documentos do projeto {ProjectId} do Chroma", projectId); + throw; + } + } + + public async Task GetDocumentCountAsync(string? projectId = null) + { + try + { + return await _vectorSearchService.GetDocumentCountAsync(projectId); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao contar documentos no Chroma"); + return 0; + } + } + + // ======================================== + // OPERAÇÕES EM LOTE + // ======================================== + + public async Task> SaveDocumentsBatchAsync(List documents) + { + var ids = new List(); + var errors = new List(); + + // Processa em lotes menores para performance + var batchSize = 10; + for (int i = 0; i < documents.Count; i += batchSize) + { + var batch = documents.Skip(i).Take(batchSize); + var tasks = batch.Select(async doc => + { + try + { + var id = await SaveDocumentAsync(doc); + return id; + } + catch (Exception ex) + { + errors.Add(ex); + _logger.LogError(ex, "Erro ao salvar documento '{Title}' em lote", doc.Title); + return null; + } + }); + + var batchResults = await Task.WhenAll(tasks); + ids.AddRange(batchResults.Where(id => id != null)!); + } + + if (errors.Any()) + { + _logger.LogWarning("Batch save completado com {ErrorCount} erros de {TotalCount} documentos", + errors.Count, documents.Count); + } + + _logger.LogInformation("Batch save: {SuccessCount}/{TotalCount} documentos salvos no Chroma", + ids.Count, documents.Count); + + return ids; + } + + public async Task DeleteDocumentsBatchAsync(List ids) + { + var errors = new List(); + + // Processa em lotes para não sobrecarregar + var batchSize = 20; + for (int i = 0; i < ids.Count; i += batchSize) + { + var batch = ids.Skip(i).Take(batchSize); + var tasks = batch.Select(async id => + { + try + { + await DeleteDocumentAsync(id); + return true; + } + catch (Exception ex) + { + errors.Add(ex); + _logger.LogError(ex, "Erro ao deletar documento {Id} em lote", id); + return false; + } + }); + + await Task.WhenAll(tasks); + } + + if (errors.Any()) + { + _logger.LogWarning("Batch delete completado com {ErrorCount} erros de {TotalCount} documentos", + errors.Count, ids.Count); + } + else + { + _logger.LogInformation("Batch delete: {TotalCount} documentos removidos do Chroma", ids.Count); + } + } + + // ======================================== + // ESTATÍSTICAS DO PROVIDER + // ======================================== + + public async Task> GetProviderStatsAsync() + { + try + { + var baseStats = await _vectorSearchService.GetStatsAsync(); + var totalDocs = await GetDocumentCountAsync(); + + // Adiciona estatísticas específicas do TextData + var projectIds = await GetAllProjectIds(); + var projectStats = new Dictionary(); + + foreach (var projectId in projectIds) + { + var count = await GetDocumentCountAsync(projectId); + projectStats[projectId] = count; + } + + var enhancedStats = new Dictionary(baseStats) + { + ["text_service_provider"] = "Chroma", + ["total_documents_via_text_service"] = totalDocs, + ["projects_count"] = projectIds.Count, + ["documents_by_project"] = projectStats, + ["supports_batch_operations"] = true, + ["supports_metadata"] = true, + ["embedding_auto_generation"] = true + }; + + return enhancedStats; + } + catch (Exception ex) + { + return new Dictionary + { + ["provider"] = "Chroma", + ["text_service_provider"] = "Chroma", + ["health"] = "error", + ["error"] = ex.Message, + ["last_check"] = DateTime.UtcNow + }; + } + } + + // ======================================== + // MÉTODOS AUXILIARES PRIVADOS + // ======================================== + + private static TextoComEmbedding ConvertToTextoComEmbedding(VectorSearchResult result) + { + return new TextoComEmbedding + { + Id = result.Id, + Titulo = result.Metadata?.GetValueOrDefault("title")?.ToString() ?? "", + Conteudo = result.Content, + ProjetoId = result.Metadata?.GetValueOrDefault("project_id")?.ToString() ?? "", + Embedding = Array.Empty(), // Chroma não retorna embedding na busca + // Campos que podem não existir no Chroma + ProjetoNome = result.Metadata?.GetValueOrDefault("project_name")?.ToString() ?? "", + TipoDocumento = result.Metadata?.GetValueOrDefault("document_type")?.ToString() ?? "", + Categoria = result.Metadata?.GetValueOrDefault("category")?.ToString() ?? "", + Tags = result.Metadata?.GetValueOrDefault("tags") as string[] ?? Array.Empty() + }; + } + + private async Task> GetAllProjectIds() + { + try + { + // Esta é uma operação custosa no Chroma + // Em produção, seria melhor manter um cache de project IDs + // ou usar uma estrutura de dados separada + + // Por agora, vamos usar uma busca com um vetor dummy para pegar todos os documentos + var dummyVector = new double[384]; // Assumindo embeddings padrão + var allResults = await _vectorSearchService.SearchSimilarAsync( + dummyVector, + projectId: null, + threshold: 0.0, + limit: 10000); + + return allResults + .Select(r => r.Metadata?.GetValueOrDefault("project_id")?.ToString()) + .Where(pid => !string.IsNullOrEmpty(pid)) + .Distinct() + .ToList()!; + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao recuperar IDs de projetos do Chroma"); + return new List(); + } + } + + private static DateTime? ParseDateTime(string? dateString) + { + if (string.IsNullOrEmpty(dateString)) + return null; + + if (DateTime.TryParse(dateString, out var date)) + return date; + + return null; + } + } +} +#pragma warning restore SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + diff --git a/Services/TextServices/QdrantTextDataService.cs b/Services/TextServices/QdrantTextDataService.cs index f4a034a..9064a61 100644 --- a/Services/TextServices/QdrantTextDataService.cs +++ b/Services/TextServices/QdrantTextDataService.cs @@ -6,6 +6,7 @@ using ChatRAG.Models; using ChatRAG.Services.Contracts; using Microsoft.SemanticKernel.Embeddings; using System.Text; +using System.Collections.Concurrent; namespace ChatRAG.Services.TextServices { @@ -15,6 +16,10 @@ namespace ChatRAG.Services.TextServices private readonly ITextEmbeddingGenerationService _embeddingService; private readonly ILogger _logger; + // Cache para project IDs para evitar buscas custosas + private readonly ConcurrentDictionary _projectIdCache = new(); + private readonly TimeSpan _cacheTimeout = TimeSpan.FromMinutes(5); + public QdrantTextDataService( IVectorSearchService vectorSearchService, ITextEmbeddingGenerationService embeddingService, @@ -42,20 +47,23 @@ namespace ChatRAG.Services.TextServices { var conteudo = $"**{titulo}** \n\n {texto}"; - // Gera embedding - var embedding = await _embeddingService.GenerateEmbeddingAsync(conteudo); - var embeddingArray = embedding.ToArray().Select(e => (double)e).ToArray(); + // Gera embedding uma única vez + var embedding = await GenerateEmbeddingOptimized(conteudo); if (string.IsNullOrEmpty(id)) { // Cria novo documento - await _vectorSearchService.AddDocumentAsync(titulo, texto, projectId, embeddingArray); - _logger.LogDebug("Documento '{Title}' criado no Qdrant", titulo); + var newId = await _vectorSearchService.AddDocumentAsync(titulo, texto, projectId, embedding); + + // Atualiza cache de project IDs + _projectIdCache.TryAdd(projectId, DateTime.UtcNow); + + _logger.LogDebug("Documento '{Title}' criado no Qdrant com ID {Id}", titulo, newId); } else { // Atualiza documento existente - await _vectorSearchService.UpdateDocumentAsync(id, titulo, texto, projectId, embeddingArray); + await _vectorSearchService.UpdateDocumentAsync(id, titulo, texto, projectId, embedding); _logger.LogDebug("Documento '{Id}' atualizado no Qdrant", id); } } @@ -70,43 +78,29 @@ namespace ChatRAG.Services.TextServices { try { - var textoArray = new List(); - string[] textolinhas = textoCompleto.Split( - new string[] { "\n" }, - StringSplitOptions.None - ); + var textoArray = ParseTextIntoSections(textoCompleto); - var title = textolinhas[0]; - var builder = new StringBuilder(); - - foreach (string line in textolinhas) + // Processa seções em paralelo com limite de concorrência + var semaphore = new SemaphoreSlim(5, 5); // Máximo 5 operações simultâneas + var tasks = textoArray.Select(async item => { - if (line.StartsWith("**") || line.StartsWith("\r**")) + await semaphore.WaitAsync(); + try { - if (builder.Length > 0) - { - textoArray.Add(title.Replace("**", "").Replace("\r", "") + ": " + Environment.NewLine + builder.ToString()); - builder = new StringBuilder(); - title = line; - } + var lines = item.Split('\n', 2); + var title = lines[0].Replace("**", "").Replace("\r", "").Trim(); + var content = lines.Length > 1 ? lines[1] : ""; + + await SalvarNoMongoDB(title, content, projectId); } - else + finally { - builder.AppendLine(line); + semaphore.Release(); } - } + }); - // Adiciona último bloco se houver - if (builder.Length > 0) - { - textoArray.Add(title.Replace("**", "").Replace("\r", "") + ": " + Environment.NewLine + builder.ToString()); - } - - // Processa cada seção - foreach (var item in textoArray) - { - await SalvarNoMongoDB(title.Replace("**", "").Replace("\r", ""), item, projectId); - } + await Task.WhenAll(tasks); + semaphore.Dispose(); _logger.LogInformation("Texto completo processado: {SectionCount} seções salvas no Qdrant", textoArray.Count); } @@ -121,16 +115,41 @@ namespace ChatRAG.Services.TextServices { try { - // Busca todos os projetos e depois todos os documentos + // Usa cache de project IDs quando possível + var projectIds = await GetAllProjectIdsOptimized(); + + if (!projectIds.Any()) + { + return Enumerable.Empty(); + } + var allDocuments = new List(); - // Como Qdrant não tem um "GetAll" direto, vamos usar scroll - // Isso é uma limitação vs MongoDB, mas é mais eficiente - var projects = await GetAllProjectIds(); - - foreach (var projectId in projects) + // Busca documentos em paralelo por projeto + var semaphore = new SemaphoreSlim(3, 3); // Máximo 3 projetos simultâneos + var tasks = projectIds.Select(async projectId => + { + await semaphore.WaitAsync(); + try + { + return await _vectorSearchService.GetDocumentsByProjectAsync(projectId); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Erro ao buscar documentos do projeto {ProjectId}", projectId); + return new List(); + } + finally + { + semaphore.Release(); + } + }); + + var results = await Task.WhenAll(tasks); + semaphore.Dispose(); + + foreach (var projectDocs in results) { - var projectDocs = await _vectorSearchService.GetDocumentsByProjectAsync(projectId); allDocuments.AddRange(projectDocs); } @@ -184,9 +203,7 @@ namespace ChatRAG.Services.TextServices { try { - var conteudo = $"**{document.Title}** \n\n {document.Content}"; - var embedding = await _embeddingService.GenerateEmbeddingAsync(conteudo); - var embeddingArray = embedding.ToArray().Select(e => (double)e).ToArray(); + var embedding = await GenerateEmbeddingOptimized($"**{document.Title}** \n\n {document.Content}"); string id; if (!string.IsNullOrEmpty(document.Id)) @@ -197,7 +214,7 @@ namespace ChatRAG.Services.TextServices document.Title, document.Content, document.ProjectId, - embeddingArray, + embedding, document.Metadata); id = document.Id; } @@ -208,10 +225,13 @@ namespace ChatRAG.Services.TextServices document.Title, document.Content, document.ProjectId, - embeddingArray, + embedding, document.Metadata); } + // Atualiza cache de project IDs + _projectIdCache.TryAdd(document.ProjectId, DateTime.UtcNow); + _logger.LogDebug("Documento {Id} salvo no Qdrant via SaveDocumentAsync", id); return id; } @@ -226,16 +246,14 @@ namespace ChatRAG.Services.TextServices { try { - var conteudo = $"**{document.Title}** \n\n {document.Content}"; - var embedding = await _embeddingService.GenerateEmbeddingAsync(conteudo); - var embeddingArray = embedding.ToArray().Select(e => (double)e).ToArray(); + var embedding = await GenerateEmbeddingOptimized($"**{document.Title}** \n\n {document.Content}"); await _vectorSearchService.UpdateDocumentAsync( id, document.Title, document.Content, document.ProjectId, - embeddingArray, + embedding, document.Metadata); _logger.LogDebug("Documento {Id} atualizado no Qdrant", id); @@ -339,7 +357,7 @@ namespace ChatRAG.Services.TextServices } // ======================================== - // OPERAÇÕES EM LOTE + // OPERAÇÕES EM LOTE OTIMIZADAS // ======================================== public async Task> SaveDocumentsBatchAsync(List documents) @@ -347,28 +365,83 @@ namespace ChatRAG.Services.TextServices var ids = new List(); var errors = new List(); - // Processa em lotes menores para performance - var batchSize = 10; - for (int i = 0; i < documents.Count; i += batchSize) - { - var batch = documents.Skip(i).Take(batchSize); - var tasks = batch.Select(async doc => - { - try - { - var id = await SaveDocumentAsync(doc); - return id; - } - catch (Exception ex) - { - errors.Add(ex); - _logger.LogError(ex, "Erro ao salvar documento '{Title}' em lote", doc.Title); - return null; - } - }); + // Agrupa documentos por projeto para otimizar embeddings + var documentsByProject = documents.GroupBy(d => d.ProjectId).ToList(); - var batchResults = await Task.WhenAll(tasks); - ids.AddRange(batchResults.Where(id => id != null)!); + foreach (var projectGroup in documentsByProject) + { + var projectDocs = projectGroup.ToList(); + + // Processa em lotes menores dentro do projeto + var batchSize = 5; // Reduzido para evitar timeout + for (int i = 0; i < projectDocs.Count; i += batchSize) + { + var batch = projectDocs.Skip(i).Take(batchSize); + + // Gera embeddings em paralelo para o lote + var embeddingTasks = batch.Select(async doc => + { + try + { + var embedding = await GenerateEmbeddingOptimized($"**{doc.Title}** \n\n {doc.Content}"); + return new { Document = doc, Embedding = embedding, Error = (Exception?)null }; + } + catch (Exception ex) + { + return new { Document = doc, Embedding = (double[]?)null, Error = ex }; + } + }); + + var embeddingResults = await Task.WhenAll(embeddingTasks); + + // Salva documentos com embeddings gerados + var saveTasks = embeddingResults.Select(async result => + { + if (result.Error != null) + { + errors.Add(result.Error); + return null; + } + + try + { + string id; + if (!string.IsNullOrEmpty(result.Document.Id)) + { + await _vectorSearchService.UpdateDocumentAsync( + result.Document.Id, + result.Document.Title, + result.Document.Content, + result.Document.ProjectId, + result.Embedding!, + result.Document.Metadata); + id = result.Document.Id; + } + else + { + id = await _vectorSearchService.AddDocumentAsync( + result.Document.Title, + result.Document.Content, + result.Document.ProjectId, + result.Embedding!, + result.Document.Metadata); + } + return id; + } + catch (Exception ex) + { + errors.Add(ex); + _logger.LogError(ex, "Erro ao salvar documento '{Title}' em lote", result.Document.Title); + return null; + } + }); + + var batchResults = await Task.WhenAll(saveTasks); + ids.AddRange(batchResults.Where(id => id != null)!); + } + + // Atualiza cache para o projeto + _projectIdCache.TryAdd(projectGroup.Key, DateTime.UtcNow); } if (errors.Any()) @@ -387,8 +460,8 @@ namespace ChatRAG.Services.TextServices { var errors = new List(); - // Processa em lotes para não sobrecarregar - var batchSize = 20; + // Processa em lotes pequenos para não sobrecarregar + var batchSize = 10; // Reduzido para melhor estabilidade for (int i = 0; i < ids.Count; i += batchSize) { var batch = ids.Skip(i).Take(batchSize); @@ -396,7 +469,7 @@ namespace ChatRAG.Services.TextServices { try { - await DeleteDocumentAsync(id); + await _vectorSearchService.DeleteDocumentAsync(id); return true; } catch (Exception ex) @@ -432,14 +505,28 @@ namespace ChatRAG.Services.TextServices var baseStats = await _vectorSearchService.GetStatsAsync(); var totalDocs = await GetDocumentCountAsync(); - // Adiciona estatísticas específicas do TextData - var projectIds = await GetAllProjectIds(); + // Usa cache para project IDs + var projectIds = await GetAllProjectIdsOptimized(); var projectStats = new Dictionary(); - foreach (var projectId in projectIds) + // Busca contadores em paralelo + var countTasks = projectIds.Select(async projectId => { - var count = await GetDocumentCountAsync(projectId); - projectStats[projectId] = count; + try + { + var count = await GetDocumentCountAsync(projectId); + return new { ProjectId = projectId, Count = count }; + } + catch + { + return new { ProjectId = projectId, Count = 0 }; + } + }); + + var countResults = await Task.WhenAll(countTasks); + foreach (var result in countResults) + { + projectStats[result.ProjectId] = result.Count; } var enhancedStats = new Dictionary(baseStats) @@ -450,7 +537,9 @@ namespace ChatRAG.Services.TextServices ["documents_by_project"] = projectStats, ["supports_batch_operations"] = true, ["supports_metadata"] = true, - ["embedding_auto_generation"] = true + ["embedding_auto_generation"] = true, + ["cache_enabled"] = true, + ["cached_project_ids"] = _projectIdCache.Count }; return enhancedStats; @@ -469,9 +558,100 @@ namespace ChatRAG.Services.TextServices } // ======================================== - // MÉTODOS AUXILIARES PRIVADOS + // MÉTODOS AUXILIARES PRIVADOS OTIMIZADOS // ======================================== + private async Task GenerateEmbeddingOptimized(string content) + { + var embedding = await _embeddingService.GenerateEmbeddingAsync(content); + return embedding.ToArray().Select(e => (double)e).ToArray(); + } + + private static List ParseTextIntoSections(string textoCompleto) + { + var textoArray = new List(); + string[] textolinhas = textoCompleto.Split(new string[] { "\n" }, StringSplitOptions.None); + + var title = textolinhas[0]; + var builder = new StringBuilder(); + + foreach (string line in textolinhas) + { + if (line.StartsWith("**") || line.StartsWith("\r**")) + { + if (builder.Length > 0) + { + textoArray.Add(title.Replace("**", "").Replace("\r", "") + ": " + Environment.NewLine + builder.ToString()); + builder = new StringBuilder(); + title = line; + } + } + else + { + builder.AppendLine(line); + } + } + + // Adiciona último bloco se houver + if (builder.Length > 0) + { + textoArray.Add(title.Replace("**", "").Replace("\r", "") + ": " + Environment.NewLine + builder.ToString()); + } + + return textoArray; + } + + private async Task> GetAllProjectIdsOptimized() + { + // Remove entradas expiradas do cache + var now = DateTime.UtcNow; + var expiredKeys = _projectIdCache + .Where(kvp => now - kvp.Value > _cacheTimeout) + .Select(kvp => kvp.Key) + .ToList(); + + foreach (var key in expiredKeys) + { + _projectIdCache.TryRemove(key, out _); + } + + // Se temos dados no cache e não estão muito antigos, usa o cache + if (_projectIdCache.Any()) + { + return _projectIdCache.Keys.ToList(); + } + + // Caso contrário, busca no Qdrant + try + { + // Esta busca é custosa, mas só será executada quando o cache estiver vazio + var allResults = await _vectorSearchService.SearchSimilarAsync( + new double[384], // Vector dummy menor + projectId: null, + threshold: 0.0, + limit: 1000); // Limit menor para melhor performance + + var projectIds = allResults + .Select(r => r.ProjectId) + .Where(pid => !string.IsNullOrEmpty(pid)) + .Distinct() + .ToList(); + + // Atualiza cache + foreach (var projectId in projectIds) + { + _projectIdCache.TryAdd(projectId, now); + } + + return projectIds; + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao recuperar IDs de projetos do Qdrant"); + return new List(); + } + } + private static TextoComEmbedding ConvertToTextoComEmbedding(VectorSearchResult result) { return new TextoComEmbedding @@ -488,35 +668,6 @@ namespace ChatRAG.Services.TextServices Tags = result.Metadata?.GetValueOrDefault("tags") as string[] ?? Array.Empty() }; } - - private async Task> GetAllProjectIds() - { - try - { - // Esta é uma operação custosa no Qdrant - // Em produção, seria melhor manter um cache de project IDs - // ou usar uma estrutura de dados separada - - // Por agora, vamos usar uma busca com um vetor dummy para pegar todos os documentos - var dummyVector = new double[1536]; // Assumindo embeddings OpenAI - var allResults = await _vectorSearchService.SearchSimilarAsync( - dummyVector, - projectId: null, - threshold: 0.0, - limit: 10000); - - return allResults - .Select(r => r.ProjectId) - .Where(pid => !string.IsNullOrEmpty(pid)) - .Distinct() - .ToList(); - } - catch (Exception ex) - { - _logger.LogError(ex, "Erro ao recuperar IDs de projetos do Qdrant"); - return new List(); - } - } } } diff --git a/Settings/VectorDatabaseSettings.cs b/Settings/VectorDatabaseSettings.cs index 5be374c..0d7f601 100644 --- a/Settings/VectorDatabaseSettings.cs +++ b/Settings/VectorDatabaseSettings.cs @@ -1,156 +1,59 @@ -namespace ChatRAG.Settings +using Microsoft.Extensions.AI; +using Qdrant.Client.Grpc; + +namespace ChatRAG.Settings.ChatRAG.Configuration { - // ============================================================================ - // 📁 Configuration/VectorDatabaseSettings.cs - // Settings unificados para todos os providers (MongoDB, Qdrant, etc.) - // ============================================================================ - - namespace ChatRAG.Configuration + public class VectorDatabaseSettings { + public string Provider { get; set; } = "Qdrant"; + public MongoDBSettings? MongoDB { get; set; } + public QdrantSettings? Qdrant { get; set; } + public ChromaSettings? Chroma { get; set; } + public EmbeddingSettings Embedding { get; set; } = new(); + /// - /// Configurações principais do sistema de Vector Database + /// Retorna erros de validação /// - public class VectorDatabaseSettings + public List GetValidationErrors() { - /// - /// Provider ativo (MongoDB, Qdrant, Pinecone, etc.) - /// - public string Provider { get; set; } = "MongoDB"; + var errors = new List(); - /// - /// Configurações específicas do MongoDB - /// - public MongoDbSettings MongoDB { get; set; } = new(); + if (string.IsNullOrWhiteSpace(Provider)) + errors.Add("Provider é obrigatório"); - /// - /// Configurações específicas do Qdrant - /// - public QdrantSettings Qdrant { get; set; } = new(); - - /// - /// Configurações globais de embedding - /// - public EmbeddingSettings Embedding { get; set; } = new(); - - /// - /// Configurações de performance e cache - /// - public PerformanceSettings Performance { get; set; } = new(); - - /// - /// Configurações de logging e monitoramento - /// - public MonitoringSettings Monitoring { get; set; } = new(); - - /// - /// Valida se as configurações estão corretas - /// - public bool IsValid() + switch (Provider.ToLower()) { - if (string.IsNullOrWhiteSpace(Provider)) - return false; - - return Provider.ToLower() switch - { - "mongodb" => MongoDB.IsValid(), - "qdrant" => Qdrant.IsValid(), - _ => false - }; + case "mongodb": + errors.AddRange(MongoDB.GetValidationErrors()); + break; + case "qdrant": + errors.AddRange(Qdrant.GetValidationErrors()); + break; + case "chroma": + errors.AddRange(Chroma.GetValidationErrors()); + break; + default: + errors.Add($"Provider '{Provider}' não é suportado"); + break; } - /// - /// Retorna erros de validação - /// - public List GetValidationErrors() - { - var errors = new List(); + errors.AddRange(Embedding.GetValidationErrors()); - if (string.IsNullOrWhiteSpace(Provider)) - errors.Add("Provider é obrigatório"); - - switch (Provider.ToLower()) - { - case "mongodb": - errors.AddRange(MongoDB.GetValidationErrors()); - break; - case "qdrant": - errors.AddRange(Qdrant.GetValidationErrors()); - break; - default: - errors.Add($"Provider '{Provider}' não é suportado"); - break; - } - - errors.AddRange(Embedding.GetValidationErrors()); - - return errors; - } + return errors; } + } - /// - /// Configurações específicas do MongoDB - /// - public class MongoDbSettings + public class MongoDBSettings + { + public string ConnectionString { get; set; } = ""; + public string DatabaseName { get; set; } = ""; + public string TextCollectionName { get; set; } = "Texts"; + public string ProjectCollectionName { get; set; } = "Groups"; + public string UserDataName { get; set; } = "UserData"; + public int ConnectionTimeoutSeconds { get; set; } = 30; + + public List GetValidationErrors() { - /// - /// String de conexão do MongoDB - /// - public string ConnectionString { get; set; } = string.Empty; - - /// - /// Nome do banco de dados - /// - public string DatabaseName { get; set; } = string.Empty; - - /// - /// Nome da coleção de textos/documentos - /// - public string TextCollectionName { get; set; } = "texts"; - - /// - /// Nome da coleção de projetos - /// - public string ProjectCollectionName { get; set; } = "projects"; - - /// - /// Nome da coleção de dados de usuário - /// - public string UserDataName { get; set; } = "users"; - - /// - /// Timeout de conexão em segundos - /// - public int ConnectionTimeoutSeconds { get; set; } = 30; - - /// - /// Timeout de operação em segundos - /// - public int OperationTimeoutSeconds { get; set; } = 60; - - /// - /// Tamanho máximo do pool de conexões - /// - public int MaxConnectionPoolSize { get; set; } = 100; - - /// - /// Habilitar índices de busca vetorial - /// - public bool EnableVectorSearch { get; set; } = true; - - /// - /// Configurações específicas do Atlas Search - /// - public MongoAtlasSearchSettings AtlasSearch { get; set; } = new(); - - public bool IsValid() - { - return !string.IsNullOrWhiteSpace(ConnectionString) && - !string.IsNullOrWhiteSpace(DatabaseName) && - !string.IsNullOrWhiteSpace(TextCollectionName); - } - - public List GetValidationErrors() - { var errors = new List(); if (string.IsNullOrWhiteSpace(ConnectionString)) @@ -166,329 +69,110 @@ errors.Add("MongoDB ConnectionTimeoutSeconds deve ser maior que 0"); return errors; - } - } - - /// - /// Configurações do MongoDB Atlas Search - /// - public class MongoAtlasSearchSettings - { - /// - /// Nome do índice de busca vetorial - /// - public string VectorIndexName { get; set; } = "vector_index"; - - /// - /// Número de candidatos para busca aproximada - /// - public int NumCandidates { get; set; } = 200; - - /// - /// Limite de resultados do Atlas Search - /// - public int SearchLimit { get; set; } = 100; - } - - /// - /// Configurações específicas do Qdrant - /// - public class QdrantSettings - { - /// - /// Host do servidor Qdrant - /// - public string Host { get; set; } = "localhost"; - - /// - /// Porta do servidor Qdrant (REST API) - /// - public int Port { get; set; } = 6333; - - /// - /// Porta gRPC (opcional, para performance) - /// - public int? GrpcPort { get; set; } = 6334; - - /// - /// Chave de API (se autenticação estiver habilitada) - /// - public string? ApiKey { get; set; } - - /// - /// Usar TLS/SSL - /// - public bool UseTls { get; set; } = false; - - /// - /// Nome da coleção principal - /// - public string CollectionName { get; set; } = "documents"; - - /// - /// Tamanho do vetor de embedding - /// - public int VectorSize { get; set; } = 1536; // OpenAI embedding size - - /// - /// Métrica de distância (Cosine, Euclid, Dot, Manhattan) - /// - public string Distance { get; set; } = "Cosine"; - - // ======================================== - // CONFIGURAÇÕES DE PERFORMANCE - // ======================================== - - /// - /// Parâmetro M do algoritmo HNSW (conectividade) - /// Valores típicos: 16-48, maior = melhor recall, mais memória - /// - public int HnswM { get; set; } = 16; - - /// - /// Parâmetro ef_construct do HNSW (construção do índice) - /// Valores típicos: 100-800, maior = melhor qualidade, mais lento - /// - public int HnswEfConstruct { get; set; } = 200; - - /// - /// Parâmetro ef do HNSW (busca) - /// Valores típicos: igual ou maior que o número de resultados desejados - /// - public int HnswEf { get; set; } = 128; - - /// - /// Armazenar vetores em disco (economiza RAM) - /// - public bool OnDisk { get; set; } = false; - - /// - /// Fator de replicação (para clusters) - /// - public int ReplicationFactor { get; set; } = 1; - - /// - /// Número de shards (para distribuição) - /// - public int ShardNumber { get; set; } = 1; - - /// - /// Usar quantização para reduzir uso de memória - /// - public bool UseQuantization { get; set; } = false; - - /// - /// Configurações de quantização - /// - public QuantizationSettings Quantization { get; set; } = new(); - - /// - /// Timeout de conexão em segundos - /// - public int ConnectionTimeoutSeconds { get; set; } = 30; - - /// - /// Timeout de operação em segundos - /// - public int OperationTimeoutSeconds { get; set; } = 60; - - /// - /// URL completa calculada - /// - public string GetConnectionUrl() - { - var protocol = UseTls ? "https" : "http"; - return $"{protocol}://{Host}:{Port}"; - } - - public bool IsValid() - { - return !string.IsNullOrWhiteSpace(Host) && - Port > 0 && - !string.IsNullOrWhiteSpace(CollectionName) && - VectorSize > 0; - } - - public List GetValidationErrors() - { - var errors = new List(); - - if (string.IsNullOrWhiteSpace(Host)) - errors.Add("Qdrant Host é obrigatório"); - - if (Port <= 0) - errors.Add("Qdrant Port deve ser maior que 0"); - - if (string.IsNullOrWhiteSpace(CollectionName)) - errors.Add("Qdrant CollectionName é obrigatório"); - - if (VectorSize <= 0) - errors.Add("Qdrant VectorSize deve ser maior que 0"); - - if (HnswM <= 0) - errors.Add("Qdrant HnswM deve ser maior que 0"); - - if (HnswEfConstruct <= 0) - errors.Add("Qdrant HnswEfConstruct deve ser maior que 0"); - - var validDistances = new[] { "Cosine", "Euclid", "Dot", "Manhattan" }; - if (!validDistances.Contains(Distance)) - errors.Add($"Qdrant Distance deve ser um de: {string.Join(", ", validDistances)}"); - - return errors; - } - } - - /// - /// Configurações de quantização para Qdrant - /// - public class QuantizationSettings - { - /// - /// Tipo de quantização (scalar, product, binary) - /// - public string Type { get; set; } = "scalar"; - - /// - /// Quantil para quantização scalar - /// - public double Quantile { get; set; } = 0.99; - - /// - /// Sempre usar RAM para quantização - /// - public bool AlwaysRam { get; set; } = false; - } - - /// - /// Configurações globais de embedding - /// - public class EmbeddingSettings - { - /// - /// Provider de embedding (OpenAI, Ollama, Azure, etc.) - /// - public string Provider { get; set; } = "OpenAI"; - - /// - /// Modelo de embedding - /// - public string Model { get; set; } = "text-embedding-ada-002"; - - /// - /// Tamanho esperado do embedding - /// - public int ExpectedSize { get; set; } = 1536; - - /// - /// Batch size para processamento em lote - /// - public int BatchSize { get; set; } = 100; - - /// - /// Cache de embeddings em memória - /// - public bool EnableCache { get; set; } = true; - - /// - /// TTL do cache em minutos - /// - public int CacheTtlMinutes { get; set; } = 60; - - public List GetValidationErrors() - { - var errors = new List(); - - if (ExpectedSize <= 0) - errors.Add("Embedding ExpectedSize deve ser maior que 0"); - - if (BatchSize <= 0) - errors.Add("Embedding BatchSize deve ser maior que 0"); - - return errors; - } - } - - /// - /// Configurações de performance e otimização - /// - public class PerformanceSettings - { - /// - /// Habilitar paralelização em operações de lote - /// - public bool EnableParallelization { get; set; } = true; - - /// - /// Número máximo de threads paralelas - /// - public int MaxParallelism { get; set; } = Environment.ProcessorCount; - - /// - /// Tamanho do batch para operações em lote - /// - public int BatchSize { get; set; } = 100; - - /// - /// Timeout padrão para operações em segundos - /// - public int DefaultTimeoutSeconds { get; set; } = 30; - - /// - /// Habilitar retry automático - /// - public bool EnableRetry { get; set; } = true; - - /// - /// Número máximo de tentativas - /// - public int MaxRetryAttempts { get; set; } = 3; - - /// - /// Delay entre tentativas em segundos - /// - public int RetryDelaySeconds { get; set; } = 2; - } - - /// - /// Configurações de monitoramento e logging - /// - public class MonitoringSettings - { - /// - /// Habilitar logging detalhado - /// - public bool EnableDetailedLogging { get; set; } = false; - - /// - /// Logar tempos de operação - /// - public bool LogOperationTimes { get; set; } = true; - - /// - /// Threshold para log de operações lentas (ms) - /// - public int SlowOperationThresholdMs { get; set; } = 1000; - - /// - /// Habilitar métricas de performance - /// - public bool EnableMetrics { get; set; } = true; - - /// - /// Intervalo de coleta de métricas em segundos - /// - public int MetricsIntervalSeconds { get; set; } = 60; - - /// - /// Habilitar health checks - /// - public bool EnableHealthChecks { get; set; } = true; - - /// - /// Intervalo de health check em segundos - /// - public int HealthCheckIntervalSeconds { get; set; } = 30; } } -} \ No newline at end of file + + public class QdrantSettings + { + public string Host { get; set; } = "localhost"; + public int Port { get; set; } = 6334; + public string CollectionName { get; set; } = "texts"; + public string GroupsCollectionName { get; set; } = "projects"; + public int VectorSize { get; set; } = 384; + public string Distance { get; set; } = "Cosine"; + public int HnswM { get; set; } = 16; + public int HnswEfConstruct { get; set; } = 200; + public bool OnDisk { get; set; } = false; + public bool UseTls { get; set; } = false; + + public List GetValidationErrors() + { + var errors = new List(); + + if (string.IsNullOrWhiteSpace(Host)) + errors.Add("Qdrant Host é obrigatório"); + + if (Port <= 0) + errors.Add("Qdrant Port deve ser maior que 0"); + + if (string.IsNullOrWhiteSpace(CollectionName)) + errors.Add("Qdrant CollectionName é obrigatório"); + + if (VectorSize <= 0) + errors.Add("Qdrant VectorSize deve ser maior que 0"); + + if (HnswM <= 0) + errors.Add("Qdrant HnswM deve ser maior que 0"); + + if (HnswEfConstruct <= 0) + errors.Add("Qdrant HnswEfConstruct deve ser maior que 0"); + + var validDistances = new[] { "Cosine", "Euclid", "Dot", "Manhattan" }; + if (!validDistances.Contains(Distance)) + errors.Add($"Qdrant Distance deve ser um de: {string.Join(", ", validDistances)}"); + + return errors; + } + } + + public class ChromaSettings + { + public string Host { get; set; } = "localhost"; + public int Port { get; set; } = 8000; + public string CollectionName { get; set; } = "rag_documents"; + public string ApiVersion { get; set; } = "v1"; + + public List GetValidationErrors() + { + var errors = new List(); + + return errors; + } + } + + public class EmbeddingSettings + { + /// + /// Provider de embedding (OpenAI, Ollama, Azure, etc.) + /// + public string Provider { get; set; } = "OpenAI"; + + /// + /// Modelo de embedding + /// + public string Model { get; set; } = "text-embedding-ada-002"; + + /// + /// Tamanho esperado do embedding + /// + public int ExpectedSize { get; set; } = 1536; + + /// + /// Batch size para processamento em lote + /// + public int BatchSize { get; set; } = 100; + + /// + /// Cache de embeddings em memória + /// + public bool EnableCache { get; set; } = true; + + /// + /// TTL do cache em minutos + /// + public int CacheTtlMinutes { get; set; } = 60; + + public List GetValidationErrors() + { + var errors = new List(); + + if (ExpectedSize <= 0) + errors.Add("Embedding ExpectedSize deve ser maior que 0"); + + if (BatchSize <= 0) + errors.Add("Embedding BatchSize deve ser maior que 0"); + + return errors; + } + } +} diff --git a/Settings/gckijn3t.ir5~ b/Settings/gckijn3t.ir5~ new file mode 100644 index 0000000..a6dea5d --- /dev/null +++ b/Settings/gckijn3t.ir5~ @@ -0,0 +1,128 @@ +using Microsoft.Extensions.AI; +using Qdrant.Client.Grpc; + +namespace ChatRAG.Settings.ChatRAG.Configuration +{ + public class VectorDatabaseSettings + { + public string Provider { get; set; } = "Qdrant"; + public MongoDBSettings? MongoDB { get; set; } + public QdrantSettings? Qdrant { get; set; } + public ChromaSettings? Chroma { get; set; } + + /// + /// Retorna erros de validação + /// + public List GetValidationErrors() + { + var errors = new List(); + + if (string.IsNullOrWhiteSpace(Provider)) + errors.Add("Provider é obrigatório"); + + switch (Provider.ToLower()) + { + case "mongodb": + errors.AddRange(MongoDB.GetValidationErrors()); + break; + case "qdrant": + errors.AddRange(Qdrant.GetValidationErrors()); + break; + default: + errors.Add($"Provider '{Provider}' não é suportado"); + break; + } + + errors.AddRange(Embedding.GetValidationErrors()); + + return errors; + } + } +} + + public class MongoDBSettings + { + public string ConnectionString { get; set; } = ""; + public string DatabaseName { get; set; } = ""; + public string TextCollectionName { get; set; } = "Texts"; + public string ProjectCollectionName { get; set; } = "Groups"; + public string UserDataName { get; set; } = "UserData"; + public int ConnectionTimeoutSeconds { get; set; } = 30; + + public List GetValidationErrors() + { + var errors = new List(); + + if (string.IsNullOrWhiteSpace(ConnectionString)) + errors.Add("MongoDB ConnectionString é obrigatória"); + + if (string.IsNullOrWhiteSpace(DatabaseName)) + errors.Add("MongoDB DatabaseName é obrigatório"); + + if (string.IsNullOrWhiteSpace(TextCollectionName)) + errors.Add("MongoDB TextCollectionName é obrigatório"); + + if (ConnectionTimeoutSeconds <= 0) + errors.Add("MongoDB ConnectionTimeoutSeconds deve ser maior que 0"); + + return errors; + } + } + + public class QdrantSettings + { + public string Host { get; set; } = "localhost"; + public int Port { get; set; } = 6334; + public string CollectionName { get; set; } = "texts"; + public int VectorSize { get; set; } = 384; + public string Distance { get; set; } = "Cosine"; + public int HnswM { get; set; } = 16; + public int HnswEfConstruct { get; set; } = 200; + public bool OnDisk { get; set; } = false; + public bool UseTls { get; set; } = false; + + public List GetValidationErrors() + { + var errors = new List(); + + if (string.IsNullOrWhiteSpace(Host)) + errors.Add("Qdrant Host é obrigatório"); + + if (Port <= 0) + errors.Add("Qdrant Port deve ser maior que 0"); + + if (string.IsNullOrWhiteSpace(CollectionName)) + errors.Add("Qdrant CollectionName é obrigatório"); + + if (VectorSize <= 0) + errors.Add("Qdrant VectorSize deve ser maior que 0"); + + if (HnswM <= 0) + errors.Add("Qdrant HnswM deve ser maior que 0"); + + if (HnswEfConstruct <= 0) + errors.Add("Qdrant HnswEfConstruct deve ser maior que 0"); + + var validDistances = new[] { "Cosine", "Euclid", "Dot", "Manhattan" }; + if (!validDistances.Contains(Distance)) + errors.Add($"Qdrant Distance deve ser um de: {string.Join(", ", validDistances)}"); + + return errors; + } + } + + public class ChromaSettings + { + public string Host { get; set; } = "localhost"; + public int Port { get; set; } = 8000; + public string CollectionName { get; set; } = "rag_documents"; + public string ApiVersion { get; set; } = "v1"; + + public List GetValidationErrors() + { + var errors = new List(); + + return errors; + } + } +} \ No newline at end of file diff --git a/Settings/ghcutjxi.wn3~ b/Settings/ghcutjxi.wn3~ new file mode 100644 index 0000000..283ffaa --- /dev/null +++ b/Settings/ghcutjxi.wn3~ @@ -0,0 +1,120 @@ +using Microsoft.Extensions.AI; + +namespace ChatRAG.Settings.ChatRAG.Configuration +{ + public class VectorDatabaseSettings + { + public string Provider { get; set; } = "Qdrant"; + public MongoDBSettings? MongoDB { get; set; } + public QdrantSettings? Qdrant { get; set; } + public ChromaSettings? Chroma { get; set; } + + /// + /// Retorna erros de validação + /// + public List GetValidationErrors() + { + var errors = new List(); + + if (string.IsNullOrWhiteSpace(Provider)) + errors.Add("Provider é obrigatório"); + + switch (Provider.ToLower()) + { + case "mongodb": + errors.AddRange(MongoDB.GetValidationErrors()); + break; + case "qdrant": + errors.AddRange(Qdrant.GetValidationErrors()); + break; + default: + errors.Add($"Provider '{Provider}' não é suportado"); + break; + } + + errors.AddRange(Embedding.GetValidationErrors()); + + return errors; + } + } +} + + public class MongoDBSettings + { + public string ConnectionString { get; set; } = ""; + public string DatabaseName { get; set; } = ""; + public string TextCollectionName { get; set; } = "Texts"; + public string ProjectCollectionName { get; set; } = "Groups"; + public string UserDataName { get; set; } = "UserData"; + public int ConnectionTimeoutSeconds { get; set; } = 30; + + public List GetValidationErrors() + { + var errors = new List(); + + if (string.IsNullOrWhiteSpace(ConnectionString)) + errors.Add("MongoDB ConnectionString é obrigatória"); + + if (string.IsNullOrWhiteSpace(DatabaseName)) + errors.Add("MongoDB DatabaseName é obrigatório"); + + if (string.IsNullOrWhiteSpace(TextCollectionName)) + errors.Add("MongoDB TextCollectionName é obrigatório"); + + if (ConnectionTimeoutSeconds <= 0) + errors.Add("MongoDB ConnectionTimeoutSeconds deve ser maior que 0"); + + return errors; + } + } + + public class QdrantSettings + { + public string Host { get; set; } = "localhost"; + public int Port { get; set; } = 6334; + public string CollectionName { get; set; } = "texts"; + public int VectorSize { get; set; } = 384; + public string Distance { get; set; } = "Cosine"; + public int HnswM { get; set; } = 16; + public int HnswEfConstruct { get; set; } = 200; + public bool OnDisk { get; set; } = false; + public bool UseTls { get; set; } = false; + + public List GetValidationErrors() + { + var errors = new List(); + + if (string.IsNullOrWhiteSpace(Host)) + errors.Add("Qdrant Host é obrigatório"); + + if (Port <= 0) + errors.Add("Qdrant Port deve ser maior que 0"); + + if (string.IsNullOrWhiteSpace(CollectionName)) + errors.Add("Qdrant CollectionName é obrigatório"); + + if (VectorSize <= 0) + errors.Add("Qdrant VectorSize deve ser maior que 0"); + + if (HnswM <= 0) + errors.Add("Qdrant HnswM deve ser maior que 0"); + + if (HnswEfConstruct <= 0) + errors.Add("Qdrant HnswEfConstruct deve ser maior que 0"); + + var validDistances = new[] { "Cosine", "Euclid", "Dot", "Manhattan" }; + if (!validDistances.Contains(Distance)) + errors.Add($"Qdrant Distance deve ser um de: {string.Join(", ", validDistances)}"); + + return errors; + } + } + + public class ChromaSettings + { + public string Host { get; set; } = "localhost"; + public int Port { get; set; } = 8000; + public string CollectionName { get; set; } = "rag_documents"; + public string ApiVersion { get; set; } = "v1"; + } +} \ No newline at end of file diff --git a/Settings/ixb5gark.btp~ b/Settings/ixb5gark.btp~ new file mode 100644 index 0000000..764ff4c --- /dev/null +++ b/Settings/ixb5gark.btp~ @@ -0,0 +1,91 @@ +using Microsoft.Extensions.AI; + +namespace ChatRAG.Settings.ChatRAG.Configuration +{ + public class VectorDatabaseSettings + { + public string Provider { get; set; } = "Qdrant"; + public MongoDBSettings? MongoDB { get; set; } + public QdrantSettings? Qdrant { get; set; } + public ChromaSettings? Chroma { get; set; } + + /// + /// Retorna erros de validação + /// + public List GetValidationErrors() + { + var errors = new List(); + + if (string.IsNullOrWhiteSpace(Provider)) + errors.Add("Provider é obrigatório"); + + switch (Provider.ToLower()) + { + case "mongodb": + errors.AddRange(MongoDB.GetValidationErrors()); + break; + case "qdrant": + errors.AddRange(Qdrant.GetValidationErrors()); + break; + default: + errors.Add($"Provider '{Provider}' não é suportado"); + break; + } + + errors.AddRange(Embedding.GetValidationErrors()); + + return errors; + } + } +} + + public class MongoDBSettings + { + public string ConnectionString { get; set; } = ""; + public string DatabaseName { get; set; } = ""; + public string TextCollectionName { get; set; } = "Texts"; + public string ProjectCollectionName { get; set; } = "Groups"; + public string UserDataName { get; set; } = "UserData"; + public int ConnectionTimeoutSeconds { get; set; } = 30; + + public List GetValidationErrors() + { + var errors = new List(); + + if (string.IsNullOrWhiteSpace(ConnectionString)) + errors.Add("MongoDB ConnectionString é obrigatória"); + + if (string.IsNullOrWhiteSpace(DatabaseName)) + errors.Add("MongoDB DatabaseName é obrigatório"); + + if (string.IsNullOrWhiteSpace(TextCollectionName)) + errors.Add("MongoDB TextCollectionName é obrigatório"); + + if (ConnectionTimeoutSeconds <= 0) + errors.Add("MongoDB ConnectionTimeoutSeconds deve ser maior que 0"); + + return errors; + } + } + + public class QdrantSettings + { + public string Host { get; set; } = "localhost"; + public int Port { get; set; } = 6334; + public string CollectionName { get; set; } = "texts"; + public int VectorSize { get; set; } = 384; + public string Distance { get; set; } = "Cosine"; + public int HnswM { get; set; } = 16; + public int HnswEfConstruct { get; set; } = 200; + public bool OnDisk { get; set; } = false; + public bool UseTls { get; set; } = false; + } + + public class ChromaSettings + { + public string Host { get; set; } = "localhost"; + public int Port { get; set; } = 8000; + public string CollectionName { get; set; } = "rag_documents"; + public string ApiVersion { get; set; } = "v1"; + } +} \ No newline at end of file diff --git a/Settings/vwuy0ebd.cjy~ b/Settings/vwuy0ebd.cjy~ new file mode 100644 index 0000000..fab97cf --- /dev/null +++ b/Settings/vwuy0ebd.cjy~ @@ -0,0 +1,174 @@ +using Microsoft.Extensions.AI; +using Qdrant.Client.Grpc; + +namespace ChatRAG.Settings.ChatRAG.Configuration +{ + public class VectorDatabaseSettings + { + public string Provider { get; set; } = "Qdrant"; + public MongoDBSettings? MongoDB { get; set; } + public QdrantSettings? Qdrant { get; set; } + public ChromaSettings? Chroma { get; set; } + public EmbeddingSettings Embedding { get; set; } = new(); + + /// + /// Retorna erros de validação + /// + public List GetValidationErrors() + { + var errors = new List(); + + if (string.IsNullOrWhiteSpace(Provider)) + errors.Add("Provider é obrigatório"); + + switch (Provider.ToLower()) + { + case "mongodb": + errors.AddRange(MongoDB.GetValidationErrors()); + break; + case "qdrant": + errors.AddRange(Qdrant.GetValidationErrors()); + break; + default: + errors.Add($"Provider '{Provider}' não é suportado"); + break; + } + + errors.AddRange(Embedding.GetValidationErrors()); + + return errors; + } + } + + public class MongoDBSettings + { + public string ConnectionString { get; set; } = ""; + public string DatabaseName { get; set; } = ""; + public string TextCollectionName { get; set; } = "Texts"; + public string ProjectCollectionName { get; set; } = "Groups"; + public string UserDataName { get; set; } = "UserData"; + public int ConnectionTimeoutSeconds { get; set; } = 30; + + public List GetValidationErrors() + { + var errors = new List(); + + if (string.IsNullOrWhiteSpace(ConnectionString)) + errors.Add("MongoDB ConnectionString é obrigatória"); + + if (string.IsNullOrWhiteSpace(DatabaseName)) + errors.Add("MongoDB DatabaseName é obrigatório"); + + if (string.IsNullOrWhiteSpace(TextCollectionName)) + errors.Add("MongoDB TextCollectionName é obrigatório"); + + if (ConnectionTimeoutSeconds <= 0) + errors.Add("MongoDB ConnectionTimeoutSeconds deve ser maior que 0"); + + return errors; + } + } + + public class QdrantSettings + { + public string Host { get; set; } = "localhost"; + public int Port { get; set; } = 6334; + public string CollectionName { get; set; } = "texts"; + public int VectorSize { get; set; } = 384; + public string Distance { get; set; } = "Cosine"; + public int HnswM { get; set; } = 16; + public int HnswEfConstruct { get; set; } = 200; + public bool OnDisk { get; set; } = false; + public bool UseTls { get; set; } = false; + + public List GetValidationErrors() + { + var errors = new List(); + + if (string.IsNullOrWhiteSpace(Host)) + errors.Add("Qdrant Host é obrigatório"); + + if (Port <= 0) + errors.Add("Qdrant Port deve ser maior que 0"); + + if (string.IsNullOrWhiteSpace(CollectionName)) + errors.Add("Qdrant CollectionName é obrigatório"); + + if (VectorSize <= 0) + errors.Add("Qdrant VectorSize deve ser maior que 0"); + + if (HnswM <= 0) + errors.Add("Qdrant HnswM deve ser maior que 0"); + + if (HnswEfConstruct <= 0) + errors.Add("Qdrant HnswEfConstruct deve ser maior que 0"); + + var validDistances = new[] { "Cosine", "Euclid", "Dot", "Manhattan" }; + if (!validDistances.Contains(Distance)) + errors.Add($"Qdrant Distance deve ser um de: {string.Join(", ", validDistances)}"); + + return errors; + } + } + + public class ChromaSettings + { + public string Host { get; set; } = "localhost"; + public int Port { get; set; } = 8000; + public string CollectionName { get; set; } = "rag_documents"; + public string ApiVersion { get; set; } = "v1"; + + public List GetValidationErrors() + { + var errors = new List(); + + return errors; + } + } + + public class EmbeddingSettings + { + /// + /// Provider de embedding (OpenAI, Ollama, Azure, etc.) + /// + public string Provider { get; set; } = "OpenAI"; + + /// + /// Modelo de embedding + /// + public string Model { get; set; } = "text-embedding-ada-002"; + + /// + /// Tamanho esperado do embedding + /// + public int ExpectedSize { get; set; } = 1536; + + /// + /// Batch size para processamento em lote + /// + public int BatchSize { get; set; } = 100; + + /// + /// Cache de embeddings em memória + /// + public bool EnableCache { get; set; } = true; + + /// + /// TTL do cache em minutos + /// + public int CacheTtlMinutes { get; set; } = 60; + + public List GetValidationErrors() + { + var errors = new List(); + + if (ExpectedSize <= 0) + errors.Add("Embedding ExpectedSize deve ser maior que 0"); + + if (BatchSize <= 0) + errors.Add("Embedding BatchSize deve ser maior que 0"); + + return errors; + } + } +} diff --git a/appsettings.json b/appsettings.json index b4ae3c3..540a24e 100644 --- a/appsettings.json +++ b/appsettings.json @@ -7,7 +7,7 @@ } }, "VectorDatabase": { - "Provider": "Qdrant", + "Provider": "Qdrant", "MongoDB": { "ConnectionString": "mongodb://admin:c4rn31r0@k3sw2:27017,k3ss1:27017/?authSource=admin", "DatabaseName": "RAGProjects-dev-pt", @@ -19,15 +19,22 @@ "Host": "localhost", "Port": 6334, "CollectionName": "texts", + "GroupsCollectionName": "projects", "VectorSize": 384, "Distance": "Cosine", "HnswM": 16, "HnswEfConstruct": 200, "OnDisk": false + }, + "Chroma": { + "Host": "localhost", + "Port": 8000, + "CollectionName": "rag_documents" } }, "Features": { - "UseQdrant": true + "UseQdrant": true, + "UseHierarchicalRAG": true }, "AllowedHosts": "*", "AppTenantId": "20190830-5fd4-4a72-b8fd-1c1cb35b25bc",