diff --git a/ChatRAG.csproj b/ChatRAG.csproj index 750d5fe..e0d4285 100644 --- a/ChatRAG.csproj +++ b/ChatRAG.csproj @@ -26,6 +26,7 @@ + diff --git a/ChatRAG.csproj.user b/ChatRAG.csproj.user index e5a2ec0..b2208ee 100644 --- a/ChatRAG.csproj.user +++ b/ChatRAG.csproj.user @@ -2,8 +2,8 @@ http - ApiControllerEmptyScaffolder - root/Common/Api + MvcControllerEmptyScaffolder + root/Common/MVC/Controller ProjectDebugger diff --git a/Controllers/ChatController.cs b/Controllers/ChatController.cs index 9f6b7f7..8e18666 100644 --- a/Controllers/ChatController.cs +++ b/Controllers/ChatController.cs @@ -7,6 +7,7 @@ using ChatRAG.Data; using ChatRAG.Models; using ChatRAG.Requests; using BlazMapper; +using ChatRAG.Services.Contracts; #pragma warning disable SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. @@ -21,23 +22,23 @@ namespace ChatApi.Controllers private readonly TextFilter _textFilter; private readonly UserDataRepository _userDataRepository; private readonly ProjectDataRepository _projectDataRepository; - private readonly TextData _textData; + private readonly ITextDataService _textDataService; private readonly IHttpClientFactory _httpClientFactory; public ChatController( ILogger logger, IResponseService responseService, UserDataRepository userDataRepository, - TextData textData, + ITextDataService textDataService, ProjectDataRepository projectDataRepository, IHttpClientFactory httpClientFactory) { _logger = logger; _responseService = responseService; _userDataRepository = userDataRepository; - _textData = textData; + _textDataService = textDataService; _projectDataRepository = projectDataRepository; - this._httpClientFactory = httpClientFactory; + _httpClientFactory = httpClientFactory; } [HttpPost] @@ -80,7 +81,13 @@ namespace ChatApi.Controllers { try { - await _textData.SalvarNoMongoDB(request.Id, request.Title, request.Content, request.ProjectId); + await _textDataService.SaveDocumentAsync(new DocumentInput + { + Id = request.Id, + Title = request.Title, + Content = request.Content, + ProjectId = request.ProjectId + }); return Created(); } catch (Exception ex) @@ -97,7 +104,13 @@ namespace ChatApi.Controllers { foreach(var request in requests) { - await _textData.SalvarNoMongoDB(request.Id, request.Title, request.Content, request.ProjectId); + await _textDataService.SaveDocumentAsync(new DocumentInput + { + Id = request.Id, + Title = request.Title, + Content = request.Content, + ProjectId = request.ProjectId + }); } return Created(); } @@ -111,7 +124,7 @@ namespace ChatApi.Controllers [Route("texts")] public async Task> GetTexts() { - var texts = await _textData.GetAll(); + var texts = await _textDataService.GetAll(); return texts.Select(t => { return new TextResponse { @@ -126,7 +139,7 @@ namespace ChatApi.Controllers [Route("texts/id/{id}")] public async Task GetText([FromRoute] string id) { - var textItem = await _textData.GetById(id); + var textItem = await _textDataService.GetById(id); return new TextResponse { Id = textItem.Id, diff --git a/Controllers/MigrationController.cs b/Controllers/MigrationController.cs new file mode 100644 index 0000000..ee104a9 --- /dev/null +++ b/Controllers/MigrationController.cs @@ -0,0 +1,419 @@ +using ChatApi.Data; +using ChatRAG.Contracts.VectorSearch; +using ChatRAG.Data; +using ChatRAG.Models; +using ChatRAG.Services.Contracts; +using ChatRAG.Services.SearchVectors; +using ChatRAG.Settings.ChatRAG.Configuration; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Options; +using Microsoft.SemanticKernel.Embeddings; + +#pragma warning disable SKEXP0001 + +namespace ChatApi.Controllers +{ + [ApiController] + [Route("api/[controller]")] + public class MigrationController : ControllerBase + { + private readonly IVectorDatabaseFactory _factory; + private readonly ILogger _logger; + private readonly VectorDatabaseSettings _settings; + private readonly ITextEmbeddingGenerationService _embeddingService; + private readonly TextDataRepository _mongoRepository; + + public MigrationController( + IVectorDatabaseFactory factory, + ILogger logger, + IOptions settings, + ITextEmbeddingGenerationService embeddingService, + TextDataRepository mongoRepository) + { + _factory = factory; + _logger = logger; + _settings = settings.Value; + _embeddingService = embeddingService; + _mongoRepository = mongoRepository; + } + + /// + /// Status da migração e informações dos providers + /// + [HttpGet("status")] + public async Task GetMigrationStatus() + { + try + { + var currentProvider = _factory.GetActiveProvider(); + + var status = new + { + CurrentProvider = currentProvider, + Settings = new + { + Provider = _settings.Provider, + MongoDB = _settings.MongoDB != null ? new + { + DatabaseName = _settings.MongoDB.DatabaseName, + TextCollection = _settings.MongoDB.TextCollectionName + } : null, + Qdrant = _settings.Qdrant != null ? new + { + Host = _settings.Qdrant.Host, + Port = _settings.Qdrant.Port, + Collection = _settings.Qdrant.CollectionName, + VectorSize = _settings.Qdrant.VectorSize + } : null + }, + Stats = await GetProvidersStats() + }; + + return Ok(status); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao obter status da migração"); + return StatusCode(500, new { error = ex.Message }); + } + } + + /// + /// Migra dados do MongoDB para Qdrant + /// + [HttpPost("mongo-to-qdrant")] + public async Task MigrateMongoToQdrant( + [FromQuery] string? projectId = null, + [FromQuery] int batchSize = 50, + [FromQuery] bool dryRun = false) + { + try + { + if (_settings.Provider != "MongoDB") + { + return BadRequest("Migração só funciona quando o provider atual é MongoDB"); + } + + _logger.LogInformation("Iniciando migração MongoDB → Qdrant. ProjectId: {ProjectId}, DryRun: {DryRun}", + projectId, dryRun); + + var result = await PerformMigration(projectId, batchSize, dryRun); + + return Ok(result); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro durante migração MongoDB → Qdrant"); + return StatusCode(500, new { error = ex.Message }); + } + } + + /// + /// Migra dados do Qdrant para MongoDB + /// + [HttpPost("qdrant-to-mongo")] + public async Task MigrateQdrantToMongo( + [FromQuery] string? projectId = null, + [FromQuery] int batchSize = 50, + [FromQuery] bool dryRun = false) + { + try + { + if (_settings.Provider != "Qdrant") + { + return BadRequest("Migração só funciona quando o provider atual é Qdrant"); + } + + _logger.LogInformation("Iniciando migração Qdrant → MongoDB. ProjectId: {ProjectId}, DryRun: {DryRun}", + projectId, dryRun); + + // TODO: Implementar migração reversa se necessário + return BadRequest("Migração Qdrant → MongoDB ainda não implementada"); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro durante migração Qdrant → MongoDB"); + return StatusCode(500, new { error = ex.Message }); + } + } + + /// + /// Compara dados entre MongoDB e Qdrant + /// + [HttpPost("compare")] + public async Task CompareProviders([FromQuery] string? projectId = null) + { + try + { + // Cria serviços para ambos os providers manualmente + var mongoService = CreateMongoService(); + var qdrantService = await CreateQdrantService(); + + var comparison = await CompareData(mongoService, qdrantService, projectId); + + return Ok(comparison); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao comparar providers"); + return StatusCode(500, new { error = ex.Message }); + } + } + + /// + /// Limpa todos os dados do provider atual + /// + [HttpDelete("clear-current")] + public async Task ClearCurrentProvider([FromQuery] string? projectId = null) + { + try + { + var vectorSearchService = _factory.CreateVectorSearchService(); + + if (string.IsNullOrEmpty(projectId)) + { + // Limpar tudo - PERIGOSO! + return BadRequest("Limpeza completa requer confirmação. Use /clear-current/confirm"); + } + + // Limpar apenas um projeto + var documents = await vectorSearchService.GetDocumentsByProjectAsync(projectId); + var ids = documents.Select(d => d.Id).ToList(); + + foreach (var id in ids) + { + await vectorSearchService.DeleteDocumentAsync(id); + } + + return Ok(new + { + provider = _settings.Provider, + projectId, + deletedCount = ids.Count + }); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao limpar dados do provider"); + return StatusCode(500, new { error = ex.Message }); + } + } + + /// + /// Testa conectividade dos providers + /// + [HttpGet("test-connections")] + public async Task TestConnections() + { + var results = new Dictionary(); + + // Testar MongoDB + try + { + var mongoService = CreateMongoService(); + var mongoHealth = await mongoService.IsHealthyAsync(); + var mongoStats = await mongoService.GetStatsAsync(); + + results["MongoDB"] = new + { + healthy = mongoHealth, + stats = mongoStats + }; + } + catch (Exception ex) + { + results["MongoDB"] = new { healthy = false, error = ex.Message }; + } + + // Testar Qdrant + try + { + var qdrantService = await CreateQdrantService(); + var qdrantHealth = await qdrantService.IsHealthyAsync(); + var qdrantStats = await qdrantService.GetStatsAsync(); + + results["Qdrant"] = new + { + healthy = qdrantHealth, + stats = qdrantStats + }; + } + catch (Exception ex) + { + results["Qdrant"] = new { healthy = false, error = ex.Message }; + } + + return Ok(results); + } + + // ======================================== + // MÉTODOS PRIVADOS + // ======================================== + + private async Task PerformMigration(string? projectId, int batchSize, bool dryRun) + { + var mongoService = CreateMongoService(); + var qdrantService = await CreateQdrantService(); + + // 1. Buscar dados do MongoDB + List documents; + if (string.IsNullOrEmpty(projectId)) + { + var allDocs = await _mongoRepository.GetAsync(); + documents = allDocs.ToList(); + } + else + { + var projectDocs = await _mongoRepository.GetByProjectIdAsync(projectId); + documents = projectDocs.ToList(); + } + + _logger.LogInformation("Encontrados {Count} documentos para migração", documents.Count); + + if (dryRun) + { + return new + { + dryRun = true, + documentsFound = documents.Count, + projects = documents.GroupBy(d => d.ProjetoId).Select(g => new { + projectId = g.Key, + count = g.Count() + }).ToList() + }; + } + + // 2. Migrar em batches + var migrated = 0; + var errors = new List(); + + for (int i = 0; i < documents.Count; i += batchSize) + { + var batch = documents.Skip(i).Take(batchSize); + + foreach (var doc in batch) + { + try + { + // Verificar se já existe no Qdrant + var exists = await qdrantService.DocumentExistsAsync(doc.Id); + if (exists) + { + _logger.LogDebug("Documento {Id} já existe no Qdrant, pulando", doc.Id); + continue; + } + + // Migrar documento + await qdrantService.AddDocumentAsync( + title: doc.Titulo, + content: doc.Conteudo, + projectId: doc.ProjetoId, + embedding: doc.Embedding, + metadata: new Dictionary + { + ["project_name"] = doc.ProjetoNome ?? "", + ["document_type"] = doc.TipoDocumento ?? "", + ["category"] = doc.Categoria ?? "", + ["tags"] = doc.Tags ?? Array.Empty(), + ["migrated_from"] = "MongoDB", + ["migrated_at"] = DateTime.UtcNow.ToString("O") + } + ); + + migrated++; + + if (migrated % 10 == 0) + { + _logger.LogInformation("Migrados {Migrated}/{Total} documentos", migrated, documents.Count); + } + } + catch (Exception ex) + { + var error = $"Erro ao migrar documento {doc.Id}: {ex.Message}"; + errors.Add(error); + _logger.LogError(ex, error); + } + } + } + + return new + { + totalDocuments = documents.Count, + migrated, + errors = errors.Count, + errorDetails = errors.Take(10).ToList(), // Primeiros 10 erros + batchSize, + duration = "Completed" + }; + } + + private async Task CompareData( + IVectorSearchService mongoService, + IVectorSearchService qdrantService, + string? projectId) + { + var mongoStats = await mongoService.GetStatsAsync(); + var qdrantStats = await qdrantService.GetStatsAsync(); + + var mongoCount = await mongoService.GetDocumentCountAsync(projectId); + var qdrantCount = await qdrantService.GetDocumentCountAsync(projectId); + + return new + { + projectId, + comparison = new + { + MongoDB = new + { + documentCount = mongoCount, + healthy = await mongoService.IsHealthyAsync(), + stats = mongoStats + }, + Qdrant = new + { + documentCount = qdrantCount, + healthy = await qdrantService.IsHealthyAsync(), + stats = qdrantStats + } + }, + differences = new + { + documentCountDiff = qdrantCount - mongoCount, + inSync = mongoCount == qdrantCount + } + }; + } + + private MongoVectorSearchService CreateMongoService() + { + return new MongoVectorSearchService(_mongoRepository, _embeddingService); + } + + private async Task CreateQdrantService() + { + var qdrantSettings = Microsoft.Extensions.Options.Options.Create(_settings); + var logger = HttpContext.RequestServices.GetService>()!; + + return new QdrantVectorSearchService(qdrantSettings, logger); + } + + private async Task> GetProvidersStats() + { + var stats = new Dictionary(); + + try + { + var currentService = _factory.CreateVectorSearchService(); + stats["current"] = await currentService.GetStatsAsync(); + } + catch (Exception ex) + { + stats["current"] = new { error = ex.Message }; + } + + return stats; + } + } +} + +#pragma warning restore SKEXP0001 \ No newline at end of file diff --git a/Data/MongoTextDataService.cs b/Data/MongoTextDataService.cs new file mode 100644 index 0000000..a0d1201 --- /dev/null +++ b/Data/MongoTextDataService.cs @@ -0,0 +1,178 @@ +using ChatApi.Data; +using ChatRAG.Models; +using ChatRAG.Services.Contracts; +using Microsoft.SemanticKernel.Embeddings; + +#pragma warning disable SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. +namespace ChatRAG.Data +{ + public class MongoTextDataService : ITextDataService + { + private readonly TextData _textData; // Sua classe existente! + private readonly ITextEmbeddingGenerationService _embeddingService; + + public MongoTextDataService( + TextData textData, + ITextEmbeddingGenerationService embeddingService) + { + _textData = textData; + _embeddingService = embeddingService; + } + + public string ProviderName => "MongoDB"; + + // ======================================== + // MÉTODOS ORIGINAIS - Delega para TextData + // ======================================== + + public async Task SalvarNoMongoDB(string titulo, string texto, string projectId) + { + await _textData.SalvarNoMongoDB(titulo, texto, projectId); + } + + public async Task SalvarNoMongoDB(string? id, string titulo, string texto, string projectId) + { + await _textData.SalvarNoMongoDB(id, titulo, texto, projectId); + } + + public async Task SalvarTextoComEmbeddingNoMongoDB(string textoCompleto, string projectId) + { + await _textData.SalvarTextoComEmbeddingNoMongoDB(textoCompleto, projectId); + } + + public async Task> GetAll() + { + return await _textData.GetAll(); + } + + public async Task> GetByPorjectId(string projectId) + { + return await _textData.GetByPorjectId(projectId); + } + + public async Task GetById(string id) + { + return await _textData.GetById(id); + } + + // ======================================== + // NOVOS MÉTODOS UNIFICADOS + // ======================================== + + public async Task SaveDocumentAsync(DocumentInput document) + { + var id = document.Id ?? Guid.NewGuid().ToString(); + await _textData.SalvarNoMongoDB(id, document.Title, document.Content, document.ProjectId); + return id; + } + + public async Task UpdateDocumentAsync(string id, DocumentInput document) + { + await _textData.SalvarNoMongoDB(id, document.Title, document.Content, document.ProjectId); + } + + public async Task DeleteDocumentAsync(string id) + { + // Implementar quando necessário ou usar TextDataRepository diretamente + throw new NotImplementedException("Delete será implementado quando migrar para interface"); + } + + public async Task DocumentExistsAsync(string id) + { + try + { + var doc = await _textData.GetById(id); + return doc != null; + } + catch + { + return false; + } + } + + public async Task GetDocumentAsync(string id) + { + try + { + var doc = await _textData.GetById(id); + if (doc == null) return null; + + return new DocumentOutput + { + Id = doc.Id, + Title = doc.Titulo, + Content = doc.Conteudo, + ProjectId = doc.ProjetoId, + Embedding = doc.Embedding, + CreatedAt = DateTime.UtcNow, // MongoDB não tem essa info no modelo atual + UpdatedAt = DateTime.UtcNow + }; + } + catch + { + return null; + } + } + + public async Task> GetDocumentsByProjectAsync(string projectId) + { + var docs = await _textData.GetByPorjectId(projectId); + return docs.Select(doc => new DocumentOutput + { + Id = doc.Id, + Title = doc.Titulo, + Content = doc.Conteudo, + ProjectId = doc.ProjetoId, + Embedding = doc.Embedding, + CreatedAt = DateTime.UtcNow, + UpdatedAt = DateTime.UtcNow + }).ToList(); + } + + public async Task GetDocumentCountAsync(string? projectId = null) + { + if (string.IsNullOrEmpty(projectId)) + { + var all = await _textData.GetAll(); + return all.Count(); + } + else + { + var byProject = await _textData.GetByPorjectId(projectId); + return byProject.Count(); + } + } + + public async Task> SaveDocumentsBatchAsync(List documents) + { + var ids = new List(); + foreach (var doc in documents) + { + var id = await SaveDocumentAsync(doc); + ids.Add(id); + } + return ids; + } + + public async Task DeleteDocumentsBatchAsync(List ids) + { + foreach (var id in ids) + { + await DeleteDocumentAsync(id); + } + } + + public async Task> GetProviderStatsAsync() + { + var totalDocs = await GetDocumentCountAsync(); + return new Dictionary + { + ["provider"] = "MongoDB", + ["total_documents"] = totalDocs, + ["health"] = "ok", + ["last_check"] = DateTime.UtcNow + }; + } + } +} +#pragma warning restore SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. diff --git a/Data/TextData.cs b/Data/TextData.cs index f816ac2..61bd8e7 100644 --- a/Data/TextData.cs +++ b/Data/TextData.cs @@ -1,5 +1,6 @@ using ChatRAG.Data; using ChatRAG.Models; +using ChatRAG.Services.Contracts; using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.Embeddings; using System.Text; @@ -8,17 +9,23 @@ using System.Text; namespace ChatApi.Data { - public class TextData + public class TextData : ITextDataService { private readonly ITextEmbeddingGenerationService _textEmbeddingGenerationService; private readonly TextDataRepository _textDataService; - + public TextData(ITextEmbeddingGenerationService textEmbeddingGenerationService, TextDataRepository textDataService) { _textEmbeddingGenerationService = textEmbeddingGenerationService; _textDataService = textDataService; } + public string ProviderName => "MongoDB"; + + // ======================================== + // MÉTODOS ORIGINAIS (já implementados) + // ======================================== + public async Task SalvarTextoComEmbeddingNoMongoDB(string textoCompleto, string projectId) { var textoArray = new List(); @@ -47,7 +54,7 @@ namespace ChatApi.Data } } - foreach(var item in textoArray) + foreach (var item in textoArray) { await SalvarNoMongoDB(title, item, projectId); } @@ -55,7 +62,7 @@ namespace ChatApi.Data public async Task SalvarNoMongoDB(string titulo, string texto, string projectId) { - await SalvarNoMongoDB(null, titulo, texto); + await SalvarNoMongoDB(null, titulo, texto, projectId); } public async Task SalvarNoMongoDB(string? id, string titulo, string texto, string projectId) @@ -67,12 +74,13 @@ namespace ChatApi.Data // Converter embedding para um formato serializável (como um array de floats) var embeddingArray = embedding.ToArray().Select(e => (double)e).ToArray(); - var exists = id!=null ? await this.GetById(id) : null; + var exists = id != null ? await this.GetById(id) : null; if (exists == null) { var documento = new TextoComEmbedding { + Id = id ?? Guid.NewGuid().ToString(), Titulo = titulo, Conteudo = texto, ProjetoId = projectId, @@ -85,14 +93,14 @@ namespace ChatApi.Data { var documento = new TextoComEmbedding { - Id = id, + Id = id!, Titulo = titulo, Conteudo = texto, ProjetoId = projectId, Embedding = embeddingArray }; - await _textDataService.UpdateAsync(id, documento); + await _textDataService.UpdateAsync(id!, documento); } } @@ -108,8 +116,173 @@ namespace ChatApi.Data public async Task GetById(string id) { - return await _textDataService.GetAsync(id); + return (await _textDataService.GetAsync(id))!; + } + + // ======================================== + // MÉTODOS NOVOS DA INTERFACE (implementação completa) + // ======================================== + + public async Task SaveDocumentAsync(DocumentInput document) + { + var id = document.Id ?? Guid.NewGuid().ToString(); + await SalvarNoMongoDB(id, document.Title, document.Content, document.ProjectId); + return id; + } + + public async Task UpdateDocumentAsync(string id, DocumentInput document) + { + await SalvarNoMongoDB(id, document.Title, document.Content, document.ProjectId); + } + + public async Task DeleteDocumentAsync(string id) + { + await _textDataService.RemoveAsync(id); + } + + public async Task DocumentExistsAsync(string id) + { + try + { + var doc = await GetById(id); + return doc != null; + } + catch + { + return false; + } + } + + public async Task GetDocumentAsync(string id) + { + try + { + var doc = await GetById(id); + if (doc == null) return null; + + return new DocumentOutput + { + Id = doc.Id, + Title = doc.Titulo, + Content = doc.Conteudo, + ProjectId = doc.ProjetoId, + Embedding = doc.Embedding, + CreatedAt = DateTime.UtcNow, + UpdatedAt = DateTime.UtcNow, + Metadata = new Dictionary + { + ["source"] = "MongoDB", + ["has_embedding"] = doc.Embedding != null, + ["embedding_size"] = doc.Embedding?.Length ?? 0 + } + }; + } + catch + { + return null; + } + } + + public async Task> GetDocumentsByProjectAsync(string projectId) + { + var docs = await GetByPorjectId(projectId); + return docs.Select(doc => new DocumentOutput + { + Id = doc.Id, + Title = doc.Titulo, + Content = doc.Conteudo, + ProjectId = doc.ProjetoId, + Embedding = doc.Embedding, + CreatedAt = DateTime.UtcNow, + UpdatedAt = DateTime.UtcNow, + Metadata = new Dictionary + { + ["source"] = "MongoDB", + ["has_embedding"] = doc.Embedding != null, + ["embedding_size"] = doc.Embedding?.Length ?? 0 + } + }).ToList(); + } + + public async Task GetDocumentCountAsync(string? projectId = null) + { + if (string.IsNullOrEmpty(projectId)) + { + var all = await GetAll(); + return all.Count(); + } + else + { + var byProject = await GetByPorjectId(projectId); + return byProject.Count(); + } + } + + public async Task> SaveDocumentsBatchAsync(List documents) + { + var ids = new List(); + + foreach (var doc in documents) + { + var id = await SaveDocumentAsync(doc); + ids.Add(id); + } + + return ids; + } + + public async Task DeleteDocumentsBatchAsync(List ids) + { + foreach (var id in ids) + { + await DeleteDocumentAsync(id); + } + } + + public async Task> GetProviderStatsAsync() + { + try + { + var totalDocs = await GetDocumentCountAsync(); + var allDocs = await GetAll(); + + var docsWithEmbedding = allDocs.Count(d => d.Embedding != null && d.Embedding.Length > 0); + var avgContentLength = allDocs.Any() ? allDocs.Average(d => d.Conteudo?.Length ?? 0) : 0; + + var projectStats = allDocs + .GroupBy(d => d.ProjetoId) + .ToDictionary( + g => g.Key ?? "unknown", + g => g.Count() + ); + + return new Dictionary + { + ["provider"] = "MongoDB", + ["total_documents"] = totalDocs, + ["documents_with_embedding"] = docsWithEmbedding, + ["embedding_coverage"] = totalDocs > 0 ? (double)docsWithEmbedding / totalDocs : 0, + ["average_content_length"] = Math.Round(avgContentLength, 1), + ["projects_count"] = projectStats.Count, + ["documents_by_project"] = projectStats, + ["health"] = "ok", + ["last_check"] = DateTime.UtcNow, + ["connection_status"] = "connected" + }; + } + catch (Exception ex) + { + return new Dictionary + { + ["provider"] = "MongoDB", + ["health"] = "error", + ["error"] = ex.Message, + ["last_check"] = DateTime.UtcNow, + ["connection_status"] = "error" + }; + } } } } -#pragma warning restore SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + +#pragma warning restore SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. \ No newline at end of file diff --git a/Extensions/ServiceCollectionExtensions.cs b/Extensions/ServiceCollectionExtensions.cs new file mode 100644 index 0000000..aa5048c --- /dev/null +++ b/Extensions/ServiceCollectionExtensions.cs @@ -0,0 +1,100 @@ +using ChatApi.Data; +using ChatRAG.Contracts.VectorSearch; +using ChatRAG.Data; +using ChatRAG.Services; +using ChatRAG.Services.Contracts; +using ChatRAG.Services.ResponseService; +using ChatRAG.Services.SearchVectors; +using ChatRAG.Services.TextServices; +using ChatRAG.Settings; +using ChatRAG.Settings.ChatRAG.Configuration; +using Microsoft.Extensions.Options; +using Qdrant.Client; + +namespace ChatRAG.Extensions +{ + public static class ServiceCollectionExtensions + { + /// + /// Registra o sistema completo de Vector Database + /// + public static IServiceCollection AddVectorDatabase( + this IServiceCollection services, + IConfiguration configuration) + { + // Registra e valida configurações + services.Configure( + configuration.GetSection("VectorDatabase")); + + services.AddSingleton, + ChatRAG.Settings.VectorDatabaseSettingsValidator>(); + + // Registra factory principal + services.AddScoped(); + + // Registra implementações de todos os providers + services.AddMongoDbProvider(); + services.AddQdrantProvider(); // 👈 Agora ativo! + + // Registra interfaces principais usando factory + services.AddScoped(provider => + { + var factory = provider.GetRequiredService(); + return factory.CreateVectorSearchService(); + }); + + services.AddScoped(provider => + { + var factory = provider.GetRequiredService(); + return factory.CreateTextDataService(); + }); + + services.AddScoped(provider => + { + var factory = provider.GetRequiredService(); + return factory.CreateResponseService(); + }); + + return services; + } + + /// + /// Registra implementações MongoDB (suas classes atuais) + /// + private static IServiceCollection AddMongoDbProvider(this IServiceCollection services) + { + services.AddScoped(); // Implementa ITextDataService + services.AddScoped(); + services.AddScoped(); // Implementa IResponseService + services.AddScoped(); // Wrapper para IVectorSearchService + + return services; + } + + /// + /// Registra implementações Qdrant + /// + private static IServiceCollection AddQdrantProvider(this IServiceCollection services) + { + // ✅ Cliente Qdrant configurado + services.AddScoped(provider => + { + var settings = provider.GetRequiredService>(); + var qdrantSettings = settings.Value.Qdrant; + + return new QdrantClient( + host: qdrantSettings.Host, + port: qdrantSettings.Port, + https: qdrantSettings.UseTls + ); + }); + + // ✅ Serviços Qdrant + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + + return services; + } + } +} \ No newline at end of file diff --git a/Models/DocumentInput.cs b/Models/DocumentInput.cs new file mode 100644 index 0000000..b13d4b1 --- /dev/null +++ b/Models/DocumentInput.cs @@ -0,0 +1,43 @@ +namespace ChatRAG.Models +{ + /// + /// Modelo para entrada de dados (CREATE/UPDATE) + /// + public class DocumentInput + { + public string? Id { get; set; } + public string Title { get; set; } = string.Empty; + public string Content { get; set; } = string.Empty; + public string ProjectId { get; set; } = string.Empty; + public Dictionary? Metadata { get; set; } + public DateTime? CreatedAt { get; set; } + public DateTime? UpdatedAt { get; set; } + + public static DocumentInput FromTextoComEmbedding(TextoComEmbedding texto) + { + return new DocumentInput + { + Id = texto.Id, + Title = texto.Titulo, + Content = texto.Conteudo, + ProjectId = texto.ProjetoId, + CreatedAt = DateTime.UtcNow, + UpdatedAt = DateTime.UtcNow + }; + } + + public DocumentInput WithMetadata(string key, object value) + { + Metadata ??= new Dictionary(); + Metadata[key] = value; + return this; + } + + public bool IsValid() + { + return !string.IsNullOrWhiteSpace(Title) && + !string.IsNullOrWhiteSpace(Content) && + !string.IsNullOrWhiteSpace(ProjectId); + } + } +} diff --git a/Models/DocumentOutput.cs b/Models/DocumentOutput.cs new file mode 100644 index 0000000..44ce81f --- /dev/null +++ b/Models/DocumentOutput.cs @@ -0,0 +1,70 @@ +namespace ChatRAG.Models +{ + /// + /// Modelo para saída de dados (READ) + /// + public class DocumentOutput + { + public string Id { get; set; } = string.Empty; + public string Title { get; set; } = string.Empty; + public string Content { get; set; } = string.Empty; + public string ProjectId { get; set; } = string.Empty; + public Dictionary? Metadata { get; set; } + public DateTime CreatedAt { get; set; } + public DateTime UpdatedAt { get; set; } + public double[]? Embedding { get; set; } + + public TextoComEmbedding ToTextoComEmbedding() + { + return new TextoComEmbedding + { + Id = Id, + Titulo = Title, + Conteudo = Content, + ProjetoId = ProjectId, + Embedding = Embedding + }; + } + + public static DocumentOutput FromTextoComEmbedding(TextoComEmbedding texto) + { + return new DocumentOutput + { + Id = texto.Id, + Title = texto.Titulo, + Content = texto.Conteudo, + ProjectId = texto.ProjetoId, + Embedding = texto.Embedding, + CreatedAt = DateTime.UtcNow, + UpdatedAt = DateTime.UtcNow + }; + } + + public string GetContentPreview(int maxLength = 200) + { + if (string.IsNullOrEmpty(Content)) + return string.Empty; + + if (Content.Length <= maxLength) + return Content; + + return Content.Substring(0, maxLength) + "..."; + } + + public bool HasEmbedding() => Embedding != null && Embedding.Length > 0; + + public DocumentInput ToInput() + { + return new DocumentInput + { + Id = Id, + Title = Title, + Content = Content, + ProjectId = ProjectId, + Metadata = Metadata, + CreatedAt = CreatedAt, + UpdatedAt = UpdatedAt + }; + } + } +} diff --git a/Models/MigrationResult.cs b/Models/MigrationResult.cs new file mode 100644 index 0000000..789bb7b --- /dev/null +++ b/Models/MigrationResult.cs @@ -0,0 +1,22 @@ +namespace ChatRAG.Models +{ + public class MigrationResult + { + public bool Success { get; set; } + public string Message { get; set; } = string.Empty; + public DateTime StartTime { get; set; } + public TimeSpan Duration { get; set; } + public int TotalDocuments { get; set; } + public int MigratedDocuments { get; set; } + public List Errors { get; set; } = new(); + public ValidationResult? ValidationResult { get; set; } + + public double SuccessRate => TotalDocuments > 0 ? (double)MigratedDocuments / TotalDocuments : 0; + } + + public class ValidationResult + { + public bool IsValid { get; set; } + public List Issues { get; set; } = new(); + } +} diff --git a/Models/Models.cs b/Models/Models.cs new file mode 100644 index 0000000..67b5f6c --- /dev/null +++ b/Models/Models.cs @@ -0,0 +1,47 @@ +namespace ChatRAG.Models +{ + public class ResponseOptions + { + public int MaxContextDocuments { get; set; } = 3; + public double SimilarityThreshold { get; set; } = 0.3; + public bool IncludeSourceDetails { get; set; } = false; + public bool IncludeTiming { get; set; } = true; + public Dictionary? Filters { get; set; } + } + + public class ResponseResult + { + public string Content { get; set; } = string.Empty; + public List Sources { get; set; } = new(); + public ResponseMetrics Metrics { get; set; } = new(); + public string Provider { get; set; } = string.Empty; + } + + public class SourceDocument + { + public string Id { get; set; } = string.Empty; + public string Title { get; set; } = string.Empty; + public string Content { get; set; } = string.Empty; + public double Similarity { get; set; } + public Dictionary? Metadata { get; set; } + } + + public class ResponseMetrics + { + public long TotalTimeMs { get; set; } + public long SearchTimeMs { get; set; } + public long LlmTimeMs { get; set; } + public int DocumentsFound { get; set; } + public int DocumentsUsed { get; set; } + public double AverageSimilarity { get; set; } + } + + public class ResponseStats + { + public int TotalRequests { get; set; } + public double AverageResponseTime { get; set; } + public Dictionary RequestsByProject { get; set; } = new(); + public DateTime LastRequest { get; set; } + public string Provider { get; set; } = string.Empty; + } +} diff --git a/Models/VectorSearchResult.cs b/Models/VectorSearchResult.cs new file mode 100644 index 0000000..ac7c697 --- /dev/null +++ b/Models/VectorSearchResult.cs @@ -0,0 +1,144 @@ +namespace ChatRAG.Models +{ + /// + /// Resultado padronizado de busca vetorial + /// Funciona com qualquer provider (MongoDB, Qdrant, etc.) + /// + public class VectorSearchResult + { + /// + /// ID único do documento + /// + public string Id { get; set; } = string.Empty; + + /// + /// Título do documento + /// + public string Title { get; set; } = string.Empty; + + /// + /// Conteúdo completo do documento + /// + public string Content { get; set; } = string.Empty; + + /// + /// ID do projeto ao qual pertence + /// + public string ProjectId { get; set; } = string.Empty; + + /// + /// Score de similaridade (0.0 a 1.0, onde 1.0 é idêntico) + /// + public double Score { get; set; } + + /// + /// Embedding vetorial (opcional - nem sempre retornado por performance) + /// + public double[]? Embedding { get; set; } + + /// + /// Metadados adicionais (tags, categoria, autor, etc.) + /// + public Dictionary? Metadata { get; set; } + + /// + /// Data de criação do documento + /// + public DateTime CreatedAt { get; set; } + + /// + /// Data da última atualização + /// + public DateTime UpdatedAt { get; set; } + + // ======================================== + // INFORMAÇÕES DO PROVIDER + // ======================================== + + /// + /// Nome do provider que retornou este resultado (MongoDB, Qdrant, etc.) + /// + public string Provider { get; set; } = string.Empty; + + /// + /// Informações específicas do provider (índices, shards, etc.) + /// + public Dictionary? ProviderSpecific { get; set; } + + // ======================================== + // MÉTODOS DE CONVENIÊNCIA + // ======================================== + + /// + /// Preview do conteúdo (primeiros N caracteres) + /// + public string GetContentPreview(int maxLength = 200) + { + if (string.IsNullOrEmpty(Content)) + return string.Empty; + + if (Content.Length <= maxLength) + return Content; + + return Content.Substring(0, maxLength) + "..."; + } + + /// + /// Score formatado como percentual + /// + public string GetScorePercentage() + { + return $"{Score:P1}"; // Ex: "85.3%" + } + + /// + /// Indica se é um resultado relevante (score alto) + /// + public bool IsHighRelevance(double threshold = 0.7) + { + return Score >= threshold; + } + + /// + /// Converte para o modelo atual do sistema (compatibilidade) + /// + public TextoComEmbedding ToTextoComEmbedding() + { + return new TextoComEmbedding + { + Id = Id, + Titulo = Title, + Conteudo = Content, + ProjetoId = ProjectId, + Embedding = Embedding + }; + } + + /// + /// Converte do modelo atual do sistema + /// + public static VectorSearchResult FromTextoComEmbedding( + TextoComEmbedding texto, + double score = 1.0, + string provider = "Unknown") + { + return new VectorSearchResult + { + Id = texto.Id, + Title = texto.Titulo, + Content = texto.Conteudo, + ProjectId = texto.ProjetoId, + Score = score, + Embedding = texto.Embedding, + Provider = provider, + CreatedAt = DateTime.UtcNow, + UpdatedAt = DateTime.UtcNow + }; + } + + public override string ToString() + { + return $"{Title} (Score: {GetScorePercentage()}, Provider: {Provider})"; + } + } +} diff --git a/Program.cs b/Program.cs index 60c9f4f..1e60c4e 100644 --- a/Program.cs +++ b/Program.cs @@ -3,9 +3,13 @@ using ChatApi.Data; using ChatApi.Middlewares; using ChatApi.Services.Crypt; using ChatApi.Settings; +using ChatRAG.Contracts.VectorSearch; using ChatRAG.Data; +using ChatRAG.Extensions; using ChatRAG.Services; +using ChatRAG.Services.Contracts; using ChatRAG.Services.ResponseService; +using ChatRAG.Services.SearchVectors; using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.Http.Features; using Microsoft.AspNetCore.Server.Kestrel.Core; @@ -75,6 +79,17 @@ builder.Configuration.GetSection("DomvsDatabase")); builder.Services.Configure( builder.Configuration.GetSection("ChatRHSettings")); +builder.Services.AddScoped(); + +builder.Services.AddVectorDatabase(builder.Configuration); + +builder.Services.AddScoped(provider => +{ + var useQdrant = builder.Configuration["Features:UseQdrant"] == "true"; + var factory = provider.GetRequiredService(); + return factory.CreateVectorSearchService(); +}); + builder.Services.AddSingleton(); builder.Services.AddScoped(); builder.Services.AddScoped(); diff --git a/Services/ResponseService/IResponseService.cs b/Services/Contracts/IResponseService.cs similarity index 81% rename from Services/ResponseService/IResponseService.cs rename to Services/Contracts/IResponseService.cs index f61508e..029e887 100644 --- a/Services/ResponseService/IResponseService.cs +++ b/Services/Contracts/IResponseService.cs @@ -1,6 +1,6 @@ using ChatApi.Models; -namespace ChatRAG.Services.ResponseService +namespace ChatRAG.Services.Contracts { public interface IResponseService { diff --git a/Services/Contracts/IResponseServiceExtended.cs b/Services/Contracts/IResponseServiceExtended.cs new file mode 100644 index 0000000..a7b2039 --- /dev/null +++ b/Services/Contracts/IResponseServiceExtended.cs @@ -0,0 +1,17 @@ +using ChatApi.Models; +using ChatRAG.Models; + +namespace ChatRAG.Services.Contracts +{ + public interface IResponseServiceExtended : IResponseService + { + Task GetResponseDetailed( + UserData userData, + string projectId, + string sessionId, + string question, + ResponseOptions? options = null); + + Task GetStatsAsync(); + } +} diff --git a/Services/Contracts/ITextDataService.cs b/Services/Contracts/ITextDataService.cs new file mode 100644 index 0000000..d8e8ebf --- /dev/null +++ b/Services/Contracts/ITextDataService.cs @@ -0,0 +1,143 @@ +using ChatRAG.Models; + +namespace ChatRAG.Services.Contracts +{ + /// + /// Interface unificada para operações de documentos de texto. + /// Permite alternar entre MongoDB, Qdrant, ou outros providers sem quebrar código. + /// + public interface ITextDataService + { + // ======================================== + // MÉTODOS ORIGINAIS (compatibilidade com TextData.cs atual) + // ======================================== + + /// + /// Salva texto no banco (método original do seu TextData.cs) + /// + /// Título do documento + /// Conteúdo do documento + /// ID do projeto + Task SalvarNoMongoDB(string titulo, string texto, string projectId); + + /// + /// Salva ou atualiza texto com ID específico (método original) + /// + /// ID do documento (null para criar novo) + /// Título do documento + /// Conteúdo do documento + /// ID do projeto + Task SalvarNoMongoDB(string? id, string titulo, string texto, string projectId); + + /// + /// Processa texto completo dividindo por seções (método original) + /// + /// Texto com divisões marcadas por ** + /// ID do projeto + Task SalvarTextoComEmbeddingNoMongoDB(string textoCompleto, string projectId); + + /// + /// Recupera todos os documentos (método original) + /// + /// Lista de todos os documentos + Task> GetAll(); + + /// + /// Recupera documentos por projeto (método original) + /// + /// ID do projeto + /// Lista de documentos do projeto + Task> GetByPorjectId(string projectId); + + /// + /// Recupera documento por ID (método original) + /// + /// ID do documento + /// Documento ou null se não encontrado + Task GetById(string id); + + // ======================================== + // MÉTODOS NOVOS (interface moderna e unificada) + // ======================================== + + /// + /// Salva documento usando modelo unificado + /// + /// Dados do documento + /// ID do documento criado + Task SaveDocumentAsync(DocumentInput document); + + /// + /// Atualiza documento existente + /// + /// ID do documento + /// Novos dados do documento + Task UpdateDocumentAsync(string id, DocumentInput document); + + /// + /// Remove documento + /// + /// ID do documento + Task DeleteDocumentAsync(string id); + + /// + /// Verifica se documento existe + /// + /// ID do documento + /// True se existe, False caso contrário + Task DocumentExistsAsync(string id); + + /// + /// Recupera documento por ID (formato moderno) + /// + /// ID do documento + /// Documento ou null se não encontrado + Task GetDocumentAsync(string id); + + /// + /// Lista documentos por projeto (formato moderno) + /// + /// ID do projeto + /// Lista de documentos do projeto + Task> GetDocumentsByProjectAsync(string projectId); + + /// + /// Conta documentos + /// + /// Filtrar por projeto (opcional) + /// Número de documentos + Task GetDocumentCountAsync(string? projectId = null); + + // ======================================== + // OPERAÇÕES EM LOTE + // ======================================== + + /// + /// Salva múltiplos documentos de uma vez + /// + /// Lista de documentos + /// Lista de IDs dos documentos criados + Task> SaveDocumentsBatchAsync(List documents); + + /// + /// Remove múltiplos documentos de uma vez + /// + /// Lista de IDs para remover + Task DeleteDocumentsBatchAsync(List ids); + + // ======================================== + // INFORMAÇÕES DO PROVIDER + // ======================================== + + /// + /// Nome do provider (MongoDB, Qdrant, etc.) + /// + string ProviderName { get; } + + /// + /// Estatísticas e métricas do provider + /// + /// Informações sobre performance, saúde, etc. + Task> GetProviderStatsAsync(); + } +} diff --git a/Services/Contracts/IVectorDatabaseFactory.cs b/Services/Contracts/IVectorDatabaseFactory.cs new file mode 100644 index 0000000..85a8f69 --- /dev/null +++ b/Services/Contracts/IVectorDatabaseFactory.cs @@ -0,0 +1,14 @@ +using ChatRAG.Contracts.VectorSearch; +using ChatRAG.Settings.ChatRAG.Configuration; + +namespace ChatRAG.Services.Contracts +{ + public interface IVectorDatabaseFactory + { + IVectorSearchService CreateVectorSearchService(); + ITextDataService CreateTextDataService(); + IResponseService CreateResponseService(); + string GetActiveProvider(); + VectorDatabaseSettings GetSettings(); + } +} diff --git a/Services/Contracts/IVectorSearchService.cs b/Services/Contracts/IVectorSearchService.cs new file mode 100644 index 0000000..4035b04 --- /dev/null +++ b/Services/Contracts/IVectorSearchService.cs @@ -0,0 +1,138 @@ +using ChatRAG.Models; +using Microsoft.Extensions.VectorData; + +namespace ChatRAG.Contracts.VectorSearch +{ + /// + /// Interface unificada para operações de busca vetorial. + /// Pode ser implementada por MongoDB, Qdrant, Pinecone, etc. + /// + public interface IVectorSearchService + { + // ======================================== + // BUSCA VETORIAL + // ======================================== + + /// + /// Busca documentos similares usando embedding vetorial + /// + /// Embedding da query (ex: 1536 dimensões para OpenAI) + /// Filtrar por projeto específico (opcional) + /// Score mínimo de similaridade (0.0 a 1.0) + /// Número máximo de resultados + /// Filtros adicionais (metadata, data, etc.) + /// Lista de documentos ordenados por similaridade + Task> SearchSimilarAsync( + double[] queryEmbedding, + string? projectId = null, + double threshold = 0.3, + int limit = 5, + Dictionary? filters = null); + + /// + /// Busca adaptativa - relaxa threshold se não encontrar resultados suficientes + /// (Implementa a mesma lógica do seu ResponseRAGService atual) + /// + /// Embedding da query + /// ID do projeto + /// Threshold inicial (será reduzido se necessário) + /// Número máximo de resultados + /// Lista de documentos com busca adaptativa + Task> SearchSimilarDynamicAsync( + double[] queryEmbedding, + string projectId, + double minThreshold = 0.5, + int limit = 5); + + // ======================================== + // CRUD DE DOCUMENTOS + // ======================================== + + /// + /// Adiciona um novo documento com embedding + /// + /// Título do documento + /// Conteúdo do documento + /// ID do projeto + /// Embedding pré-calculado + /// Metadados adicionais (tags, data, autor, etc.) + /// ID do documento criado + Task AddDocumentAsync( + string title, + string content, + string projectId, + double[] embedding, + Dictionary? metadata = null); + + /// + /// Atualiza um documento existente + /// + /// ID do documento + /// Novo título + /// Novo conteúdo + /// ID do projeto + /// Novo embedding + /// Novos metadados + Task UpdateDocumentAsync( + string id, + string title, + string content, + string projectId, + double[] embedding, + Dictionary? metadata = null); + + /// + /// Remove um documento + /// + /// ID do documento + Task DeleteDocumentAsync(string id); + + // ======================================== + // CONSULTAS AUXILIARES + // ======================================== + + /// + /// Verifica se um documento existe + /// + /// ID do documento + /// True se existe, False caso contrário + Task DocumentExistsAsync(string id); + + /// + /// Recupera um documento específico por ID + /// + /// ID do documento + /// Documento ou null se não encontrado + Task GetDocumentAsync(string id); + + /// + /// Lista todos os documentos de um projeto + /// + /// ID do projeto + /// Lista de documentos do projeto + Task> GetDocumentsByProjectAsync(string projectId); + + /// + /// Conta total de documentos + /// + /// Filtrar por projeto (opcional) + /// Número de documentos + Task GetDocumentCountAsync(string? projectId = null); + + // ======================================== + // HEALTH CHECK E MÉTRICAS + // ======================================== + + /// + /// Verifica se o serviço está saudável + /// + /// True se está funcionando, False caso contrário + Task IsHealthyAsync(); + + /// + /// Retorna estatísticas e métricas do provider + /// + /// Dicionário com estatísticas (documentos, performance, etc.) + Task> GetStatsAsync(); + } +} diff --git a/Services/ResponseService/MongoResponseService.cs b/Services/ResponseService/MongoResponseService.cs new file mode 100644 index 0000000..8bc3d4a --- /dev/null +++ b/Services/ResponseService/MongoResponseService.cs @@ -0,0 +1,113 @@ +using ChatApi.Models; +using ChatRAG.Contracts.VectorSearch; +using ChatRAG.Models; +using ChatRAG.Services.Contracts; +using Microsoft.SemanticKernel.Embeddings; + +#pragma warning disable SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. +namespace ChatRAG.Services.ResponseService +{ + public class MongoResponseService : IResponseService + { + private readonly ResponseRAGService _originalService; // Sua classe atual! + private readonly IVectorSearchService _vectorSearchService; + private readonly ITextEmbeddingGenerationService _embeddingService; + private readonly TextFilter _textFilter; + + public MongoResponseService( + ResponseRAGService originalService, + IVectorSearchService vectorSearchService, + ITextEmbeddingGenerationService embeddingService, + TextFilter textFilter) + { + _originalService = originalService; + _vectorSearchService = vectorSearchService; + _embeddingService = embeddingService; + _textFilter = textFilter; + } + + public string ProviderName => "MongoDB"; + + // ======================================== + // MÉTODO ORIGINAL - Delega para ResponseRAGService + // ======================================== + public async Task GetResponse(UserData userData, string projectId, string sessionId, string question) + { + return await _originalService.GetResponse(userData, projectId, sessionId, question); + } + + // ======================================== + // MÉTODO ESTENDIDO COM MAIS DETALHES + // ======================================== + public async Task GetResponseDetailed( + UserData userData, + string projectId, + string sessionId, + string question, + ResponseOptions? options = null) + { + options ??= new ResponseOptions(); + var stopwatch = System.Diagnostics.Stopwatch.StartNew(); + + // Gera embedding da pergunta + var embeddingPergunta = await _embeddingService.GenerateEmbeddingAsync( + _textFilter.ToLowerAndWithoutAccents(question)); + var embeddingArray = embeddingPergunta.ToArray().Select(e => (double)e).ToArray(); + + var searchStart = stopwatch.ElapsedMilliseconds; + + // Busca documentos similares usando a interface + var documentos = await _vectorSearchService.SearchSimilarDynamicAsync( + embeddingArray, + projectId, + options.SimilarityThreshold, + options.MaxContextDocuments); + + var searchTime = stopwatch.ElapsedMilliseconds - searchStart; + var llmStart = stopwatch.ElapsedMilliseconds; + + // Chama o método original para gerar resposta + var response = await _originalService.GetResponse(userData, projectId, sessionId, question); + + var llmTime = stopwatch.ElapsedMilliseconds - llmStart; + stopwatch.Stop(); + + // Monta resultado detalhado + return new ResponseResult + { + Content = response, + Provider = "MongoDB", + Sources = documentos.Select(d => new SourceDocument + { + Id = d.Id, + Title = d.Title, + Content = d.Content, + Similarity = d.Score, + Metadata = d.Metadata + }).ToList(), + Metrics = new ResponseMetrics + { + TotalTimeMs = stopwatch.ElapsedMilliseconds, + SearchTimeMs = searchTime, + LlmTimeMs = llmTime, + DocumentsFound = documentos.Count, + DocumentsUsed = documentos.Count, + AverageSimilarity = documentos.Any() ? documentos.Average(d => d.Score) : 0 + } + }; + } + + public async Task GetStatsAsync() + { + // Implementação básica - pode ser expandida + return new ResponseStats + { + TotalRequests = 0, + AverageResponseTime = 0, + RequestsByProject = new Dictionary(), + LastRequest = DateTime.UtcNow + }; + } + } +} +#pragma warning restore SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. diff --git a/Services/ResponseService/QdrantResponseService.cs b/Services/ResponseService/QdrantResponseService.cs new file mode 100644 index 0000000..e291b09 --- /dev/null +++ b/Services/ResponseService/QdrantResponseService.cs @@ -0,0 +1,206 @@ +#pragma warning disable SKEXP0001 + +using ChatApi.Models; +using ChatRAG.Contracts.VectorSearch; +using ChatRAG.Models; +using ChatRAG.Services.Contracts; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.ChatCompletion; +using Microsoft.SemanticKernel.Embeddings; + +namespace ChatRAG.Services.ResponseService +{ + public class QdrantResponseService : IResponseService + { + private readonly IVectorSearchService _vectorSearchService; + private readonly ITextEmbeddingGenerationService _embeddingService; + private readonly IChatCompletionService _chatService; + private readonly ILogger _logger; + + public QdrantResponseService( + IVectorSearchService vectorSearchService, + ITextEmbeddingGenerationService embeddingService, + IChatCompletionService chatService, + ILogger logger) + { + _vectorSearchService = vectorSearchService; + _embeddingService = embeddingService; + _chatService = chatService; + _logger = logger; + } + + public string ProviderName => "Qdrant"; + + public async Task GetResponse( + UserData userData, + string projectId, + string sessionId, + string userMessage) + { + try + { + _logger.LogInformation("Processando consulta RAG com Qdrant para projeto {ProjectId}", projectId); + + // 1. Gerar embedding da pergunta do usuário + var questionEmbedding = await _embeddingService.GenerateEmbeddingAsync(userMessage); + var embeddingArray = questionEmbedding.ToArray().Select(e => (double)e).ToArray(); + + // 2. Buscar documentos similares no Qdrant + var searchResults = await _vectorSearchService.SearchSimilarDynamicAsync( + queryEmbedding: embeddingArray, + projectId: projectId, + minThreshold: 0.5, + limit: 5 + ); + + // 3. Construir contexto a partir dos resultados + var context = BuildContextFromResults(searchResults); + + // 4. Criar prompt com contexto + var prompt = BuildRagPrompt(userMessage, context); + + // 5. Gerar resposta usando LLM + var response = await _chatService.GetChatMessageContentAsync(prompt); + + _logger.LogDebug("Resposta RAG gerada com {ResultCount} documentos do Qdrant", + searchResults.Count); + + return response.Content ?? "Desculpe, não foi possível gerar uma resposta."; + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao processar consulta RAG com Qdrant"); + return "Ocorreu um erro ao processar sua consulta. Tente novamente."; + } + } + + public async Task GetResponseWithHistory( + UserData userData, + string projectId, + string sessionId, + string userMessage, + List conversationHistory) + { + try + { + // Combina histórico com mensagem atual para melhor contexto + var enhancedMessage = BuildEnhancedMessageWithHistory(userMessage, conversationHistory); + + return await GetResponse(userData, projectId, sessionId, enhancedMessage); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao processar consulta RAG com histórico"); + return "Ocorreu um erro ao processar sua consulta. Tente novamente."; + } + } + + // ======================================== + // MÉTODOS AUXILIARES PRIVADOS + // ======================================== + + private string BuildContextFromResults(List results) + { + if (!results.Any()) + { + return "Nenhum documento relevante encontrado."; + } + + var contextBuilder = new System.Text.StringBuilder(); + contextBuilder.AppendLine("=== CONTEXTO DOS DOCUMENTOS ==="); + + foreach (var result in results.Take(5)) // Limita a 5 documentos + { + contextBuilder.AppendLine($"\n--- Documento: {result.Title} (Relevância: {result.GetScorePercentage()}) ---"); + contextBuilder.AppendLine(result.Content); + contextBuilder.AppendLine(); + } + + return contextBuilder.ToString(); + } + + private string BuildRagPrompt(string userQuestion, string context) + { + return $@" + Você é um assistente especializado que responde perguntas baseado nos documentos fornecidos. + + CONTEXTO DOS DOCUMENTOS: + {context} + + PERGUNTA DO USUÁRIO: + {userQuestion} + + INSTRUÇÕES: + - Responda baseado APENAS nas informações dos documentos fornecidos + - Se a informação não estiver nos documentos, diga que não encontrou a informação + - Seja preciso e cite trechos relevantes quando possível + - Mantenha um tom profissional e prestativo + - Se houver múltiplas informações relevantes, organize-as de forma clara + + RESPOSTA: + "; + } + + private string BuildEnhancedMessageWithHistory(string currentMessage, List history) + { + if (!history.Any()) + return currentMessage; + + var enhancedMessage = new System.Text.StringBuilder(); + + enhancedMessage.AppendLine("HISTÓRICO DA CONVERSA:"); + foreach (var message in history.TakeLast(3)) // Últimas 3 mensagens para contexto + { + enhancedMessage.AppendLine($"- {message}"); + } + + enhancedMessage.AppendLine($"\nPERGUNTA ATUAL: {currentMessage}"); + + return enhancedMessage.ToString(); + } + + // ======================================== + // MÉTODOS DE ESTATÍSTICAS + // ======================================== + + public async Task> GetProviderStatsAsync() + { + try + { + var vectorStats = await _vectorSearchService.GetStatsAsync(); + + return new Dictionary(vectorStats) + { + ["response_service_provider"] = "Qdrant", + ["supports_history"] = true, + ["supports_dynamic_threshold"] = true, + ["last_check"] = DateTime.UtcNow + }; + } + catch (Exception ex) + { + return new Dictionary + { + ["response_service_provider"] = "Qdrant", + ["health"] = "error", + ["error"] = ex.Message, + ["last_check"] = DateTime.UtcNow + }; + } + } + + public async Task IsHealthyAsync() + { + try + { + return await _vectorSearchService.IsHealthyAsync(); + } + catch + { + return false; + } + } + } +} + +#pragma warning restore SKEXP0001 \ No newline at end of file diff --git a/Services/ResponseService/ResponseCompanyService.cs b/Services/ResponseService/ResponseCompanyService.cs index eba07e5..21fd4c6 100644 --- a/Services/ResponseService/ResponseCompanyService.cs +++ b/Services/ResponseService/ResponseCompanyService.cs @@ -1,8 +1,10 @@  using ChatApi; using ChatApi.Models; +using ChatRAG.Contracts.VectorSearch; using ChatRAG.Data; using ChatRAG.Models; +using ChatRAG.Services.Contracts; using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.ChatCompletion; using Microsoft.SemanticKernel.Embeddings; @@ -19,6 +21,7 @@ namespace ChatRAG.Services.ResponseService private readonly TextDataRepository _textDataRepository; private readonly ProjectDataRepository _projectDataRepository; private readonly IChatCompletionService _chatCompletionService; + private readonly IVectorSearchService _vectorSearchService; public ResponseRAGService( ChatHistoryService chatHistoryService, @@ -26,7 +29,9 @@ namespace ChatRAG.Services.ResponseService TextFilter textFilter, TextDataRepository textDataRepository, ProjectDataRepository projectDataRepository, - IChatCompletionService chatCompletionService) + IChatCompletionService chatCompletionService, + IVectorSearchService vectorSearchService, + ITextDataService textDataService) { this._chatHistoryService = chatHistoryService; this._kernel = kernel; @@ -34,6 +39,7 @@ namespace ChatRAG.Services.ResponseService this._textDataRepository = textDataRepository; this._projectDataRepository = projectDataRepository; this._chatCompletionService = chatCompletionService; + this._vectorSearchService = vectorSearchService; } public async Task GetResponse(UserData userData, string projectId, string sessionId, string question) @@ -43,7 +49,9 @@ namespace ChatRAG.Services.ResponseService //var resposta = await BuscarTextoRelacionado(question); //var resposta = await BuscarTopTextosRelacionados(question, projectId); - var resposta = await BuscarTopTextosRelacionadosDinamico(question, projectId); + //var resposta = await BuscarTopTextosRelacionadosDinamico(question, projectId); + var resposta = await BuscarTopTextosRelacionadosComInterface(question, projectId); + var projectData = (await _projectDataRepository.GetAsync()).FirstOrDefault(); @@ -101,7 +109,30 @@ namespace ChatRAG.Services.ResponseService return melhorTexto != null ? melhorTexto.Conteudo : "Não encontrei uma resposta adequada."; } - // Adicione esta nova rotina no seu ResponseRAGService + private async Task BuscarTopTextosRelacionadosComInterface(string pergunta, string projectId) + { + var embeddingService = _kernel.GetRequiredService(); + var embeddingPergunta = await embeddingService.GenerateEmbeddingAsync( + _textFilter.ToLowerAndWithoutAccents(pergunta)); + var embeddingArray = embeddingPergunta.ToArray().Select(e => (double)e).ToArray(); + + var resultados = await _vectorSearchService.SearchSimilarDynamicAsync(embeddingArray, projectId, 0.5, 3); + + if (!resultados.Any()) + return "Não encontrei respostas adequadas para a pergunta fornecida."; + + var cabecalho = $"Contexto encontrado para: '{pergunta}' ({resultados.Count} resultado(s)):\n\n"; + + var resultadosFormatados = resultados + .Select((item, index) => + $"=== CONTEXTO {index + 1} ===\n" + + $"Relevância: {item.Score:P1}\n" + + $"Conteúdo:\n{item.Content}") + .ToList(); + + return cabecalho + string.Join("\n\n", resultadosFormatados); + } + async Task BuscarTopTextosRelacionadosDinamico(string pergunta, string projectId, int size = 3) { diff --git a/Services/SearchVectors/MongoVectorSearchService.cs b/Services/SearchVectors/MongoVectorSearchService.cs new file mode 100644 index 0000000..9cf588e --- /dev/null +++ b/Services/SearchVectors/MongoVectorSearchService.cs @@ -0,0 +1,220 @@ +using ChatRAG.Contracts.VectorSearch; +using ChatRAG.Data; +using ChatRAG.Models; +using Microsoft.Extensions.VectorData; +using Microsoft.SemanticKernel.Embeddings; + +#pragma warning disable SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. +namespace ChatRAG.Services.SearchVectors +{ + public class MongoVectorSearchService : IVectorSearchService + { + private readonly TextDataRepository _textDataRepository; + private readonly ITextEmbeddingGenerationService _embeddingService; + + public MongoVectorSearchService( + TextDataRepository textDataRepository, + ITextEmbeddingGenerationService embeddingService) + { + _textDataRepository = textDataRepository; + _embeddingService = embeddingService; + } + + // ... resto da implementação permanece igual ... + // (copiar do código anterior) + + public async Task> SearchSimilarAsync( + double[] queryEmbedding, + string? projectId = null, + double threshold = 0.3, + int limit = 5, + Dictionary? filters = null) + { + var textos = string.IsNullOrEmpty(projectId) + ? await _textDataRepository.GetAsync() + : await _textDataRepository.GetByProjectIdAsync(projectId); + + var resultados = textos + .Select(texto => new VectorSearchResult + { + Id = texto.Id, + Title = texto.Titulo, + Content = texto.Conteudo, + ProjectId = texto.ProjetoId, + Score = CalcularSimilaridadeCoseno(queryEmbedding, texto.Embedding), + Embedding = texto.Embedding, + Provider = "MongoDB", + CreatedAt = DateTime.UtcNow, + UpdatedAt = DateTime.UtcNow + }) + .Where(r => r.Score >= threshold) + .OrderByDescending(r => r.Score) + .Take(limit) + .ToList(); + + return resultados; + } + + public async Task> SearchSimilarDynamicAsync( + double[] queryEmbedding, + string projectId, + double minThreshold = 0.5, + int limit = 5) + { + var resultados = await SearchSimilarAsync(queryEmbedding, projectId, minThreshold, limit); + + if (resultados.Count < 3) + { + resultados = await SearchSimilarAsync(queryEmbedding, projectId, minThreshold * 0.7, limit); + } + + if (resultados.Count < 3) + { + resultados = await SearchSimilarAsync(queryEmbedding, projectId, 0.2, limit); + } + + return resultados.Take(limit).ToList(); + } + + public async Task AddDocumentAsync( + string title, + string content, + string projectId, + double[] embedding, + Dictionary? metadata = null) + { + var documento = new TextoComEmbedding + { + Id = Guid.NewGuid().ToString(), + Titulo = title, + Conteudo = content, + ProjetoId = projectId, + Embedding = embedding + }; + + await _textDataRepository.CreateAsync(documento); + return documento.Id; + } + + public async Task UpdateDocumentAsync( + string id, + string title, + string content, + string projectId, + double[] embedding, + Dictionary? metadata = null) + { + var documento = new TextoComEmbedding + { + Id = id, + Titulo = title, + Conteudo = content, + ProjetoId = projectId, + Embedding = embedding + }; + + await _textDataRepository.UpdateAsync(id, documento); + } + + public async Task DeleteDocumentAsync(string id) + { + await _textDataRepository.RemoveAsync(id); + } + + public async Task DocumentExistsAsync(string id) + { + var doc = await _textDataRepository.GetAsync(id); + return doc != null; + } + + public async Task GetDocumentAsync(string id) + { + var doc = await _textDataRepository.GetAsync(id); + if (doc == null) return null; + + return new VectorSearchResult + { + Id = doc.Id, + Title = doc.Titulo, + Content = doc.Conteudo, + ProjectId = doc.ProjetoId, + Score = 1.0, + Embedding = doc.Embedding, + Provider = "MongoDB", + CreatedAt = DateTime.UtcNow, + UpdatedAt = DateTime.UtcNow + }; + } + + public async Task> GetDocumentsByProjectAsync(string projectId) + { + var docs = await _textDataRepository.GetByProjectIdAsync(projectId); + return docs.Select(doc => new VectorSearchResult + { + Id = doc.Id, + Title = doc.Titulo, + Content = doc.Conteudo, + ProjectId = doc.ProjetoId, + Score = 1.0, + Embedding = doc.Embedding, + Provider = "MongoDB", + CreatedAt = DateTime.UtcNow, + UpdatedAt = DateTime.UtcNow + }).ToList(); + } + + public async Task GetDocumentCountAsync(string? projectId = null) + { + if (string.IsNullOrEmpty(projectId)) + { + var all = await _textDataRepository.GetAsync(); + return all.Count; + } + else + { + var byProject = await _textDataRepository.GetByProjectIdAsync(projectId); + return byProject.Count; + } + } + + public async Task IsHealthyAsync() + { + try + { + var count = await GetDocumentCountAsync(); + return true; + } + catch + { + return false; + } + } + + public async Task> GetStatsAsync() + { + var totalDocs = await GetDocumentCountAsync(); + return new Dictionary + { + ["provider"] = "MongoDB", + ["total_documents"] = totalDocs, + ["health"] = await IsHealthyAsync(), + ["last_check"] = DateTime.UtcNow + }; + } + + private double CalcularSimilaridadeCoseno(double[] embedding1, double[] embedding2) + { + double dotProduct = 0.0; + double normA = 0.0; + double normB = 0.0; + for (int i = 0; i < embedding1.Length; i++) + { + dotProduct += embedding1[i] * embedding2[i]; + normA += Math.Pow(embedding1[i], 2); + normB += Math.Pow(embedding2[i], 2); + } + return dotProduct / (Math.Sqrt(normA) * Math.Sqrt(normB)); + } + } +} +#pragma warning restore SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. diff --git a/Services/SearchVectors/QdrantVectorSearchService.cs b/Services/SearchVectors/QdrantVectorSearchService.cs new file mode 100644 index 0000000..f59adbd --- /dev/null +++ b/Services/SearchVectors/QdrantVectorSearchService.cs @@ -0,0 +1,527 @@ +using ChatRAG.Contracts.VectorSearch; +using ChatRAG.Settings.ChatRAG.Configuration; +using Microsoft.Extensions.Options; +using Qdrant.Client.Grpc; +using ChatRAG.Models; +using ChatRAG.Services.Contracts; +using Qdrant.Client; +using static Qdrant.Client.Grpc.Conditions; +using System.Drawing; + +#pragma warning disable SKEXP0001 + + +namespace ChatRAG.Services.SearchVectors +{ + public class QdrantVectorSearchService : IVectorSearchService + { + private readonly QdrantClient _client; + private readonly QdrantSettings _settings; + private readonly ILogger _logger; + private bool _collectionInitialized = false; + + public QdrantVectorSearchService( + IOptions settings, + ILogger logger) + { + _settings = settings.Value.Qdrant; + _logger = logger; + + _client = new QdrantClient(_settings.Host, _settings.Port, https: _settings.UseTls); + + _logger.LogInformation("QdrantVectorSearchService inicializado para {Host}:{Port}", + _settings.Host, _settings.Port); + } + + private async Task EnsureCollectionExistsAsync() + { + if (_collectionInitialized) return; + + try + { + var collectionExists = await _client.CollectionExistsAsync(_settings.CollectionName); + + if (!collectionExists) + { + _logger.LogInformation("Criando collection {CollectionName}...", _settings.CollectionName); + + var vectorsConfig = new VectorParams + { + Size = (ulong)_settings.VectorSize, + Distance = _settings.Distance.ToLower() switch + { + "cosine" => Distance.Cosine, + "euclid" => Distance.Euclid, + "dot" => Distance.Dot, + "manhattan" => Distance.Manhattan, + _ => Distance.Cosine + } + }; + + // Configurações HNSW opcionais + if (_settings.HnswM > 0) + { + vectorsConfig.HnswConfig = new HnswConfigDiff + { + M = (ulong)_settings.HnswM, + EfConstruct = (ulong)_settings.HnswEfConstruct, + OnDisk = _settings.OnDisk + }; + } + + await _client.CreateCollectionAsync( + collectionName: _settings.CollectionName, + vectorsConfig: vectorsConfig + ); + + _logger.LogInformation("✅ Collection {CollectionName} criada", _settings.CollectionName); + } + + _collectionInitialized = true; + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao inicializar collection {CollectionName}", _settings.CollectionName); + throw; + } + } + + public async Task> SearchSimilarAsync( + double[] queryEmbedding, + string? projectId = null, + double threshold = 0.3, + int limit = 5, + Dictionary? filters = null) + { + await EnsureCollectionExistsAsync(); + + try + { + var vector = queryEmbedding.Select(x => (float)x).ToArray(); + + Filter? filter = null; + if (!string.IsNullOrEmpty(projectId) || filters?.Any() == true) + { + var mustConditions = new List(); + + if (!string.IsNullOrEmpty(projectId)) + { + mustConditions.Add(MatchKeyword("project_id", projectId)); + } + + if (filters?.Any() == true) + { + foreach (var kvp in filters) + { + mustConditions.Add(MatchKeyword(kvp.Key, kvp.Value.ToString()!)); + } + } + + if (mustConditions.Any()) + { + filter = new Filter(); + filter.Must.AddRange(mustConditions); + } + } + + var searchResult = await _client.SearchAsync( + collectionName: _settings.CollectionName, + vector: vector, + filter: filter, + limit: (ulong)limit, + scoreThreshold: (float)threshold, + payloadSelector: true, + vectorsSelector: true + ); + + return searchResult.Select(point => new VectorSearchResult + { + Id = point.Id.Uuid ?? point.Id.Num.ToString(), + Title = GetStringFromPayload(point.Payload, "title"), + Content = GetStringFromPayload(point.Payload, "content"), + ProjectId = GetStringFromPayload(point.Payload, "project_id"), + Score = point.Score, + Provider = "Qdrant", + CreatedAt = GetDateTimeFromPayload(point.Payload, "created_at"), + UpdatedAt = GetDateTimeFromPayload(point.Payload, "updated_at"), + Metadata = ConvertPayloadToMetadata(point.Payload) + }).ToList(); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro na busca vetorial Qdrant"); + throw; + } + } + + public async Task> SearchSimilarDynamicAsync( + double[] queryEmbedding, + string projectId, + double minThreshold = 0.5, + int limit = 5) + { + var results = await SearchSimilarAsync(queryEmbedding, projectId, minThreshold, limit); + + if (results.Count < 3 && minThreshold > 0.2) + { + results = await SearchSimilarAsync(queryEmbedding, projectId, minThreshold * 0.7, limit); + } + + if (results.Count < 3) + { + results = await SearchSimilarAsync(queryEmbedding, projectId, 0.2, limit); + } + + return results.Take(limit).ToList(); + } + + public async Task AddDocumentAsync( + string title, + string content, + string projectId, + double[] embedding, + Dictionary? metadata = null) + { + await EnsureCollectionExistsAsync(); + + try + { + var id = Guid.NewGuid().ToString(); + var vector = embedding.Select(x => (float)x).ToArray(); + + var payload = new Dictionary + { + ["title"] = title, + ["content"] = content, + ["project_id"] = projectId, + ["created_at"] = DateTime.UtcNow.ToString("O"), + ["updated_at"] = DateTime.UtcNow.ToString("O") + }; + + if (metadata?.Any() == true) + { + foreach (var kvp in metadata) + { + payload[$"meta_{kvp.Key}"] = ConvertToValue(kvp.Value); + } + } + + var point = new PointStruct + { + Id = new PointId { Uuid = id }, + Vectors = vector, + Payload = { payload } + }; + + await _client.UpsertAsync( + collectionName: _settings.CollectionName, + points: new[] { point } + ); + + _logger.LogDebug("Documento {Id} adicionado ao Qdrant", id); + return id; + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao adicionar documento no Qdrant"); + throw; + } + } + + public async Task UpdateDocumentAsync( + string id, + string title, + string content, + string projectId, + double[] embedding, + Dictionary? metadata = null) + { + await EnsureCollectionExistsAsync(); + + try + { + var vector = embedding.Select(x => (float)x).ToArray(); + + var payload = new Dictionary + { + ["title"] = title, + ["content"] = content, + ["project_id"] = projectId, + ["updated_at"] = DateTime.UtcNow.ToString("O") + }; + + if (metadata?.Any() == true) + { + foreach (var kvp in metadata) + { + payload[$"meta_{kvp.Key}"] = ConvertToValue(kvp.Value); + } + } + + var point = new PointStruct + { + Id = new PointId { Uuid = id }, + Vectors = vector, + Payload = { payload } + }; + + await _client.UpsertAsync( + collectionName: _settings.CollectionName, + points: new[] { point } + ); + + _logger.LogDebug("Documento {Id} atualizado no Qdrant", id); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao atualizar documento {Id} no Qdrant", id); + throw; + } + } + + public async Task DeleteDocumentAsync(string id) + { + await EnsureCollectionExistsAsync(); + + try + { + var pointId = new PointId { Uuid = id } ; + + await _client.DeleteAsync( + collectionName: _settings.CollectionName, + ids: new ulong[] { pointId.Num } + ); + + _logger.LogDebug("Documento {Id} removido do Qdrant", id); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao remover documento {Id} do Qdrant", id); + throw; + } + } + + public async Task DocumentExistsAsync(string id) + { + try + { + var result = await GetDocumentAsync(id); + return result != null; + } + catch + { + return false; + } + } + + public async Task GetDocumentAsync(string id) + { + await EnsureCollectionExistsAsync(); + + try + { + var pointId = new PointId { Uuid = id }; + + var results = await _client.RetrieveAsync( + collectionName: _settings.CollectionName, + ids: new PointId[] { pointId }, + withPayload: true, + withVectors: false + ); + + var point = results.FirstOrDefault(); + if (point == null) return null; + + return new VectorSearchResult + { + Id = point.Id.Uuid ?? point.Id.Num.ToString(), + Title = GetStringFromPayload(point.Payload, "title"), + Content = GetStringFromPayload(point.Payload, "content"), + ProjectId = GetStringFromPayload(point.Payload, "project_id"), + Score = 1.0, + Provider = "Qdrant", + CreatedAt = GetDateTimeFromPayload(point.Payload, "created_at"), + UpdatedAt = GetDateTimeFromPayload(point.Payload, "updated_at"), + Metadata = ConvertPayloadToMetadata(point.Payload) + }; + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao recuperar documento {Id} do Qdrant", id); + return null; + } + } + + public async Task> GetDocumentsByProjectAsync(string projectId) + { + await EnsureCollectionExistsAsync(); + + try + { + var filter = new Filter(); + filter.Must.Add(MatchKeyword("project_id", projectId)); + + var results = await _client.ScrollAsync( + collectionName: _settings.CollectionName, + filter: filter, + limit: 10000, + payloadSelector: true, + vectorsSelector: true + ); + + return results.Result.Select(point => new VectorSearchResult + { + Id = point.Id.Uuid ?? point.Id.Num.ToString(), + Title = GetStringFromPayload(point.Payload, "title"), + Content = GetStringFromPayload(point.Payload, "content"), + ProjectId = GetStringFromPayload(point.Payload, "project_id"), + Score = 1.0, + Provider = "Qdrant", + CreatedAt = GetDateTimeFromPayload(point.Payload, "created_at"), + UpdatedAt = GetDateTimeFromPayload(point.Payload, "updated_at"), + Metadata = ConvertPayloadToMetadata(point.Payload) + }).ToList(); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao buscar documentos do projeto {ProjectId} no Qdrant", projectId); + throw; + } + } + + public async Task GetDocumentCountAsync(string? projectId = null) + { + await EnsureCollectionExistsAsync(); + + try + { + Filter? filter = null; + if (!string.IsNullOrEmpty(projectId)) + { + filter = new Filter(); + filter.Must.Add(MatchKeyword("project_id", projectId)); + } + + var result = await _client.CountAsync(_settings.CollectionName, filter); + return (int)result; + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao contar documentos no Qdrant"); + return 0; + } + } + + public async Task IsHealthyAsync() + { + try + { + var collections = await _client.ListCollectionsAsync(); + return collections != null; + } + catch + { + return false; + } + } + + public async Task> GetStatsAsync() + { + try + { + await EnsureCollectionExistsAsync(); + + var collectionInfo = await _client.GetCollectionInfoAsync(_settings.CollectionName); + var totalDocs = await GetDocumentCountAsync(); + + return new Dictionary + { + ["provider"] = "Qdrant", + ["total_documents"] = totalDocs, + ["collection_name"] = _settings.CollectionName, + ["vector_size"] = _settings.VectorSize, + ["distance_metric"] = _settings.Distance, + ["points_count"] = collectionInfo.PointsCount, + ["segments_count"] = collectionInfo.SegmentsCount, + ["health"] = await IsHealthyAsync(), + ["last_check"] = DateTime.UtcNow + }; + } + catch (Exception ex) + { + return new Dictionary + { + ["provider"] = "Qdrant", + ["health"] = false, + ["error"] = ex.Message, + ["last_check"] = DateTime.UtcNow + }; + } + } + + private static Value ConvertToValue(object value) + { + return value switch + { + string s => s, + int i => i, + long l => l, + double d => d, + float f => f, + bool b => b, + DateTime dt => dt.ToString("O"), + _ => value?.ToString() ?? "" + }; + } + + private static string GetStringFromPayload( + IDictionary payload, + string key, + string defaultValue = "") + { + return payload.TryGetValue(key, out var value) ? value.StringValue : defaultValue; + } + + private static DateTime GetDateTimeFromPayload( + IDictionary payload, + string key) + { + if (payload.TryGetValue(key, out var value) && + DateTime.TryParse(value.StringValue, out var date)) + { + return date; + } + return DateTime.UtcNow; + } + + private static Dictionary? ConvertPayloadToMetadata( + IDictionary payload) + { + var metadata = new Dictionary(); + + foreach (var kvp in payload.Where(p => p.Key.StartsWith("meta_"))) + { + var key = kvp.Key.Substring(5); + var value = kvp.Value; + + metadata[key] = value.KindCase switch + { + Value.KindOneofCase.StringValue => value.StringValue, + Value.KindOneofCase.IntegerValue => value.IntegerValue, + Value.KindOneofCase.DoubleValue => value.DoubleValue, + Value.KindOneofCase.BoolValue => value.BoolValue, + _ => value.StringValue + }; + } + + return metadata.Any() ? metadata : null; + } + + public void Dispose() + { + _client?.Dispose(); + } + } +} + +#pragma warning restore SKEXP0001 \ No newline at end of file diff --git a/Services/SearchVectors/VectorDatabaseFactory.cs b/Services/SearchVectors/VectorDatabaseFactory.cs new file mode 100644 index 0000000..64316c7 --- /dev/null +++ b/Services/SearchVectors/VectorDatabaseFactory.cs @@ -0,0 +1,114 @@ +using ChatApi.Data; +using ChatRAG.Contracts.VectorSearch; +using ChatRAG.Services.Contracts; +using ChatRAG.Services.ResponseService; +using ChatRAG.Services.TextServices; +using ChatRAG.Settings.ChatRAG.Configuration; +using Microsoft.Extensions.Options; + +namespace ChatRAG.Services.SearchVectors +{ + /// + /// Factory principal que cria implementações baseadas na configuração + /// + /// + /// Factory principal que cria implementações baseadas na configuração + /// + public class VectorDatabaseFactory : IVectorDatabaseFactory + { + private readonly VectorDatabaseSettings _settings; + private readonly IServiceProvider _serviceProvider; + private readonly ILogger _logger; + + public VectorDatabaseFactory( + IOptions settings, + IServiceProvider serviceProvider, + ILogger logger) + { + _settings = settings.Value; + _serviceProvider = serviceProvider; + _logger = logger; + + // Valida configurações na inicialização + ValidateSettings(); + } + + public string GetActiveProvider() => _settings.Provider; + public VectorDatabaseSettings GetSettings() => _settings; + + public IVectorSearchService CreateVectorSearchService() + { + _logger.LogInformation("Criando VectorSearchService para provider: {Provider}", _settings.Provider); + + return _settings.Provider.ToLower() switch + { + "qdrant" => GetService(), + "mongodb" => GetService(), + _ => throw new ArgumentException($"Provider de VectorSearch não suportado: {_settings.Provider}") + }; + } + + public ITextDataService CreateTextDataService() + { + _logger.LogInformation("Criando TextDataService para provider: {Provider}", _settings.Provider); + + return _settings.Provider.ToLower() switch + { + // ✅ CORRIGIDO: Usa os namespaces corretos + "qdrant" => GetService(), + "mongodb" => GetService(), // Sua classe atual! + _ => throw new ArgumentException($"Provider de TextData não suportado: {_settings.Provider}") + }; + } + + public IResponseService CreateResponseService() + { + _logger.LogInformation("Criando ResponseService para provider: {Provider}", _settings.Provider); + + return _settings.Provider.ToLower() switch + { + // ✅ CORRIGIDO: Usa os namespaces corretos + "qdrant" => GetService(), + "mongodb" => GetService(), // Sua classe atual! + _ => throw new ArgumentException($"Provider de Response não suportado: {_settings.Provider}") + }; + } + + // ======================================== + // MÉTODOS AUXILIARES + // ======================================== + + private T GetService() where T : class + { + try + { + var service = _serviceProvider.GetRequiredService(); + _logger.LogDebug("Serviço {ServiceType} criado com sucesso", typeof(T).Name); + return service; + } + catch (InvalidOperationException ex) + { + _logger.LogError(ex, "Erro ao criar serviço {ServiceType} para provider {Provider}", + typeof(T).Name, _settings.Provider); + + throw new InvalidOperationException( + $"Serviço {typeof(T).Name} não está registrado para provider {_settings.Provider}. " + + $"Certifique-se de chamar services.Add{_settings.Provider}Provider() no DI.", ex); + } + } + + private void ValidateSettings() + { + if (!_settings.IsValid()) + { + var errors = _settings.GetValidationErrors(); + var errorMessage = $"Configurações inválidas para VectorDatabase: {string.Join(", ", errors)}"; + + _logger.LogError(errorMessage); + throw new InvalidOperationException(errorMessage); + } + + _logger.LogInformation("Configurações validadas com sucesso para provider: {Provider}", _settings.Provider); + } + } +} diff --git a/Services/TextServices/QdrantTextDataService.cs b/Services/TextServices/QdrantTextDataService.cs new file mode 100644 index 0000000..f4a034a --- /dev/null +++ b/Services/TextServices/QdrantTextDataService.cs @@ -0,0 +1,523 @@ +#pragma warning disable SKEXP0001 + +using ChatRAG.Contracts.VectorSearch; +using ChatRAG.Data; +using ChatRAG.Models; +using ChatRAG.Services.Contracts; +using Microsoft.SemanticKernel.Embeddings; +using System.Text; + +namespace ChatRAG.Services.TextServices +{ + public class QdrantTextDataService : ITextDataService + { + private readonly IVectorSearchService _vectorSearchService; + private readonly ITextEmbeddingGenerationService _embeddingService; + private readonly ILogger _logger; + + public QdrantTextDataService( + IVectorSearchService vectorSearchService, + ITextEmbeddingGenerationService embeddingService, + ILogger logger) + { + _vectorSearchService = vectorSearchService; + _embeddingService = embeddingService; + _logger = logger; + } + + public string ProviderName => "Qdrant"; + + // ======================================== + // MÉTODOS ORIGINAIS (compatibilidade com MongoDB) + // ======================================== + + public async Task SalvarNoMongoDB(string titulo, string texto, string projectId) + { + await SalvarNoMongoDB(null, titulo, texto, projectId); + } + + public async Task SalvarNoMongoDB(string? id, string titulo, string texto, string projectId) + { + try + { + var conteudo = $"**{titulo}** \n\n {texto}"; + + // Gera embedding + var embedding = await _embeddingService.GenerateEmbeddingAsync(conteudo); + var embeddingArray = embedding.ToArray().Select(e => (double)e).ToArray(); + + if (string.IsNullOrEmpty(id)) + { + // Cria novo documento + await _vectorSearchService.AddDocumentAsync(titulo, texto, projectId, embeddingArray); + _logger.LogDebug("Documento '{Title}' criado no Qdrant", titulo); + } + else + { + // Atualiza documento existente + await _vectorSearchService.UpdateDocumentAsync(id, titulo, texto, projectId, embeddingArray); + _logger.LogDebug("Documento '{Id}' atualizado no Qdrant", id); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao salvar documento '{Title}' no Qdrant", titulo); + throw; + } + } + + public async Task SalvarTextoComEmbeddingNoMongoDB(string textoCompleto, string projectId) + { + try + { + var textoArray = new List(); + string[] textolinhas = textoCompleto.Split( + new string[] { "\n" }, + StringSplitOptions.None + ); + + var title = textolinhas[0]; + var builder = new StringBuilder(); + + foreach (string line in textolinhas) + { + if (line.StartsWith("**") || line.StartsWith("\r**")) + { + if (builder.Length > 0) + { + textoArray.Add(title.Replace("**", "").Replace("\r", "") + ": " + Environment.NewLine + builder.ToString()); + builder = new StringBuilder(); + title = line; + } + } + else + { + builder.AppendLine(line); + } + } + + // Adiciona último bloco se houver + if (builder.Length > 0) + { + textoArray.Add(title.Replace("**", "").Replace("\r", "") + ": " + Environment.NewLine + builder.ToString()); + } + + // Processa cada seção + foreach (var item in textoArray) + { + await SalvarNoMongoDB(title.Replace("**", "").Replace("\r", ""), item, projectId); + } + + _logger.LogInformation("Texto completo processado: {SectionCount} seções salvas no Qdrant", textoArray.Count); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao processar texto completo no Qdrant"); + throw; + } + } + + public async Task> GetAll() + { + try + { + // Busca todos os projetos e depois todos os documentos + var allDocuments = new List(); + + // Como Qdrant não tem um "GetAll" direto, vamos usar scroll + // Isso é uma limitação vs MongoDB, mas é mais eficiente + var projects = await GetAllProjectIds(); + + foreach (var projectId in projects) + { + var projectDocs = await _vectorSearchService.GetDocumentsByProjectAsync(projectId); + allDocuments.AddRange(projectDocs); + } + + return allDocuments.Select(ConvertToTextoComEmbedding); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao recuperar todos os documentos do Qdrant"); + throw; + } + } + + public async Task> GetByPorjectId(string projectId) + { + try + { + var documents = await _vectorSearchService.GetDocumentsByProjectAsync(projectId); + return documents.Select(ConvertToTextoComEmbedding); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao recuperar documentos do projeto {ProjectId} no Qdrant", projectId); + throw; + } + } + + public async Task GetById(string id) + { + try + { + var document = await _vectorSearchService.GetDocumentAsync(id); + if (document == null) + { + throw new ArgumentException($"Documento {id} não encontrado no Qdrant"); + } + + return ConvertToTextoComEmbedding(document); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao recuperar documento {Id} do Qdrant", id); + throw; + } + } + + // ======================================== + // MÉTODOS NOVOS DA INTERFACE + // ======================================== + + public async Task SaveDocumentAsync(DocumentInput document) + { + try + { + var conteudo = $"**{document.Title}** \n\n {document.Content}"; + var embedding = await _embeddingService.GenerateEmbeddingAsync(conteudo); + var embeddingArray = embedding.ToArray().Select(e => (double)e).ToArray(); + + string id; + if (!string.IsNullOrEmpty(document.Id)) + { + // Atualizar documento existente + await _vectorSearchService.UpdateDocumentAsync( + document.Id, + document.Title, + document.Content, + document.ProjectId, + embeddingArray, + document.Metadata); + id = document.Id; + } + else + { + // Criar novo documento + id = await _vectorSearchService.AddDocumentAsync( + document.Title, + document.Content, + document.ProjectId, + embeddingArray, + document.Metadata); + } + + _logger.LogDebug("Documento {Id} salvo no Qdrant via SaveDocumentAsync", id); + return id; + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao salvar documento no Qdrant"); + throw; + } + } + + public async Task UpdateDocumentAsync(string id, DocumentInput document) + { + try + { + var conteudo = $"**{document.Title}** \n\n {document.Content}"; + var embedding = await _embeddingService.GenerateEmbeddingAsync(conteudo); + var embeddingArray = embedding.ToArray().Select(e => (double)e).ToArray(); + + await _vectorSearchService.UpdateDocumentAsync( + id, + document.Title, + document.Content, + document.ProjectId, + embeddingArray, + document.Metadata); + + _logger.LogDebug("Documento {Id} atualizado no Qdrant", id); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao atualizar documento {Id} no Qdrant", id); + throw; + } + } + + public async Task DeleteDocumentAsync(string id) + { + try + { + await _vectorSearchService.DeleteDocumentAsync(id); + _logger.LogDebug("Documento {Id} deletado do Qdrant", id); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao deletar documento {Id} do Qdrant", id); + throw; + } + } + + public async Task DocumentExistsAsync(string id) + { + try + { + return await _vectorSearchService.DocumentExistsAsync(id); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao verificar existência do documento {Id} no Qdrant", id); + return false; + } + } + + public async Task GetDocumentAsync(string id) + { + try + { + var result = await _vectorSearchService.GetDocumentAsync(id); + if (result == null) return null; + + return new DocumentOutput + { + Id = result.Id, + Title = result.Title, + Content = result.Content, + ProjectId = result.ProjectId, + Embedding = result.Embedding, + CreatedAt = result.CreatedAt, + UpdatedAt = result.UpdatedAt, + Metadata = result.Metadata + }; + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao recuperar documento {Id} do Qdrant", id); + return null; + } + } + + public async Task> GetDocumentsByProjectAsync(string projectId) + { + try + { + var results = await _vectorSearchService.GetDocumentsByProjectAsync(projectId); + + return results.Select(result => new DocumentOutput + { + Id = result.Id, + Title = result.Title, + Content = result.Content, + ProjectId = result.ProjectId, + Embedding = result.Embedding, + CreatedAt = result.CreatedAt, + UpdatedAt = result.UpdatedAt, + Metadata = result.Metadata + }).ToList(); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao recuperar documentos do projeto {ProjectId} do Qdrant", projectId); + throw; + } + } + + public async Task GetDocumentCountAsync(string? projectId = null) + { + try + { + return await _vectorSearchService.GetDocumentCountAsync(projectId); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao contar documentos no Qdrant"); + return 0; + } + } + + // ======================================== + // OPERAÇÕES EM LOTE + // ======================================== + + public async Task> SaveDocumentsBatchAsync(List documents) + { + var ids = new List(); + var errors = new List(); + + // Processa em lotes menores para performance + var batchSize = 10; + for (int i = 0; i < documents.Count; i += batchSize) + { + var batch = documents.Skip(i).Take(batchSize); + var tasks = batch.Select(async doc => + { + try + { + var id = await SaveDocumentAsync(doc); + return id; + } + catch (Exception ex) + { + errors.Add(ex); + _logger.LogError(ex, "Erro ao salvar documento '{Title}' em lote", doc.Title); + return null; + } + }); + + var batchResults = await Task.WhenAll(tasks); + ids.AddRange(batchResults.Where(id => id != null)!); + } + + if (errors.Any()) + { + _logger.LogWarning("Batch save completado com {ErrorCount} erros de {TotalCount} documentos", + errors.Count, documents.Count); + } + + _logger.LogInformation("Batch save: {SuccessCount}/{TotalCount} documentos salvos no Qdrant", + ids.Count, documents.Count); + + return ids; + } + + public async Task DeleteDocumentsBatchAsync(List ids) + { + var errors = new List(); + + // Processa em lotes para não sobrecarregar + var batchSize = 20; + for (int i = 0; i < ids.Count; i += batchSize) + { + var batch = ids.Skip(i).Take(batchSize); + var tasks = batch.Select(async id => + { + try + { + await DeleteDocumentAsync(id); + return true; + } + catch (Exception ex) + { + errors.Add(ex); + _logger.LogError(ex, "Erro ao deletar documento {Id} em lote", id); + return false; + } + }); + + await Task.WhenAll(tasks); + } + + if (errors.Any()) + { + _logger.LogWarning("Batch delete completado com {ErrorCount} erros de {TotalCount} documentos", + errors.Count, ids.Count); + } + else + { + _logger.LogInformation("Batch delete: {TotalCount} documentos removidos do Qdrant", ids.Count); + } + } + + // ======================================== + // ESTATÍSTICAS DO PROVIDER + // ======================================== + + public async Task> GetProviderStatsAsync() + { + try + { + var baseStats = await _vectorSearchService.GetStatsAsync(); + var totalDocs = await GetDocumentCountAsync(); + + // Adiciona estatísticas específicas do TextData + var projectIds = await GetAllProjectIds(); + var projectStats = new Dictionary(); + + foreach (var projectId in projectIds) + { + var count = await GetDocumentCountAsync(projectId); + projectStats[projectId] = count; + } + + var enhancedStats = new Dictionary(baseStats) + { + ["text_service_provider"] = "Qdrant", + ["total_documents_via_text_service"] = totalDocs, + ["projects_count"] = projectIds.Count, + ["documents_by_project"] = projectStats, + ["supports_batch_operations"] = true, + ["supports_metadata"] = true, + ["embedding_auto_generation"] = true + }; + + return enhancedStats; + } + catch (Exception ex) + { + return new Dictionary + { + ["provider"] = "Qdrant", + ["text_service_provider"] = "Qdrant", + ["health"] = "error", + ["error"] = ex.Message, + ["last_check"] = DateTime.UtcNow + }; + } + } + + // ======================================== + // MÉTODOS AUXILIARES PRIVADOS + // ======================================== + + private static TextoComEmbedding ConvertToTextoComEmbedding(VectorSearchResult result) + { + return new TextoComEmbedding + { + Id = result.Id, + Titulo = result.Title, + Conteudo = result.Content, + ProjetoId = result.ProjectId, + Embedding = result.Embedding, + // Campos que podem não existir no Qdrant + ProjetoNome = result.Metadata?.GetValueOrDefault("project_name")?.ToString() ?? "", + TipoDocumento = result.Metadata?.GetValueOrDefault("document_type")?.ToString() ?? "", + Categoria = result.Metadata?.GetValueOrDefault("category")?.ToString() ?? "", + Tags = result.Metadata?.GetValueOrDefault("tags") as string[] ?? Array.Empty() + }; + } + + private async Task> GetAllProjectIds() + { + try + { + // Esta é uma operação custosa no Qdrant + // Em produção, seria melhor manter um cache de project IDs + // ou usar uma estrutura de dados separada + + // Por agora, vamos usar uma busca com um vetor dummy para pegar todos os documentos + var dummyVector = new double[1536]; // Assumindo embeddings OpenAI + var allResults = await _vectorSearchService.SearchSimilarAsync( + dummyVector, + projectId: null, + threshold: 0.0, + limit: 10000); + + return allResults + .Select(r => r.ProjectId) + .Where(pid => !string.IsNullOrEmpty(pid)) + .Distinct() + .ToList(); + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro ao recuperar IDs de projetos do Qdrant"); + return new List(); + } + } + } +} + +#pragma warning restore SKEXP0001 \ No newline at end of file diff --git a/Services/VectorDatabaseHealthCheck.cs b/Services/VectorDatabaseHealthCheck.cs new file mode 100644 index 0000000..14cd240 --- /dev/null +++ b/Services/VectorDatabaseHealthCheck.cs @@ -0,0 +1,64 @@ +using ChatRAG.Services.Contracts; +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace ChatRAG.Services +{ + public class VectorDatabaseHealthCheck : IHealthCheck + { + private readonly IVectorDatabaseFactory _factory; + private readonly ILogger _logger; + + public VectorDatabaseHealthCheck( + IVectorDatabaseFactory factory, + ILogger logger) + { + _factory = factory; + _logger = logger; + } + + public async Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default) + { + try + { + var provider = _factory.GetActiveProvider(); + var vectorService = _factory.CreateVectorSearchService(); + var textService = _factory.CreateTextDataService(); + + // Testa conectividade básica + var isHealthy = await vectorService.IsHealthyAsync(); + var stats = await vectorService.GetStatsAsync(); + var providerStats = await textService.GetProviderStatsAsync(); + + var data = new Dictionary + { + ["provider"] = provider, + ["vector_service_healthy"] = isHealthy, + ["total_documents"] = stats.GetValueOrDefault("total_documents", 0), + ["provider_stats"] = providerStats + }; + + if (isHealthy) + { + _logger.LogDebug("Vector Database health check passou para provider {Provider}", provider); + return HealthCheckResult.Healthy($"Vector Database ({provider}) está saudável", data); + } + else + { + _logger.LogWarning("Vector Database health check falhou para provider {Provider}", provider); + return HealthCheckResult.Unhealthy($"Vector Database ({provider}) não está saudável", data: data); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Erro no health check do Vector Database"); + return HealthCheckResult.Unhealthy("Erro no Vector Database", ex, new Dictionary + { + ["provider"] = _factory.GetActiveProvider(), + ["error"] = ex.Message + }); + } + } + } +} diff --git a/Settings/VectorDatabaseSettings.cs b/Settings/VectorDatabaseSettings.cs new file mode 100644 index 0000000..5be374c --- /dev/null +++ b/Settings/VectorDatabaseSettings.cs @@ -0,0 +1,494 @@ +namespace ChatRAG.Settings +{ + // ============================================================================ + // 📁 Configuration/VectorDatabaseSettings.cs + // Settings unificados para todos os providers (MongoDB, Qdrant, etc.) + // ============================================================================ + + namespace ChatRAG.Configuration + { + /// + /// Configurações principais do sistema de Vector Database + /// + public class VectorDatabaseSettings + { + /// + /// Provider ativo (MongoDB, Qdrant, Pinecone, etc.) + /// + public string Provider { get; set; } = "MongoDB"; + + /// + /// Configurações específicas do MongoDB + /// + public MongoDbSettings MongoDB { get; set; } = new(); + + /// + /// Configurações específicas do Qdrant + /// + public QdrantSettings Qdrant { get; set; } = new(); + + /// + /// Configurações globais de embedding + /// + public EmbeddingSettings Embedding { get; set; } = new(); + + /// + /// Configurações de performance e cache + /// + public PerformanceSettings Performance { get; set; } = new(); + + /// + /// Configurações de logging e monitoramento + /// + public MonitoringSettings Monitoring { get; set; } = new(); + + /// + /// Valida se as configurações estão corretas + /// + public bool IsValid() + { + if (string.IsNullOrWhiteSpace(Provider)) + return false; + + return Provider.ToLower() switch + { + "mongodb" => MongoDB.IsValid(), + "qdrant" => Qdrant.IsValid(), + _ => false + }; + } + + /// + /// Retorna erros de validação + /// + public List GetValidationErrors() + { + var errors = new List(); + + if (string.IsNullOrWhiteSpace(Provider)) + errors.Add("Provider é obrigatório"); + + switch (Provider.ToLower()) + { + case "mongodb": + errors.AddRange(MongoDB.GetValidationErrors()); + break; + case "qdrant": + errors.AddRange(Qdrant.GetValidationErrors()); + break; + default: + errors.Add($"Provider '{Provider}' não é suportado"); + break; + } + + errors.AddRange(Embedding.GetValidationErrors()); + + return errors; + } + } + + /// + /// Configurações específicas do MongoDB + /// + public class MongoDbSettings + { + /// + /// String de conexão do MongoDB + /// + public string ConnectionString { get; set; } = string.Empty; + + /// + /// Nome do banco de dados + /// + public string DatabaseName { get; set; } = string.Empty; + + /// + /// Nome da coleção de textos/documentos + /// + public string TextCollectionName { get; set; } = "texts"; + + /// + /// Nome da coleção de projetos + /// + public string ProjectCollectionName { get; set; } = "projects"; + + /// + /// Nome da coleção de dados de usuário + /// + public string UserDataName { get; set; } = "users"; + + /// + /// Timeout de conexão em segundos + /// + public int ConnectionTimeoutSeconds { get; set; } = 30; + + /// + /// Timeout de operação em segundos + /// + public int OperationTimeoutSeconds { get; set; } = 60; + + /// + /// Tamanho máximo do pool de conexões + /// + public int MaxConnectionPoolSize { get; set; } = 100; + + /// + /// Habilitar índices de busca vetorial + /// + public bool EnableVectorSearch { get; set; } = true; + + /// + /// Configurações específicas do Atlas Search + /// + public MongoAtlasSearchSettings AtlasSearch { get; set; } = new(); + + public bool IsValid() + { + return !string.IsNullOrWhiteSpace(ConnectionString) && + !string.IsNullOrWhiteSpace(DatabaseName) && + !string.IsNullOrWhiteSpace(TextCollectionName); + } + + public List GetValidationErrors() + { + var errors = new List(); + + if (string.IsNullOrWhiteSpace(ConnectionString)) + errors.Add("MongoDB ConnectionString é obrigatória"); + + if (string.IsNullOrWhiteSpace(DatabaseName)) + errors.Add("MongoDB DatabaseName é obrigatório"); + + if (string.IsNullOrWhiteSpace(TextCollectionName)) + errors.Add("MongoDB TextCollectionName é obrigatório"); + + if (ConnectionTimeoutSeconds <= 0) + errors.Add("MongoDB ConnectionTimeoutSeconds deve ser maior que 0"); + + return errors; + } + } + + /// + /// Configurações do MongoDB Atlas Search + /// + public class MongoAtlasSearchSettings + { + /// + /// Nome do índice de busca vetorial + /// + public string VectorIndexName { get; set; } = "vector_index"; + + /// + /// Número de candidatos para busca aproximada + /// + public int NumCandidates { get; set; } = 200; + + /// + /// Limite de resultados do Atlas Search + /// + public int SearchLimit { get; set; } = 100; + } + + /// + /// Configurações específicas do Qdrant + /// + public class QdrantSettings + { + /// + /// Host do servidor Qdrant + /// + public string Host { get; set; } = "localhost"; + + /// + /// Porta do servidor Qdrant (REST API) + /// + public int Port { get; set; } = 6333; + + /// + /// Porta gRPC (opcional, para performance) + /// + public int? GrpcPort { get; set; } = 6334; + + /// + /// Chave de API (se autenticação estiver habilitada) + /// + public string? ApiKey { get; set; } + + /// + /// Usar TLS/SSL + /// + public bool UseTls { get; set; } = false; + + /// + /// Nome da coleção principal + /// + public string CollectionName { get; set; } = "documents"; + + /// + /// Tamanho do vetor de embedding + /// + public int VectorSize { get; set; } = 1536; // OpenAI embedding size + + /// + /// Métrica de distância (Cosine, Euclid, Dot, Manhattan) + /// + public string Distance { get; set; } = "Cosine"; + + // ======================================== + // CONFIGURAÇÕES DE PERFORMANCE + // ======================================== + + /// + /// Parâmetro M do algoritmo HNSW (conectividade) + /// Valores típicos: 16-48, maior = melhor recall, mais memória + /// + public int HnswM { get; set; } = 16; + + /// + /// Parâmetro ef_construct do HNSW (construção do índice) + /// Valores típicos: 100-800, maior = melhor qualidade, mais lento + /// + public int HnswEfConstruct { get; set; } = 200; + + /// + /// Parâmetro ef do HNSW (busca) + /// Valores típicos: igual ou maior que o número de resultados desejados + /// + public int HnswEf { get; set; } = 128; + + /// + /// Armazenar vetores em disco (economiza RAM) + /// + public bool OnDisk { get; set; } = false; + + /// + /// Fator de replicação (para clusters) + /// + public int ReplicationFactor { get; set; } = 1; + + /// + /// Número de shards (para distribuição) + /// + public int ShardNumber { get; set; } = 1; + + /// + /// Usar quantização para reduzir uso de memória + /// + public bool UseQuantization { get; set; } = false; + + /// + /// Configurações de quantização + /// + public QuantizationSettings Quantization { get; set; } = new(); + + /// + /// Timeout de conexão em segundos + /// + public int ConnectionTimeoutSeconds { get; set; } = 30; + + /// + /// Timeout de operação em segundos + /// + public int OperationTimeoutSeconds { get; set; } = 60; + + /// + /// URL completa calculada + /// + public string GetConnectionUrl() + { + var protocol = UseTls ? "https" : "http"; + return $"{protocol}://{Host}:{Port}"; + } + + public bool IsValid() + { + return !string.IsNullOrWhiteSpace(Host) && + Port > 0 && + !string.IsNullOrWhiteSpace(CollectionName) && + VectorSize > 0; + } + + public List GetValidationErrors() + { + var errors = new List(); + + if (string.IsNullOrWhiteSpace(Host)) + errors.Add("Qdrant Host é obrigatório"); + + if (Port <= 0) + errors.Add("Qdrant Port deve ser maior que 0"); + + if (string.IsNullOrWhiteSpace(CollectionName)) + errors.Add("Qdrant CollectionName é obrigatório"); + + if (VectorSize <= 0) + errors.Add("Qdrant VectorSize deve ser maior que 0"); + + if (HnswM <= 0) + errors.Add("Qdrant HnswM deve ser maior que 0"); + + if (HnswEfConstruct <= 0) + errors.Add("Qdrant HnswEfConstruct deve ser maior que 0"); + + var validDistances = new[] { "Cosine", "Euclid", "Dot", "Manhattan" }; + if (!validDistances.Contains(Distance)) + errors.Add($"Qdrant Distance deve ser um de: {string.Join(", ", validDistances)}"); + + return errors; + } + } + + /// + /// Configurações de quantização para Qdrant + /// + public class QuantizationSettings + { + /// + /// Tipo de quantização (scalar, product, binary) + /// + public string Type { get; set; } = "scalar"; + + /// + /// Quantil para quantização scalar + /// + public double Quantile { get; set; } = 0.99; + + /// + /// Sempre usar RAM para quantização + /// + public bool AlwaysRam { get; set; } = false; + } + + /// + /// Configurações globais de embedding + /// + public class EmbeddingSettings + { + /// + /// Provider de embedding (OpenAI, Ollama, Azure, etc.) + /// + public string Provider { get; set; } = "OpenAI"; + + /// + /// Modelo de embedding + /// + public string Model { get; set; } = "text-embedding-ada-002"; + + /// + /// Tamanho esperado do embedding + /// + public int ExpectedSize { get; set; } = 1536; + + /// + /// Batch size para processamento em lote + /// + public int BatchSize { get; set; } = 100; + + /// + /// Cache de embeddings em memória + /// + public bool EnableCache { get; set; } = true; + + /// + /// TTL do cache em minutos + /// + public int CacheTtlMinutes { get; set; } = 60; + + public List GetValidationErrors() + { + var errors = new List(); + + if (ExpectedSize <= 0) + errors.Add("Embedding ExpectedSize deve ser maior que 0"); + + if (BatchSize <= 0) + errors.Add("Embedding BatchSize deve ser maior que 0"); + + return errors; + } + } + + /// + /// Configurações de performance e otimização + /// + public class PerformanceSettings + { + /// + /// Habilitar paralelização em operações de lote + /// + public bool EnableParallelization { get; set; } = true; + + /// + /// Número máximo de threads paralelas + /// + public int MaxParallelism { get; set; } = Environment.ProcessorCount; + + /// + /// Tamanho do batch para operações em lote + /// + public int BatchSize { get; set; } = 100; + + /// + /// Timeout padrão para operações em segundos + /// + public int DefaultTimeoutSeconds { get; set; } = 30; + + /// + /// Habilitar retry automático + /// + public bool EnableRetry { get; set; } = true; + + /// + /// Número máximo de tentativas + /// + public int MaxRetryAttempts { get; set; } = 3; + + /// + /// Delay entre tentativas em segundos + /// + public int RetryDelaySeconds { get; set; } = 2; + } + + /// + /// Configurações de monitoramento e logging + /// + public class MonitoringSettings + { + /// + /// Habilitar logging detalhado + /// + public bool EnableDetailedLogging { get; set; } = false; + + /// + /// Logar tempos de operação + /// + public bool LogOperationTimes { get; set; } = true; + + /// + /// Threshold para log de operações lentas (ms) + /// + public int SlowOperationThresholdMs { get; set; } = 1000; + + /// + /// Habilitar métricas de performance + /// + public bool EnableMetrics { get; set; } = true; + + /// + /// Intervalo de coleta de métricas em segundos + /// + public int MetricsIntervalSeconds { get; set; } = 60; + + /// + /// Habilitar health checks + /// + public bool EnableHealthChecks { get; set; } = true; + + /// + /// Intervalo de health check em segundos + /// + public int HealthCheckIntervalSeconds { get; set; } = 30; + } + } +} \ No newline at end of file diff --git a/Settings/VectorDatabaseSettingsValidator.cs b/Settings/VectorDatabaseSettingsValidator.cs new file mode 100644 index 0000000..1d61ec9 --- /dev/null +++ b/Settings/VectorDatabaseSettingsValidator.cs @@ -0,0 +1,23 @@ +using ChatRAG.Settings.ChatRAG.Configuration; +using Microsoft.Extensions.Options; + +namespace ChatRAG.Settings +{ + /// + /// Validador para VectorDatabaseSettings + /// + public class VectorDatabaseSettingsValidator : IValidateOptions + { + public ValidateOptionsResult Validate(string name, VectorDatabaseSettings options) + { + var errors = options.GetValidationErrors(); + + if (errors.Any()) + { + return ValidateOptionsResult.Fail(errors); + } + + return ValidateOptionsResult.Success; + } + } +} diff --git a/Tools/MigrationService.cs b/Tools/MigrationService.cs new file mode 100644 index 0000000..9e928a3 --- /dev/null +++ b/Tools/MigrationService.cs @@ -0,0 +1,264 @@ +using ChatRAG.Contracts.VectorSearch; +using ChatRAG.Data; +using ChatRAG.Models; +using ChatRAG.Services.Contracts; +using ChatRAG.Services.Migration; +using ChatRAG.Services.SearchVectors; +using ChatRAG.Settings.ChatRAG.Configuration; +using ChatRAG.Settings; +using Microsoft.Extensions.Options; +using System.Diagnostics; + +namespace ChatRAG.Services.Migration +{ + public class MigrationService + { + private readonly ILogger _logger; + private readonly IServiceProvider _serviceProvider; + + public MigrationService( + ILogger logger, + IServiceProvider serviceProvider) + { + _logger = logger; + _serviceProvider = serviceProvider; + } + + /// + /// Migra todos os dados do MongoDB para Qdrant + /// + public async Task MigrateFromMongoToQdrantAsync( + bool validateData = true, + int batchSize = 50) + { + var stopwatch = Stopwatch.StartNew(); + var result = new MigrationResult { StartTime = DateTime.UtcNow }; + + try + { + _logger.LogInformation("🚀 Iniciando migração MongoDB → Qdrant..."); + + // Cria serviços específicos para migração + var mongoService = CreateMongoService(); + var qdrantService = CreateQdrantService(); + + // 1. Exporta dados do MongoDB + _logger.LogInformation("📤 Exportando dados do MongoDB..."); + var mongoDocuments = await mongoService.GetAll(); + var documentsList = mongoDocuments.ToList(); + + result.TotalDocuments = documentsList.Count; + _logger.LogInformation("✅ {Count} documentos encontrados no MongoDB", result.TotalDocuments); + + if (!documentsList.Any()) + { + _logger.LogWarning("⚠️ Nenhum documento encontrado no MongoDB"); + result.Success = true; + result.Message = "Migração concluída - nenhum documento para migrar"; + return result; + } + + // 2. Agrupa por projeto para migração organizada + var documentsByProject = documentsList.GroupBy(d => d.ProjetoId).ToList(); + _logger.LogInformation("📁 Documentos organizados em {ProjectCount} projetos", documentsByProject.Count); + + // 3. Migra por projeto em lotes + foreach (var projectGroup in documentsByProject) + { + var projectId = projectGroup.Key; + var projectDocs = projectGroup.ToList(); + + _logger.LogInformation("📂 Migrando projeto {ProjectId}: {DocCount} documentos", + projectId, projectDocs.Count); + + // Processa em lotes para não sobrecarregar + for (int i = 0; i < projectDocs.Count; i += batchSize) + { + var batch = projectDocs.Skip(i).Take(batchSize).ToList(); + + try + { + await MigrateBatch(batch, qdrantService); + result.MigratedDocuments += batch.Count; + + _logger.LogDebug("✅ Lote {BatchNum}: {BatchCount} documentos migrados", + (i / batchSize) + 1, batch.Count); + } + catch (Exception ex) + { + _logger.LogError(ex, "❌ Erro no lote {BatchNum} do projeto {ProjectId}", + (i / batchSize) + 1, projectId); + + result.Errors.Add($"Projeto {projectId}, lote {(i / batchSize) + 1}: {ex.Message}"); + } + } + } + + // 4. Validação (se solicitada) + if (validateData) + { + _logger.LogInformation("🔍 Validando dados migrados..."); + var validationResult = await ValidateMigration(mongoService, qdrantService); + result.ValidationResult = validationResult; + + if (!validationResult.IsValid) + { + _logger.LogWarning("⚠️ Validação encontrou inconsistências: {Issues}", + string.Join(", ", validationResult.Issues)); + } + else + { + _logger.LogInformation("✅ Validação passou - dados consistentes"); + } + } + + stopwatch.Stop(); + result.Duration = stopwatch.Elapsed; + result.Success = true; + result.Message = $"Migração concluída: {result.MigratedDocuments}/{result.TotalDocuments} documentos"; + + _logger.LogInformation("🎉 Migração concluída em {Duration}s: {MigratedCount}/{TotalCount} documentos", + result.Duration.TotalSeconds, result.MigratedDocuments, result.TotalDocuments); + + return result; + } + catch (Exception ex) + { + stopwatch.Stop(); + result.Duration = stopwatch.Elapsed; + result.Success = false; + result.Message = $"Erro na migração: {ex.Message}"; + result.Errors.Add(ex.ToString()); + + _logger.LogError(ex, "💥 Erro fatal na migração"); + return result; + } + } + + /// + /// Rollback - remove todos os dados do Qdrant + /// + public async Task RollbackQdrantAsync() + { + try + { + _logger.LogWarning("🔄 Iniciando rollback - removendo dados do Qdrant..."); + + var qdrantService = CreateQdrantService(); + + // Busca todos os documentos + var allDocuments = await qdrantService.GetAll(); + var documentIds = allDocuments.Select(d => d.Id).ToList(); + + if (!documentIds.Any()) + { + _logger.LogInformation("ℹ️ Nenhum documento encontrado no Qdrant para rollback"); + return true; + } + + // Remove em lotes + var batchSize = 100; + for (int i = 0; i < documentIds.Count; i += batchSize) + { + var batch = documentIds.Skip(i).Take(batchSize).ToList(); + await qdrantService.DeleteDocumentsBatchAsync(batch); + + _logger.LogDebug("🗑️ Lote {BatchNum}: {BatchCount} documentos removidos", + (i / batchSize) + 1, batch.Count); + } + + _logger.LogInformation("✅ Rollback concluído: {Count} documentos removidos do Qdrant", documentIds.Count); + return true; + } + catch (Exception ex) + { + _logger.LogError(ex, "❌ Erro no rollback"); + return false; + } + } + + // ======================================== + // MÉTODOS AUXILIARES + // ======================================== + + private async Task MigrateBatch(List batch, ITextDataService qdrantService) + { + var documents = batch.Select(doc => new DocumentInput + { + Id = doc.Id, + Title = doc.Titulo, + Content = doc.Conteudo, + ProjectId = doc.ProjetoId, + CreatedAt = DateTime.UtcNow, + UpdatedAt = DateTime.UtcNow, + Metadata = new Dictionary + { + ["migrated_from"] = "mongodb", + ["migration_date"] = DateTime.UtcNow.ToString("O"), + ["original_id"] = doc.Id, + ["project_name"] = doc.ProjetoNome ?? "", + ["document_type"] = doc.TipoDocumento ?? "", + ["category"] = doc.Categoria ?? "" + } + }).ToList(); + + await qdrantService.SaveDocumentsBatchAsync(documents); + } + + private async Task ValidateMigration(ITextDataService mongoService, ITextDataService qdrantService) + { + var result = new ValidationResult(); + + try + { + // Compara contagens + var mongoCount = await mongoService.GetDocumentCountAsync(); + var qdrantCount = await qdrantService.GetDocumentCountAsync(); + + if (mongoCount != qdrantCount) + { + result.Issues.Add($"Contagem divergente: MongoDB({mongoCount}) vs Qdrant({qdrantCount})"); + } + + // Valida alguns documentos aleatoriamente + var mongoDocuments = await mongoService.GetAll(); + var sampleDocs = mongoDocuments.Take(10).ToList(); + + foreach (var mongoDoc in sampleDocs) + { + var qdrantDoc = await qdrantService.GetDocumentAsync(mongoDoc.Id); + + if (qdrantDoc == null) + { + result.Issues.Add($"Documento {mongoDoc.Id} não encontrado no Qdrant"); + } + else if (qdrantDoc.Title != mongoDoc.Titulo || qdrantDoc.Content != mongoDoc.Conteudo) + { + result.Issues.Add($"Conteúdo divergente no documento {mongoDoc.Id}"); + } + } + + result.IsValid = !result.Issues.Any(); + return result; + } + catch (Exception ex) + { + result.Issues.Add($"Erro na validação: {ex.Message}"); + result.IsValid = false; + return result; + } + } + + private ITextDataService CreateMongoService() + { + // Força usar MongoDB independente da configuração + return _serviceProvider.GetRequiredService(); + } + + private ITextDataService CreateQdrantService() + { + // Força usar Qdrant independente da configuração + return _serviceProvider.GetRequiredService(); + } + } +} diff --git a/Tools/PerformanceTester.cs b/Tools/PerformanceTester.cs new file mode 100644 index 0000000..4eb5b80 --- /dev/null +++ b/Tools/PerformanceTester.cs @@ -0,0 +1,6 @@ +namespace ChatRAG.Tools +{ + public class PerformanceTester + { + } +} diff --git a/appsettings.json b/appsettings.json index bf802ce..987e7d1 100644 --- a/appsettings.json +++ b/appsettings.json @@ -13,7 +13,30 @@ "Microsoft.AspNetCore.DataProtection": "None" } }, + "VectorDatabase": { + "Provider": "MongoDB", // 👈 Mude para "Qdrant" quando quiser testar + "MongoDB": { + "ConnectionString": "sua_connection_string_atual", + "DatabaseName": "seu_database_atual", + "TextCollectionName": "seu_collection_atual", + "ProjectCollectionName": "seu_project_collection", + "UserDataName": "seu_user_collection" + }, + "Qdrant": { + "Host": "localhost", + "Port": 6334, + "CollectionName": "documents", + "VectorSize": 1536, + "Distance": "Cosine", + "HnswM": 16, + "HnswEfConstruct": 200, + "OnDisk": false + } + }, + "Features": { + "UseQdrant": true + }, "AllowedHosts": "*", "AppTenantId": "20190830-5fd4-4a72-b8fd-1c1cb35b25bc", - "AppClientID": "8f4248fc-ee30-4f54-8793-66edcca3fd20", + "AppClientID": "8f4248fc-ee30-4f54-8793-66edcca3fd20" }