419 lines
15 KiB
C#
419 lines
15 KiB
C#
using ChatApi.Data;
|
|
using ChatRAG.Contracts.VectorSearch;
|
|
using ChatRAG.Data;
|
|
using ChatRAG.Models;
|
|
using ChatRAG.Services.Contracts;
|
|
using ChatRAG.Services.SearchVectors;
|
|
using ChatRAG.Settings.ChatRAG.Configuration;
|
|
using Microsoft.AspNetCore.Mvc;
|
|
using Microsoft.Extensions.Options;
|
|
using Microsoft.SemanticKernel.Embeddings;
|
|
|
|
#pragma warning disable SKEXP0001
|
|
|
|
namespace ChatApi.Controllers
|
|
{
|
|
[ApiController]
|
|
[Route("api/[controller]")]
|
|
public class MigrationController : ControllerBase
|
|
{
|
|
private readonly IVectorDatabaseFactory _factory;
|
|
private readonly ILogger<MigrationController> _logger;
|
|
private readonly VectorDatabaseSettings _settings;
|
|
private readonly ITextEmbeddingGenerationService _embeddingService;
|
|
private readonly TextDataRepository _mongoRepository;
|
|
|
|
public MigrationController(
|
|
IVectorDatabaseFactory factory,
|
|
ILogger<MigrationController> logger,
|
|
IOptions<VectorDatabaseSettings> settings,
|
|
ITextEmbeddingGenerationService embeddingService,
|
|
TextDataRepository mongoRepository)
|
|
{
|
|
_factory = factory;
|
|
_logger = logger;
|
|
_settings = settings.Value;
|
|
_embeddingService = embeddingService;
|
|
_mongoRepository = mongoRepository;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Status da migração e informações dos providers
|
|
/// </summary>
|
|
[HttpGet("status")]
|
|
public async Task<IActionResult> GetMigrationStatus()
|
|
{
|
|
try
|
|
{
|
|
var currentProvider = _factory.GetActiveProvider();
|
|
|
|
var status = new
|
|
{
|
|
CurrentProvider = currentProvider,
|
|
Settings = new
|
|
{
|
|
Provider = _settings.Provider,
|
|
MongoDB = _settings.MongoDB != null ? new
|
|
{
|
|
DatabaseName = _settings.MongoDB.DatabaseName,
|
|
TextCollection = _settings.MongoDB.TextCollectionName
|
|
} : null,
|
|
Qdrant = _settings.Qdrant != null ? new
|
|
{
|
|
Host = _settings.Qdrant.Host,
|
|
Port = _settings.Qdrant.Port,
|
|
Collection = _settings.Qdrant.CollectionName,
|
|
VectorSize = _settings.Qdrant.VectorSize
|
|
} : null
|
|
},
|
|
Stats = await GetProvidersStats()
|
|
};
|
|
|
|
return Ok(status);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex, "Erro ao obter status da migração");
|
|
return StatusCode(500, new { error = ex.Message });
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Migra dados do MongoDB para Qdrant
|
|
/// </summary>
|
|
[HttpPost("mongo-to-qdrant")]
|
|
public async Task<IActionResult> MigrateMongoToQdrant(
|
|
[FromQuery] string? projectId = null,
|
|
[FromQuery] int batchSize = 50,
|
|
[FromQuery] bool dryRun = false)
|
|
{
|
|
try
|
|
{
|
|
if (_settings.Provider != "MongoDB")
|
|
{
|
|
return BadRequest("Migração só funciona quando o provider atual é MongoDB");
|
|
}
|
|
|
|
_logger.LogInformation("Iniciando migração MongoDB → Qdrant. ProjectId: {ProjectId}, DryRun: {DryRun}",
|
|
projectId, dryRun);
|
|
|
|
var result = await PerformMigration(projectId, batchSize, dryRun);
|
|
|
|
return Ok(result);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex, "Erro durante migração MongoDB → Qdrant");
|
|
return StatusCode(500, new { error = ex.Message });
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Migra dados do Qdrant para MongoDB
|
|
/// </summary>
|
|
[HttpPost("qdrant-to-mongo")]
|
|
public async Task<IActionResult> MigrateQdrantToMongo(
|
|
[FromQuery] string? projectId = null,
|
|
[FromQuery] int batchSize = 50,
|
|
[FromQuery] bool dryRun = false)
|
|
{
|
|
try
|
|
{
|
|
if (_settings.Provider != "Qdrant")
|
|
{
|
|
return BadRequest("Migração só funciona quando o provider atual é Qdrant");
|
|
}
|
|
|
|
_logger.LogInformation("Iniciando migração Qdrant → MongoDB. ProjectId: {ProjectId}, DryRun: {DryRun}",
|
|
projectId, dryRun);
|
|
|
|
// TODO: Implementar migração reversa se necessário
|
|
return BadRequest("Migração Qdrant → MongoDB ainda não implementada");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex, "Erro durante migração Qdrant → MongoDB");
|
|
return StatusCode(500, new { error = ex.Message });
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Compara dados entre MongoDB e Qdrant
|
|
/// </summary>
|
|
[HttpPost("compare")]
|
|
public async Task<IActionResult> CompareProviders([FromQuery] string? projectId = null)
|
|
{
|
|
try
|
|
{
|
|
// Cria serviços para ambos os providers manualmente
|
|
var mongoService = CreateMongoService();
|
|
var qdrantService = await CreateQdrantService();
|
|
|
|
var comparison = await CompareData(mongoService, qdrantService, projectId);
|
|
|
|
return Ok(comparison);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex, "Erro ao comparar providers");
|
|
return StatusCode(500, new { error = ex.Message });
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Limpa todos os dados do provider atual
|
|
/// </summary>
|
|
[HttpDelete("clear-current")]
|
|
public async Task<IActionResult> ClearCurrentProvider([FromQuery] string? projectId = null)
|
|
{
|
|
try
|
|
{
|
|
var vectorSearchService = _factory.CreateVectorSearchService();
|
|
|
|
if (string.IsNullOrEmpty(projectId))
|
|
{
|
|
// Limpar tudo - PERIGOSO!
|
|
return BadRequest("Limpeza completa requer confirmação. Use /clear-current/confirm");
|
|
}
|
|
|
|
// Limpar apenas um projeto
|
|
var documents = await vectorSearchService.GetDocumentsByProjectAsync(projectId);
|
|
var ids = documents.Select(d => d.Id).ToList();
|
|
|
|
foreach (var id in ids)
|
|
{
|
|
await vectorSearchService.DeleteDocumentAsync(id);
|
|
}
|
|
|
|
return Ok(new
|
|
{
|
|
provider = _settings.Provider,
|
|
projectId,
|
|
deletedCount = ids.Count
|
|
});
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex, "Erro ao limpar dados do provider");
|
|
return StatusCode(500, new { error = ex.Message });
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Testa conectividade dos providers
|
|
/// </summary>
|
|
[HttpGet("test-connections")]
|
|
public async Task<IActionResult> TestConnections()
|
|
{
|
|
var results = new Dictionary<string, object>();
|
|
|
|
// Testar MongoDB
|
|
try
|
|
{
|
|
var mongoService = CreateMongoService();
|
|
var mongoHealth = await mongoService.IsHealthyAsync();
|
|
var mongoStats = await mongoService.GetStatsAsync();
|
|
|
|
results["MongoDB"] = new
|
|
{
|
|
healthy = mongoHealth,
|
|
stats = mongoStats
|
|
};
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
results["MongoDB"] = new { healthy = false, error = ex.Message };
|
|
}
|
|
|
|
// Testar Qdrant
|
|
try
|
|
{
|
|
var qdrantService = await CreateQdrantService();
|
|
var qdrantHealth = await qdrantService.IsHealthyAsync();
|
|
var qdrantStats = await qdrantService.GetStatsAsync();
|
|
|
|
results["Qdrant"] = new
|
|
{
|
|
healthy = qdrantHealth,
|
|
stats = qdrantStats
|
|
};
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
results["Qdrant"] = new { healthy = false, error = ex.Message };
|
|
}
|
|
|
|
return Ok(results);
|
|
}
|
|
|
|
// ========================================
|
|
// MÉTODOS PRIVADOS
|
|
// ========================================
|
|
|
|
private async Task<object> PerformMigration(string? projectId, int batchSize, bool dryRun)
|
|
{
|
|
var mongoService = CreateMongoService();
|
|
var qdrantService = await CreateQdrantService();
|
|
|
|
// 1. Buscar dados do MongoDB
|
|
List<TextoComEmbedding> documents;
|
|
if (string.IsNullOrEmpty(projectId))
|
|
{
|
|
var allDocs = await _mongoRepository.GetAsync();
|
|
documents = allDocs.ToList();
|
|
}
|
|
else
|
|
{
|
|
var projectDocs = await _mongoRepository.GetByProjectIdAsync(projectId);
|
|
documents = projectDocs.ToList();
|
|
}
|
|
|
|
_logger.LogInformation("Encontrados {Count} documentos para migração", documents.Count);
|
|
|
|
if (dryRun)
|
|
{
|
|
return new
|
|
{
|
|
dryRun = true,
|
|
documentsFound = documents.Count,
|
|
projects = documents.GroupBy(d => d.ProjetoId).Select(g => new {
|
|
projectId = g.Key,
|
|
count = g.Count()
|
|
}).ToList()
|
|
};
|
|
}
|
|
|
|
// 2. Migrar em batches
|
|
var migrated = 0;
|
|
var errors = new List<string>();
|
|
|
|
for (int i = 0; i < documents.Count; i += batchSize)
|
|
{
|
|
var batch = documents.Skip(i).Take(batchSize);
|
|
|
|
foreach (var doc in batch)
|
|
{
|
|
try
|
|
{
|
|
// Verificar se já existe no Qdrant
|
|
var exists = await qdrantService.DocumentExistsAsync(doc.Id);
|
|
if (exists)
|
|
{
|
|
_logger.LogDebug("Documento {Id} já existe no Qdrant, pulando", doc.Id);
|
|
continue;
|
|
}
|
|
|
|
// Migrar documento
|
|
await qdrantService.AddDocumentAsync(
|
|
title: doc.Titulo,
|
|
content: doc.Conteudo,
|
|
projectId: doc.ProjetoId,
|
|
embedding: doc.Embedding,
|
|
metadata: new Dictionary<string, object>
|
|
{
|
|
["project_name"] = doc.ProjetoNome ?? "",
|
|
["document_type"] = doc.TipoDocumento ?? "",
|
|
["category"] = doc.Categoria ?? "",
|
|
["tags"] = doc.Tags ?? Array.Empty<string>(),
|
|
["migrated_from"] = "MongoDB",
|
|
["migrated_at"] = DateTime.UtcNow.ToString("O")
|
|
}
|
|
);
|
|
|
|
migrated++;
|
|
|
|
if (migrated % 10 == 0)
|
|
{
|
|
_logger.LogInformation("Migrados {Migrated}/{Total} documentos", migrated, documents.Count);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
var error = $"Erro ao migrar documento {doc.Id}: {ex.Message}";
|
|
errors.Add(error);
|
|
_logger.LogError(ex, error);
|
|
}
|
|
}
|
|
}
|
|
|
|
return new
|
|
{
|
|
totalDocuments = documents.Count,
|
|
migrated,
|
|
errors = errors.Count,
|
|
errorDetails = errors.Take(10).ToList(), // Primeiros 10 erros
|
|
batchSize,
|
|
duration = "Completed"
|
|
};
|
|
}
|
|
|
|
private async Task<object> CompareData(
|
|
IVectorSearchService mongoService,
|
|
IVectorSearchService qdrantService,
|
|
string? projectId)
|
|
{
|
|
var mongoStats = await mongoService.GetStatsAsync();
|
|
var qdrantStats = await qdrantService.GetStatsAsync();
|
|
|
|
var mongoCount = await mongoService.GetDocumentCountAsync(projectId);
|
|
var qdrantCount = await qdrantService.GetDocumentCountAsync(projectId);
|
|
|
|
return new
|
|
{
|
|
projectId,
|
|
comparison = new
|
|
{
|
|
MongoDB = new
|
|
{
|
|
documentCount = mongoCount,
|
|
healthy = await mongoService.IsHealthyAsync(),
|
|
stats = mongoStats
|
|
},
|
|
Qdrant = new
|
|
{
|
|
documentCount = qdrantCount,
|
|
healthy = await qdrantService.IsHealthyAsync(),
|
|
stats = qdrantStats
|
|
}
|
|
},
|
|
differences = new
|
|
{
|
|
documentCountDiff = qdrantCount - mongoCount,
|
|
inSync = mongoCount == qdrantCount
|
|
}
|
|
};
|
|
}
|
|
|
|
private MongoVectorSearchService CreateMongoService()
|
|
{
|
|
return new MongoVectorSearchService(_mongoRepository, _embeddingService);
|
|
}
|
|
|
|
private async Task<QdrantVectorSearchService> CreateQdrantService()
|
|
{
|
|
var qdrantSettings = Microsoft.Extensions.Options.Options.Create(_settings);
|
|
var logger = HttpContext.RequestServices.GetService<ILogger<QdrantVectorSearchService>>()!;
|
|
|
|
return new QdrantVectorSearchService(qdrantSettings, logger);
|
|
}
|
|
|
|
private async Task<Dictionary<string, object>> GetProvidersStats()
|
|
{
|
|
var stats = new Dictionary<string, object>();
|
|
|
|
try
|
|
{
|
|
var currentService = _factory.CreateVectorSearchService();
|
|
stats["current"] = await currentService.GetStatsAsync();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
stats["current"] = new { error = ex.Message };
|
|
}
|
|
|
|
return stats;
|
|
}
|
|
}
|
|
}
|
|
|
|
#pragma warning restore SKEXP0001 |