Merge pull request 'fix: ajustes de bypass novamente...' (#29) from release/V1.0.0 into main

Reviewed-on: http://git.carneiro.ddnsfree.com/ricardo/YTExtractor/pulls/29

.gitea/workflows/deploy.yml

@@ -0,0 +1,114 @@
+name: Build and Deploy ASP.NET API
+
+on:
+  push:
+    branches: [ main ]
+
+jobs:
+  build-and-deploy:
+    runs-on: localACDC
+    
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+      
+    - name: Setup .NET
+      uses: actions/setup-dotnet@v4
+      with:
+        dotnet-version: '8.0.x' # ou sua versão
+        
+    - name: Restore dependencies
+      run: dotnet restore 
+      
+    - name: Build application
+      run: dotnet build --configuration Release --no-restore
+      
+    - name: Run tests (opcional)
+      run: dotnet test --no-build --verbosity normal
+      
+    - name: Publish application
+      run: dotnet publish --configuration Release --output ./publish
+
+    # - name: Build Docker image for ARM64
+    #   run: |
+    #     # Criar builder se não existir
+    #     docker buildx create --name arm-builder --use --platform linux/arm64 || docker buildx use arm-builder
+        
+    #     # Build específico para ARM64
+    #     docker buildx build \
+    #       --platform linux/arm64 \
+    #       --tag registry.redecarneir.us/ytextractor:latest \
+    #       --tag registry.redecarneir.us/ytextractor:${{ github.sha }} \
+    #       -f YTExtractor/Dockerfile \
+    #       --push .
+
+    # - name: Build Docker image 1
+    #   run: |
+    #     docker build -t ytextractor:${{ github.sha }} -f YTExtractor/Dockerfile .
+    #     docker tag ytextractor:${{ github.sha }} registry.redecarneir.us/ytextractor:latest
+    #     docker tag ytextractor:${{ github.sha }} registry.redecarneir.us/ytextractor:${{ github.sha }}
+        
+    # - name: Push to registry
+    #   run: |
+    #     COMMIT_SHA=$(git rev-parse --short HEAD)
+    #     docker push registry.redecarneir.us/ytextractor:latest
+    #     docker push registry.redecarneir.us/ytextractor:${{ github.sha }}
+        
+    # - name: Deploy to remote VPS
+    #   run: |
+    #     ssh -o StrictHostKeyChecking=no ubuntu@137.131.63.61 << 'EOF'
+    #       # Pull da nova imagem
+    #       docker pull registry.redecarneir.us/ytextractor:latest
+          
+    #       # Parar container atual se existir
+    #       docker stop ytextractor-api || true
+    #       docker rm ytextractor-api || true
+          
+    #       # Rodar novo container em produção
+    #       # Porta 80 interna (do container) mapeada para 5000 externa
+    #       docker run -d \
+    #         --name ytextractor-api \
+    #         --restart unless-stopped \
+    #         -p 5000:80 \
+    #         -e ASPNETCORE_ENVIRONMENT=Production \
+    #         -v /tmp/ytextractor:/app/temp \
+    #         registry.redecarneir.us/ytextractor:latest
+            
+    #       # Verificar se está rodando
+    #       sleep 5
+    #       docker ps | grep ytextractor-api
+          
+    #       # Limpeza de imagens antigas (opcional)
+    #       docker image prune -f
+    #     EOF
+
+    - name: Deploy and build on VPS
+      run: |
+        # Enviar código para VPS
+        rsync -avz --delete \
+          --exclude '.git' \
+          --exclude 'bin' \
+          --exclude 'obj' \
+          ./ ubuntu@137.131.63.61:~/ytextractor/
+          
+        # Build no VPS (que já é ARM64)
+        ssh ubuntu@137.131.63.61 << 'EOF'
+          cd ~/ytextractor
+          docker build -t registry.redecarneir.us/ytextractor:latest -f YTExtractor/Dockerfile .
+          docker stop ytextractor-api || true
+          docker rm ytextractor-api || true
+          docker run -d \
+            --name ytextractor-api \
+            --restart unless-stopped \
+            -p 8082:8080 \
+            --add-host="k3sw2:172.17.0.1" \
+            --add-host="k3ss1:172.17.0.1" \
+            -e ASPNETCORE_ENVIRONMENT=Production \
+            -v /tmp/ytextractor:/app/temp \
+            registry.redecarneir.us/ytextractor:latest
+        EOF
+        
+    - name: Verify deployment
+      run: |
+        ssh ubuntu@137.131.63.61 'docker ps | grep ytextractor-api'
+        echo "✅ Deploy completed successfully!"

YTExtractor.sln

@@ -5,6 +5,11 @@ VisualStudioVersion = 17.11.35327.3
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "YTExtractor", "YTExtractor\YTExtractor.csproj", "{7DA7D783-153F-42EF-87E4-239DEC80F91A}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "pipeline", "pipeline", "{5F17A7E6-48F5-4970-B8F6-310BBF9A3C50}"
+	ProjectSection(SolutionItems) = preProject
+		.gitea\workflows\deploy.yml = .gitea\workflows\deploy.yml
+	EndProjectSection
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU

YTExtractor/.gitea/workflow/main.yml

@@ -1,99 +0,0 @@
-name: CI/CD Pipeline para YTExtractor
-
-on:
-  push:
-    branches: [ main, 'release/*' ]
-  pull_request:
-    branches: [ main, 'release/*' ]
-
-jobs:
-  build:
-    runs-on: ubuntu-22.04
-    
-    steps:
-      - uses: actions/checkout@v3
-      
-      - name: Setup .NET
-        uses: actions/setup-dotnet@v3
-        with:
-          dotnet-version: '8.0.x'
-          
-      - name: Restore dependencies
-        run: dotnet restore
-        
-      - name: Build
-        run: dotnet build --no-restore
-        
-      - name: Test
-        run: dotnet test --no-build --verbosity normal
-        
-      - name: Build Docker image
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          push: false
-          load: true
-          tags: ytextractor:${{ github.sha }}
-          platforms: linux/amd64,linux/arm64
-          
-      - name: Save Docker image
-        run: docker save ytextractor:${{ github.sha }} > ytextractor-image.tar
-          
-      - name: Upload artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: docker-image
-          path: ytextractor-image.tar
-  
-  deploy-localACDC:
-    needs: build
-    runs-on: ubuntu-22.04
-    if: github.ref == 'refs/heads/main'
-    
-    steps:
-      - name: Download artifact
-        uses: actions/download-artifact@v3
-        with:
-          name: docker-image
-          
-      - name: Deploy to localACDC
-        run: |
-          echo "Implantando no servidor localACDC"
-          scp ytextractor-image.tar user@localACDC:/tmp/
-          ssh user@localACDC "docker load < /tmp/ytextractor-image.tar && \
-                             docker stop ytextractor || true && \
-                             docker rm ytextractor || true && \
-                             docker run -d --name ytextractor -p 80:80 ytextractor:${{ github.sha }}"
-  
-  deploy-pi2Zero:
-    needs: build
-    runs-on: ubuntu-22.04
-    if: startsWith(github.ref, 'refs/heads/release/')
-    
-    steps:
-      - name: Download artifact
-        uses: actions/download-artifact@v3
-        with:
-          name: docker-image
-          
-      - name: Deploy to pi2Zero (Orange Pi Zero)
-        run: |
-          echo "Implantando no Orange Pi Zero (recursos limitados)"
-          # Transfere a imagem para o servidor
-          scp ytextractor-image.tar user@pi2Zero:/tmp/
-          
-          # Comandos específicos para o Orange Pi Zero (otimizados para baixa memória)
-          ssh user@pi2Zero "
-            # Limpar recursos não utilizados
-            docker system prune -f
-            
-            # Parar e remover contêiner existente
-            docker stop ytextractor || true
-            docker rm ytextractor || true
-            
-            # Carregar a nova imagem
-            docker load < /tmp/ytextractor-image.tar
-            
-            # Iniciar o serviço com limites de memória
-            docker run -d --name ytextractor -p 80:80 --memory=300m --memory-swap=600m ytextractor:${{ github.sha }}
-          "

YTExtractor/Data/MongoDbConnector.cs

@@ -7,11 +7,15 @@ namespace YTExtractor.Data
         private readonly IMongoDatabase _database;
         private readonly IMongoCollection<VideoData> _collection;
 
-        public MongoDBConnector(IConfiguration configuration)
+        public MongoDBConnector(IConfiguration configuration, Serilog.ILogger logger)
         {
             var connectionString = configuration.GetSection("MongoDbConnection").Value;
+            var databaseName = configuration.GetSection("MongoDbDatabase").Value;
+
+            logger.Information($"ConnString: {connectionString}");
+            logger.Information($"DbName: {databaseName}");
             var client = new MongoClient(connectionString);
-            _database = client.GetDatabase("YTExtractor");
+            _database = client.GetDatabase(databaseName);
             _collection = _database.GetCollection<VideoData>("videos");
         }
 

YTExtractor/Dockerfile

@@ -1,16 +1,19 @@
+# Build stage
 FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build
 WORKDIR /src
 COPY . .
 RUN dotnet restore
 RUN dotnet publish -c Release -o /app
 
+# Runtime stage
 FROM mcr.microsoft.com/dotnet/aspnet:8.0
 WORKDIR /app
 COPY --from=build /app .
 
+# Install yt-dlp with --break-system-packages flag (necessário no Debian 12)
 RUN apt-get update && \
     apt-get install -y python3-pip && \
-    pip3 install yt-dlp && \
+    pip3 install --break-system-packages yt-dlp && \
     mkdir /app/temp && \
     chmod 777 /app/temp && \
     which yt-dlp || (echo "yt-dlp not found" && exit 1)

YTExtractor/Logging/Configuration/SerilogConfiguration.cs

@@ -7,24 +7,119 @@ namespace YTExtractor.Logging.Configuration
 {
     public static class SerilogConfiguration
     {
-        public static LoggerConfiguration SetLoggerConfiguration(this WebApplicationBuilder builder, LoggerConfiguration config, IServiceProvider services, IConfiguration configuration)
+        public static LoggerConfiguration SetLoggerConfiguration(
+            this WebApplicationBuilder builder,
+            LoggerConfiguration config,
+            IServiceProvider services,
+            IConfiguration configuration)
         {
             var workspace = configuration["Serilog:Properties:Workspace"];
-            var seqServer = configuration.GetValue<string>("Serilog:WriteTo:2:Args:serverUrl"); ;
+            var seqServer = configuration.GetValue<string>("Serilog:WriteTo:2:Args:serverUrl");
 
-            config
-                .ReadFrom.Configuration(configuration)
-                .ReadFrom.Services(services)
-                .Enrich.FromLogContext()
-                .Enrich.WithEnvironmentName()
-                //.Enrich.WithMachineName()
-                .Enrich.WithProperty("Application", "SumaTube")
-                .Enrich.WithProperty("Workspace", workspace)
-                .WriteTo.Seq(seqServer)
-                ;
+            try
+            {
+                // Usa a configuração do appsettings.json (mais elegante)
+                config
+                    .ReadFrom.Configuration(configuration)
+                    .ReadFrom.Services(services)
+                    .Enrich.FromLogContext()
+                    .Enrich.WithEnvironmentName()
+                    //.Enrich.WithMachineName()
+                    .Enrich.WithProperty("Application", "YTExtractor")
+                    .Enrich.WithProperty("Workspace", workspace)
+                    .WriteTo.Console();
 
-            return config;
+                // Testa a conectividade com o Seq se estiver configurado
+                if (!string.IsNullOrEmpty(seqServer))
+                {
+                    TestSeqConnectivity(seqServer);
+                }
+
+                Console.WriteLine("✅ Serilog configurado com sucesso a partir do appsettings.json");
+                if (!string.IsNullOrEmpty(seqServer))
+                {
+                    Console.WriteLine($"✅ Seq configurado: {seqServer}");
+                }
+
+                return config;
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"❌ ERRO ao configurar Serilog: {ex.Message}");
+                Console.WriteLine("🔄 Tentando configuração de fallback...");
+
+                // Configuração de fallback se a configuração principal falhar
+                return CreateFallbackConfiguration(config, workspace, seqServer);
+            }
         }
 
+        private static LoggerConfiguration CreateFallbackConfiguration(
+            LoggerConfiguration config,
+            string workspace,
+            string seqServer)
+        {
+            try
+            {
+                config
+                    .MinimumLevel.Information()
+                    .MinimumLevel.Override("Microsoft", LogEventLevel.Warning)
+                    .MinimumLevel.Override("Microsoft.Hosting.Lifetime", LogEventLevel.Information)
+                    .Enrich.FromLogContext()
+                    .Enrich.WithEnvironmentName()
+                    .Enrich.WithProperty("Application", "YTExtractor")
+                    .Enrich.WithProperty("Workspace", workspace ?? "Unknown")
+                    .WriteTo.Console()
+                    .WriteTo.File("logs/fallback-app-.log", rollingInterval: RollingInterval.Day);
+
+                // Tenta adicionar o Seq apenas se a URL estiver disponível
+                if (!string.IsNullOrEmpty(seqServer))
+                {
+                    try
+                    {
+                        config.WriteTo.Seq(seqServer);
+                        Console.WriteLine($"✅ Seq configurado no fallback: {seqServer}");
+                    }
+                    catch (Exception seqEx)
+                    {
+                        Console.WriteLine($"⚠️  AVISO: Seq não pôde ser configurado no fallback: {seqEx.Message}");
+                        Console.WriteLine("📝 Continuando apenas com console e arquivo de log");
+                    }
+                }
+
+                Console.WriteLine("✅ Configuração de fallback aplicada com sucesso");
+                return config;
+            }
+            catch (Exception fallbackEx)
+            {
+                Console.WriteLine($"❌ ERRO CRÍTICO: Falha na configuração de fallback: {fallbackEx.Message}");
+                throw; // Re-throw porque se o fallback falhar, temos um problema sério
+            }
+        }
+
+        private static void TestSeqConnectivity(string seqUrl)
+        {
+            try
+            {
+                using var httpClient = new HttpClient();
+                httpClient.Timeout = TimeSpan.FromSeconds(10); 
+                var response = httpClient.GetAsync($"{seqUrl}/api").Result;
+
+                if (response.IsSuccessStatusCode)
+                {
+                    Console.WriteLine($"🌐 Conectividade com Seq OK: {seqUrl}");
+                }
+                else
+                {                               
+                    Console.WriteLine($"⚠️  Seq configurado mas pode não estar respondendo: {response.StatusCode}");
+                }
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"⚠️  Seq configurado mas conexão falhou: {ex.Message}");
+                Console.WriteLine($" StackTrace: {ex.StackTrace}");
+                Console.WriteLine($"💡 Dica: Verifique se o Seq está rodando em {seqUrl}");
+                throw new Exception("Erro ao configurar Seq. Configurar console.", ex);
+            }
+        }
     }
 }

YTExtractor/Program.cs

@@ -42,9 +42,11 @@ app.MapPost("/api/video-info", async (VideoRequest request, MongoDBConnector mon
 {
     try
     {
+        Log.Information("Requisição recebida");
         if (!youtubeService.IsValidYouTubeUrl(request.Url))
             return Results.BadRequest("Invalid YouTube URL");
 
+        Log.Information("Obtendo pasta atual");
         var tempDir = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
         Directory.CreateDirectory(tempDir);
 
@@ -52,9 +54,11 @@ app.MapPost("/api/video-info", async (VideoRequest request, MongoDBConnector mon
         {
             Log.Information($"Obtendo dados do video: {request.Url}");
             var service = new ConvertTranscriptService();
+            Log.Information("Obtendo dados da url no mongo");
             var videoExists = await mongo.GetVideoByUrl(request.Url);
             if (videoExists != null)
             {
+                Log.Information($"Retorno via mongo db da url: {request.Url}");
                 return Results.Ok(new VideoInfo(
                     videoExists.Url,
                     videoExists.Titulo,
@@ -63,9 +67,11 @@ app.MapPost("/api/video-info", async (VideoRequest request, MongoDBConnector mon
                 ));
             }
 
+            Log.Information("Obtendo dados da url na api do youtube");
             var info = await youtubeService.GetVideoInfo(request.Url, tempDir);
             var subtitles = service.ExtractPlainText(await youtubeService.GetSubtitles(request.Url, request.Language, tempDir));
 
+            Log.Information("Guardar no mongodb");
             await mongo.InsertVideo(new VideoData
             {
                 Id = Guid.NewGuid().ToString(),

YTExtractor/Services/Handlers/YtDlpHandler.cs

@@ -21,10 +21,24 @@ namespace YTExtractor.Services.Handlers
             {
                 _logger.LogInformation("Getting video info using yt-dlp for {Url}", url);
 
+                var arguments = $"--dump-json " +
+                                $"--user-agent \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36\" " +
+                                $"--add-header \"Accept-Language:pt-BR,pt;q=0.9,en;q=0.8\" " +
+                                $"--add-header \"Accept:text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\" " +
+                                $"--add-header \"Accept-Encoding:gzip, deflate, br\" " +
+                                $"--add-header \"DNT:1\" " +
+                                $"--add-header \"Connection:keep-alive\" " +
+                                $"--add-header \"Upgrade-Insecure-Requests:1\" " +
+                                $"--no-check-certificate " +
+                                $"--extractor-retries 3 " +
+                                $"--socket-timeout 30 " +
+                                $"{url}";
+
                 var startInfo = new ProcessStartInfo
                 {
                     FileName = "yt-dlp",
-                    Arguments = $"--dump-json {url}",
+                    //Arguments = $"--dump-json {url}",
+                    Arguments = arguments,                    
                     RedirectStandardOutput = true,
                     UseShellExecute = false,
                     CreateNoWindow = true,

YTExtractor/Services/YoutubeExplode/YoutubeExplodeClient.cs

@@ -16,8 +16,12 @@ namespace YTExtractor.Services.YoutubeExplode
 
         public YoutubeExplodeClient(ILogger<YoutubeExplodeClient> logger)
         {
+            var httpClient = new HttpClient();
+            httpClient.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36");
+            httpClient.DefaultRequestHeaders.Add("Accept-Language", "pt-BR,pt;q=0.9,en;q=0.8");
+
             _logger = logger;
-            _youtube = new YoutubeClient();
+            _youtube = new YoutubeClient(httpClient);
         }
 
         /// <summary>

YTExtractor/YTExtractor.csproj

@@ -10,6 +10,10 @@
     <Compile Remove="Services\Handlers\YouExposeHandler.cs" />
   </ItemGroup>
 
+  <ItemGroup>
+    <_ContentIncludedByDefault Remove="appsettings.Production.json" />
+  </ItemGroup>
+
   <ItemGroup>
     <PackageReference Include="Google.Apis.YouTube.v3" Version="1.69.0.3707" />
     <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="8.0.12" />

YTExtractor/appsettings.Development.json

@@ -23,5 +23,7 @@
       "Workspace": "Dev",
       "Application": "YTExtractor"
     }
-  }
+  },
+  "MongoDbConnection": "mongodb://admin:c4rn31r0@192.168.0.82:27017,192.168.0.81:27017/?replicaSet=rs0",
+  "MongoDbDatabase": "YTExtractor"
 }

YTExtractor/appsettings.Production.json

@@ -0,0 +1,31 @@
+{
+  "Serilog": {
+    "WriteTo": [
+      { "Name": "Console" },
+      {
+        "Name": "File",
+        "Args": {
+          "path": "logs/prod-app-.log",
+          "rollingInterval": "Day"
+        }
+      },
+      {
+        "Name": "Seq",
+        "Args": {
+          "serverUrl": "http://localhost:5341",
+          "compact": true,
+          "batchPostingLimit": 100
+        }
+      }
+    ],
+    "Properties": {
+      "Environment": "Production",
+      "Workspace": "VPS",
+      "Application": "YTExtractor"
+    }
+  },
+  "AllowedHosts": "*",
+  //"MongoDbConnection": "mongodb://admin:c4rn31r0@172.17.0.1:27017,172.17.0.1:27018/?replicaSet=rs0",
+  "MongoDbConnection": "mongodb://admin:c4rn31r0@k3sw2:27017,k3ss1:27018/?replicaSet=rs0",
+  "MongoDbDatabase": "YTExtractor-Prod"
+}

YTExtractor/appsettings.json

@@ -20,8 +20,5 @@
   },
   "AllowedHosts": "*",
   "MongoDbConnection": "mongodb://admin:c4rn31r0@192.168.0.82:27017,192.168.0.81:27017/?replicaSet=rs0",
-  "YouExpose": {
-    "ApiKey": "sua-chave-api-aqui",
-    "ApiUrl": "https://api.youexpose.com/"
-  }
+  "MongoDbDatabase": "YTExtractor"
 }