NALU/src/Nalu.Api/Services/LlmRouter/LlmRouter.cs

namespace Nalu.Web.Services.LlmRouter;

public class LlmRouter(IReadOnlyList<ILlmProvider> providers, ILogger<LlmRouter> logger) : ILlmRouter
{
    public async Task<LlmResponse> CompleteAsync(LlmRequest request, CancellationToken ct)
    {
        foreach (var provider in providers)
        {
            try
            {
                return await provider.CompleteAsync(request, ct);
            }
            catch (RateLimitException ex)
            {
                logger.LogWarning("Provider {Provider} rate limited, trying next", ex.Provider);
            }
            catch (TimeoutException)
            {
                logger.LogWarning("Provider {Provider} timed out, trying next", provider.Name);
            }
            catch (Exception ex) when (ex is not OperationCanceledException)
            {
                logger.LogWarning(ex, "Provider {Provider} failed, trying next", provider.Name);
            }
        }

        throw new ServiceUnavailableException("All LLM providers unavailable");
    }
}