From 2ac72d15880efe0c5fe964aeb306c61622bc2af1 Mon Sep 17 00:00:00 2001 From: Mark McDowall Date: Thu, 13 Apr 2023 19:54:37 -0700 Subject: [PATCH] New: Updated Rarbg request limits (cherry picked from commit 47cf8e6430b7f7704ce2f1524fa9e3c8e6f9b47a) Closes #8337 Fixes #7700 --- .../Http/HttpRequestBuilder.cs | 2 + .../Http/TooManyRequestsException.cs | 2 +- .../IndexerTests/RarbgTests/RarbgFixture.cs | 53 +++++++++++++++++++ .../RequestLimitReachedException.cs | 11 +++- src/NzbDrone.Core/Indexers/HttpIndexerBase.cs | 12 ++++- src/NzbDrone.Core/Indexers/Rarbg/Rarbg.cs | 4 +- .../Indexers/Rarbg/RarbgParser.cs | 12 ++++- .../Indexers/Rarbg/RarbgRequestGenerator.cs | 3 ++ .../Indexers/Rarbg/RarbgResponse.cs | 1 + 9 files changed, 94 insertions(+), 6 deletions(-) diff --git a/src/NzbDrone.Common/Http/HttpRequestBuilder.cs b/src/NzbDrone.Common/Http/HttpRequestBuilder.cs index 32fafd56f..93eebd039 100644 --- a/src/NzbDrone.Common/Http/HttpRequestBuilder.cs +++ b/src/NzbDrone.Common/Http/HttpRequestBuilder.cs @@ -20,6 +20,7 @@ public class HttpRequestBuilder public Dictionary Segments { get; private set; } public HttpHeader Headers { get; private set; } public bool SuppressHttpError { get; set; } + public IEnumerable SuppressHttpErrorStatusCodes { get; set; } public bool LogHttpError { get; set; } public bool UseSimplifiedUserAgent { get; set; } public bool AllowAutoRedirect { get; set; } @@ -102,6 +103,7 @@ protected virtual void Apply(HttpRequest request) { request.Method = Method; request.SuppressHttpError = SuppressHttpError; + request.SuppressHttpErrorStatusCodes = SuppressHttpErrorStatusCodes; request.LogHttpError = LogHttpError; request.UseSimplifiedUserAgent = UseSimplifiedUserAgent; request.AllowAutoRedirect = AllowAutoRedirect; diff --git a/src/NzbDrone.Common/Http/TooManyRequestsException.cs b/src/NzbDrone.Common/Http/TooManyRequestsException.cs index dcb261efa..117188b4e 100644 --- a/src/NzbDrone.Common/Http/TooManyRequestsException.cs +++ b/src/NzbDrone.Common/Http/TooManyRequestsException.cs @@ -1,4 +1,4 @@ -using System; +using System; namespace NzbDrone.Common.Http { diff --git a/src/NzbDrone.Core.Test/IndexerTests/RarbgTests/RarbgFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/RarbgTests/RarbgFixture.cs index eab77a41d..fc4e6750e 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/RarbgTests/RarbgFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/RarbgTests/RarbgFixture.cs @@ -1,5 +1,6 @@ using System; using System.Linq; +using System.Net; using System.Net.Http; using FluentAssertions; using Moq; @@ -84,5 +85,57 @@ public void should_warn_on_unknown_error() ExceptionVerification.ExpectedWarns(1); } + + [Test] + public void should_warn_and_record_failure_on_429_response() + { + Mocker.GetMock() + .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.Get))) + .Returns(r => new HttpResponse(r, new HttpHeader(), "", HttpStatusCode.TooManyRequests)); + + var releases = Subject.FetchRecent(); + + releases.Should().HaveCount(0); + + ExceptionVerification.ExpectedWarns(1); + + Mocker.GetMock() + .Verify(v => v.RecordFailure(It.IsAny(), It.Is(t => t == TimeSpan.FromMinutes(2)))); + } + + [Test] + public void should_warn_and_record_failure_on_520_response() + { + Mocker.GetMock() + .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.Get))) + .Returns(r => new HttpResponse(r, new HttpHeader(), "", (HttpStatusCode)520)); + + var releases = Subject.FetchRecent(); + + releases.Should().HaveCount(0); + + ExceptionVerification.ExpectedWarns(1); + + Mocker.GetMock() + .Verify(v => v.RecordFailure(It.IsAny(), It.Is(t => t == TimeSpan.FromMinutes(3)))); + } + + // Uncomment when RarbgParser is updated + // [Test] + // public void should_warn_and_record_failure_on_200_response_with_rate_limit() + // { + // Mocker.GetMock() + // .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.Get))) + // .Returns(r => new HttpResponse(r, new HttpHeader(), "{ rate_limit: 1 }")); + // + // var releases = Subject.FetchRecent(); + // + // releases.Should().HaveCount(0); + // + // ExceptionVerification.ExpectedWarns(1); + // + // Mocker.GetMock() + // .Verify(v => v.RecordFailure(It.IsAny(), It.Is(t => t == TimeSpan.FromMinutes(5)))); + // } } } diff --git a/src/NzbDrone.Core/Indexers/Exceptions/RequestLimitReachedException.cs b/src/NzbDrone.Core/Indexers/Exceptions/RequestLimitReachedException.cs index 2ad6ecd81..04c245f7b 100644 --- a/src/NzbDrone.Core/Indexers/Exceptions/RequestLimitReachedException.cs +++ b/src/NzbDrone.Core/Indexers/Exceptions/RequestLimitReachedException.cs @@ -1,9 +1,12 @@ -using NzbDrone.Common.Exceptions; +using System; +using NzbDrone.Common.Exceptions; namespace NzbDrone.Core.Indexers.Exceptions { public class RequestLimitReachedException : NzbDroneException { + public TimeSpan RetryAfter { get; private set; } + public RequestLimitReachedException(string message, params object[] args) : base(message, args) { @@ -13,5 +16,11 @@ public RequestLimitReachedException(string message) : base(message) { } + + public RequestLimitReachedException(string message, TimeSpan retryAfter) + : base(message) + { + RetryAfter = retryAfter; + } } } diff --git a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs index 267bb3a4f..0799140f9 100644 --- a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs @@ -223,9 +223,17 @@ protected virtual IList FetchReleases(Func public override string Name => "Rarbg"; public override DownloadProtocol Protocol => DownloadProtocol.Torrent; - public override TimeSpan RateLimit => TimeSpan.FromSeconds(2); + public override TimeSpan RateLimit => TimeSpan.FromSeconds(4); public Rarbg(IRarbgTokenProvider tokenProvider, IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger) : base(httpClient, indexerStatusService, configService, parsingService, logger) diff --git a/src/NzbDrone.Core/Indexers/Rarbg/RarbgParser.cs b/src/NzbDrone.Core/Indexers/Rarbg/RarbgParser.cs index 8d5b4f28e..249adf261 100644 --- a/src/NzbDrone.Core/Indexers/Rarbg/RarbgParser.cs +++ b/src/NzbDrone.Core/Indexers/Rarbg/RarbgParser.cs @@ -18,10 +18,14 @@ public IList ParseResponse(IndexerResponse indexerResponse) switch (indexerResponse.HttpResponse.StatusCode) { + case HttpStatusCode.TooManyRequests: + throw new RequestLimitReachedException("Indexer API limit reached", TimeSpan.FromMinutes(2)); + case (HttpStatusCode)520: + throw new RequestLimitReachedException("Indexer API error. Likely rate limited by origin server", TimeSpan.FromMinutes(3)); default: if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK) { - throw new IndexerException(indexerResponse, "Indexer API call returned an unexpected StatusCode [{0}]", indexerResponse.HttpResponse.StatusCode); + throw new IndexerException(indexerResponse, "Indexer API call returned an unexpected status code [{0}]", indexerResponse.HttpResponse.StatusCode); } break; @@ -45,6 +49,12 @@ public IList ParseResponse(IndexerResponse indexerResponse) if (jsonResponse.Resource.torrent_results == null) { + // Despite this being the requested behaviour it appears to be problematic, commenting it out for now + // if (jsonResponse.Resource.rate_limit == 1) + // { + // throw new RequestLimitReachedException("Indexer API limit reached", TimeSpan.FromMinutes(5)); + // } + return results; } diff --git a/src/NzbDrone.Core/Indexers/Rarbg/RarbgRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Rarbg/RarbgRequestGenerator.cs index 12f6111fc..de495fa02 100644 --- a/src/NzbDrone.Core/Indexers/Rarbg/RarbgRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/Rarbg/RarbgRequestGenerator.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Net; using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; @@ -39,6 +40,8 @@ private IEnumerable GetPagedRequests(string mode, int? imdbId, s .Resource("/pubapi_v2.php") .Accept(HttpAccept.Json); + requestBuilder.SuppressHttpErrorStatusCodes = new[] { HttpStatusCode.TooManyRequests, (HttpStatusCode)520 }; + if (Settings.CaptchaToken.IsNotNullOrWhiteSpace()) { requestBuilder.UseSimplifiedUserAgent = true; diff --git a/src/NzbDrone.Core/Indexers/Rarbg/RarbgResponse.cs b/src/NzbDrone.Core/Indexers/Rarbg/RarbgResponse.cs index 4087fc5ca..9b53bdc90 100644 --- a/src/NzbDrone.Core/Indexers/Rarbg/RarbgResponse.cs +++ b/src/NzbDrone.Core/Indexers/Rarbg/RarbgResponse.cs @@ -7,6 +7,7 @@ public class RarbgResponse { public string error { get; set; } public int? error_code { get; set; } + public int? rate_limit { get; set; } public List torrent_results { get; set; } }