Mirrors/Radarr
1
0
Fork 0
mirror of https://github.com/Radarr/Radarr.git synced 2024-11-04 10:02:40 +01:00
Code Issues Releases Wiki Activity

New: Updated Rarbg request limits

Browse Source 
(cherry picked from commit 47cf8e6430b7f7704ce2f1524fa9e3c8e6f9b47a)

Closes #8337
Fixes #7700
This commit is contained in:
Mark McDowall 2023-04-13 19:54:37 -07:00 committed by Qstick
parent a77cb25513
commit 2ac72d1588
9 changed files with 94 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/NzbDrone.Common/Http/HttpRequestBuilder.cs
View File

@ -20,6 +20,7 @@ public class HttpRequestBuilder
public Dictionary<string, string> Segments { get; private set; }
public HttpHeader Headers { get; private set; }
public bool SuppressHttpError { get; set; }
public IEnumerable<HttpStatusCode> SuppressHttpErrorStatusCodes { get; set; }
public bool LogHttpError { get; set; }
public bool UseSimplifiedUserAgent { get; set; }
public bool AllowAutoRedirect { get; set; }
@ -102,6 +103,7 @@ protected virtual void Apply(HttpRequest request)
{
request.Method = Method;
request.SuppressHttpError = SuppressHttpError;
request.SuppressHttpErrorStatusCodes = SuppressHttpErrorStatusCodes;
request.LogHttpError = LogHttpError;
request.UseSimplifiedUserAgent = UseSimplifiedUserAgent;
request.AllowAutoRedirect = AllowAutoRedirect;

2
src/NzbDrone.Common/Http/TooManyRequestsException.cs
View File

@ -1,4 +1,4 @@
using System;
using System;
namespace NzbDrone.Common.Http
{

53
src/NzbDrone.Core.Test/IndexerTests/RarbgTests/RarbgFixture.cs
View File

@ -1,5 +1,6 @@
using System;
using System.Linq;
using System.Net;
using System.Net.Http;
using FluentAssertions;
using Moq;
@ -84,5 +85,57 @@ public void should_warn_on_unknown_error()
ExceptionVerification.ExpectedWarns(1);
}
[Test]
public void should_warn_and_record_failure_on_429_response()
{
Mocker.GetMock<IHttpClient>()
.Setup(o => o.Execute(It.Is<HttpRequest>(v => v.Method == HttpMethod.Get)))
.Returns<HttpRequest>(r => new HttpResponse(r, new HttpHeader(), "", HttpStatusCode.TooManyRequests));
var releases = Subject.FetchRecent();
releases.Should().HaveCount(0);
ExceptionVerification.ExpectedWarns(1);
Mocker.GetMock<IIndexerStatusService>()
.Verify(v => v.RecordFailure(It.IsAny<int>(), It.Is<TimeSpan>(t => t == TimeSpan.FromMinutes(2))));
}
[Test]
public void should_warn_and_record_failure_on_520_response()
{
Mocker.GetMock<IHttpClient>()
.Setup(o => o.Execute(It.Is<HttpRequest>(v => v.Method == HttpMethod.Get)))
.Returns<HttpRequest>(r => new HttpResponse(r, new HttpHeader(), "", (HttpStatusCode)520));
var releases = Subject.FetchRecent();
releases.Should().HaveCount(0);
ExceptionVerification.ExpectedWarns(1);
Mocker.GetMock<IIndexerStatusService>()
.Verify(v => v.RecordFailure(It.IsAny<int>(), It.Is<TimeSpan>(t => t == TimeSpan.FromMinutes(3))));
}
// Uncomment when RarbgParser is updated
// [Test]
// public void should_warn_and_record_failure_on_200_response_with_rate_limit()
// {
// Mocker.GetMock<IHttpClient>()
// .Setup(o => o.Execute(It.Is<HttpRequest>(v => v.Method == HttpMethod.Get)))
// .Returns<HttpRequest>(r => new HttpResponse(r, new HttpHeader(), "{ rate_limit: 1 }"));
//
// var releases = Subject.FetchRecent();
//
// releases.Should().HaveCount(0);
//
// ExceptionVerification.ExpectedWarns(1);
//
// Mocker.GetMock<IIndexerStatusService>()
// .Verify(v => v.RecordFailure(It.IsAny<int>(), It.Is<TimeSpan>(t => t == TimeSpan.FromMinutes(5))));
// }
}
}

11
src/NzbDrone.Core/Indexers/Exceptions/RequestLimitReachedException.cs
View File

@ -1,9 +1,12 @@
using NzbDrone.Common.Exceptions;
using System;
using NzbDrone.Common.Exceptions;
namespace NzbDrone.Core.Indexers.Exceptions
{
public class RequestLimitReachedException : NzbDroneException
{
public TimeSpan RetryAfter { get; private set; }
public RequestLimitReachedException(string message, params object[] args)
: base(message, args)
{
@ -13,5 +16,11 @@ public RequestLimitReachedException(string message)
: base(message)
{
}
public RequestLimitReachedException(string message, TimeSpan retryAfter)
: base(message)
{
RetryAfter = retryAfter;
}
}
}

12
src/NzbDrone.Core/Indexers/HttpIndexerBase.cs
View File

@ -223,9 +223,17 @@ protected virtual IList<ReleaseInfo> FetchReleases(Func<IIndexerRequestGenerator
_indexerStatusService.RecordFailure(Definition.Id);
_logger.Warn("{0} {1}", this, ex.Message);
}
catch (RequestLimitReachedException)
catch (RequestLimitReachedException ex)
{
_indexerStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1));
if (ex.RetryAfter != TimeSpan.Zero)
{
_indexerStatusService.RecordFailure(Definition.Id, ex.RetryAfter);
}
else
{
_indexerStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1));
}
_logger.Warn("API Request Limit reached for {0}", this);
}
catch (ApiKeyException)

4
src/NzbDrone.Core/Indexers/Rarbg/Rarbg.cs
View File

@ -1,5 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using NLog;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
@ -18,7 +20,7 @@ public class Rarbg : HttpIndexerBase<RarbgSettings>
public override string Name => "Rarbg";
public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
public override TimeSpan RateLimit => TimeSpan.FromSeconds(2);
public override TimeSpan RateLimit => TimeSpan.FromSeconds(4);
public Rarbg(IRarbgTokenProvider tokenProvider, IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
: base(httpClient, indexerStatusService, configService, parsingService, logger)

12
src/NzbDrone.Core/Indexers/Rarbg/RarbgParser.cs
View File

@ -18,10 +18,14 @@ public IList<ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
switch (indexerResponse.HttpResponse.StatusCode)
{
case HttpStatusCode.TooManyRequests:
throw new RequestLimitReachedException("Indexer API limit reached", TimeSpan.FromMinutes(2));
case (HttpStatusCode)520:
throw new RequestLimitReachedException("Indexer API error. Likely rate limited by origin server", TimeSpan.FromMinutes(3));
default:
if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
{
throw new IndexerException(indexerResponse, "Indexer API call returned an unexpected StatusCode [{0}]", indexerResponse.HttpResponse.StatusCode);
throw new IndexerException(indexerResponse, "Indexer API call returned an unexpected status code [{0}]", indexerResponse.HttpResponse.StatusCode);
}
break;
@ -45,6 +49,12 @@ public IList<ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
if (jsonResponse.Resource.torrent_results == null)
{
// Despite this being the requested behaviour it appears to be problematic, commenting it out for now
// if (jsonResponse.Resource.rate_limit == 1)
// {
// throw new RequestLimitReachedException("Indexer API limit reached", TimeSpan.FromMinutes(5));
// }
return results;
}

3
src/NzbDrone.Core/Indexers/Rarbg/RarbgRequestGenerator.cs
View File

@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using NzbDrone.Common.EnvironmentInfo;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
@ -39,6 +40,8 @@ private IEnumerable<IndexerRequest> GetPagedRequests(string mode, int? imdbId, s
.Resource("/pubapi_v2.php")
.Accept(HttpAccept.Json);
requestBuilder.SuppressHttpErrorStatusCodes = new[] { HttpStatusCode.TooManyRequests, (HttpStatusCode)520 };
if (Settings.CaptchaToken.IsNotNullOrWhiteSpace())
{
requestBuilder.UseSimplifiedUserAgent = true;

1
src/NzbDrone.Core/Indexers/Rarbg/RarbgResponse.cs
View File

@ -7,6 +7,7 @@ public class RarbgResponse
{
public string error { get; set; }
public int? error_code { get; set; }
public int? rate_limit { get; set; }
public List<RarbgTorrent> torrent_results { get; set; }
}