mirror of
https://github.com/Radarr/Radarr.git
synced 2024-09-11 12:02:35 +02:00
New: Updated Rarbg request limits
(cherry picked from commit 47cf8e6430b7f7704ce2f1524fa9e3c8e6f9b47a) Closes #8337 Fixes #7700
This commit is contained in:
parent
a77cb25513
commit
2ac72d1588
@ -20,6 +20,7 @@ public class HttpRequestBuilder
|
||||
public Dictionary<string, string> Segments { get; private set; }
|
||||
public HttpHeader Headers { get; private set; }
|
||||
public bool SuppressHttpError { get; set; }
|
||||
public IEnumerable<HttpStatusCode> SuppressHttpErrorStatusCodes { get; set; }
|
||||
public bool LogHttpError { get; set; }
|
||||
public bool UseSimplifiedUserAgent { get; set; }
|
||||
public bool AllowAutoRedirect { get; set; }
|
||||
@ -102,6 +103,7 @@ protected virtual void Apply(HttpRequest request)
|
||||
{
|
||||
request.Method = Method;
|
||||
request.SuppressHttpError = SuppressHttpError;
|
||||
request.SuppressHttpErrorStatusCodes = SuppressHttpErrorStatusCodes;
|
||||
request.LogHttpError = LogHttpError;
|
||||
request.UseSimplifiedUserAgent = UseSimplifiedUserAgent;
|
||||
request.AllowAutoRedirect = AllowAutoRedirect;
|
||||
|
@ -1,4 +1,4 @@
|
||||
using System;
|
||||
using System;
|
||||
|
||||
namespace NzbDrone.Common.Http
|
||||
{
|
||||
|
@ -1,5 +1,6 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using FluentAssertions;
|
||||
using Moq;
|
||||
@ -84,5 +85,57 @@ public void should_warn_on_unknown_error()
|
||||
|
||||
ExceptionVerification.ExpectedWarns(1);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_warn_and_record_failure_on_429_response()
|
||||
{
|
||||
Mocker.GetMock<IHttpClient>()
|
||||
.Setup(o => o.Execute(It.Is<HttpRequest>(v => v.Method == HttpMethod.Get)))
|
||||
.Returns<HttpRequest>(r => new HttpResponse(r, new HttpHeader(), "", HttpStatusCode.TooManyRequests));
|
||||
|
||||
var releases = Subject.FetchRecent();
|
||||
|
||||
releases.Should().HaveCount(0);
|
||||
|
||||
ExceptionVerification.ExpectedWarns(1);
|
||||
|
||||
Mocker.GetMock<IIndexerStatusService>()
|
||||
.Verify(v => v.RecordFailure(It.IsAny<int>(), It.Is<TimeSpan>(t => t == TimeSpan.FromMinutes(2))));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_warn_and_record_failure_on_520_response()
|
||||
{
|
||||
Mocker.GetMock<IHttpClient>()
|
||||
.Setup(o => o.Execute(It.Is<HttpRequest>(v => v.Method == HttpMethod.Get)))
|
||||
.Returns<HttpRequest>(r => new HttpResponse(r, new HttpHeader(), "", (HttpStatusCode)520));
|
||||
|
||||
var releases = Subject.FetchRecent();
|
||||
|
||||
releases.Should().HaveCount(0);
|
||||
|
||||
ExceptionVerification.ExpectedWarns(1);
|
||||
|
||||
Mocker.GetMock<IIndexerStatusService>()
|
||||
.Verify(v => v.RecordFailure(It.IsAny<int>(), It.Is<TimeSpan>(t => t == TimeSpan.FromMinutes(3))));
|
||||
}
|
||||
|
||||
// Uncomment when RarbgParser is updated
|
||||
// [Test]
|
||||
// public void should_warn_and_record_failure_on_200_response_with_rate_limit()
|
||||
// {
|
||||
// Mocker.GetMock<IHttpClient>()
|
||||
// .Setup(o => o.Execute(It.Is<HttpRequest>(v => v.Method == HttpMethod.Get)))
|
||||
// .Returns<HttpRequest>(r => new HttpResponse(r, new HttpHeader(), "{ rate_limit: 1 }"));
|
||||
//
|
||||
// var releases = Subject.FetchRecent();
|
||||
//
|
||||
// releases.Should().HaveCount(0);
|
||||
//
|
||||
// ExceptionVerification.ExpectedWarns(1);
|
||||
//
|
||||
// Mocker.GetMock<IIndexerStatusService>()
|
||||
// .Verify(v => v.RecordFailure(It.IsAny<int>(), It.Is<TimeSpan>(t => t == TimeSpan.FromMinutes(5))));
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
@ -1,9 +1,12 @@
|
||||
using NzbDrone.Common.Exceptions;
|
||||
using System;
|
||||
using NzbDrone.Common.Exceptions;
|
||||
|
||||
namespace NzbDrone.Core.Indexers.Exceptions
|
||||
{
|
||||
public class RequestLimitReachedException : NzbDroneException
|
||||
{
|
||||
public TimeSpan RetryAfter { get; private set; }
|
||||
|
||||
public RequestLimitReachedException(string message, params object[] args)
|
||||
: base(message, args)
|
||||
{
|
||||
@ -13,5 +16,11 @@ public RequestLimitReachedException(string message)
|
||||
: base(message)
|
||||
{
|
||||
}
|
||||
|
||||
public RequestLimitReachedException(string message, TimeSpan retryAfter)
|
||||
: base(message)
|
||||
{
|
||||
RetryAfter = retryAfter;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -223,9 +223,17 @@ protected virtual IList<ReleaseInfo> FetchReleases(Func<IIndexerRequestGenerator
|
||||
_indexerStatusService.RecordFailure(Definition.Id);
|
||||
_logger.Warn("{0} {1}", this, ex.Message);
|
||||
}
|
||||
catch (RequestLimitReachedException)
|
||||
catch (RequestLimitReachedException ex)
|
||||
{
|
||||
_indexerStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1));
|
||||
if (ex.RetryAfter != TimeSpan.Zero)
|
||||
{
|
||||
_indexerStatusService.RecordFailure(Definition.Id, ex.RetryAfter);
|
||||
}
|
||||
else
|
||||
{
|
||||
_indexerStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1));
|
||||
}
|
||||
|
||||
_logger.Warn("API Request Limit reached for {0}", this);
|
||||
}
|
||||
catch (ApiKeyException)
|
||||
|
@ -1,5 +1,7 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using NLog;
|
||||
using NzbDrone.Common.Extensions;
|
||||
using NzbDrone.Common.Http;
|
||||
@ -18,7 +20,7 @@ public class Rarbg : HttpIndexerBase<RarbgSettings>
|
||||
public override string Name => "Rarbg";
|
||||
|
||||
public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
|
||||
public override TimeSpan RateLimit => TimeSpan.FromSeconds(2);
|
||||
public override TimeSpan RateLimit => TimeSpan.FromSeconds(4);
|
||||
|
||||
public Rarbg(IRarbgTokenProvider tokenProvider, IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
|
||||
: base(httpClient, indexerStatusService, configService, parsingService, logger)
|
||||
|
@ -18,10 +18,14 @@ public IList<ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
|
||||
|
||||
switch (indexerResponse.HttpResponse.StatusCode)
|
||||
{
|
||||
case HttpStatusCode.TooManyRequests:
|
||||
throw new RequestLimitReachedException("Indexer API limit reached", TimeSpan.FromMinutes(2));
|
||||
case (HttpStatusCode)520:
|
||||
throw new RequestLimitReachedException("Indexer API error. Likely rate limited by origin server", TimeSpan.FromMinutes(3));
|
||||
default:
|
||||
if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
|
||||
{
|
||||
throw new IndexerException(indexerResponse, "Indexer API call returned an unexpected StatusCode [{0}]", indexerResponse.HttpResponse.StatusCode);
|
||||
throw new IndexerException(indexerResponse, "Indexer API call returned an unexpected status code [{0}]", indexerResponse.HttpResponse.StatusCode);
|
||||
}
|
||||
|
||||
break;
|
||||
@ -45,6 +49,12 @@ public IList<ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
|
||||
|
||||
if (jsonResponse.Resource.torrent_results == null)
|
||||
{
|
||||
// Despite this being the requested behaviour it appears to be problematic, commenting it out for now
|
||||
// if (jsonResponse.Resource.rate_limit == 1)
|
||||
// {
|
||||
// throw new RequestLimitReachedException("Indexer API limit reached", TimeSpan.FromMinutes(5));
|
||||
// }
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
|
@ -1,6 +1,7 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using NzbDrone.Common.EnvironmentInfo;
|
||||
using NzbDrone.Common.Extensions;
|
||||
using NzbDrone.Common.Http;
|
||||
@ -39,6 +40,8 @@ private IEnumerable<IndexerRequest> GetPagedRequests(string mode, int? imdbId, s
|
||||
.Resource("/pubapi_v2.php")
|
||||
.Accept(HttpAccept.Json);
|
||||
|
||||
requestBuilder.SuppressHttpErrorStatusCodes = new[] { HttpStatusCode.TooManyRequests, (HttpStatusCode)520 };
|
||||
|
||||
if (Settings.CaptchaToken.IsNotNullOrWhiteSpace())
|
||||
{
|
||||
requestBuilder.UseSimplifiedUserAgent = true;
|
||||
|
@ -7,6 +7,7 @@ public class RarbgResponse
|
||||
{
|
||||
public string error { get; set; }
|
||||
public int? error_code { get; set; }
|
||||
public int? rate_limit { get; set; }
|
||||
public List<RarbgTorrent> torrent_results { get; set; }
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user