mirror of
https://github.com/Radarr/Radarr.git
synced 2024-09-17 15:02:34 +02:00
New: Updated Rarbg request limits
(cherry picked from commit 47cf8e6430b7f7704ce2f1524fa9e3c8e6f9b47a) Closes #8337 Fixes #7700
This commit is contained in:
parent
a77cb25513
commit
2ac72d1588
@ -20,6 +20,7 @@ public class HttpRequestBuilder
|
|||||||
public Dictionary<string, string> Segments { get; private set; }
|
public Dictionary<string, string> Segments { get; private set; }
|
||||||
public HttpHeader Headers { get; private set; }
|
public HttpHeader Headers { get; private set; }
|
||||||
public bool SuppressHttpError { get; set; }
|
public bool SuppressHttpError { get; set; }
|
||||||
|
public IEnumerable<HttpStatusCode> SuppressHttpErrorStatusCodes { get; set; }
|
||||||
public bool LogHttpError { get; set; }
|
public bool LogHttpError { get; set; }
|
||||||
public bool UseSimplifiedUserAgent { get; set; }
|
public bool UseSimplifiedUserAgent { get; set; }
|
||||||
public bool AllowAutoRedirect { get; set; }
|
public bool AllowAutoRedirect { get; set; }
|
||||||
@ -102,6 +103,7 @@ protected virtual void Apply(HttpRequest request)
|
|||||||
{
|
{
|
||||||
request.Method = Method;
|
request.Method = Method;
|
||||||
request.SuppressHttpError = SuppressHttpError;
|
request.SuppressHttpError = SuppressHttpError;
|
||||||
|
request.SuppressHttpErrorStatusCodes = SuppressHttpErrorStatusCodes;
|
||||||
request.LogHttpError = LogHttpError;
|
request.LogHttpError = LogHttpError;
|
||||||
request.UseSimplifiedUserAgent = UseSimplifiedUserAgent;
|
request.UseSimplifiedUserAgent = UseSimplifiedUserAgent;
|
||||||
request.AllowAutoRedirect = AllowAutoRedirect;
|
request.AllowAutoRedirect = AllowAutoRedirect;
|
||||||
|
@ -1,4 +1,4 @@
|
|||||||
using System;
|
using System;
|
||||||
|
|
||||||
namespace NzbDrone.Common.Http
|
namespace NzbDrone.Common.Http
|
||||||
{
|
{
|
||||||
|
@ -1,5 +1,6 @@
|
|||||||
using System;
|
using System;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
|
using System.Net;
|
||||||
using System.Net.Http;
|
using System.Net.Http;
|
||||||
using FluentAssertions;
|
using FluentAssertions;
|
||||||
using Moq;
|
using Moq;
|
||||||
@ -84,5 +85,57 @@ public void should_warn_on_unknown_error()
|
|||||||
|
|
||||||
ExceptionVerification.ExpectedWarns(1);
|
ExceptionVerification.ExpectedWarns(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void should_warn_and_record_failure_on_429_response()
|
||||||
|
{
|
||||||
|
Mocker.GetMock<IHttpClient>()
|
||||||
|
.Setup(o => o.Execute(It.Is<HttpRequest>(v => v.Method == HttpMethod.Get)))
|
||||||
|
.Returns<HttpRequest>(r => new HttpResponse(r, new HttpHeader(), "", HttpStatusCode.TooManyRequests));
|
||||||
|
|
||||||
|
var releases = Subject.FetchRecent();
|
||||||
|
|
||||||
|
releases.Should().HaveCount(0);
|
||||||
|
|
||||||
|
ExceptionVerification.ExpectedWarns(1);
|
||||||
|
|
||||||
|
Mocker.GetMock<IIndexerStatusService>()
|
||||||
|
.Verify(v => v.RecordFailure(It.IsAny<int>(), It.Is<TimeSpan>(t => t == TimeSpan.FromMinutes(2))));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void should_warn_and_record_failure_on_520_response()
|
||||||
|
{
|
||||||
|
Mocker.GetMock<IHttpClient>()
|
||||||
|
.Setup(o => o.Execute(It.Is<HttpRequest>(v => v.Method == HttpMethod.Get)))
|
||||||
|
.Returns<HttpRequest>(r => new HttpResponse(r, new HttpHeader(), "", (HttpStatusCode)520));
|
||||||
|
|
||||||
|
var releases = Subject.FetchRecent();
|
||||||
|
|
||||||
|
releases.Should().HaveCount(0);
|
||||||
|
|
||||||
|
ExceptionVerification.ExpectedWarns(1);
|
||||||
|
|
||||||
|
Mocker.GetMock<IIndexerStatusService>()
|
||||||
|
.Verify(v => v.RecordFailure(It.IsAny<int>(), It.Is<TimeSpan>(t => t == TimeSpan.FromMinutes(3))));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Uncomment when RarbgParser is updated
|
||||||
|
// [Test]
|
||||||
|
// public void should_warn_and_record_failure_on_200_response_with_rate_limit()
|
||||||
|
// {
|
||||||
|
// Mocker.GetMock<IHttpClient>()
|
||||||
|
// .Setup(o => o.Execute(It.Is<HttpRequest>(v => v.Method == HttpMethod.Get)))
|
||||||
|
// .Returns<HttpRequest>(r => new HttpResponse(r, new HttpHeader(), "{ rate_limit: 1 }"));
|
||||||
|
//
|
||||||
|
// var releases = Subject.FetchRecent();
|
||||||
|
//
|
||||||
|
// releases.Should().HaveCount(0);
|
||||||
|
//
|
||||||
|
// ExceptionVerification.ExpectedWarns(1);
|
||||||
|
//
|
||||||
|
// Mocker.GetMock<IIndexerStatusService>()
|
||||||
|
// .Verify(v => v.RecordFailure(It.IsAny<int>(), It.Is<TimeSpan>(t => t == TimeSpan.FromMinutes(5))));
|
||||||
|
// }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1,9 +1,12 @@
|
|||||||
using NzbDrone.Common.Exceptions;
|
using System;
|
||||||
|
using NzbDrone.Common.Exceptions;
|
||||||
|
|
||||||
namespace NzbDrone.Core.Indexers.Exceptions
|
namespace NzbDrone.Core.Indexers.Exceptions
|
||||||
{
|
{
|
||||||
public class RequestLimitReachedException : NzbDroneException
|
public class RequestLimitReachedException : NzbDroneException
|
||||||
{
|
{
|
||||||
|
public TimeSpan RetryAfter { get; private set; }
|
||||||
|
|
||||||
public RequestLimitReachedException(string message, params object[] args)
|
public RequestLimitReachedException(string message, params object[] args)
|
||||||
: base(message, args)
|
: base(message, args)
|
||||||
{
|
{
|
||||||
@ -13,5 +16,11 @@ public RequestLimitReachedException(string message)
|
|||||||
: base(message)
|
: base(message)
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public RequestLimitReachedException(string message, TimeSpan retryAfter)
|
||||||
|
: base(message)
|
||||||
|
{
|
||||||
|
RetryAfter = retryAfter;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -223,9 +223,17 @@ protected virtual IList<ReleaseInfo> FetchReleases(Func<IIndexerRequestGenerator
|
|||||||
_indexerStatusService.RecordFailure(Definition.Id);
|
_indexerStatusService.RecordFailure(Definition.Id);
|
||||||
_logger.Warn("{0} {1}", this, ex.Message);
|
_logger.Warn("{0} {1}", this, ex.Message);
|
||||||
}
|
}
|
||||||
catch (RequestLimitReachedException)
|
catch (RequestLimitReachedException ex)
|
||||||
{
|
{
|
||||||
_indexerStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1));
|
if (ex.RetryAfter != TimeSpan.Zero)
|
||||||
|
{
|
||||||
|
_indexerStatusService.RecordFailure(Definition.Id, ex.RetryAfter);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
_indexerStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1));
|
||||||
|
}
|
||||||
|
|
||||||
_logger.Warn("API Request Limit reached for {0}", this);
|
_logger.Warn("API Request Limit reached for {0}", this);
|
||||||
}
|
}
|
||||||
catch (ApiKeyException)
|
catch (ApiKeyException)
|
||||||
|
@ -1,5 +1,7 @@
|
|||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Net;
|
||||||
using NLog;
|
using NLog;
|
||||||
using NzbDrone.Common.Extensions;
|
using NzbDrone.Common.Extensions;
|
||||||
using NzbDrone.Common.Http;
|
using NzbDrone.Common.Http;
|
||||||
@ -18,7 +20,7 @@ public class Rarbg : HttpIndexerBase<RarbgSettings>
|
|||||||
public override string Name => "Rarbg";
|
public override string Name => "Rarbg";
|
||||||
|
|
||||||
public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
|
public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
|
||||||
public override TimeSpan RateLimit => TimeSpan.FromSeconds(2);
|
public override TimeSpan RateLimit => TimeSpan.FromSeconds(4);
|
||||||
|
|
||||||
public Rarbg(IRarbgTokenProvider tokenProvider, IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
|
public Rarbg(IRarbgTokenProvider tokenProvider, IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
|
||||||
: base(httpClient, indexerStatusService, configService, parsingService, logger)
|
: base(httpClient, indexerStatusService, configService, parsingService, logger)
|
||||||
|
@ -18,10 +18,14 @@ public IList<ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
|
|||||||
|
|
||||||
switch (indexerResponse.HttpResponse.StatusCode)
|
switch (indexerResponse.HttpResponse.StatusCode)
|
||||||
{
|
{
|
||||||
|
case HttpStatusCode.TooManyRequests:
|
||||||
|
throw new RequestLimitReachedException("Indexer API limit reached", TimeSpan.FromMinutes(2));
|
||||||
|
case (HttpStatusCode)520:
|
||||||
|
throw new RequestLimitReachedException("Indexer API error. Likely rate limited by origin server", TimeSpan.FromMinutes(3));
|
||||||
default:
|
default:
|
||||||
if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
|
if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
|
||||||
{
|
{
|
||||||
throw new IndexerException(indexerResponse, "Indexer API call returned an unexpected StatusCode [{0}]", indexerResponse.HttpResponse.StatusCode);
|
throw new IndexerException(indexerResponse, "Indexer API call returned an unexpected status code [{0}]", indexerResponse.HttpResponse.StatusCode);
|
||||||
}
|
}
|
||||||
|
|
||||||
break;
|
break;
|
||||||
@ -45,6 +49,12 @@ public IList<ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
|
|||||||
|
|
||||||
if (jsonResponse.Resource.torrent_results == null)
|
if (jsonResponse.Resource.torrent_results == null)
|
||||||
{
|
{
|
||||||
|
// Despite this being the requested behaviour it appears to be problematic, commenting it out for now
|
||||||
|
// if (jsonResponse.Resource.rate_limit == 1)
|
||||||
|
// {
|
||||||
|
// throw new RequestLimitReachedException("Indexer API limit reached", TimeSpan.FromMinutes(5));
|
||||||
|
// }
|
||||||
|
|
||||||
return results;
|
return results;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1,6 +1,7 @@
|
|||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
|
using System.Net;
|
||||||
using NzbDrone.Common.EnvironmentInfo;
|
using NzbDrone.Common.EnvironmentInfo;
|
||||||
using NzbDrone.Common.Extensions;
|
using NzbDrone.Common.Extensions;
|
||||||
using NzbDrone.Common.Http;
|
using NzbDrone.Common.Http;
|
||||||
@ -39,6 +40,8 @@ private IEnumerable<IndexerRequest> GetPagedRequests(string mode, int? imdbId, s
|
|||||||
.Resource("/pubapi_v2.php")
|
.Resource("/pubapi_v2.php")
|
||||||
.Accept(HttpAccept.Json);
|
.Accept(HttpAccept.Json);
|
||||||
|
|
||||||
|
requestBuilder.SuppressHttpErrorStatusCodes = new[] { HttpStatusCode.TooManyRequests, (HttpStatusCode)520 };
|
||||||
|
|
||||||
if (Settings.CaptchaToken.IsNotNullOrWhiteSpace())
|
if (Settings.CaptchaToken.IsNotNullOrWhiteSpace())
|
||||||
{
|
{
|
||||||
requestBuilder.UseSimplifiedUserAgent = true;
|
requestBuilder.UseSimplifiedUserAgent = true;
|
||||||
|
@ -7,6 +7,7 @@ public class RarbgResponse
|
|||||||
{
|
{
|
||||||
public string error { get; set; }
|
public string error { get; set; }
|
||||||
public int? error_code { get; set; }
|
public int? error_code { get; set; }
|
||||||
|
public int? rate_limit { get; set; }
|
||||||
public List<RarbgTorrent> torrent_results { get; set; }
|
public List<RarbgTorrent> torrent_results { get; set; }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user