mirror of
https://github.com/Radarr/Radarr.git
synced 2024-09-17 15:02:34 +02:00
New: Will now temporarily stop using an indexer if the indexer reported an error.
This commit is contained in:
parent
6d046a8df8
commit
f2a70677e4
@ -16,6 +16,7 @@ public class ReleaseResource : RestResource
|
||||
public Double AgeHours { get; set; }
|
||||
public Double AgeMinutes { get; set; }
|
||||
public Int64 Size { get; set; }
|
||||
public Int32 IndexerId { get; set; }
|
||||
public String Indexer { get; set; }
|
||||
public String ReleaseGroup { get; set; }
|
||||
public String SubGroup { get; set; }
|
||||
|
@ -249,6 +249,16 @@ public void should_overwrite_response_cookie()
|
||||
|
||||
ExceptionVerification.IgnoreErrors();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_throw_on_http429_too_many_requests()
|
||||
{
|
||||
var request = new HttpRequest("http://eu.httpbin.org/status/429");
|
||||
|
||||
Assert.Throws<TooManyRequestsException>(() => Subject.Get(request));
|
||||
|
||||
ExceptionVerification.IgnoreWarns();
|
||||
}
|
||||
}
|
||||
|
||||
public class HttpBinResource
|
||||
|
@ -94,7 +94,15 @@ public HttpResponse Execute(HttpRequest request)
|
||||
if (!request.SuppressHttpError && response.HasHttpError)
|
||||
{
|
||||
_logger.Warn("HTTP Error - {0}", response);
|
||||
throw new HttpException(request, response);
|
||||
|
||||
if ((int)response.StatusCode == 429)
|
||||
{
|
||||
throw new TooManyRequestsException(request, response);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new HttpException(request, response);
|
||||
}
|
||||
}
|
||||
|
||||
return response;
|
||||
|
21
src/NzbDrone.Common/Http/TooManyRequestsException.cs
Normal file
21
src/NzbDrone.Common/Http/TooManyRequestsException.cs
Normal file
@ -0,0 +1,21 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
|
||||
namespace NzbDrone.Common.Http
|
||||
{
|
||||
public class TooManyRequestsException : HttpException
|
||||
{
|
||||
public TimeSpan RetryAfter { get; private set; }
|
||||
|
||||
public TooManyRequestsException(HttpRequest request, HttpResponse response)
|
||||
: base(request, response)
|
||||
{
|
||||
if (response.Headers.ContainsKey("Retry-After"))
|
||||
{
|
||||
RetryAfter = TimeSpan.FromSeconds(int.Parse(response.Headers["Retry-After"].ToString()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
@ -158,6 +158,7 @@
|
||||
<SubType>Component</SubType>
|
||||
</Compile>
|
||||
<Compile Include="Http\HttpRequestBuilder.cs" />
|
||||
<Compile Include="Http\TooManyRequestsException.cs" />
|
||||
<Compile Include="Http\UriExtensions.cs" />
|
||||
<Compile Include="Extensions\IEnumerableExtensions.cs" />
|
||||
<Compile Include="Http\UserAgentBuilder.cs" />
|
||||
|
@ -1,15 +1,20 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using FizzWare.NBuilder;
|
||||
using Moq;
|
||||
using NUnit.Framework;
|
||||
using NzbDrone.Common.Http;
|
||||
using NzbDrone.Common.TPL;
|
||||
using NzbDrone.Core.Download;
|
||||
using NzbDrone.Core.Download.Clients;
|
||||
using NzbDrone.Core.Exceptions;
|
||||
using NzbDrone.Core.Indexers;
|
||||
using NzbDrone.Core.Parser.Model;
|
||||
using NzbDrone.Core.Test.Framework;
|
||||
using NzbDrone.Core.Tv;
|
||||
using NzbDrone.Test.Common;
|
||||
using System.Collections.Generic;
|
||||
using NzbDrone.Core.Indexers;
|
||||
|
||||
namespace NzbDrone.Core.Test.Download
|
||||
{
|
||||
@ -107,6 +112,53 @@ public void Download_report_should_not_publish_on_failed_grab_event()
|
||||
VerifyEventNotPublished<EpisodeGrabbedEvent>();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void Download_report_should_trigger_indexer_backoff_on_indexer_error()
|
||||
{
|
||||
var mock = WithUsenetClient();
|
||||
mock.Setup(s => s.Download(It.IsAny<RemoteEpisode>()))
|
||||
.Callback<RemoteEpisode>(v => {
|
||||
throw new ReleaseDownloadException(v.Release, "Error", new WebException());
|
||||
});
|
||||
|
||||
Assert.Throws<ReleaseDownloadException>(() => Subject.DownloadReport(_parseResult));
|
||||
|
||||
Mocker.GetMock<IIndexerStatusService>()
|
||||
.Verify(v => v.RecordFailure(It.IsAny<int>(), It.IsAny<TimeSpan>()), Times.Once());
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void Download_report_should_trigger_indexer_backoff_on_http429_with_long_time()
|
||||
{
|
||||
var request = new HttpRequest("http://my.indexer.com");
|
||||
var response = new HttpResponse(request, new HttpHeader(), new byte[0], (HttpStatusCode)429);
|
||||
response.Headers["Retry-After"] = "300";
|
||||
|
||||
var mock = WithUsenetClient();
|
||||
mock.Setup(s => s.Download(It.IsAny<RemoteEpisode>()))
|
||||
.Callback<RemoteEpisode>(v => {
|
||||
throw new ReleaseDownloadException(v.Release, "Error", new TooManyRequestsException(request, response));
|
||||
});
|
||||
|
||||
Assert.Throws<ReleaseDownloadException>(() => Subject.DownloadReport(_parseResult));
|
||||
|
||||
Mocker.GetMock<IIndexerStatusService>()
|
||||
.Verify(v => v.RecordFailure(It.IsAny<int>(), TimeSpan.FromMinutes(5)), Times.Once());
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void Download_report_should_not_trigger_indexer_backoff_on_downloadclient_error()
|
||||
{
|
||||
var mock = WithUsenetClient();
|
||||
mock.Setup(s => s.Download(It.IsAny<RemoteEpisode>()))
|
||||
.Throws(new DownloadClientException("Some Error"));
|
||||
|
||||
Assert.Throws<DownloadClientException>(() => Subject.DownloadReport(_parseResult));
|
||||
|
||||
Mocker.GetMock<IIndexerStatusService>()
|
||||
.Verify(v => v.RecordFailure(It.IsAny<int>(), It.IsAny<TimeSpan>()), Times.Never());
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_not_attempt_download_if_client_isnt_configure()
|
||||
{
|
||||
|
@ -0,0 +1,63 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using FizzWare.NBuilder;
|
||||
using Marr.Data;
|
||||
using Moq;
|
||||
using NUnit.Framework;
|
||||
using NzbDrone.Common.Extensions;
|
||||
using NzbDrone.Core.DecisionEngine;
|
||||
using NzbDrone.Core.Download.Pending;
|
||||
using NzbDrone.Core.Indexers;
|
||||
using NzbDrone.Core.Parser;
|
||||
using NzbDrone.Core.Parser.Model;
|
||||
using NzbDrone.Core.Profiles;
|
||||
using NzbDrone.Core.Qualities;
|
||||
using NzbDrone.Core.Test.Framework;
|
||||
using NzbDrone.Core.Tv;
|
||||
using FluentAssertions;
|
||||
|
||||
namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
|
||||
{
|
||||
[TestFixture]
|
||||
public class PendingReleaseServiceFixture : CoreTest<PendingReleaseService>
|
||||
{
|
||||
private void GivenPendingRelease()
|
||||
{
|
||||
Mocker.GetMock<IPendingReleaseRepository>()
|
||||
.Setup(v => v.All())
|
||||
.Returns(new List<PendingRelease> {
|
||||
new PendingRelease { Release = new ReleaseInfo { IndexerId = 1 } }
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_not_ignore_pending_items_from_available_indexer()
|
||||
{
|
||||
Mocker.GetMock<IIndexerStatusService>()
|
||||
.Setup(v => v.GetBlockedIndexers())
|
||||
.Returns(new List<IndexerStatus>());
|
||||
|
||||
GivenPendingRelease();
|
||||
|
||||
var results = Subject.GetPending();
|
||||
|
||||
results.Should().NotBeEmpty();
|
||||
Mocker.GetMock<IMakeDownloadDecision>()
|
||||
.Verify(v => v.GetRssDecision(It.Is<List<ReleaseInfo>>(d => d.Count == 0)), Times.Never());
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_ignore_pending_items_from_unavailable_indexer()
|
||||
{
|
||||
Mocker.GetMock<IIndexerStatusService>()
|
||||
.Setup(v => v.GetBlockedIndexers())
|
||||
.Returns(new List<IndexerStatus> { new IndexerStatus { IndexerId = 1, DisabledTill = DateTime.UtcNow.AddHours(2) } });
|
||||
|
||||
GivenPendingRelease();
|
||||
|
||||
var results = Subject.GetPending();
|
||||
|
||||
results.Should().BeEmpty();
|
||||
}
|
||||
}
|
||||
}
|
@ -0,0 +1,99 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using FizzWare.NBuilder;
|
||||
using Moq;
|
||||
using NUnit.Framework;
|
||||
using NzbDrone.Common.Disk;
|
||||
using NzbDrone.Core.HealthCheck.Checks;
|
||||
using NzbDrone.Core.Indexers;
|
||||
using NzbDrone.Core.Test.Framework;
|
||||
using NzbDrone.Core.Tv;
|
||||
|
||||
namespace NzbDrone.Core.Test.HealthCheck.Checks
|
||||
{
|
||||
[TestFixture]
|
||||
public class IndexerStatusCheckFixture : CoreTest<IndexerStatusCheck>
|
||||
{
|
||||
private List<IIndexer> _indexers = new List<IIndexer>();
|
||||
private List<IndexerStatus> _blockedIndexers = new List<IndexerStatus>();
|
||||
|
||||
[SetUp]
|
||||
public void SetUp()
|
||||
{
|
||||
Mocker.GetMock<IIndexerFactory>()
|
||||
.Setup(v => v.GetAvailableProviders())
|
||||
.Returns(_indexers);
|
||||
|
||||
Mocker.GetMock<IIndexerStatusService>()
|
||||
.Setup(v => v.GetBlockedIndexers())
|
||||
.Returns(_blockedIndexers);
|
||||
}
|
||||
|
||||
private Mock<IIndexer> GivenIndexer(int i, double backoffHours, double failureHours)
|
||||
{
|
||||
var id = i;
|
||||
|
||||
var mockIndexer = new Mock<IIndexer>();
|
||||
mockIndexer.SetupGet(s => s.Definition).Returns(new IndexerDefinition { Id = id });
|
||||
mockIndexer.SetupGet(s => s.SupportsSearch).Returns(true);
|
||||
|
||||
_indexers.Add(mockIndexer.Object);
|
||||
|
||||
if (backoffHours != 0.0)
|
||||
{
|
||||
_blockedIndexers.Add(new IndexerStatus
|
||||
{
|
||||
IndexerId = id,
|
||||
InitialFailure = DateTime.UtcNow.AddHours(-failureHours),
|
||||
MostRecentFailure = DateTime.UtcNow.AddHours(-0.1),
|
||||
EscalationLevel = 5,
|
||||
DisabledTill = DateTime.UtcNow.AddHours(backoffHours)
|
||||
});
|
||||
}
|
||||
|
||||
return mockIndexer;
|
||||
}
|
||||
|
||||
|
||||
[Test]
|
||||
public void should_not_return_error_when_no_indexers()
|
||||
{
|
||||
Subject.Check().ShouldBeOk();
|
||||
}
|
||||
[Test]
|
||||
public void should_not_return_error_when_indexer_failed_less_than_an_hour()
|
||||
{
|
||||
GivenIndexer(1, 0.1, 0.5);
|
||||
|
||||
Subject.Check().ShouldBeOk();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_return_warning_if_indexer_unavailable()
|
||||
{
|
||||
GivenIndexer(1, 10.0, 24.0);
|
||||
GivenIndexer(2, 0.0, 0.0);
|
||||
|
||||
Subject.Check().ShouldBeWarning();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_return_error_if_all_indexers_unavailable()
|
||||
{
|
||||
GivenIndexer(1, 10.0, 24.0);
|
||||
|
||||
Subject.Check().ShouldBeError();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_return_warning_if_few_indexers_unavailable()
|
||||
{
|
||||
GivenIndexer(1, 10.0, 24.0);
|
||||
GivenIndexer(2, 10.0, 24.0);
|
||||
GivenIndexer(3, 0.0, 0.0);
|
||||
|
||||
Subject.Check().ShouldBeWarning();
|
||||
}
|
||||
}
|
||||
}
|
@ -0,0 +1,56 @@
|
||||
using FizzWare.NBuilder;
|
||||
using FluentAssertions;
|
||||
using NUnit.Framework;
|
||||
using NzbDrone.Core.Housekeeping.Housekeepers;
|
||||
using NzbDrone.Core.Indexers;
|
||||
using NzbDrone.Core.Qualities;
|
||||
using NzbDrone.Core.Test.Framework;
|
||||
using NzbDrone.Core.Tv;
|
||||
|
||||
namespace NzbDrone.Core.Test.Housekeeping.Housekeepers
|
||||
{
|
||||
[TestFixture]
|
||||
public class CleanupOrphanedIndexerStatusFixture : DbTest<CleanupOrphanedIndexerStatus, IndexerStatus>
|
||||
{
|
||||
private IndexerDefinition _indexer;
|
||||
|
||||
[SetUp]
|
||||
public void Setup()
|
||||
{
|
||||
_indexer = Builder<IndexerDefinition>.CreateNew()
|
||||
.BuildNew();
|
||||
}
|
||||
|
||||
private void GivenIndexer()
|
||||
{
|
||||
Db.Insert(_indexer);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_delete_orphaned_indexerstatus()
|
||||
{
|
||||
var status = Builder<IndexerStatus>.CreateNew()
|
||||
.With(h => h.IndexerId = _indexer.Id)
|
||||
.BuildNew();
|
||||
Db.Insert(status);
|
||||
|
||||
Subject.Clean();
|
||||
AllStoredModels.Should().BeEmpty();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_not_delete_unorphaned_indexerstatus()
|
||||
{
|
||||
GivenIndexer();
|
||||
|
||||
var status = Builder<IndexerStatus>.CreateNew()
|
||||
.With(h => h.IndexerId = _indexer.Id)
|
||||
.BuildNew();
|
||||
Db.Insert(status);
|
||||
|
||||
Subject.Clean();
|
||||
AllStoredModels.Should().HaveCount(1);
|
||||
AllStoredModels.Should().Contain(h => h.IndexerId == _indexer.Id);
|
||||
}
|
||||
}
|
||||
}
|
@ -25,6 +25,7 @@ public class NzbSearchServiceFixture : CoreTest<NzbSearchService>
|
||||
public void SetUp()
|
||||
{
|
||||
_mockIndexer = Mocker.GetMock<IIndexer>();
|
||||
_mockIndexer.SetupGet(s => s.Definition).Returns(new IndexerDefinition { Id = 1 });
|
||||
_mockIndexer.SetupGet(s => s.SupportsSearch).Returns(true);
|
||||
|
||||
Mocker.GetMock<IIndexerFactory>()
|
||||
|
@ -59,7 +59,8 @@ public void should_parse_recent_feed_from_BroadcastheNet()
|
||||
|
||||
private void VerifyBackOff()
|
||||
{
|
||||
// TODO How to detect (and implement) back-off logic.
|
||||
Mocker.GetMock<IIndexerStatusService>()
|
||||
.Verify(v => v.RecordFailure(It.IsAny<int>(), It.IsAny<TimeSpan>()), Times.Once());
|
||||
}
|
||||
|
||||
[Test]
|
||||
@ -89,8 +90,6 @@ public void should_back_off_and_report_api_key_invalid()
|
||||
|
||||
results.Should().BeEmpty();
|
||||
|
||||
results.Should().BeEmpty();
|
||||
|
||||
VerifyBackOff();
|
||||
|
||||
ExceptionVerification.ExpectedWarns(1);
|
||||
|
@ -0,0 +1,94 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Linq.Expressions;
|
||||
using System.Text;
|
||||
using FluentAssertions;
|
||||
using Moq;
|
||||
using NUnit.Framework;
|
||||
using NzbDrone.Core.Indexers;
|
||||
using NzbDrone.Core.Test.Framework;
|
||||
|
||||
namespace NzbDrone.Core.Test.IndexerTests
|
||||
{
|
||||
public class IndexerStatusServiceFixture : CoreTest<IndexerStatusService>
|
||||
{
|
||||
private DateTime _epoch;
|
||||
|
||||
[SetUp]
|
||||
public void SetUp()
|
||||
{
|
||||
_epoch = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
private void WithStatus(IndexerStatus status)
|
||||
{
|
||||
Mocker.GetMock<IIndexerStatusRepository>()
|
||||
.Setup(v => v.FindByIndexerId(1))
|
||||
.Returns(status);
|
||||
|
||||
Mocker.GetMock<IIndexerStatusRepository>()
|
||||
.Setup(v => v.All())
|
||||
.Returns(new[] { status });
|
||||
}
|
||||
|
||||
private void VerifyUpdate(bool updated = true)
|
||||
{
|
||||
Mocker.GetMock<IIndexerStatusRepository>()
|
||||
.Verify(v => v.Upsert(It.IsAny<IndexerStatus>()), Times.Exactly(updated ? 1 : 0));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_start_backoff_on_first_failure()
|
||||
{
|
||||
WithStatus(new IndexerStatus());
|
||||
|
||||
Subject.RecordFailure(1);
|
||||
|
||||
VerifyUpdate();
|
||||
|
||||
var status = Subject.GetBlockedIndexers().FirstOrDefault();
|
||||
status.Should().NotBeNull();
|
||||
status.DisabledTill.Should().HaveValue();
|
||||
status.DisabledTill.Value.Should().BeCloseTo(_epoch + TimeSpan.FromMinutes(5), 500);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_cancel_backoff_on_success()
|
||||
{
|
||||
WithStatus(new IndexerStatus { EscalationLevel = 2 });
|
||||
|
||||
Subject.RecordSuccess(1);
|
||||
|
||||
VerifyUpdate();
|
||||
|
||||
var status = Subject.GetBlockedIndexers().FirstOrDefault();
|
||||
status.Should().BeNull();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_not_store_update_if_already_okay()
|
||||
{
|
||||
WithStatus(new IndexerStatus { EscalationLevel = 0 });
|
||||
|
||||
Subject.RecordSuccess(1);
|
||||
|
||||
VerifyUpdate(false);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_preserve_escalation_on_intermittent_success()
|
||||
{
|
||||
WithStatus(new IndexerStatus { MostRecentFailure = _epoch - TimeSpan.FromSeconds(4), EscalationLevel = 3 });
|
||||
|
||||
Subject.RecordSuccess(1);
|
||||
Subject.RecordSuccess(1);
|
||||
Subject.RecordFailure(1);
|
||||
|
||||
var status = Subject.GetBlockedIndexers().FirstOrDefault();
|
||||
status.Should().NotBeNull();
|
||||
status.DisabledTill.Should().HaveValue();
|
||||
status.DisabledTill.Value.Should().BeCloseTo(_epoch + TimeSpan.FromMinutes(15), 500);
|
||||
}
|
||||
}
|
||||
}
|
@ -18,6 +18,7 @@ public void Setup()
|
||||
{
|
||||
Subject.Definition = new IndexerDefinition()
|
||||
{
|
||||
Id = 5,
|
||||
Name = "Newznab",
|
||||
Settings = new NewznabSettings()
|
||||
{
|
||||
@ -47,6 +48,7 @@ public void should_parse_recent_feed_from_newznab_nzb_su()
|
||||
releaseInfo.DownloadUrl.Should().Be("http://nzb.su/getnzb/24967ef4c2e26296c65d3bbfa97aa8fe.nzb&i=37292&r=xxx");
|
||||
releaseInfo.InfoUrl.Should().Be("http://nzb.su/details/24967ef4c2e26296c65d3bbfa97aa8fe");
|
||||
releaseInfo.CommentUrl.Should().Be("http://nzb.su/details/24967ef4c2e26296c65d3bbfa97aa8fe#comments");
|
||||
releaseInfo.IndexerId.Should().Be(Subject.Definition.Id);
|
||||
releaseInfo.Indexer.Should().Be(Subject.Definition.Name);
|
||||
releaseInfo.PublishDate.Should().Be(DateTime.Parse("2012/02/27 16:09:39"));
|
||||
releaseInfo.Size.Should().Be(1183105773);
|
||||
|
@ -30,18 +30,6 @@ public void SetUp()
|
||||
AbsoluteEpisodeNumber = 100
|
||||
};
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_return_one_page_for_feed()
|
||||
{
|
||||
var results = Subject.GetRecentRequests();
|
||||
|
||||
results.Should().HaveCount(1);
|
||||
|
||||
var pages = results.First().Take(10).ToList();
|
||||
|
||||
pages.Should().HaveCount(1);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_use_all_categories_for_feed()
|
||||
|
@ -22,8 +22,8 @@ public override string Name
|
||||
public Int32 _supportedPageSize;
|
||||
public override Int32 PageSize { get { return _supportedPageSize; } }
|
||||
|
||||
public TestIndexer(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger)
|
||||
: base(httpClient, configService, parsingService, logger)
|
||||
public TestIndexer(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
|
||||
: base(httpClient, indexerStatusService, configService, parsingService, logger)
|
||||
{
|
||||
|
||||
}
|
||||
|
@ -13,8 +13,8 @@ namespace NzbDrone.Core.Test.IndexerTests.TorrentRssIndexerTests
|
||||
{
|
||||
public class TestTorrentRssIndexer : TorrentRssIndexer
|
||||
{
|
||||
public TestTorrentRssIndexer(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, ITorrentRssParserFactory torrentRssParserFactory, Logger logger)
|
||||
: base(httpClient, configService, parsingService, torrentRssParserFactory, logger)
|
||||
public TestTorrentRssIndexer(ITorrentRssParserFactory torrentRssParserFactory, IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
|
||||
: base(torrentRssParserFactory, httpClient, indexerStatusService, configService, parsingService, logger)
|
||||
{
|
||||
}
|
||||
|
||||
|
@ -48,18 +48,6 @@ public void SetUp()
|
||||
.Setup(v => v.GetCapabilities(It.IsAny<TorznabSettings>()))
|
||||
.Returns(_capabilities);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_return_one_page_for_feed()
|
||||
{
|
||||
var results = Subject.GetRecentRequests();
|
||||
|
||||
results.Should().HaveCount(1);
|
||||
|
||||
var pages = results.First().Take(10).ToList();
|
||||
|
||||
pages.Should().HaveCount(1);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_use_all_categories_for_feed()
|
||||
|
@ -168,6 +168,7 @@
|
||||
<Compile Include="Download\DownloadClientTests\UTorrentTests\UTorrentFixture.cs" />
|
||||
<Compile Include="Download\DownloadServiceFixture.cs" />
|
||||
<Compile Include="Download\FailedDownloadServiceFixture.cs" />
|
||||
<Compile Include="Download\Pending\PendingReleaseServiceTests\PendingReleaseServiceFixture.cs" />
|
||||
<Compile Include="Download\Pending\PendingReleaseServiceTests\RemovePendingFixture.cs" />
|
||||
<Compile Include="Download\Pending\PendingReleaseServiceTests\RemoveRejectedFixture.cs" />
|
||||
<Compile Include="Download\Pending\PendingReleaseServiceTests\RemoveGrabbedFixture.cs" />
|
||||
@ -193,6 +194,7 @@
|
||||
<Compile Include="HealthCheck\Checks\ImportMechanismCheckFixture.cs" />
|
||||
<Compile Include="HealthCheck\Checks\IndexerCheckFixture.cs" />
|
||||
<Compile Include="HealthCheck\Checks\MonoVersionCheckFixture.cs" />
|
||||
<Compile Include="HealthCheck\Checks\IndexerStatusCheckFixture.cs" />
|
||||
<Compile Include="HealthCheck\Checks\RootFolderCheckFixture.cs" />
|
||||
<Compile Include="HealthCheck\Checks\UpdateCheckFixture.cs" />
|
||||
<Compile Include="HealthCheck\HealthCheckFixture.cs" />
|
||||
@ -203,6 +205,7 @@
|
||||
<Compile Include="Housekeeping\Housekeepers\CleanupOrphanedBlacklistFixture.cs" />
|
||||
<Compile Include="Housekeeping\Housekeepers\CleanupOrphanedEpisodeFilesFixture.cs" />
|
||||
<Compile Include="Housekeeping\Housekeepers\CleanupOrphanedEpisodesFixture.cs" />
|
||||
<Compile Include="Housekeeping\Housekeepers\CleanupOrphanedIndexerStatusFixture.cs" />
|
||||
<Compile Include="Housekeeping\Housekeepers\CleanupOrphanedHistoryItemsFixture.cs" />
|
||||
<Compile Include="Housekeeping\Housekeepers\CleanupOrphanedMetadataFilesFixture.cs" />
|
||||
<Compile Include="Housekeeping\Housekeepers\CleanupOrphanedPendingReleasesFixture.cs" />
|
||||
@ -215,6 +218,7 @@
|
||||
<Compile Include="IndexerTests\BroadcastheNetTests\BroadcastheNetFixture.cs" />
|
||||
<Compile Include="IndexerTests\HDBitsTests\HDBitsFixture.cs" />
|
||||
<Compile Include="IndexerTests\IndexerServiceFixture.cs" />
|
||||
<Compile Include="IndexerTests\IndexerStatusServiceFixture.cs" />
|
||||
<Compile Include="IndexerTests\IntegrationTests\IndexerIntegrationTests.cs" />
|
||||
<Compile Include="IndexerTests\RarbgTests\RarbgFixture.cs" />
|
||||
<Compile Include="IndexerTests\TorrentRssIndexerTests\TorrentRssParserFactoryFixture.cs" />
|
||||
|
@ -0,0 +1,25 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Data;
|
||||
using FluentMigrator;
|
||||
using NzbDrone.Common.Extensions;
|
||||
using NzbDrone.Common.Serializer;
|
||||
using NzbDrone.Core.Datastore.Migration.Framework;
|
||||
|
||||
namespace NzbDrone.Core.Datastore.Migration
|
||||
{
|
||||
[Migration(91)]
|
||||
public class added_indexerstatus : NzbDroneMigrationBase
|
||||
{
|
||||
protected override void MainDbUpgrade()
|
||||
{
|
||||
Create.TableForModel("IndexerStatus")
|
||||
.WithColumn("IndexerId").AsInt32().NotNullable().Unique()
|
||||
.WithColumn("InitialFailure").AsDateTime().Nullable()
|
||||
.WithColumn("MostRecentFailure").AsDateTime().Nullable()
|
||||
.WithColumn("EscalationLevel").AsInt32().NotNullable()
|
||||
.WithColumn("DisabledTill").AsDateTime().Nullable()
|
||||
.WithColumn("LastRssSyncReleaseInfo").AsString().Nullable();
|
||||
}
|
||||
}
|
||||
}
|
@ -111,6 +111,8 @@ public static void Map()
|
||||
Mapper.Entity<User>().RegisterModel("Users");
|
||||
Mapper.Entity<CommandModel>().RegisterModel("Commands")
|
||||
.Ignore(c => c.Message);
|
||||
|
||||
Mapper.Entity<IndexerStatus>().RegisterModel("IndexerStatus");
|
||||
}
|
||||
|
||||
private static void RegisterMappers()
|
||||
|
@ -2,8 +2,11 @@
|
||||
using NLog;
|
||||
using NzbDrone.Common.EnsureThat;
|
||||
using NzbDrone.Common.Extensions;
|
||||
using NzbDrone.Common.Http;
|
||||
using NzbDrone.Common.Instrumentation.Extensions;
|
||||
using NzbDrone.Common.TPL;
|
||||
using NzbDrone.Core.Exceptions;
|
||||
using NzbDrone.Core.Indexers;
|
||||
using NzbDrone.Core.Messaging.Events;
|
||||
using NzbDrone.Core.Parser.Model;
|
||||
|
||||
@ -18,16 +21,19 @@ public interface IDownloadService
|
||||
public class DownloadService : IDownloadService
|
||||
{
|
||||
private readonly IProvideDownloadClient _downloadClientProvider;
|
||||
private readonly IIndexerStatusService _indexerStatusService;
|
||||
private readonly IRateLimitService _rateLimitService;
|
||||
private readonly IEventAggregator _eventAggregator;
|
||||
private readonly Logger _logger;
|
||||
|
||||
public DownloadService(IProvideDownloadClient downloadClientProvider,
|
||||
IIndexerStatusService indexerStatusService,
|
||||
IRateLimitService rateLimitService,
|
||||
IEventAggregator eventAggregator,
|
||||
Logger logger)
|
||||
{
|
||||
_downloadClientProvider = downloadClientProvider;
|
||||
_indexerStatusService = indexerStatusService;
|
||||
_rateLimitService = rateLimitService;
|
||||
_eventAggregator = eventAggregator;
|
||||
_logger = logger;
|
||||
@ -54,7 +60,26 @@ public void DownloadReport(RemoteEpisode remoteEpisode)
|
||||
_rateLimitService.WaitAndPulse(uri.Host, TimeSpan.FromSeconds(2));
|
||||
}
|
||||
|
||||
var downloadClientId = downloadClient.Download(remoteEpisode);
|
||||
string downloadClientId;
|
||||
try
|
||||
{
|
||||
downloadClientId = downloadClient.Download(remoteEpisode);
|
||||
_indexerStatusService.RecordSuccess(remoteEpisode.Release.IndexerId);
|
||||
}
|
||||
catch (ReleaseDownloadException ex)
|
||||
{
|
||||
var http429 = ex.InnerException as TooManyRequestsException;
|
||||
if (http429 != null)
|
||||
{
|
||||
_indexerStatusService.RecordFailure(remoteEpisode.Release.IndexerId, http429.RetryAfter);
|
||||
}
|
||||
else
|
||||
{
|
||||
_indexerStatusService.RecordFailure(remoteEpisode.Release.IndexerId);
|
||||
}
|
||||
throw;
|
||||
}
|
||||
|
||||
var episodeGrabbedEvent = new EpisodeGrabbedEvent(remoteEpisode);
|
||||
episodeGrabbedEvent.DownloadClient = downloadClient.GetType().Name;
|
||||
|
||||
|
@ -35,6 +35,7 @@ public class PendingReleaseService : IPendingReleaseService,
|
||||
IHandle<EpisodeGrabbedEvent>,
|
||||
IHandle<RssSyncCompleteEvent>
|
||||
{
|
||||
private readonly IIndexerStatusService _indexerStatusService;
|
||||
private readonly IPendingReleaseRepository _repository;
|
||||
private readonly ISeriesService _seriesService;
|
||||
private readonly IParsingService _parsingService;
|
||||
@ -44,7 +45,8 @@ public class PendingReleaseService : IPendingReleaseService,
|
||||
private readonly IEventAggregator _eventAggregator;
|
||||
private readonly Logger _logger;
|
||||
|
||||
public PendingReleaseService(IPendingReleaseRepository repository,
|
||||
public PendingReleaseService(IIndexerStatusService indexerStatusService,
|
||||
IPendingReleaseRepository repository,
|
||||
ISeriesService seriesService,
|
||||
IParsingService parsingService,
|
||||
IDelayProfileService delayProfileService,
|
||||
@ -53,6 +55,7 @@ public PendingReleaseService(IPendingReleaseRepository repository,
|
||||
IEventAggregator eventAggregator,
|
||||
Logger logger)
|
||||
{
|
||||
_indexerStatusService = indexerStatusService;
|
||||
_repository = repository;
|
||||
_seriesService = seriesService;
|
||||
_parsingService = parsingService;
|
||||
@ -86,7 +89,21 @@ public void Add(DownloadDecision decision)
|
||||
|
||||
public List<ReleaseInfo> GetPending()
|
||||
{
|
||||
return _repository.All().Select(p => p.Release).ToList();
|
||||
var releases = _repository.All().Select(p => p.Release).ToList();
|
||||
|
||||
if (releases.Any())
|
||||
{
|
||||
releases = FilterBlockedIndexers(releases);
|
||||
}
|
||||
|
||||
return releases;
|
||||
}
|
||||
|
||||
private List<ReleaseInfo> FilterBlockedIndexers(List<ReleaseInfo> releases)
|
||||
{
|
||||
var blockedIndexers = new HashSet<int>(_indexerStatusService.GetBlockedIndexers().Select(v => v.IndexerId));
|
||||
|
||||
return releases.Where(release => !blockedIndexers.Contains(release.IndexerId)).ToList();
|
||||
}
|
||||
|
||||
public List<RemoteEpisode> GetPendingRemoteEpisodes(int seriesId)
|
||||
|
@ -126,10 +126,16 @@ private string DownloadFromWebUrl(RemoteEpisode remoteEpisode, String torrentUrl
|
||||
|
||||
torrentFile = response.ResponseData;
|
||||
}
|
||||
catch (HttpException ex)
|
||||
{
|
||||
_logger.ErrorException(String.Format("Downloading torrent file for episode '{0}' failed ({1})",
|
||||
remoteEpisode.Release.Title, torrentUrl), ex);
|
||||
|
||||
throw new ReleaseDownloadException(remoteEpisode.Release, "Downloading torrent failed", ex);
|
||||
}
|
||||
catch (WebException ex)
|
||||
{
|
||||
_logger.ErrorException(String.Format("Downloading torrentfile for episode '{0}' failed ({1})",
|
||||
_logger.ErrorException(String.Format("Downloading torrent file for episode '{0}' failed ({1})",
|
||||
remoteEpisode.Release.Title, torrentUrl), ex);
|
||||
|
||||
throw new ReleaseDownloadException(remoteEpisode.Release, "Downloading torrent failed", ex);
|
||||
|
@ -49,6 +49,13 @@ public override String Download(RemoteEpisode remoteEpisode)
|
||||
{
|
||||
nzbData = _httpClient.Get(new HttpRequest(url)).ResponseData;
|
||||
}
|
||||
catch (HttpException ex)
|
||||
{
|
||||
_logger.ErrorException(String.Format("Downloading nzb for episode '{0}' failed ({1})",
|
||||
remoteEpisode.Release.Title, url), ex);
|
||||
|
||||
throw new ReleaseDownloadException(remoteEpisode.Release, "Downloading nzb failed", ex);
|
||||
}
|
||||
catch (WebException ex)
|
||||
{
|
||||
_logger.ErrorException(String.Format("Downloading nzb for episode '{0}' failed ({1})",
|
||||
|
48
src/NzbDrone.Core/HealthCheck/Checks/IndexerStatusCheck.cs
Normal file
48
src/NzbDrone.Core/HealthCheck/Checks/IndexerStatusCheck.cs
Normal file
@ -0,0 +1,48 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using NzbDrone.Common.Extensions;
|
||||
using NzbDrone.Core.Indexers;
|
||||
|
||||
namespace NzbDrone.Core.HealthCheck.Checks
|
||||
{
|
||||
public class IndexerStatusCheck : HealthCheckBase
|
||||
{
|
||||
private readonly IIndexerFactory _indexerFactory;
|
||||
private readonly IIndexerStatusService _indexerStatusService;
|
||||
|
||||
public IndexerStatusCheck(IIndexerFactory indexerFactory, IIndexerStatusService indexerStatusService)
|
||||
{
|
||||
_indexerFactory = indexerFactory;
|
||||
_indexerStatusService = indexerStatusService;
|
||||
}
|
||||
|
||||
public override HealthCheck Check()
|
||||
{
|
||||
var enabledIndexers = _indexerFactory.GetAvailableProviders();
|
||||
var backOffIndexers = enabledIndexers.Join(_indexerStatusService.GetBlockedIndexers(),
|
||||
i => i.Definition.Id,
|
||||
s => s.IndexerId,
|
||||
(i, s) => new { Indexer = i, Status = s })
|
||||
.Where(v => (v.Status.MostRecentFailure - v.Status.InitialFailure) > TimeSpan.FromHours(1))
|
||||
.ToList();
|
||||
|
||||
if (backOffIndexers.Empty())
|
||||
{
|
||||
return new HealthCheck(GetType());
|
||||
}
|
||||
|
||||
if (backOffIndexers.Count == enabledIndexers.Count)
|
||||
{
|
||||
return new HealthCheck(GetType(), HealthCheckResult.Error, "All indexers are unavailable due to failures", "#indexers-are-unavailable-due-to-failures");
|
||||
}
|
||||
|
||||
if (backOffIndexers.Count > 1)
|
||||
{
|
||||
return new HealthCheck(GetType(), HealthCheckResult.Warning, string.Format("{0} indexers are unavailable due to failures", backOffIndexers.Count), "#indexers-are-unavailable-due-to-failures");
|
||||
}
|
||||
|
||||
var indexer = backOffIndexers.First();
|
||||
return new HealthCheck(GetType(), HealthCheckResult.Warning, string.Format("Indexer {0} is unavailable due to failures", indexer.Indexer.Definition.Name), "#indexers-are-unavailable-due-to-failures");
|
||||
}
|
||||
}
|
||||
}
|
@ -0,0 +1,26 @@
|
||||
using NzbDrone.Core.Datastore;
|
||||
|
||||
namespace NzbDrone.Core.Housekeeping.Housekeepers
|
||||
{
|
||||
public class CleanupOrphanedIndexerStatus : IHousekeepingTask
|
||||
{
|
||||
private readonly IMainDatabase _database;
|
||||
|
||||
public CleanupOrphanedIndexerStatus(IMainDatabase database)
|
||||
{
|
||||
_database = database;
|
||||
}
|
||||
|
||||
public void Clean()
|
||||
{
|
||||
var mapper = _database.GetDataMapper();
|
||||
|
||||
mapper.ExecuteNonQuery(@"DELETE FROM IndexerStatus
|
||||
WHERE Id IN (
|
||||
SELECT IndexerStatus.Id FROM IndexerStatus
|
||||
LEFT OUTER JOIN Indexers
|
||||
ON IndexerStatus.IndexerId = Indexers.Id
|
||||
WHERE Indexers.Id IS NULL)");
|
||||
}
|
||||
}
|
||||
}
|
@ -16,14 +16,7 @@
|
||||
|
||||
namespace NzbDrone.Core.IndexerSearch
|
||||
{
|
||||
public interface IEpisodeSearchService
|
||||
{
|
||||
void MissingEpisodesAiredAfter(DateTime dateTime, IEnumerable<Int32> grabbed);
|
||||
}
|
||||
|
||||
public class EpisodeSearchService : IEpisodeSearchService,
|
||||
IExecute<EpisodeSearchCommand>,
|
||||
IExecute<MissingEpisodeSearchCommand>
|
||||
public class EpisodeSearchService : IExecute<EpisodeSearchCommand>, IExecute<MissingEpisodeSearchCommand>
|
||||
{
|
||||
private readonly ISearchForNzb _nzbSearchService;
|
||||
private readonly IProcessDownloadDecisions _processDownloadDecisions;
|
||||
@ -44,28 +37,6 @@ public EpisodeSearchService(ISearchForNzb nzbSearchService,
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public void MissingEpisodesAiredAfter(DateTime dateTime, IEnumerable<Int32> grabbed)
|
||||
{
|
||||
var missing = _episodeService.EpisodesBetweenDates(dateTime, DateTime.UtcNow, false)
|
||||
.Where(e => !e.HasFile &&
|
||||
!_queueService.GetQueue().Select(q => q.Episode.Id).Contains(e.Id) &&
|
||||
!grabbed.Contains(e.Id))
|
||||
.ToList();
|
||||
|
||||
var downloadedCount = 0;
|
||||
_logger.Info("Searching for {0} missing episodes since last RSS Sync", missing.Count);
|
||||
|
||||
foreach (var episode in missing)
|
||||
{
|
||||
//TODO: Add a flag to the search to state it is a "scheduled" search
|
||||
var decisions = _nzbSearchService.EpisodeSearch(episode);
|
||||
var processed = _processDownloadDecisions.ProcessDecisions(decisions);
|
||||
downloadedCount += processed.Grabbed.Count;
|
||||
}
|
||||
|
||||
_logger.ProgressInfo("Completed search for {0} episodes. {1} reports downloaded.", missing.Count, downloadedCount);
|
||||
}
|
||||
|
||||
private void SearchForMissingEpisodes(List<Episode> episodes)
|
||||
{
|
||||
_logger.ProgressInfo("Performing missing search for {0} episodes", episodes.Count);
|
||||
|
@ -247,7 +247,7 @@ private List<DownloadDecision> SearchAnimeSeason(Series series, List<Episode> ep
|
||||
|
||||
private List<DownloadDecision> Dispatch(Func<IIndexer, IEnumerable<ReleaseInfo>> searchAction, SearchCriteriaBase criteriaBase)
|
||||
{
|
||||
var indexers = _indexerFactory.SearchEnabled().ToList();
|
||||
var indexers = _indexerFactory.SearchEnabled();
|
||||
var reports = new List<ReleaseInfo>();
|
||||
|
||||
_logger.ProgressInfo("Searching {0} indexers for {1}", indexers.Count, criteriaBase);
|
||||
|
@ -20,8 +20,8 @@ public override string Name
|
||||
public override Boolean SupportsSearch { get { return false; } }
|
||||
public override Int32 PageSize { get { return 0; } }
|
||||
|
||||
public BitMeTv(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger)
|
||||
: base(httpClient, configService, parsingService, logger)
|
||||
public BitMeTv(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
|
||||
: base(httpClient, indexerStatusService, configService, parsingService, logger)
|
||||
{
|
||||
|
||||
}
|
||||
|
@ -20,8 +20,8 @@ public override string Name
|
||||
public override bool SupportsSearch { get { return true; } }
|
||||
public override int PageSize { get { return 100; } }
|
||||
|
||||
public BroadcastheNet(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger)
|
||||
: base(httpClient, configService, parsingService, logger)
|
||||
public BroadcastheNet(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
|
||||
: base(httpClient, indexerStatusService, configService, parsingService, logger)
|
||||
{
|
||||
|
||||
}
|
||||
|
@ -23,7 +23,7 @@ public virtual IList<IEnumerable<IndexerRequest>> GetRecentRequests()
|
||||
{
|
||||
var pageableRequests = new List<IEnumerable<IndexerRequest>>();
|
||||
|
||||
pageableRequests.AddIfNotNull(GetPagedRequests(1, null));
|
||||
pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, null));
|
||||
|
||||
return pageableRequests;
|
||||
}
|
||||
|
@ -17,8 +17,8 @@ public override string Name
|
||||
|
||||
public override DownloadProtocol Protocol { get { return DownloadProtocol.Usenet; } }
|
||||
|
||||
public Fanzub(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger)
|
||||
: base(httpClient, configService, parsingService, logger)
|
||||
public Fanzub(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
|
||||
: base(httpClient, indexerStatusService, configService, parsingService, logger)
|
||||
{
|
||||
|
||||
}
|
||||
|
@ -4,7 +4,8 @@
|
||||
using NLog;
|
||||
using NzbDrone.Core.Parser.Model;
|
||||
using NzbDrone.Common.TPL;
|
||||
|
||||
using System.Collections;
|
||||
using System;
|
||||
namespace NzbDrone.Core.Indexers
|
||||
{
|
||||
public interface IFetchAndParseRss
|
||||
@ -27,7 +28,7 @@ public List<ReleaseInfo> Fetch()
|
||||
{
|
||||
var result = new List<ReleaseInfo>();
|
||||
|
||||
var indexers = _indexerFactory.RssEnabled().ToList();
|
||||
var indexers = _indexerFactory.RssEnabled();
|
||||
|
||||
if (!indexers.Any())
|
||||
{
|
||||
|
@ -14,11 +14,8 @@ public class HDBits : HttpIndexerBase<HDBitsSettings>
|
||||
public override bool SupportsSearch { get { return true; } }
|
||||
public override int PageSize { get { return 30; } }
|
||||
|
||||
public HDBits(IHttpClient httpClient,
|
||||
IConfigService configService,
|
||||
IParsingService parsingService,
|
||||
Logger logger)
|
||||
: base(httpClient, configService, parsingService, logger)
|
||||
public HDBits(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
|
||||
: base(httpClient, indexerStatusService, configService, parsingService, logger)
|
||||
{ }
|
||||
|
||||
public override IIndexerRequestGenerator GetRequestGenerator()
|
||||
|
@ -33,8 +33,8 @@ public abstract class HttpIndexerBase<TSettings> : IndexerBase<TSettings>
|
||||
public abstract IIndexerRequestGenerator GetRequestGenerator();
|
||||
public abstract IParseIndexerResponse GetParser();
|
||||
|
||||
public HttpIndexerBase(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger)
|
||||
: base(configService, parsingService, logger)
|
||||
public HttpIndexerBase(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
|
||||
: base(indexerStatusService, configService, parsingService, logger)
|
||||
{
|
||||
_httpClient = httpClient;
|
||||
}
|
||||
@ -48,7 +48,7 @@ public override IList<ReleaseInfo> FetchRecent()
|
||||
|
||||
var generator = GetRequestGenerator();
|
||||
|
||||
return FetchReleases(generator.GetRecentRequests());
|
||||
return FetchReleases(generator.GetRecentRequests(), true);
|
||||
}
|
||||
|
||||
public override IList<ReleaseInfo> Fetch(SingleEpisodeSearchCriteria searchCriteria)
|
||||
@ -111,7 +111,7 @@ public override IList<ReleaseInfo> Fetch(SpecialEpisodeSearchCriteria searchCrit
|
||||
return FetchReleases(generator.GetSearchRequests(searchCriteria));
|
||||
}
|
||||
|
||||
protected virtual IList<ReleaseInfo> FetchReleases(IList<IEnumerable<IndexerRequest>> pageableRequests)
|
||||
protected virtual IList<ReleaseInfo> FetchReleases(IList<IEnumerable<IndexerRequest>> pageableRequests, bool isRecent = false)
|
||||
{
|
||||
var releases = new List<ReleaseInfo>();
|
||||
var url = String.Empty;
|
||||
@ -120,6 +120,13 @@ protected virtual IList<ReleaseInfo> FetchReleases(IList<IEnumerable<IndexerRequ
|
||||
|
||||
try
|
||||
{
|
||||
var fullyUpdated = false;
|
||||
ReleaseInfo lastReleaseInfo = null;
|
||||
if (isRecent)
|
||||
{
|
||||
lastReleaseInfo = _indexerStatusService.GetLastRssSyncReleaseInfo(Definition.Id);
|
||||
}
|
||||
|
||||
foreach (var pageableRequest in pageableRequests)
|
||||
{
|
||||
var pagedReleases = new List<ReleaseInfo>();
|
||||
@ -132,7 +139,33 @@ protected virtual IList<ReleaseInfo> FetchReleases(IList<IEnumerable<IndexerRequ
|
||||
|
||||
pagedReleases.AddRange(page);
|
||||
|
||||
if (!IsFullPage(page) || pagedReleases.Count >= MaxNumResultsPerQuery)
|
||||
if (isRecent)
|
||||
{
|
||||
if (lastReleaseInfo == null)
|
||||
{
|
||||
fullyUpdated = true;
|
||||
break;
|
||||
}
|
||||
var oldestReleaseDate = page.Select(v => v.PublishDate).Min();
|
||||
if (oldestReleaseDate < lastReleaseInfo.PublishDate || page.Any(v => v.DownloadUrl == lastReleaseInfo.DownloadUrl))
|
||||
{
|
||||
fullyUpdated = true;
|
||||
break;
|
||||
}
|
||||
|
||||
if (pagedReleases.Count >= MaxNumResultsPerQuery &&
|
||||
oldestReleaseDate < DateTime.UtcNow - TimeSpan.FromHours(24))
|
||||
{
|
||||
fullyUpdated = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
else if (pagedReleases.Count >= MaxNumResultsPerQuery)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if (!IsFullPage(page))
|
||||
{
|
||||
break;
|
||||
}
|
||||
@ -140,9 +173,26 @@ protected virtual IList<ReleaseInfo> FetchReleases(IList<IEnumerable<IndexerRequ
|
||||
|
||||
releases.AddRange(pagedReleases);
|
||||
}
|
||||
|
||||
if (isRecent && !releases.Empty())
|
||||
{
|
||||
var ordered = releases.OrderByDescending(v => v.PublishDate).ToList();
|
||||
|
||||
if (!fullyUpdated && lastReleaseInfo != null)
|
||||
{
|
||||
var gapStart = lastReleaseInfo.PublishDate;
|
||||
var gapEnd = ordered.Last();
|
||||
_logger.Warn("Indexer {0} rss sync didn't cover the period between {1} and {2} UTC. Search may be required.", Definition.Name, gapStart, gapEnd);
|
||||
}
|
||||
lastReleaseInfo = ordered.First();
|
||||
_indexerStatusService.UpdateRssSyncStatus(Definition.Id, lastReleaseInfo);
|
||||
}
|
||||
|
||||
_indexerStatusService.RecordSuccess(Definition.Id);
|
||||
}
|
||||
catch (WebException webException)
|
||||
{
|
||||
_indexerStatusService.RecordFailure(Definition.Id);
|
||||
if (webException.Message.Contains("502") || webException.Message.Contains("503") ||
|
||||
webException.Message.Contains("timed out"))
|
||||
{
|
||||
@ -155,29 +205,36 @@ protected virtual IList<ReleaseInfo> FetchReleases(IList<IEnumerable<IndexerRequ
|
||||
}
|
||||
catch (HttpException httpException)
|
||||
{
|
||||
if ((int) httpException.Response.StatusCode == 429)
|
||||
if ((int)httpException.Response.StatusCode == 429)
|
||||
{
|
||||
_indexerStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1));
|
||||
_logger.Warn("API Request Limit reached for {0}", this);
|
||||
}
|
||||
|
||||
_logger.Warn("{0} {1}", this, httpException.Message);
|
||||
else
|
||||
{
|
||||
_indexerStatusService.RecordFailure(Definition.Id);
|
||||
_logger.Warn("{0} {1}", this, httpException.Message);
|
||||
}
|
||||
}
|
||||
catch (RequestLimitReachedException)
|
||||
{
|
||||
// TODO: Backoff for x period.
|
||||
_indexerStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1));
|
||||
_logger.Warn("API Request Limit reached for {0}", this);
|
||||
}
|
||||
catch (ApiKeyException)
|
||||
{
|
||||
_indexerStatusService.RecordFailure(Definition.Id);
|
||||
_logger.Warn("Invalid API Key for {0} {1}", this, url);
|
||||
}
|
||||
catch (IndexerException ex)
|
||||
{
|
||||
_indexerStatusService.RecordFailure(Definition.Id);
|
||||
var message = String.Format("{0} - {1}", ex.Message, url);
|
||||
_logger.WarnException(message, ex);
|
||||
}
|
||||
catch (Exception feedEx)
|
||||
{
|
||||
_indexerStatusService.RecordFailure(Definition.Id);
|
||||
feedEx.Data.Add("FeedUrl", url);
|
||||
_logger.ErrorException("An error occurred while processing feed. " + url, feedEx);
|
||||
}
|
||||
|
@ -20,8 +20,8 @@ public override string Name
|
||||
public override Boolean SupportsSearch { get { return false; } }
|
||||
public override Int32 PageSize { get { return 0; } }
|
||||
|
||||
public IPTorrents(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger)
|
||||
: base(httpClient, configService, parsingService, logger)
|
||||
public IPTorrents(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
|
||||
: base(httpClient, indexerStatusService, configService, parsingService, logger)
|
||||
{
|
||||
|
||||
}
|
||||
|
@ -15,6 +15,7 @@ namespace NzbDrone.Core.Indexers
|
||||
public abstract class IndexerBase<TSettings> : IIndexer
|
||||
where TSettings : IProviderConfig, new()
|
||||
{
|
||||
protected readonly IIndexerStatusService _indexerStatusService;
|
||||
protected readonly IConfigService _configService;
|
||||
protected readonly IParsingService _parsingService;
|
||||
protected readonly Logger _logger;
|
||||
@ -25,8 +26,9 @@ public abstract class IndexerBase<TSettings> : IIndexer
|
||||
public abstract Boolean SupportsRss { get; }
|
||||
public abstract Boolean SupportsSearch { get; }
|
||||
|
||||
public IndexerBase(IConfigService configService, IParsingService parsingService, Logger logger)
|
||||
public IndexerBase(IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
|
||||
{
|
||||
_indexerStatusService = indexerStatusService;
|
||||
_configService = configService;
|
||||
_parsingService = parsingService;
|
||||
_logger = logger;
|
||||
@ -85,6 +87,7 @@ protected virtual IList<ReleaseInfo> CleanupReleases(IEnumerable<ReleaseInfo> re
|
||||
|
||||
result.ForEach(c =>
|
||||
{
|
||||
c.IndexerId = Definition.Id;
|
||||
c.Indexer = Definition.Name;
|
||||
c.DownloadProtocol = Protocol;
|
||||
});
|
||||
@ -106,6 +109,11 @@ public ValidationResult Test()
|
||||
failures.Add(new ValidationFailure(string.Empty, "Test was aborted due to an error: " + ex.Message));
|
||||
}
|
||||
|
||||
if (Definition.Id != 0)
|
||||
{
|
||||
_indexerStatusService.RecordSuccess(Definition.Id);
|
||||
}
|
||||
|
||||
return new ValidationResult(failures);
|
||||
}
|
||||
|
||||
|
@ -18,5 +18,7 @@ public override Boolean Enable
|
||||
return EnableRss || EnableSearch;
|
||||
}
|
||||
}
|
||||
|
||||
public IndexerStatus Status { get; set; }
|
||||
}
|
||||
}
|
||||
|
@ -15,21 +15,21 @@ public interface IIndexerFactory : IProviderFactory<IIndexer, IndexerDefinition>
|
||||
|
||||
public class IndexerFactory : ProviderFactory<IIndexer, IndexerDefinition>, IIndexerFactory
|
||||
{
|
||||
private readonly IIndexerStatusService _indexerStatusService;
|
||||
private readonly IIndexerRepository _providerRepository;
|
||||
private readonly Logger _logger;
|
||||
|
||||
public IndexerFactory(IIndexerRepository providerRepository,
|
||||
public IndexerFactory(IIndexerStatusService indexerStatusService,
|
||||
IIndexerRepository providerRepository,
|
||||
IEnumerable<IIndexer> providers,
|
||||
IContainer container,
|
||||
IEventAggregator eventAggregator,
|
||||
Logger logger)
|
||||
: base(providerRepository, providers, container, eventAggregator, logger)
|
||||
{
|
||||
_indexerStatusService = indexerStatusService;
|
||||
_providerRepository = providerRepository;
|
||||
}
|
||||
|
||||
protected override void InitializeProviders()
|
||||
{
|
||||
//_providerRepository.DeleteImplementations("Animezb");
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
protected override List<IndexerDefinition> Active()
|
||||
@ -50,12 +50,37 @@ public override IndexerDefinition GetProviderCharacteristics(IIndexer provider,
|
||||
|
||||
public List<IIndexer> RssEnabled()
|
||||
{
|
||||
return GetAvailableProviders().Where(n => ((IndexerDefinition)n.Definition).EnableRss).ToList();
|
||||
var enabledIndexers = GetAvailableProviders().Where(n => ((IndexerDefinition)n.Definition).EnableRss);
|
||||
|
||||
var indexers = FilterBlockedIndexers(enabledIndexers);
|
||||
|
||||
return indexers.ToList();
|
||||
}
|
||||
|
||||
public List<IIndexer> SearchEnabled()
|
||||
{
|
||||
return GetAvailableProviders().Where(n => ((IndexerDefinition)n.Definition).EnableSearch).ToList();
|
||||
}
|
||||
var enabledIndexers = GetAvailableProviders().Where(n => ((IndexerDefinition)n.Definition).EnableSearch);
|
||||
|
||||
var indexers = FilterBlockedIndexers(enabledIndexers);
|
||||
|
||||
return indexers.ToList();
|
||||
}
|
||||
|
||||
private IEnumerable<IIndexer> FilterBlockedIndexers(IEnumerable<IIndexer> indexers)
|
||||
{
|
||||
var blockedIndexers = _indexerStatusService.GetBlockedIndexers().ToDictionary(v => v.IndexerId, v => v);
|
||||
|
||||
foreach (var indexer in indexers)
|
||||
{
|
||||
IndexerStatus blockedIndexerStatus;
|
||||
if (blockedIndexers.TryGetValue(indexer.Definition.Id, out blockedIndexerStatus))
|
||||
{
|
||||
_logger.Debug("Temporarily ignoring indexer {0} till {1} due to recent failures.", indexer.Definition.Name, blockedIndexerStatus.DisabledTill.Value);
|
||||
continue;
|
||||
}
|
||||
|
||||
yield return indexer;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
25
src/NzbDrone.Core/Indexers/IndexerStatus.cs
Normal file
25
src/NzbDrone.Core/Indexers/IndexerStatus.cs
Normal file
@ -0,0 +1,25 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using NzbDrone.Core.Datastore;
|
||||
using NzbDrone.Core.Parser.Model;
|
||||
|
||||
namespace NzbDrone.Core.Indexers
|
||||
{
|
||||
public class IndexerStatus : ModelBase
|
||||
{
|
||||
public int IndexerId { get; set; }
|
||||
|
||||
public DateTime? InitialFailure { get; set; }
|
||||
public DateTime? MostRecentFailure { get; set; }
|
||||
public int EscalationLevel { get; set; }
|
||||
public DateTime? DisabledTill { get; set; }
|
||||
|
||||
public ReleaseInfo LastRssSyncReleaseInfo { get; set; }
|
||||
|
||||
public bool IsDisabled()
|
||||
{
|
||||
return DisabledTill.HasValue && DisabledTill.Value > DateTime.UtcNow;
|
||||
}
|
||||
}
|
||||
}
|
26
src/NzbDrone.Core/Indexers/IndexerStatusRepository.cs
Normal file
26
src/NzbDrone.Core/Indexers/IndexerStatusRepository.cs
Normal file
@ -0,0 +1,26 @@
|
||||
using System.Linq;
|
||||
using NzbDrone.Core.Datastore;
|
||||
using NzbDrone.Core.Messaging.Events;
|
||||
using NzbDrone.Core.ThingiProvider;
|
||||
|
||||
|
||||
namespace NzbDrone.Core.Indexers
|
||||
{
|
||||
public interface IIndexerStatusRepository : IProviderRepository<IndexerStatus>
|
||||
{
|
||||
IndexerStatus FindByIndexerId(int indexerId);
|
||||
}
|
||||
|
||||
public class IndexerStatusRepository : ProviderRepository<IndexerStatus>, IIndexerStatusRepository
|
||||
{
|
||||
public IndexerStatusRepository(IMainDatabase database, IEventAggregator eventAggregator)
|
||||
: base(database, eventAggregator)
|
||||
{
|
||||
}
|
||||
|
||||
public IndexerStatus FindByIndexerId(int indexerId)
|
||||
{
|
||||
return Query.Where(c => c.IndexerId == indexerId).SingleOrDefault();
|
||||
}
|
||||
}
|
||||
}
|
140
src/NzbDrone.Core/Indexers/IndexerStatusService.cs
Normal file
140
src/NzbDrone.Core/Indexers/IndexerStatusService.cs
Normal file
@ -0,0 +1,140 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using NLog;
|
||||
using NzbDrone.Common.Cache;
|
||||
using NzbDrone.Core.Messaging.Events;
|
||||
using NzbDrone.Core.Parser.Model;
|
||||
using NzbDrone.Core.ThingiProvider.Events;
|
||||
|
||||
namespace NzbDrone.Core.Indexers
|
||||
{
|
||||
public interface IIndexerStatusService
|
||||
{
|
||||
List<IndexerStatus> GetBlockedIndexers();
|
||||
ReleaseInfo GetLastRssSyncReleaseInfo(int indexerId);
|
||||
void RecordSuccess(int indexerId);
|
||||
void RecordFailure(int indexerId, TimeSpan minimumBackOff = default(TimeSpan));
|
||||
|
||||
void UpdateRssSyncStatus(int indexerId, ReleaseInfo releaseInfo);
|
||||
}
|
||||
|
||||
public class IndexerStatusService : IIndexerStatusService, IHandleAsync<ProviderDeletedEvent<IIndexer>>
|
||||
{
|
||||
private static readonly int[] EscalationBackOffPeriods = {
|
||||
0,
|
||||
5 * 60,
|
||||
15 * 60,
|
||||
30 * 60,
|
||||
60 * 60,
|
||||
3 * 60 * 60,
|
||||
6 * 60 * 60,
|
||||
12 * 60 * 60,
|
||||
24 * 60 * 60
|
||||
};
|
||||
private static readonly int MaximumEscalationLevel = EscalationBackOffPeriods.Length - 1;
|
||||
|
||||
private static readonly object _syncRoot = new object();
|
||||
|
||||
private readonly IIndexerStatusRepository _indexerStatusRepository;
|
||||
private readonly Logger _logger;
|
||||
|
||||
public IndexerStatusService(IIndexerStatusRepository indexerStatusRepository, Logger logger)
|
||||
{
|
||||
_indexerStatusRepository = indexerStatusRepository;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public List<IndexerStatus> GetBlockedIndexers()
|
||||
{
|
||||
return _indexerStatusRepository.All().Where(v => v.IsDisabled()).ToList();
|
||||
}
|
||||
|
||||
public ReleaseInfo GetLastRssSyncReleaseInfo(int indexerId)
|
||||
{
|
||||
return GetIndexerStatus(indexerId).LastRssSyncReleaseInfo;
|
||||
}
|
||||
|
||||
private IndexerStatus GetIndexerStatus(int indexerId)
|
||||
{
|
||||
return _indexerStatusRepository.FindByIndexerId(indexerId) ?? new IndexerStatus { IndexerId = indexerId };
|
||||
}
|
||||
|
||||
private TimeSpan CalculateBackOffPeriod(IndexerStatus status)
|
||||
{
|
||||
var level = Math.Min(MaximumEscalationLevel, status.EscalationLevel);
|
||||
|
||||
return TimeSpan.FromSeconds(EscalationBackOffPeriods[level]);
|
||||
}
|
||||
|
||||
public void RecordSuccess(int indexerId)
|
||||
{
|
||||
lock (_syncRoot)
|
||||
{
|
||||
var status = GetIndexerStatus(indexerId);
|
||||
|
||||
if (status.EscalationLevel == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
status.EscalationLevel--;
|
||||
status.DisabledTill = null;
|
||||
|
||||
_indexerStatusRepository.Upsert(status);
|
||||
}
|
||||
}
|
||||
|
||||
public void RecordFailure(int indexerId, TimeSpan minimumBackOff = default(TimeSpan))
|
||||
{
|
||||
lock (_syncRoot)
|
||||
{
|
||||
var status = GetIndexerStatus(indexerId);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
|
||||
if (status.EscalationLevel == 0)
|
||||
{
|
||||
status.InitialFailure = now;
|
||||
}
|
||||
|
||||
status.MostRecentFailure = now;
|
||||
status.EscalationLevel = Math.Min(MaximumEscalationLevel, status.EscalationLevel + 1);
|
||||
|
||||
if (minimumBackOff != TimeSpan.Zero)
|
||||
{
|
||||
while (status.EscalationLevel < MaximumEscalationLevel && CalculateBackOffPeriod(status) < minimumBackOff)
|
||||
{
|
||||
status.EscalationLevel++;
|
||||
}
|
||||
}
|
||||
|
||||
status.DisabledTill = now + CalculateBackOffPeriod(status);
|
||||
|
||||
_indexerStatusRepository.Upsert(status);
|
||||
}
|
||||
}
|
||||
|
||||
public void UpdateRssSyncStatus(int indexerId, ReleaseInfo releaseInfo)
|
||||
{
|
||||
lock (_syncRoot)
|
||||
{
|
||||
var status = GetIndexerStatus(indexerId);
|
||||
|
||||
status.LastRssSyncReleaseInfo = releaseInfo;
|
||||
|
||||
_indexerStatusRepository.Upsert(status);
|
||||
}
|
||||
}
|
||||
|
||||
public void HandleAsync(ProviderDeletedEvent<IIndexer> message)
|
||||
{
|
||||
var indexerStatus = _indexerStatusRepository.FindByIndexerId(message.ProviderId);
|
||||
|
||||
if (indexerStatus != null)
|
||||
{
|
||||
_indexerStatusRepository.Delete(indexerStatus);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
@ -19,8 +19,8 @@ public override string Name
|
||||
public override DownloadProtocol Protocol { get { return DownloadProtocol.Torrent; } }
|
||||
public override Int32 PageSize { get { return 25; } }
|
||||
|
||||
public KickassTorrents(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger)
|
||||
: base(httpClient, configService, parsingService, logger)
|
||||
public KickassTorrents(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
|
||||
: base(httpClient, indexerStatusService, configService, parsingService, logger)
|
||||
{
|
||||
|
||||
}
|
||||
|
@ -24,8 +24,7 @@ public virtual IList<IEnumerable<IndexerRequest>> GetRecentRequests()
|
||||
{
|
||||
var pageableRequests = new List<IEnumerable<IndexerRequest>>();
|
||||
|
||||
// We give kat a bit more pages to get to 100 total for recent, coz users have been missing releases.
|
||||
pageableRequests.AddIfNotNull(GetPagedRequests(4, "tv"));
|
||||
pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, "tv"));
|
||||
|
||||
return pageableRequests;
|
||||
}
|
||||
|
@ -50,8 +50,8 @@ public override IEnumerable<ProviderDefinition> DefaultDefinitions
|
||||
}
|
||||
}
|
||||
|
||||
public Newznab(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger)
|
||||
: base(httpClient, configService, parsingService, logger)
|
||||
public Newznab(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
|
||||
: base(httpClient, indexerStatusService, configService, parsingService, logger)
|
||||
{
|
||||
|
||||
}
|
||||
|
@ -23,8 +23,7 @@ public virtual IList<IEnumerable<IndexerRequest>> GetRecentRequests()
|
||||
{
|
||||
var pageableRequests = new List<IEnumerable<IndexerRequest>>();
|
||||
|
||||
// TODO: We might consider getting multiple pages in the future, but atm we limit it to 1 page.
|
||||
pageableRequests.AddIfNotNull(GetPagedRequests(1, Settings.Categories.Concat(Settings.AnimeCategories), "tvsearch", ""));
|
||||
pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, Settings.Categories.Concat(Settings.AnimeCategories), "tvsearch", ""));
|
||||
|
||||
return pageableRequests;
|
||||
}
|
||||
|
@ -19,8 +19,8 @@ public override string Name
|
||||
public override DownloadProtocol Protocol { get { return DownloadProtocol.Torrent; } }
|
||||
public override Int32 PageSize { get { return 100; } }
|
||||
|
||||
public Nyaa(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger)
|
||||
: base(httpClient, configService, parsingService, logger)
|
||||
public Nyaa(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
|
||||
: base(httpClient, indexerStatusService, configService, parsingService, logger)
|
||||
{
|
||||
|
||||
}
|
||||
|
@ -23,7 +23,7 @@ public virtual IList<IEnumerable<IndexerRequest>> GetRecentRequests()
|
||||
{
|
||||
var pageableRequests = new List<IEnumerable<IndexerRequest>>();
|
||||
|
||||
pageableRequests.AddIfNotNull(GetPagedRequests(1, null));
|
||||
pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, null));
|
||||
|
||||
return pageableRequests;
|
||||
}
|
||||
|
@ -17,8 +17,8 @@ public override string Name
|
||||
|
||||
public override DownloadProtocol Protocol { get { return DownloadProtocol.Usenet; } }
|
||||
|
||||
public Omgwtfnzbs(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger)
|
||||
: base(httpClient, configService, parsingService, logger)
|
||||
public Omgwtfnzbs(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
|
||||
: base(httpClient, indexerStatusService, configService, parsingService, logger)
|
||||
{
|
||||
|
||||
}
|
||||
|
@ -15,8 +15,8 @@ public class Rarbg : HttpIndexerBase<RarbgSettings>
|
||||
public override DownloadProtocol Protocol { get { return DownloadProtocol.Torrent; } }
|
||||
public override TimeSpan RateLimit { get { return TimeSpan.FromSeconds(2); } }
|
||||
|
||||
public Rarbg(IRarbgTokenProvider tokenProvider, IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger)
|
||||
: base(httpClient, configService, parsingService, logger)
|
||||
public Rarbg(IRarbgTokenProvider tokenProvider, IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
|
||||
: base(httpClient, indexerStatusService, configService, parsingService, logger)
|
||||
{
|
||||
_tokenProvider = tokenProvider;
|
||||
}
|
||||
|
@ -9,31 +9,35 @@
|
||||
using NzbDrone.Core.IndexerSearch;
|
||||
using NzbDrone.Core.Messaging.Commands;
|
||||
using NzbDrone.Core.Messaging.Events;
|
||||
using NzbDrone.Core.Parser.Model;
|
||||
|
||||
namespace NzbDrone.Core.Indexers
|
||||
{
|
||||
public class RssSyncService : IExecute<RssSyncCommand>
|
||||
{
|
||||
private readonly IIndexerStatusService _indexerStatusService;
|
||||
private readonly IIndexerFactory _indexerFactory;
|
||||
private readonly IFetchAndParseRss _rssFetcherAndParser;
|
||||
private readonly IMakeDownloadDecision _downloadDecisionMaker;
|
||||
private readonly IProcessDownloadDecisions _processDownloadDecisions;
|
||||
private readonly IEpisodeSearchService _episodeSearchService;
|
||||
private readonly IPendingReleaseService _pendingReleaseService;
|
||||
private readonly IEventAggregator _eventAggregator;
|
||||
private readonly Logger _logger;
|
||||
|
||||
public RssSyncService(IFetchAndParseRss rssFetcherAndParser,
|
||||
public RssSyncService(IIndexerStatusService indexerStatusService,
|
||||
IIndexerFactory indexerFactory,
|
||||
IFetchAndParseRss rssFetcherAndParser,
|
||||
IMakeDownloadDecision downloadDecisionMaker,
|
||||
IProcessDownloadDecisions processDownloadDecisions,
|
||||
IEpisodeSearchService episodeSearchService,
|
||||
IPendingReleaseService pendingReleaseService,
|
||||
IEventAggregator eventAggregator,
|
||||
Logger logger)
|
||||
{
|
||||
_indexerStatusService = indexerStatusService;
|
||||
_indexerFactory = indexerFactory;
|
||||
_rssFetcherAndParser = rssFetcherAndParser;
|
||||
_downloadDecisionMaker = downloadDecisionMaker;
|
||||
_processDownloadDecisions = processDownloadDecisions;
|
||||
_episodeSearchService = episodeSearchService;
|
||||
_pendingReleaseService = pendingReleaseService;
|
||||
_eventAggregator = eventAggregator;
|
||||
_logger = logger;
|
||||
@ -44,7 +48,10 @@ private ProcessedDecisions Sync()
|
||||
{
|
||||
_logger.ProgressInfo("Starting RSS Sync");
|
||||
|
||||
var reports = _rssFetcherAndParser.Fetch().Concat(_pendingReleaseService.GetPending()).ToList();
|
||||
var rssReleases = _rssFetcherAndParser.Fetch();
|
||||
var pendingReleases = _pendingReleaseService.GetPending();
|
||||
|
||||
var reports = rssReleases.Concat(pendingReleases).ToList();
|
||||
var decisions = _downloadDecisionMaker.GetRssDecision(reports);
|
||||
var processed = _processDownloadDecisions.ProcessDecisions(decisions);
|
||||
|
||||
@ -65,12 +72,6 @@ public void Execute(RssSyncCommand message)
|
||||
var processed = Sync();
|
||||
var grabbedOrPending = processed.Grabbed.Concat(processed.Pending).ToList();
|
||||
|
||||
if (message.LastExecutionTime.HasValue && DateTime.UtcNow.Subtract(message.LastExecutionTime.Value).TotalHours > 3)
|
||||
{
|
||||
_logger.Info("RSS Sync hasn't run since: {0}. Searching for any missing episodes since then.", message.LastExecutionTime.Value);
|
||||
_episodeSearchService.MissingEpisodesAiredAfter(message.LastExecutionTime.Value.AddDays(-1), grabbedOrPending.SelectMany(d => d.RemoteEpisode.Episodes).Select(e => e.Id));
|
||||
}
|
||||
|
||||
_eventAggregator.PublishEvent(new RssSyncCompleteEvent(processed));
|
||||
}
|
||||
}
|
||||
|
@ -22,8 +22,8 @@ public override string Name
|
||||
|
||||
private readonly ITorrentRssParserFactory _torrentRssParserFactory;
|
||||
|
||||
public TorrentRssIndexer(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, ITorrentRssParserFactory torrentRssParserFactory, Logger logger)
|
||||
: base(httpClient, configService, parsingService, logger)
|
||||
public TorrentRssIndexer(ITorrentRssParserFactory torrentRssParserFactory, IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
|
||||
: base(httpClient, indexerStatusService, configService, parsingService, logger)
|
||||
{
|
||||
_torrentRssParserFactory = torrentRssParserFactory;
|
||||
}
|
||||
|
@ -20,8 +20,8 @@ public override string Name
|
||||
public override Boolean SupportsSearch { get { return false; } }
|
||||
public override Int32 PageSize { get { return 0; } }
|
||||
|
||||
public Torrentleech(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger)
|
||||
: base(httpClient, configService, parsingService, logger)
|
||||
public Torrentleech(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
|
||||
: base(httpClient, indexerStatusService, configService, parsingService, logger)
|
||||
{
|
||||
|
||||
}
|
||||
|
@ -48,8 +48,8 @@ public override IEnumerable<ProviderDefinition> DefaultDefinitions
|
||||
}
|
||||
}
|
||||
|
||||
public Torznab(ITorznabCapabilitiesProvider torznabCapabilitiesProvider, IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger)
|
||||
: base(httpClient, configService, parsingService, logger)
|
||||
public Torznab(ITorznabCapabilitiesProvider torznabCapabilitiesProvider, IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
|
||||
: base(httpClient, indexerStatusService, configService, parsingService, logger)
|
||||
{
|
||||
_torznabCapabilitiesProvider = torznabCapabilitiesProvider;
|
||||
}
|
||||
|
@ -70,8 +70,7 @@ public virtual IList<IEnumerable<IndexerRequest>> GetRecentRequests()
|
||||
|
||||
if (capabilities.SupportedTvSearchParameters != null)
|
||||
{
|
||||
// TODO: We might consider getting multiple pages in the future, but atm we limit it to 1 page.
|
||||
pageableRequests.AddIfNotNull(GetPagedRequests(1, Settings.Categories.Concat(Settings.AnimeCategories), "tvsearch", ""));
|
||||
pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, Settings.Categories.Concat(Settings.AnimeCategories), "tvsearch", ""));
|
||||
}
|
||||
|
||||
return pageableRequests;
|
||||
|
@ -29,8 +29,8 @@ public override IIndexerRequestGenerator GetRequestGenerator()
|
||||
return new RssIndexerRequestGenerator("http://newshost.co.za/rss/?sec=TV&fr=false");
|
||||
}
|
||||
|
||||
public Wombles(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger)
|
||||
: base(httpClient, configService, parsingService, logger)
|
||||
public Wombles(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
|
||||
: base(httpClient, indexerStatusService, configService, parsingService, logger)
|
||||
{
|
||||
|
||||
}
|
||||
|
@ -235,37 +235,38 @@
|
||||
<Compile Include="Datastore\Migration\057_convert_episode_file_path_to_relative.cs" />
|
||||
<Compile Include="Datastore\Migration\058_drop_epsiode_file_path.cs" />
|
||||
<Compile Include="Datastore\Migration\059_add_enable_options_to_indexers.cs" />
|
||||
<Compile Include="Datastore\Migration\060_remove_enable_from_indexers.cs" />
|
||||
<Compile Include="Datastore\Migration\061_clear_bad_scene_names.cs" />
|
||||
<Compile Include="Datastore\Migration\062_convert_quality_models.cs" />
|
||||
<Compile Include="Datastore\Migration\063_add_remotepathmappings.cs" />
|
||||
<Compile Include="Datastore\Migration\064_add_remove_method_from_logs.cs" />
|
||||
<Compile Include="Datastore\Migration\061_clear_bad_scene_names.cs" />
|
||||
<Compile Include="Datastore\Migration\060_remove_enable_from_indexers.cs" />
|
||||
<Compile Include="Datastore\Migration\062_convert_quality_models.cs" />
|
||||
<Compile Include="Datastore\Migration\065_make_scene_numbering_nullable.cs" />
|
||||
<Compile Include="Datastore\Migration\087_remove_eztv.cs" />
|
||||
<Compile Include="Datastore\Migration\078_add_commands_table.cs" />
|
||||
<Compile Include="Datastore\Migration\068_add_release_restrictions.cs" />
|
||||
<Compile Include="Datastore\Migration\066_add_tags.cs" />
|
||||
<Compile Include="Datastore\Migration\067_add_added_to_series.cs" />
|
||||
<Compile Include="Datastore\Migration\068_add_release_restrictions.cs" />
|
||||
<Compile Include="Datastore\Migration\069_quality_proper.cs" />
|
||||
<Compile Include="Datastore\Migration\070_delay_profile.cs" />
|
||||
<Compile Include="Datastore\Migration\071_unknown_quality_in_profile.cs" />
|
||||
<Compile Include="Datastore\Migration\072_history_grabid.cs" />
|
||||
<Compile Include="Datastore\Migration\076_add_users_table.cs" />
|
||||
<Compile Include="Datastore\Migration\075_force_lib_update.cs" />
|
||||
<Compile Include="Datastore\Migration\074_disable_eztv.cs" />
|
||||
<Compile Include="Datastore\Migration\073_clear_ratings.cs" />
|
||||
<Compile Include="Datastore\Migration\074_disable_eztv.cs" />
|
||||
<Compile Include="Datastore\Migration\075_force_lib_update.cs" />
|
||||
<Compile Include="Datastore\Migration\076_add_users_table.cs" />
|
||||
<Compile Include="Datastore\Migration\077_add_add_options_to_series.cs" />
|
||||
<Compile Include="Datastore\Migration\089_add_on_rename_to_notifcations.cs" />
|
||||
<Compile Include="Datastore\Migration\078_add_commands_table.cs" />
|
||||
<Compile Include="Datastore\Migration\079_dedupe_tags.cs" />
|
||||
<Compile Include="Datastore\Migration\081_move_dot_prefix_to_transmission_category.cs" />
|
||||
<Compile Include="Datastore\Migration\082_add_fanzub_settings.cs" />
|
||||
<Compile Include="Datastore\Migration\083_additonal_blacklist_columns.cs" />
|
||||
<Compile Include="Datastore\Migration\084_update_quality_minmax_size.cs" />
|
||||
<Compile Include="Datastore\Migration\085_expand_transmission_urlbase.cs" />
|
||||
<Compile Include="Datastore\Migration\086_pushbullet_device_ids.cs" />
|
||||
<Compile Include="Datastore\Migration\081_move_dot_prefix_to_transmission_category.cs" />
|
||||
<Compile Include="Datastore\Migration\079_dedupe_tags.cs" />
|
||||
<Compile Include="Datastore\Migration\070_delay_profile.cs" />
|
||||
<Compile Include="Datastore\Migration\084_update_quality_minmax_size.cs" />
|
||||
<Compile Include="Datastore\Migration\083_additonal_blacklist_columns.cs" />
|
||||
<Compile Include="Datastore\Migration\082_add_fanzub_settings.cs" />
|
||||
<Compile Include="Datastore\Migration\087_remove_eztv.cs" />
|
||||
<Compile Include="Datastore\Migration\088_pushbullet_devices_channels_list.cs" />
|
||||
<Compile Include="Datastore\Migration\092_add_unverifiedscenenumbering.cs" />
|
||||
<Compile Include="Datastore\Migration\089_add_on_rename_to_notifcations.cs" />
|
||||
<Compile Include="Datastore\Migration\090_update_kickass_url.cs" />
|
||||
<Compile Include="Datastore\Migration\091_added_indexerstatus.cs" />
|
||||
<Compile Include="Datastore\Migration\092_add_unverifiedscenenumbering.cs" />
|
||||
<Compile Include="Datastore\Migration\Framework\MigrationContext.cs" />
|
||||
<Compile Include="Datastore\Migration\Framework\MigrationController.cs" />
|
||||
<Compile Include="Datastore\Migration\Framework\MigrationDbFactory.cs" />
|
||||
@ -434,6 +435,7 @@
|
||||
<Compile Include="HealthCheck\Checks\DownloadClientCheck.cs" />
|
||||
<Compile Include="HealthCheck\Checks\DroneFactoryCheck.cs" />
|
||||
<Compile Include="HealthCheck\Checks\ImportMechanismCheck.cs" />
|
||||
<Compile Include="HealthCheck\Checks\IndexerStatusCheck.cs" />
|
||||
<Compile Include="HealthCheck\Checks\IndexerCheck.cs" />
|
||||
<Compile Include="HealthCheck\Checks\MediaInfoDllCheck.cs" />
|
||||
<Compile Include="HealthCheck\Checks\MonoVersionCheck.cs" />
|
||||
@ -453,6 +455,7 @@
|
||||
<Compile Include="Housekeeping\Housekeepers\CleanupOrphanedBlacklist.cs" />
|
||||
<Compile Include="Housekeeping\Housekeepers\CleanupOrphanedEpisodeFiles.cs" />
|
||||
<Compile Include="Housekeeping\Housekeepers\CleanupOrphanedEpisodes.cs" />
|
||||
<Compile Include="Housekeeping\Housekeepers\CleanupOrphanedIndexerStatus.cs" />
|
||||
<Compile Include="Housekeeping\Housekeepers\CleanupOrphanedHistoryItems.cs" />
|
||||
<Compile Include="Housekeeping\Housekeepers\CleanupOrphanedMetadataFiles.cs" />
|
||||
<Compile Include="Housekeeping\Housekeepers\CleanupOrphanedPendingReleases.cs" />
|
||||
@ -493,10 +496,13 @@
|
||||
<Compile Include="Indexers\IndexerBase.cs" />
|
||||
<Compile Include="Indexers\IndexerDefinition.cs" />
|
||||
<Compile Include="Indexers\IndexerFactory.cs" />
|
||||
<Compile Include="Indexers\IndexerStatusRepository.cs" />
|
||||
<Compile Include="Indexers\IndexerRepository.cs" />
|
||||
<Compile Include="Indexers\IndexerRequest.cs" />
|
||||
<Compile Include="Indexers\IndexerResponse.cs" />
|
||||
<Compile Include="Indexers\IndexerSettingUpdatedEvent.cs" />
|
||||
<Compile Include="Indexers\IndexerStatus.cs" />
|
||||
<Compile Include="Indexers\IndexerStatusService.cs" />
|
||||
<Compile Include="Indexers\IProcessIndexerResponse.cs" />
|
||||
<Compile Include="Indexers\IPTorrents\IPTorrentsRequestGenerator.cs" />
|
||||
<Compile Include="Indexers\IPTorrents\IPTorrents.cs" />
|
||||
@ -890,6 +896,7 @@
|
||||
<Compile Include="Tags\TagService.cs" />
|
||||
<Compile Include="Tags\TagsUpdatedEvent.cs" />
|
||||
<Compile Include="ThingiProvider\ConfigContractNotFoundException.cs" />
|
||||
<Compile Include="ThingiProvider\Events\ProviderDeletedEvent.cs" />
|
||||
<Compile Include="ThingiProvider\Events\ProviderUpdatedEvent.cs" />
|
||||
<Compile Include="ThingiProvider\IProvider.cs" />
|
||||
<Compile Include="ThingiProvider\IProviderConfig.cs" />
|
||||
|
@ -1,24 +1,24 @@
|
||||
using System;
|
||||
using System.Text;
|
||||
using NzbDrone.Core.Indexers;
|
||||
|
||||
namespace NzbDrone.Core.Parser.Model
|
||||
{
|
||||
using System.Text;
|
||||
|
||||
public class ReleaseInfo
|
||||
{
|
||||
public String Guid { get; set; }
|
||||
public String Title { get; set; }
|
||||
public Int64 Size { get; set; }
|
||||
public String DownloadUrl { get; set; }
|
||||
public String InfoUrl { get; set; }
|
||||
public String CommentUrl { get; set; }
|
||||
public String Indexer { get; set; }
|
||||
public string Guid { get; set; }
|
||||
public string Title { get; set; }
|
||||
public long Size { get; set; }
|
||||
public string DownloadUrl { get; set; }
|
||||
public string InfoUrl { get; set; }
|
||||
public string CommentUrl { get; set; }
|
||||
public int IndexerId { get; set; }
|
||||
public string Indexer { get; set; }
|
||||
public DownloadProtocol DownloadProtocol { get; set; }
|
||||
public Int32 TvRageId { get; set; }
|
||||
public int TvRageId { get; set; }
|
||||
public DateTime PublishDate { get; set; }
|
||||
|
||||
public Int32 Age
|
||||
public int Age
|
||||
{
|
||||
get
|
||||
{
|
||||
@ -30,7 +30,7 @@ public Int32 Age
|
||||
private set { }
|
||||
}
|
||||
|
||||
public Double AgeHours
|
||||
public double AgeHours
|
||||
{
|
||||
get
|
||||
{
|
||||
@ -42,7 +42,7 @@ public Double AgeHours
|
||||
private set { }
|
||||
}
|
||||
|
||||
public Double AgeMinutes
|
||||
public double AgeMinutes
|
||||
{
|
||||
get
|
||||
{
|
||||
@ -56,7 +56,7 @@ private set { }
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
return String.Format("[{0}] {1} [{2}]", PublishDate, Title, Size);
|
||||
return string.Format("[{0}] {1} [{2}]", PublishDate, Title, Size);
|
||||
}
|
||||
|
||||
public virtual string ToString(string format)
|
||||
|
@ -0,0 +1,14 @@
|
||||
using NzbDrone.Common.Messaging;
|
||||
|
||||
namespace NzbDrone.Core.ThingiProvider.Events
|
||||
{
|
||||
public class ProviderDeletedEvent<TProvider> : IEvent
|
||||
{
|
||||
public int ProviderId { get; private set; }
|
||||
|
||||
public ProviderDeletedEvent(int id)
|
||||
{
|
||||
ProviderId = id;
|
||||
}
|
||||
}
|
||||
}
|
@ -4,5 +4,11 @@ namespace NzbDrone.Core.ThingiProvider.Events
|
||||
{
|
||||
public class ProviderUpdatedEvent<TProvider> : IEvent
|
||||
{
|
||||
public ProviderDefinition Definition { get; private set; }
|
||||
|
||||
public ProviderUpdatedEvent(ProviderDefinition definition)
|
||||
{
|
||||
Definition = definition;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
@ -99,12 +99,13 @@ public virtual TProviderDefinition Create(TProviderDefinition definition)
|
||||
public virtual void Update(TProviderDefinition definition)
|
||||
{
|
||||
_providerRepository.Update(definition);
|
||||
_eventAggregator.PublishEvent(new ProviderUpdatedEvent<TProvider>());
|
||||
_eventAggregator.PublishEvent(new ProviderUpdatedEvent<TProvider>(definition));
|
||||
}
|
||||
|
||||
public void Delete(int id)
|
||||
{
|
||||
_providerRepository.Delete(id);
|
||||
_eventAggregator.PublishEvent(new ProviderDeletedEvent<TProvider>(id));
|
||||
}
|
||||
|
||||
public TProvider GetInstance(TProviderDefinition definition)
|
||||
|
Loading…
Reference in New Issue
Block a user