diff --git a/frontend/src/System/Status/Health/Health.js b/frontend/src/System/Status/Health/Health.js index d04772a4b..01b53d1dc 100644 --- a/frontend/src/System/Status/Health/Health.js +++ b/frontend/src/System/Status/Health/Health.js @@ -19,6 +19,7 @@ function getInternalLink(source) { case 'IndexerRssCheck': case 'IndexerSearchCheck': case 'IndexerStatusCheck': + case 'IndexerLongTermStatusCheck': return ( + { + private List _indexers = new List(); + private List _blockedIndexers = new List(); + + [SetUp] + public void SetUp() + { + Mocker.GetMock() + .Setup(v => v.GetAvailableProviders()) + .Returns(_indexers); + + Mocker.GetMock() + .Setup(v => v.GetBlockedProviders()) + .Returns(_blockedIndexers); + + Mocker.GetMock() + .Setup(v => v.GetLocalizedString(It.IsAny())) + .Returns("Error"); + } + + private Mock GivenIndexer(int id, double backoffHours, double failureHours) + { + var mockIndexer = new Mock(); + mockIndexer.SetupGet(s => s.Definition).Returns(new IndexerDefinition { Id = id }); + mockIndexer.SetupGet(s => s.SupportsSearch).Returns(true); + + _indexers.Add(mockIndexer.Object); + + if (backoffHours != 0.0) + { + _blockedIndexers.Add(new IndexerStatus + { + ProviderId = id, + InitialFailure = DateTime.UtcNow.AddHours(-failureHours), + MostRecentFailure = DateTime.UtcNow.AddHours(-0.1), + EscalationLevel = 5, + DisabledTill = DateTime.UtcNow.AddHours(backoffHours) + }); + } + + return mockIndexer; + } + + [Test] + public void should_not_return_error_when_no_indexers() + { + Subject.Check().ShouldBeOk(); + } + + [Test] + public void should_return_warning_if_indexer_unavailable() + { + GivenIndexer(1, 10.0, 24.0); + GivenIndexer(2, 0.0, 0.0); + + Subject.Check().ShouldBeWarning(); + } + + [Test] + public void should_return_error_if_all_indexers_unavailable() + { + GivenIndexer(1, 10.0, 24.0); + + Subject.Check().ShouldBeError(); + } + + [Test] + public void should_return_warning_if_few_indexers_unavailable() + { + GivenIndexer(1, 10.0, 24.0); + GivenIndexer(2, 10.0, 24.0); + GivenIndexer(3, 0.0, 0.0); + + Subject.Check().ShouldBeWarning(); + } + } +} diff --git a/src/NzbDrone.Core.Test/HealthCheck/Checks/IndexerStatusCheckFixture.cs b/src/NzbDrone.Core.Test/HealthCheck/Checks/IndexerStatusCheckFixture.cs index acf4f054c..ed54dc0f7 100644 --- a/src/NzbDrone.Core.Test/HealthCheck/Checks/IndexerStatusCheckFixture.cs +++ b/src/NzbDrone.Core.Test/HealthCheck/Checks/IndexerStatusCheckFixture.cs @@ -65,7 +65,7 @@ public void should_not_return_error_when_no_indexers() [Test] public void should_return_warning_if_indexer_unavailable() { - GivenIndexer(1, 10.0, 24.0); + GivenIndexer(1, 2.0, 4.0); GivenIndexer(2, 0.0, 0.0); Subject.Check().ShouldBeWarning(); @@ -74,7 +74,7 @@ public void should_return_warning_if_indexer_unavailable() [Test] public void should_return_error_if_all_indexers_unavailable() { - GivenIndexer(1, 10.0, 24.0); + GivenIndexer(1, 2.0, 4.0); Subject.Check().ShouldBeError(); } @@ -82,8 +82,8 @@ public void should_return_error_if_all_indexers_unavailable() [Test] public void should_return_warning_if_few_indexers_unavailable() { - GivenIndexer(1, 10.0, 24.0); - GivenIndexer(2, 10.0, 24.0); + GivenIndexer(1, 2.0, 4.0); + GivenIndexer(2, 2.0, 4.0); GivenIndexer(3, 0.0, 0.0); Subject.Check().ShouldBeWarning(); diff --git a/src/NzbDrone.Core/HealthCheck/Checks/IndexerLongTermStatusCheck.cs b/src/NzbDrone.Core/HealthCheck/Checks/IndexerLongTermStatusCheck.cs new file mode 100644 index 000000000..9ba7ffd1c --- /dev/null +++ b/src/NzbDrone.Core/HealthCheck/Checks/IndexerLongTermStatusCheck.cs @@ -0,0 +1,59 @@ +using System; +using System.Linq; +using NzbDrone.Common.Extensions; +using NzbDrone.Core.Indexers; +using NzbDrone.Core.Localization; +using NzbDrone.Core.ThingiProvider.Events; + +namespace NzbDrone.Core.HealthCheck.Checks +{ + [CheckOn(typeof(ProviderUpdatedEvent))] + [CheckOn(typeof(ProviderDeletedEvent))] + [CheckOn(typeof(ProviderStatusChangedEvent))] + public class IndexerLongTermStatusCheck : HealthCheckBase + { + private readonly IIndexerFactory _providerFactory; + private readonly IIndexerStatusService _providerStatusService; + + public IndexerLongTermStatusCheck(IIndexerFactory providerFactory, + IIndexerStatusService providerStatusService, + ILocalizationService localizationService) + : base(localizationService) + { + _providerFactory = providerFactory; + _providerStatusService = providerStatusService; + } + + public override HealthCheck Check() + { + var enabledProviders = _providerFactory.GetAvailableProviders(); + var backOffProviders = enabledProviders.Join(_providerStatusService.GetBlockedProviders(), + i => i.Definition.Id, + s => s.ProviderId, + (i, s) => new { Provider = i, Status = s }) + .Where(p => p.Status.InitialFailure.HasValue && + p.Status.InitialFailure.Value.Before( + DateTime.UtcNow.AddHours(-6))) + .ToList(); + + if (backOffProviders.Empty()) + { + return new HealthCheck(GetType()); + } + + if (backOffProviders.Count == enabledProviders.Count) + { + return new HealthCheck(GetType(), + HealthCheckResult.Error, + _localizationService.GetLocalizedString("IndexerLongTermStatusCheckAllClientMessage"), + "#indexers-are-unavailable-due-to-failures"); + } + + return new HealthCheck(GetType(), + HealthCheckResult.Warning, + string.Format(_localizationService.GetLocalizedString("IndexerLongTermStatusCheckSingleClientMessage"), + string.Join(", ", backOffProviders.Select(v => v.Provider.Definition.Name))), + "#indexers-are-unavailable-due-to-failures"); + } + } +} diff --git a/src/NzbDrone.Core/HealthCheck/Checks/IndexerStatusCheck.cs b/src/NzbDrone.Core/HealthCheck/Checks/IndexerStatusCheck.cs index 1d4b9ba95..f7d6ee541 100644 --- a/src/NzbDrone.Core/HealthCheck/Checks/IndexerStatusCheck.cs +++ b/src/NzbDrone.Core/HealthCheck/Checks/IndexerStatusCheck.cs @@ -1,3 +1,4 @@ +using System; using System.Linq; using NzbDrone.Common.Extensions; using NzbDrone.Core.Indexers; @@ -25,10 +26,13 @@ public override HealthCheck Check() { var enabledProviders = _providerFactory.GetAvailableProviders(); var backOffProviders = enabledProviders.Join(_providerStatusService.GetBlockedProviders(), - i => i.Definition.Id, - s => s.ProviderId, - (i, s) => new { Provider = i, Status = s }) - .ToList(); + i => i.Definition.Id, + s => s.ProviderId, + (i, s) => new { Provider = i, Status = s }) + .Where(p => p.Status.InitialFailure.HasValue && + p.Status.InitialFailure.Value.After( + DateTime.UtcNow.AddHours(-6))) + .ToList(); if (backOffProviders.Empty()) { @@ -37,10 +41,17 @@ public override HealthCheck Check() if (backOffProviders.Count == enabledProviders.Count) { - return new HealthCheck(GetType(), HealthCheckResult.Error, _localizationService.GetLocalizedString("IndexerStatusCheckAllClientMessage"), "#indexers-are-unavailable-due-to-failures"); + return new HealthCheck(GetType(), + HealthCheckResult.Error, + _localizationService.GetLocalizedString("IndexerStatusCheckAllClientMessage"), + "#indexers-are-unavailable-due-to-failures"); } - return new HealthCheck(GetType(), HealthCheckResult.Warning, string.Format(_localizationService.GetLocalizedString("IndexerStatusCheckSingleClientMessage"), string.Join(", ", backOffProviders.Select(v => v.Provider.Definition.Name))), "#indexers-are-unavailable-due-to-failures"); + return new HealthCheck(GetType(), + HealthCheckResult.Warning, + string.Format(_localizationService.GetLocalizedString("IndexerStatusCheckSingleClientMessage"), + string.Join(", ", backOffProviders.Select(v => v.Provider.Definition.Name))), + "#indexers-are-unavailable-due-to-failures"); } } } diff --git a/src/NzbDrone.Core/Localization/Core/en.json b/src/NzbDrone.Core/Localization/Core/en.json index bc47f73c7..c85bbca12 100644 --- a/src/NzbDrone.Core/Localization/Core/en.json +++ b/src/NzbDrone.Core/Localization/Core/en.json @@ -343,6 +343,8 @@ "IndexersSettingsSummary": "Indexers and release restrictions", "IndexerStatusCheckAllClientMessage": "All indexers are unavailable due to failures", "IndexerStatusCheckSingleClientMessage": "Indexers unavailable due to failures: {0}", + "IndexerLongTermStatusCheckAllClientMessage": "All indexers are unavailable due to failures for more than 6 hours", + "IndexerLongTermStatusCheckSingleClientMessage": "Indexers unavailable due to failures for more than 6 hours: {0}", "Info": "Info", "InteractiveImport": "Interactive Import", "InteractiveSearch": "Interactive Search",