New: Differentiate between short term and long term (more than 6 hours) indexer failures (#5202)

* New: Differentiate between short term and long term (more than 6 hours) indexer failures

(cherry picked from commit 2adedb97da5ad31b65f0dc2eec5c263efe95731f)

* fixup! Mock Localization

Co-authored-by: Mark McDowall <mark@mcdowall.ca>
Co-authored-by: Qstick <qstick@gmail.com>

frontend/src/System/Status/Health/Health.js

@@ -19,6 +19,7 @@ function getInternalLink(source) {
     case 'IndexerRssCheck':
     case 'IndexerSearchCheck':
     case 'IndexerStatusCheck':
+    case 'IndexerLongTermStatusCheck':
       return (
         <IconButton
           name={icons.SETTINGS}

src/NzbDrone.Core.Test/HealthCheck/Checks/IndexerLongTermStatusCheckFixture.cs

@@ -0,0 +1,90 @@
+using System;
+using System.Collections.Generic;
+using Moq;
+using NUnit.Framework;
+using NzbDrone.Core.HealthCheck.Checks;
+using NzbDrone.Core.Indexers;
+using NzbDrone.Core.Localization;
+using NzbDrone.Core.Test.Framework;
+
+namespace NzbDrone.Core.Test.HealthCheck.Checks
+{
+    [TestFixture]
+    public class IndexerLongTermStatusCheckFixture : CoreTest<IndexerLongTermStatusCheck>
+    {
+        private List<IIndexer> _indexers = new List<IIndexer>();
+        private List<IndexerStatus> _blockedIndexers = new List<IndexerStatus>();
+
+        [SetUp]
+        public void SetUp()
+        {
+            Mocker.GetMock<IIndexerFactory>()
+                  .Setup(v => v.GetAvailableProviders())
+                  .Returns(_indexers);
+
+            Mocker.GetMock<IIndexerStatusService>()
+                   .Setup(v => v.GetBlockedProviders())
+                   .Returns(_blockedIndexers);
+
+            Mocker.GetMock<ILocalizationService>()
+                   .Setup(v => v.GetLocalizedString(It.IsAny<string>()))
+                   .Returns("Error");
+        }
+
+        private Mock<IIndexer> GivenIndexer(int id, double backoffHours, double failureHours)
+        {
+            var mockIndexer = new Mock<IIndexer>();
+            mockIndexer.SetupGet(s => s.Definition).Returns(new IndexerDefinition { Id = id });
+            mockIndexer.SetupGet(s => s.SupportsSearch).Returns(true);
+
+            _indexers.Add(mockIndexer.Object);
+
+            if (backoffHours != 0.0)
+            {
+                _blockedIndexers.Add(new IndexerStatus
+                    {
+                        ProviderId = id,
+                        InitialFailure = DateTime.UtcNow.AddHours(-failureHours),
+                        MostRecentFailure = DateTime.UtcNow.AddHours(-0.1),
+                        EscalationLevel = 5,
+                        DisabledTill = DateTime.UtcNow.AddHours(backoffHours)
+                    });
+            }
+
+            return mockIndexer;
+        }
+
+        [Test]
+        public void should_not_return_error_when_no_indexers()
+        {
+            Subject.Check().ShouldBeOk();
+        }
+
+        [Test]
+        public void should_return_warning_if_indexer_unavailable()
+        {
+            GivenIndexer(1, 10.0, 24.0);
+            GivenIndexer(2, 0.0, 0.0);
+
+            Subject.Check().ShouldBeWarning();
+        }
+
+        [Test]
+        public void should_return_error_if_all_indexers_unavailable()
+        {
+            GivenIndexer(1, 10.0, 24.0);
+
+            Subject.Check().ShouldBeError();
+        }
+
+        [Test]
+        public void should_return_warning_if_few_indexers_unavailable()
+        {
+            GivenIndexer(1, 10.0, 24.0);
+            GivenIndexer(2, 10.0, 24.0);
+            GivenIndexer(3, 0.0, 0.0);
+
+            Subject.Check().ShouldBeWarning();
+        }
+    }
+}

src/NzbDrone.Core.Test/HealthCheck/Checks/IndexerStatusCheckFixture.cs

@@ -65,7 +65,7 @@ public void should_not_return_error_when_no_indexers()
         [Test]
         public void should_return_warning_if_indexer_unavailable()
         {
-            GivenIndexer(1, 10.0, 24.0);
+            GivenIndexer(1, 2.0, 4.0);
             GivenIndexer(2, 0.0, 0.0);
 
             Subject.Check().ShouldBeWarning();
@@ -74,7 +74,7 @@ public void should_return_warning_if_indexer_unavailable()
         [Test]
         public void should_return_error_if_all_indexers_unavailable()
         {
-            GivenIndexer(1, 10.0, 24.0);
+            GivenIndexer(1, 2.0, 4.0);
 
             Subject.Check().ShouldBeError();
         }
@@ -82,8 +82,8 @@ public void should_return_error_if_all_indexers_unavailable()
         [Test]
         public void should_return_warning_if_few_indexers_unavailable()
         {
-            GivenIndexer(1, 10.0, 24.0);
+            GivenIndexer(1, 2.0, 4.0);
-            GivenIndexer(2, 10.0, 24.0);
+            GivenIndexer(2, 2.0, 4.0);
             GivenIndexer(3, 0.0, 0.0);
 
             Subject.Check().ShouldBeWarning();

src/NzbDrone.Core/HealthCheck/Checks/IndexerLongTermStatusCheck.cs

@@ -0,0 +1,59 @@
+using System;
+using System.Linq;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Core.Indexers;
+using NzbDrone.Core.Localization;
+using NzbDrone.Core.ThingiProvider.Events;
+
+namespace NzbDrone.Core.HealthCheck.Checks
+{
+    [CheckOn(typeof(ProviderUpdatedEvent<IIndexer>))]
+    [CheckOn(typeof(ProviderDeletedEvent<IIndexer>))]
+    [CheckOn(typeof(ProviderStatusChangedEvent<IIndexer>))]
+    public class IndexerLongTermStatusCheck : HealthCheckBase
+    {
+        private readonly IIndexerFactory _providerFactory;
+        private readonly IIndexerStatusService _providerStatusService;
+
+        public IndexerLongTermStatusCheck(IIndexerFactory providerFactory,
+                                          IIndexerStatusService providerStatusService,
+                                          ILocalizationService localizationService)
+            : base(localizationService)
+        {
+            _providerFactory = providerFactory;
+            _providerStatusService = providerStatusService;
+        }
+
+        public override HealthCheck Check()
+        {
+            var enabledProviders = _providerFactory.GetAvailableProviders();
+            var backOffProviders = enabledProviders.Join(_providerStatusService.GetBlockedProviders(),
+                                                       i => i.Definition.Id,
+                                                       s => s.ProviderId,
+                                                       (i, s) => new { Provider = i, Status = s })
+                                                   .Where(p => p.Status.InitialFailure.HasValue &&
+                                                               p.Status.InitialFailure.Value.Before(
+                                                                   DateTime.UtcNow.AddHours(-6)))
+                                                   .ToList();
+
+            if (backOffProviders.Empty())
+            {
+                return new HealthCheck(GetType());
+            }
+
+            if (backOffProviders.Count == enabledProviders.Count)
+            {
+                return new HealthCheck(GetType(),
+                    HealthCheckResult.Error,
+                    _localizationService.GetLocalizedString("IndexerLongTermStatusCheckAllClientMessage"),
+                    "#indexers-are-unavailable-due-to-failures");
+            }
+
+            return new HealthCheck(GetType(),
+                HealthCheckResult.Warning,
+                string.Format(_localizationService.GetLocalizedString("IndexerLongTermStatusCheckSingleClientMessage"),
+                    string.Join(", ", backOffProviders.Select(v => v.Provider.Definition.Name))),
+                "#indexers-are-unavailable-due-to-failures");
+        }
+    }
+}

src/NzbDrone.Core/HealthCheck/Checks/IndexerStatusCheck.cs

@@ -1,3 +1,4 @@
+using System;
 using System.Linq;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Core.Indexers;
@@ -28,6 +29,9 @@ public override HealthCheck Check()
                                                        i => i.Definition.Id,
                                                        s => s.ProviderId,
                                                        (i, s) => new { Provider = i, Status = s })
+                                                   .Where(p => p.Status.InitialFailure.HasValue &&
+                                                               p.Status.InitialFailure.Value.After(
+                                                                   DateTime.UtcNow.AddHours(-6)))
                                                    .ToList();
 
             if (backOffProviders.Empty())
@@ -37,10 +41,17 @@ public override HealthCheck Check()
 
             if (backOffProviders.Count == enabledProviders.Count)
             {
-                return new HealthCheck(GetType(), HealthCheckResult.Error, _localizationService.GetLocalizedString("IndexerStatusCheckAllClientMessage"), "#indexers-are-unavailable-due-to-failures");
+                return new HealthCheck(GetType(),
+                    HealthCheckResult.Error,
+                    _localizationService.GetLocalizedString("IndexerStatusCheckAllClientMessage"),
+                    "#indexers-are-unavailable-due-to-failures");
             }
 
-            return new HealthCheck(GetType(), HealthCheckResult.Warning, string.Format(_localizationService.GetLocalizedString("IndexerStatusCheckSingleClientMessage"), string.Join(", ", backOffProviders.Select(v => v.Provider.Definition.Name))), "#indexers-are-unavailable-due-to-failures");
+            return new HealthCheck(GetType(),
+                HealthCheckResult.Warning,
+                string.Format(_localizationService.GetLocalizedString("IndexerStatusCheckSingleClientMessage"),
+                    string.Join(", ", backOffProviders.Select(v => v.Provider.Definition.Name))),
+                "#indexers-are-unavailable-due-to-failures");
         }
     }
 }

src/NzbDrone.Core/Localization/Core/en.json

@@ -343,6 +343,8 @@
   "IndexersSettingsSummary": "Indexers and release restrictions",
   "IndexerStatusCheckAllClientMessage": "All indexers are unavailable due to failures",
   "IndexerStatusCheckSingleClientMessage": "Indexers unavailable due to failures: {0}",
+  "IndexerLongTermStatusCheckAllClientMessage": "All indexers are unavailable due to failures for more than 6 hours",
+  "IndexerLongTermStatusCheckSingleClientMessage": "Indexers unavailable due to failures for more than 6 hours: {0}",
   "Info": "Info",
   "InteractiveImport": "Interactive Import",
   "InteractiveSearch": "Interactive Search",