Failed downloads are added to history

src/NzbDrone.Core.Test/Download/FailedDownloadServiceFixture.cs

@@ -0,0 +1,198 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using FizzWare.NBuilder;
+using Moq;
+using NUnit.Framework;
+using NzbDrone.Core.Download;
+using NzbDrone.Core.History;
+using NzbDrone.Core.Messaging.Events;
+using NzbDrone.Core.Test.Framework;
+using NzbDrone.Core.Tv;
+
+namespace NzbDrone.Core.Test.Download
+{
+    [TestFixture]
+    public class FailedDownloadServiceFixture : CoreTest<FailedDownloadService>
+    {
+        private Series _series;
+        private Episode _episode;
+        private List<HistoryItem> _completed;
+        private List<HistoryItem> _failed;
+
+        [SetUp]
+        public void Setup()
+        {
+            _series = Builder<Series>.CreateNew().Build();
+            _episode = Builder<Episode>.CreateNew().Build();
+
+            _completed = Builder<HistoryItem>.CreateListOfSize(5)
+                                             .All()
+                                             .With(h => h.Status = HistoryStatus.Completed)
+                                             .Build()
+                                             .ToList();
+
+            _failed = Builder<HistoryItem>.CreateListOfSize(1)
+                                          .All()
+                                          .With(h => h.Status = HistoryStatus.Failed)
+                                          .Build()
+                                          .ToList();
+
+            Mocker.GetMock<IProvideDownloadClient>()
+                  .Setup(c => c.GetDownloadClient()).Returns(Mocker.GetMock<IDownloadClient>().Object);
+        }
+
+        private void GivenNoRecentHistory()
+        {
+            Mocker.GetMock<IHistoryService>()
+                  .Setup(s => s.BetweenDates(It.IsAny<DateTime>(), It.IsAny<DateTime>(), HistoryEventType.Grabbed))
+                  .Returns(new List<History.History>());
+        }
+
+        private void GivenRecentHistory(List<History.History> history)
+        {
+            Mocker.GetMock<IHistoryService>()
+                  .Setup(s => s.BetweenDates(It.IsAny<DateTime>(), It.IsAny<DateTime>(), HistoryEventType.Grabbed))
+                  .Returns(history);
+        }
+
+        private void GivenNoFailedHistory()
+        {
+            Mocker.GetMock<IHistoryService>()
+                  .Setup(s => s.Failed())
+                  .Returns(new List<History.History>());
+        }
+
+        private void GivenFailedHistory(List<History.History> failedHistory)
+        {
+            Mocker.GetMock<IHistoryService>()
+                  .Setup(s => s.Failed())
+                  .Returns(failedHistory);
+        }
+
+        private void GivenFailedDownloadClientHistory()
+        {
+            Mocker.GetMock<IDownloadClient>()
+                  .Setup(s => s.GetHistory(0, 20))
+                  .Returns(_failed);
+        }
+
+        private void VerifyNoFailedDownloads()
+        {
+            Mocker.GetMock<IEventAggregator>()
+                .Verify(v => v.PublishEvent(It.IsAny<DownloadFailedEvent>()), Times.Never());
+        }
+
+        private void VerifyFailedDownloads(int count = 1)
+        {
+            Mocker.GetMock<IEventAggregator>()
+                .Verify(v => v.PublishEvent(It.IsAny<DownloadFailedEvent>()), Times.Exactly(count));
+        }
+
+        [Test]
+        public void should_not_process_if_no_download_client_history()
+        {
+            Mocker.GetMock<IDownloadClient>()
+                  .Setup(s => s.GetHistory(0, 20))
+                  .Returns(new List<HistoryItem>());
+
+            Subject.Execute(new FailedDownloadCommand());
+
+            Mocker.GetMock<IHistoryService>()
+                  .Verify(s => s.BetweenDates(It.IsAny<DateTime>(), It.IsAny<DateTime>(), HistoryEventType.Grabbed),
+                      Times.Never());
+
+            VerifyNoFailedDownloads();
+        }
+
+        [Test]
+        public void should_not_process_if_no_failed_items_in_download_client_history()
+        {
+            Mocker.GetMock<IDownloadClient>()
+                  .Setup(s => s.GetHistory(0, 20))
+                  .Returns(_completed);
+
+            Subject.Execute(new FailedDownloadCommand());
+
+            Mocker.GetMock<IHistoryService>()
+                  .Verify(s => s.BetweenDates(It.IsAny<DateTime>(), It.IsAny<DateTime>(), HistoryEventType.Grabbed),
+                      Times.Never());
+
+            VerifyNoFailedDownloads();
+        }
+
+        [Test]
+        public void should_not_process_if_matching_history_is_not_found()
+        {
+            GivenNoRecentHistory();
+            GivenFailedDownloadClientHistory();
+
+            Subject.Execute(new FailedDownloadCommand());
+
+            VerifyNoFailedDownloads();
+        }
+
+        [Test]
+        public void should_not_process_if_already_added_to_history_as_failed()
+        {
+            GivenFailedDownloadClientHistory();
+            
+            var history = Builder<History.History>.CreateListOfSize(1)
+                                                  .Build()
+                                                  .ToList();
+            
+            GivenRecentHistory(history);
+            GivenFailedHistory(history);
+
+            history.First().Data.Add("downloadClient", "SabnzbdClient");
+            history.First().Data.Add("downloadClientId", _failed.First().Id);
+
+            Subject.Execute(new FailedDownloadCommand());
+
+            VerifyNoFailedDownloads();
+        }
+
+        [Test]
+        public void should_process_if_not_already_in_failed_history()
+        {
+            GivenFailedDownloadClientHistory();
+
+            var history = Builder<History.History>.CreateListOfSize(1)
+                                                  .Build()
+                                                  .ToList();
+
+            GivenRecentHistory(history);
+            GivenNoFailedHistory();
+
+            history.First().Data.Add("downloadClient", "SabnzbdClient");
+            history.First().Data.Add("downloadClientId", _failed.First().Id);
+
+            Subject.Execute(new FailedDownloadCommand());
+
+            VerifyFailedDownloads();
+        }
+
+        [Test]
+        public void should_process_for_each_failed_episode()
+        {
+            GivenFailedDownloadClientHistory();
+
+            var history = Builder<History.History>.CreateListOfSize(2)
+                                                  .Build()
+                                                  .ToList();
+
+            GivenRecentHistory(history);
+            GivenNoFailedHistory();
+
+            history.ForEach(h =>
+            {
+                h.Data.Add("downloadClient", "SabnzbdClient");
+                h.Data.Add("downloadClientId", _failed.First().Id);
+            });
+
+            Subject.Execute(new FailedDownloadCommand());
+
+            VerifyFailedDownloads(2);
+        }
+    }
+}

src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj

@@ -124,6 +124,7 @@
     <Compile Include="Download\DownloadClientTests\SabProviderTests\QueueFixture.cs" />
     <Compile Include="Download\DownloadClientTests\SabProviderTests\SabProviderFixture.cs" />
     <Compile Include="Download\DownloadServiceFixture.cs" />
+    <Compile Include="Download\FailedDownloadServiceFixture.cs" />
     <Compile Include="Framework\CoreTest.cs" />
     <Compile Include="Framework\DbTest.cs" />
     <Compile Include="Framework\NBuilderExtensions.cs" />

src/NzbDrone.Core/Download/Clients/BlackholeProvider.cs

@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.IO;
 using NLog;
 using NzbDrone.Common;
@@ -51,5 +52,10 @@ namespace NzbDrone.Core.Download.Clients
         {
             return new QueueItem[0];
         }
+
+        public IEnumerable<HistoryItem> GetHistory(int start = 0, int limit = 0)
+        {
+            return new HistoryItem[0];
+        }
     }
 }

src/NzbDrone.Core/Download/Clients/Nzbget/NzbgetClient.cs

@@ -91,6 +91,11 @@ namespace NzbDrone.Core.Download.Clients.Nzbget
             }
         }
 
+        public IEnumerable<HistoryItem> GetHistory(int start = 0, int limit = 0)
+        {
+            return new HistoryItem[0];
+        }
+
         public virtual VersionModel GetVersion(string host = null, int port = 0, string username = null, string password = null)
         {
             //Get saved values if any of these are defaults

src/NzbDrone.Core/Download/Clients/PneumaticClient.cs

@@ -65,6 +65,11 @@ namespace NzbDrone.Core.Download.Clients
             return new QueueItem[0];
         }
 
+        public IEnumerable<HistoryItem> GetHistory(int start = 0, int limit = 0)
+        {
+            return new HistoryItem[0];
+        }
+
         public virtual bool IsInQueue(RemoteEpisode newEpisode)
         {
             return false;

src/NzbDrone.Core/Download/Clients/Sabnzbd/SabnzbdClient.cs

@@ -38,6 +38,15 @@ namespace NzbDrone.Core.Download.Clients.Sabnzbd
             _logger = logger;
         }
 
+        public bool IsConfigured
+        {
+            get
+            {
+                return !string.IsNullOrWhiteSpace(_configService.SabHost)
+                    && _configService.SabPort != 0;
+            }
+        }
+
         public string DownloadNzb(RemoteEpisode remoteEpisode)
         {
             var url = remoteEpisode.Release.DownloadUrl;
@@ -56,15 +65,6 @@ namespace NzbDrone.Core.Download.Clients.Sabnzbd
             }
         }
 
-        public bool IsConfigured
-        {
-            get
-            {
-                return !string.IsNullOrWhiteSpace(_configService.SabHost)
-                    && _configService.SabPort != 0;
-            }
-        }
-
         public IEnumerable<QueueItem> GetQueue()
         {
             return _queueCache.Get("queue", () =>
@@ -104,7 +104,7 @@ namespace NzbDrone.Core.Download.Clients.Sabnzbd
             }, TimeSpan.FromSeconds(10));
         }
 
-        public virtual List<SabHistoryItem> GetHistory(int start = 0, int limit = 0)
+        public IEnumerable<HistoryItem> GetHistory(int start = 0, int limit = 0)
         {
             string action = String.Format("mode=history&output=json&start={0}&limit={1}", start, limit);
             string request = GetSabRequest(action);
@@ -113,7 +113,23 @@ namespace NzbDrone.Core.Download.Clients.Sabnzbd
             CheckForError(response);
 
             var items = Json.Deserialize<SabHistory>(JObject.Parse(response).SelectToken("history").ToString()).Items;
-            return items ?? new List<SabHistoryItem>();
+            var historyItems = new List<HistoryItem>();
+
+            foreach (var sabHistoryItem in items)
+            {
+                var historyItem = new HistoryItem();
+                historyItem.Id = sabHistoryItem.Id;
+                historyItem.Title = sabHistoryItem.Title;
+                historyItem.Size = sabHistoryItem.Size;
+                historyItem.DownloadTime = sabHistoryItem.DownloadTime;
+                historyItem.Storage = sabHistoryItem.Storage;
+                historyItem.Category = sabHistoryItem.Category;
+                historyItem.Status = sabHistoryItem.Status == "Failed" ? HistoryStatus.Failed : HistoryStatus.Completed;
+
+                historyItems.Add(historyItem);
+            }
+
+            return historyItems;
         }
 
         public virtual SabCategoryModel GetCategories(string host = null, int port = 0, string apiKey = null, string username = null, string password = null)

src/NzbDrone.Core/Download/DownloadFailedEvent.cs

@@ -0,0 +1,16 @@
+using System;
+using NzbDrone.Common.Messaging;
+using NzbDrone.Core.Tv;
+
+namespace NzbDrone.Core.Download
+{
+    public class DownloadFailedEvent : IEvent
+    {
+        public Series Series { get; set; }
+        public Episode Episode { get; set; }
+        public QualityModel Quality { get; set; }
+        public String SourceTitle { get; set; }
+        public String DownloadClient { get; set; }
+        public String DownloadClientId { get; set; }
+    }
+}

src/NzbDrone.Core/Download/FailedDownloadCommand.cs

@@ -0,0 +1,9 @@
+using NzbDrone.Core.Messaging.Commands;
+
+namespace NzbDrone.Core.Download
+{
+    public class FailedDownloadCommand : Command
+    {
+
+    }
+}

src/NzbDrone.Core/Download/FailedDownloadService.cs

@@ -0,0 +1,87 @@
+using System;
+using System.Linq;
+using NLog;
+using NzbDrone.Core.History;
+using NzbDrone.Core.Messaging.Commands;
+using NzbDrone.Core.Messaging.Events;
+
+namespace NzbDrone.Core.Download
+{
+    public class FailedDownloadService : IExecute<FailedDownloadCommand>
+    {
+        private readonly IProvideDownloadClient _downloadClientProvider;
+        private readonly IHistoryService _historyService;
+        private readonly IEventAggregator _eventAggregator;
+        private readonly Logger _logger;
+
+        private static string DOWNLOAD_CLIENT = "downloadClient";
+        private static string DOWNLOAD_CLIENT_ID = "downloadClientId";
+
+        public FailedDownloadService(IProvideDownloadClient downloadClientProvider,
+                                     IHistoryService historyService,
+                                     IEventAggregator eventAggregator,
+                                     Logger logger)
+        {
+            _downloadClientProvider = downloadClientProvider;
+            _historyService = historyService;
+            _eventAggregator = eventAggregator;
+            _logger = logger;
+        }
+
+        private void CheckForFailedDownloads()
+        {
+            var downloadClient = _downloadClientProvider.GetDownloadClient();
+            var downloadClientHistory = downloadClient.GetHistory(0, 20).ToList();
+
+            var failedItems = downloadClientHistory.Where(h => h.Status == HistoryStatus.Failed).ToList();
+
+            if (!failedItems.Any())
+            {
+                _logger.Trace("Yay! No failed downloads");
+                return;
+            }
+
+            var recentHistory = _historyService.BetweenDates(DateTime.UtcNow.AddDays(-1), DateTime.UtcNow, HistoryEventType.Grabbed);
+            var failedHistory = _historyService.Failed();
+
+            foreach (var failedItem in failedItems)
+            {
+                var failedLocal = failedItem;
+                var historyItems = recentHistory.Where(h => h.Data.ContainsKey(DOWNLOAD_CLIENT) &&
+                                                            h.Data[DOWNLOAD_CLIENT_ID].Equals(failedLocal.Id))
+                                                .ToList();
+
+                if (!historyItems.Any())
+                {
+                    _logger.Trace("Unable to find matching history item");
+                    continue;
+                }
+
+                if (failedHistory.Any(h => h.Data.ContainsKey(DOWNLOAD_CLIENT_ID) &&
+                                           h.Data[DOWNLOAD_CLIENT_ID].Equals(failedLocal.Id)))
+                {
+                    _logger.Trace("Already added to history as failed");
+                    continue;
+                }
+
+                foreach (var historyItem in historyItems)
+                {
+                    _eventAggregator.PublishEvent(new DownloadFailedEvent
+                    {
+                        Series = historyItem.Series,
+                        Episode = historyItem.Episode,
+                        Quality = historyItem.Quality,
+                        SourceTitle = historyItem.SourceTitle,
+                        DownloadClient = historyItem.Data[DOWNLOAD_CLIENT],
+                        DownloadClientId = historyItem.Data[DOWNLOAD_CLIENT_ID]
+                    });
+                }
+            }
+        }
+
+        public void Execute(FailedDownloadCommand message)
+        {
+            CheckForFailedDownloads();
+        }
+    }
+}

src/NzbDrone.Core/Download/HistoryItem.cs

@@ -0,0 +1,22 @@
+using System;
+using NzbDrone.Core.Parser.Model;
+
+namespace NzbDrone.Core.Download
+{
+    public class HistoryItem
+    {
+        public String Id { get; set; }
+        public String Title { get; set; }
+        public String Size { get; set; }
+        public String Category { get; set; }
+        public Int32 DownloadTime { get; set; }
+        public String Storage { get; set; }
+        public HistoryStatus Status { get; set; }
+    }
+
+    public enum HistoryStatus
+    {
+        Completed = 0,
+        Failed = 1
+    }
+}

src/NzbDrone.Core/Download/IDownloadClient.cs

@@ -8,5 +8,6 @@ namespace NzbDrone.Core.Download
         string DownloadNzb(RemoteEpisode remoteEpisode);
         bool IsConfigured { get; }
         IEnumerable<QueueItem> GetQueue();
+        IEnumerable<HistoryItem> GetHistory(int start = 0, int limit = 0);
     }
 }

src/NzbDrone.Core/History/History.cs

@@ -17,12 +17,9 @@ namespace NzbDrone.Core.History
         public string SourceTitle { get; set; }
         public QualityModel Quality { get; set; }
         public DateTime Date { get; set; }
-
         public Episode Episode { get; set; }
         public Series Series { get; set; }
-
         public HistoryEventType EventType { get; set; }
-
         public Dictionary<string, string> Data { get; set; }
     }
 
@@ -32,7 +29,8 @@ namespace NzbDrone.Core.History
         Unknown = 0,
         Grabbed = 1,
         SeriesFolderImported = 2,
-        DownloadFolderImported = 3
+        DownloadFolderImported = 3,
+        DownloadFailed = 4
     }
 
 }

src/NzbDrone.Core/History/HistoryRepository.cs

@@ -13,6 +13,8 @@ namespace NzbDrone.Core.History
     {
         void Trim();
         List<QualityModel> GetBestQualityInHistory(int episodeId);
+        List<History> BetweenDates(DateTime startDate, DateTime endDate, HistoryEventType eventType);
+        List<History> Failed();
     }
 
     public class HistoryRepository : BasicRepository<History>, IHistoryRepository
@@ -38,6 +40,20 @@ namespace NzbDrone.Core.History
             return history.Select(h => h.Quality).ToList();
         }
 
+        public List<History> BetweenDates(DateTime startDate, DateTime endDate, HistoryEventType eventType)
+        {
+            return Query.Join<History, Series>(JoinType.Inner, h => h.Series, (h, s) => h.SeriesId == s.Id)
+                        .Join<History, Episode>(JoinType.Inner, h => h.Episode, (h, e) => h.EpisodeId == e.Id)
+                        .Where(h => h.Date >= startDate)
+                        .AndWhere(h => h.Date <= endDate)
+                        .AndWhere(h => h.EventType == eventType);
+        }
+
+        public List<History> Failed()
+        {
+            return Query.Where(h => h.EventType == HistoryEventType.DownloadFailed);
+        }
+
         public override PagingSpec<History> GetPaged(PagingSpec<History> pagingSpec)
         {
             pagingSpec.Records = GetPagedQuery(pagingSpec).ToList();

src/NzbDrone.Core/History/HistoryService.cs

@@ -18,9 +18,11 @@ namespace NzbDrone.Core.History
         void Trim();
         QualityModel GetBestQualityInHistory(int episodeId);
         PagingSpec<History> Paged(PagingSpec<History> pagingSpec);
+        List<History> BetweenDates(DateTime startDate, DateTime endDate, HistoryEventType eventType);
+        List<History> Failed();
     }
 
-    public class HistoryService : IHistoryService, IHandle<EpisodeGrabbedEvent>, IHandle<EpisodeImportedEvent>
+    public class HistoryService : IHistoryService, IHandle<EpisodeGrabbedEvent>, IHandle<EpisodeImportedEvent>, IHandle<DownloadFailedEvent>
     {
         private readonly IHistoryRepository _historyRepository;
         private readonly Logger _logger;
@@ -41,6 +43,16 @@ namespace NzbDrone.Core.History
             return _historyRepository.GetPaged(pagingSpec);
         }
 
+        public List<History> BetweenDates(DateTime startDate, DateTime endDate, HistoryEventType eventType)
+        {
+            return _historyRepository.BetweenDates(startDate, endDate, eventType);
+        }
+
+        public List<History> Failed()
+        {
+            return _historyRepository.Failed();
+        }
+
         public void Purge()
         {
             _historyRepository.Purge();
@@ -51,7 +63,7 @@ namespace NzbDrone.Core.History
             _historyRepository.Trim();
         }
 
-        public virtual QualityModel GetBestQualityInHistory(int episodeId)
+        public QualityModel GetBestQualityInHistory(int episodeId)
         {
             return _historyRepository.GetBestQualityInHistory(episodeId).OrderByDescending(q => q).FirstOrDefault();
         }
@@ -107,5 +119,23 @@ namespace NzbDrone.Core.History
                 _historyRepository.Insert(history);
             }
         }
+
+        public void Handle(DownloadFailedEvent message)
+        {
+            var history = new History
+            {
+                EventType = HistoryEventType.DownloadFailed,
+                Date = DateTime.UtcNow,
+                Quality = message.Quality,
+                SourceTitle = message.SourceTitle,
+                SeriesId = message.Series.Id,
+                EpisodeId = message.Episode.Id,
+            };
+
+            history.Data.Add("DownloadClient", message.DownloadClient);
+            history.Data.Add("DownloadClientId", message.DownloadClientId);
+
+            _historyRepository.Insert(history);
+        }
     }
 }

src/NzbDrone.Core/Jobs/TaskManager.cs

@@ -7,6 +7,7 @@ using NzbDrone.Core.Configuration.Events;
 using NzbDrone.Core.DataAugmentation;
 using NzbDrone.Core.DataAugmentation.Scene;
 using NzbDrone.Core.DataAugmentation.Xem;
+using NzbDrone.Core.Download;
 using NzbDrone.Core.Housekeeping;
 using NzbDrone.Core.Indexers;
 using NzbDrone.Core.Instrumentation.Commands;
@@ -54,7 +55,8 @@ namespace NzbDrone.Core.Jobs
                     new ScheduledTask{ Interval = 1*60, TypeName = typeof(TrimLogCommand).FullName},
                     new ScheduledTask{ Interval = 3*60, TypeName = typeof(UpdateSceneMappingCommand).FullName},
                     new ScheduledTask{ Interval = 1, TypeName = typeof(TrackedCommandCleanupCommand).FullName},
-                    new ScheduledTask{ Interval = 24*60, TypeName = typeof(HousekeepingCommand).FullName}
+                    new ScheduledTask{ Interval = 24*60, TypeName = typeof(HousekeepingCommand).FullName},
+                    new ScheduledTask{ Interval = 1, TypeName = typeof(FailedDownloadCommand).FullName}
                 };
 
             var currentTasks = _scheduledTaskRepository.All();

src/NzbDrone.Core/NzbDrone.Core.csproj

@@ -226,9 +226,13 @@
     <Compile Include="Download\Clients\Sabnzbd\JsonConverters\SabnzbdQueueTimeConverter.cs" />
     <Compile Include="Download\Clients\Sabnzbd\SabAutoConfigureService.cs" />
     <Compile Include="Download\Clients\Sabnzbd\SabCommunicationProxy.cs" />
+    <Compile Include="Download\FailedDownloadCommand.cs" />
+    <Compile Include="Download\HistoryItem.cs" />
+    <Compile Include="Download\DownloadFailedEvent.cs" />
     <Compile Include="Download\DownloadApprovedReports.cs" />
     <Compile Include="Download\DownloadClientProvider.cs" />
     <Compile Include="Download\DownloadClientType.cs" />
+    <Compile Include="Download\FailedDownloadService.cs" />
     <Compile Include="Download\QueueItem.cs" />
     <Compile Include="Exceptions\BadRequestException.cs" />
     <Compile Include="Exceptions\DownstreamException.cs" />

src/UI/Cells/EventTypeCell.js

@@ -29,6 +29,10 @@ define(
                             icon = 'icon-nd-imported';
                             toolTip = 'Episode downloaded successfully and picked up from download client';
                             break;
+                        case 'downloadFailed':
+                            icon = 'icon-nd-download-failed';
+                            toolTip = 'Episode download failed';
+                            break;
                         default :
                             icon = 'icon-question';
                             toolTip = 'unknown event';

src/UI/Content/icons.less

@@ -157,3 +157,8 @@
   .icon(@remove-sign);
   color : purple;
 }
+
+.icon-nd-download-failed:before {
+  .icon(@cloud-download);
+  color: @errorText;
+}