Store releases when download client is unavailable

New: Retry releases when download client was unavailable
Closes #949

src/NzbDrone.Core.Test/Download/DownloadApprovedReportsTests/DownloadApprovedFixture.cs

@@ -209,7 +209,7 @@ public void should_not_add_to_pending_if_episode_was_grabbed()
             decisions.Add(new DownloadDecision(remoteEpisode, new Rejection("Failure!", RejectionType.Temporary)));
 
             Subject.ProcessDecisions(decisions);
-            Mocker.GetMock<IPendingReleaseService>().Verify(v => v.Add(It.IsAny<DownloadDecision>()), Times.Never());
+            Mocker.GetMock<IPendingReleaseService>().Verify(v => v.Add(It.IsAny<DownloadDecision>(), It.IsAny<PendingReleaseReason>()), Times.Never());
         }
 
         [Test]
@@ -223,7 +223,7 @@ public void should_add_to_pending_even_if_already_added_to_pending()
             decisions.Add(new DownloadDecision(remoteEpisode, new Rejection("Failure!", RejectionType.Temporary)));
 
             Subject.ProcessDecisions(decisions);
-            Mocker.GetMock<IPendingReleaseService>().Verify(v => v.Add(It.IsAny<DownloadDecision>()), Times.Exactly(2));
+            Mocker.GetMock<IPendingReleaseService>().Verify(v => v.Add(It.IsAny<DownloadDecision>(), It.IsAny<PendingReleaseReason>()), Times.Exactly(2));
         }
     }
 }

src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/AddFixture.cs

@@ -102,7 +102,7 @@ private void GivenHeldRelease(string title, string indexer, DateTime publishDate
         [Test]
         public void should_add()
         {
-            Subject.Add(_temporarilyRejected);
+            Subject.Add(_temporarilyRejected, PendingReleaseReason.Delay);
 
             VerifyInsert();
         }
@@ -112,7 +112,7 @@ public void should_not_add_if_it_is_the_same_release_from_the_same_indexer()
         {
             GivenHeldRelease(_release.Title, _release.Indexer, _release.PublishDate);
 
-            Subject.Add(_temporarilyRejected);
+            Subject.Add(_temporarilyRejected, PendingReleaseReason.Delay);
 
             VerifyNoInsert();
         }
@@ -122,7 +122,7 @@ public void should_add_if_title_is_different()
         {
             GivenHeldRelease(_release.Title + "-RP", _release.Indexer, _release.PublishDate);
 
-            Subject.Add(_temporarilyRejected);
+            Subject.Add(_temporarilyRejected, PendingReleaseReason.Delay);
 
             VerifyInsert();
         }
@@ -132,7 +132,7 @@ public void should_add_if_indexer_is_different()
         {
             GivenHeldRelease(_release.Title, "AnotherIndexer", _release.PublishDate);
 
-            Subject.Add(_temporarilyRejected);
+            Subject.Add(_temporarilyRejected, PendingReleaseReason.Delay);
 
             VerifyInsert();
         }
@@ -142,7 +142,7 @@ public void should_add_if_publish_date_is_different()
         {
             GivenHeldRelease(_release.Title, _release.Indexer, _release.PublishDate.AddHours(1));
 
-            Subject.Add(_temporarilyRejected);
+            Subject.Add(_temporarilyRejected, PendingReleaseReason.Delay);
 
             VerifyInsert();
         }

src/NzbDrone.Core.Test/Housekeeping/Housekeepers/CleanupDownloadClientUnavailablePendingReleasesFixture.cs

@@ -0,0 +1,60 @@
+using System;
+using FizzWare.NBuilder;
+using FluentAssertions;
+using NUnit.Framework;
+using NzbDrone.Core.Download.Pending;
+using NzbDrone.Core.Housekeeping.Housekeepers;
+using NzbDrone.Core.Parser.Model;
+using NzbDrone.Core.Test.Framework;
+
+namespace NzbDrone.Core.Test.Housekeeping.Housekeepers
+{
+    [TestFixture]
+    public class CleanupDownloadClientUnavailablePendingReleasesFixture : DbTest<CleanupDownloadClientUnavailablePendingReleases, PendingRelease>
+    {
+        [Test]
+        public void should_delete_old_DownloadClientUnavailable_pending_items()
+        {
+            var pendingRelease = Builder<PendingRelease>.CreateNew()
+                .With(h => h.Reason = PendingReleaseReason.DownloadClientUnavailable)
+                .With(h => h.Added = DateTime.UtcNow.AddDays(-21))
+                .With(h => h.ParsedEpisodeInfo = new ParsedEpisodeInfo())
+                .With(h => h.Release = new ReleaseInfo())
+                .BuildNew();
+
+            Db.Insert(pendingRelease);
+            Subject.Clean();
+            AllStoredModels.Should().BeEmpty();
+        }
+
+        [Test]
+        public void should_delete_old_Fallback_pending_items()
+        {
+            var pendingRelease = Builder<PendingRelease>.CreateNew()
+                .With(h => h.Reason = PendingReleaseReason.Fallback)
+                .With(h => h.Added = DateTime.UtcNow.AddDays(-21))
+                .With(h => h.ParsedEpisodeInfo = new ParsedEpisodeInfo())
+                .With(h => h.Release = new ReleaseInfo())
+                .BuildNew();
+
+            Db.Insert(pendingRelease);
+            Subject.Clean();
+            AllStoredModels.Should().BeEmpty();
+        }
+
+        [Test]
+        public void should_not_delete_old_Delay_pending_items()
+        {
+            var pendingRelease = Builder<PendingRelease>.CreateNew()
+                .With(h => h.Reason = PendingReleaseReason.Delay)
+                .With(h => h.Added = DateTime.UtcNow.AddDays(-21))
+                .With(h => h.ParsedEpisodeInfo = new ParsedEpisodeInfo())
+                .With(h => h.Release = new ReleaseInfo())
+                .BuildNew();
+
+            Db.Insert(pendingRelease);
+            Subject.Clean();
+            AllStoredModels.Should().HaveCount(1);
+        }
+    }
+}

src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj

@@ -237,6 +237,7 @@
     <Compile Include="Housekeeping\Housekeepers\CleanupOrphanedIndexerStatusFixture.cs" />
     <Compile Include="Housekeeping\Housekeepers\CleanupOrphanedHistoryItemsFixture.cs" />
     <Compile Include="Housekeeping\Housekeepers\CleanupOrphanedMetadataFilesFixture.cs" />
+    <Compile Include="Housekeeping\Housekeepers\CleanupDownloadClientUnavailablePendingReleasesFixture.cs" />
     <Compile Include="Housekeeping\Housekeepers\CleanupUnusedTagsFixture.cs" />
     <Compile Include="Housekeeping\Housekeepers\CleanupOrphanedPendingReleasesFixture.cs" />
     <Compile Include="Housekeeping\Housekeepers\FixFutureRunScheduledTasksFixture.cs" />

src/NzbDrone.Core/Datastore/Migration/097_add_release_to_pending_releases.cs

@@ -0,0 +1,14 @@
+using FluentMigrator;
+using NzbDrone.Core.Datastore.Migration.Framework;
+
+namespace NzbDrone.Core.Datastore.Migration
+{
+    [Migration(97)]
+    public class add_reason_to_pending_releases : NzbDroneMigrationBase
+    {
+        protected override void MainDbUpgrade()
+        {
+            Alter.Table("PendingReleases").AddColumn("Reason").AsInt32().WithDefaultValue(0);
+        }
+    }
+}

src/NzbDrone.Core/Download/Clients/Deluge/DelugeProxy.cs

@@ -231,7 +231,7 @@ private JsonRpcResponse<TResult> ExecuteRequest<TResult>(JsonRpcRequestBuilder r
             }
             catch (WebException ex)
             {
-                throw new DownloadClientException("Unable to connect to Deluge, please check your settings", ex);
+                throw new DownloadClientUnavailableException("Unable to connect to Deluge, please check your settings", ex);
             }
         }
 

src/NzbDrone.Core/Download/Clients/DownloadClientException.cs

@@ -8,19 +8,16 @@ public class DownloadClientException : NzbDroneException
         public DownloadClientException(string message, params object[] args)
             : base(string.Format(message, args))
         {
-
         }
 
         public DownloadClientException(string message)
             : base(message)
         {
-
         }
 
         public DownloadClientException(string message, Exception innerException, params object[] args)
             : base(string.Format(message, args), innerException)
         {
-
         }
 
         public DownloadClientException(string message, Exception innerException)

src/NzbDrone.Core/Download/Clients/DownloadClientUnavailableException.cs

@@ -0,0 +1,27 @@
+using System;
+
+namespace NzbDrone.Core.Download.Clients
+{
+    public class DownloadClientUnavailableException : DownloadClientException
+    {
+        public DownloadClientUnavailableException(string message, params object[] args)
+            : base(string.Format(message, args))
+        {
+        }
+
+        public DownloadClientUnavailableException(string message)
+            : base(message)
+        {
+        }
+
+        public DownloadClientUnavailableException(string message, Exception innerException, params object[] args)
+            : base(string.Format(message, args), innerException)
+        {
+        }
+
+        public DownloadClientUnavailableException(string message, Exception innerException)
+            : base(message, innerException)
+        {
+        }
+    }
+}

src/NzbDrone.Core/Download/Clients/DownloadStation/Proxies/DiskStationProxyBase.cs

@@ -72,7 +72,20 @@ private DiskStationResponse<T> ProcessRequest<T>(HttpRequestBuilder requestBuild
                                                          DownloadStationSettings settings) where T : new()
         {
             var request = requestBuilder.Build();
-            var response = _httpClient.Execute(request);
+            HttpResponse response;
+
+            try
+            {
+                response = _httpClient.Execute(request);
+            }
+            catch (HttpException ex)
+            {
+                throw new DownloadClientException("Unable to connect to Diskstation, please check your settings", ex);
+            }
+            catch (WebException ex)
+            {
+                throw new DownloadClientUnavailableException("Unable to connect to Diskstation, please check your settings", ex);
+            }
 
             _logger.Debug("Trying to {0}", operation);
 

src/NzbDrone.Core/Download/Clients/Hadouken/HadoukenProxy.cs

@@ -77,7 +77,21 @@ private T ProcessRequest<T>(HadoukenSettings settings, string method, params obj
             requestBuilder.Headers.Add("Accept-Encoding", "gzip,deflate");
 
             var httpRequest = requestBuilder.Build();
-            var response = _httpClient.Execute(httpRequest);
+            HttpResponse response;
+
+            try
+            {
+                response = _httpClient.Execute(httpRequest);
+            }
+            catch (HttpException ex)
+            {
+                throw new DownloadClientException("Unable to connect to Hadouken, please check your settings", ex);
+            }
+            catch (WebException ex)
+            {
+                throw new DownloadClientUnavailableException("Unable to connect to Hadouken, please check your settings", ex);
+            }
+
             var result = Json.Deserialize<JsonRpcResponse<T>>(response.Content);
 
             if (result.Error != null)

src/NzbDrone.Core/Download/Clients/NzbVortex/NzbVortexProxy.cs

@@ -164,7 +164,7 @@ private T ProcessRequest<T>(HttpRequestBuilder requestBuilder, bool requiresAuth
             }
             catch (WebException ex)
             {
-                throw new DownloadClientException("Unable to connect to NZBVortex, please check your settings", ex);
+                throw new DownloadClientUnavailableException("Unable to connect to NZBVortex, please check your settings", ex);
             }
         }
 

src/NzbDrone.Core/Download/Clients/Nzbget/NzbgetProxy.cs

@@ -235,14 +235,14 @@ private T ProcessRequest<T>(NzbgetSettings settings, string method, params objec
             {
                 if (ex.Response.StatusCode == HttpStatusCode.Unauthorized)
                 {
-                    throw new DownloadClientException("Authentication failed for NzbGet, please check your settings", ex);
+                    throw new DownloadClientAuthenticationException("Authentication failed for NzbGet, please check your settings", ex);
                 }
 
                 throw new DownloadClientException("Unable to connect to NzbGet. " + ex.Message, ex);
             }
             catch (WebException ex)
             {
-                throw new DownloadClientException("Unable to connect to NzbGet. " + ex.Message, ex);
+                throw new DownloadClientUnavailableException("Unable to connect to NzbGet. " + ex.Message, ex);
             }
 
             var result = Json.Deserialize<JsonRpcResponse<T>>(response.Content);

src/NzbDrone.Core/Download/Clients/QBittorrent/QBittorrentProxy.cs

@@ -225,7 +225,7 @@ private void AuthenticateClient(HttpRequestBuilder requestBuilder, QBittorrentSe
                 }
                 catch (WebException ex)
                 {
-                    throw new DownloadClientException("Failed to connect to qBitTorrent, please check your settings.", ex);
+                    throw new DownloadClientUnavailableException("Failed to connect to qBitTorrent, please check your settings.", ex);
                 }
 
                 if (response.Content != "Ok.") // returns "Fails." on bad login

src/NzbDrone.Core/Download/Clients/Sabnzbd/SabnzbdProxy.cs

@@ -183,7 +183,7 @@ private string ProcessRequest(HttpRequestBuilder requestBuilder, SabnzbdSettings
             }
             catch (WebException ex)
             {
-                throw new DownloadClientException("Unable to connect to SABnzbd, please check your settings", ex);
+                throw new DownloadClientUnavailableException("Unable to connect to SABnzbd, please check your settings", ex);
             }
 
             CheckForError(response);

src/NzbDrone.Core/Download/Clients/Transmission/TransmissionBase.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Net;
@@ -211,17 +211,14 @@ protected ValidationFailure TestConnection()
                     DetailedDescription = string.Format("Please verify your username and password. Also verify if the host running Sonarr isn't blocked from accessing {0} by WhiteList limitations in the {0} configuration.", Name)
                 };
             }
-            catch (WebException ex)
+            catch (DownloadClientUnavailableException ex)
             {
                 _logger.Error(ex);
-                if (ex.Status == WebExceptionStatus.ConnectFailure)
+
+                return new NzbDroneValidationFailure("Host", "Unable to connect")
                 {
-                    return new NzbDroneValidationFailure("Host", "Unable to connect")
-                    {
-                        DetailedDescription = "Please verify the hostname and port."
-                    };
-                }
-                return new NzbDroneValidationFailure(string.Empty, "Unknown exception: " + ex.Message);
+                    DetailedDescription = "Please verify the hostname and port."
+                };
             }
             catch (Exception ex)
             {

src/NzbDrone.Core/Download/Clients/Transmission/TransmissionProxy.cs

@@ -238,54 +238,66 @@ private void AuthenticateClient(HttpRequestBuilder requestBuilder, TransmissionS
         
         public TransmissionResponse ProcessRequest(string action, object arguments, TransmissionSettings settings)
         {
-            var requestBuilder = BuildRequest(settings);
-            requestBuilder.Headers.ContentType = "application/json";
-            requestBuilder.SuppressHttpError = true;
-
-            AuthenticateClient(requestBuilder, settings);
-
-            var request = requestBuilder.Post().Build();
-
-            var data = new Dictionary<string, object>();
-            data.Add("method", action);
-
-            if (arguments != null)
+            try
             {
-                data.Add("arguments", arguments);
-            }
+                var requestBuilder = BuildRequest(settings);
+                requestBuilder.Headers.ContentType = "application/json";
+                requestBuilder.SuppressHttpError = true;
 
-            request.SetContent(data.ToJson());
-            request.ContentSummary = string.Format("{0}(...)", action);
+                AuthenticateClient(requestBuilder, settings);
 
-            var response = _httpClient.Execute(request);            
-            if (response.StatusCode == HttpStatusCode.Conflict)
-            {
-                AuthenticateClient(requestBuilder, settings, true);
+                var request = requestBuilder.Post().Build();
 
-                request = requestBuilder.Post().Build();
+                var data = new Dictionary<string, object>();
+                data.Add("method", action);
+
+                if (arguments != null)
+                {
+                    data.Add("arguments", arguments);
+                }
 
                 request.SetContent(data.ToJson());
                 request.ContentSummary = string.Format("{0}(...)", action);
 
-                response = _httpClient.Execute(request);
-            }
-            else if (response.StatusCode == HttpStatusCode.Unauthorized)
-            {
-                throw new DownloadClientAuthenticationException("User authentication failed.");
-            }
+                var response = _httpClient.Execute(request);
 
-            var transmissionResponse = Json.Deserialize<TransmissionResponse>(response.Content);
+                if (response.StatusCode == HttpStatusCode.Conflict)
+                {
+                    AuthenticateClient(requestBuilder, settings, true);
 
-            if (transmissionResponse == null)
-            {
-                throw new TransmissionException("Unexpected response");
-            }
-            else if (transmissionResponse.Result != "success")
-            {
-                throw new TransmissionException(transmissionResponse.Result);
-            }
+                    request = requestBuilder.Post().Build();
 
-            return transmissionResponse;
+                    request.SetContent(data.ToJson());
+                    request.ContentSummary = string.Format("{0}(...)", action);
+
+                    response = _httpClient.Execute(request);
+                }
+                else if (response.StatusCode == HttpStatusCode.Unauthorized)
+                {
+                    throw new DownloadClientAuthenticationException("User authentication failed.");
+                }
+
+                var transmissionResponse = Json.Deserialize<TransmissionResponse>(response.Content);
+
+                if (transmissionResponse == null)
+                {
+                    throw new TransmissionException("Unexpected response");
+                }
+                else if (transmissionResponse.Result != "success")
+                {
+                    throw new TransmissionException(transmissionResponse.Result);
+                }
+
+                return transmissionResponse;
+            }
+            catch (HttpException ex)
+            {
+                throw new DownloadClientException("Unable to connect to Transmission, please check your settings", ex);
+            }
+            catch (WebException ex)
+            {
+                throw new DownloadClientUnavailableException("Unable to connect to Transmission, please check your settings", ex);
+            }
         }
     }
 }

src/NzbDrone.Core/Download/Clients/rTorrent/RTorrentProxy.cs

@@ -2,6 +2,8 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Net;
+using System.Net.Sockets;
+using System.Runtime.InteropServices.ComTypes;
 using NLog;
 using NzbDrone.Common.Extensions;
 using CookComputing.XmlRpc;
@@ -54,8 +56,7 @@ public string GetVersion(RTorrentSettings settings)
             _logger.Debug("Executing remote method: system.client_version");
 
             var client = BuildClient(settings);
-
-            var version = client.GetVersion();
+            var version = ExecuteRequest(() => client.GetVersion());
 
             return version;
         }
@@ -65,20 +66,22 @@ public List<RTorrentTorrent> GetTorrents(RTorrentSettings settings)
             _logger.Debug("Executing remote method: d.multicall2");
 
             var client = BuildClient(settings);
-            var ret = client.TorrentMulticall("", "",
-                "d.name=", // string
-                "d.hash=", // string
-                "d.base_path=", // string
-                "d.custom1=", // string (label)
-                "d.size_bytes=", // long
-                "d.left_bytes=", // long
-                "d.down.rate=", // long (in bytes / s)
-                "d.ratio=", // long
-                "d.is_open=", // long
-                "d.is_active=", // long
-                "d.complete="); //long
+            var ret = ExecuteRequest(() => client.TorrentMulticall("", "",
+                    "d.name=", // string
+                    "d.hash=", // string
+                    "d.base_path=", // string
+                    "d.custom1=", // string (label)
+                    "d.size_bytes=", // long
+                    "d.left_bytes=", // long
+                    "d.down.rate=", // long (in bytes / s)
+                    "d.ratio=", // long
+                    "d.is_open=", // long
+                    "d.is_active=", // long
+                    "d.complete=") //long
+            );
 
             var items = new List<RTorrentTorrent>();
+
             foreach (object[] torrent in ret)
             {
                 var labelDecoded = System.Web.HttpUtility.UrlDecode((string) torrent[3]);
@@ -107,8 +110,8 @@ public void AddTorrentFromUrl(string torrentUrl, string label, RTorrentPriority
             _logger.Debug("Executing remote method: load.normal");
 
             var client = BuildClient(settings);
+            var response = ExecuteRequest(() => client.LoadStart("", torrentUrl, GetCommands(label, priority, directory)));
 
-            var response = client.LoadStart("", torrentUrl, GetCommands(label, priority, directory));
             if (response != 0)
             {
                 throw new DownloadClientException("Could not add torrent: {0}.", torrentUrl);
@@ -120,8 +123,8 @@ public void AddTorrentFromFile(string fileName, byte[] fileContent, string label
             _logger.Debug("Executing remote method: load.raw");
 
             var client = BuildClient(settings);
+            var response = ExecuteRequest(() => client.LoadRawStart("", fileContent, GetCommands(label, priority, directory)));
 
-            var response = client.LoadRawStart("", fileContent, GetCommands(label, priority, directory));
             if (response != 0)
             {
                 throw new DownloadClientException("Could not add torrent: {0}.", fileName);
@@ -133,14 +136,39 @@ public void RemoveTorrent(string hash, RTorrentSettings settings)
             _logger.Debug("Executing remote method: d.erase");
 
             var client = BuildClient(settings);
+            var response = ExecuteRequest(() => client.Remove(hash));
 
-            var response = client.Remove(hash);
             if (response != 0)
             {
                 throw new DownloadClientException("Could not remove torrent: {0}.", hash);
             }
         }
 
+        public bool HasHashTorrent(string hash, RTorrentSettings settings)
+        {
+            _logger.Debug("Executing remote method: d.name");
+
+            var client = BuildClient(settings);
+
+            try
+            {
+                var name = ExecuteRequest(() => client.GetName(hash));
+
+                if (name.IsNullOrWhiteSpace())
+                {
+                    return false;
+                }
+
+                var metaTorrent = name == (hash + ".meta");
+
+                return !metaTorrent;
+            }
+            catch (Exception)
+            {
+                return false;
+            }
+        }
+
         private string[] GetCommands(string label, RTorrentPriority priority, string directory)
         {
             var result = new List<string>();
@@ -163,25 +191,6 @@ private string[] GetCommands(string label, RTorrentPriority priority, string dir
             return result.ToArray();
         }
 
-        public bool HasHashTorrent(string hash, RTorrentSettings settings)
-        {
-            _logger.Debug("Executing remote method: d.name");
-
-            var client = BuildClient(settings);
-
-            try
-            {
-                var name = client.GetName(hash);
-                if (name.IsNullOrWhiteSpace()) return false;
-                bool metaTorrent = name == (hash + ".meta");
-                return !metaTorrent;
-            }
-            catch (Exception)
-            {
-                return false;
-            }
-        }
-
         private IRTorrent BuildClient(RTorrentSettings settings)
         {
             var client = XmlRpcProxyGen.Create<IRTorrent>();
@@ -201,5 +210,21 @@ private IRTorrent BuildClient(RTorrentSettings settings)
 
             return client;
         }
+
+        private T ExecuteRequest<T>(Func<T> task)
+        {
+            try
+            {
+                return task();
+            }
+            catch (XmlRpcServerException ex)
+            {
+                throw new DownloadClientException("Unable to connect to rTorrent, please check your settings", ex);
+            }
+            catch (WebException ex)
+            {
+                throw new DownloadClientUnavailableException("Unable to connect to rTorrent, please check your settings", ex);
+            }
+        }
     }
 }

src/NzbDrone.Core/Download/Clients/uTorrent/UTorrentProxy.cs

@@ -244,7 +244,7 @@ private void AuthenticateClient(HttpRequestBuilder requestBuilder, UTorrentSetti
                 }
                 catch (WebException ex)
                 {
-                    throw new DownloadClientException("Unable to connect to uTorrent, please check your settings", ex);
+                    throw new DownloadClientUnavailableException("Unable to connect to uTorrent, please check your settings", ex);
                 }
 
                 cookies = response.GetCookies();

src/NzbDrone.Core/Download/Pending/PendingRelease.cs

@@ -11,6 +11,7 @@ public class PendingRelease : ModelBase
         public DateTime Added { get; set; }
         public ParsedEpisodeInfo ParsedEpisodeInfo { get; set; }
         public ReleaseInfo Release { get; set; }
+        public PendingReleaseReason Reason { get; set; }
 
         //Not persisted
         public RemoteEpisode RemoteEpisode { get; set; }

src/NzbDrone.Core/Download/Pending/PendingReleaseReason.cs

@@ -0,0 +1,9 @@
+namespace NzbDrone.Core.Download.Pending
+{
+    public enum PendingReleaseReason
+    {
+        Delay = 0,
+        DownloadClientUnavailable = 1,
+        Fallback = 2
+    }
+}

src/NzbDrone.Core/Download/Pending/PendingReleaseService.cs

@@ -20,8 +20,7 @@ namespace NzbDrone.Core.Download.Pending
 {
     public interface IPendingReleaseService
     {
-        void Add(DownloadDecision decision);
-
+        void Add(DownloadDecision decision, PendingReleaseReason reason);
         List<ReleaseInfo> GetPending();
         List<RemoteEpisode> GetPendingRemoteEpisodes(int seriesId);
         List<Queue.Queue> GetPendingQueue();
@@ -67,7 +66,7 @@ public PendingReleaseService(IIndexerStatusService indexerStatusService,
         }
 
 
-        public void Add(DownloadDecision decision)
+        public void Add(DownloadDecision decision, PendingReleaseReason reason)
         {
             var alreadyPending = GetPendingReleases();
 
@@ -77,14 +76,32 @@ public void Add(DownloadDecision decision)
                                                              .Intersect(episodeIds)
                                                              .Any());
 
-            if (existingReports.Any(MatchingReleasePredicate(decision.RemoteEpisode.Release)))
+            var matchingReports = existingReports.Where(MatchingReleasePredicate(decision.RemoteEpisode.Release)).ToList();
+
+            if (matchingReports.Any())
             {
-                _logger.Debug("This release is already pending, not adding again");
-                return;
+                var sameReason = true;
+
+                foreach (var matchingReport in matchingReports)
+                {
+                    if (matchingReport.Reason != reason)
+                    {
+                        _logger.Debug("This release is already pending with reason {0}, changing to {1}", matchingReport.Reason, reason);
+                        matchingReport.Reason = reason;
+                        _repository.Update(matchingReport);
+                        sameReason = false;
+                    }
+                }
+
+                if (sameReason)
+                {
+                    _logger.Debug("This release is already pending with reason {0}, not adding again", reason);
+                    return;
+                }
             }
 
-            _logger.Debug("Adding release to pending releases");
-            Insert(decision);
+            _logger.Debug("Adding release to pending releases with reason {0}", reason);
+            Insert(decision, reason);
         }
 
         public List<ReleaseInfo> GetPending()
@@ -117,7 +134,7 @@ public List<RemoteEpisode> GetPendingRemoteEpisodes(int seriesId)
 
             var nextRssSync = new Lazy<DateTime>(() => _taskManager.GetNextExecution(typeof(RssSyncCommand)));
 
-            foreach (var pendingRelease in GetPendingReleases())
+            foreach (var pendingRelease in GetPendingReleases().Where(p => p.Reason != PendingReleaseReason.Fallback))
             {
                 foreach (var episode in pendingRelease.RemoteEpisode.Episodes)
                 {
@@ -132,6 +149,13 @@ public List<RemoteEpisode> GetPendingRemoteEpisodes(int seriesId)
                         ect = ect.AddMinutes(_configService.RssSyncInterval);
                     }
 
+                    var timeleft = ect.Subtract(DateTime.UtcNow);
+
+                    if (timeleft.TotalSeconds < 0)
+                    {
+                        timeleft = TimeSpan.Zero;
+                    }
+
                     var queue = new Queue.Queue
                                 {
                                     Id = GetQueueId(pendingRelease, episode),
@@ -142,11 +166,12 @@ public List<RemoteEpisode> GetPendingRemoteEpisodes(int seriesId)
                                     Size = pendingRelease.RemoteEpisode.Release.Size,
                                     Sizeleft = pendingRelease.RemoteEpisode.Release.Size,
                                     RemoteEpisode = pendingRelease.RemoteEpisode,
-                                    Timeleft = ect.Subtract(DateTime.UtcNow),
+                                    Timeleft = timeleft,
                                     EstimatedCompletionTime = ect,
-                                    Status = "Pending",
+                                    Status = pendingRelease.Reason.ToString(),
                                     Protocol = pendingRelease.RemoteEpisode.Release.DownloadProtocol
                                 };
+
                     queued.Add(queue);
                 }
             }
@@ -224,7 +249,7 @@ private RemoteEpisode GetRemoteEpisode(PendingRelease release)
             };
         }
 
-        private void Insert(DownloadDecision decision)
+        private void Insert(DownloadDecision decision, PendingReleaseReason reason)
         {
             _repository.Insert(new PendingRelease
             {
@@ -232,7 +257,8 @@ private void Insert(DownloadDecision decision)
                 ParsedEpisodeInfo = decision.RemoteEpisode.ParsedEpisodeInfo,
                 Release = decision.RemoteEpisode.Release,
                 Title = decision.RemoteEpisode.Release.Title,
-                Added = DateTime.UtcNow
+                Added = DateTime.UtcNow,
+                Reason = reason
             });
 
             _eventAggregator.PublishEvent(new PendingReleasesUpdatedEvent());

src/NzbDrone.Core/Download/ProcessDownloadDecisions.cs

@@ -1,9 +1,12 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Net;
 using NLog;
 using NzbDrone.Core.DecisionEngine;
+using NzbDrone.Core.Download.Clients;
 using NzbDrone.Core.Download.Pending;
+using NzbDrone.Core.Indexers;
 
 namespace NzbDrone.Core.Download
 {
@@ -36,37 +39,33 @@ public ProcessedDecisions ProcessDecisions(List<DownloadDecision> decisions)
             var prioritizedDecisions = _prioritizeDownloadDecision.PrioritizeDecisions(qualifiedReports);
             var grabbed = new List<DownloadDecision>();
             var pending = new List<DownloadDecision>();
+            var failed = new List<DownloadDecision>();
+
+            var usenetFailed = false;
+            var torrentFailed = false;
 
             foreach (var report in prioritizedDecisions)
             {
                 var remoteEpisode = report.RemoteEpisode;
+                var downloadProtocol = report.RemoteEpisode.Release.DownloadProtocol;
 
-                var episodeIds = remoteEpisode.Episodes.Select(e => e.Id).ToList();
-
-                //Skip if already grabbed
-                if (grabbed.SelectMany(r => r.RemoteEpisode.Episodes)
-                                .Select(e => e.Id)
-                                .ToList()
-                                .Intersect(episodeIds)
-                                .Any())
+                // Skip if already grabbed
+                if (IsEpisodeProcessed(grabbed, report))
                 {
                     continue;
                 }
 
                 if (report.TemporarilyRejected)
                 {
-                    _pendingReleaseService.Add(report);
+                    _pendingReleaseService.Add(report, PendingReleaseReason.Delay);
                     pending.Add(report);
                     continue;
                 }
 
-                if (pending.SelectMany(r => r.RemoteEpisode.Episodes)
-                        .Select(e => e.Id)
-                        .ToList()
-                        .Intersect(episodeIds)
-                        .Any())
+                if (downloadProtocol == DownloadProtocol.Usenet && usenetFailed ||
+                    downloadProtocol == DownloadProtocol.Torrent && torrentFailed)
                 {
-                    continue;
+                    failed.Add(report);
                 }
 
                 try
@@ -74,14 +73,31 @@ public ProcessedDecisions ProcessDecisions(List<DownloadDecision> decisions)
                     _downloadService.DownloadReport(remoteEpisode);
                     grabbed.Add(report);
                 }
-                catch (Exception e)
+                catch (Exception ex)
                 {
-                    //TODO: support for store & forward
-                    //We'll need to differentiate between a download client error and an indexer error
-                    _logger.Warn(e, "Couldn't add report to download queue. " + remoteEpisode);
+                    if (ex is DownloadClientUnavailableException || ex is DownloadClientAuthenticationException)
+                    {
+                        _logger.Debug("Failed to send release to download client, storing until later");
+                        failed.Add(report);
+
+                        if (downloadProtocol == DownloadProtocol.Usenet)
+                        {
+                            usenetFailed = true;
+                        }
+                        else if (downloadProtocol == DownloadProtocol.Torrent)
+                        {
+                            torrentFailed = true;
+                        }
+                    }
+                    else
+                    {
+                        _logger.Warn(ex, "Couldn't add report to download queue. " + remoteEpisode);
+                    }
                 }
             }
 
+            pending.AddRange(ProcessFailedGrabs(grabbed, failed));
+
             return new ProcessedDecisions(grabbed, pending, decisions.Where(d => d.Rejected).ToList());
         }
 
@@ -90,5 +106,50 @@ internal List<DownloadDecision> GetQualifiedReports(IEnumerable<DownloadDecision
             //Process both approved and temporarily rejected
             return decisions.Where(c => (c.Approved || c.TemporarilyRejected) && c.RemoteEpisode.Episodes.Any()).ToList();
         }
+
+        private bool IsEpisodeProcessed(List<DownloadDecision> decisions, DownloadDecision report)
+        {
+            var episodeIds = report.RemoteEpisode.Episodes.Select(e => e.Id).ToList();
+
+            return decisions.SelectMany(r => r.RemoteEpisode.Episodes)
+                            .Select(e => e.Id)
+                            .ToList()
+                            .Intersect(episodeIds)
+                            .Any();
+        }
+
+        private List<DownloadDecision> ProcessFailedGrabs(List<DownloadDecision> grabbed, List<DownloadDecision> failed)
+        {
+            var pending = new List<DownloadDecision>();
+            var stored = new List<DownloadDecision>();
+
+            foreach (var report in failed)
+            {
+                // If a release was already grabbed with matching episodes we should store it as a fallback
+                // and filter it out the next time it is processed incase a higher quality release failed to
+                // add to the download client, but a lower quality release was sent to another client
+                // If the release wasn't grabbed already, but was already stored, store it as a fallback,
+                // otherwise store it as DownloadClientUnavailable.
+
+                if (IsEpisodeProcessed(grabbed, report))
+                {
+                    _pendingReleaseService.Add(report, PendingReleaseReason.Fallback);
+                    pending.Add(report);
+                }
+                else if (IsEpisodeProcessed(stored, report))
+                {
+                    _pendingReleaseService.Add(report, PendingReleaseReason.Fallback);
+                    pending.Add(report);
+                }
+                else
+                {
+                    _pendingReleaseService.Add(report, PendingReleaseReason.DownloadClientUnavailable);
+                    pending.Add(report);
+                    stored.Add(report);
+                }
+            }
+
+            return pending;
+        }
     }
 }

src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupDownloadClientUnavailablePendingReleases.cs

@@ -0,0 +1,32 @@
+using System;
+using NzbDrone.Core.Datastore;
+using NzbDrone.Core.Download.Pending;
+
+namespace NzbDrone.Core.Housekeeping.Housekeepers
+{
+    public class CleanupDownloadClientUnavailablePendingReleases : IHousekeepingTask
+    {
+        private readonly IMainDatabase _database;
+
+        public CleanupDownloadClientUnavailablePendingReleases(IMainDatabase database)
+        {
+            _database = database;
+        }
+
+        public void Clean()
+        {
+            var mapper = _database.GetDataMapper();
+            var twoWeeksAgo = DateTime.UtcNow.AddDays(-14);
+
+            mapper.Delete<PendingRelease>(p => p.Added < twoWeeksAgo &&
+                                               (p.Reason == PendingReleaseReason.DownloadClientUnavailable ||
+                                                p.Reason == PendingReleaseReason.Fallback));
+
+//            mapper.AddParameter("twoWeeksAgo", $"{DateTime.UtcNow.AddDays(-14).ToString("s")}Z");
+
+//            mapper.ExecuteNonQuery(@"DELETE FROM PendingReleases
+//                                     WHERE Added < @twoWeeksAgo
+//                                     AND (Reason = 'DownloadClientUnavailable' OR Reason = 'Fallback')");
+        }
+    }
+}

src/NzbDrone.Core/NzbDrone.Core.csproj

@@ -287,6 +287,7 @@
     <Compile Include="Datastore\Migration\092_add_unverifiedscenenumbering.cs" />
     <Compile Include="Datastore\Migration\100_add_scene_season_number.cs" />
     <Compile Include="Datastore\Migration\099_extra_and_subtitle_files.cs" />
+    <Compile Include="Datastore\Migration\097_add_release_to_pending_releases.cs" />
     <Compile Include="Datastore\Migration\094_add_tvmazeid.cs" />
     <Compile Include="Datastore\Migration\098_remove_titans_of_tv.cs">
       <SubType>Code</SubType>
@@ -365,6 +366,7 @@
     <Compile Include="Download\Clients\Deluge\DelugePriority.cs" />
     <Compile Include="Download\Clients\Deluge\DelugeUpdateUIResult.cs" />
     <Compile Include="Download\Clients\DownloadClientAuthenticationException.cs" />
+    <Compile Include="Download\Clients\DownloadClientUnavailableException.cs" />
     <Compile Include="Download\Clients\DownloadClientException.cs" />
     <Compile Include="Download\Clients\DownloadStation\Proxies\DownloadStationInfoProxy.cs" />
     <Compile Include="Download\Clients\DownloadStation\TorrentDownloadStation.cs" />
@@ -499,6 +501,7 @@
     <Compile Include="Download\DownloadEventHub.cs" />
     <Compile Include="Download\DownloadClientStatusRepository.cs" />
     <Compile Include="Download\DownloadClientStatusService.cs" />
+    <Compile Include="Download\Pending\PendingReleaseReason.cs" />
     <Compile Include="Download\TrackedDownloads\DownloadMonitoringService.cs" />
     <Compile Include="Download\TrackedDownloads\TrackedDownload.cs" />
     <Compile Include="Download\TrackedDownloads\TrackedDownloadService.cs" />
@@ -594,6 +597,7 @@
     <Compile Include="Housekeeping\Housekeepers\CleanupOrphanedIndexerStatus.cs" />
     <Compile Include="Housekeeping\Housekeepers\CleanupOrphanedHistoryItems.cs" />
     <Compile Include="Housekeeping\Housekeepers\CleanupOrphanedMetadataFiles.cs" />
+    <Compile Include="Housekeeping\Housekeepers\CleanupDownloadClientUnavailablePendingReleases.cs" />
     <Compile Include="Housekeeping\Housekeepers\CleanupUnusedTags.cs" />
     <Compile Include="Housekeeping\Housekeepers\CleanupOrphanedPendingReleases.cs" />
     <Compile Include="Housekeeping\Housekeepers\DeleteBadMediaCovers.cs" />

src/UI/Activity/Queue/QueueStatusCell.js

@@ -36,6 +36,11 @@ module.exports = NzbDroneCell.extend({
                 title = 'Pending';
             }
 
+            if (status === 'downloadclientunavailable') {
+                icon = 'icon-sonarr-client-unavailable';
+                title = 'Download pending, download client is unavailable';
+            }
+
             if (status === 'failed') {
                 icon = 'icon-sonarr-download-failed';
                 title = 'Download failed';

src/UI/Activity/Queue/TimeleftCell.js

@@ -10,24 +10,39 @@ module.exports = NzbDroneCell.extend({
         this.$el.empty();
 
         if (this.cellValue) {
-            if (this.cellValue.get('status').toLowerCase() === 'pending') {
-                var ect = this.cellValue.get('estimatedCompletionTime');
-                var time = '{0} at {1}'.format(FormatHelpers.relativeDate(ect), moment(ect).format(UiSettingsModel.time(true, false)));
+            var status = this.cellValue.get('status').toLowerCase();
+            var ect = this.cellValue.get('estimatedCompletionTime');
+            var time = '{0} at {1}'.format(FormatHelpers.relativeDate(ect), moment(ect).format(UiSettingsModel.time(true, false)));
+
+            if (status === 'pending') {
                 this.$el.html('<div title="Delaying download till {0}">-</div>'.format(time));
-                return this;
-            }
-
-            var timeleft = this.cellValue.get('timeleft');
-            var totalSize = FormatHelpers.bytes(this.cellValue.get('size'), 2);
-            var remainingSize = FormatHelpers.bytes(this.cellValue.get('sizeleft'), 2);
-
-            if (timeleft === undefined) {
-                this.$el.html('-');
+            } else if (status === 'downloadclientunavailable') {
+                this.$el.html('<div title="Retrying download at {0}">-</div>'.format(time));
             } else {
-                this.$el.html('<span title="{1} / {2}">{0}</span>'.format(timeleft, remainingSize, totalSize));
+                var timeleft = this.cellValue.get('timeleft');
+                var totalSize = FormatHelpers.bytes(this.cellValue.get('size'), 2);
+                var remainingSize = FormatHelpers.bytes(this.cellValue.get('sizeleft'), 2);
+
+                if (timeleft === undefined) {
+                    this.$el.html('-');
+                } else {
+                    var duration = moment.duration(timeleft);
+                    var days = duration.get('days');
+                    var hours = FormatHelpers.pad(duration.get('hours'), 2);
+                    var minutes = FormatHelpers.pad(duration.get('minutes'), 2);
+                    var seconds = FormatHelpers.pad(duration.get('seconds'), 2);
+
+                    var formattedTime = '{0}:{1}:{2}'.format(hours, minutes, seconds);
+
+                    if (days > 0) {
+                        formattedTime = days + 'd ' + formattedTime;
+                    }
+
+                    this.$el.html('<span title="{1} / {2}">{0}</span>'.format(formattedTime, remainingSize, totalSize));
+                }
             }
         }
 
         return this;
     }
-});
+});

src/UI/Content/icons.less

@@ -151,6 +151,11 @@
   .fa-icon-content(@fa-var-clock-o);
 }
 
+.icon-sonarr-client-unavailable {
+  .fa-icon-content(@fa-var-clock-o);
+  .fa-icon-color(@brand-warning);
+}
+
 .icon-sonarr-queued {
   .fa-icon-content(@fa-var-cloud);
 }