From 9a46d5165c6fd5b2ef98d59706884f4372621cf1 Mon Sep 17 00:00:00 2001 From: nitsua Date: Sat, 12 Sep 2020 02:25:32 -0400 Subject: [PATCH] New: Add support for prioritizing indexers (#5000) --- .../Indexers/EditIndexerModalContent.js | 20 ++++++++- .../src/Settings/Indexers/Indexers/Indexer.js | 20 ++++++--- .../Settings/Indexers/Indexers/Indexers.js | 3 ++ src/NzbDrone.Api/Indexers/IndexerModule.cs | 2 + src/NzbDrone.Api/Indexers/IndexerResource.cs | 1 + .../PrioritizeDownloadDecisionFixture.cs | 43 ++++++++++++++++--- .../Migration/184_add_priority_to_indexers.cs | 14 ++++++ .../DownloadDecisionComparer.cs | 12 ++++++ .../Download/ProcessDownloadDecisions.cs | 13 +++--- src/NzbDrone.Core/Indexers/IndexerBase.cs | 2 + .../Indexers/IndexerDefinition.cs | 1 + src/NzbDrone.Core/Localization/Core/en.json | 8 ++++ src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs | 1 + src/Radarr.Api.V3/Indexers/IndexerResource.cs | 3 ++ 14 files changed, 125 insertions(+), 18 deletions(-) create mode 100644 src/NzbDrone.Core/Datastore/Migration/184_add_priority_to_indexers.cs diff --git a/frontend/src/Settings/Indexers/Indexers/EditIndexerModalContent.js b/frontend/src/Settings/Indexers/Indexers/EditIndexerModalContent.js index a198a20a5..470b84120 100644 --- a/frontend/src/Settings/Indexers/Indexers/EditIndexerModalContent.js +++ b/frontend/src/Settings/Indexers/Indexers/EditIndexerModalContent.js @@ -43,13 +43,14 @@ function EditIndexerModalContent(props) { enableInteractiveSearch, supportsRss, supportsSearch, - fields + fields, + priority } = item; return ( - {`${id ? 'Edit' : 'Add'} Indexer - ${implementationName}`} + {`${id ? translate('EditIndexer') : translate('AddIndexer')} - ${implementationName}`} @@ -134,7 +135,22 @@ function EditIndexerModalContent(props) { ); }) } + + {translate('IndexerPriority')} + + } diff --git a/frontend/src/Settings/Indexers/Indexers/Indexer.js b/frontend/src/Settings/Indexers/Indexers/Indexer.js index 069b9f668..4ef739689 100644 --- a/frontend/src/Settings/Indexers/Indexers/Indexer.js +++ b/frontend/src/Settings/Indexers/Indexers/Indexer.js @@ -69,7 +69,9 @@ class Indexer extends Component { enableAutomaticSearch, enableInteractiveSearch, supportsRss, - supportsSearch + supportsSearch, + priority, + showPriority } = this.props; return ( @@ -103,24 +105,30 @@ class Indexer extends Component { { supportsSearch && enableAutomaticSearch && } { supportsSearch && enableInteractiveSearch && } + { + showPriority && + + } { !enableRss && !enableAutomaticSearch && !enableInteractiveSearch && } @@ -155,7 +163,9 @@ Indexer.propTypes = { supportsRss: PropTypes.bool.isRequired, supportsSearch: PropTypes.bool.isRequired, onCloneIndexerPress: PropTypes.func.isRequired, - onConfirmDeleteIndexer: PropTypes.func.isRequired + onConfirmDeleteIndexer: PropTypes.func.isRequired, + priority: PropTypes.number.isRequired, + showPriority: PropTypes.bool.isRequired }; export default Indexer; diff --git a/frontend/src/Settings/Indexers/Indexers/Indexers.js b/frontend/src/Settings/Indexers/Indexers/Indexers.js index 2ef4a9a80..1ff6d211e 100644 --- a/frontend/src/Settings/Indexers/Indexers/Indexers.js +++ b/frontend/src/Settings/Indexers/Indexers/Indexers.js @@ -64,6 +64,8 @@ class Indexers extends Component { isEditIndexerModalOpen } = this.state; + const showPriority = items.some((index) => index.priority !== 25); + return (
diff --git a/src/NzbDrone.Api/Indexers/IndexerModule.cs b/src/NzbDrone.Api/Indexers/IndexerModule.cs index b22c8a7e1..f12b3d7aa 100644 --- a/src/NzbDrone.Api/Indexers/IndexerModule.cs +++ b/src/NzbDrone.Api/Indexers/IndexerModule.cs @@ -18,6 +18,7 @@ protected override void MapToResource(IndexerResource resource, IndexerDefinitio resource.SupportsRss = definition.SupportsRss; resource.SupportsSearch = definition.SupportsSearch; resource.Protocol = definition.Protocol; + resource.Priority = definition.Priority; } protected override void MapToModel(IndexerDefinition definition, IndexerResource resource) @@ -27,6 +28,7 @@ protected override void MapToModel(IndexerDefinition definition, IndexerResource definition.EnableRss = resource.EnableRss; definition.EnableAutomaticSearch = resource.EnableSearch; definition.EnableInteractiveSearch = resource.EnableSearch; + definition.Priority = resource.Priority; } protected override void Validate(IndexerDefinition definition, bool includeWarnings) diff --git a/src/NzbDrone.Api/Indexers/IndexerResource.cs b/src/NzbDrone.Api/Indexers/IndexerResource.cs index 6a1fad8f8..6d79362e6 100644 --- a/src/NzbDrone.Api/Indexers/IndexerResource.cs +++ b/src/NzbDrone.Api/Indexers/IndexerResource.cs @@ -9,5 +9,6 @@ public class IndexerResource : ProviderResource public bool SupportsRss { get; set; } public bool SupportsSearch { get; set; } public DownloadProtocol Protocol { get; set; } + public int Priority { get; set; } } } diff --git a/src/NzbDrone.Core.Test/DecisionEngineTests/PrioritizeDownloadDecisionFixture.cs b/src/NzbDrone.Core.Test/DecisionEngineTests/PrioritizeDownloadDecisionFixture.cs index 4de1b8066..bd6c2304a 100644 --- a/src/NzbDrone.Core.Test/DecisionEngineTests/PrioritizeDownloadDecisionFixture.cs +++ b/src/NzbDrone.Core.Test/DecisionEngineTests/PrioritizeDownloadDecisionFixture.cs @@ -51,7 +51,7 @@ private void GivenPreferredSize(double? size) .Returns(new QualityDefinition { PreferredSize = size }); } - private RemoteMovie GivenRemoteMovie(QualityModel quality, int age = 0, long size = 0, DownloadProtocol downloadProtocol = DownloadProtocol.Usenet, int runtime = 150) + private RemoteMovie GivenRemoteMovie(QualityModel quality, int age = 0, long size = 0, DownloadProtocol downloadProtocol = DownloadProtocol.Usenet, int runtime = 150, int indexerPriority = 25) { var remoteMovie = new RemoteMovie(); remoteMovie.ParsedMovieInfo = new ParsedMovieInfo(); @@ -73,6 +73,7 @@ private RemoteMovie GivenRemoteMovie(QualityModel quality, int age = 0, long siz remoteMovie.Release.Size = size; remoteMovie.Release.DownloadProtocol = downloadProtocol; remoteMovie.Release.Title = "A Movie 1998"; + remoteMovie.Release.IndexerPriority = indexerPriority; remoteMovie.CustomFormats = new List(); remoteMovie.CustomFormatScore = 0; @@ -449,13 +450,11 @@ public void should_prefer_better_custom_format2() [Test] public void should_prefer_2_custom_formats() { - var quality1 = new QualityModel(Quality.Bluray720p); - var remoteMovie1 = GivenRemoteMovie(quality1); + var remoteMovie1 = GivenRemoteMovie(new QualityModel(Quality.Bluray720p)); remoteMovie1.CustomFormats.Add(_customFormat1); remoteMovie1.CustomFormatScore = remoteMovie1.Movie.Profile.CalculateCustomFormatScore(remoteMovie1.CustomFormats); - var quality2 = new QualityModel(Quality.Bluray720p); - var remoteMovie2 = GivenRemoteMovie(quality2); + var remoteMovie2 = GivenRemoteMovie(new QualityModel(Quality.Bluray720p)); remoteMovie2.CustomFormats.AddRange(new List { _customFormat1, _customFormat2 }); remoteMovie2.CustomFormatScore = remoteMovie2.Movie.Profile.CalculateCustomFormatScore(remoteMovie2.CustomFormats); @@ -555,5 +554,39 @@ public void should_prefer_score_over_real_when_download_propers_is_do_not_prefer qualifiedReports.First().RemoteMovie.ParsedMovieInfo.Quality.Revision.Real.Should().Be(0); qualifiedReports.First().RemoteMovie.CustomFormatScore.Should().Be(10); } + + [Test] + public void sort_download_decisions_based_on_indexer_priority() + { + var remoteMovie1 = GivenRemoteMovie(new QualityModel(Quality.WEBDL1080p), indexerPriority: 25); + var remoteMovie2 = GivenRemoteMovie(new QualityModel(Quality.WEBDL1080p), indexerPriority: 50); + var remoteMovie3 = GivenRemoteMovie(new QualityModel(Quality.WEBDL1080p), indexerPriority: 1); + + var decisions = new List(); + decisions.AddRange(new[] { new DownloadDecision(remoteMovie1), new DownloadDecision(remoteMovie2), new DownloadDecision(remoteMovie3) }); + + var qualifiedReports = Subject.PrioritizeDecisionsForMovies(decisions); + qualifiedReports.First().RemoteMovie.Should().Be(remoteMovie3); + qualifiedReports.Skip(1).First().RemoteMovie.Should().Be(remoteMovie1); + qualifiedReports.Last().RemoteMovie.Should().Be(remoteMovie2); + } + + [Test] + public void ensure_download_decisions_indexer_priority_is_not_perfered_over_quality() + { + var remoteMovie1 = GivenRemoteMovie(new QualityModel(Quality.HDTV720p), indexerPriority: 25); + var remoteMovie2 = GivenRemoteMovie(new QualityModel(Quality.WEBDL1080p), indexerPriority: 50); + var remoteMovie3 = GivenRemoteMovie(new QualityModel(Quality.SDTV), indexerPriority: 1); + var remoteMovie4 = GivenRemoteMovie(new QualityModel(Quality.WEBDL1080p), indexerPriority: 25); + + var decisions = new List(); + decisions.AddRange(new[] { new DownloadDecision(remoteMovie1), new DownloadDecision(remoteMovie2), new DownloadDecision(remoteMovie3), new DownloadDecision(remoteMovie4) }); + + var qualifiedReports = Subject.PrioritizeDecisionsForMovies(decisions); + qualifiedReports.First().RemoteMovie.Should().Be(remoteMovie4); + qualifiedReports.Skip(1).First().RemoteMovie.Should().Be(remoteMovie2); + qualifiedReports.Skip(2).First().RemoteMovie.Should().Be(remoteMovie1); + qualifiedReports.Last().RemoteMovie.Should().Be(remoteMovie3); + } } } diff --git a/src/NzbDrone.Core/Datastore/Migration/184_add_priority_to_indexers.cs b/src/NzbDrone.Core/Datastore/Migration/184_add_priority_to_indexers.cs new file mode 100644 index 000000000..f1e5cbcee --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/184_add_priority_to_indexers.cs @@ -0,0 +1,14 @@ +using FluentMigrator; +using NzbDrone.Core.Datastore.Migration.Framework; + +namespace NzbDrone.Core.Datastore.Migration +{ + [Migration(184)] + public class add_priority_to_indexers : NzbDroneMigrationBase + { + protected override void MainDbUpgrade() + { + Alter.Table("Indexers").AddColumn("Priority").AsInt32().NotNullable().WithDefaultValue(25); + } + } +} diff --git a/src/NzbDrone.Core/DecisionEngine/DownloadDecisionComparer.cs b/src/NzbDrone.Core/DecisionEngine/DownloadDecisionComparer.cs index 13414c2fd..bc11d86e9 100644 --- a/src/NzbDrone.Core/DecisionEngine/DownloadDecisionComparer.cs +++ b/src/NzbDrone.Core/DecisionEngine/DownloadDecisionComparer.cs @@ -32,6 +32,7 @@ public int Compare(DownloadDecision x, DownloadDecision y) CompareQuality, CompareCustomFormatScore, CompareProtocol, + CompareIndexerPriority, CompareIndexerFlags, ComparePeersIfTorrent, CompareAgeIfUsenet, @@ -50,11 +51,22 @@ private int CompareBy(TSubject left, TSubject right, Func(TSubject left, TSubject right, Func funcValue) + where TValue : IComparable + { + return CompareBy(left, right, funcValue) * -1; + } + private int CompareAll(params int[] comparers) { return comparers.Select(comparer => comparer).FirstOrDefault(result => result != 0); } + private int CompareIndexerPriority(DownloadDecision x, DownloadDecision y) + { + return CompareByReverse(x.RemoteMovie.Release, y.RemoteMovie.Release, release => release.IndexerPriority); + } + private int CompareQuality(DownloadDecision x, DownloadDecision y) { if (_configService.DownloadPropersAndRepacks == ProperDownloadTypes.DoNotPrefer) diff --git a/src/NzbDrone.Core/Download/ProcessDownloadDecisions.cs b/src/NzbDrone.Core/Download/ProcessDownloadDecisions.cs index 78ef519e1..8412bd477 100644 --- a/src/NzbDrone.Core/Download/ProcessDownloadDecisions.cs +++ b/src/NzbDrone.Core/Download/ProcessDownloadDecisions.cs @@ -48,7 +48,7 @@ public ProcessedDecisions ProcessDecisions(List decisions) foreach (var report in prioritizedDecisions) { - var remoteEpisode = report.RemoteMovie; + var remoteMovie = report.RemoteMovie; var downloadProtocol = report.RemoteMovie.Release.DownloadProtocol; // Skip if already grabbed @@ -72,19 +72,20 @@ public ProcessedDecisions ProcessDecisions(List decisions) try { - _downloadService.DownloadReport(remoteEpisode); + _logger.Trace("Grabbing from Indexer {0} at priority {1}.", remoteMovie.Release.Indexer, remoteMovie.Release.IndexerPriority); + _downloadService.DownloadReport(remoteMovie); grabbed.Add(report); } catch (ReleaseUnavailableException) { - _logger.Warn("Failed to download release from indexer, no longer available. " + remoteEpisode); + _logger.Warn("Failed to download release from indexer, no longer available. " + remoteMovie); rejected.Add(report); } catch (Exception ex) { if (ex is DownloadClientUnavailableException || ex is DownloadClientAuthenticationException) { - _logger.Debug(ex, "Failed to send release to download client, storing until later. " + remoteEpisode); + _logger.Debug(ex, "Failed to send release to download client, storing until later. " + remoteMovie); PreparePending(pendingAddQueue, grabbed, pending, report, PendingReleaseReason.DownloadClientUnavailable); if (downloadProtocol == DownloadProtocol.Usenet) @@ -98,7 +99,7 @@ public ProcessedDecisions ProcessDecisions(List decisions) } else { - _logger.Warn(ex, "Couldn't add report to download queue. " + remoteEpisode); + _logger.Warn(ex, "Couldn't add report to download queue. " + remoteMovie); } } } @@ -129,7 +130,7 @@ private bool IsMovieProcessed(List decisions, DownloadDecision private void PreparePending(List> queue, List grabbed, List pending, DownloadDecision report, PendingReleaseReason reason) { - // If a release was already grabbed with matching episodes we should store it as a fallback + // If a release was already grabbed with a matching movie we should store it as a fallback // and filter it out the next time it is processed. // If a higher quality release failed to add to the download client, but a lower quality release // was sent to another client we still list it normally so it apparent that it'll grab next time. diff --git a/src/NzbDrone.Core/Indexers/IndexerBase.cs b/src/NzbDrone.Core/Indexers/IndexerBase.cs index 265a1b144..11b489245 100644 --- a/src/NzbDrone.Core/Indexers/IndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/IndexerBase.cs @@ -22,6 +22,7 @@ public abstract class IndexerBase : IIndexer public abstract string Name { get; } public abstract DownloadProtocol Protocol { get; } + public int Priority { get; set; } public abstract bool SupportsRss { get; } public abstract bool SupportsSearch { get; } @@ -77,6 +78,7 @@ protected virtual IList CleanupReleases(IEnumerable re c.IndexerId = Definition.Id; c.Indexer = Definition.Name; c.DownloadProtocol = Protocol; + c.IndexerPriority = ((IndexerDefinition)Definition).Priority; }); return result; diff --git a/src/NzbDrone.Core/Indexers/IndexerDefinition.cs b/src/NzbDrone.Core/Indexers/IndexerDefinition.cs index 748a117e5..d43c18198 100644 --- a/src/NzbDrone.Core/Indexers/IndexerDefinition.cs +++ b/src/NzbDrone.Core/Indexers/IndexerDefinition.cs @@ -10,6 +10,7 @@ public class IndexerDefinition : ProviderDefinition public DownloadProtocol Protocol { get; set; } public bool SupportsRss { get; set; } public bool SupportsSearch { get; set; } + public int Priority { get; set; } = 25; public override bool Enable => EnableRss || EnableAutomaticSearch || EnableInteractiveSearch; diff --git a/src/NzbDrone.Core/Localization/Core/en.json b/src/NzbDrone.Core/Localization/Core/en.json index 9a2cc4c1a..55ad1800b 100644 --- a/src/NzbDrone.Core/Localization/Core/en.json +++ b/src/NzbDrone.Core/Localization/Core/en.json @@ -6,6 +6,7 @@ "AddExclusion": "Add Exclusion", "AddImportExclusionHelpText": "Prevent movie from being added to Radarr by lists", "AddingTag": "Adding tag", + "AddIndexer": "Add Indexer", "AddList": "Add List", "AddListExclusion": "Add List Exclusion", "AddMovies": "Add Movies", @@ -47,6 +48,7 @@ "Authentication": "Authentication", "AuthenticationMethodHelpText": "Require Username and Password to access Radarr", "Automatic": "Automatic", + "AutomaticSearch": "Automatic Search", "AutoRedownloadFailedHelpText": "Automatically search for and attempt to download a different release", "AutoUnmonitorPreviouslyDownloadedMoviesHelpText": "Movies deleted from disk are automatically unmonitored in Radarr", "AvailabilityDelay": "Availability Delay", @@ -165,6 +167,7 @@ "DetailedProgressBarHelpText": "Show text on progess bar", "Details": "Details", "DigitalRelease": "Digital Release", + "Disabled": "Disabled", "Discover": "Discover", "DiskSpace": "Disk Space", "Docker": "Docker", @@ -194,6 +197,7 @@ "DownloadWarningCheckDownloadClientForMoreDetails": "Download warning: check download client for more details", "Edit": "Edit", "Edition": "Edition", + "EditIndexer": "Edit Indexer", "EditMovie": "Edit Movie", "EditPerson": "Edit Person", "EditRemotePathMapping": "Edit Remote Path Mapping", @@ -317,6 +321,8 @@ "IncludeUnmonitored": "Include Unmonitored", "Indexer": "Indexer", "IndexerFlags": "Indexer Flags", + "IndexerPriority": "Indexer Priority", + "IndexerPriorityHelpText": "Indexer Priority from 1 (Highest) to 50 (Lowest). Default: 25.", "IndexerRssHealthCheckNoAvailableIndexers": "All rss-capable indexers are temporarily unavailable due to recent indexer errors", "IndexerRssHealthCheckNoIndexers": "No indexers available with RSS sync enabled, Radarr will not grab new releases automatically", "Indexers": "Indexers", @@ -329,6 +335,7 @@ "IndexerStatusCheckSingleClientMessage": "Indexers unavailable due to failures: {0}", "Info": "Info", "InteractiveImport": "Interactive Import", + "InteractiveSearch": "Interactive Search", "Interval": "Interval", "KeyboardShortcuts": "Keyboard Shortcuts", "Language": "Language", @@ -494,6 +501,7 @@ "PreferIndexerFlagsHelpText": "Prioritize releases with special flags", "PreferredSize": "Preferred Size", "PreviewRename": "Preview Rename", + "Priority": "Priority", "PriorityHelpText": "Prioritize multiple Download Clients. Round-Robin is used for clients with the same priority.", "Profiles": "Profiles", "ProfilesSettingsSummary": "Quality, Language and Delay profiles", diff --git a/src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs b/src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs index b55e7a614..c9f6069f5 100644 --- a/src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs +++ b/src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs @@ -14,6 +14,7 @@ public class ReleaseInfo public string CommentUrl { get; set; } public int IndexerId { get; set; } public string Indexer { get; set; } + public int IndexerPriority { get; set; } public DownloadProtocol DownloadProtocol { get; set; } public int TvdbId { get; set; } public int TvRageId { get; set; } diff --git a/src/Radarr.Api.V3/Indexers/IndexerResource.cs b/src/Radarr.Api.V3/Indexers/IndexerResource.cs index ae4e9413c..f490a31ab 100644 --- a/src/Radarr.Api.V3/Indexers/IndexerResource.cs +++ b/src/Radarr.Api.V3/Indexers/IndexerResource.cs @@ -10,6 +10,7 @@ public class IndexerResource : ProviderResource public bool SupportsRss { get; set; } public bool SupportsSearch { get; set; } public DownloadProtocol Protocol { get; set; } + public int Priority { get; set; } } public class IndexerResourceMapper : ProviderResourceMapper @@ -29,6 +30,7 @@ public override IndexerResource ToResource(IndexerDefinition definition) resource.SupportsRss = definition.SupportsRss; resource.SupportsSearch = definition.SupportsSearch; resource.Protocol = definition.Protocol; + resource.Priority = definition.Priority; return resource; } @@ -45,6 +47,7 @@ public override IndexerDefinition ToModel(IndexerResource resource) definition.EnableRss = resource.EnableRss; definition.EnableAutomaticSearch = resource.EnableAutomaticSearch; definition.EnableInteractiveSearch = resource.EnableInteractiveSearch; + definition.Priority = resource.Priority; return definition; }