From efd4abfa3eaab0b476ff8d69a7ef4fc98ae1fd7d Mon Sep 17 00:00:00 2001 From: Qstick Date: Sun, 23 Apr 2023 18:11:43 -0500 Subject: [PATCH] New: Use languages from Torznab/Newznab attributes if given --- .../Aggregators/AggregateLanguagesFixture.cs | 1 + .../Aggregators/AggregateLanguages.cs | 41 +++++++++++------- src/NzbDrone.Core/Indexers/IndexerBase.cs | 12 ++++++ .../Indexers/Newznab/NewznabRssParser.cs | 43 +++++++++++++++++++ src/NzbDrone.Core/Indexers/RssParser.cs | 9 +++- .../Indexers/Torznab/TorznabRssParser.cs | 43 +++++++++++++++++++ src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs | 8 ++++ 7 files changed, 141 insertions(+), 16 deletions(-) diff --git a/src/NzbDrone.Core.Test/Download/Aggregation/Aggregators/AggregateLanguagesFixture.cs b/src/NzbDrone.Core.Test/Download/Aggregation/Aggregators/AggregateLanguagesFixture.cs index 6fa686164..81d1eaf03 100644 --- a/src/NzbDrone.Core.Test/Download/Aggregation/Aggregators/AggregateLanguagesFixture.cs +++ b/src/NzbDrone.Core.Test/Download/Aggregation/Aggregators/AggregateLanguagesFixture.cs @@ -32,6 +32,7 @@ public void Setup() _remoteMovie = Builder.CreateNew() .With(l => l.ParsedMovieInfo = null) .With(l => l.Movie = _movie) + .With(l => l.Release = new ReleaseInfo()) .Build(); } diff --git a/src/NzbDrone.Core/Download/Aggregation/Aggregators/AggregateLanguages.cs b/src/NzbDrone.Core/Download/Aggregation/Aggregators/AggregateLanguages.cs index 010a2ec16..7c5cd7a63 100644 --- a/src/NzbDrone.Core/Download/Aggregation/Aggregators/AggregateLanguages.cs +++ b/src/NzbDrone.Core/Download/Aggregation/Aggregators/AggregateLanguages.cs @@ -20,6 +20,7 @@ public AggregateLanguages(Logger logger) public RemoteMovie Aggregate(RemoteMovie remoteMovie) { var parsedMovieInfo = remoteMovie.ParsedMovieInfo; + var releaseInfo = remoteMovie.Release; var languages = parsedMovieInfo.Languages; var movie = remoteMovie.Movie; var releaseTokens = parsedMovieInfo.SimpleReleaseTitle ?? parsedMovieInfo.ReleaseTitle; @@ -30,30 +31,40 @@ public RemoteMovie Aggregate(RemoteMovie remoteMovie) { _logger.Debug("Unable to aggregate languages, using parsed values: {0}", string.Join(", ", languages.ToList())); - remoteMovie.Languages = languages; + remoteMovie.Languages = releaseInfo.Languages.Any() ? releaseInfo.Languages : languages; return remoteMovie; } - var movieTitleLanguage = LanguageParser.ParseLanguages(movie.Title); - - if (!movieTitleLanguage.Contains(Language.Unknown)) + if (releaseInfo != null && releaseInfo.Languages.Any()) { - var normalizedEpisodeTitle = Parser.Parser.NormalizeEpisodeTitle(movie.Title); - var movieTitleIndex = normalizedReleaseTokens.IndexOf(normalizedEpisodeTitle, StringComparison.CurrentCultureIgnoreCase); + _logger.Debug("Languages provided by indexer, using release values: {0}", string.Join(", ", releaseInfo.Languages)); - if (movieTitleIndex >= 0) - { - releaseTokens = releaseTokens.Remove(movieTitleIndex, normalizedEpisodeTitle.Length); - languagesToRemove.AddRange(movieTitleLanguage); - } + // Use languages from release (given by indexer or user) if available + languages = releaseInfo.Languages; } + else + { + var movieTitleLanguage = LanguageParser.ParseLanguages(movie.Title); - // Remove any languages still in the title that would normally be removed - languagesToRemove = languagesToRemove.Except(LanguageParser.ParseLanguages(releaseTokens)).ToList(); + if (!movieTitleLanguage.Contains(Language.Unknown)) + { + var normalizedEpisodeTitle = Parser.Parser.NormalizeEpisodeTitle(movie.Title); + var movieTitleIndex = normalizedReleaseTokens.IndexOf(normalizedEpisodeTitle, StringComparison.CurrentCultureIgnoreCase); - // Remove all languages that aren't part of the updated releaseTokens - languages = languages.Except(languagesToRemove).ToList(); + if (movieTitleIndex >= 0) + { + releaseTokens = releaseTokens.Remove(movieTitleIndex, normalizedEpisodeTitle.Length); + languagesToRemove.AddRange(movieTitleLanguage); + } + } + + // Remove any languages still in the title that would normally be removed + languagesToRemove = languagesToRemove.Except(LanguageParser.ParseLanguages(releaseTokens)).ToList(); + + // Remove all languages that aren't part of the updated releaseTokens + languages = languages.Except(languagesToRemove).ToList(); + } // Use movie language as fallback if we couldn't parse a language if (languages.Count == 0 || (languages.Count == 1 && languages.First() == Language.Unknown)) diff --git a/src/NzbDrone.Core/Indexers/IndexerBase.cs b/src/NzbDrone.Core/Indexers/IndexerBase.cs index 79e846e6b..7707b1d70 100644 --- a/src/NzbDrone.Core/Indexers/IndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/IndexerBase.cs @@ -1,11 +1,14 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text.RegularExpressions; using FluentValidation.Results; using NLog; +using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.IndexerSearch.Definitions; +using NzbDrone.Core.Languages; using NzbDrone.Core.Parser; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.ThingiProvider; @@ -15,6 +18,8 @@ namespace NzbDrone.Core.Indexers public abstract class IndexerBase : IIndexer where TSettings : IIndexerSettings, new() { + private static readonly Regex MultiRegex = new (@"\b(?multi)\b", RegexOptions.Compiled | RegexOptions.IgnoreCase); + protected readonly IIndexerStatusService _indexerStatusService; protected readonly IConfigService _configService; protected readonly IParsingService _parsingService; @@ -73,9 +78,16 @@ public virtual object RequestAction(string action, IDictionary q protected virtual IList CleanupReleases(IEnumerable releases) { var result = releases.DistinctBy(v => v.Guid).ToList(); + var settings = Definition.Settings as IIndexerSettings; result.ForEach(c => { + // Use multi languages from setting if ReleaseInfo languages is empty + if (c.Languages.Empty() && MultiRegex.IsMatch(c.Title) && settings.MultiLanguages.Any()) + { + c.Languages = settings.MultiLanguages.Select(i => (Language)i).ToList(); + } + c.IndexerId = Definition.Id; c.Indexer = Definition.Name; c.DownloadProtocol = Protocol; diff --git a/src/NzbDrone.Core/Indexers/Newznab/NewznabRssParser.cs b/src/NzbDrone.Core/Indexers/Newznab/NewznabRssParser.cs index 6a54424e9..c7544e8db 100644 --- a/src/NzbDrone.Core/Indexers/Newznab/NewznabRssParser.cs +++ b/src/NzbDrone.Core/Indexers/Newznab/NewznabRssParser.cs @@ -5,6 +5,8 @@ using System.Xml.Linq; using NzbDrone.Common.Extensions; using NzbDrone.Core.Indexers.Exceptions; +using NzbDrone.Core.Languages; +using NzbDrone.Core.Parser; using NzbDrone.Core.Parser.Model; namespace NzbDrone.Core.Indexers.Newznab @@ -97,6 +99,30 @@ protected override string GetCommentUrl(XElement item) return ParseUrl(item.TryGetValue("comments")); } + protected override List GetLanguages(XElement item) + { + var languges = TryGetMultipleNewznabAttributes(item, "language"); + var results = new List(); + + // Try to find elements for some indexers that suck at following the rules. + if (languges.Count == 0) + { + languges = item.Elements("language").Select(e => e.Value).ToList(); + } + + foreach (var language in languges) + { + var mappedLanguage = IsoLanguages.FindByName(language)?.Language ?? null; + + if (mappedLanguage != null) + { + results.Add(mappedLanguage); + } + } + + return results; + } + protected override long GetSize(XElement item) { long size; @@ -188,5 +214,22 @@ protected string TryGetNewznabAttribute(XElement item, string key, string defaul return defaultValue; } + + protected List TryGetMultipleNewznabAttributes(XElement item, string key) + { + var attrElements = item.Elements(ns + "attr").Where(e => e.Attribute("name").Value.Equals(key, StringComparison.OrdinalIgnoreCase)); + var results = new List(); + + foreach (var element in attrElements) + { + var attrValue = element.Attribute("value"); + if (attrValue != null) + { + results.Add(attrValue.Value); + } + } + + return results; + } } } diff --git a/src/NzbDrone.Core/Indexers/RssParser.cs b/src/NzbDrone.Core/Indexers/RssParser.cs index e30a2b1c2..1c3a41c42 100644 --- a/src/NzbDrone.Core/Indexers/RssParser.cs +++ b/src/NzbDrone.Core/Indexers/RssParser.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Globalization; using System.IO; @@ -12,6 +12,7 @@ using NzbDrone.Common.Http; using NzbDrone.Common.Instrumentation; using NzbDrone.Core.Indexers.Exceptions; +using NzbDrone.Core.Languages; using NzbDrone.Core.Parser.Model; namespace NzbDrone.Core.Indexers @@ -160,6 +161,7 @@ protected virtual ReleaseInfo ProcessItem(XElement item, ReleaseInfo releaseInfo releaseInfo.DownloadUrl = GetDownloadUrl(item); releaseInfo.InfoUrl = GetInfoUrl(item); releaseInfo.CommentUrl = GetCommentUrl(item); + releaseInfo.Languages = GetLanguages(item); try { @@ -226,6 +228,11 @@ protected virtual string GetCommentUrl(XElement item) return ParseUrl((string)item.Element("comments")); } + protected virtual List GetLanguages(XElement item) + { + return new List(); + } + protected virtual long GetSize(XElement item) { if (UseEnclosureLength) diff --git a/src/NzbDrone.Core/Indexers/Torznab/TorznabRssParser.cs b/src/NzbDrone.Core/Indexers/Torznab/TorznabRssParser.cs index c82e1bf4e..5d5f32213 100644 --- a/src/NzbDrone.Core/Indexers/Torznab/TorznabRssParser.cs +++ b/src/NzbDrone.Core/Indexers/Torznab/TorznabRssParser.cs @@ -4,6 +4,8 @@ using System.Xml.Linq; using NzbDrone.Common.Extensions; using NzbDrone.Core.Indexers.Exceptions; +using NzbDrone.Core.Languages; +using NzbDrone.Core.Parser; using NzbDrone.Core.Parser.Model; namespace NzbDrone.Core.Indexers.Torznab @@ -93,6 +95,30 @@ protected override string GetCommentUrl(XElement item) return ParseUrl(item.TryGetValue("comments")); } + protected override List GetLanguages(XElement item) + { + var languges = TryGetMultipleTorznabAttributes(item, "language"); + var results = new List(); + + // Try to find elements for some indexers that suck at following the rules. + if (languges.Count == 0) + { + languges = item.Elements("language").Select(e => e.Value).ToList(); + } + + foreach (var language in languges) + { + var mappedLanguage = IsoLanguages.FindByName(language)?.Language ?? null; + + if (mappedLanguage != null) + { + results.Add(mappedLanguage); + } + } + + return results; + } + protected override long GetSize(XElement item) { long size; @@ -224,5 +250,22 @@ protected float TryGetFloatTorznabAttribute(XElement item, string key, float def return defaultValue; } + + protected List TryGetMultipleTorznabAttributes(XElement item, string key) + { + var attrElements = item.Elements(ns + "attr").Where(e => e.Attribute("name").Value.Equals(key, StringComparison.OrdinalIgnoreCase)); + var results = new List(); + + foreach (var element in attrElements) + { + var attrValue = element.Attribute("value"); + if (attrValue != null) + { + results.Add(attrValue.Value); + } + } + + return results; + } } } diff --git a/src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs b/src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs index 11e8b56c7..404f08c0a 100644 --- a/src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs +++ b/src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs @@ -1,11 +1,18 @@ using System; +using System.Collections.Generic; using System.Text; using NzbDrone.Core.Indexers; +using NzbDrone.Core.Languages; namespace NzbDrone.Core.Parser.Model { public class ReleaseInfo { + public ReleaseInfo() + { + Languages = new List(); + } + public string Guid { get; set; } public string Title { get; set; } public long Size { get; set; } @@ -27,6 +34,7 @@ public class ReleaseInfo public string Resolution { get; set; } public IndexerFlags IndexerFlags { get; set; } + public List Languages { get; set; } public int Age {