From ac4669dfc168dca66657b69760731be78189f261 Mon Sep 17 00:00:00 2001 From: Jendrik Weise Date: Sun, 21 Jan 2024 00:19:33 +0100 Subject: [PATCH] New: Parse subtitle titles (cherry picked from commit 69f99373e56a2fca49a2be645e6640624cf12339) Includes: * Fixed: Subtitle title migration when original title is null (cherry picked from commit 8921c5d7a079c58b0c74713355bf82846abc43ab) * Fixed: Migrating subtitle files with unexpectedly large number at end (cherry picked from commit f95dd00b51e61a96a0e6c094ec922c8f5cbc5334) * Fixed: Parsing of subtitle languages separated by dash (cherry picked from commit cb72e752f9e24e5691292d86ac1f46c8fa35a844) * Fix incorrect subtitle copy regex (cherry picked from commit 2ef46e5b902b42cdbafbd80b3154a5d22b090f29) --- ...les_from_existing_subtitle_filesFixture.cs | 98 +++++++++++++++++++ .../AggregateSubtitleInfoFixture.cs | 48 +++++++++ .../ParserTests/LanguageParserFixture.cs | 38 +++++++ ...rse_titles_from_existing_subtitle_files.cs | 88 +++++++++++++++++ .../Subtitles/ExistingSubtitleImporter.cs | 30 +++++- .../Extras/Subtitles/SubtitleFile.cs | 30 +++++- .../Extras/Subtitles/SubtitleService.cs | 29 ++++-- .../Aggregation/AggregationService.cs | 3 +- .../Aggregators/AggregateEdition.cs | 2 + .../Aggregators/AggregateLanguage.cs | 2 + .../Aggregators/AggregateQuality.cs | 2 + .../Aggregators/AggregateReleaseGroup.cs | 2 + .../Aggregators/AggregateReleaseInfo.cs | 2 + .../Aggregators/AggregateSubtitleInfo.cs | 63 ++++++++++++ .../Aggregators/IAggregateLocalMovie.cs | 2 + src/NzbDrone.Core/Parser/LanguageParser.cs | 79 ++++++++++++++- src/NzbDrone.Core/Parser/Model/LocalMovie.cs | 1 + .../Parser/Model/SubtitleTitleInfo.cs | 15 +++ 18 files changed, 518 insertions(+), 16 deletions(-) create mode 100644 src/NzbDrone.Core.Test/Datastore/Migration/238_parse_titles_from_existing_subtitle_filesFixture.cs create mode 100644 src/NzbDrone.Core.Test/MediaFiles/MovieImport/Aggregation/Aggregators/AggregateSubtitleInfoFixture.cs create mode 100644 src/NzbDrone.Core/Datastore/Migration/238_parse_titles_from_existing_subtitle_files.cs create mode 100644 src/NzbDrone.Core/MediaFiles/MovieImport/Aggregation/Aggregators/AggregateSubtitleInfo.cs create mode 100644 src/NzbDrone.Core/Parser/Model/SubtitleTitleInfo.cs diff --git a/src/NzbDrone.Core.Test/Datastore/Migration/238_parse_titles_from_existing_subtitle_filesFixture.cs b/src/NzbDrone.Core.Test/Datastore/Migration/238_parse_titles_from_existing_subtitle_filesFixture.cs new file mode 100644 index 000000000..7f6605452 --- /dev/null +++ b/src/NzbDrone.Core.Test/Datastore/Migration/238_parse_titles_from_existing_subtitle_filesFixture.cs @@ -0,0 +1,98 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Dapper; +using FluentAssertions; +using NUnit.Framework; +using NzbDrone.Common.Serializer; +using NzbDrone.Core.Datastore.Migration; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.Datastore.Migration +{ + [TestFixture] + public class parse_title_from_existing_subtitle_filesFixture : MigrationTest + { + [TestCase("Name (2020) - [AAC 2.0].testtitle.default.eng.forced.ass", "Name (2020)/Name (2020) - [AAC 2.0].mkv", "testtitle", 0)] + [TestCase("Name (2020) - [AAC 2.0].eng.default.testtitle.forced.ass", "Name (2020)/Name (2020) - [AAC 2.0].mkv", "testtitle", 0)] + [TestCase("Name (2020) - [AAC 2.0].default.eng.testtitle.forced.ass", "Name (2020)/Name (2020).mkv", "testtitle", 0)] + [TestCase("Name (2020) - [AAC 2.0].testtitle.forced.eng.ass", "Name (2020)/Name (2020) - [AAC 2.0].mkv", "testtitle", 0)] + [TestCase("Name (2020) - [AAC 2.0].eng.forced.testtitle.ass", "Name (2020)/Name (2020) - [AAC 2.0].mkv", "testtitle", 0)] + [TestCase("Name (2020) - [AAC 2.0].forced.eng.testtitle.ass", "Name (2020)/Name (2020).mkv", "testtitle", 0)] + [TestCase("Name (2020) - [AAC 2.0].testtitle.default.fra.forced.ass", "Name (2020)/Name (2020) - [AAC 2.0].mkv", "testtitle", 0)] + [TestCase("Name (2020) - [AAC 2.0].fra.default.testtitle.forced.ass", "Name (2020)/Name (2020) - [AAC 2.0].mkv", "testtitle", 0)] + [TestCase("Name (2020) - [AAC 2.0].default.fra.testtitle.forced.ass", "Name (2020)/Name (2020).mkv", "testtitle", 0)] + [TestCase("Name (2020) - [AAC 2.0].testtitle.forced.fra.ass", "Name (2020)/Name (2020) - [AAC 2.0].mkv", "testtitle", 0)] + [TestCase("Name (2020) - [AAC 2.0].fra.forced.testtitle.ass", "Name (2020)/Name (2020) - [AAC 2.0].mkv", "testtitle", 0)] + [TestCase("Name (2020) - [AAC 2.0].forced.fra.testtitle.ass", "Name (2020)/Name (2020).mkv", "testtitle", 0)] + [TestCase("Name (2020) - [AAC 2.0].default.forced.ass", "Name (2020)/Name (2020) - [AAC 2.0].mkv", null, 0)] + [TestCase("Name (2020) - [AAC 2.0].default.ass", "Name (2020)/Name (2020) - [AAC 2.0].mkv", null, 0)] + [TestCase("Name (2020) - [AAC 2.0].ass", "Name (2020)/Name (2020) - [AAC 2.0].mkv", null, 0)] + [TestCase("Name (2020) - [AAC 2.0].testtitle.ass", "Name (2020)/Name (2020) - [AAC 2.0].mkv", null, 0)] + [TestCase("Name (2020) - [AAC 2.0].testtitle - 3.default.eng.forced.ass", "Name (2020)/Name (2020) - [AAC 2.0].mkv", "testtitle", 3)] + [TestCase("Name (2020) - [AAC 2.0].testtitle - 3.forced.eng.ass", "Name (2020)/Name (2020) - [AAC 2.0].mkv", "testtitle", 3)] + [TestCase("Name (2020) - [AAC 2.0].eng.forced.testtitle - 3.ass", "Name (2020)/Name (2020) - [AAC 2.0].mkv", "testtitle", 3)] + [TestCase("Name (2020) - [AAC 2.0].fra.default.testtitle - 3.forced.ass", "Name (2020)/Name (2020) - [AAC 2.0].mkv", "testtitle", 3)] + [TestCase("Name (2020) - [AAC 2.0].3.default.eng.forced.ass", "Name (2020)/Name (2020) - [AAC 2.0].mkv", null, 3)] + [TestCase("Name (2020) - [AAC 2.0].3.forced.eng.ass", "Name (2020)/Name (2020) - [AAC 2.0].mkv", null, 3)] + [TestCase("Name (2020) - [AAC 2.0].eng.forced.3.ass", "Name (2020)/Name (2020) - [AAC 2.0].mkv", null, 3)] + [TestCase("Name (2020) - [AAC 2.0].fra.default.3.forced.ass", "Name (2020)/Name (2020) - [AAC 2.0].mkv", null, 3)] + [TestCase("Name (2020) - Name.2020.S01E03.REAL.PROPER.1080p.HEVC.x265-MeGusta - 0609901d2ea34acd81c9030980406065.en.forced.srt", "Name (2020)/Name (2020) - Name.2020.S01E03.REAL.PROPER.1080p.HEVC.x265-MeGusta - 0609901d2ea34acd81c9030980406065.mkv", null, 0)] + public void should_process_file_with_missing_title(string subtitlePath, string moviePath, string title, int copy) + { + var now = DateTime.UtcNow; + + var db = WithDapperMigrationTestDb(c => + { + c.Insert.IntoTable("SubtitleFiles").Row(new + { + MovieId = 1, + MovieFileId = 1, + RelativePath = subtitlePath, + Added = now, + LastUpdated = now, + Extension = Path.GetExtension(subtitlePath), + Language = 10, + LanguageTags = new List { "sdh" }.ToJson() + }); + + c.Insert.IntoTable("MovieFiles").Row(new + { + Id = 1, + MovieId = 1, + RelativePath = moviePath, + Quality = new { }.ToJson(), + IndexerFlags = 0, + Size = 0, + DateAdded = now, + Languages = new List { 1 }.ToJson() + }); + }); + + var files = db.Query("SELECT * FROM \"SubtitleFiles\"").ToList(); + + files.Should().HaveCount(1); + + files.First().Title.Should().Be(title); + files.First().Copy.Should().Be(copy); + files.First().LanguageTags.Should().NotContain("sdh"); + files.First().Language.Should().NotBe(10); + } + } + + public class SubtitleFile238 + { + public int Id { get; set; } + public int MovieId { get; set; } + public int? MovieFileId { get; set; } + public string RelativePath { get; set; } + public DateTime Added { get; set; } + public DateTime LastUpdated { get; set; } + public string Extension { get; set; } + public int Language { get; set; } + public int Copy { get; set; } + public string Title { get; set; } + public List LanguageTags { get; set; } + } +} diff --git a/src/NzbDrone.Core.Test/MediaFiles/MovieImport/Aggregation/Aggregators/AggregateSubtitleInfoFixture.cs b/src/NzbDrone.Core.Test/MediaFiles/MovieImport/Aggregation/Aggregators/AggregateSubtitleInfoFixture.cs new file mode 100644 index 000000000..54a98c7b5 --- /dev/null +++ b/src/NzbDrone.Core.Test/MediaFiles/MovieImport/Aggregation/Aggregators/AggregateSubtitleInfoFixture.cs @@ -0,0 +1,48 @@ +using FluentAssertions; +using NUnit.Framework; +using NzbDrone.Core.MediaFiles; +using NzbDrone.Core.MediaFiles.MovieImport.Aggregation.Aggregators; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.MediaFiles.movieImport.Aggregation.Aggregators +{ + [TestFixture] + public class AggregateSubtitleInfoFixture : CoreTest + { + [TestCase("Name (2020)/Name (2020) - [AAC 2.0].mkv", "", "Name (2020) - [AAC 2.0].default.eng.forced.ass")] + [TestCase("Name (2020)/Name (2020) - [AAC 2.0].mkv", "", "Name (2020) - [AAC 2.0].eng.default.ass")] + [TestCase("Name (2020)/Name (2020) - [AAC 2.0].mkv", "", "Name (2020) - [AAC 2.0].fra.ass")] + [TestCase("", "Name (2020)/Name (2020) - [AAC 2.0].mkv", "Name (2020) - [AAC 2.0].default.eng.forced.ass")] + [TestCase("", "Name (2020)/Name (2020) - [AAC 2.0].mkv", "Name (2020) - [AAC 2.0].eng.default.ass")] + [TestCase("", "Name (2020)/Name (2020) - [AAC 2.0].mkv", "Name (2020) - [AAC 2.0].fra.ass")] + public void should_do_basic_parse(string relativePath, string originalFilePath, string path) + { + var movieFile = new MovieFile + { + RelativePath = relativePath, + OriginalFilePath = originalFilePath + }; + + var subtitleTitleInfo = Subject.CleanSubtitleTitleInfo(movieFile, path); + + subtitleTitleInfo.Title.Should().BeNull(); + subtitleTitleInfo.Copy.Should().Be(0); + } + + [TestCase("Default (2020)/Default (2020) - [AAC 2.0].mkv", "Default (2020) - [AAC 2.0].default.eng.forced.ass")] + [TestCase("Default (2020)/Default (2020) - [AAC 2.0].mkv", "Default (2020) - [AAC 2.0].eng.default.ass")] + [TestCase("Default (2020)/Default (2020) - [AAC 2.0].mkv", "Default (2020) - [AAC 2.0].default.eng.testtitle.forced.ass")] + [TestCase("Default (2020)/Default (2020) - [AAC 2.0].mkv", "Default (2020) - [AAC 2.0].testtitle.eng.default.ass")] + public void should_not_parse_default(string relativePath, string path) + { + var movieFile = new MovieFile + { + RelativePath = relativePath + }; + + var subtitleTitleInfo = Subject.CleanSubtitleTitleInfo(movieFile, path); + + subtitleTitleInfo.LanguageTags.Should().NotContain("default"); + } + } +} diff --git a/src/NzbDrone.Core.Test/ParserTests/LanguageParserFixture.cs b/src/NzbDrone.Core.Test/ParserTests/LanguageParserFixture.cs index 33c2cf53c..724d0961e 100644 --- a/src/NzbDrone.Core.Test/ParserTests/LanguageParserFixture.cs +++ b/src/NzbDrone.Core.Test/ParserTests/LanguageParserFixture.cs @@ -482,5 +482,43 @@ public void should_add_original_language_and_english_to_german_release_with_ml_t result.Languages.Should().Contain(Language.Original); result.Languages.Should().Contain(Language.English); } + + [TestCase("Name (2020) - [AAC 2.0].testtitle.default.eng.forced.ass", new[] { "default", "forced" }, "testtitle", "English")] + [TestCase("Name (2020) - [AAC 2.0].eng.default.testtitle.forced.ass", new[] { "default", "forced" }, "testtitle", "English")] + [TestCase("Name (2020) - [AAC 2.0].default.eng.testtitle.forced.ass", new[] { "default", "forced" }, "testtitle", "English")] + [TestCase("Name (2020) - [AAC 2.0].testtitle.forced.eng.ass", new[] { "forced" }, "testtitle", "English")] + [TestCase("Name (2020) - [AAC 2.0].eng.forced.testtitle.ass", new[] { "forced" }, "testtitle", "English")] + [TestCase("Name (2020) - [AAC 2.0].forced.eng.testtitle.ass", new[] { "forced" }, "testtitle", "English")] + [TestCase("Name (2020) - [AAC 2.0].testtitle.default.fra.forced.ass", new[] { "default", "forced" }, "testtitle", "French")] + [TestCase("Name (2020) - [AAC 2.0].fra.default.testtitle.forced.ass", new[] { "default", "forced" }, "testtitle", "French")] + [TestCase("Name (2020) - [AAC 2.0].default.fra.testtitle.forced.ass", new[] { "default", "forced" }, "testtitle", "French")] + [TestCase("Name (2020) - [AAC 2.0].testtitle.forced.fra.ass", new[] { "forced" }, "testtitle", "French")] + [TestCase("Name (2020) - [AAC 2.0].fra.forced.testtitle.ass", new[] { "forced" }, "testtitle", "French")] + [TestCase("Name (2020) - [AAC 2.0].forced.fra.testtitle.ass", new[] { "forced" }, "testtitle", "French")] + [TestCase("Name (2020) - [AAC 2.0].ru-something-else.srt", new string[0], "something-else", "Russian")] + [TestCase("Name (2020) - [AAC 2.0].Full Subtitles.eng.ass", new string[0], "Full Subtitles", "English")] + [TestCase("Name (2020) - [AAC 2.0].mytitle - 1.en.ass", new string[0], "mytitle", "English")] + [TestCase("Name (2020) - [AAC 2.0].mytitle 1.en.ass", new string[0], "mytitle 1", "English")] + [TestCase("Name (2020) - [AAC 2.0].mytitle.en.ass", new string[0], "mytitle", "English")] + public void should_parse_title_and_tags(string postTitle, string[] expectedTags, string expectedTitle, string expectedLanguage) + { + var subtitleTitleInfo = LanguageParser.ParseSubtitleLanguageInformation(postTitle); + + subtitleTitleInfo.LanguageTags.Should().BeEquivalentTo(expectedTags); + subtitleTitleInfo.Title.Should().BeEquivalentTo(expectedTitle); + subtitleTitleInfo.Language.Should().BeEquivalentTo((Language)expectedLanguage); + } + + [TestCase("Name (2020) - [AAC 2.0].default.forced.ass")] + [TestCase("Name (2020) - [AAC 2.0].default.ass")] + [TestCase("Name (2020) - [AAC 2.0].ass")] + [TestCase("Name (2020) - [AAC 2.0].testtitle.ass")] + public void should_not_parse_false_title(string postTitle) + { + var subtitleTitleInfo = LanguageParser.ParseSubtitleLanguageInformation(postTitle); + subtitleTitleInfo.Language.Should().Be(Language.Unknown); + subtitleTitleInfo.LanguageTags.Should().BeEmpty(); + subtitleTitleInfo.RawTitle.Should().BeNull(); + } } } diff --git a/src/NzbDrone.Core/Datastore/Migration/238_parse_titles_from_existing_subtitle_files.cs b/src/NzbDrone.Core/Datastore/Migration/238_parse_titles_from_existing_subtitle_files.cs new file mode 100644 index 000000000..dbb1d63c1 --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/238_parse_titles_from_existing_subtitle_files.cs @@ -0,0 +1,88 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.IO; +using System.Linq; +using Dapper; +using FluentMigrator; +using NLog; +using NzbDrone.Common.Instrumentation; +using NzbDrone.Core.Datastore.Migration.Framework; +using NzbDrone.Core.MediaFiles.MovieImport.Aggregation.Aggregators; +using NzbDrone.Core.Parser; +using NzbDrone.Core.Parser.Model; + +namespace NzbDrone.Core.Datastore.Migration +{ + [Migration(238)] + public class parse_title_from_existing_subtitle_files : NzbDroneMigrationBase + { + private static readonly Logger Logger = NzbDroneLogger.GetLogger(typeof(AggregateSubtitleInfo)); + + protected override void MainDbUpgrade() + { + Alter.Table("SubtitleFiles").AddColumn("Title").AsString().Nullable(); + Alter.Table("SubtitleFiles").AddColumn("Copy").AsInt32().WithDefaultValue(0); + Execute.WithConnection(UpdateTitles); + } + + private void UpdateTitles(IDbConnection conn, IDbTransaction tran) + { + var updates = new List(); + + using (var cmd = conn.CreateCommand()) + { + cmd.Transaction = tran; + cmd.CommandText = "SELECT \"SubtitleFiles\".\"Id\", \"SubtitleFiles\".\"RelativePath\", \"MovieFiles\".\"RelativePath\", \"MovieFiles\".\"OriginalFilePath\" FROM \"SubtitleFiles\" JOIN \"MovieFiles\" ON \"SubtitleFiles\".\"MovieFileId\" = \"MovieFiles\".\"Id\""; + + using var reader = cmd.ExecuteReader(); + while (reader.Read()) + { + var id = reader.GetInt32(0); + var relativePath = reader.GetString(1); + var movieFileRelativePath = reader.GetString(2); + var movieFileOriginalFilePath = reader[3] as string; + + var subtitleTitleInfo = CleanSubtitleTitleInfo(movieFileRelativePath, movieFileOriginalFilePath, relativePath); + + updates.Add(new + { + Id = id, + Title = subtitleTitleInfo.Title, + Language = subtitleTitleInfo.Language, + LanguageTags = subtitleTitleInfo.LanguageTags, + Copy = subtitleTitleInfo.Copy + }); + } + } + + var updateSubtitleFilesSql = "UPDATE \"SubtitleFiles\" SET \"Title\" = @Title, \"Copy\" = @Copy, \"Language\" = @Language, \"LanguageTags\" = @LanguageTags, \"LastUpdated\" = CURRENT_TIMESTAMP WHERE \"Id\" = @Id"; + conn.Execute(updateSubtitleFilesSql, updates, transaction: tran); + } + + private static SubtitleTitleInfo CleanSubtitleTitleInfo(string relativePath, string originalFilePath, string path) + { + var subtitleTitleInfo = LanguageParser.ParseSubtitleLanguageInformation(path); + + var movieFileTitle = Path.GetFileNameWithoutExtension(relativePath); + var originalMovieFileTitle = Path.GetFileNameWithoutExtension(originalFilePath) ?? string.Empty; + + if (subtitleTitleInfo.TitleFirst && (movieFileTitle.Contains(subtitleTitleInfo.RawTitle, StringComparison.OrdinalIgnoreCase) || originalMovieFileTitle.Contains(subtitleTitleInfo.RawTitle, StringComparison.OrdinalIgnoreCase))) + { + Logger.Debug("Subtitle title '{0}' is in movie file title '{1}'. Removing from subtitle title.", subtitleTitleInfo.RawTitle, movieFileTitle); + + subtitleTitleInfo = LanguageParser.ParseBasicSubtitle(path); + } + + var cleanedTags = subtitleTitleInfo.LanguageTags.Where(t => !movieFileTitle.Contains(t, StringComparison.OrdinalIgnoreCase)).ToList(); + + if (cleanedTags.Count != subtitleTitleInfo.LanguageTags.Count) + { + Logger.Debug("Removed language tags '{0}' from subtitle title '{1}'.", string.Join(", ", subtitleTitleInfo.LanguageTags.Except(cleanedTags)), subtitleTitleInfo.RawTitle); + subtitleTitleInfo.LanguageTags = cleanedTags; + } + + return subtitleTitleInfo; + } + } +} diff --git a/src/NzbDrone.Core/Extras/Subtitles/ExistingSubtitleImporter.cs b/src/NzbDrone.Core/Extras/Subtitles/ExistingSubtitleImporter.cs index b80b356ef..66597eb91 100644 --- a/src/NzbDrone.Core/Extras/Subtitles/ExistingSubtitleImporter.cs +++ b/src/NzbDrone.Core/Extras/Subtitles/ExistingSubtitleImporter.cs @@ -4,23 +4,28 @@ using NLog; using NzbDrone.Common.Extensions; using NzbDrone.Core.Extras.Files; +using NzbDrone.Core.MediaFiles.MovieImport.Aggregation; using NzbDrone.Core.Movies; using NzbDrone.Core.Parser; +using NzbDrone.Core.Parser.Model; namespace NzbDrone.Core.Extras.Subtitles { public class ExistingSubtitleImporter : ImportExistingExtraFilesBase { private readonly IExtraFileService _subtitleFileService; + private readonly IAggregationService _aggregationService; private readonly IParsingService _parsingService; private readonly Logger _logger; public ExistingSubtitleImporter(IExtraFileService subtitleFileService, + IAggregationService aggregationService, IParsingService parsingService, Logger logger) : base(subtitleFileService) { _subtitleFileService = subtitleFileService; + _aggregationService = aggregationService; _parsingService = parsingService; _logger = logger; } @@ -48,14 +53,33 @@ public override IEnumerable ProcessFiles(Movie movie, List fi continue; } + var localMovie = new LocalMovie + { + FileMovieInfo = minimalInfo, + Movie = movie, + Path = possibleSubtitleFile + }; + + try + { + _aggregationService.Augment(localMovie, null); + } + catch (AugmentingFailedException) + { + _logger.Debug("Unable to parse extra file: {0}", possibleSubtitleFile); + continue; + } + var subtitleFile = new SubtitleFile { MovieId = movie.Id, MovieFileId = movie.MovieFileId, RelativePath = movie.Path.GetRelativePath(possibleSubtitleFile), - Language = LanguageParser.ParseSubtitleLanguage(possibleSubtitleFile), - LanguageTags = LanguageParser.ParseLanguageTags(possibleSubtitleFile), - Extension = extension + Language = localMovie.SubtitleInfo.Language, + LanguageTags = localMovie.SubtitleInfo.LanguageTags, + Title = localMovie.SubtitleInfo.Title, + Extension = extension, + Copy = localMovie.SubtitleInfo.Copy }; subtitleFiles.Add(subtitleFile); diff --git a/src/NzbDrone.Core/Extras/Subtitles/SubtitleFile.cs b/src/NzbDrone.Core/Extras/Subtitles/SubtitleFile.cs index 41a28ef4a..c3a3b7db1 100644 --- a/src/NzbDrone.Core/Extras/Subtitles/SubtitleFile.cs +++ b/src/NzbDrone.Core/Extras/Subtitles/SubtitleFile.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.Text; using NzbDrone.Core.Extras.Files; using NzbDrone.Core.Languages; @@ -13,15 +14,40 @@ public SubtitleFile() public Language Language { get; set; } - public string AggregateString => Language + LanguageTagsAsString + Extension; + public string AggregateString => Language + Title + LanguageTagsAsString + Extension; + + public int Copy { get; set; } public List LanguageTags { get; set; } + public string Title { get; set; } + private string LanguageTagsAsString => string.Join(".", LanguageTags); public override string ToString() { - return $"[{Id}] {RelativePath} ({Language}{(LanguageTags.Count > 0 ? "." : "")}{LanguageTagsAsString}{Extension})"; + var stringBuilder = new StringBuilder(); + stringBuilder.AppendFormat("[{0}] ", Id); + stringBuilder.Append(RelativePath); + + stringBuilder.Append(" ("); + stringBuilder.Append(Language); + if (Title is not null) + { + stringBuilder.Append('.'); + stringBuilder.Append(Title); + } + + if (LanguageTags.Count > 0) + { + stringBuilder.Append('.'); + stringBuilder.Append(LanguageTagsAsString); + } + + stringBuilder.Append(Extension); + stringBuilder.Append(')'); + + return stringBuilder.ToString(); } } } diff --git a/src/NzbDrone.Core/Extras/Subtitles/SubtitleService.cs b/src/NzbDrone.Core/Extras/Subtitles/SubtitleService.cs index dac8e780e..00fea36aa 100644 --- a/src/NzbDrone.Core/Extras/Subtitles/SubtitleService.cs +++ b/src/NzbDrone.Core/Extras/Subtitles/SubtitleService.cs @@ -76,12 +76,18 @@ public override IEnumerable MoveFilesAfterRename(Movie movie, List 1; + var orderedGroup = group.OrderBy(s => -s.Copy).ToList(); + var copy = group.First().Copy; - foreach (var subtitleFile in group) + foreach (var subtitleFile in orderedGroup) { - var suffix = GetSuffix(subtitleFile.Language, copy, subtitleFile.LanguageTags, groupCount > 1); + if (multipleCopies && subtitleFile.Copy == 0) + { + subtitleFile.Copy = ++copy; + } + + var suffix = GetSuffix(subtitleFile.Language, subtitleFile.Copy, subtitleFile.LanguageTags, multipleCopies, subtitleFile.Title); movedFiles.AddIfNotNull(MoveFile(movie, movieFile, subtitleFile, suffix)); @@ -229,11 +235,22 @@ public override IEnumerable ImportFiles(LocalMovie localMovie, MovieF return importedFiles; } - private string GetSuffix(Language language, int copy, List languageTags, bool multipleCopies = false) + private string GetSuffix(Language language, int copy, List languageTags, bool multipleCopies = false, string title = null) { var suffixBuilder = new StringBuilder(); - if (multipleCopies) + if (title is not null) + { + suffixBuilder.Append('.'); + suffixBuilder.Append(title); + + if (multipleCopies) + { + suffixBuilder.Append(" - "); + suffixBuilder.Append(copy); + } + } + else if (multipleCopies) { suffixBuilder.Append('.'); suffixBuilder.Append(copy); diff --git a/src/NzbDrone.Core/MediaFiles/MovieImport/Aggregation/AggregationService.cs b/src/NzbDrone.Core/MediaFiles/MovieImport/Aggregation/AggregationService.cs index ab1685b50..f141317dc 100644 --- a/src/NzbDrone.Core/MediaFiles/MovieImport/Aggregation/AggregationService.cs +++ b/src/NzbDrone.Core/MediaFiles/MovieImport/Aggregation/AggregationService.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using NLog; using NzbDrone.Common.Disk; using NzbDrone.Core.Configuration; @@ -30,7 +31,7 @@ public AggregationService(IEnumerable augmenters, IConfigService configService, Logger logger) { - _augmenters = augmenters; + _augmenters = augmenters.OrderBy(a => a.Order).ToList(); _diskProvider = diskProvider; _videoFileInfoReader = videoFileInfoReader; _configService = configService; diff --git a/src/NzbDrone.Core/MediaFiles/MovieImport/Aggregation/Aggregators/AggregateEdition.cs b/src/NzbDrone.Core/MediaFiles/MovieImport/Aggregation/Aggregators/AggregateEdition.cs index e4417afd3..ec3093e8a 100644 --- a/src/NzbDrone.Core/MediaFiles/MovieImport/Aggregation/Aggregators/AggregateEdition.cs +++ b/src/NzbDrone.Core/MediaFiles/MovieImport/Aggregation/Aggregators/AggregateEdition.cs @@ -6,6 +6,8 @@ namespace NzbDrone.Core.MediaFiles.MovieImport.Aggregation.Aggregators { public class AggregateEdition : IAggregateLocalMovie { + public int Order => 1; + public LocalMovie Aggregate(LocalMovie localMovie, DownloadClientItem downloadClientItem) { var movieEdition = localMovie.DownloadClientMovieInfo?.Edition; diff --git a/src/NzbDrone.Core/MediaFiles/MovieImport/Aggregation/Aggregators/AggregateLanguage.cs b/src/NzbDrone.Core/MediaFiles/MovieImport/Aggregation/Aggregators/AggregateLanguage.cs index 135d59c07..320cd9674 100644 --- a/src/NzbDrone.Core/MediaFiles/MovieImport/Aggregation/Aggregators/AggregateLanguage.cs +++ b/src/NzbDrone.Core/MediaFiles/MovieImport/Aggregation/Aggregators/AggregateLanguage.cs @@ -10,6 +10,8 @@ namespace NzbDrone.Core.MediaFiles.MovieImport.Aggregation.Aggregators { public class AggregateLanguage : IAggregateLocalMovie { + public int Order => 1; + private readonly List _augmentLanguages; private readonly Logger _logger; diff --git a/src/NzbDrone.Core/MediaFiles/MovieImport/Aggregation/Aggregators/AggregateQuality.cs b/src/NzbDrone.Core/MediaFiles/MovieImport/Aggregation/Aggregators/AggregateQuality.cs index b271f0703..8ae8fa4bb 100644 --- a/src/NzbDrone.Core/MediaFiles/MovieImport/Aggregation/Aggregators/AggregateQuality.cs +++ b/src/NzbDrone.Core/MediaFiles/MovieImport/Aggregation/Aggregators/AggregateQuality.cs @@ -10,6 +10,8 @@ namespace NzbDrone.Core.MediaFiles.MovieImport.Aggregation.Aggregators { public class AggregateQuality : IAggregateLocalMovie { + public int Order => 1; + private readonly List _augmentQualities; private readonly Logger _logger; diff --git a/src/NzbDrone.Core/MediaFiles/MovieImport/Aggregation/Aggregators/AggregateReleaseGroup.cs b/src/NzbDrone.Core/MediaFiles/MovieImport/Aggregation/Aggregators/AggregateReleaseGroup.cs index 256e4acb7..ab7202d11 100644 --- a/src/NzbDrone.Core/MediaFiles/MovieImport/Aggregation/Aggregators/AggregateReleaseGroup.cs +++ b/src/NzbDrone.Core/MediaFiles/MovieImport/Aggregation/Aggregators/AggregateReleaseGroup.cs @@ -6,6 +6,8 @@ namespace NzbDrone.Core.MediaFiles.MovieImport.Aggregation.Aggregators { public class AggregateReleaseGroup : IAggregateLocalMovie { + public int Order => 1; + public LocalMovie Aggregate(LocalMovie localMovie, DownloadClientItem downloadClientItem) { var releaseGroup = localMovie.DownloadClientMovieInfo?.ReleaseGroup; diff --git a/src/NzbDrone.Core/MediaFiles/MovieImport/Aggregation/Aggregators/AggregateReleaseInfo.cs b/src/NzbDrone.Core/MediaFiles/MovieImport/Aggregation/Aggregators/AggregateReleaseInfo.cs index 50d2f8522..a5395572c 100644 --- a/src/NzbDrone.Core/MediaFiles/MovieImport/Aggregation/Aggregators/AggregateReleaseInfo.cs +++ b/src/NzbDrone.Core/MediaFiles/MovieImport/Aggregation/Aggregators/AggregateReleaseInfo.cs @@ -8,6 +8,8 @@ namespace NzbDrone.Core.MediaFiles.MovieImport.Aggregation.Aggregators { public class AggregateReleaseInfo : IAggregateLocalMovie { + public int Order => 1; + private readonly IHistoryService _historyService; public AggregateReleaseInfo(IHistoryService historyService) diff --git a/src/NzbDrone.Core/MediaFiles/MovieImport/Aggregation/Aggregators/AggregateSubtitleInfo.cs b/src/NzbDrone.Core/MediaFiles/MovieImport/Aggregation/Aggregators/AggregateSubtitleInfo.cs new file mode 100644 index 000000000..606cf2609 --- /dev/null +++ b/src/NzbDrone.Core/MediaFiles/MovieImport/Aggregation/Aggregators/AggregateSubtitleInfo.cs @@ -0,0 +1,63 @@ +using System; +using System.IO; +using System.Linq; +using NLog; +using NzbDrone.Core.Download; +using NzbDrone.Core.Extras.Subtitles; +using NzbDrone.Core.Parser; +using NzbDrone.Core.Parser.Model; + +namespace NzbDrone.Core.MediaFiles.MovieImport.Aggregation.Aggregators +{ + public class AggregateSubtitleInfo : IAggregateLocalMovie + { + public int Order => 2; + + private readonly Logger _logger; + + public AggregateSubtitleInfo(Logger logger) + { + _logger = logger; + } + + public LocalMovie Aggregate(LocalMovie localMovie, DownloadClientItem downloadClientItem) + { + var path = localMovie.Path; + var isSubtitleFile = SubtitleFileExtensions.Extensions.Contains(Path.GetExtension(path)); + + if (!isSubtitleFile) + { + return localMovie; + } + + localMovie.SubtitleInfo = CleanSubtitleTitleInfo(localMovie.Movie.MovieFile, path); + + return localMovie; + } + + public SubtitleTitleInfo CleanSubtitleTitleInfo(MovieFile movieFile, string path) + { + var subtitleTitleInfo = LanguageParser.ParseSubtitleLanguageInformation(path); + + var movieFileTitle = Path.GetFileNameWithoutExtension(movieFile.RelativePath); + var originalMovieFileTitle = Path.GetFileNameWithoutExtension(movieFile.OriginalFilePath) ?? string.Empty; + + if (subtitleTitleInfo.TitleFirst && (movieFileTitle.Contains(subtitleTitleInfo.RawTitle, StringComparison.OrdinalIgnoreCase) || originalMovieFileTitle.Contains(subtitleTitleInfo.RawTitle, StringComparison.OrdinalIgnoreCase))) + { + _logger.Debug("Subtitle title '{0}' is in movie file title '{1}'. Removing from subtitle title.", subtitleTitleInfo.RawTitle, movieFileTitle); + + subtitleTitleInfo = LanguageParser.ParseBasicSubtitle(path); + } + + var cleanedTags = subtitleTitleInfo.LanguageTags.Where(t => !movieFileTitle.Contains(t, StringComparison.OrdinalIgnoreCase)).ToList(); + + if (cleanedTags.Count != subtitleTitleInfo.LanguageTags.Count) + { + _logger.Debug("Removed language tags '{0}' from subtitle title '{1}'.", string.Join(", ", subtitleTitleInfo.LanguageTags.Except(cleanedTags)), subtitleTitleInfo.RawTitle); + subtitleTitleInfo.LanguageTags = cleanedTags; + } + + return subtitleTitleInfo; + } + } +} diff --git a/src/NzbDrone.Core/MediaFiles/MovieImport/Aggregation/Aggregators/IAggregateLocalMovie.cs b/src/NzbDrone.Core/MediaFiles/MovieImport/Aggregation/Aggregators/IAggregateLocalMovie.cs index 6ec399ae3..7b1e892d0 100644 --- a/src/NzbDrone.Core/MediaFiles/MovieImport/Aggregation/Aggregators/IAggregateLocalMovie.cs +++ b/src/NzbDrone.Core/MediaFiles/MovieImport/Aggregation/Aggregators/IAggregateLocalMovie.cs @@ -5,6 +5,8 @@ namespace NzbDrone.Core.MediaFiles.MovieImport.Aggregation.Aggregators { public interface IAggregateLocalMovie { + int Order { get; } + LocalMovie Aggregate(LocalMovie localMovie, DownloadClientItem downloadClientItem); } } diff --git a/src/NzbDrone.Core/Parser/LanguageParser.cs b/src/NzbDrone.Core/Parser/LanguageParser.cs index 5609067ad..06fcc5ae0 100644 --- a/src/NzbDrone.Core/Parser/LanguageParser.cs +++ b/src/NzbDrone.Core/Parser/LanguageParser.cs @@ -7,6 +7,7 @@ using NzbDrone.Common.Extensions; using NzbDrone.Common.Instrumentation; using NzbDrone.Core.Languages; +using NzbDrone.Core.Parser.Model; namespace NzbDrone.Core.Parser { @@ -45,7 +46,11 @@ public static class LanguageParser private static readonly Regex GermanDualLanguageRegex = new (@"(?[a-z]{2,3})([-_. ](?full|forced|foreign|default|cc|psdh|sdh))*$", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex SubtitleLanguageRegex = new Regex(".+?([-_. ](?forced|foreign|default|cc|psdh|sdh))*[-_. ](?[a-z]{2,3})([-_. ](?forced|foreign|default|cc|psdh|sdh))*$", RegexOptions.Compiled | RegexOptions.IgnoreCase); + + private static readonly Regex SubtitleLanguageTitleRegex = new Regex(@".+?(\.((?forced|foreign|default|cc|psdh|sdh)|(?[a-z]{2,3})))*[-_. ](?[^.]*)(\.((?<tags2>forced|foreign|default|cc|psdh|sdh)|(?<iso_code>[a-z]{2,3})))*$", RegexOptions.Compiled | RegexOptions.IgnoreCase); + + private static readonly Regex SubtitleTitleRegex = new Regex(@"^((?<title>.+) - )?(?<copy>(?<!\d+)\d{1,3}(?!\d+))$", RegexOptions.Compiled); public static List<Language> ParseLanguages(string title) { @@ -430,14 +435,80 @@ public static Language ParseSubtitleLanguage(string fileName) } } - Logger.Debug("Unable to parse langauge from subtitle file: {0}", fileName); + Logger.Debug("Unable to parse language from subtitle file: {0}", fileName); } - catch (Exception) + catch (Exception ex) { - Logger.Debug("Failed parsing langauge from subtitle file: {0}", fileName); + Logger.Debug(ex, "Failed parsing language from subtitle file: {0}", fileName); } return Language.Unknown; } + + public static SubtitleTitleInfo ParseBasicSubtitle(string fileName) + { + return new SubtitleTitleInfo + { + TitleFirst = false, + LanguageTags = ParseLanguageTags(fileName), + Language = ParseSubtitleLanguage(fileName) + }; + } + + public static SubtitleTitleInfo ParseSubtitleLanguageInformation(string fileName) + { + var simpleFilename = Path.GetFileNameWithoutExtension(fileName); + var matchTitle = SubtitleLanguageTitleRegex.Match(simpleFilename); + + if (!matchTitle.Groups["title"].Success || (matchTitle.Groups["iso_code"].Captures.Count is var languageCodeNumber && languageCodeNumber != 1)) + { + Logger.Debug("Could not parse a title from subtitle file: {0}. Falling back to parsing without title.", fileName); + + return ParseBasicSubtitle(fileName); + } + + var isoCode = matchTitle.Groups["iso_code"].Value; + var isoLanguage = IsoLanguages.Find(isoCode.ToLower()); + + var language = isoLanguage?.Language ?? Language.Unknown; + + var languageTags = matchTitle.Groups["tags1"].Captures + .Union(matchTitle.Groups["tags2"].Captures) + .Cast<Capture>() + .Where(tag => !tag.Value.Empty()) + .Select(tag => tag.Value.ToLower()); + var rawTitle = matchTitle.Groups["title"].Value; + + var subtitleTitleInfo = new SubtitleTitleInfo + { + TitleFirst = matchTitle.Groups["tags1"].Captures.Empty(), + LanguageTags = languageTags.ToList(), + RawTitle = rawTitle, + Language = language + }; + + UpdateTitleAndCopyFromTitle(subtitleTitleInfo); + + return subtitleTitleInfo; + } + + public static void UpdateTitleAndCopyFromTitle(SubtitleTitleInfo subtitleTitleInfo) + { + if (subtitleTitleInfo.RawTitle is null) + { + subtitleTitleInfo.Title = null; + subtitleTitleInfo.Copy = 0; + } + else if (SubtitleTitleRegex.Match(subtitleTitleInfo.RawTitle) is var match && match.Success) + { + subtitleTitleInfo.Title = match.Groups["title"].Success ? match.Groups["title"].ToString() : null; + subtitleTitleInfo.Copy = int.Parse(match.Groups["copy"].ToString()); + } + else + { + subtitleTitleInfo.Title = subtitleTitleInfo.RawTitle; + subtitleTitleInfo.Copy = 0; + } + } } } diff --git a/src/NzbDrone.Core/Parser/Model/LocalMovie.cs b/src/NzbDrone.Core/Parser/Model/LocalMovie.cs index 3d89d349e..6e78538bb 100644 --- a/src/NzbDrone.Core/Parser/Model/LocalMovie.cs +++ b/src/NzbDrone.Core/Parser/Model/LocalMovie.cs @@ -41,6 +41,7 @@ public LocalMovie() public bool FileRenamedAfterScriptImport { get; set; } public bool ShouldImportExtras { get; set; } public List<string> PossibleExtraFiles { get; set; } + public SubtitleTitleInfo SubtitleInfo { get; set; } public override string ToString() { diff --git a/src/NzbDrone.Core/Parser/Model/SubtitleTitleInfo.cs b/src/NzbDrone.Core/Parser/Model/SubtitleTitleInfo.cs new file mode 100644 index 000000000..29ea84377 --- /dev/null +++ b/src/NzbDrone.Core/Parser/Model/SubtitleTitleInfo.cs @@ -0,0 +1,15 @@ +using System.Collections.Generic; +using NzbDrone.Core.Languages; + +namespace NzbDrone.Core.Parser.Model +{ + public class SubtitleTitleInfo + { + public List<string> LanguageTags { get; set; } + public Language Language { get; set; } + public string RawTitle { get; set; } + public string Title { get; set; } + public int Copy { get; set; } + public bool TitleFirst { get; set; } + } +}