From d835358f75709cbacf454997725ae70051c86acc Mon Sep 17 00:00:00 2001 From: Qstick Date: Sun, 11 Oct 2020 02:52:27 -0400 Subject: [PATCH] Fixed: Clean DB dupes and add unique constrain --- .../186_fix_tmdb_duplicatesFixture.cs | 170 ++++++++++++++++++ .../Migration/186_fix_tmdb_duplicates.cs | 82 +++++++++ 2 files changed, 252 insertions(+) create mode 100644 src/NzbDrone.Core.Test/Datastore/Migration/186_fix_tmdb_duplicatesFixture.cs create mode 100644 src/NzbDrone.Core/Datastore/Migration/186_fix_tmdb_duplicates.cs diff --git a/src/NzbDrone.Core.Test/Datastore/Migration/186_fix_tmdb_duplicatesFixture.cs b/src/NzbDrone.Core.Test/Datastore/Migration/186_fix_tmdb_duplicatesFixture.cs new file mode 100644 index 000000000..9a4f1971b --- /dev/null +++ b/src/NzbDrone.Core.Test/Datastore/Migration/186_fix_tmdb_duplicatesFixture.cs @@ -0,0 +1,170 @@ +using System; +using System.Linq; +using FluentAssertions; +using NUnit.Framework; +using NzbDrone.Common.Serializer; +using NzbDrone.Core.Datastore.Migration; +using NzbDrone.Core.Movies; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.Datastore.Migration +{ + [TestFixture] + public class fix_tmdb_duplicatesFixture : MigrationTest + { + private void AddMovie(fix_tmdb_duplicates m, int id, string movieTitle, string titleSlug, int tmdbId, int movieFileId, DateTime? lastInfo, DateTime added) + { + var movie = new + { + Id = id, + Monitored = true, + Title = movieTitle, + CleanTitle = movieTitle, + Status = MovieStatusType.Announced, + MinimumAvailability = MovieStatusType.Announced, + Images = new[] { new { CoverType = "Poster" } }.ToJson(), + Recommendations = new[] { 1 }.ToJson(), + HasPreDBEntry = false, + Runtime = 90, + OriginalLanguage = 1, + ProfileId = 1, + MovieFileId = movieFileId, + Path = string.Format("/Movies/{0}", movieTitle), + TitleSlug = titleSlug, + TmdbId = tmdbId, + Added = added, + LastInfoSync = lastInfo, + }; + + m.Insert.IntoTable("Movies").Row(movie); + } + + [Test] + public void should_clean_duplicate_movies() + { + var tmdbId = 123465; + var dateAdded = DateTime.UtcNow; + + var db = WithMigrationTestDb(c => + { + AddMovie(c, 1, "movie", "slug", tmdbId, 0, dateAdded, dateAdded); + AddMovie(c, 2, "movie", "slug1", tmdbId, 0, dateAdded, dateAdded); + AddMovie(c, 3, "movie", "slug2", tmdbId, 0, dateAdded, dateAdded); + AddMovie(c, 4, "movie", "slug3", tmdbId, 0, dateAdded, dateAdded); + }); + + var items = db.Query("SELECT Id, TmdbId, MovieFileId FROM Movies"); + + items.Should().HaveCount(1); + } + + [Test] + public void should_not_clean_non_duplicate_movies() + { + var tmdbId = 123465; + var dateAdded = DateTime.UtcNow; + + var db = WithMigrationTestDb(c => + { + AddMovie(c, 1, "movie", "slug", tmdbId, 0, dateAdded, dateAdded); + AddMovie(c, 2, "movie", "slug1", tmdbId, 0, dateAdded, dateAdded); + AddMovie(c, 3, "movie", "slug2", tmdbId, 0, dateAdded, dateAdded); + AddMovie(c, 4, "movie", "slug3", tmdbId, 0, dateAdded, dateAdded); + AddMovie(c, 5, "movie2", "slug4", 123457, 0, dateAdded, dateAdded); + }); + + var items = db.Query("SELECT Id, TmdbId, MovieFileId FROM Movies"); + + items.Should().HaveCount(2); + items.Where(i => i.TmdbId == tmdbId).Should().HaveCount(1); + } + + [Test] + public void should_not_clean_any_if_no_duplicate_movies() + { + var dateAdded = DateTime.UtcNow; + + var db = WithMigrationTestDb(c => + { + AddMovie(c, 1, "movie1", "slug", 1, 0, dateAdded, dateAdded); + AddMovie(c, 2, "movie2", "slug1", 2, 0, dateAdded, dateAdded); + AddMovie(c, 3, "movie3", "slug2", 3, 0, dateAdded, dateAdded); + AddMovie(c, 4, "movie4", "slug3", 4, 0, dateAdded, dateAdded); + AddMovie(c, 5, "movie5", "slug4", 123457, 0, dateAdded, dateAdded); + }); + + var items = db.Query("SELECT Id, TmdbId, MovieFileId FROM Movies"); + + items.Should().HaveCount(5); + } + + [Test] + public void should_keep_movie_with_file_when_duplicates() + { + var tmdbId = 123465; + var dateAdded = DateTime.UtcNow; + + var db = WithMigrationTestDb(c => + { + AddMovie(c, 1, "movie", "slug", tmdbId, 0, dateAdded, dateAdded); + AddMovie(c, 2, "movie", "slug1", tmdbId, 1, dateAdded, dateAdded); + AddMovie(c, 3, "movie", "slug2", tmdbId, 0, dateAdded, dateAdded); + AddMovie(c, 4, "movie", "slug3", tmdbId, 0, dateAdded, dateAdded); + }); + + var items = db.Query("SELECT Id, TmdbId, MovieFileId FROM Movies"); + + items.Should().HaveCount(1); + items.First().Id.Should().Be(2); + } + + [Test] + public void should_keep_earliest_added_a_movie_with_file_when_duplicates_and_multiple_have_file() + { + var tmdbId = 123465; + var dateAdded = DateTime.UtcNow; + + var db = WithMigrationTestDb(c => + { + AddMovie(c, 1, "movie", "slug", tmdbId, 0, dateAdded, dateAdded); + AddMovie(c, 2, "movie", "slug1", tmdbId, 1, dateAdded, dateAdded.AddSeconds(200)); + AddMovie(c, 3, "movie", "slug2", tmdbId, 0, dateAdded, dateAdded); + AddMovie(c, 4, "movie", "slug3", tmdbId, 2, dateAdded, dateAdded); + }); + + var items = db.Query("SELECT Id, TmdbId, MovieFileId FROM Movies"); + + items.Should().HaveCount(1); + items.First().MovieFileId.Should().BeGreaterThan(0); + items.First().Id.Should().Be(4); + } + + [Test] + public void should_keep_a_movie_with_info_when_duplicates_and_no_file() + { + var tmdbId = 123465; + var dateAdded = DateTime.UtcNow; + + var db = WithMigrationTestDb(c => + { + AddMovie(c, 1, "movie", "slug", tmdbId, 0, null, dateAdded); + AddMovie(c, 2, "movie", "slug1", tmdbId, 0, null, dateAdded); + AddMovie(c, 3, "movie", "slug2", tmdbId, 0, dateAdded, dateAdded); + AddMovie(c, 4, "movie", "slug3", tmdbId, 0, null, dateAdded); + }); + + var items = db.Query("SELECT Id, LastInfoSync, TmdbId, MovieFileId FROM Movies"); + + items.Should().HaveCount(1); + items.First().LastInfoSync.Should().NotBeNull(); + } + + public class Movie185 + { + public int Id { get; set; } + public int TmdbId { get; set; } + public int MovieFileId { get; set; } + public DateTime? LastInfoSync { get; set; } + } + } +} diff --git a/src/NzbDrone.Core/Datastore/Migration/186_fix_tmdb_duplicates.cs b/src/NzbDrone.Core/Datastore/Migration/186_fix_tmdb_duplicates.cs new file mode 100644 index 000000000..46fd2acf0 --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/186_fix_tmdb_duplicates.cs @@ -0,0 +1,82 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.Linq; +using Dapper; +using FluentMigrator; +using NzbDrone.Core.Datastore.Migration.Framework; + +namespace NzbDrone.Core.Datastore.Migration +{ + [Migration(186)] + public class fix_tmdb_duplicates : NzbDroneMigrationBase + { + protected override void MainDbUpgrade() + { + Execute.WithConnection(FixMovies); + Delete.Index("IX_Movies_TmdbId").OnTable("Movies"); + Alter.Table("Movies").AlterColumn("TmdbId").AsInt32().Unique(); + } + + private void FixMovies(IDbConnection conn, IDbTransaction tran) + { + var movieRows = conn.Query($"SELECT Id, TmdbId, Added, LastInfoSync, MovieFileId FROM Movies"); + + // Only process if there are movies existing in the DB + if (movieRows.Any()) + { + var movieGroups = movieRows.GroupBy(m => m.TmdbId); + var problemMovies = movieGroups.Where(g => g.Count() > 1); + var purgeMovies = new List(); + + // Don't do anything if there are no duplicate movies + if (!problemMovies.Any()) + { + return; + } + + //Process duplicates to pick which to purge + foreach (var problemGroup in problemMovies) + { + var moviesWithFiles = problemGroup.Where(m => m.MovieFileId > 0); + var moviesWithInfo = problemGroup.Where(m => m.LastInfoSync != null); + + // If we only have one with file keep it + if (moviesWithFiles.Count() == 1) + { + purgeMovies.AddRange(problemGroup.Where(m => m.MovieFileId == 0).Select(m => m)); + continue; + } + + // If we only have one with info keep it + if (moviesWithInfo.Count() == 1) + { + purgeMovies.AddRange(problemGroup.Where(m => m.LastInfoSync == null).Select(m => m)); + continue; + } + + // Else Prioritize by having file then Added + purgeMovies.AddRange(problemGroup.OrderByDescending(m => m.MovieFileId > 0 ? 1 : 0).ThenBy(m => m.Added).Skip(1).Select(m => m)); + } + + if (purgeMovies.Count > 0) + { + var deleteSql = "DELETE FROM Movies WHERE Id = @Id"; + conn.Execute(deleteSql, purgeMovies, transaction: tran); + } + + // Delete duplicates, files, metadata, history, etc... + // (Or just the movie and let housekeeper take the rest) + } + } + + private class MovieEntity185 + { + public int Id { get; set; } + public int TmdbId { get; set; } + public DateTime Added { get; set; } + public DateTime? LastInfoSync { get; set; } + public int MovieFileId { get; set; } + } + } +}