mirror of
https://github.com/Radarr/Radarr.git
synced 2024-08-18 16:29:41 +02:00
Fixed: Clean DB dupes and add unique constrain
This commit is contained in:
parent
45d329423c
commit
d835358f75
@ -0,0 +1,170 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using FluentAssertions;
|
||||
using NUnit.Framework;
|
||||
using NzbDrone.Common.Serializer;
|
||||
using NzbDrone.Core.Datastore.Migration;
|
||||
using NzbDrone.Core.Movies;
|
||||
using NzbDrone.Core.Test.Framework;
|
||||
|
||||
namespace NzbDrone.Core.Test.Datastore.Migration
|
||||
{
|
||||
[TestFixture]
|
||||
public class fix_tmdb_duplicatesFixture : MigrationTest<fix_tmdb_duplicates>
|
||||
{
|
||||
private void AddMovie(fix_tmdb_duplicates m, int id, string movieTitle, string titleSlug, int tmdbId, int movieFileId, DateTime? lastInfo, DateTime added)
|
||||
{
|
||||
var movie = new
|
||||
{
|
||||
Id = id,
|
||||
Monitored = true,
|
||||
Title = movieTitle,
|
||||
CleanTitle = movieTitle,
|
||||
Status = MovieStatusType.Announced,
|
||||
MinimumAvailability = MovieStatusType.Announced,
|
||||
Images = new[] { new { CoverType = "Poster" } }.ToJson(),
|
||||
Recommendations = new[] { 1 }.ToJson(),
|
||||
HasPreDBEntry = false,
|
||||
Runtime = 90,
|
||||
OriginalLanguage = 1,
|
||||
ProfileId = 1,
|
||||
MovieFileId = movieFileId,
|
||||
Path = string.Format("/Movies/{0}", movieTitle),
|
||||
TitleSlug = titleSlug,
|
||||
TmdbId = tmdbId,
|
||||
Added = added,
|
||||
LastInfoSync = lastInfo,
|
||||
};
|
||||
|
||||
m.Insert.IntoTable("Movies").Row(movie);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_clean_duplicate_movies()
|
||||
{
|
||||
var tmdbId = 123465;
|
||||
var dateAdded = DateTime.UtcNow;
|
||||
|
||||
var db = WithMigrationTestDb(c =>
|
||||
{
|
||||
AddMovie(c, 1, "movie", "slug", tmdbId, 0, dateAdded, dateAdded);
|
||||
AddMovie(c, 2, "movie", "slug1", tmdbId, 0, dateAdded, dateAdded);
|
||||
AddMovie(c, 3, "movie", "slug2", tmdbId, 0, dateAdded, dateAdded);
|
||||
AddMovie(c, 4, "movie", "slug3", tmdbId, 0, dateAdded, dateAdded);
|
||||
});
|
||||
|
||||
var items = db.Query<Movie185>("SELECT Id, TmdbId, MovieFileId FROM Movies");
|
||||
|
||||
items.Should().HaveCount(1);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_not_clean_non_duplicate_movies()
|
||||
{
|
||||
var tmdbId = 123465;
|
||||
var dateAdded = DateTime.UtcNow;
|
||||
|
||||
var db = WithMigrationTestDb(c =>
|
||||
{
|
||||
AddMovie(c, 1, "movie", "slug", tmdbId, 0, dateAdded, dateAdded);
|
||||
AddMovie(c, 2, "movie", "slug1", tmdbId, 0, dateAdded, dateAdded);
|
||||
AddMovie(c, 3, "movie", "slug2", tmdbId, 0, dateAdded, dateAdded);
|
||||
AddMovie(c, 4, "movie", "slug3", tmdbId, 0, dateAdded, dateAdded);
|
||||
AddMovie(c, 5, "movie2", "slug4", 123457, 0, dateAdded, dateAdded);
|
||||
});
|
||||
|
||||
var items = db.Query<Movie185>("SELECT Id, TmdbId, MovieFileId FROM Movies");
|
||||
|
||||
items.Should().HaveCount(2);
|
||||
items.Where(i => i.TmdbId == tmdbId).Should().HaveCount(1);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_not_clean_any_if_no_duplicate_movies()
|
||||
{
|
||||
var dateAdded = DateTime.UtcNow;
|
||||
|
||||
var db = WithMigrationTestDb(c =>
|
||||
{
|
||||
AddMovie(c, 1, "movie1", "slug", 1, 0, dateAdded, dateAdded);
|
||||
AddMovie(c, 2, "movie2", "slug1", 2, 0, dateAdded, dateAdded);
|
||||
AddMovie(c, 3, "movie3", "slug2", 3, 0, dateAdded, dateAdded);
|
||||
AddMovie(c, 4, "movie4", "slug3", 4, 0, dateAdded, dateAdded);
|
||||
AddMovie(c, 5, "movie5", "slug4", 123457, 0, dateAdded, dateAdded);
|
||||
});
|
||||
|
||||
var items = db.Query<Movie185>("SELECT Id, TmdbId, MovieFileId FROM Movies");
|
||||
|
||||
items.Should().HaveCount(5);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_keep_movie_with_file_when_duplicates()
|
||||
{
|
||||
var tmdbId = 123465;
|
||||
var dateAdded = DateTime.UtcNow;
|
||||
|
||||
var db = WithMigrationTestDb(c =>
|
||||
{
|
||||
AddMovie(c, 1, "movie", "slug", tmdbId, 0, dateAdded, dateAdded);
|
||||
AddMovie(c, 2, "movie", "slug1", tmdbId, 1, dateAdded, dateAdded);
|
||||
AddMovie(c, 3, "movie", "slug2", tmdbId, 0, dateAdded, dateAdded);
|
||||
AddMovie(c, 4, "movie", "slug3", tmdbId, 0, dateAdded, dateAdded);
|
||||
});
|
||||
|
||||
var items = db.Query<Movie185>("SELECT Id, TmdbId, MovieFileId FROM Movies");
|
||||
|
||||
items.Should().HaveCount(1);
|
||||
items.First().Id.Should().Be(2);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_keep_earliest_added_a_movie_with_file_when_duplicates_and_multiple_have_file()
|
||||
{
|
||||
var tmdbId = 123465;
|
||||
var dateAdded = DateTime.UtcNow;
|
||||
|
||||
var db = WithMigrationTestDb(c =>
|
||||
{
|
||||
AddMovie(c, 1, "movie", "slug", tmdbId, 0, dateAdded, dateAdded);
|
||||
AddMovie(c, 2, "movie", "slug1", tmdbId, 1, dateAdded, dateAdded.AddSeconds(200));
|
||||
AddMovie(c, 3, "movie", "slug2", tmdbId, 0, dateAdded, dateAdded);
|
||||
AddMovie(c, 4, "movie", "slug3", tmdbId, 2, dateAdded, dateAdded);
|
||||
});
|
||||
|
||||
var items = db.Query<Movie185>("SELECT Id, TmdbId, MovieFileId FROM Movies");
|
||||
|
||||
items.Should().HaveCount(1);
|
||||
items.First().MovieFileId.Should().BeGreaterThan(0);
|
||||
items.First().Id.Should().Be(4);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_keep_a_movie_with_info_when_duplicates_and_no_file()
|
||||
{
|
||||
var tmdbId = 123465;
|
||||
var dateAdded = DateTime.UtcNow;
|
||||
|
||||
var db = WithMigrationTestDb(c =>
|
||||
{
|
||||
AddMovie(c, 1, "movie", "slug", tmdbId, 0, null, dateAdded);
|
||||
AddMovie(c, 2, "movie", "slug1", tmdbId, 0, null, dateAdded);
|
||||
AddMovie(c, 3, "movie", "slug2", tmdbId, 0, dateAdded, dateAdded);
|
||||
AddMovie(c, 4, "movie", "slug3", tmdbId, 0, null, dateAdded);
|
||||
});
|
||||
|
||||
var items = db.Query<Movie185>("SELECT Id, LastInfoSync, TmdbId, MovieFileId FROM Movies");
|
||||
|
||||
items.Should().HaveCount(1);
|
||||
items.First().LastInfoSync.Should().NotBeNull();
|
||||
}
|
||||
|
||||
public class Movie185
|
||||
{
|
||||
public int Id { get; set; }
|
||||
public int TmdbId { get; set; }
|
||||
public int MovieFileId { get; set; }
|
||||
public DateTime? LastInfoSync { get; set; }
|
||||
}
|
||||
}
|
||||
}
|
@ -0,0 +1,82 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Data;
|
||||
using System.Linq;
|
||||
using Dapper;
|
||||
using FluentMigrator;
|
||||
using NzbDrone.Core.Datastore.Migration.Framework;
|
||||
|
||||
namespace NzbDrone.Core.Datastore.Migration
|
||||
{
|
||||
[Migration(186)]
|
||||
public class fix_tmdb_duplicates : NzbDroneMigrationBase
|
||||
{
|
||||
protected override void MainDbUpgrade()
|
||||
{
|
||||
Execute.WithConnection(FixMovies);
|
||||
Delete.Index("IX_Movies_TmdbId").OnTable("Movies");
|
||||
Alter.Table("Movies").AlterColumn("TmdbId").AsInt32().Unique();
|
||||
}
|
||||
|
||||
private void FixMovies(IDbConnection conn, IDbTransaction tran)
|
||||
{
|
||||
var movieRows = conn.Query<MovieEntity185>($"SELECT Id, TmdbId, Added, LastInfoSync, MovieFileId FROM Movies");
|
||||
|
||||
// Only process if there are movies existing in the DB
|
||||
if (movieRows.Any())
|
||||
{
|
||||
var movieGroups = movieRows.GroupBy(m => m.TmdbId);
|
||||
var problemMovies = movieGroups.Where(g => g.Count() > 1);
|
||||
var purgeMovies = new List<MovieEntity185>();
|
||||
|
||||
// Don't do anything if there are no duplicate movies
|
||||
if (!problemMovies.Any())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
//Process duplicates to pick which to purge
|
||||
foreach (var problemGroup in problemMovies)
|
||||
{
|
||||
var moviesWithFiles = problemGroup.Where(m => m.MovieFileId > 0);
|
||||
var moviesWithInfo = problemGroup.Where(m => m.LastInfoSync != null);
|
||||
|
||||
// If we only have one with file keep it
|
||||
if (moviesWithFiles.Count() == 1)
|
||||
{
|
||||
purgeMovies.AddRange(problemGroup.Where(m => m.MovieFileId == 0).Select(m => m));
|
||||
continue;
|
||||
}
|
||||
|
||||
// If we only have one with info keep it
|
||||
if (moviesWithInfo.Count() == 1)
|
||||
{
|
||||
purgeMovies.AddRange(problemGroup.Where(m => m.LastInfoSync == null).Select(m => m));
|
||||
continue;
|
||||
}
|
||||
|
||||
// Else Prioritize by having file then Added
|
||||
purgeMovies.AddRange(problemGroup.OrderByDescending(m => m.MovieFileId > 0 ? 1 : 0).ThenBy(m => m.Added).Skip(1).Select(m => m));
|
||||
}
|
||||
|
||||
if (purgeMovies.Count > 0)
|
||||
{
|
||||
var deleteSql = "DELETE FROM Movies WHERE Id = @Id";
|
||||
conn.Execute(deleteSql, purgeMovies, transaction: tran);
|
||||
}
|
||||
|
||||
// Delete duplicates, files, metadata, history, etc...
|
||||
// (Or just the movie and let housekeeper take the rest)
|
||||
}
|
||||
}
|
||||
|
||||
private class MovieEntity185
|
||||
{
|
||||
public int Id { get; set; }
|
||||
public int TmdbId { get; set; }
|
||||
public DateTime Added { get; set; }
|
||||
public DateTime? LastInfoSync { get; set; }
|
||||
public int MovieFileId { get; set; }
|
||||
}
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue
Block a user