From 3f6f4fc65fa91368126a866a15955f8bebdb09c1 Mon Sep 17 00:00:00 2001 From: Qstick Date: Sun, 8 Oct 2023 11:59:17 -0500 Subject: [PATCH] Remove db calls from list threads --- .../ImportListSyncServiceFixture.cs | 4 +- .../FetchAndParseImportListService.cs | 74 +++++-------------- .../ImportLists/ImportListBase.cs | 5 +- .../ImportLists/ImportListSyncService.cs | 62 ++++++++++++++-- 4 files changed, 81 insertions(+), 64 deletions(-) diff --git a/src/NzbDrone.Core.Test/ImportListTests/ImportListSyncServiceFixture.cs b/src/NzbDrone.Core.Test/ImportListTests/ImportListSyncServiceFixture.cs index 95283449c..a58aa25e5 100644 --- a/src/NzbDrone.Core.Test/ImportListTests/ImportListSyncServiceFixture.cs +++ b/src/NzbDrone.Core.Test/ImportListTests/ImportListSyncServiceFixture.cs @@ -60,7 +60,7 @@ public void Setup() { Movies = _list1Movies, AnyFailure = false, - SyncedLists = 1 + SyncedLists = new List { 1 } }; _commandAll = new ImportListSyncCommand @@ -100,7 +100,7 @@ private void GivenListFailure() private void GivenNoListSync() { - _importListFetch.SyncedLists = 0; + _importListFetch.SyncedLists = new List(); } private void GivenCleanLevel(string cleanLevel) diff --git a/src/NzbDrone.Core/ImportLists/FetchAndParseImportListService.cs b/src/NzbDrone.Core/ImportLists/FetchAndParseImportListService.cs index d47aeb71f..828c3b428 100644 --- a/src/NzbDrone.Core/ImportLists/FetchAndParseImportListService.cs +++ b/src/NzbDrone.Core/ImportLists/FetchAndParseImportListService.cs @@ -5,9 +5,6 @@ using NLog; using NzbDrone.Common.Instrumentation.Extensions; using NzbDrone.Common.TPL; -using NzbDrone.Core.ImportLists.ImportListMovies; -using NzbDrone.Core.MetadataSource; -using NzbDrone.Core.Movies; namespace NzbDrone.Core.ImportLists { @@ -21,26 +18,14 @@ public class FetchAndParseImportListService : IFetchAndParseImportList { private readonly IImportListFactory _importListFactory; private readonly IImportListStatusService _importListStatusService; - private readonly IImportListMovieService _listMovieService; - private readonly ISearchForNewMovie _movieSearch; - private readonly IProvideMovieInfo _movieInfoService; - private readonly IMovieMetadataService _movieMetadataService; private readonly Logger _logger; public FetchAndParseImportListService(IImportListFactory importListFactory, IImportListStatusService importListStatusService, - IImportListMovieService listMovieService, - ISearchForNewMovie movieSearch, - IProvideMovieInfo movieInfoService, - IMovieMetadataService movieMetadataService, Logger logger) { _importListFactory = importListFactory; _importListStatusService = importListStatusService; - _listMovieService = listMovieService; - _movieSearch = movieSearch; - _movieInfoService = movieInfoService; - _movieMetadataService = movieMetadataService; _logger = logger; } @@ -101,21 +86,17 @@ public ImportListFetchResult Fetch() if (!importListReports.AnyFailure) { - var alreadyMapped = result.Movies.Where(x => importListReports.Movies.Any(r => r.TmdbId == x.TmdbId)); - var listMovies = MapMovieReports(importListReports.Movies.Where(x => result.Movies.All(r => r.TmdbId != x.TmdbId))).Where(x => x.TmdbId > 0).ToList(); + var listMovies = importListReports.Movies; - listMovies.AddRange(alreadyMapped); - listMovies = listMovies.DistinctBy(x => x.TmdbId).ToList(); listMovies.ForEach(m => m.ListId = importList.Definition.Id); result.Movies.AddRange(listMovies); - _listMovieService.SyncMoviesForList(listMovies, importList.Definition.Id); + + result.SyncedWithoutFailure.Add(importList.Definition.Id); } result.AnyFailure |= importListReports.AnyFailure; - result.SyncedLists++; - - _importListStatusService.UpdateListSyncStatus(importList.Definition.Id); + result.SyncedLists.Add(importList.Definition.Id); } } catch (Exception e) @@ -129,9 +110,17 @@ public ImportListFetchResult Fetch() Task.WaitAll(taskList.ToArray()); + foreach (var list in importLists) + { + if (result.SyncedLists.Contains(list.Definition.Id)) + { + _importListStatusService.UpdateListSyncStatus(list.Definition.Id); + } + } + result.Movies = result.Movies.DistinctBy(r => new { r.TmdbId, r.ImdbId, r.Title }).ToList(); - _logger.Debug("Found {0} total reports from {1} lists", result.Movies.Count, result.SyncedLists); + _logger.Debug("Found {0} total reports from {1} lists", result.Movies.Count, result.SyncedLists.Count); return result; } @@ -160,19 +149,19 @@ public ImportListFetchResult FetchSingleList(ImportListDefinition definition) if (!importListReports.AnyFailure) { - var listMovies = MapMovieReports(importListReports.Movies) - .Where(x => x.TmdbId > 0) - .DistinctBy(x => x.TmdbId) - .ToList(); + var listMovies = importListReports.Movies; listMovies.ForEach(m => m.ListId = importList.Definition.Id); result.Movies.AddRange(listMovies); - _listMovieService.SyncMoviesForList(listMovies, importList.Definition.Id); + + result.SyncedWithoutFailure.Add(importList.Definition.Id); } result.AnyFailure |= importListReports.AnyFailure; + result.SyncedLists.Add(importList.Definition.Id); + _importListStatusService.UpdateListSyncStatus(importList.Definition.Id); } } @@ -187,32 +176,5 @@ public ImportListFetchResult FetchSingleList(ImportListDefinition definition) return result; } - - private List MapMovieReports(IEnumerable reports) - { - var mappedMovies = reports.Select(m => _movieSearch.MapMovieToTmdbMovie(new MovieMetadata { Title = m.Title, TmdbId = m.TmdbId, ImdbId = m.ImdbId, Year = m.Year })) - .Where(x => x != null) - .DistinctBy(x => x.TmdbId) - .ToList(); - - _movieMetadataService.UpsertMany(mappedMovies); - - var mappedListMovies = new List(); - - foreach (var movieMeta in mappedMovies) - { - var mappedListMovie = new ImportListMovie(); - - if (movieMeta != null) - { - mappedListMovie.MovieMetadata = movieMeta; - mappedListMovie.MovieMetadataId = movieMeta.Id; - } - - mappedListMovies.Add(mappedListMovie); - } - - return mappedListMovies; - } } } diff --git a/src/NzbDrone.Core/ImportLists/ImportListBase.cs b/src/NzbDrone.Core/ImportLists/ImportListBase.cs index e8c962317..247d8d611 100644 --- a/src/NzbDrone.Core/ImportLists/ImportListBase.cs +++ b/src/NzbDrone.Core/ImportLists/ImportListBase.cs @@ -15,11 +15,14 @@ public class ImportListFetchResult public ImportListFetchResult() { Movies = new List(); + SyncedLists = new List(); + SyncedWithoutFailure = new List(); } public List Movies { get; set; } public bool AnyFailure { get; set; } - public int SyncedLists { get; set; } + public List SyncedLists { get; set; } + public List SyncedWithoutFailure { get; set; } } public abstract class ImportListBase : IImportList diff --git a/src/NzbDrone.Core/ImportLists/ImportListSyncService.cs b/src/NzbDrone.Core/ImportLists/ImportListSyncService.cs index f73d0b7dd..4a031b47e 100644 --- a/src/NzbDrone.Core/ImportLists/ImportListSyncService.cs +++ b/src/NzbDrone.Core/ImportLists/ImportListSyncService.cs @@ -7,6 +7,7 @@ using NzbDrone.Core.ImportLists.ImportExclusions; using NzbDrone.Core.ImportLists.ImportListMovies; using NzbDrone.Core.Messaging.Commands; +using NzbDrone.Core.MetadataSource; using NzbDrone.Core.Movies; namespace NzbDrone.Core.ImportLists @@ -17,6 +18,8 @@ public class ImportListSyncService : IExecute private readonly IImportListFactory _importListFactory; private readonly IFetchAndParseImportList _listFetcherAndParser; private readonly IMovieService _movieService; + private readonly IMovieMetadataService _movieMetadataService; + private readonly ISearchForNewMovie _movieSearch; private readonly IAddMovieService _addMovieService; private readonly IConfigService _configService; private readonly IImportExclusionsService _exclusionService; @@ -25,6 +28,8 @@ public class ImportListSyncService : IExecute public ImportListSyncService(IImportListFactory importListFactory, IFetchAndParseImportList listFetcherAndParser, IMovieService movieService, + IMovieMetadataService movieMetadataService, + ISearchForNewMovie movieSearch, IAddMovieService addMovieService, IConfigService configService, IImportExclusionsService exclusionService, @@ -34,6 +39,8 @@ public ImportListSyncService(IImportListFactory importListFactory, _importListFactory = importListFactory; _listFetcherAndParser = listFetcherAndParser; _movieService = movieService; + _movieMetadataService = movieMetadataService; + _movieSearch = movieSearch; _addMovieService = addMovieService; _exclusionService = exclusionService; _listMovieService = listMovieService; @@ -52,17 +59,17 @@ private void SyncAll() var listItemsResult = _listFetcherAndParser.Fetch(); - if (listItemsResult.SyncedLists == 0) + if (listItemsResult.SyncedLists.Count == 0) { return; } + ProcessListItems(listItemsResult); + if (!listItemsResult.AnyFailure) { CleanLibrary(); } - - ProcessListItems(listItemsResult); } private void SyncList(ImportListDefinition definition) @@ -125,7 +132,25 @@ private void ProcessMovieReport(ImportListDefinition importList, ImportListMovie private void ProcessListItems(ImportListFetchResult listFetchResult) { - listFetchResult.Movies = listFetchResult.Movies.DistinctBy(x => + var allMappedMovies = new List(); + + // Sync ListMovies table for Discovery view and Cleaning task + foreach (var listId in listFetchResult.SyncedWithoutFailure) + { + var listMovies = listFetchResult.Movies.Where(x => x.ListId == listId); + var alreadyMapped = allMappedMovies.Where(x => listMovies.Any(r => r.TmdbId == x.TmdbId)); + var mappedListMovies = MapMovieReports(listMovies.Where(x => allMappedMovies.All(r => r.TmdbId != x.TmdbId)).ToList()).Where(x => x.TmdbId > 0).ToList(); + + mappedListMovies.AddRange(alreadyMapped); + mappedListMovies = mappedListMovies.DistinctBy(x => x.TmdbId).ToList(); + mappedListMovies.ForEach(m => m.ListId = listId); + + allMappedMovies.AddRange(mappedListMovies); + + _listMovieService.SyncMoviesForList(mappedListMovies, listId); + } + + allMappedMovies = allMappedMovies.DistinctBy(x => { if (x.TmdbId != 0) { @@ -140,7 +165,7 @@ private void ProcessListItems(ImportListFetchResult listFetchResult) return x.Title; }).ToList(); - var listedMovies = listFetchResult.Movies.ToList(); + var listedMovies = allMappedMovies; var importExclusions = _exclusionService.GetAllExclusions(); var dbMovies = _movieService.AllMovieTmdbIds(); @@ -168,6 +193,33 @@ private void ProcessListItems(ImportListFetchResult listFetchResult) } } + private List MapMovieReports(IEnumerable reports) + { + var mappedMovies = reports.Select(m => _movieSearch.MapMovieToTmdbMovie(new MovieMetadata { Title = m.Title, TmdbId = m.TmdbId, ImdbId = m.ImdbId, Year = m.Year })) + .Where(x => x != null) + .DistinctBy(x => x.TmdbId) + .ToList(); + + _movieMetadataService.UpsertMany(mappedMovies); + + var mappedListMovies = new List(); + + foreach (var movieMeta in mappedMovies) + { + var mappedListMovie = new ImportListMovie(); + + if (movieMeta != null) + { + mappedListMovie.MovieMetadata = movieMeta; + mappedListMovie.MovieMetadataId = movieMeta.Id; + } + + mappedListMovies.Add(mappedListMovie); + } + + return mappedListMovies; + } + public void Execute(ImportListSyncCommand message) { if (message.DefinitionId.HasValue)