From 95ca863697880a25efbc29cac84206040443d85b Mon Sep 17 00:00:00 2001 From: Leonardo Galli Date: Tue, 30 Jan 2018 18:29:07 +0100 Subject: [PATCH] Fixed: Clean Library being to agressive when lists are having failures. Fixes #2455 --- .../NetImport/HttpNetImportBase.cs | 11 +++-- src/NzbDrone.Core/NetImport/INetImport.cs | 2 +- src/NzbDrone.Core/NetImport/NetImportBase.cs | 8 +++- .../NetImport/NetImportSearchService.cs | 48 ++++++++++++------- 4 files changed, 46 insertions(+), 23 deletions(-) diff --git a/src/NzbDrone.Core/NetImport/HttpNetImportBase.cs b/src/NzbDrone.Core/NetImport/HttpNetImportBase.cs index 524d3bd13..82cc96f3b 100644 --- a/src/NzbDrone.Core/NetImport/HttpNetImportBase.cs +++ b/src/NzbDrone.Core/NetImport/HttpNetImportBase.cs @@ -34,19 +34,21 @@ public HttpNetImportBase(IHttpClient httpClient, IConfigService configService, I _httpClient = httpClient; } - public override IList Fetch() + public override NetImportFetchResult Fetch() { var generator = GetRequestGenerator(); return FetchMovies(generator.GetMovies()); } - protected virtual IList FetchMovies(NetImportPageableRequestChain pageableRequestChain, bool isRecent = false) + protected virtual NetImportFetchResult FetchMovies(NetImportPageableRequestChain pageableRequestChain, bool isRecent = false) { var movies = new List(); var url = string.Empty; var parser = GetParser(); + var anyFailure = false; + try { for (int i = 0; i < pageableRequestChain.Tiers; i++) @@ -73,6 +75,7 @@ protected virtual IList FetchMovies(NetImportPageableRequestChain pageabl } catch (WebException webException) { + anyFailure = true; if (webException.Message.Contains("502") || webException.Message.Contains("503") || webException.Message.Contains("timed out")) { @@ -85,6 +88,7 @@ protected virtual IList FetchMovies(NetImportPageableRequestChain pageabl } catch (HttpException httpException) { + anyFailure = true; if ((int)httpException.Response.StatusCode == 429) { _logger.Warn("API Request Limit reached for {0}", this); @@ -96,11 +100,12 @@ protected virtual IList FetchMovies(NetImportPageableRequestChain pageabl } catch (Exception feedEx) { + anyFailure = true; feedEx.Data.Add("FeedUrl", url); _logger.Error(feedEx, "An error occurred while processing feed. " + url); } - return movies; + return new NetImportFetchResult {Movies = movies, AnyFailure = anyFailure}; } protected virtual IList FetchPage(NetImportRequest request, IParseNetImportResponse parser) diff --git a/src/NzbDrone.Core/NetImport/INetImport.cs b/src/NzbDrone.Core/NetImport/INetImport.cs index 2e03fa2a5..d024cf495 100644 --- a/src/NzbDrone.Core/NetImport/INetImport.cs +++ b/src/NzbDrone.Core/NetImport/INetImport.cs @@ -9,6 +9,6 @@ public interface INetImport : IProvider bool Enabled { get; } bool EnableAuto { get; } - IList Fetch(); + NetImportFetchResult Fetch(); } } \ No newline at end of file diff --git a/src/NzbDrone.Core/NetImport/NetImportBase.cs b/src/NzbDrone.Core/NetImport/NetImportBase.cs index 007818c06..b01cce2ed 100644 --- a/src/NzbDrone.Core/NetImport/NetImportBase.cs +++ b/src/NzbDrone.Core/NetImport/NetImportBase.cs @@ -9,6 +9,12 @@ namespace NzbDrone.Core.NetImport { + public class NetImportFetchResult + { + public IList Movies { get; set; } + public bool AnyFailure { get; set; } + } + public abstract class NetImportBase : INetImport where TSettings : IProviderConfig, new() { @@ -20,7 +26,7 @@ public abstract class NetImportBase : INetImport public abstract bool Enabled { get; } public abstract bool EnableAuto { get; } - public abstract IList Fetch(); + public abstract NetImportFetchResult Fetch(); public NetImportBase(IConfigService configService, IParsingService parsingService, Logger logger) { diff --git a/src/NzbDrone.Core/NetImport/NetImportSearchService.cs b/src/NzbDrone.Core/NetImport/NetImportSearchService.cs index 4d0dad8bd..5326ab816 100644 --- a/src/NzbDrone.Core/NetImport/NetImportSearchService.cs +++ b/src/NzbDrone.Core/NetImport/NetImportSearchService.cs @@ -54,21 +54,22 @@ public NetImportSearchService(INetImportFactory netImportFactory, IMovieService } - public List Fetch(int listId, bool onlyEnableAuto = false) + public NetImportFetchResult Fetch(int listId, bool onlyEnableAuto = false) { return MovieListSearch(listId, onlyEnableAuto); } public List FetchAndFilter(int listId, bool onlyEnableAuto) { - var movies = MovieListSearch(listId, onlyEnableAuto); + var movies = MovieListSearch(listId, onlyEnableAuto).Movies; - return _movieService.FilterExistingMovies(movies); + return _movieService.FilterExistingMovies(movies.ToList()); } - public List MovieListSearch(int listId, bool onlyEnableAuto = false) + public NetImportFetchResult MovieListSearch(int listId, bool onlyEnableAuto = false) { var movies = new List(); + var anyFailure = false; var importLists = _netImportFactory.GetAvailableProviders(); @@ -81,24 +82,31 @@ public List MovieListSearch(int listId, bool onlyEnableAuto = false) foreach (var list in lists) { - movies.AddRange(list.Fetch()); + var result = list.Fetch(); + movies.AddRange(result.Movies); + anyFailure |= result.AnyFailure; } _logger.Debug("Found {0} movies from list(s) {1}", movies.Count, string.Join(", ", lists.Select(l => l.Definition.Name))); - return movies.DistinctBy(x => { - if (x.TmdbId != 0) - { - return x.TmdbId.ToString(); - } + return new NetImportFetchResult + { + Movies = movies.DistinctBy(x => + { + if (x.TmdbId != 0) + { + return x.TmdbId.ToString(); + } - if (x.ImdbId.IsNotNullOrWhiteSpace()) - { - return x.ImdbId; - } + if (x.ImdbId.IsNotNullOrWhiteSpace()) + { + return x.ImdbId; + } - return x.Title; - }).ToList(); + return x.Title; + }).ToList(), + AnyFailure = anyFailure + }; } @@ -112,9 +120,13 @@ public void Execute(NetImportSyncCommand message) return; } - var listedMovies = Fetch(0, true); + var result = Fetch(0, true); + var listedMovies = result.Movies.ToList(); - CleanLibrary(listedMovies); + if (!result.AnyFailure) + { + CleanLibrary(listedMovies); + } listedMovies = listedMovies.Where(x => !_movieService.MovieExists(x)).ToList(); if (listedMovies.Any())