From 68c326ae27b9f72bae165f611cfd6f592b901be1 Mon Sep 17 00:00:00 2001 From: The Dark <12370876+CheAle14@users.noreply.github.com> Date: Sat, 27 Jan 2024 05:55:52 +0000 Subject: [PATCH] New: Import list clean library option Closes #5201 --- frontend/src/App/State/SettingsAppState.ts | 7 + .../ImportLists/ImportListSettings.js | 22 +- .../ImportLists/Options/ImportListOptions.tsx | 148 +++++++ .../Actions/Settings/importListOptions.js | 64 +++ frontend/src/Store/Actions/settingsActions.js | 5 + .../createSettingsSectionSelector.js | 32 -- .../createSettingsSectionSelector.ts | 49 +++ .../src/typings/ImportListOptionsSettings.ts | 10 + frontend/src/typings/pending.ts | 9 + .../FetchAndParseImportListFixture.cs | 219 +++++++++++ .../ImportListItemServiceFixture.cs | 57 +++ .../ImportListSyncServiceFixture.cs | 363 +++++++++++++++++- .../Configuration/ConfigService.cs | 13 + .../Configuration/IConfigService.cs | 4 + .../Migration/193_add_import_list_items.cs | 26 ++ src/NzbDrone.Core/Datastore/TableMapping.cs | 3 + .../ImportLists/AniList/AniListImportBase.cs | 3 +- .../ImportLists/AniList/List/AniListImport.cs | 6 +- .../ImportLists/Custom/CustomImport.cs | 6 +- .../FetchAndParseImportListService.cs | 56 ++- .../ImportLists/HttpImportListBase.cs | 8 +- src/NzbDrone.Core/ImportLists/IImportList.cs | 4 +- .../ImportLists/ImportListBase.cs | 19 +- .../ImportListItemRepository.cs | 43 +++ .../ImportListItems/ImportListItemService.cs | 59 +++ .../ImportLists/ImportListStatus.cs | 1 + .../ImportLists/ImportListStatusService.cs | 29 +- .../ImportLists/ImportListSyncService.cs | 130 ++++++- .../ImportLists/ListSyncLevelType.cs | 10 + .../ImportLists/Plex/PlexImport.cs | 3 +- .../ImportLists/Rss/RssImportBase.cs | 4 +- .../ImportLists/Simkl/SimklImportBase.cs | 7 +- .../ImportLists/Sonarr/SonarrImport.cs | 7 +- .../ImportLists/Trakt/TraktImportBase.cs | 3 +- .../ImportLists/Trakt/TraktParser.cs | 3 +- src/NzbDrone.Core/Localization/Core/en.json | 9 + .../Parser/Model/ImportListItemInfo.cs | 3 +- .../Config/ImportListConfigController.cs | 27 ++ .../Config/ImportListConfigResource.cs | 24 ++ 39 files changed, 1383 insertions(+), 112 deletions(-) create mode 100644 frontend/src/Settings/ImportLists/Options/ImportListOptions.tsx create mode 100644 frontend/src/Store/Actions/Settings/importListOptions.js delete mode 100644 frontend/src/Store/Selectors/createSettingsSectionSelector.js create mode 100644 frontend/src/Store/Selectors/createSettingsSectionSelector.ts create mode 100644 frontend/src/typings/ImportListOptionsSettings.ts create mode 100644 frontend/src/typings/pending.ts create mode 100644 src/NzbDrone.Core.Test/ImportListTests/FetchAndParseImportListFixture.cs create mode 100644 src/NzbDrone.Core.Test/ImportListTests/ImportListItemServiceFixture.cs create mode 100644 src/NzbDrone.Core/Datastore/Migration/193_add_import_list_items.cs create mode 100644 src/NzbDrone.Core/ImportLists/ImportListItems/ImportListItemRepository.cs create mode 100644 src/NzbDrone.Core/ImportLists/ImportListItems/ImportListItemService.cs create mode 100644 src/NzbDrone.Core/ImportLists/ListSyncLevelType.cs create mode 100644 src/Sonarr.Api.V3/Config/ImportListConfigController.cs create mode 100644 src/Sonarr.Api.V3/Config/ImportListConfigResource.cs diff --git a/frontend/src/App/State/SettingsAppState.ts b/frontend/src/App/State/SettingsAppState.ts index e249f2d20..cb0c78ba8 100644 --- a/frontend/src/App/State/SettingsAppState.ts +++ b/frontend/src/App/State/SettingsAppState.ts @@ -7,6 +7,7 @@ import AppSectionState, { import Language from 'Language/Language'; import DownloadClient from 'typings/DownloadClient'; import ImportList from 'typings/ImportList'; +import ImportListOptionsSettings from 'typings/ImportListOptionsSettings'; import Indexer from 'typings/Indexer'; import Notification from 'typings/Notification'; import QualityProfile from 'typings/QualityProfile'; @@ -35,10 +36,15 @@ export interface QualityProfilesAppState extends AppSectionState, AppSectionSchemaState {} +export interface ImportListOptionsSettingsAppState + extends AppSectionItemState, + AppSectionSaveState {} + export type LanguageSettingsAppState = AppSectionState; export type UiSettingsAppState = AppSectionItemState; interface SettingsAppState { + advancedSettings: boolean; downloadClients: DownloadClientAppState; importLists: ImportListAppState; indexers: IndexerAppState; @@ -46,6 +52,7 @@ interface SettingsAppState { notifications: NotificationAppState; qualityProfiles: QualityProfilesAppState; ui: UiSettingsAppState; + importListOptions: ImportListOptionsSettingsAppState; } export default SettingsAppState; diff --git a/frontend/src/Settings/ImportLists/ImportListSettings.js b/frontend/src/Settings/ImportLists/ImportListSettings.js index 32a365860..de1d486b6 100644 --- a/frontend/src/Settings/ImportLists/ImportListSettings.js +++ b/frontend/src/Settings/ImportLists/ImportListSettings.js @@ -10,6 +10,7 @@ import translate from 'Utilities/String/translate'; import ImportListsExclusionsConnector from './ImportListExclusions/ImportListExclusionsConnector'; import ImportListsConnector from './ImportLists/ImportListsConnector'; import ManageImportListsModal from './ImportLists/Manage/ManageImportListsModal'; +import ImportListOptions from './Options/ImportListOptions'; class ImportListSettings extends Component { @@ -19,7 +20,10 @@ class ImportListSettings extends Component { constructor(props, context) { super(props, context); + this._saveCallback = null; + this.state = { + isSaving: false, hasPendingChanges: false, isManageImportListsOpen: false }; @@ -28,6 +32,14 @@ class ImportListSettings extends Component { // // Listeners + setChildSave = (saveCallback) => { + this._saveCallback = saveCallback; + }; + + onChildStateChange = (payload) => { + this.setState(payload); + }; + setListOptionsRef = (ref) => { this._listOptions = ref; }; @@ -47,7 +59,9 @@ class ImportListSettings extends Component { }; onSavePress = () => { - this._listOptions.getWrappedInstance().save(); + if (this._saveCallback) { + this._saveCallback(); + } }; // @@ -93,6 +107,12 @@ class ImportListSettings extends Component { + + + translate('Disabled') }, + { key: 'logOnly', value: () => translate('LogOnly') }, + { key: 'keepAndUnmonitor', value: () => translate('KeepAndUnmonitorSeries') }, + { key: 'keepAndTag', value: () => translate('KeepAndTagSeries') }, +]; + +function createImportListOptionsSelector() { + return createSelector( + (state: AppState) => state.settings.advancedSettings, + createSettingsSectionSelector(SECTION), + (advancedSettings, sectionSettings) => { + return { + advancedSettings, + save: sectionSettings.isSaving, + ...sectionSettings, + }; + } + ); +} + +interface ImportListOptionsPageProps { + setChildSave(saveCallback: () => void): void; + onChildStateChange(payload: unknown): void; +} + +function ImportListOptions(props: ImportListOptionsPageProps) { + const { setChildSave, onChildStateChange } = props; + const selected = useSelector(createImportListOptionsSelector()); + + const { + isSaving, + hasPendingChanges, + advancedSettings, + isFetching, + error, + settings, + hasSettings, + } = selected; + + const { listSyncLevel, listSyncTag } = settings; + + const dispatch = useDispatch(); + + const onInputChange = useCallback( + ({ name, value }: { name: string; value: unknown }) => { + // @ts-expect-error 'setImportListOptionsValue' isn't typed yet + dispatch(setImportListOptionsValue({ name, value })); + }, + [dispatch] + ); + + const onTagChange = useCallback( + ({ name, value }: { name: string; value: number[] }) => { + const id = value.length === 0 ? 0 : value.pop(); + // @ts-expect-error 'setImportListOptionsValue' isn't typed yet + dispatch(setImportListOptionsValue({ name, value: id })); + }, + [dispatch] + ); + + useEffect(() => { + dispatch(fetchImportListOptions()); + setChildSave(() => dispatch(saveImportListOptions())); + + return () => { + dispatch(clearPendingChanges({ section: SECTION })); + }; + }, [dispatch, setChildSave]); + + useEffect(() => { + onChildStateChange({ + isSaving, + hasPendingChanges, + }); + }, [onChildStateChange, isSaving, hasPendingChanges]); + + const translatedLevelOptions = cleanLibraryLevelOptions.map( + ({ key, value }) => { + return { + key, + value: value(), + }; + } + ); + + return advancedSettings ? ( +
+ {isFetching ? : null} + + {!isFetching && error ? ( +
{translate('UnableToLoadListOptions')}
+ ) : null} + + {hasSettings && !isFetching && !error ? ( +
+ + {translate('CleanLibraryLevel')} + + + {listSyncLevel.value === 'keepAndTag' ? ( + + {translate('ListSyncTag')} + + + ) : null} +
+ ) : null} +
+ ) : null; +} + +export default ImportListOptions; diff --git a/frontend/src/Store/Actions/Settings/importListOptions.js b/frontend/src/Store/Actions/Settings/importListOptions.js new file mode 100644 index 000000000..e33c80770 --- /dev/null +++ b/frontend/src/Store/Actions/Settings/importListOptions.js @@ -0,0 +1,64 @@ +import { createAction } from 'redux-actions'; +import createFetchHandler from 'Store/Actions/Creators/createFetchHandler'; +import createSaveHandler from 'Store/Actions/Creators/createSaveHandler'; +import createSetSettingValueReducer from 'Store/Actions/Creators/Reducers/createSetSettingValueReducer'; +import { createThunk } from 'Store/thunks'; + +// +// Variables + +const section = 'settings.importListOptions'; + +// +// Actions Types + +export const FETCH_IMPORT_LIST_OPTIONS = 'settings/importListOptions/fetchImportListOptions'; +export const SAVE_IMPORT_LIST_OPTIONS = 'settings/importListOptions/saveImportListOptions'; +export const SET_IMPORT_LIST_OPTIONS_VALUE = 'settings/importListOptions/setImportListOptionsValue'; + +// +// Action Creators + +export const fetchImportListOptions = createThunk(FETCH_IMPORT_LIST_OPTIONS); +export const saveImportListOptions = createThunk(SAVE_IMPORT_LIST_OPTIONS); +export const setImportListOptionsValue = createAction(SET_IMPORT_LIST_OPTIONS_VALUE, (payload) => { + return { + section, + ...payload + }; +}); + +// +// Details + +export default { + + // + // State + + defaultState: { + isFetching: false, + isPopulated: false, + error: null, + pendingChanges: {}, + isSaving: false, + saveError: null, + item: {} + }, + + // + // Action Handlers + + actionHandlers: { + [FETCH_IMPORT_LIST_OPTIONS]: createFetchHandler(section, '/config/importlist'), + [SAVE_IMPORT_LIST_OPTIONS]: createSaveHandler(section, '/config/importlist') + }, + + // + // Reducers + + reducers: { + [SET_IMPORT_LIST_OPTIONS_VALUE]: createSetSettingValueReducer(section) + } + +}; diff --git a/frontend/src/Store/Actions/settingsActions.js b/frontend/src/Store/Actions/settingsActions.js index c1e6e5135..32ec41f8a 100644 --- a/frontend/src/Store/Actions/settingsActions.js +++ b/frontend/src/Store/Actions/settingsActions.js @@ -10,6 +10,7 @@ import downloadClientOptions from './Settings/downloadClientOptions'; import downloadClients from './Settings/downloadClients'; import general from './Settings/general'; import importListExclusions from './Settings/importListExclusions'; +import importListOptions from './Settings/importListOptions'; import importLists from './Settings/importLists'; import indexerOptions from './Settings/indexerOptions'; import indexers from './Settings/indexers'; @@ -33,6 +34,7 @@ export * from './Settings/delayProfiles'; export * from './Settings/downloadClients'; export * from './Settings/downloadClientOptions'; export * from './Settings/general'; +export * from './Settings/importListOptions'; export * from './Settings/importLists'; export * from './Settings/importListExclusions'; export * from './Settings/indexerOptions'; @@ -69,6 +71,7 @@ export const defaultState = { general: general.defaultState, importLists: importLists.defaultState, importListExclusions: importListExclusions.defaultState, + importListOptions: importListOptions.defaultState, indexerOptions: indexerOptions.defaultState, indexers: indexers.defaultState, languages: languages.defaultState, @@ -112,6 +115,7 @@ export const actionHandlers = handleThunks({ ...general.actionHandlers, ...importLists.actionHandlers, ...importListExclusions.actionHandlers, + ...importListOptions.actionHandlers, ...indexerOptions.actionHandlers, ...indexers.actionHandlers, ...languages.actionHandlers, @@ -146,6 +150,7 @@ export const reducers = createHandleActions({ ...general.reducers, ...importLists.reducers, ...importListExclusions.reducers, + ...importListOptions.reducers, ...indexerOptions.reducers, ...indexers.reducers, ...languages.reducers, diff --git a/frontend/src/Store/Selectors/createSettingsSectionSelector.js b/frontend/src/Store/Selectors/createSettingsSectionSelector.js deleted file mode 100644 index a9f6cbff6..000000000 --- a/frontend/src/Store/Selectors/createSettingsSectionSelector.js +++ /dev/null @@ -1,32 +0,0 @@ -import { createSelector } from 'reselect'; -import selectSettings from 'Store/Selectors/selectSettings'; - -function createSettingsSectionSelector(section) { - return createSelector( - (state) => state.settings[section], - (sectionSettings) => { - const { - isFetching, - isPopulated, - error, - item, - pendingChanges, - isSaving, - saveError - } = sectionSettings; - - const settings = selectSettings(item, pendingChanges, saveError); - - return { - isFetching, - isPopulated, - error, - isSaving, - saveError, - ...settings - }; - } - ); -} - -export default createSettingsSectionSelector; diff --git a/frontend/src/Store/Selectors/createSettingsSectionSelector.ts b/frontend/src/Store/Selectors/createSettingsSectionSelector.ts new file mode 100644 index 000000000..f43e4e59b --- /dev/null +++ b/frontend/src/Store/Selectors/createSettingsSectionSelector.ts @@ -0,0 +1,49 @@ +import { createSelector } from 'reselect'; +import AppSectionState, { + AppSectionItemState, +} from 'App/State/AppSectionState'; +import AppState from 'App/State/AppState'; +import selectSettings from 'Store/Selectors/selectSettings'; +import { PendingSection } from 'typings/pending'; + +type SettingNames = keyof Omit; +type GetSectionState = AppState['settings'][Name]; +type GetSettingsSectionItemType = + GetSectionState extends AppSectionItemState + ? R + : GetSectionState extends AppSectionState + ? R + : never; + +type AppStateWithPending = { + item?: GetSettingsSectionItemType; + pendingChanges?: Partial>; + saveError?: Error; +} & GetSectionState; + +function createSettingsSectionSelector( + section: Name +) { + return createSelector( + (state: AppState) => state.settings[section], + (sectionSettings) => { + const { item, pendingChanges, saveError, ...other } = + sectionSettings as AppStateWithPending; + + const { settings, ...rest } = selectSettings( + item, + pendingChanges, + saveError + ); + + return { + ...other, + saveError, + settings: settings as PendingSection>, + ...rest, + }; + } + ); +} + +export default createSettingsSectionSelector; diff --git a/frontend/src/typings/ImportListOptionsSettings.ts b/frontend/src/typings/ImportListOptionsSettings.ts new file mode 100644 index 000000000..4eb7d2039 --- /dev/null +++ b/frontend/src/typings/ImportListOptionsSettings.ts @@ -0,0 +1,10 @@ +export type ListSyncLevel = + | 'disabled' + | 'logOnly' + | 'keepAndUnmonitor' + | 'keepAndTag'; + +export default interface ImportListOptionsSettings { + listSyncLevel: ListSyncLevel; + listSyncTag: number; +} diff --git a/frontend/src/typings/pending.ts b/frontend/src/typings/pending.ts new file mode 100644 index 000000000..53e885bcb --- /dev/null +++ b/frontend/src/typings/pending.ts @@ -0,0 +1,9 @@ +export interface Pending { + value: T; + errors: any[]; + warnings: any[]; +} + +export type PendingSection = { + [K in keyof T]: Pending; +}; diff --git a/src/NzbDrone.Core.Test/ImportListTests/FetchAndParseImportListFixture.cs b/src/NzbDrone.Core.Test/ImportListTests/FetchAndParseImportListFixture.cs new file mode 100644 index 000000000..16600d698 --- /dev/null +++ b/src/NzbDrone.Core.Test/ImportListTests/FetchAndParseImportListFixture.cs @@ -0,0 +1,219 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using FizzWare.NBuilder; +using FluentAssertions; +using Moq; +using NUnit.Framework; +using NzbDrone.Core.ImportLists; +using NzbDrone.Core.ImportLists.ImportListItems; +using NzbDrone.Core.MetadataSource; +using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.ImportListTests +{ + [TestFixture] + public class FetchAndParseImportListServiceFixture : CoreTest + { + private List _importLists; + private List _listSeries; + + [SetUp] + public void Setup() + { + _importLists = new List(); + + Mocker.GetMock() + .Setup(v => v.AutomaticAddEnabled(It.IsAny())) + .Returns(_importLists); + + _listSeries = Builder.CreateListOfSize(5) + .Build().ToList(); + + Mocker.GetMock() + .Setup(v => v.SearchForNewSeriesByImdbId(It.IsAny())) + .Returns((string value) => new List() { new Tv.Series() { ImdbId = value } }); + } + + private Mock WithList(int id, bool enabled, bool enabledAuto, ImportListFetchResult fetchResult, TimeSpan? minRefresh = null, int? lastSyncOffset = null, int? syncDeletedCount = null) + { + return CreateListResult(id, enabled, enabledAuto, fetchResult, minRefresh, lastSyncOffset, syncDeletedCount); + } + + private Mock CreateListResult(int id, bool enabled, bool enabledAuto, ImportListFetchResult fetchResult, TimeSpan? minRefresh = null, int? lastSyncOffset = null, int? syncDeletedCount = null) + { + var refreshInterval = minRefresh ?? TimeSpan.FromHours(12); + var importListDefinition = new ImportListDefinition { Id = id, Enable = enabled, EnableAutomaticAdd = enabledAuto, MinRefreshInterval = refreshInterval }; + + var mockImportList = new Mock(); + mockImportList.SetupGet(s => s.Definition).Returns(importListDefinition); + mockImportList.Setup(s => s.Fetch()).Returns(fetchResult); + mockImportList.SetupGet(s => s.MinRefreshInterval).Returns(refreshInterval); + + DateTime? lastSync = lastSyncOffset.HasValue ? DateTime.UtcNow.AddHours(lastSyncOffset.Value) : null; + Mocker.GetMock() + .Setup(v => v.GetListStatus(id)) + .Returns(new ImportListStatus() { LastInfoSync = lastSync }); + + if (syncDeletedCount.HasValue) + { + Mocker.GetMock() + .Setup(v => v.SyncSeriesForList(It.IsAny>(), id)) + .Returns(syncDeletedCount.Value); + } + + _importLists.Add(mockImportList.Object); + + return mockImportList; + } + + [Test] + public void should_skip_recently_fetched_list() + { + var fetchResult = new ImportListFetchResult(); + var list = WithList(1, true, true, fetchResult, lastSyncOffset: 0); + + var result = Subject.Fetch(); + + list.Verify(f => f.Fetch(), Times.Never()); + result.Series.Count.Should().Be(0); + result.AnyFailure.Should().BeFalse(); + } + + [Test] + public void should_skip_recent_and_fetch_good() + { + var fetchResult = new ImportListFetchResult(); + var recent = WithList(1, true, true, fetchResult, lastSyncOffset: 0); + var old = WithList(2, true, true, fetchResult); + + var result = Subject.Fetch(); + + recent.Verify(f => f.Fetch(), Times.Never()); + old.Verify(f => f.Fetch(), Times.Once()); + result.AnyFailure.Should().BeFalse(); + } + + [Test] + public void should_return_failure_if_single_list_fails() + { + var fetchResult = new ImportListFetchResult { Series = _listSeries, AnyFailure = true }; + WithList(1, true, true, fetchResult); + + var listResult = Subject.Fetch(); + listResult.AnyFailure.Should().BeTrue(); + + Mocker.GetMock() + .Verify(v => v.UpdateListSyncStatus(It.IsAny(), It.IsAny()), Times.Never()); + } + + [Test] + public void should_return_failure_if_any_list_fails() + { + var fetchResult1 = new ImportListFetchResult { Series = _listSeries, AnyFailure = true }; + WithList(1, true, true, fetchResult1); + var fetchResult2 = new ImportListFetchResult { Series = _listSeries, AnyFailure = false }; + WithList(2, true, true, fetchResult2); + + var listResult = Subject.Fetch(); + listResult.AnyFailure.Should().BeTrue(); + } + + [Test] + public void should_return_early_if_no_available_lists() + { + var listResult = Subject.Fetch(); + + Mocker.GetMock() + .Verify(v => v.GetListStatus(It.IsAny()), Times.Never()); + + listResult.Series.Count.Should().Be(0); + listResult.AnyFailure.Should().BeFalse(); + } + + [Test] + public void should_store_series_if_list_doesnt_fail() + { + var listId = 1; + var fetchResult = new ImportListFetchResult { Series = _listSeries, AnyFailure = false }; + WithList(listId, true, true, fetchResult); + + var listResult = Subject.Fetch(); + listResult.AnyFailure.Should().BeFalse(); + + Mocker.GetMock() + .Verify(v => v.UpdateListSyncStatus(listId, false), Times.Once()); + Mocker.GetMock() + .Verify(v => v.SyncSeriesForList(_listSeries, listId), Times.Once()); + } + + [Test] + public void should_not_store_series_if_list_fails() + { + var listId = 1; + var fetchResult = new ImportListFetchResult { Series = _listSeries, AnyFailure = true }; + WithList(listId, true, true, fetchResult); + + var listResult = Subject.Fetch(); + listResult.AnyFailure.Should().BeTrue(); + + Mocker.GetMock() + .Verify(v => v.UpdateListSyncStatus(listId, false), Times.Never()); + Mocker.GetMock() + .Verify(v => v.SyncSeriesForList(It.IsAny>(), listId), Times.Never()); + } + + [Test] + public void should_only_store_series_for_lists_that_dont_fail() + { + var passedListId = 1; + var fetchResult1 = new ImportListFetchResult { Series = _listSeries, AnyFailure = false }; + WithList(passedListId, true, true, fetchResult1); + var failedListId = 2; + var fetchResult2 = new ImportListFetchResult { Series = _listSeries, AnyFailure = true }; + WithList(failedListId, true, true, fetchResult2); + + var listResult = Subject.Fetch(); + listResult.AnyFailure.Should().BeTrue(); + + Mocker.GetMock() + .Verify(v => v.UpdateListSyncStatus(passedListId, false), Times.Once()); + Mocker.GetMock() + .Verify(v => v.SyncSeriesForList(_listSeries, passedListId), Times.Once()); + Mocker.GetMock() + .Verify(v => v.UpdateListSyncStatus(failedListId, false), Times.Never()); + Mocker.GetMock() + .Verify(v => v.SyncSeriesForList(It.IsAny>(), failedListId), Times.Never()); + } + + [Test] + public void should_return_all_results_for_all_lists() + { + var passedListId = 1; + var fetchResult1 = new ImportListFetchResult { Series = _listSeries, AnyFailure = false }; + WithList(passedListId, true, true, fetchResult1); + var secondListId = 2; + var fetchResult2 = new ImportListFetchResult { Series = _listSeries, AnyFailure = false }; + WithList(secondListId, true, true, fetchResult2); + + var listResult = Subject.Fetch(); + listResult.AnyFailure.Should().BeFalse(); + listResult.Series.Count.Should().Be(5); + } + + [Test] + public void should_set_removed_flag_if_list_has_removed_items() + { + var listId = 1; + var fetchResult = new ImportListFetchResult { Series = _listSeries, AnyFailure = false }; + WithList(listId, true, true, fetchResult, syncDeletedCount: 500); + + var result = Subject.Fetch(); + result.AnyFailure.Should().BeFalse(); + + Mocker.GetMock() + .Verify(v => v.UpdateListSyncStatus(listId, true), Times.Once()); + } + } +} diff --git a/src/NzbDrone.Core.Test/ImportListTests/ImportListItemServiceFixture.cs b/src/NzbDrone.Core.Test/ImportListTests/ImportListItemServiceFixture.cs new file mode 100644 index 000000000..661e2b357 --- /dev/null +++ b/src/NzbDrone.Core.Test/ImportListTests/ImportListItemServiceFixture.cs @@ -0,0 +1,57 @@ +using System.Collections.Generic; +using System.Linq; +using FizzWare.NBuilder; +using FluentAssertions; +using Moq; +using NUnit.Framework; +using NzbDrone.Core.ImportLists.ImportListItems; +using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.ImportListTests +{ + public class ImportListItemServiceFixture : CoreTest + { + [SetUp] + public void SetUp() + { + var existing = Builder.CreateListOfSize(3) + .TheFirst(1) + .With(s => s.TvdbId = 6) + .With(s => s.ImdbId = "6") + .TheNext(1) + .With(s => s.TvdbId = 7) + .With(s => s.ImdbId = "7") + .TheNext(1) + .With(s => s.TvdbId = 8) + .With(s => s.ImdbId = "8") + .Build().ToList(); + Mocker.GetMock() + .Setup(v => v.GetAllForLists(It.IsAny>())) + .Returns(existing); + } + + [Test] + public void should_insert_new_update_existing_and_delete_missing() + { + var newItems = Builder.CreateListOfSize(3) + .TheFirst(1) + .With(s => s.TvdbId = 5) + .TheNext(1) + .With(s => s.TvdbId = 6) + .TheNext(1) + .With(s => s.TvdbId = 7) + .Build().ToList(); + + var numDeleted = Subject.SyncSeriesForList(newItems, 1); + + numDeleted.Should().Be(1); + Mocker.GetMock() + .Verify(v => v.InsertMany(It.Is>(s => s.Count == 1 && s[0].TvdbId == 5)), Times.Once()); + Mocker.GetMock() + .Verify(v => v.UpdateMany(It.Is>(s => s.Count == 2 && s[0].TvdbId == 6 && s[1].TvdbId == 7)), Times.Once()); + Mocker.GetMock() + .Verify(v => v.DeleteMany(It.Is>(s => s.Count == 1 && s[0].TvdbId == 8)), Times.Once()); + } + } +} diff --git a/src/NzbDrone.Core.Test/ImportListTests/ImportListSyncServiceFixture.cs b/src/NzbDrone.Core.Test/ImportListTests/ImportListSyncServiceFixture.cs index 870dd66fc..3cf62f649 100644 --- a/src/NzbDrone.Core.Test/ImportListTests/ImportListSyncServiceFixture.cs +++ b/src/NzbDrone.Core.Test/ImportListTests/ImportListSyncServiceFixture.cs @@ -1,9 +1,13 @@ +using System; using System.Collections.Generic; using System.Linq; +using FizzWare.NBuilder; using Moq; using NUnit.Framework; +using NzbDrone.Core.Configuration; using NzbDrone.Core.ImportLists; using NzbDrone.Core.ImportLists.Exclusions; +using NzbDrone.Core.ImportLists.ImportListItems; using NzbDrone.Core.MetadataSource; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Test.Framework; @@ -13,17 +17,61 @@ namespace NzbDrone.Core.Test.ImportListTests { public class ImportListSyncServiceFixture : CoreTest { - private List _importListReports; + private ImportListFetchResult _importListFetch; + private List _list1Series; + private List _list2Series; + + private List _existingSeries; + private List _importLists; + private ImportListSyncCommand _commandAll; + private ImportListSyncCommand _commandSingle; [SetUp] public void SetUp() { - var importListItem1 = new ImportListItemInfo + _importLists = new List(); + + var item1 = new ImportListItemInfo() { Title = "Breaking Bad" }; - _importListReports = new List { importListItem1 }; + _list1Series = new List() { item1 }; + + _existingSeries = Builder.CreateListOfSize(3) + .TheFirst(1) + .With(s => s.TvdbId = 6) + .With(s => s.ImdbId = "6") + .TheNext(1) + .With(s => s.TvdbId = 7) + .With(s => s.ImdbId = "7") + .TheNext(1) + .With(s => s.TvdbId = 8) + .With(s => s.ImdbId = "8") + .Build().ToList(); + + _list2Series = Builder.CreateListOfSize(3) + .TheFirst(1) + .With(s => s.TvdbId = 6) + .With(s => s.ImdbId = "6") + .TheNext(1) + .With(s => s.TvdbId = 7) + .With(s => s.ImdbId = "7") + .TheNext(1) + .With(s => s.TvdbId = 8) + .With(s => s.ImdbId = "8") + .Build().ToList(); + + _importListFetch = new ImportListFetchResult(_list1Series, false); + + _commandAll = new ImportListSyncCommand + { + }; + + _commandSingle = new ImportListSyncCommand + { + DefinitionId = 1 + }; var mockImportList = new Mock(); @@ -31,6 +79,10 @@ namespace NzbDrone.Core.Test.ImportListTests .Setup(v => v.AllSeriesTvdbIds()) .Returns(new List()); + Mocker.GetMock() + .Setup(v => v.GetAllSeries()) + .Returns(_existingSeries); + Mocker.GetMock() .Setup(v => v.SearchForNewSeries(It.IsAny())) .Returns(new List()); @@ -41,15 +93,19 @@ namespace NzbDrone.Core.Test.ImportListTests Mocker.GetMock() .Setup(v => v.All()) - .Returns(new List { new ImportListDefinition { ShouldMonitor = MonitorTypes.All } }); + .Returns(() => _importLists.Select(x => x.Definition as ImportListDefinition).ToList()); + + Mocker.GetMock() + .Setup(v => v.GetAvailableProviders()) + .Returns(_importLists); Mocker.GetMock() .Setup(v => v.AutomaticAddEnabled(It.IsAny())) - .Returns(new List { mockImportList.Object }); + .Returns(() => _importLists.Where(x => (x.Definition as ImportListDefinition).EnableAutomaticAdd).ToList()); Mocker.GetMock() .Setup(v => v.Fetch()) - .Returns(_importListReports); + .Returns(_importListFetch); Mocker.GetMock() .Setup(v => v.All()) @@ -58,19 +114,19 @@ namespace NzbDrone.Core.Test.ImportListTests private void WithTvdbId() { - _importListReports.First().TvdbId = 81189; + _list1Series.First().TvdbId = 81189; } private void WithImdbId() { - _importListReports.First().ImdbId = "tt0496424"; + _list1Series.First().ImdbId = "tt0496424"; } private void WithExistingSeries() { Mocker.GetMock() .Setup(v => v.AllSeriesTvdbIds()) - .Returns(new List { _importListReports.First().TvdbId }); + .Returns(new List { _list1Series.First().TvdbId }); } private void WithExcludedSeries() @@ -81,22 +137,281 @@ namespace NzbDrone.Core.Test.ImportListTests { new ImportListExclusion { - TvdbId = 81189 + TvdbId = _list1Series.First().TvdbId } }); } private void WithMonitorType(MonitorTypes monitor) { + _importLists.ForEach(li => (li.Definition as ImportListDefinition).ShouldMonitor = monitor); + } + + private void WithCleanLevel(ListSyncLevelType cleanLevel, int? tagId = null) + { + Mocker.GetMock() + .SetupGet(v => v.ListSyncLevel) + .Returns(cleanLevel); + if (tagId.HasValue) + { + Mocker.GetMock() + .SetupGet(v => v.ListSyncTag) + .Returns(tagId.Value); + } + } + + private void WithList(int id, bool enabledAuto, int lastSyncHoursOffset = 0, bool pendingRemovals = true, DateTime? disabledTill = null) + { + var importListDefinition = new ImportListDefinition { Id = id, EnableAutomaticAdd = enabledAuto }; + Mocker.GetMock() - .Setup(v => v.All()) - .Returns(new List { new ImportListDefinition { ShouldMonitor = monitor } }); + .Setup(v => v.Get(id)) + .Returns(importListDefinition); + + var mockImportList = new Mock(); + mockImportList.SetupGet(s => s.Definition).Returns(importListDefinition); + mockImportList.SetupGet(s => s.MinRefreshInterval).Returns(TimeSpan.FromHours(12)); + + var status = new ImportListStatus() + { + LastInfoSync = DateTime.UtcNow.AddHours(lastSyncHoursOffset), + HasRemovedItemSinceLastClean = pendingRemovals, + DisabledTill = disabledTill + }; + + if (disabledTill.HasValue) + { + _importListFetch.AnyFailure = true; + } + + Mocker.GetMock() + .Setup(v => v.GetListStatus(id)) + .Returns(status); + + _importLists.Add(mockImportList.Object); + } + + private void VerifyDidAddTag(int expectedSeriesCount, int expectedTagId) + { + Mocker.GetMock() + .Verify(v => v.UpdateSeries(It.Is>(x => x.Count == expectedSeriesCount && x.All(series => series.Tags.Contains(expectedTagId))), true), Times.Once()); + } + + [Test] + public void should_not_clean_library_if_lists_have_not_removed_any_items() + { + _importListFetch.Series = _existingSeries.Select(x => new ImportListItemInfo() { TvdbId = x.TvdbId }).ToList(); + _importListFetch.Series.ForEach(m => m.ImportListId = 1); + WithList(1, true, pendingRemovals: false); + WithCleanLevel(ListSyncLevelType.KeepAndUnmonitor); + + Subject.Execute(_commandAll); + + Mocker.GetMock() + .Verify(v => v.GetAllSeries(), Times.Never()); + + Mocker.GetMock() + .Verify(v => v.UpdateSeries(It.IsAny>(), true), Times.Never()); + } + + [Test] + public void should_not_clean_library_if_config_value_disable() + { + _importListFetch.Series.ForEach(m => m.ImportListId = 1); + WithList(1, true); + WithCleanLevel(ListSyncLevelType.Disabled); + + Subject.Execute(_commandAll); + + Mocker.GetMock() + .Verify(v => v.GetAllSeries(), Times.Never()); + + Mocker.GetMock() + .Verify(v => v.UpdateSeries(new List(), true), Times.Never()); + } + + [Test] + public void should_log_only_on_clean_library_if_config_value_logonly() + { + _importListFetch.Series.ForEach(m => m.ImportListId = 1); + WithList(1, true); + WithCleanLevel(ListSyncLevelType.LogOnly); + + Subject.Execute(_commandAll); + + Mocker.GetMock() + .Verify(v => v.GetAllSeries(), Times.Once()); + + Mocker.GetMock() + .Verify(v => v.DeleteSeries(It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never()); + + Mocker.GetMock() + .Verify(v => v.UpdateSeries(new List(), true), Times.Once()); + } + + [Test] + public void should_unmonitor_on_clean_library_if_config_value_keepAndUnmonitor() + { + _importListFetch.Series.ForEach(m => m.ImportListId = 1); + WithList(1, true); + WithCleanLevel(ListSyncLevelType.KeepAndUnmonitor); + var monitored = _existingSeries.Count(x => x.Monitored); + + Subject.Execute(_commandAll); + + Mocker.GetMock() + .Verify(v => v.GetAllSeries(), Times.Once()); + + Mocker.GetMock() + .Verify(v => v.DeleteSeries(It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never()); + + Mocker.GetMock() + .Verify(v => v.UpdateSeries(It.Is>(s => s.Count == monitored && s.All(m => !m.Monitored)), true), Times.Once()); + } + + [Test] + public void should_not_clean_on_clean_library_if_tvdb_match() + { + WithList(1, true); + WithCleanLevel(ListSyncLevelType.KeepAndUnmonitor); + _importListFetch.Series.ForEach(m => m.ImportListId = 1); + + Mocker.GetMock() + .Setup(v => v.Exists(6, It.IsAny())) + .Returns(true); + + Subject.Execute(_commandAll); + + Mocker.GetMock() + .Verify(v => v.UpdateSeries(It.Is>(s => s.Count > 0 && s.All(m => !m.Monitored)), true), Times.Once()); + } + + [Test] + public void should_not_clean_on_clean_library_if_imdb_match() + { + WithList(1, true); + WithCleanLevel(ListSyncLevelType.KeepAndUnmonitor); + _importListFetch.Series.ForEach(m => m.ImportListId = 1); + + var x = _importLists; + + Mocker.GetMock() + .Setup(v => v.Exists(It.IsAny(), "6")) + .Returns(true); + + Subject.Execute(_commandAll); + + Mocker.GetMock() + .Verify(v => v.UpdateSeries(It.Is>(s => s.Count > 0 && s.All(m => !m.Monitored)), true), Times.Once()); + } + + [Test] + public void should_tag_series_on_clean_library_if_config_value_keepAndTag() + { + _importListFetch.Series.ForEach(m => m.ImportListId = 1); + WithList(1, true); + WithCleanLevel(ListSyncLevelType.KeepAndTag, 1); + + Subject.Execute(_commandAll); + + Mocker.GetMock() + .Verify(v => v.GetAllSeries(), Times.Once()); + + VerifyDidAddTag(_existingSeries.Count, 1); + } + + [Test] + public void should_not_clean_if_list_failures() + { + _importListFetch.Series.ForEach(m => m.ImportListId = 1); + WithList(1, true, disabledTill: DateTime.UtcNow.AddHours(1)); + WithCleanLevel(ListSyncLevelType.LogOnly); + + Subject.Execute(_commandAll); + + Mocker.GetMock() + .Verify(v => v.GetAllSeries(), Times.Never()); + + Mocker.GetMock() + .Verify(v => v.UpdateSeries(It.IsAny>(), It.IsAny()), Times.Never()); + Mocker.GetMock() + .Verify(v => v.DeleteSeries(It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never()); + } + + [Test] + public void should_add_new_series_from_single_list_to_library() + { + _importListFetch.Series.ForEach(m => m.ImportListId = 1); + WithList(1, true); + WithCleanLevel(ListSyncLevelType.Disabled); + + Subject.Execute(_commandAll); + + Mocker.GetMock() + .Verify(v => v.AddSeries(It.Is>(s => s.Count == 1), true), Times.Once()); + } + + [Test] + public void should_add_new_series_from_multiple_list_to_library() + { + _list2Series.ForEach(m => m.ImportListId = 2); + _importListFetch.Series.ForEach(m => m.ImportListId = 1); + _importListFetch.Series.AddRange(_list2Series); + + WithList(1, true); + WithList(2, true); + + WithCleanLevel(ListSyncLevelType.Disabled); + + Subject.Execute(_commandAll); + + Mocker.GetMock() + .Verify(v => v.AddSeries(It.Is>(s => s.Count == 4), true), Times.Once()); + } + + [Test] + public void should_add_new_series_to_library_only_from_enabled_lists() + { + _list2Series.ForEach(m => m.ImportListId = 2); + _importListFetch.Series.ForEach(m => m.ImportListId = 1); + _importListFetch.Series.AddRange(_list2Series); + + WithList(1, true); + WithList(2, false); + + WithCleanLevel(ListSyncLevelType.Disabled); + + Subject.Execute(_commandAll); + + Mocker.GetMock() + .Verify(v => v.AddSeries(It.Is>(s => s.Count == 1), true), Times.Once()); + } + + [Test] + public void should_not_add_duplicate_series_from_seperate_lists() + { + _list2Series.ForEach(m => m.ImportListId = 2); + _importListFetch.Series.ForEach(m => m.ImportListId = 1); + _importListFetch.Series.AddRange(_list2Series); + _importListFetch.Series[0].TvdbId = 6; + + WithList(1, true); + WithList(2, true); + + WithCleanLevel(ListSyncLevelType.Disabled); + + Subject.Execute(_commandAll); + + Mocker.GetMock() + .Verify(v => v.AddSeries(It.Is>(s => s.Count == 3), true), Times.Once()); } [Test] public void should_search_if_series_title_and_no_series_id() { - Subject.Execute(new ImportListSyncCommand()); + _importListFetch.Series.ForEach(m => m.ImportListId = 1); + WithList(1, true); + Subject.Execute(_commandAll); Mocker.GetMock() .Verify(v => v.SearchForNewSeries(It.IsAny()), Times.Once()); @@ -105,8 +420,10 @@ namespace NzbDrone.Core.Test.ImportListTests [Test] public void should_not_search_if_series_title_and_series_id() { + _importListFetch.Series.ForEach(m => m.ImportListId = 1); + WithList(1, true); WithTvdbId(); - Subject.Execute(new ImportListSyncCommand()); + Subject.Execute(_commandAll); Mocker.GetMock() .Verify(v => v.SearchForNewSeries(It.IsAny()), Times.Never()); @@ -115,8 +432,10 @@ namespace NzbDrone.Core.Test.ImportListTests [Test] public void should_search_by_imdb_if_series_title_and_series_imdb() { + _importListFetch.Series.ForEach(m => m.ImportListId = 1); + WithList(1, true); WithImdbId(); - Subject.Execute(new ImportListSyncCommand()); + Subject.Execute(_commandAll); Mocker.GetMock() .Verify(v => v.SearchForNewSeriesByImdbId(It.IsAny()), Times.Once()); @@ -125,10 +444,12 @@ namespace NzbDrone.Core.Test.ImportListTests [Test] public void should_not_add_if_existing_series() { + _importListFetch.Series.ForEach(m => m.ImportListId = 1); + WithList(1, true); WithTvdbId(); WithExistingSeries(); - Subject.Execute(new ImportListSyncCommand()); + Subject.Execute(_commandAll); Mocker.GetMock() .Verify(v => v.AddSeries(It.Is>(t => t.Count == 0), It.IsAny())); @@ -138,10 +459,12 @@ namespace NzbDrone.Core.Test.ImportListTests [TestCase(MonitorTypes.All, true)] public void should_add_if_not_existing_series(MonitorTypes monitor, bool expectedSeriesMonitored) { + _importListFetch.Series.ForEach(m => m.ImportListId = 1); + WithList(1, true); WithTvdbId(); WithMonitorType(monitor); - Subject.Execute(new ImportListSyncCommand()); + Subject.Execute(_commandAll); Mocker.GetMock() .Verify(v => v.AddSeries(It.Is>(t => t.Count == 1 && t.First().Monitored == expectedSeriesMonitored), It.IsAny())); @@ -150,10 +473,12 @@ namespace NzbDrone.Core.Test.ImportListTests [Test] public void should_not_add_if_excluded_series() { + _importListFetch.Series.ForEach(m => m.ImportListId = 1); + WithList(1, true); WithTvdbId(); WithExcludedSeries(); - Subject.Execute(new ImportListSyncCommand()); + Subject.Execute(_commandAll); Mocker.GetMock() .Verify(v => v.AddSeries(It.Is>(t => t.Count == 0), It.IsAny())); @@ -177,7 +502,7 @@ namespace NzbDrone.Core.Test.ImportListTests { Mocker.GetMock() .Setup(v => v.Fetch()) - .Returns(new List()); + .Returns(new ImportListFetchResult()); Subject.Execute(new ImportListSyncCommand()); diff --git a/src/NzbDrone.Core/Configuration/ConfigService.cs b/src/NzbDrone.Core/Configuration/ConfigService.cs index 2193b182b..65eb7a5be 100644 --- a/src/NzbDrone.Core/Configuration/ConfigService.cs +++ b/src/NzbDrone.Core/Configuration/ConfigService.cs @@ -6,6 +6,7 @@ using NLog; using NzbDrone.Common.EnsureThat; using NzbDrone.Common.Http.Proxy; using NzbDrone.Core.Configuration.Events; +using NzbDrone.Core.ImportLists; using NzbDrone.Core.Languages; using NzbDrone.Core.MediaFiles; using NzbDrone.Core.MediaFiles.EpisodeImport; @@ -276,6 +277,18 @@ namespace NzbDrone.Core.Configuration set { SetValue("ChownGroup", value); } } + public ListSyncLevelType ListSyncLevel + { + get { return GetValueEnum("ListSyncLevel", ListSyncLevelType.Disabled); } + set { SetValue("ListSyncLevel", value); } + } + + public int ListSyncTag + { + get { return GetValueInt("ListSyncTag"); } + set { SetValue("ListSyncTag", value); } + } + public int FirstDayOfWeek { get { return GetValueInt("FirstDayOfWeek", (int)CultureInfo.CurrentCulture.DateTimeFormat.FirstDayOfWeek); } diff --git a/src/NzbDrone.Core/Configuration/IConfigService.cs b/src/NzbDrone.Core/Configuration/IConfigService.cs index 2bcd7b923..00aaf94c3 100644 --- a/src/NzbDrone.Core/Configuration/IConfigService.cs +++ b/src/NzbDrone.Core/Configuration/IConfigService.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; using NzbDrone.Common.Http.Proxy; +using NzbDrone.Core.ImportLists; using NzbDrone.Core.MediaFiles; using NzbDrone.Core.MediaFiles.EpisodeImport; using NzbDrone.Core.Qualities; @@ -52,6 +53,9 @@ namespace NzbDrone.Core.Configuration int MaximumSize { get; set; } int MinimumAge { get; set; } + ListSyncLevelType ListSyncLevel { get; set; } + int ListSyncTag { get; set; } + // UI int FirstDayOfWeek { get; set; } string CalendarWeekColumnHeader { get; set; } diff --git a/src/NzbDrone.Core/Datastore/Migration/193_add_import_list_items.cs b/src/NzbDrone.Core/Datastore/Migration/193_add_import_list_items.cs new file mode 100644 index 000000000..3f0ed5003 --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/193_add_import_list_items.cs @@ -0,0 +1,26 @@ +using FluentMigrator; +using NzbDrone.Core.Datastore.Migration.Framework; + +namespace NzbDrone.Core.Datastore.Migration +{ + [Migration(193)] + public class add_import_list_items : NzbDroneMigrationBase + { + protected override void MainDbUpgrade() + { + Create.TableForModel("ImportListItems") + .WithColumn("ImportListId").AsInt32() + .WithColumn("Title").AsString() + .WithColumn("TvdbId").AsInt32() + .WithColumn("Year").AsInt32().Nullable() + .WithColumn("TmdbId").AsInt32().Nullable() + .WithColumn("ImdbId").AsString().Nullable() + .WithColumn("MalId").AsInt32().Nullable() + .WithColumn("AniListId").AsInt32().Nullable() + .WithColumn("ReleaseDate").AsDateTimeOffset().Nullable(); + + Alter.Table("ImportListStatus") + .AddColumn("HasRemovedItemSinceLastClean").AsBoolean().WithDefaultValue(false); + } + } +} diff --git a/src/NzbDrone.Core/Datastore/TableMapping.cs b/src/NzbDrone.Core/Datastore/TableMapping.cs index 4ada8a42b..6496c5466 100644 --- a/src/NzbDrone.Core/Datastore/TableMapping.cs +++ b/src/NzbDrone.Core/Datastore/TableMapping.cs @@ -81,6 +81,9 @@ namespace NzbDrone.Core.Datastore .Ignore(i => i.MinRefreshInterval) .Ignore(i => i.Enable); + Mapper.Entity("ImportListItems").RegisterModel() + .Ignore(i => i.ImportList); + Mapper.Entity("Notifications").RegisterModel() .Ignore(x => x.ImplementationName) .Ignore(i => i.SupportsOnGrab) diff --git a/src/NzbDrone.Core/ImportLists/AniList/AniListImportBase.cs b/src/NzbDrone.Core/ImportLists/AniList/AniListImportBase.cs index b5818775b..b6bd395a3 100644 --- a/src/NzbDrone.Core/ImportLists/AniList/AniListImportBase.cs +++ b/src/NzbDrone.Core/ImportLists/AniList/AniListImportBase.cs @@ -5,7 +5,6 @@ using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.Localization; using NzbDrone.Core.Parser; -using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Validation; namespace NzbDrone.Core.ImportLists.AniList @@ -65,7 +64,7 @@ namespace NzbDrone.Core.ImportLists.AniList return new { }; } - public override IList Fetch() + public override ImportListFetchResult Fetch() { CheckToken(); return base.Fetch(); diff --git a/src/NzbDrone.Core/ImportLists/AniList/List/AniListImport.cs b/src/NzbDrone.Core/ImportLists/AniList/List/AniListImport.cs index ad62eb1a3..f2ecb26e7 100644 --- a/src/NzbDrone.Core/ImportLists/AniList/List/AniListImport.cs +++ b/src/NzbDrone.Core/ImportLists/AniList/List/AniListImport.cs @@ -44,10 +44,11 @@ namespace NzbDrone.Core.ImportLists.AniList.List return new AniListParser(Settings); } - protected override IList FetchItems(Func pageableRequestChainSelector, bool isRecent = false) + protected override ImportListFetchResult FetchItems(Func pageableRequestChainSelector, bool isRecent = false) { var releases = new List(); var url = string.Empty; + var anyFailure = true; try { @@ -77,6 +78,7 @@ namespace NzbDrone.Core.ImportLists.AniList.List while (hasNextPage); _importListStatusService.RecordSuccess(Definition.Id); + anyFailure = false; } catch (WebException webException) { @@ -149,7 +151,7 @@ namespace NzbDrone.Core.ImportLists.AniList.List _logger.Error(ex, "An error occurred while processing feed. {0}", url); } - return CleanupListItems(releases); + return new ImportListFetchResult(CleanupListItems(releases), anyFailure); } } } diff --git a/src/NzbDrone.Core/ImportLists/Custom/CustomImport.cs b/src/NzbDrone.Core/ImportLists/Custom/CustomImport.cs index 3d4645c49..f82e2f9f9 100644 --- a/src/NzbDrone.Core/ImportLists/Custom/CustomImport.cs +++ b/src/NzbDrone.Core/ImportLists/Custom/CustomImport.cs @@ -30,9 +30,10 @@ namespace NzbDrone.Core.ImportLists.Custom _customProxy = customProxy; } - public override IList Fetch() + public override ImportListFetchResult Fetch() { var series = new List(); + var anyFailure = false; try { @@ -50,12 +51,13 @@ namespace NzbDrone.Core.ImportLists.Custom } catch (Exception ex) { + anyFailure = true; _logger.Debug(ex, "Failed to fetch data for list {0} ({1})", Definition.Name, Name); _importListStatusService.RecordFailure(Definition.Id); } - return CleanupListItems(series); + return new ImportListFetchResult(CleanupListItems(series), anyFailure); } public override object RequestAction(string action, IDictionary query) diff --git a/src/NzbDrone.Core/ImportLists/FetchAndParseImportListService.cs b/src/NzbDrone.Core/ImportLists/FetchAndParseImportListService.cs index d8cb3f218..5c6250d40 100644 --- a/src/NzbDrone.Core/ImportLists/FetchAndParseImportListService.cs +++ b/src/NzbDrone.Core/ImportLists/FetchAndParseImportListService.cs @@ -4,32 +4,34 @@ using System.Linq; using System.Threading.Tasks; using NLog; using NzbDrone.Common.TPL; -using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.ImportLists.ImportListItems; namespace NzbDrone.Core.ImportLists { public interface IFetchAndParseImportList { - List Fetch(); - List FetchSingleList(ImportListDefinition definition); + ImportListFetchResult Fetch(); + ImportListFetchResult FetchSingleList(ImportListDefinition definition); } public class FetchAndParseImportListService : IFetchAndParseImportList { private readonly IImportListFactory _importListFactory; private readonly IImportListStatusService _importListStatusService; + private readonly IImportListItemService _importListItemService; private readonly Logger _logger; - public FetchAndParseImportListService(IImportListFactory importListFactory, IImportListStatusService importListStatusService, Logger logger) + public FetchAndParseImportListService(IImportListFactory importListFactory, IImportListStatusService importListStatusService, IImportListItemService importListItemService, Logger logger) { _importListFactory = importListFactory; _importListStatusService = importListStatusService; + _importListItemService = importListItemService; _logger = logger; } - public List Fetch() + public ImportListFetchResult Fetch() { - var result = new List(); + var result = new ImportListFetchResult(); var importLists = _importListFactory.AutomaticAddEnabled(); @@ -47,7 +49,7 @@ namespace NzbDrone.Core.ImportLists foreach (var importList in importLists) { var importListLocal = importList; - var importListStatus = _importListStatusService.GetLastSyncListInfo(importListLocal.Definition.Id); + var importListStatus = _importListStatusService.GetListStatus(importListLocal.Definition.Id).LastInfoSync; if (importListStatus.HasValue) { @@ -64,16 +66,23 @@ namespace NzbDrone.Core.ImportLists { try { - var importListReports = importListLocal.Fetch(); + var fetchResult = importListLocal.Fetch(); + var importListReports = fetchResult.Series; lock (result) { _logger.Debug("Found {0} reports from {1} ({2})", importListReports.Count, importList.Name, importListLocal.Definition.Name); - result.AddRange(importListReports); - } + if (!fetchResult.AnyFailure) + { + importListReports.ForEach(s => s.ImportListId = importList.Definition.Id); + result.Series.AddRange(importListReports); + var removed = _importListItemService.SyncSeriesForList(importListReports, importList.Definition.Id); + _importListStatusService.UpdateListSyncStatus(importList.Definition.Id, removed > 0); + } - _importListStatusService.UpdateListSyncStatus(importList.Definition.Id); + result.AnyFailure |= fetchResult.AnyFailure; + } } catch (Exception e) { @@ -86,16 +95,16 @@ namespace NzbDrone.Core.ImportLists Task.WaitAll(taskList.ToArray()); - result = result.DistinctBy(r => new { r.TvdbId, r.ImdbId, r.Title }).ToList(); + result.Series = result.Series.DistinctBy(r => new { r.TvdbId, r.ImdbId, r.Title }).ToList(); - _logger.Debug("Found {0} total reports from {1} lists", result.Count, importLists.Count); + _logger.Debug("Found {0} total reports from {1} lists", result.Series.Count, importLists.Count); return result; } - public List FetchSingleList(ImportListDefinition definition) + public ImportListFetchResult FetchSingleList(ImportListDefinition definition) { - var result = new List(); + var result = new ImportListFetchResult(); var importList = _importListFactory.GetInstance(definition); @@ -114,16 +123,25 @@ namespace NzbDrone.Core.ImportLists { try { - var importListReports = importListLocal.Fetch(); + var fetchResult = importListLocal.Fetch(); + var importListReports = fetchResult.Series; lock (result) { _logger.Debug("Found {0} reports from {1} ({2})", importListReports.Count, importList.Name, importListLocal.Definition.Name); - result.AddRange(importListReports); + if (!fetchResult.AnyFailure) + { + importListReports.ForEach(s => s.ImportListId = importList.Definition.Id); + result.Series.AddRange(importListReports); + var removed = _importListItemService.SyncSeriesForList(importListReports, importList.Definition.Id); + _importListStatusService.UpdateListSyncStatus(importList.Definition.Id, removed > 0); + } + + result.AnyFailure |= fetchResult.AnyFailure; } - _importListStatusService.UpdateListSyncStatus(importList.Definition.Id); + result.AnyFailure |= fetchResult.AnyFailure; } catch (Exception e) { @@ -135,8 +153,6 @@ namespace NzbDrone.Core.ImportLists Task.WaitAll(taskList.ToArray()); - result = result.DistinctBy(r => new { r.TvdbId, r.ImdbId, r.Title }).ToList(); - return result; } } diff --git a/src/NzbDrone.Core/ImportLists/HttpImportListBase.cs b/src/NzbDrone.Core/ImportLists/HttpImportListBase.cs index 25057c2fd..ebb948a89 100644 --- a/src/NzbDrone.Core/ImportLists/HttpImportListBase.cs +++ b/src/NzbDrone.Core/ImportLists/HttpImportListBase.cs @@ -38,15 +38,16 @@ namespace NzbDrone.Core.ImportLists _httpClient = httpClient; } - public override IList Fetch() + public override ImportListFetchResult Fetch() { return FetchItems(g => g.GetListItems(), true); } - protected virtual IList FetchItems(Func pageableRequestChainSelector, bool isRecent = false) + protected virtual ImportListFetchResult FetchItems(Func pageableRequestChainSelector, bool isRecent = false) { var releases = new List(); var url = string.Empty; + var anyFailure = true; try { @@ -92,6 +93,7 @@ namespace NzbDrone.Core.ImportLists } _importListStatusService.RecordSuccess(Definition.Id); + anyFailure = false; } catch (WebException webException) { @@ -163,7 +165,7 @@ namespace NzbDrone.Core.ImportLists _logger.Error(ex, "An error occurred while processing feed. {0}", url); } - return CleanupListItems(releases); + return new ImportListFetchResult(CleanupListItems(releases), anyFailure); } protected virtual bool IsValidItem(ImportListItemInfo listItem) diff --git a/src/NzbDrone.Core/ImportLists/IImportList.cs b/src/NzbDrone.Core/ImportLists/IImportList.cs index 4970c5261..43047f97a 100644 --- a/src/NzbDrone.Core/ImportLists/IImportList.cs +++ b/src/NzbDrone.Core/ImportLists/IImportList.cs @@ -1,6 +1,4 @@ using System; -using System.Collections.Generic; -using NzbDrone.Core.Parser.Model; using NzbDrone.Core.ThingiProvider; namespace NzbDrone.Core.ImportLists @@ -9,6 +7,6 @@ namespace NzbDrone.Core.ImportLists { ImportListType ListType { get; } TimeSpan MinRefreshInterval { get; } - IList Fetch(); + ImportListFetchResult Fetch(); } } diff --git a/src/NzbDrone.Core/ImportLists/ImportListBase.cs b/src/NzbDrone.Core/ImportLists/ImportListBase.cs index 5806e3095..9011a9213 100644 --- a/src/NzbDrone.Core/ImportLists/ImportListBase.cs +++ b/src/NzbDrone.Core/ImportLists/ImportListBase.cs @@ -11,6 +11,23 @@ using NzbDrone.Core.ThingiProvider; namespace NzbDrone.Core.ImportLists { + public class ImportListFetchResult + { + public ImportListFetchResult() + { + Series = new List(); + } + + public ImportListFetchResult(IEnumerable series, bool anyFailure) + { + Series = series.ToList(); + AnyFailure = anyFailure; + } + + public List Series { get; set; } + public bool AnyFailure { get; set; } + } + public abstract class ImportListBase : IImportList where TSettings : IImportListSettings, new() { @@ -63,7 +80,7 @@ namespace NzbDrone.Core.ImportLists protected TSettings Settings => (TSettings)Definition.Settings; - public abstract IList Fetch(); + public abstract ImportListFetchResult Fetch(); protected virtual IList CleanupListItems(IEnumerable releases) { diff --git a/src/NzbDrone.Core/ImportLists/ImportListItems/ImportListItemRepository.cs b/src/NzbDrone.Core/ImportLists/ImportListItems/ImportListItemRepository.cs new file mode 100644 index 000000000..abe546026 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/ImportListItems/ImportListItemRepository.cs @@ -0,0 +1,43 @@ +using System.Collections.Generic; +using System.Linq; +using NzbDrone.Core.Datastore; +using NzbDrone.Core.Messaging.Events; +using NzbDrone.Core.Parser.Model; + +namespace NzbDrone.Core.ImportLists.ImportListItems +{ + public interface IImportListItemInfoRepository : IBasicRepository + { + List GetAllForLists(List listIds); + bool Exists(int tvdbId, string imdbId); + } + + public class ImportListItemRepository : BasicRepository, IImportListItemInfoRepository + { + public ImportListItemRepository(IMainDatabase database, IEventAggregator eventAggregator) + : base(database, eventAggregator) + { + } + + public List GetAllForLists(List listIds) + { + return Query(x => listIds.Contains(x.ImportListId)); + } + + public bool Exists(int tvdbId, string imdbId) + { + List items; + + if (string.IsNullOrWhiteSpace(imdbId)) + { + items = Query(x => x.TvdbId == tvdbId); + } + else + { + items = Query(x => x.TvdbId == tvdbId || x.ImdbId == imdbId); + } + + return items.Any(); + } + } +} diff --git a/src/NzbDrone.Core/ImportLists/ImportListItems/ImportListItemService.cs b/src/NzbDrone.Core/ImportLists/ImportListItems/ImportListItemService.cs new file mode 100644 index 000000000..852a30ee5 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/ImportListItems/ImportListItemService.cs @@ -0,0 +1,59 @@ +using System.Collections.Generic; +using System.Linq; +using NLog; +using NzbDrone.Core.Messaging.Events; +using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.ThingiProvider.Events; + +namespace NzbDrone.Core.ImportLists.ImportListItems +{ + public interface IImportListItemService + { + List GetAllForLists(List listIds); + int SyncSeriesForList(List listSeries, int listId); + bool Exists(int tvdbId, string imdbId); + } + + public class ImportListItemService : IImportListItemService, IHandleAsync> + { + private readonly IImportListItemInfoRepository _importListSeriesRepository; + private readonly Logger _logger; + + public ImportListItemService(IImportListItemInfoRepository importListSeriesRepository, + Logger logger) + { + _importListSeriesRepository = importListSeriesRepository; + _logger = logger; + } + + public int SyncSeriesForList(List listSeries, int listId) + { + var existingListSeries = GetAllForLists(new List { listId }); + + listSeries.ForEach(l => l.Id = existingListSeries.FirstOrDefault(e => e.TvdbId == l.TvdbId)?.Id ?? 0); + + _importListSeriesRepository.InsertMany(listSeries.Where(l => l.Id == 0).ToList()); + _importListSeriesRepository.UpdateMany(listSeries.Where(l => l.Id > 0).ToList()); + var toDelete = existingListSeries.Where(l => !listSeries.Any(x => x.TvdbId == l.TvdbId)).ToList(); + _importListSeriesRepository.DeleteMany(toDelete); + + return toDelete.Count; + } + + public List GetAllForLists(List listIds) + { + return _importListSeriesRepository.GetAllForLists(listIds).ToList(); + } + + public void HandleAsync(ProviderDeletedEvent message) + { + var seriesOnList = _importListSeriesRepository.GetAllForLists(new List { message.ProviderId }); + _importListSeriesRepository.DeleteMany(seriesOnList); + } + + public bool Exists(int tvdbId, string imdbId) + { + return _importListSeriesRepository.Exists(tvdbId, imdbId); + } + } +} diff --git a/src/NzbDrone.Core/ImportLists/ImportListStatus.cs b/src/NzbDrone.Core/ImportLists/ImportListStatus.cs index 60d6aa3a0..b7eeabe7b 100644 --- a/src/NzbDrone.Core/ImportLists/ImportListStatus.cs +++ b/src/NzbDrone.Core/ImportLists/ImportListStatus.cs @@ -6,5 +6,6 @@ namespace NzbDrone.Core.ImportLists public class ImportListStatus : ProviderStatusBase { public DateTime? LastInfoSync { get; set; } + public bool HasRemovedItemSinceLastClean { get; set; } } } diff --git a/src/NzbDrone.Core/ImportLists/ImportListStatusService.cs b/src/NzbDrone.Core/ImportLists/ImportListStatusService.cs index bbef4b179..35005d145 100644 --- a/src/NzbDrone.Core/ImportLists/ImportListStatusService.cs +++ b/src/NzbDrone.Core/ImportLists/ImportListStatusService.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using NLog; using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Core.Messaging.Events; @@ -8,9 +9,10 @@ namespace NzbDrone.Core.ImportLists { public interface IImportListStatusService : IProviderStatusServiceBase { - DateTime? GetLastSyncListInfo(int importListId); + ImportListStatus GetListStatus(int importListId); - void UpdateListSyncStatus(int importListId); + void UpdateListSyncStatus(int importListId, bool removedItems); + void MarkListsAsCleaned(); } public class ImportListStatusService : ProviderStatusServiceBase, IImportListStatusService @@ -20,21 +22,38 @@ namespace NzbDrone.Core.ImportLists { } - public DateTime? GetLastSyncListInfo(int importListId) + public ImportListStatus GetListStatus(int importListId) { - return GetProviderStatus(importListId).LastInfoSync; + return GetProviderStatus(importListId); } - public void UpdateListSyncStatus(int importListId) + public void UpdateListSyncStatus(int importListId, bool removedItems) { lock (_syncRoot) { var status = GetProviderStatus(importListId); status.LastInfoSync = DateTime.UtcNow; + status.HasRemovedItemSinceLastClean |= removedItems; _providerStatusRepository.Upsert(status); } } + + public void MarkListsAsCleaned() + { + lock (_syncRoot) + { + var toUpdate = new List(); + + foreach (var status in _providerStatusRepository.All()) + { + status.HasRemovedItemSinceLastClean = false; + toUpdate.Add(status); + } + + _providerStatusRepository.UpdateMany(toUpdate); + } + } } } diff --git a/src/NzbDrone.Core/ImportLists/ImportListSyncService.cs b/src/NzbDrone.Core/ImportLists/ImportListSyncService.cs index 12109a94a..f6d414562 100644 --- a/src/NzbDrone.Core/ImportLists/ImportListSyncService.cs +++ b/src/NzbDrone.Core/ImportLists/ImportListSyncService.cs @@ -3,41 +3,85 @@ using System.Linq; using NLog; using NzbDrone.Common.Extensions; using NzbDrone.Common.Instrumentation.Extensions; +using NzbDrone.Core.Configuration; using NzbDrone.Core.ImportLists.Exclusions; +using NzbDrone.Core.ImportLists.ImportListItems; +using NzbDrone.Core.Jobs; using NzbDrone.Core.Messaging.Commands; +using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.MetadataSource; using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.ThingiProvider.Events; using NzbDrone.Core.Tv; namespace NzbDrone.Core.ImportLists { - public class ImportListSyncService : IExecute + public class ImportListSyncService : IExecute, IHandleAsync> { private readonly IImportListFactory _importListFactory; + private readonly IImportListStatusService _importListStatusService; private readonly IImportListExclusionService _importListExclusionService; + private readonly IImportListItemService _importListItemService; private readonly IFetchAndParseImportList _listFetcherAndParser; private readonly ISearchForNewSeries _seriesSearchService; private readonly ISeriesService _seriesService; private readonly IAddSeriesService _addSeriesService; + private readonly IConfigService _configService; + private readonly ITaskManager _taskManager; private readonly Logger _logger; public ImportListSyncService(IImportListFactory importListFactory, + IImportListStatusService importListStatusService, IImportListExclusionService importListExclusionService, + IImportListItemService importListItemService, IFetchAndParseImportList listFetcherAndParser, ISearchForNewSeries seriesSearchService, ISeriesService seriesService, IAddSeriesService addSeriesService, + IConfigService configService, + ITaskManager taskManager, Logger logger) { _importListFactory = importListFactory; + _importListStatusService = importListStatusService; _importListExclusionService = importListExclusionService; + _importListItemService = importListItemService; _listFetcherAndParser = listFetcherAndParser; _seriesSearchService = seriesSearchService; _seriesService = seriesService; _addSeriesService = addSeriesService; + _configService = configService; + _taskManager = taskManager; _logger = logger; } + private bool AllListsSuccessfulWithAPendingClean() + { + var lists = _importListFactory.AutomaticAddEnabled(false); + var anyRemoved = false; + + foreach (var list in lists) + { + var status = _importListStatusService.GetListStatus(list.Definition.Id); + + if (status.DisabledTill.HasValue) + { + // list failed the last time it was synced. + return false; + } + + if (!status.LastInfoSync.HasValue) + { + // list has never been synced. + return false; + } + + anyRemoved |= status.HasRemovedItemSinceLastClean; + } + + return anyRemoved; + } + private void SyncAll() { if (_importListFactory.AutomaticAddEnabled().Empty()) @@ -49,18 +93,26 @@ namespace NzbDrone.Core.ImportLists _logger.ProgressInfo("Starting Import List Sync"); - var listItems = _listFetcherAndParser.Fetch().ToList(); + var result = _listFetcherAndParser.Fetch(); + + var listItems = result.Series.ToList(); ProcessListItems(listItems); + + TryCleanLibrary(); } private void SyncList(ImportListDefinition definition) { _logger.ProgressInfo(string.Format("Starting Import List Refresh for List {0}", definition.Name)); - var listItems = _listFetcherAndParser.FetchSingleList(definition).ToList(); + var result = _listFetcherAndParser.FetchSingleList(definition); + + var listItems = result.Series.ToList(); ProcessListItems(listItems); + + TryCleanLibrary(); } private void ProcessListItems(List items) @@ -90,6 +142,11 @@ namespace NzbDrone.Core.ImportLists var importList = importLists.Single(x => x.Id == item.ImportListId); + if (!importList.EnableAutomaticAdd) + { + continue; + } + // Map by IMDb ID if we have it if (item.TvdbId <= 0 && item.ImdbId.IsNotNullOrWhiteSpace()) { @@ -180,10 +237,10 @@ namespace NzbDrone.Core.ImportLists SeasonFolder = importList.SeasonFolder, Tags = importList.Tags, AddOptions = new AddSeriesOptions - { - SearchForMissingEpisodes = importList.SearchForMissingEpisodes, - Monitor = importList.ShouldMonitor - } + { + SearchForMissingEpisodes = importList.SearchForMissingEpisodes, + Monitor = importList.ShouldMonitor + } }); } } @@ -206,5 +263,64 @@ namespace NzbDrone.Core.ImportLists SyncAll(); } } + + private void TryCleanLibrary() + { + if (_configService.ListSyncLevel == ListSyncLevelType.Disabled) + { + return; + } + + if (AllListsSuccessfulWithAPendingClean()) + { + CleanLibrary(); + } + } + + private void CleanLibrary() + { + if (_configService.ListSyncLevel == ListSyncLevelType.Disabled) + { + return; + } + + var seriesToUpdate = new List(); + var seriesInLibrary = _seriesService.GetAllSeries(); + + foreach (var series in seriesInLibrary) + { + var seriesExists = _importListItemService.Exists(series.TvdbId, series.ImdbId); + + if (!seriesExists) + { + switch (_configService.ListSyncLevel) + { + case ListSyncLevelType.LogOnly: + _logger.Info("{0} was in your library, but not found in your lists --> You might want to unmonitor or remove it", series); + break; + case ListSyncLevelType.KeepAndUnmonitor when series.Monitored: + _logger.Info("{0} was in your library, but not found in your lists --> Keeping in library but unmonitoring it", series); + series.Monitored = false; + seriesToUpdate.Add(series); + break; + case ListSyncLevelType.KeepAndTag when !series.Tags.Contains(_configService.ListSyncTag): + _logger.Info("{0} was in your library, but not found in your lists --> Keeping in library but tagging it", series); + series.Tags.Add(_configService.ListSyncTag); + seriesToUpdate.Add(series); + break; + default: + break; + } + } + } + + _seriesService.UpdateSeries(seriesToUpdate, true); + _importListStatusService.MarkListsAsCleaned(); + } + + public void HandleAsync(ProviderDeletedEvent message) + { + TryCleanLibrary(); + } } } diff --git a/src/NzbDrone.Core/ImportLists/ListSyncLevelType.cs b/src/NzbDrone.Core/ImportLists/ListSyncLevelType.cs new file mode 100644 index 000000000..cdbf90c41 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/ListSyncLevelType.cs @@ -0,0 +1,10 @@ +namespace NzbDrone.Core.ImportLists +{ + public enum ListSyncLevelType + { + Disabled, + LogOnly, + KeepAndUnmonitor, + KeepAndTag + } +} diff --git a/src/NzbDrone.Core/ImportLists/Plex/PlexImport.cs b/src/NzbDrone.Core/ImportLists/Plex/PlexImport.cs index 393bdb692..ee50b3540 100644 --- a/src/NzbDrone.Core/ImportLists/Plex/PlexImport.cs +++ b/src/NzbDrone.Core/ImportLists/Plex/PlexImport.cs @@ -8,7 +8,6 @@ using NzbDrone.Core.Exceptions; using NzbDrone.Core.Localization; using NzbDrone.Core.Notifications.Plex.PlexTv; using NzbDrone.Core.Parser; -using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Validation; namespace NzbDrone.Core.ImportLists.Plex @@ -35,7 +34,7 @@ namespace NzbDrone.Core.ImportLists.Plex public override string Name => _localizationService.GetLocalizedString("ImportListsPlexSettingsWatchlistName"); public override int PageSize => 50; - public override IList Fetch() + public override ImportListFetchResult Fetch() { Settings.Validate().Filter("AccessToken").ThrowOnError(); diff --git a/src/NzbDrone.Core/ImportLists/Rss/RssImportBase.cs b/src/NzbDrone.Core/ImportLists/Rss/RssImportBase.cs index ef6b6a7ef..f00a66253 100644 --- a/src/NzbDrone.Core/ImportLists/Rss/RssImportBase.cs +++ b/src/NzbDrone.Core/ImportLists/Rss/RssImportBase.cs @@ -1,11 +1,9 @@ using System; -using System.Collections.Generic; using NLog; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.Localization; using NzbDrone.Core.Parser; -using NzbDrone.Core.Parser.Model; namespace NzbDrone.Core.ImportLists.Rss { @@ -26,7 +24,7 @@ namespace NzbDrone.Core.ImportLists.Rss { } - public override IList Fetch() + public override ImportListFetchResult Fetch() { return FetchItems(g => g.GetListItems()); } diff --git a/src/NzbDrone.Core/ImportLists/Simkl/SimklImportBase.cs b/src/NzbDrone.Core/ImportLists/Simkl/SimklImportBase.cs index 8592fb1cc..bc0240c07 100644 --- a/src/NzbDrone.Core/ImportLists/Simkl/SimklImportBase.cs +++ b/src/NzbDrone.Core/ImportLists/Simkl/SimklImportBase.cs @@ -36,7 +36,7 @@ namespace NzbDrone.Core.ImportLists.Simkl _importListRepository = netImportRepository; } - public override IList Fetch() + public override ImportListFetchResult Fetch() { Settings.Validate().Filter("AccessToken", "RefreshToken").ThrowOnError(); _logger.Trace($"Access token expires at {Settings.Expires}"); @@ -47,13 +47,14 @@ namespace NzbDrone.Core.ImportLists.Simkl RefreshToken(); } - var lastFetch = _importListStatusService.GetLastSyncListInfo(Definition.Id); + var lastFetch = _importListStatusService.GetListStatus(Definition.Id).LastInfoSync; var lastActivity = GetLastActivity(); // Check to see if user has any activity since last sync, if not return empty to avoid work if (lastFetch.HasValue && lastActivity < lastFetch.Value.AddHours(-2)) { - return Array.Empty(); + // mark failure to avoid deleting series due to emptyness + return new ImportListFetchResult(new List(), true); } var generator = GetRequestGenerator(); diff --git a/src/NzbDrone.Core/ImportLists/Sonarr/SonarrImport.cs b/src/NzbDrone.Core/ImportLists/Sonarr/SonarrImport.cs index 369b10e52..3a35ecf6d 100644 --- a/src/NzbDrone.Core/ImportLists/Sonarr/SonarrImport.cs +++ b/src/NzbDrone.Core/ImportLists/Sonarr/SonarrImport.cs @@ -31,10 +31,10 @@ namespace NzbDrone.Core.ImportLists.Sonarr _sonarrV3Proxy = sonarrV3Proxy; } - public override IList Fetch() + public override ImportListFetchResult Fetch() { var series = new List(); - + var anyFailure = false; try { var remoteSeries = _sonarrV3Proxy.GetSeries(Settings); @@ -75,9 +75,10 @@ namespace NzbDrone.Core.ImportLists.Sonarr _logger.Debug(ex, "Failed to fetch data for list {0} ({1})", Definition.Name, Name); _importListStatusService.RecordFailure(Definition.Id); + anyFailure = true; } - return CleanupListItems(series); + return new ImportListFetchResult(CleanupListItems(series), anyFailure); } public override object RequestAction(string action, IDictionary query) diff --git a/src/NzbDrone.Core/ImportLists/Trakt/TraktImportBase.cs b/src/NzbDrone.Core/ImportLists/Trakt/TraktImportBase.cs index 848fe9553..b7397138c 100644 --- a/src/NzbDrone.Core/ImportLists/Trakt/TraktImportBase.cs +++ b/src/NzbDrone.Core/ImportLists/Trakt/TraktImportBase.cs @@ -6,7 +6,6 @@ using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.Localization; using NzbDrone.Core.Parser; -using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Validation; namespace NzbDrone.Core.ImportLists.Trakt @@ -36,7 +35,7 @@ namespace NzbDrone.Core.ImportLists.Trakt _importListRepository = netImportRepository; } - public override IList Fetch() + public override ImportListFetchResult Fetch() { Settings.Validate().Filter("AccessToken", "RefreshToken").ThrowOnError(); _logger.Trace($"Access token expires at {Settings.Expires}"); diff --git a/src/NzbDrone.Core/ImportLists/Trakt/TraktParser.cs b/src/NzbDrone.Core/ImportLists/Trakt/TraktParser.cs index ffde5a462..46f41b6d5 100644 --- a/src/NzbDrone.Core/ImportLists/Trakt/TraktParser.cs +++ b/src/NzbDrone.Core/ImportLists/Trakt/TraktParser.cs @@ -35,7 +35,8 @@ namespace NzbDrone.Core.ImportLists.Trakt series.AddIfNotNull(new ImportListItemInfo() { Title = traktResponse.Show.Title, - TvdbId = traktResponse.Show.Ids.Tvdb.GetValueOrDefault() + TvdbId = traktResponse.Show.Ids.Tvdb.GetValueOrDefault(), + ImdbId = traktResponse.Show.Ids.Imdb }); } diff --git a/src/NzbDrone.Core/Localization/Core/en.json b/src/NzbDrone.Core/Localization/Core/en.json index f5e78bbd4..153a894cd 100644 --- a/src/NzbDrone.Core/Localization/Core/en.json +++ b/src/NzbDrone.Core/Localization/Core/en.json @@ -208,6 +208,7 @@ "ChownGroup": "chown Group", "ChownGroupHelpText": "Group name or gid. Use gid for remote file systems.", "ChownGroupHelpTextWarning": "This only works if the user running {appName} is the owner of the file. It's better to ensure the download client uses the same group as {appName}.", + "CleanLibraryLevel": "Clean Library Level", "Clear": "Clear", "ClearBlocklist": "Clear blocklist", "ClearBlocklistMessageText": "Are you sure you want to clear all items from the blocklist?", @@ -790,6 +791,7 @@ "ImportListSearchForMissingEpisodes": "Search for Missing Episodes", "ImportListSearchForMissingEpisodesHelpText": "After series is added to {appName} automatically search for missing episodes", "ImportListSettings": "Import List Settings", + "ImportListStatusAllPossiblePartialFetchHealthCheckMessage": "All lists require manual interaction due to possible partial fetches", "ImportListStatusAllUnavailableHealthCheckMessage": "All lists are unavailable due to failures", "ImportListStatusUnavailableHealthCheckMessage": "Lists unavailable due to failures: {importListNames}", "ImportLists": "Import Lists", @@ -1010,6 +1012,8 @@ "Interval": "Interval", "InvalidFormat": "Invalid Format", "InvalidUILanguage": "Your UI is set to an invalid language, correct it and save your settings", + "KeepAndTagSeries": "Keep and Tag Series", + "KeepAndUnmonitorSeries": "Keep and Unmonitor Series", "KeyboardShortcuts": "Keyboard Shortcuts", "KeyboardShortcutsCloseModal": "Close Current Modal", "KeyboardShortcutsConfirmModal": "Accept Confirmation Modal", @@ -1038,6 +1042,9 @@ "ListOptionsLoadError": "Unable to load list options", "ListQualityProfileHelpText": "Quality Profile list items will be added with", "ListRootFolderHelpText": "Root Folder list items will be added to", + "ListSyncLevelHelpText": "Series in library will be handled based on your selection if they fall off or do not appear on your list(s)", + "ListSyncTag": "List Sync Tag", + "ListSyncTagHelpText": "This tag will be added when a series falls off or is no longer on your list(s)", "ListTagsHelpText": "Tags that will be added on import from this list", "ListWillRefreshEveryInterval": "List will refresh every {refreshInterval}", "ListsLoadError": "Unable to load Lists", @@ -1050,6 +1057,7 @@ "LogFilesLocation": "Log files are located in: {location}", "LogLevel": "Log Level", "LogLevelTraceHelpTextWarning": "Trace logging should only be enabled temporarily", + "LogOnly": "Log Only", "Logging": "Logging", "Logout": "Logout", "Logs": "Logs", @@ -1946,6 +1954,7 @@ "Umask777Description": "{octal} - Everyone write", "UnableToLoadAutoTagging": "Unable to load auto tagging", "UnableToLoadBackups": "Unable to load backups", + "UnableToLoadListOptions": "Unable to load list options", "UnableToUpdateSonarrDirectly": "Unable to update {appName} directly,", "Unavailable": "Unavailable", "Underscore": "Underscore", diff --git a/src/NzbDrone.Core/Parser/Model/ImportListItemInfo.cs b/src/NzbDrone.Core/Parser/Model/ImportListItemInfo.cs index 7c521a4e6..8d0e37534 100644 --- a/src/NzbDrone.Core/Parser/Model/ImportListItemInfo.cs +++ b/src/NzbDrone.Core/Parser/Model/ImportListItemInfo.cs @@ -1,8 +1,9 @@ using System; +using NzbDrone.Core.Datastore; namespace NzbDrone.Core.Parser.Model { - public class ImportListItemInfo + public class ImportListItemInfo : ModelBase { public int ImportListId { get; set; } public string ImportList { get; set; } diff --git a/src/Sonarr.Api.V3/Config/ImportListConfigController.cs b/src/Sonarr.Api.V3/Config/ImportListConfigController.cs new file mode 100644 index 000000000..dba64a441 --- /dev/null +++ b/src/Sonarr.Api.V3/Config/ImportListConfigController.cs @@ -0,0 +1,27 @@ +using FluentValidation; +using NzbDrone.Core.Configuration; +using NzbDrone.Core.ImportLists; +using NzbDrone.Core.Validation; +using Sonarr.Http; + +namespace Sonarr.Api.V3.Config +{ + [V3ApiController("config/importlist")] + + public class ImportListConfigController : ConfigController + { + public ImportListConfigController(IConfigService configService) + : base(configService) + { + SharedValidator.RuleFor(x => x.ListSyncTag) + .ValidId() + .WithMessage("Tag must be specified") + .When(x => x.ListSyncLevel == ListSyncLevelType.KeepAndTag); + } + + protected override ImportListConfigResource ToResource(IConfigService model) + { + return ImportListConfigResourceMapper.ToResource(model); + } + } +} diff --git a/src/Sonarr.Api.V3/Config/ImportListConfigResource.cs b/src/Sonarr.Api.V3/Config/ImportListConfigResource.cs new file mode 100644 index 000000000..b3f80af54 --- /dev/null +++ b/src/Sonarr.Api.V3/Config/ImportListConfigResource.cs @@ -0,0 +1,24 @@ +using NzbDrone.Core.Configuration; +using NzbDrone.Core.ImportLists; +using Sonarr.Http.REST; + +namespace Sonarr.Api.V3.Config +{ + public class ImportListConfigResource : RestResource + { + public ListSyncLevelType ListSyncLevel { get; set; } + public int ListSyncTag { get; set; } + } + + public static class ImportListConfigResourceMapper + { + public static ImportListConfigResource ToResource(IConfigService model) + { + return new ImportListConfigResource + { + ListSyncLevel = model.ListSyncLevel, + ListSyncTag = model.ListSyncTag, + }; + } + } +}