Mirrors/Radarr
1
0
Fork 0
mirror of https://github.com/Radarr/Radarr.git synced 2024-11-05 02:22:31 +01:00
Code Issues Releases Wiki Activity

Fixed: Prevent duplicate parsing of extra files

Browse Source
This commit is contained in:
Mark McDowall 2016-09-18 11:04:56 -07:00
parent 2e96c4e798
commit a621f0d49b
8 changed files with 81 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/NzbDrone.Core/Extras/ExistingExtraFileService.cs
View File

@ -14,18 +14,18 @@ namespace NzbDrone.Core.Extras
public class ExistingExtraFileService : IHandle<SeriesScannedEvent>
{
private readonly IDiskProvider _diskProvider;
private readonly IDiskScanService _diskScanService;
private readonly List<IImportExistingExtraFiles> _existingExtraFileImporters;
private readonly List<IManageExtraFiles> _extraFileManagers;
private readonly Logger _logger;
public ExistingExtraFileService(IDiskProvider diskProvider,
IDiskScanService diskScanService,
List<IImportExistingExtraFiles> existingExtraFileImporters,
List<IManageExtraFiles> extraFileManagers,
Logger logger)
{
_diskProvider = diskProvider;
_diskScanService = diskScanService;
_existingExtraFileImporters = existingExtraFileImporters.OrderBy(e => e.Order).ToList();
_extraFileManagers = extraFileManagers.OrderBy(e => e.Order).ToList();
_logger = logger;
}
@ -41,9 +41,8 @@ public void Handle(SeriesScannedEvent message)
_logger.Debug("Looking for existing extra files in {0}", series.Path);
var filesOnDisk = _diskProvider.GetFiles(series.Path, SearchOption.AllDirectories);
var possibleExtraFiles = filesOnDisk.Where(c => !MediaFileExtensions.Extensions.Contains(Path.GetExtension(c).ToLower()) &&
!c.StartsWith(Path.Combine(series.Path, "EXTRAS"))).ToList();
var filesOnDisk = _diskScanService.GetNonVideoFiles(series.Path);
var possibleExtraFiles = _diskScanService.FilterFiles(series, filesOnDisk);
var filteredFiles = possibleExtraFiles;
var importedFiles = new List<string>();
@ -55,7 +54,7 @@ public void Handle(SeriesScannedEvent message)
importedFiles.AddRange(imported.Select(f => Path.Combine(series.Path, f.RelativePath)));
}
_logger.Info("Found {0} extra files", extraFiles);
_logger.Info("Found {0} extra files", extraFiles.Count);
}
}
}

18
src/NzbDrone.Core/Extras/ImportExistingExtraFileFilterResult.cs Normal file
View File

@ -0,0 +1,18 @@
using System.Collections.Generic;
using NzbDrone.Core.Extras.Files;
namespace NzbDrone.Core.Extras
{
public class ImportExistingExtraFileFilterResult<TExtraFile>
where TExtraFile : ExtraFile, new()
{
public ImportExistingExtraFileFilterResult(List<TExtraFile> previouslyImported, List<string> filesOnDisk)
{
PreviouslyImported = previouslyImported;
FilesOnDisk = filesOnDisk;
}
public List<TExtraFile> PreviouslyImported { get; set; }
public List<string> FilesOnDisk { get; set; }
}
}

14
src/NzbDrone.Core/Extras/ImportExistingExtraFilesBase.cs
View File

@ -21,7 +21,7 @@ public ImportExistingExtraFilesBase(IExtraFileService<TExtraFile> extraFileServi
public abstract int Order { get; }
public abstract IEnumerable<ExtraFile> ProcessFiles(Series series, List<string> filesOnDisk, List<string> importedFiles);
public virtual List<string> FilterAndClean(Series series, List<string> filesOnDisk, List<string> importedFiles)
public virtual ImportExistingExtraFileFilterResult<TExtraFile> FilterAndClean(Series series, List<string> filesOnDisk, List<string> importedFiles)
{
var seriesFiles = _extraFileService.GetFilesBySeries(series.Id);
@ -30,12 +30,16 @@ public virtual List<string> FilterAndClean(Series series, List<string> filesOnDi
return Filter(series, filesOnDisk, importedFiles, seriesFiles);
}
private List<string> Filter(Series series, List<string> filesOnDisk, List<string> importedFiles, List<TExtraFile> seriesFiles)
private ImportExistingExtraFileFilterResult<TExtraFile> Filter(Series series, List<string> filesOnDisk, List<string> importedFiles, List<TExtraFile> seriesFiles)
{
var filteredFiles = filesOnDisk;
var previouslyImported = seriesFiles.IntersectBy(s => Path.Combine(series.Path, s.RelativePath), filesOnDisk, f => f, PathEqualityComparer.Instance).ToList();
var filteredFiles = filesOnDisk.Except(previouslyImported.Select(f => Path.Combine(series.Path, f.RelativePath)).ToList(), PathEqualityComparer.Instance)
.Except(importedFiles, PathEqualityComparer.Instance)
.ToList();
filteredFiles = filteredFiles.Except(seriesFiles.Select(f => Path.Combine(series.Path, f.RelativePath)).ToList(), PathEqualityComparer.Instance).ToList();
return filteredFiles.Except(importedFiles, PathEqualityComparer.Instance).ToList();
// Return files that are already imported so they aren't imported again by other importers.
// Filter out files that were previously imported and as well as ones imported by other importers.
return new ImportExistingExtraFileFilterResult<TExtraFile>(previouslyImported, filteredFiles);
}
private void Clean(Series series, List<string> filesOnDisk, List<string> importedFiles, List<TExtraFile> seriesFiles)

17
src/NzbDrone.Core/Extras/MetaData/ExistingMetadataImporter.cs
View File

@ -5,6 +5,7 @@
using NzbDrone.Common.Extensions;
using NzbDrone.Core.Extras.Files;
using NzbDrone.Core.Extras.Metadata.Files;
using NzbDrone.Core.Extras.Subtitles;
using NzbDrone.Core.Parser;
using NzbDrone.Core.Tv;
@ -42,10 +43,17 @@ public override IEnumerable<ExtraFile> ProcessFiles(Series series, List<string>
_logger.Debug("Looking for existing metadata in {0}", series.Path);
var metadataFiles = new List<MetadataFile>();
var filteredFiles = FilterAndClean(series, filesOnDisk, importedFiles);
var filterResult = FilterAndClean(series, filesOnDisk, importedFiles);
foreach (var possibleMetadataFile in filteredFiles)
foreach (var possibleMetadataFile in filterResult.FilesOnDisk)
{
// Don't process files that have known Subtitle file extensions (saves a bit of unecessary processing)
if (SubtitleFileExtensions.Extensions.Contains(Path.GetExtension(possibleMetadataFile)))
{
continue;
}
foreach (var consumer in _consumers)
{
var metadata = consumer.FindMetadataFile(series, possibleMetadataFile);
@ -90,7 +98,10 @@ public override IEnumerable<ExtraFile> ProcessFiles(Series series, List<string>
_logger.Info("Found {0} existing metadata files", metadataFiles.Count);
_metadataFileService.Upsert(metadataFiles);
return metadataFiles;
// Return files that were just imported along with files that were
// previously imported so previously imported files aren't imported twice
return metadataFiles.Concat(filterResult.PreviouslyImported);
}
}
}

9
src/NzbDrone.Core/Extras/Others/ExistingOtherExtraImporter.cs
View File

@ -38,9 +38,9 @@ public override IEnumerable<ExtraFile> ProcessFiles(Series series, List<string>
_logger.Debug("Looking for existing extra files in {0}", series.Path);
var extraFiles = new List<OtherExtraFile>();
var filteredFiles = FilterAndClean(series, filesOnDisk, importedFiles);
var filterResult = FilterAndClean(series, filesOnDisk, importedFiles);
foreach (var possibleExtraFile in filteredFiles)
foreach (var possibleExtraFile in filterResult.FilesOnDisk)
{
var localEpisode = _parsingService.GetLocalEpisode(possibleExtraFile, series);
@ -77,7 +77,10 @@ public override IEnumerable<ExtraFile> ProcessFiles(Series series, List<string>
_logger.Info("Found {0} existing other extra files", extraFiles.Count);
_otherExtraFileService.Upsert(extraFiles);
return extraFiles;
// Return files that were just imported along with files that were
// previously imported so previously imported files aren't imported twice
return extraFiles.Concat(filterResult.PreviouslyImported);
}
}
}

9
src/NzbDrone.Core/Extras/Subtitles/ExistingSubtitleImporter.cs
View File

@ -38,9 +38,9 @@ public override IEnumerable<ExtraFile> ProcessFiles(Series series, List<string>
_logger.Debug("Looking for existing subtitle files in {0}", series.Path);
var subtitleFiles = new List<SubtitleFile>();
var filteredFiles = FilterAndClean(series, filesOnDisk, importedFiles);
var filterResult = FilterAndClean(series, filesOnDisk, importedFiles);
foreach (var possibleSubtitleFile in filteredFiles)
foreach (var possibleSubtitleFile in filterResult.FilesOnDisk)
{
var extension = Path.GetExtension(possibleSubtitleFile);
@ -83,7 +83,10 @@ public override IEnumerable<ExtraFile> ProcessFiles(Series series, List<string>
_logger.Info("Found {0} existing subtitle files", subtitleFiles.Count);
_subtitleFileService.Upsert(subtitleFiles);
return subtitleFiles;
// Return files that were just imported along with files that were
// previously imported so previously imported files aren't imported twice
return subtitleFiles.Concat(filterResult.PreviouslyImported);
}
}
}

25
src/NzbDrone.Core/MediaFiles/DiskScanService.cs
View File

@ -23,6 +23,8 @@ public interface IDiskScanService
{
void Scan(Series series);
string[] GetVideoFiles(string path, bool allDirectories = true);
string[] GetNonVideoFiles(string path, bool allDirectories = true);
List<string> FilterFiles(Series series, IEnumerable<string> files);
}
public class DiskScanService :
@ -59,7 +61,7 @@ public DiskScanService(IDiskProvider diskProvider,
}
private static readonly Regex ExcludedSubFoldersRegex = new Regex(@"(?:\\|\/|^)(extras|@eadir|\..+)(?:\\|\/)", RegexOptions.Compiled | RegexOptions.IgnoreCase);
private static readonly Regex ExcludedFilesRegex = new Regex(@"^\._", RegexOptions.Compiled | RegexOptions.IgnoreCase);
private static readonly Regex ExcludedFilesRegex = new Regex(@"^\._|Thumbs\.db", RegexOptions.Compiled | RegexOptions.IgnoreCase);
public void Scan(Series series)
{
@ -133,10 +135,25 @@ public string[] GetVideoFiles(string path, bool allDirectories = true)
return mediaFileList.ToArray();
}
private IEnumerable<string> FilterFiles(Series series, IEnumerable<string> videoFiles)
public string[] GetNonVideoFiles(string path, bool allDirectories = true)
{
return videoFiles.Where(file => !ExcludedSubFoldersRegex.IsMatch(series.Path.GetRelativePath(file)))
.Where(file => !ExcludedFilesRegex.IsMatch(Path.GetFileName(file)));
_logger.Debug("Scanning '{0}' for non-video files", path);
var searchOption = allDirectories ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly;
var filesOnDisk = _diskProvider.GetFiles(path, searchOption);
var mediaFileList = filesOnDisk.Where(file => !MediaFileExtensions.Extensions.Contains(Path.GetExtension(file).ToLower()))
.ToList();
_logger.Debug("{0} non-video files were found in {1}", mediaFileList.Count, path);
return mediaFileList.ToArray();
}
public List<string> FilterFiles(Series series, IEnumerable<string> files)
{
return files.Where(file => !ExcludedSubFoldersRegex.IsMatch(series.Path.GetRelativePath(file)))
.Where(file => !ExcludedFilesRegex.IsMatch(Path.GetFileName(file)))
.ToList();
}
private void SetPermissions(string path)

1
src/NzbDrone.Core/NzbDrone.Core.csproj
View File

@ -495,6 +495,7 @@
<Compile Include="Extras\Files\ExtraFileRepository.cs" />
<Compile Include="Extras\ExtraService.cs" />
<Compile Include="Extras\IImportExistingExtraFiles.cs" />
<Compile Include="Extras\ImportExistingExtraFileFilterResult.cs" />
<Compile Include="Extras\ImportExistingExtraFilesBase.cs" />
<Compile Include="Extras\Metadata\Files\MetadataFile.cs" />
<Compile Include="Extras\Metadata\Files\MetadataFileRepository.cs" />