mirror of
https://github.com/Radarr/Radarr.git
synced 2024-09-11 03:52:33 +02:00
Fixed: Prevent duplicate parsing of extra files
This commit is contained in:
parent
2e96c4e798
commit
a621f0d49b
@ -14,18 +14,18 @@ namespace NzbDrone.Core.Extras
|
||||
public class ExistingExtraFileService : IHandle<SeriesScannedEvent>
|
||||
{
|
||||
private readonly IDiskProvider _diskProvider;
|
||||
private readonly IDiskScanService _diskScanService;
|
||||
private readonly List<IImportExistingExtraFiles> _existingExtraFileImporters;
|
||||
private readonly List<IManageExtraFiles> _extraFileManagers;
|
||||
private readonly Logger _logger;
|
||||
|
||||
public ExistingExtraFileService(IDiskProvider diskProvider,
|
||||
IDiskScanService diskScanService,
|
||||
List<IImportExistingExtraFiles> existingExtraFileImporters,
|
||||
List<IManageExtraFiles> extraFileManagers,
|
||||
Logger logger)
|
||||
{
|
||||
_diskProvider = diskProvider;
|
||||
_diskScanService = diskScanService;
|
||||
_existingExtraFileImporters = existingExtraFileImporters.OrderBy(e => e.Order).ToList();
|
||||
_extraFileManagers = extraFileManagers.OrderBy(e => e.Order).ToList();
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
@ -41,9 +41,8 @@ public void Handle(SeriesScannedEvent message)
|
||||
|
||||
_logger.Debug("Looking for existing extra files in {0}", series.Path);
|
||||
|
||||
var filesOnDisk = _diskProvider.GetFiles(series.Path, SearchOption.AllDirectories);
|
||||
var possibleExtraFiles = filesOnDisk.Where(c => !MediaFileExtensions.Extensions.Contains(Path.GetExtension(c).ToLower()) &&
|
||||
!c.StartsWith(Path.Combine(series.Path, "EXTRAS"))).ToList();
|
||||
var filesOnDisk = _diskScanService.GetNonVideoFiles(series.Path);
|
||||
var possibleExtraFiles = _diskScanService.FilterFiles(series, filesOnDisk);
|
||||
|
||||
var filteredFiles = possibleExtraFiles;
|
||||
var importedFiles = new List<string>();
|
||||
@ -55,7 +54,7 @@ public void Handle(SeriesScannedEvent message)
|
||||
importedFiles.AddRange(imported.Select(f => Path.Combine(series.Path, f.RelativePath)));
|
||||
}
|
||||
|
||||
_logger.Info("Found {0} extra files", extraFiles);
|
||||
_logger.Info("Found {0} extra files", extraFiles.Count);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -0,0 +1,18 @@
|
||||
using System.Collections.Generic;
|
||||
using NzbDrone.Core.Extras.Files;
|
||||
|
||||
namespace NzbDrone.Core.Extras
|
||||
{
|
||||
public class ImportExistingExtraFileFilterResult<TExtraFile>
|
||||
where TExtraFile : ExtraFile, new()
|
||||
{
|
||||
public ImportExistingExtraFileFilterResult(List<TExtraFile> previouslyImported, List<string> filesOnDisk)
|
||||
{
|
||||
PreviouslyImported = previouslyImported;
|
||||
FilesOnDisk = filesOnDisk;
|
||||
}
|
||||
|
||||
public List<TExtraFile> PreviouslyImported { get; set; }
|
||||
public List<string> FilesOnDisk { get; set; }
|
||||
}
|
||||
}
|
@ -21,7 +21,7 @@ public ImportExistingExtraFilesBase(IExtraFileService<TExtraFile> extraFileServi
|
||||
public abstract int Order { get; }
|
||||
public abstract IEnumerable<ExtraFile> ProcessFiles(Series series, List<string> filesOnDisk, List<string> importedFiles);
|
||||
|
||||
public virtual List<string> FilterAndClean(Series series, List<string> filesOnDisk, List<string> importedFiles)
|
||||
public virtual ImportExistingExtraFileFilterResult<TExtraFile> FilterAndClean(Series series, List<string> filesOnDisk, List<string> importedFiles)
|
||||
{
|
||||
var seriesFiles = _extraFileService.GetFilesBySeries(series.Id);
|
||||
|
||||
@ -30,12 +30,16 @@ public virtual List<string> FilterAndClean(Series series, List<string> filesOnDi
|
||||
return Filter(series, filesOnDisk, importedFiles, seriesFiles);
|
||||
}
|
||||
|
||||
private List<string> Filter(Series series, List<string> filesOnDisk, List<string> importedFiles, List<TExtraFile> seriesFiles)
|
||||
private ImportExistingExtraFileFilterResult<TExtraFile> Filter(Series series, List<string> filesOnDisk, List<string> importedFiles, List<TExtraFile> seriesFiles)
|
||||
{
|
||||
var filteredFiles = filesOnDisk;
|
||||
var previouslyImported = seriesFiles.IntersectBy(s => Path.Combine(series.Path, s.RelativePath), filesOnDisk, f => f, PathEqualityComparer.Instance).ToList();
|
||||
var filteredFiles = filesOnDisk.Except(previouslyImported.Select(f => Path.Combine(series.Path, f.RelativePath)).ToList(), PathEqualityComparer.Instance)
|
||||
.Except(importedFiles, PathEqualityComparer.Instance)
|
||||
.ToList();
|
||||
|
||||
filteredFiles = filteredFiles.Except(seriesFiles.Select(f => Path.Combine(series.Path, f.RelativePath)).ToList(), PathEqualityComparer.Instance).ToList();
|
||||
return filteredFiles.Except(importedFiles, PathEqualityComparer.Instance).ToList();
|
||||
// Return files that are already imported so they aren't imported again by other importers.
|
||||
// Filter out files that were previously imported and as well as ones imported by other importers.
|
||||
return new ImportExistingExtraFileFilterResult<TExtraFile>(previouslyImported, filteredFiles);
|
||||
}
|
||||
|
||||
private void Clean(Series series, List<string> filesOnDisk, List<string> importedFiles, List<TExtraFile> seriesFiles)
|
||||
|
@ -5,6 +5,7 @@
|
||||
using NzbDrone.Common.Extensions;
|
||||
using NzbDrone.Core.Extras.Files;
|
||||
using NzbDrone.Core.Extras.Metadata.Files;
|
||||
using NzbDrone.Core.Extras.Subtitles;
|
||||
using NzbDrone.Core.Parser;
|
||||
using NzbDrone.Core.Tv;
|
||||
|
||||
@ -42,10 +43,17 @@ public override IEnumerable<ExtraFile> ProcessFiles(Series series, List<string>
|
||||
_logger.Debug("Looking for existing metadata in {0}", series.Path);
|
||||
|
||||
var metadataFiles = new List<MetadataFile>();
|
||||
var filteredFiles = FilterAndClean(series, filesOnDisk, importedFiles);
|
||||
var filterResult = FilterAndClean(series, filesOnDisk, importedFiles);
|
||||
|
||||
foreach (var possibleMetadataFile in filteredFiles)
|
||||
foreach (var possibleMetadataFile in filterResult.FilesOnDisk)
|
||||
{
|
||||
// Don't process files that have known Subtitle file extensions (saves a bit of unecessary processing)
|
||||
|
||||
if (SubtitleFileExtensions.Extensions.Contains(Path.GetExtension(possibleMetadataFile)))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var consumer in _consumers)
|
||||
{
|
||||
var metadata = consumer.FindMetadataFile(series, possibleMetadataFile);
|
||||
@ -90,7 +98,10 @@ public override IEnumerable<ExtraFile> ProcessFiles(Series series, List<string>
|
||||
_logger.Info("Found {0} existing metadata files", metadataFiles.Count);
|
||||
_metadataFileService.Upsert(metadataFiles);
|
||||
|
||||
return metadataFiles;
|
||||
// Return files that were just imported along with files that were
|
||||
// previously imported so previously imported files aren't imported twice
|
||||
|
||||
return metadataFiles.Concat(filterResult.PreviouslyImported);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -38,9 +38,9 @@ public override IEnumerable<ExtraFile> ProcessFiles(Series series, List<string>
|
||||
_logger.Debug("Looking for existing extra files in {0}", series.Path);
|
||||
|
||||
var extraFiles = new List<OtherExtraFile>();
|
||||
var filteredFiles = FilterAndClean(series, filesOnDisk, importedFiles);
|
||||
var filterResult = FilterAndClean(series, filesOnDisk, importedFiles);
|
||||
|
||||
foreach (var possibleExtraFile in filteredFiles)
|
||||
foreach (var possibleExtraFile in filterResult.FilesOnDisk)
|
||||
{
|
||||
var localEpisode = _parsingService.GetLocalEpisode(possibleExtraFile, series);
|
||||
|
||||
@ -77,7 +77,10 @@ public override IEnumerable<ExtraFile> ProcessFiles(Series series, List<string>
|
||||
_logger.Info("Found {0} existing other extra files", extraFiles.Count);
|
||||
_otherExtraFileService.Upsert(extraFiles);
|
||||
|
||||
return extraFiles;
|
||||
// Return files that were just imported along with files that were
|
||||
// previously imported so previously imported files aren't imported twice
|
||||
|
||||
return extraFiles.Concat(filterResult.PreviouslyImported);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -38,9 +38,9 @@ public override IEnumerable<ExtraFile> ProcessFiles(Series series, List<string>
|
||||
_logger.Debug("Looking for existing subtitle files in {0}", series.Path);
|
||||
|
||||
var subtitleFiles = new List<SubtitleFile>();
|
||||
var filteredFiles = FilterAndClean(series, filesOnDisk, importedFiles);
|
||||
var filterResult = FilterAndClean(series, filesOnDisk, importedFiles);
|
||||
|
||||
foreach (var possibleSubtitleFile in filteredFiles)
|
||||
foreach (var possibleSubtitleFile in filterResult.FilesOnDisk)
|
||||
{
|
||||
var extension = Path.GetExtension(possibleSubtitleFile);
|
||||
|
||||
@ -83,7 +83,10 @@ public override IEnumerable<ExtraFile> ProcessFiles(Series series, List<string>
|
||||
_logger.Info("Found {0} existing subtitle files", subtitleFiles.Count);
|
||||
_subtitleFileService.Upsert(subtitleFiles);
|
||||
|
||||
return subtitleFiles;
|
||||
// Return files that were just imported along with files that were
|
||||
// previously imported so previously imported files aren't imported twice
|
||||
|
||||
return subtitleFiles.Concat(filterResult.PreviouslyImported);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -23,6 +23,8 @@ public interface IDiskScanService
|
||||
{
|
||||
void Scan(Series series);
|
||||
string[] GetVideoFiles(string path, bool allDirectories = true);
|
||||
string[] GetNonVideoFiles(string path, bool allDirectories = true);
|
||||
List<string> FilterFiles(Series series, IEnumerable<string> files);
|
||||
}
|
||||
|
||||
public class DiskScanService :
|
||||
@ -59,7 +61,7 @@ public DiskScanService(IDiskProvider diskProvider,
|
||||
}
|
||||
|
||||
private static readonly Regex ExcludedSubFoldersRegex = new Regex(@"(?:\\|\/|^)(extras|@eadir|\..+)(?:\\|\/)", RegexOptions.Compiled | RegexOptions.IgnoreCase);
|
||||
private static readonly Regex ExcludedFilesRegex = new Regex(@"^\._", RegexOptions.Compiled | RegexOptions.IgnoreCase);
|
||||
private static readonly Regex ExcludedFilesRegex = new Regex(@"^\._|Thumbs\.db", RegexOptions.Compiled | RegexOptions.IgnoreCase);
|
||||
|
||||
public void Scan(Series series)
|
||||
{
|
||||
@ -133,10 +135,25 @@ public string[] GetVideoFiles(string path, bool allDirectories = true)
|
||||
return mediaFileList.ToArray();
|
||||
}
|
||||
|
||||
private IEnumerable<string> FilterFiles(Series series, IEnumerable<string> videoFiles)
|
||||
public string[] GetNonVideoFiles(string path, bool allDirectories = true)
|
||||
{
|
||||
return videoFiles.Where(file => !ExcludedSubFoldersRegex.IsMatch(series.Path.GetRelativePath(file)))
|
||||
.Where(file => !ExcludedFilesRegex.IsMatch(Path.GetFileName(file)));
|
||||
_logger.Debug("Scanning '{0}' for non-video files", path);
|
||||
|
||||
var searchOption = allDirectories ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly;
|
||||
var filesOnDisk = _diskProvider.GetFiles(path, searchOption);
|
||||
|
||||
var mediaFileList = filesOnDisk.Where(file => !MediaFileExtensions.Extensions.Contains(Path.GetExtension(file).ToLower()))
|
||||
.ToList();
|
||||
|
||||
_logger.Debug("{0} non-video files were found in {1}", mediaFileList.Count, path);
|
||||
return mediaFileList.ToArray();
|
||||
}
|
||||
|
||||
public List<string> FilterFiles(Series series, IEnumerable<string> files)
|
||||
{
|
||||
return files.Where(file => !ExcludedSubFoldersRegex.IsMatch(series.Path.GetRelativePath(file)))
|
||||
.Where(file => !ExcludedFilesRegex.IsMatch(Path.GetFileName(file)))
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private void SetPermissions(string path)
|
||||
|
@ -495,6 +495,7 @@
|
||||
<Compile Include="Extras\Files\ExtraFileRepository.cs" />
|
||||
<Compile Include="Extras\ExtraService.cs" />
|
||||
<Compile Include="Extras\IImportExistingExtraFiles.cs" />
|
||||
<Compile Include="Extras\ImportExistingExtraFileFilterResult.cs" />
|
||||
<Compile Include="Extras\ImportExistingExtraFilesBase.cs" />
|
||||
<Compile Include="Extras\Metadata\Files\MetadataFile.cs" />
|
||||
<Compile Include="Extras\Metadata\Files\MetadataFileRepository.cs" />
|
||||
|
Loading…
Reference in New Issue
Block a user