New: Added FileList.io indexer support

src/NzbDrone.Core.Test/Files/Indexers/FileList/RecentFeed.json

@@ -0,0 +1,38 @@
+[
+  {
+    "id": 1234,
+    "name": "Mankind.Divided.2019.S01E01.1080p.WEB-DL",
+    "imdb": "tt1232322",
+    "freeleech": 0,
+    "upload_date": "2019-01-22 22:20:19",
+    "download_link": "https://filelist.io/download.php?id=1234&passkey=somepass",
+    "size": 830512414,
+    "internal": 0,
+    "moderated": 1,
+    "category": "Seriale HD",
+    "seeders": 12,
+    "leechers": 2,
+    "times_completed": 11,
+    "comments": 0,
+    "files": 3,
+    "small_description": "Much anticipated show about (redacted)"
+  },
+  {
+    "id": 1235,
+    "name": "Mankind.Divided.2019.S01E02.1080p.WEB-DL",
+    "imdb": "tt9999999",
+    "freeleech": 0,
+    "upload_date": "2019-01-22 22:19:37",
+    "download_link": "https://filelist.io/download.php?id=1235&passkey=somepass",
+    "size": 473149881,
+    "internal": 0,
+    "moderated": 1,
+    "category": "Seriale HD",
+    "seeders": 9,
+    "leechers": 1,
+    "times_completed": 8,
+    "comments": 0,
+    "files": 3,
+    "small_description": "(redacted) finds a way to unify two of the most insignificant factions"
+  }
+]

src/NzbDrone.Core.Test/IndexerTests/FileListTests/FileListFixture.cs

@@ -0,0 +1,57 @@
+using System;
+using System.Linq;
+using FluentAssertions;
+using Moq;
+using NUnit.Framework;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.Indexers;
+using NzbDrone.Core.Indexers.FileList;
+using NzbDrone.Core.Parser.Model;
+using NzbDrone.Core.Test.Framework;
+
+namespace NzbDrone.Core.Test.IndexerTests.FileListTests
+{
+    [TestFixture]
+    public class FileListFixture : CoreTest<FileList>
+    {
+        [SetUp]
+        public void Setup()
+        {
+            Subject.Definition = new IndexerDefinition()
+            {
+                Name = "FileList",
+                Settings = new FileListSettings() { Username = "someuser", Passkey = "somepass" }
+            };
+        }
+
+        [Test]
+        public void should_parse_recent_feed_from_FileList()
+        {
+            var recentFeed = ReadAllText(@"Files/Indexers/FileList/recentfeed.json");
+
+            Mocker.GetMock<IHttpClient>()
+                .Setup(o => o.Execute(It.Is<HttpRequest>(v => v.Method == HttpMethod.GET)))
+                .Returns<HttpRequest>(r => new HttpResponse(r, new HttpHeader(), recentFeed));
+
+            var releases = Subject.FetchRecent();
+
+            releases.Should().HaveCount(2);
+            releases.First().Should().BeOfType<TorrentInfo>();
+
+            var torrentInfo = releases.First() as TorrentInfo;
+
+            torrentInfo.Title.Should().Be("Mankind.Divided.2019.S01E01.1080p.WEB-DL");
+            torrentInfo.DownloadProtocol.Should().Be(DownloadProtocol.Torrent);
+            torrentInfo.DownloadUrl.Should().Be("https://filelist.io/download.php?id=1234&passkey=somepass");
+            torrentInfo.InfoUrl.Should().Be("https://filelist.io/details.php?id=1234");
+            torrentInfo.CommentUrl.Should().BeNullOrEmpty();
+            torrentInfo.Indexer.Should().Be(Subject.Definition.Name);
+            torrentInfo.PublishDate.Should().Be(DateTime.Parse("2019-01-22 22:20:19").ToUniversalTime());
+            torrentInfo.Size.Should().Be(830512414);
+            torrentInfo.InfoHash.Should().Be(null);
+            torrentInfo.MagnetUrl.Should().Be(null);
+            torrentInfo.Peers.Should().Be(2 + 12);
+            torrentInfo.Seeders.Should().Be(12);
+        }
+    }
+}

src/NzbDrone.Core/Annotations/FieldDefinitionAttribute.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Runtime.CompilerServices;
 
 namespace NzbDrone.Core.Annotations
 {
@@ -23,6 +24,20 @@ namespace NzbDrone.Core.Annotations
         public PrivacyLevel Privacy { get; set; }
     }
 
+    [AttributeUsage(AttributeTargets.Property | AttributeTargets.Field, AllowMultiple = false)]
+    public class FieldOptionAttribute : Attribute
+    {
+        public FieldOptionAttribute([CallerLineNumber] int order = 0, string label = null)
+        {
+            Order = order;
+            Label = label;
+        }
+
+        public int Order { get; private set; }
+        public string Label { get; set; }
+        public string Hint { get; set; }
+    }
+
     public enum FieldType
     {
         Textbox,

src/NzbDrone.Core/Indexers/FileList/FileList.cs

@@ -0,0 +1,30 @@
+using NLog;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.Configuration;
+using NzbDrone.Core.Parser;
+
+namespace NzbDrone.Core.Indexers.FileList
+{
+    public class FileList : HttpIndexerBase<FileListSettings>
+    {
+        public override string Name => "FileList";
+        public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
+        public override bool SupportsRss => true;
+        public override bool SupportsSearch => true;
+
+        public FileList(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
+            : base(httpClient, indexerStatusService, configService, parsingService, logger)
+        {
+        }
+
+        public override IIndexerRequestGenerator GetRequestGenerator()
+        {
+            return new FileListRequestGenerator() { Settings = Settings };
+        }
+
+        public override IParseIndexerResponse GetParser()
+        {
+            return new FileListParser(Settings);
+        }
+    }
+}

src/NzbDrone.Core/Indexers/FileList/FileListParser.cs

@@ -0,0 +1,75 @@
+using System;
+using System.Collections.Generic;
+using System.Net;
+using Newtonsoft.Json;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.Indexers.Exceptions;
+using NzbDrone.Core.Parser.Model;
+
+namespace NzbDrone.Core.Indexers.FileList
+{
+    public class FileListParser : IParseIndexerResponse
+    {
+        private readonly FileListSettings _settings;
+
+        public FileListParser(FileListSettings settings)
+        {
+            _settings = settings;
+        }
+
+        public IList<ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
+        {
+            var torrentInfos = new List<ReleaseInfo>();
+
+            if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
+            {
+                throw new IndexerException(indexerResponse,
+                    "Unexpected response status {0} code from API request",
+                    indexerResponse.HttpResponse.StatusCode);
+            }
+
+            var queryResults = JsonConvert.DeserializeObject<List<FileListTorrent>>(indexerResponse.Content);
+
+            foreach (var result in queryResults)
+            {
+                var id = result.Id;
+
+                //if (result.FreeLeech)
+
+                torrentInfos.Add(new TorrentInfo()
+                {
+                    Guid = $"FileList-{id}",
+                    Title = result.Name,
+                    Size = result.Size,
+                    DownloadUrl = GetDownloadUrl(id),
+                    InfoUrl = GetInfoUrl(id),
+                    Seeders = result.Seeders,
+                    Peers = result.Leechers + result.Seeders,
+                    PublishDate = result.UploadDate.ToUniversalTime(),
+                    ImdbId = result.ImdbId
+                });
+            }
+
+            return torrentInfos.ToArray();
+        }
+        
+        private string GetDownloadUrl(string torrentId)
+        {
+            var url = new HttpUri(_settings.BaseUrl)
+                .CombinePath("download.php")
+                .AddQueryParam("id", torrentId)
+                .AddQueryParam("passkey", _settings.Passkey);
+
+            return url.FullUri;
+        }
+
+        private string GetInfoUrl(string torrentId)
+        {
+            var url = new HttpUri(_settings.BaseUrl)
+                .CombinePath("details.php")
+                .AddQueryParam("id", torrentId);
+
+            return url.FullUri;
+        }
+    }
+}

src/NzbDrone.Core/Indexers/FileList/FileListRequestGenerator.cs

@@ -0,0 +1,117 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.IndexerSearch.Definitions;
+
+namespace NzbDrone.Core.Indexers.FileList
+{
+    public class FileListRequestGenerator : IIndexerRequestGenerator
+    {
+        public FileListSettings Settings { get; set; }
+        
+        public virtual IndexerPageableRequestChain GetRecentRequests()
+        {
+            var pageableRequests = new IndexerPageableRequestChain();
+
+            pageableRequests.Add(GetRequest("latest-torrents", Settings.Categories.Concat(Settings.AnimeCategories), ""));
+
+            return pageableRequests;
+        }
+
+        public IndexerPageableRequestChain GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria)
+        {
+            var pageableRequests = new IndexerPageableRequestChain();
+
+            AddImdbRequests(pageableRequests, searchCriteria, "search-torrents", Settings.Categories, $"&season={searchCriteria.SeasonNumber}&episode={searchCriteria.EpisodeNumber}");
+
+            pageableRequests.AddTier();
+
+            AddNameRequests(pageableRequests, searchCriteria, "search-torrents", Settings.Categories, $"&season={searchCriteria.SeasonNumber}&episode={searchCriteria.EpisodeNumber}");
+
+            return pageableRequests;
+        }
+
+        public IndexerPageableRequestChain GetSearchRequests(SeasonSearchCriteria searchCriteria)
+        {
+            var pageableRequests = new IndexerPageableRequestChain();
+
+            AddImdbRequests(pageableRequests, searchCriteria, "search-torrents", Settings.Categories, $"&season={searchCriteria.SeasonNumber}");
+
+            pageableRequests.AddTier();
+
+            AddNameRequests(pageableRequests, searchCriteria, "search-torrents", Settings.Categories, $"&season={searchCriteria.SeasonNumber}");
+
+            return pageableRequests;
+        }
+
+        public IndexerPageableRequestChain GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria)
+        {
+            return new IndexerPageableRequestChain();
+        }
+
+        public IndexerPageableRequestChain GetSearchRequests(DailySeasonSearchCriteria searchCriteria)
+        {
+            return new IndexerPageableRequestChain();
+        }
+
+        public IndexerPageableRequestChain GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria)
+        {
+            var pageableRequests = new IndexerPageableRequestChain();
+
+            // FileList has absolute releases in E01 format but also release sin S01E01 format, likely by imdb numbering but we only have tvdb numbering... so we try those as fallback to abs.
+            AddImdbRequests(pageableRequests, searchCriteria, "search-torrents", Settings.AnimeCategories, $"&season=0&episode={searchCriteria.AbsoluteEpisodeNumber}");
+            pageableRequests.AddTier();
+            foreach (var eps in searchCriteria.Episodes)
+            {
+                AddImdbRequests(pageableRequests, searchCriteria, "search-torrents", Settings.AnimeCategories, $"&season={eps.SeasonNumber}&episode={eps.EpisodeNumber}");
+            }
+
+            pageableRequests.AddTier();
+
+            AddNameRequests(pageableRequests, searchCriteria, "search-torrents", Settings.AnimeCategories, $"&season=0&episode={searchCriteria.AbsoluteEpisodeNumber}");
+            pageableRequests.AddTier();
+            foreach (var eps in searchCriteria.Episodes)
+            {
+                AddNameRequests(pageableRequests, searchCriteria, "search-torrents", Settings.AnimeCategories, $"&season={eps.SeasonNumber}&episode={eps.EpisodeNumber}");
+            }
+
+            return pageableRequests;
+
+        }
+
+        public IndexerPageableRequestChain GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria)
+        {
+            return new IndexerPageableRequestChain();
+        }
+
+        private void AddImdbRequests(IndexerPageableRequestChain chain, SearchCriteriaBase searchCriteria, string searchType, IEnumerable<int> categories, string parameters)
+        {
+            if (searchCriteria.Series.ImdbId.IsNotNullOrWhiteSpace())
+            {
+                chain.Add(GetRequest(searchType, categories, string.Format("&type=imdb&query={0}{1}", searchCriteria.Series.ImdbId, parameters)));
+            }
+        }
+
+        private void AddNameRequests(IndexerPageableRequestChain chain, SearchCriteriaBase searchCriteria, string searchType, IEnumerable<int> categories, string parameters)
+        {
+            foreach (var sceneTitle in searchCriteria.SceneTitles)
+            {
+                chain.Add(GetRequest(searchType, categories, string.Format("&type=name&query={0}{1}", Uri.EscapeDataString(sceneTitle.Trim()), parameters)));
+            }
+        }
+
+        private IEnumerable<IndexerRequest> GetRequest(string searchType, IEnumerable<int> categories, string parameters)
+        {
+            var categoriesQuery = string.Join(",", categories.Distinct());
+
+            var baseUrl = string.Format("{0}/api.php?action={1}&category={2}{3}", Settings.BaseUrl.TrimEnd('/'), searchType, categoriesQuery, parameters);
+
+            var request = new IndexerRequest(baseUrl, HttpAccept.Json);
+            request.HttpRequest.AddBasicAuthentication(Settings.Username.Trim(), Settings.Passkey.Trim());
+
+            yield return request;
+        }
+    }
+}

src/NzbDrone.Core/Indexers/FileList/FileListSettings.cs

@@ -0,0 +1,85 @@
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.ComponentModel.DataAnnotations;
+using FluentValidation;
+using Growl.Connector;
+using NzbDrone.Core.Annotations;
+using NzbDrone.Core.Parser.Model;
+using NzbDrone.Core.Validation;
+
+namespace NzbDrone.Core.Indexers.FileList
+{
+    public class FileListSettingsValidator : AbstractValidator<FileListSettings>
+    {
+        public FileListSettingsValidator()
+        {
+            RuleFor(c => c.BaseUrl).ValidRootUrl();
+            RuleFor(c => c.Username).NotEmpty();
+            RuleFor(c => c.Passkey).NotEmpty();
+
+            RuleFor(c => c.SeedCriteria).SetValidator(_ => new SeedCriteriaSettingsValidator());
+        }
+    }
+
+    public class FileListSettings : ITorrentIndexerSettings
+    {
+        private static readonly FileListSettingsValidator Validator = new FileListSettingsValidator();
+
+        public FileListSettings()
+        {
+            BaseUrl = "https://filelist.io";
+            MinimumSeeders = IndexerDefaults.MINIMUM_SEEDERS;
+
+            Categories = new int[]
+            {
+                (int)FileListCategories.TV_SD,
+                (int)FileListCategories.TV_HD,
+                (int)FileListCategories.TV_4K
+            };
+
+            AnimeCategories = new int[0];
+        }
+
+        [FieldDefinition(0, Label = "Username", Privacy = PrivacyLevel.UserName)]
+        public string Username { get; set; }
+
+        [FieldDefinition(1, Label = "Passkey", Privacy = PrivacyLevel.ApiKey)]
+        public string Passkey { get; set; }
+
+        [FieldDefinition(3, Label = "API URL", Advanced = true, HelpText = "Do not change this unless you know what you're doing. Since your API key will be sent to that host.")]
+        public string BaseUrl { get; set; }
+
+        [FieldDefinition(4, Label = "Categories", Type = FieldType.Select, SelectOptions = typeof(FileListCategories), HelpText = "Categories for use in search and feeds, leave blank to disable standard/daily shows")]
+        public IEnumerable<int> Categories { get; set; }
+
+        [FieldDefinition(5, Label = "Anime Categories", Type = FieldType.Select, SelectOptions = typeof(FileListCategories), HelpText = "Categories for use in search and feeds, leave blank to disable anime")]
+        public IEnumerable<int> AnimeCategories { get; set; }
+
+        [FieldDefinition(6, Type = FieldType.Number, Label = "Minimum Seeders", HelpText = "Minimum number of seeders required.", Advanced = true)]
+        public int MinimumSeeders { get; set; }
+
+        [FieldDefinition(7)]
+        public SeedCriteriaSettings SeedCriteria { get; set; } = new SeedCriteriaSettings();
+
+        public NzbDroneValidationResult Validate()
+        {
+            return new NzbDroneValidationResult(Validator.Validate(this));
+        }
+    }
+
+    public enum FileListCategories
+    {
+        [FieldOption]
+        Anime = 24,
+        [FieldOption]
+        Animation = 15,
+        [FieldOption]
+        TV_4K = 27,
+        [FieldOption]
+        TV_HD = 21,
+        [FieldOption]
+        TV_SD = 23,
+        [FieldOption]
+        Sport = 13
+    }
+}

src/NzbDrone.Core/Indexers/FileList/FileListTorrent.cs

@@ -0,0 +1,24 @@
+using System;
+using Newtonsoft.Json;
+
+namespace NzbDrone.Core.Indexers.FileList
+{
+    public class FileListTorrent
+    {
+        public string Id { get; set; }
+        public string Name { get; set; }
+        public long Size { get; set; }
+        public int Leechers { get; set; }
+        public int Seeders { get; set; }
+        [JsonProperty(PropertyName = "times_completed")]
+        public uint TimesCompleted { get; set; }
+        public uint Comments { get; set; }
+        public uint Files { get; set; }
+        [JsonProperty(PropertyName = "imdb")]
+        public string ImdbId { get; set; }
+        [JsonProperty(PropertyName = "freeleech")]
+        public bool FreeLeech { get; set; }
+        [JsonProperty(PropertyName = "upload_date")]
+        public DateTime UploadDate { get; set; }
+    }
+}

src/NzbDrone.Core/Instrumentation/DatabaseTarget.cs

@@ -104,7 +104,7 @@ namespace NzbDrone.Core.Instrumentation
 
         public void Handle(ApplicationShutdownRequested message)
         {
-            if (LogManager.Configuration.LoggingRules.Contains(Rule))
+            if (LogManager.Configuration != null && LogManager.Configuration.LoggingRules.Contains(Rule))
             {
                 UnRegister();
             }

src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs

@@ -18,6 +18,7 @@ namespace NzbDrone.Core.Parser.Model
         public DownloadProtocol DownloadProtocol { get; set; }
         public int TvdbId { get; set; }
         public int TvRageId { get; set; }
+        public string ImdbId { get; set; }
         public DateTime PublishDate { get; set; }
 
         public string Origin { get; set; }

src/Sonarr.Http/ClientSchema/SelectOption.cs

@@ -4,5 +4,7 @@
     {
         public int Value { get; set; }
         public string Name { get; set; }
+        public int Order { get; set; }
+        public string Hint { get; set; }
     }
 }