From 6744e0d506a7c3fb2a0edc35336f9aee352fa4ca Mon Sep 17 00:00:00 2001 From: Taloth Saldono Date: Sun, 24 May 2015 20:41:12 +0200 Subject: [PATCH] New: Added support for Rarbg as replacement for Eztv. --- src/NzbDrone.Common/Http/HttpRequest.cs | 5 + src/NzbDrone.Common/Http/UriExtensions.cs | 4 +- .../Files/Indexers/Rarbg/RecentFeed.json | 29 ++++++ .../IndexerTests/RarbgTests/RarbgFixture.cs | 59 ++++++++++++ .../NzbDrone.Core.Test.csproj | 4 + src/NzbDrone.Core/Indexers/Rarbg/Rarbg.cs | 50 ++++++++++ .../Indexers/Rarbg/RarbgParser.cs | 79 +++++++++++++++ .../Indexers/Rarbg/RarbgRequestGenerator.cs | 96 +++++++++++++++++++ .../Indexers/Rarbg/RarbgSettings.cs | 38 ++++++++ .../Indexers/Rarbg/RarbgTokenProvider.cs | 39 ++++++++ .../Indexers/Rarbg/RarbgTorrent.cs | 24 +++++ src/NzbDrone.Core/NzbDrone.Core.csproj | 6 ++ 12 files changed, 430 insertions(+), 3 deletions(-) create mode 100644 src/NzbDrone.Core.Test/Files/Indexers/Rarbg/RecentFeed.json create mode 100644 src/NzbDrone.Core.Test/IndexerTests/RarbgTests/RarbgFixture.cs create mode 100644 src/NzbDrone.Core/Indexers/Rarbg/Rarbg.cs create mode 100644 src/NzbDrone.Core/Indexers/Rarbg/RarbgParser.cs create mode 100644 src/NzbDrone.Core/Indexers/Rarbg/RarbgRequestGenerator.cs create mode 100644 src/NzbDrone.Core/Indexers/Rarbg/RarbgSettings.cs create mode 100644 src/NzbDrone.Core/Indexers/Rarbg/RarbgTokenProvider.cs create mode 100644 src/NzbDrone.Core/Indexers/Rarbg/RarbgTorrent.cs diff --git a/src/NzbDrone.Common/Http/HttpRequest.cs b/src/NzbDrone.Common/Http/HttpRequest.cs index f71e3f473..3de434fbe 100644 --- a/src/NzbDrone.Common/Http/HttpRequest.cs +++ b/src/NzbDrone.Common/Http/HttpRequest.cs @@ -70,6 +70,11 @@ public void AddSegment(string segment, string value) _segments.Add(key, value); } + public void AddQueryParam(string segment, string value) + { + UriBuilder.SetQueryParam(segment, value); + } + public void AddCookie(string key, string value) { Cookies[key] = value; diff --git a/src/NzbDrone.Common/Http/UriExtensions.cs b/src/NzbDrone.Common/Http/UriExtensions.cs index c45a5c97c..d45d8c943 100644 --- a/src/NzbDrone.Common/Http/UriExtensions.cs +++ b/src/NzbDrone.Common/Http/UriExtensions.cs @@ -14,9 +14,7 @@ public static void SetQueryParam(this UriBuilder uriBuilder, string key, object query += "&"; } - uriBuilder.Query = query.Trim('?') + (key + "=" + value); + uriBuilder.Query = query.Trim('?') + key + "=" + Uri.EscapeDataString(value.ToString()); } - - } } \ No newline at end of file diff --git a/src/NzbDrone.Core.Test/Files/Indexers/Rarbg/RecentFeed.json b/src/NzbDrone.Core.Test/Files/Indexers/Rarbg/RecentFeed.json new file mode 100644 index 000000000..2cdbd41e1 --- /dev/null +++ b/src/NzbDrone.Core.Test/Files/Indexers/Rarbg/RecentFeed.json @@ -0,0 +1,29 @@ +[ + { + "f": "Thunderbirds.Are.Go.S01E09.Slingshot.1080p.WEB-DL.AAC2.0.H.264-Coo7[rartv]", + "c": "TV HD Episodes", + "d": "magnet:?xt=urn:btih:ff4737b5230307836ec8abce6ab73727f1358bf3&dn=Thunderbirds.Are.Go.S01E09.Slingshot.1080p.WEB-DL.AAC2.0.H.264-Coo7%5Brartv%5D&tr=http%3A%2F%2Ftracker.trackerfix.com%3A80%2Fannounce&tr=udp%3A%2F%2F9.rarbg.me%3A2710&tr=udp%3A%2F%2F9.rarbg.to%3A2710&tr=udp%3A%2F%2Fopen.demonii.com%3A1337%2Fannounce", + "s": "44", + "l": "19", + "t": "896238116", + "u": "2015-05-24 19:36:09" + }, + { + "f": "Thunderbirds.Are.Go.S01E10.Tunnels.Of.Time.720p.HDTV.x264-RDVAS[rartv]", + "c": "TV HD Episodes", + "d": "magnet:?xt=urn:btih:47bf1d7bfb72a83300bbe68d0b6aa09591e7a0a1&dn=Thunderbirds.Are.Go.S01E10.Tunnels.Of.Time.720p.HDTV.x264-RDVAS%5Brartv%5D&tr=http%3A%2F%2Ftracker.trackerfix.com%3A80%2Fannounce&tr=udp%3A%2F%2F9.rarbg.me%3A2710&tr=udp%3A%2F%2F9.rarbg.to%3A2710&tr=udp%3A%2F%2Fopen.demonii.com%3A1337%2Fannounce", + "s": "179", + "l": "125", + "t": "556055350", + "u": "2015-05-24 19:07:59" + }, + { + "f": "Tatau.S01E06.1080p.WEB-DL.AAC2.0.H.264-BS[rartv]", + "c": "TV HD Episodes", + "d": "magnet:?xt=urn:btih:8857e9b011c7a0483351371721fa9f3ba356dd73&dn=Tatau.S01E06.1080p.WEB-DL.AAC2.0.H.264-BS%5Brartv%5D&tr=http%3A%2F%2Ftracker.trackerfix.com%3A80%2Fannounce&tr=udp%3A%2F%2F9.rarbg.me%3A2710&tr=udp%3A%2F%2F9.rarbg.to%3A2710&tr=udp%3A%2F%2Fopen.demonii.com%3A1337%2Fannounce", + "s": "27", + "l": "22", + "t": "1652442143", + "u": "2015-05-24 18:54:49" + } +] diff --git a/src/NzbDrone.Core.Test/IndexerTests/RarbgTests/RarbgFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/RarbgTests/RarbgFixture.cs new file mode 100644 index 000000000..c930fbabe --- /dev/null +++ b/src/NzbDrone.Core.Test/IndexerTests/RarbgTests/RarbgFixture.cs @@ -0,0 +1,59 @@ +using System; +using System.Linq; +using FluentAssertions; +using Moq; +using NUnit.Framework; +using NzbDrone.Common.Http; +using NzbDrone.Core.Indexers; +using NzbDrone.Core.Indexers.Rarbg; +using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.IndexerTests.RarbgTests +{ + [TestFixture] + public class RarbgFixture : CoreTest + { + [SetUp] + public void Setup() + { + Subject.Definition = new IndexerDefinition() + { + Name = "Rarbg", + Settings = new RarbgSettings() + }; + + Mocker.GetMock() + .Setup(v => v.GetToken(It.IsAny())) + .Returns("validtoken"); + } + + [Test] + public void should_parse_recent_feed_from_Rarbg() + { + var recentFeed = ReadAllText(@"Files/Indexers/Rarbg/RecentFeed.json"); + + Mocker.GetMock() + .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.GET))) + .Returns(r => new HttpResponse(r, new HttpHeader(), recentFeed)); + + var releases = Subject.FetchRecent(); + + releases.Should().HaveCount(3); + releases.First().Should().BeOfType(); + + var torrentInfo = releases.First() as TorrentInfo; + + torrentInfo.Title.Should().Be("Thunderbirds.Are.Go.S01E09.Slingshot.1080p.WEB-DL.AAC2.0.H.264-Coo7[rartv]"); + torrentInfo.DownloadProtocol.Should().Be(DownloadProtocol.Torrent); + torrentInfo.DownloadUrl.Should().Be("magnet:?xt=urn:btih:ff4737b5230307836ec8abce6ab73727f1358bf3&dn=Thunderbirds.Are.Go.S01E09.Slingshot.1080p.WEB-DL.AAC2.0.H.264-Coo7%5Brartv%5D&tr=http%3A%2F%2Ftracker.trackerfix.com%3A80%2Fannounce&tr=udp%3A%2F%2F9.rarbg.me%3A2710&tr=udp%3A%2F%2F9.rarbg.to%3A2710&tr=udp%3A%2F%2Fopen.demonii.com%3A1337%2Fannounce"); + torrentInfo.Indexer.Should().Be(Subject.Definition.Name); + torrentInfo.PublishDate.Should().Be(DateTime.Parse("2015-05-24 19:36:09")); + torrentInfo.Size.Should().Be(896238116); + torrentInfo.InfoHash.Should().BeNull(); + torrentInfo.MagnetUrl.Should().BeNull(); + torrentInfo.Peers.Should().Be(44+19); + torrentInfo.Seeders.Should().Be(44); + } + } +} diff --git a/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj b/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj index 788f44fd0..ceb198a2f 100644 --- a/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj +++ b/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj @@ -168,6 +168,9 @@ + + Always + @@ -207,6 +210,7 @@ + diff --git a/src/NzbDrone.Core/Indexers/Rarbg/Rarbg.cs b/src/NzbDrone.Core/Indexers/Rarbg/Rarbg.cs new file mode 100644 index 000000000..e2189aa14 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Rarbg/Rarbg.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using NLog; +using NzbDrone.Common.Http; +using NzbDrone.Core.Configuration; +using NzbDrone.Core.Parser; +using NzbDrone.Core.Parser.Model; + +namespace NzbDrone.Core.Indexers.Rarbg +{ + public class Rarbg : HttpIndexerBase + { + private readonly IRarbgTokenProvider _tokenProvider; + private static DateTime _lastFetch; + + public override string Name { get { return "Rarbg"; } } + + public override DownloadProtocol Protocol { get { return DownloadProtocol.Torrent; } } + + public Rarbg(IRarbgTokenProvider tokenProvider, IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger) + : base(httpClient, configService, parsingService, logger) + { + _tokenProvider = tokenProvider; + } + + public override IIndexerRequestGenerator GetRequestGenerator() + { + return new RarbgRequestGenerator(_tokenProvider) { Settings = Settings }; + } + + public override IParseIndexerResponse GetParser() + { + return new RarbgParser(); + } + + protected override IList FetchPage(IndexerRequest request, IParseIndexerResponse parser) + { + var delay = _lastFetch + TimeSpan.FromSeconds(10) - DateTime.UtcNow; + if (delay.TotalSeconds > 0) + { + Thread.Sleep(delay); + } + + _lastFetch = DateTime.UtcNow; + + return base.FetchPage(request, parser); + } + } +} \ No newline at end of file diff --git a/src/NzbDrone.Core/Indexers/Rarbg/RarbgParser.cs b/src/NzbDrone.Core/Indexers/Rarbg/RarbgParser.cs new file mode 100644 index 000000000..c3274adf8 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Rarbg/RarbgParser.cs @@ -0,0 +1,79 @@ +using System; +using System.Collections.Generic; +using System.Net; +using System.Text.RegularExpressions; +using Newtonsoft.Json.Linq; +using NzbDrone.Common.Http; +using NzbDrone.Core.Indexers.Exceptions; +using NzbDrone.Core.Parser.Model; + +namespace NzbDrone.Core.Indexers.Rarbg +{ + public class RarbgParser : IParseIndexerResponse + { + private static readonly Regex RegexGuid = new Regex(@"^magnet:\?xt=urn:btih:([a-f0-9]+)", RegexOptions.Compiled); + + public IList ParseResponse(IndexerResponse indexerResponse) + { + var results = new List(); + + switch (indexerResponse.HttpResponse.StatusCode) + { + default: + if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK) + { + throw new IndexerException(indexerResponse, "Indexer API call returned an unexpected StatusCode [{0}]", indexerResponse.HttpResponse.StatusCode); + } + break; + } + + var jsonResponse = new HttpResponse(indexerResponse.HttpResponse).Resource as JContainer; + + var errorResponse = jsonResponse as JObject; + if (errorResponse != null && errorResponse["error"] != null) + { + var error = errorResponse["error"].ToString(); + if (error == "No results found") + { + return results; + } + + throw new IndexerException(indexerResponse, "Indexer API call returned an error [{0}]", errorResponse["error"]); + } + + var torrentResponse = jsonResponse.ToObject>(); + + foreach (var torrent in torrentResponse) + { + var torrentInfo = new TorrentInfo(); + + torrentInfo.Guid = GetGuid(torrent); + torrentInfo.Title = torrent.Title; + torrentInfo.Size = torrent.Size; + torrentInfo.DownloadUrl = torrent.DownloadUrl; + torrentInfo.PublishDate = torrent.PublishDate; + torrentInfo.Seeders = torrent.Seeders; + torrentInfo.Peers = torrent.Leechers + torrent.Seeders; + + results.Add(torrentInfo); + } + + return results; + } + + private string GetGuid(RarbgTorrent torrent) + { + var match = RegexGuid.Match(torrent.DownloadUrl); + + if (match.Success) + { + return string.Format("rarbg-{0}", match.Groups[1].Value); + } + else + { + return string.Format("rarbg-{0}", torrent.DownloadUrl); + } + } + + } +} diff --git a/src/NzbDrone.Core/Indexers/Rarbg/RarbgRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Rarbg/RarbgRequestGenerator.cs new file mode 100644 index 000000000..673a6bef1 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Rarbg/RarbgRequestGenerator.cs @@ -0,0 +1,96 @@ +using System; +using System.Collections.Generic; +using NzbDrone.Common.Extensions; +using NzbDrone.Common.Http; +using NzbDrone.Core.IndexerSearch.Definitions; + +namespace NzbDrone.Core.Indexers.Rarbg +{ + public class RarbgRequestGenerator : IIndexerRequestGenerator + { + private readonly IRarbgTokenProvider _tokenProvider; + + public RarbgSettings Settings { get; set; } + + public RarbgRequestGenerator(IRarbgTokenProvider tokenProvider) + { + _tokenProvider = tokenProvider; + } + + public virtual IList> GetRecentRequests() + { + var pageableRequests = new List>(); + + pageableRequests.AddIfNotNull(GetPagedRequests("list", null, null)); + + return pageableRequests; + } + + public virtual IList> GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria) + { + var pageableRequests = new List>(); + + pageableRequests.AddIfNotNull(GetPagedRequests("search", searchCriteria.Series.TvdbId, "S{0:00}E{1:00}", searchCriteria.SeasonNumber, searchCriteria.EpisodeNumber)); + + return pageableRequests; + } + + public virtual IList> GetSearchRequests(SeasonSearchCriteria searchCriteria) + { + var pageableRequests = new List>(); + + pageableRequests.AddIfNotNull(GetPagedRequests("search", searchCriteria.Series.TvdbId, "S{0:00}", searchCriteria.SeasonNumber)); + + return pageableRequests; + } + + public virtual IList> GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria) + { + var pageableRequests = new List>(); + + pageableRequests.AddIfNotNull(GetPagedRequests("search", searchCriteria.Series.TvdbId, "\"{0:yyyy MM dd}\"", searchCriteria.AirDate)); + + return pageableRequests; + } + + public virtual IList> GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria) + { + return new List>(); + } + + public virtual IList> GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria) + { + return new List>(); + } + + private IEnumerable GetPagedRequests(string mode, int? tvdbId, string query, params object[] args) + { + var httpRequest = new HttpRequest(Settings.BaseUrl + "/pubapi.php", HttpAccept.Json); + + httpRequest.AddQueryParam("mode", mode); + + if (tvdbId.HasValue) + { + httpRequest.AddQueryParam("search_tvdb", tvdbId.Value.ToString()); + } + + if (query.IsNotNullOrWhiteSpace()) + { + httpRequest.AddQueryParam("search_string", string.Format(query, args)); + } + + if (!Settings.RankedOnly) + { + httpRequest.AddQueryParam("ranked", "0"); + } + + httpRequest.AddQueryParam("category", "18;41"); + httpRequest.AddQueryParam("limit", "100"); + httpRequest.AddQueryParam("token", _tokenProvider.GetToken(Settings)); + httpRequest.AddQueryParam("format", "json_extended"); + httpRequest.AddQueryParam("response_type", "json"); + + yield return new IndexerRequest(httpRequest); + } + } +} diff --git a/src/NzbDrone.Core/Indexers/Rarbg/RarbgSettings.cs b/src/NzbDrone.Core/Indexers/Rarbg/RarbgSettings.cs new file mode 100644 index 000000000..41a6573aa --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Rarbg/RarbgSettings.cs @@ -0,0 +1,38 @@ +using System; +using FluentValidation; +using NzbDrone.Core.Annotations; +using NzbDrone.Core.ThingiProvider; +using NzbDrone.Core.Validation; + +namespace NzbDrone.Core.Indexers.Rarbg +{ + public class RarbgSettingsValidator : AbstractValidator + { + public RarbgSettingsValidator() + { + RuleFor(c => c.BaseUrl).ValidRootUrl(); + } + } + + public class RarbgSettings : IProviderConfig + { + private static readonly RarbgSettingsValidator Validator = new RarbgSettingsValidator(); + + public RarbgSettings() + { + BaseUrl = "https://torrentapi.org"; + RankedOnly = false; + } + + [FieldDefinition(0, Label = "API URL", HelpText = "URL to Rarbg api, not the website.")] + public string BaseUrl { get; set; } + + [FieldDefinition(1, Label = "Ranked Only", HelpText = "Only include ranked results.")] + public bool RankedOnly { get; set; } + + public NzbDroneValidationResult Validate() + { + return new NzbDroneValidationResult(Validator.Validate(this)); + } + } +} \ No newline at end of file diff --git a/src/NzbDrone.Core/Indexers/Rarbg/RarbgTokenProvider.cs b/src/NzbDrone.Core/Indexers/Rarbg/RarbgTokenProvider.cs new file mode 100644 index 000000000..1dadcb83f --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Rarbg/RarbgTokenProvider.cs @@ -0,0 +1,39 @@ +using System; +using Newtonsoft.Json.Linq; +using NLog; +using NzbDrone.Common.Cache; +using NzbDrone.Common.Http; + +namespace NzbDrone.Core.Indexers.Rarbg +{ + public interface IRarbgTokenProvider + { + string GetToken(RarbgSettings settings); + } + + public class RarbgTokenProvider : IRarbgTokenProvider + { + private readonly IHttpClient _httpClient; + private readonly ICached _tokenCache; + private readonly Logger _logger; + + public RarbgTokenProvider(IHttpClient httpClient, ICacheManager cacheManager, Logger logger) + { + _httpClient = httpClient; + _tokenCache = cacheManager.GetCache(GetType()); + _logger = logger; + } + + public string GetToken(RarbgSettings settings) + { + return _tokenCache.Get(settings.BaseUrl, () => + { + var url = settings.BaseUrl.Trim('/') + "/pubapi.php?get_token=get_token&format=json&response_type=json"; + + var response = _httpClient.Get(new HttpRequest(url, HttpAccept.Json)); + + return response.Resource["token"].ToString(); + }, TimeSpan.FromMinutes(14.0)); + } + } +} diff --git a/src/NzbDrone.Core/Indexers/Rarbg/RarbgTorrent.cs b/src/NzbDrone.Core/Indexers/Rarbg/RarbgTorrent.cs new file mode 100644 index 000000000..304ce3abe --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Rarbg/RarbgTorrent.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace NzbDrone.Core.Indexers.Rarbg +{ + public class RarbgTorrent + { + [JsonProperty("f")] + public string Title { get; set; } + [JsonProperty("c")] + public string Category { get; set; } + [JsonProperty("d")] + public string DownloadUrl { get; set; } + [JsonProperty("s")] + public int Seeders { get; set; } + [JsonProperty("l")] + public int Leechers { get; set; } + [JsonProperty("t")] + public long Size { get; set; } + [JsonProperty("u")] + public DateTime PublishDate { get; set; } + } +} diff --git a/src/NzbDrone.Core/NzbDrone.Core.csproj b/src/NzbDrone.Core/NzbDrone.Core.csproj index c953ec0b0..c2b8d04d9 100644 --- a/src/NzbDrone.Core/NzbDrone.Core.csproj +++ b/src/NzbDrone.Core/NzbDrone.Core.csproj @@ -515,6 +515,12 @@ + + + + + +