Merge branch 'rarbg-support' into develop

src/NzbDrone.Common/Http/HttpRequest.cs

@@ -70,6 +70,11 @@ public void AddSegment(string segment, string value)
             _segments.Add(key, value);
         }
 
+        public void AddQueryParam(string segment, string value)
+        {
+            UriBuilder.SetQueryParam(segment, value);
+        }
+
         public void AddCookie(string key, string value)
         {
             Cookies[key] = value;

src/NzbDrone.Common/Http/UriExtensions.cs

@@ -14,9 +14,7 @@ public static void SetQueryParam(this UriBuilder uriBuilder, string key, object
                 query += "&";
             }
 
-            uriBuilder.Query = query.Trim('?') + (key + "=" + value);
+            uriBuilder.Query = query.Trim('?') + key + "=" + Uri.EscapeDataString(value.ToString());
         }
-
-
     }
 }

src/NzbDrone.Core.Test/Files/Indexers/Rarbg/RecentFeed.json

@@ -0,0 +1,29 @@
+[
+    {
+        "f": "Thunderbirds.Are.Go.S01E09.Slingshot.1080p.WEB-DL.AAC2.0.H.264-Coo7[rartv]",
+        "c": "TV HD Episodes",
+        "d": "magnet:?xt=urn:btih:ff4737b5230307836ec8abce6ab73727f1358bf3&dn=Thunderbirds.Are.Go.S01E09.Slingshot.1080p.WEB-DL.AAC2.0.H.264-Coo7%5Brartv%5D&tr=http%3A%2F%2Ftracker.trackerfix.com%3A80%2Fannounce&tr=udp%3A%2F%2F9.rarbg.me%3A2710&tr=udp%3A%2F%2F9.rarbg.to%3A2710&tr=udp%3A%2F%2Fopen.demonii.com%3A1337%2Fannounce",
+        "s": "44",
+        "l": "19",
+        "t": "896238116",
+        "u": "2015-05-24 19:36:09"
+    },
+    {
+        "f": "Thunderbirds.Are.Go.S01E10.Tunnels.Of.Time.720p.HDTV.x264-RDVAS[rartv]",
+        "c": "TV HD Episodes",
+        "d": "magnet:?xt=urn:btih:47bf1d7bfb72a83300bbe68d0b6aa09591e7a0a1&dn=Thunderbirds.Are.Go.S01E10.Tunnels.Of.Time.720p.HDTV.x264-RDVAS%5Brartv%5D&tr=http%3A%2F%2Ftracker.trackerfix.com%3A80%2Fannounce&tr=udp%3A%2F%2F9.rarbg.me%3A2710&tr=udp%3A%2F%2F9.rarbg.to%3A2710&tr=udp%3A%2F%2Fopen.demonii.com%3A1337%2Fannounce",
+        "s": "179",
+        "l": "125",
+        "t": "556055350",
+        "u": "2015-05-24 19:07:59"
+    },
+    {
+        "f": "Tatau.S01E06.1080p.WEB-DL.AAC2.0.H.264-BS[rartv]",
+        "c": "TV HD Episodes",
+        "d": "magnet:?xt=urn:btih:8857e9b011c7a0483351371721fa9f3ba356dd73&dn=Tatau.S01E06.1080p.WEB-DL.AAC2.0.H.264-BS%5Brartv%5D&tr=http%3A%2F%2Ftracker.trackerfix.com%3A80%2Fannounce&tr=udp%3A%2F%2F9.rarbg.me%3A2710&tr=udp%3A%2F%2F9.rarbg.to%3A2710&tr=udp%3A%2F%2Fopen.demonii.com%3A1337%2Fannounce",
+        "s": "27",
+        "l": "22",
+        "t": "1652442143",
+        "u": "2015-05-24 18:54:49"
+    }
+]

src/NzbDrone.Core.Test/IndexerTests/RarbgTests/RarbgFixture.cs

@@ -0,0 +1,59 @@
+using System;
+using System.Linq;
+using FluentAssertions;
+using Moq;
+using NUnit.Framework;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.Indexers;
+using NzbDrone.Core.Indexers.Rarbg;
+using NzbDrone.Core.Parser.Model;
+using NzbDrone.Core.Test.Framework;
+
+namespace NzbDrone.Core.Test.IndexerTests.RarbgTests
+{
+    [TestFixture]
+    public class RarbgFixture : CoreTest<Rarbg>
+    {
+        [SetUp]
+        public void Setup()
+        {
+            Subject.Definition = new IndexerDefinition()
+                {
+                    Name = "Rarbg",
+                    Settings = new RarbgSettings()
+                };
+
+            Mocker.GetMock<IRarbgTokenProvider>()
+                .Setup(v => v.GetToken(It.IsAny<RarbgSettings>()))
+                .Returns("validtoken");
+        }
+
+        [Test]
+        public void should_parse_recent_feed_from_Rarbg()
+        {
+            var recentFeed = ReadAllText(@"Files/Indexers/Rarbg/RecentFeed.json");
+
+            Mocker.GetMock<IHttpClient>()
+                .Setup(o => o.Execute(It.Is<HttpRequest>(v => v.Method == HttpMethod.GET)))
+                .Returns<HttpRequest>(r => new HttpResponse(r, new HttpHeader(), recentFeed));
+
+            var releases = Subject.FetchRecent();
+
+            releases.Should().HaveCount(3);
+            releases.First().Should().BeOfType<TorrentInfo>();
+
+            var torrentInfo = releases.First() as TorrentInfo;
+
+            torrentInfo.Title.Should().Be("Thunderbirds.Are.Go.S01E09.Slingshot.1080p.WEB-DL.AAC2.0.H.264-Coo7[rartv]");
+            torrentInfo.DownloadProtocol.Should().Be(DownloadProtocol.Torrent);
+            torrentInfo.DownloadUrl.Should().Be("magnet:?xt=urn:btih:ff4737b5230307836ec8abce6ab73727f1358bf3&dn=Thunderbirds.Are.Go.S01E09.Slingshot.1080p.WEB-DL.AAC2.0.H.264-Coo7%5Brartv%5D&tr=http%3A%2F%2Ftracker.trackerfix.com%3A80%2Fannounce&tr=udp%3A%2F%2F9.rarbg.me%3A2710&tr=udp%3A%2F%2F9.rarbg.to%3A2710&tr=udp%3A%2F%2Fopen.demonii.com%3A1337%2Fannounce");
+            torrentInfo.Indexer.Should().Be(Subject.Definition.Name);
+            torrentInfo.PublishDate.Should().Be(DateTime.Parse("2015-05-24 19:36:09"));
+            torrentInfo.Size.Should().Be(896238116);
+            torrentInfo.InfoHash.Should().BeNull();
+            torrentInfo.MagnetUrl.Should().BeNull();
+            torrentInfo.Peers.Should().Be(44+19);
+            torrentInfo.Seeders.Should().Be(44);
+        }
+    }
+}

src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj

@@ -168,6 +168,9 @@
     <Compile Include="Download\Pending\PendingReleaseServiceTests\RemoveRejectedFixture.cs" />
     <Compile Include="Download\Pending\PendingReleaseServiceTests\RemoveGrabbedFixture.cs" />
     <Compile Include="Download\Pending\PendingReleaseServiceTests\AddFixture.cs" />
+    <None Include="Files\Indexers\Rarbg\RecentFeed.json">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
     <Compile Include="FluentTest.cs" />
     <Compile Include="Framework\CoreTest.cs" />
     <Compile Include="Framework\DbTest.cs" />
@@ -207,6 +210,7 @@
     <Compile Include="IndexerTests\HDBitsTests\HDBitsFixture.cs" />
     <Compile Include="IndexerTests\IndexerServiceFixture.cs" />
     <Compile Include="IndexerTests\IntegrationTests\IndexerIntegrationTests.cs" />
+    <Compile Include="IndexerTests\RarbgTests\RarbgFixture.cs" />
     <Compile Include="IndexerTests\TorznabTests\TorznabFixture.cs" />
     <Compile Include="IndexerTests\TorznabTests\TorznabRequestGeneratorFixture.cs" />
     <Compile Include="IndexerTests\TorznabTests\TorznabSettingFixture.cs" />

src/NzbDrone.Core/Indexers/Rarbg/Rarbg.cs

@@ -0,0 +1,50 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using NLog;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.Configuration;
+using NzbDrone.Core.Parser;
+using NzbDrone.Core.Parser.Model;
+
+namespace NzbDrone.Core.Indexers.Rarbg
+{
+    public class Rarbg : HttpIndexerBase<RarbgSettings>
+    {
+        private readonly IRarbgTokenProvider _tokenProvider;
+        private static DateTime _lastFetch;
+
+        public override string Name { get { return "Rarbg"; } }
+        
+        public override DownloadProtocol Protocol { get { return DownloadProtocol.Torrent; } }
+
+        public Rarbg(IRarbgTokenProvider tokenProvider, IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger)
+            : base(httpClient, configService, parsingService, logger)
+        {
+            _tokenProvider = tokenProvider;
+        }
+
+        public override IIndexerRequestGenerator GetRequestGenerator()
+        {
+            return new RarbgRequestGenerator(_tokenProvider) { Settings = Settings };
+        }
+
+        public override IParseIndexerResponse GetParser()
+        {
+            return new RarbgParser();
+        }
+
+        protected override IList<ReleaseInfo> FetchPage(IndexerRequest request, IParseIndexerResponse parser)
+        {
+            var delay = _lastFetch + TimeSpan.FromSeconds(10) - DateTime.UtcNow;
+            if (delay.TotalSeconds > 0)
+            {
+                Thread.Sleep(delay);
+            }
+
+            _lastFetch = DateTime.UtcNow;
+
+            return base.FetchPage(request, parser);
+        }
+    }
+}

src/NzbDrone.Core/Indexers/Rarbg/RarbgParser.cs

@@ -0,0 +1,79 @@
+using System;
+using System.Collections.Generic;
+using System.Net;
+using System.Text.RegularExpressions;
+using Newtonsoft.Json.Linq;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.Indexers.Exceptions;
+using NzbDrone.Core.Parser.Model;
+
+namespace NzbDrone.Core.Indexers.Rarbg
+{
+    public class RarbgParser : IParseIndexerResponse
+    {
+        private static readonly Regex RegexGuid = new Regex(@"^magnet:\?xt=urn:btih:([a-f0-9]+)", RegexOptions.Compiled);
+
+        public IList<ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
+        {
+            var results = new List<ReleaseInfo>();
+
+            switch (indexerResponse.HttpResponse.StatusCode)
+            {
+                default:
+                    if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
+                    {
+                        throw new IndexerException(indexerResponse, "Indexer API call returned an unexpected StatusCode [{0}]", indexerResponse.HttpResponse.StatusCode);
+                    }
+                    break;
+            }
+
+            var jsonResponse = new HttpResponse<object>(indexerResponse.HttpResponse).Resource as JContainer;
+
+            var errorResponse = jsonResponse as JObject;
+            if (errorResponse != null && errorResponse["error"] != null)
+            {
+                var error = errorResponse["error"].ToString();
+                if (error == "No results found")
+                {
+                    return results;
+                }
+
+                throw new IndexerException(indexerResponse, "Indexer API call returned an error [{0}]", errorResponse["error"]);
+            }
+
+            var torrentResponse = jsonResponse.ToObject<List<RarbgTorrent>>();
+
+            foreach (var torrent in torrentResponse)
+            {
+                var torrentInfo = new TorrentInfo();
+
+                torrentInfo.Guid = GetGuid(torrent);
+                torrentInfo.Title = torrent.Title;
+                torrentInfo.Size = torrent.Size;
+                torrentInfo.DownloadUrl = torrent.DownloadUrl;
+                torrentInfo.PublishDate = torrent.PublishDate;
+                torrentInfo.Seeders = torrent.Seeders;
+                torrentInfo.Peers = torrent.Leechers + torrent.Seeders;
+
+                results.Add(torrentInfo);
+            }
+
+            return results;
+        }
+
+        private string GetGuid(RarbgTorrent torrent)
+        {
+            var match = RegexGuid.Match(torrent.DownloadUrl);
+
+            if (match.Success)
+            {
+                return string.Format("rarbg-{0}", match.Groups[1].Value);
+            }
+            else
+            {
+                return string.Format("rarbg-{0}", torrent.DownloadUrl);
+            }
+        }
+
+    }
+}

src/NzbDrone.Core/Indexers/Rarbg/RarbgRequestGenerator.cs

@@ -0,0 +1,96 @@
+using System;
+using System.Collections.Generic;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.IndexerSearch.Definitions;
+
+namespace NzbDrone.Core.Indexers.Rarbg
+{
+    public class RarbgRequestGenerator : IIndexerRequestGenerator
+    {
+        private readonly IRarbgTokenProvider _tokenProvider;
+
+        public RarbgSettings Settings { get; set; }
+
+        public RarbgRequestGenerator(IRarbgTokenProvider tokenProvider)
+        {
+            _tokenProvider = tokenProvider;
+        }
+
+        public virtual IList<IEnumerable<IndexerRequest>> GetRecentRequests()
+        {
+            var pageableRequests = new List<IEnumerable<IndexerRequest>>();
+
+            pageableRequests.AddIfNotNull(GetPagedRequests("list", null, null));
+
+            return pageableRequests;
+        }
+
+        public virtual IList<IEnumerable<IndexerRequest>> GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria)
+        {
+            var pageableRequests = new List<IEnumerable<IndexerRequest>>();
+
+            pageableRequests.AddIfNotNull(GetPagedRequests("search", searchCriteria.Series.TvdbId, "S{0:00}E{1:00}", searchCriteria.SeasonNumber, searchCriteria.EpisodeNumber));
+
+            return pageableRequests;
+        }
+
+        public virtual IList<IEnumerable<IndexerRequest>> GetSearchRequests(SeasonSearchCriteria searchCriteria)
+        {
+            var pageableRequests = new List<IEnumerable<IndexerRequest>>();
+
+            pageableRequests.AddIfNotNull(GetPagedRequests("search", searchCriteria.Series.TvdbId, "S{0:00}", searchCriteria.SeasonNumber));
+
+            return pageableRequests;
+        }
+
+        public virtual IList<IEnumerable<IndexerRequest>> GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria)
+        {
+            var pageableRequests = new List<IEnumerable<IndexerRequest>>();
+
+            pageableRequests.AddIfNotNull(GetPagedRequests("search", searchCriteria.Series.TvdbId, "\"{0:yyyy MM dd}\"", searchCriteria.AirDate));
+
+            return pageableRequests;
+        }
+
+        public virtual IList<IEnumerable<IndexerRequest>> GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria)
+        {
+            return new List<IEnumerable<IndexerRequest>>();
+        }
+
+        public virtual IList<IEnumerable<IndexerRequest>> GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria)
+        {
+            return new List<IEnumerable<IndexerRequest>>();
+        }
+
+        private IEnumerable<IndexerRequest> GetPagedRequests(string mode, int? tvdbId, string query, params object[] args)
+        {
+            var httpRequest = new HttpRequest(Settings.BaseUrl + "/pubapi.php", HttpAccept.Json);
+
+            httpRequest.AddQueryParam("mode", mode);
+
+            if (tvdbId.HasValue)
+            {
+                httpRequest.AddQueryParam("search_tvdb", tvdbId.Value.ToString());
+            }
+
+            if (query.IsNotNullOrWhiteSpace())
+            {
+                httpRequest.AddQueryParam("search_string", string.Format(query, args));
+            }
+
+            if (!Settings.RankedOnly)
+            {
+                httpRequest.AddQueryParam("ranked", "0");
+            }
+
+            httpRequest.AddQueryParam("category", "18;41");
+            httpRequest.AddQueryParam("limit", "100");
+            httpRequest.AddQueryParam("token", _tokenProvider.GetToken(Settings));
+            httpRequest.AddQueryParam("format", "json_extended");
+            httpRequest.AddQueryParam("response_type", "json");
+
+            yield return new IndexerRequest(httpRequest);
+        }
+    }
+}

src/NzbDrone.Core/Indexers/Rarbg/RarbgSettings.cs

@@ -0,0 +1,38 @@
+using System;
+using FluentValidation;
+using NzbDrone.Core.Annotations;
+using NzbDrone.Core.ThingiProvider;
+using NzbDrone.Core.Validation;
+
+namespace NzbDrone.Core.Indexers.Rarbg
+{
+    public class RarbgSettingsValidator : AbstractValidator<RarbgSettings>
+    {
+        public RarbgSettingsValidator()
+        {
+            RuleFor(c => c.BaseUrl).ValidRootUrl();
+        }
+    }
+
+    public class RarbgSettings : IProviderConfig
+    {
+        private static readonly RarbgSettingsValidator Validator = new RarbgSettingsValidator();
+
+        public RarbgSettings()
+        {
+            BaseUrl = "https://torrentapi.org";
+            RankedOnly = false;
+        }
+
+        [FieldDefinition(0, Label = "API URL", HelpText = "URL to Rarbg api, not the website.")]
+        public string BaseUrl { get; set; }
+
+        [FieldDefinition(1, Label = "Ranked Only", HelpText = "Only include ranked results.")]
+        public bool RankedOnly { get; set; }
+
+        public NzbDroneValidationResult Validate()
+        {
+            return new NzbDroneValidationResult(Validator.Validate(this));
+        }
+    }
+}

src/NzbDrone.Core/Indexers/Rarbg/RarbgTokenProvider.cs

@@ -0,0 +1,39 @@
+using System;
+using Newtonsoft.Json.Linq;
+using NLog;
+using NzbDrone.Common.Cache;
+using NzbDrone.Common.Http;
+
+namespace NzbDrone.Core.Indexers.Rarbg
+{
+    public interface IRarbgTokenProvider
+    {
+        string GetToken(RarbgSettings settings);
+    }
+
+    public class RarbgTokenProvider : IRarbgTokenProvider
+    {
+        private readonly IHttpClient _httpClient;
+        private readonly ICached<string> _tokenCache;
+        private readonly Logger _logger;
+
+        public RarbgTokenProvider(IHttpClient httpClient, ICacheManager cacheManager, Logger logger)
+        {
+            _httpClient = httpClient;
+            _tokenCache = cacheManager.GetCache<string>(GetType());
+            _logger = logger;
+        }
+
+        public string GetToken(RarbgSettings settings)
+        {
+            return _tokenCache.Get(settings.BaseUrl, () =>
+                {
+                    var url = settings.BaseUrl.Trim('/') + "/pubapi.php?get_token=get_token&format=json&response_type=json";
+
+                    var response = _httpClient.Get<JObject>(new HttpRequest(url, HttpAccept.Json));
+
+                    return response.Resource["token"].ToString();
+                }, TimeSpan.FromMinutes(14.0));
+        }
+    }
+}

src/NzbDrone.Core/Indexers/Rarbg/RarbgTorrent.cs

@@ -0,0 +1,24 @@
+using System;
+using System.Collections.Generic;
+using Newtonsoft.Json;
+
+namespace NzbDrone.Core.Indexers.Rarbg
+{
+    public class RarbgTorrent
+    {
+        [JsonProperty("f")]
+        public string Title { get; set; }
+        [JsonProperty("c")]
+        public string Category { get; set; }
+        [JsonProperty("d")]
+        public string DownloadUrl { get; set; }
+        [JsonProperty("s")]
+        public int Seeders { get; set; }
+        [JsonProperty("l")]
+        public int Leechers { get; set; }
+        [JsonProperty("t")]
+        public long Size { get; set; }
+        [JsonProperty("u")]
+        public DateTime PublishDate { get; set; }
+    }
+}

src/NzbDrone.Core/NzbDrone.Core.csproj

@@ -515,6 +515,12 @@
     <Compile Include="Indexers\Omgwtfnzbs\OmgwtfnzbsRssParser.cs" />
     <Compile Include="Indexers\Omgwtfnzbs\OmgwtfnzbsSettings.cs" />
     <Compile Include="Indexers\HttpIndexerBase.cs" />
+    <Compile Include="Indexers\Rarbg\Rarbg.cs" />
+    <Compile Include="Indexers\Rarbg\RarbgRequestGenerator.cs" />
+    <Compile Include="Indexers\Rarbg\RarbgTorrent.cs" />
+    <Compile Include="Indexers\Rarbg\RarbgSettings.cs" />
+    <Compile Include="Indexers\Rarbg\RarbgParser.cs" />
+    <Compile Include="Indexers\Rarbg\RarbgTokenProvider.cs" />
     <Compile Include="Indexers\RssIndexerRequestGenerator.cs" />
     <Compile Include="Indexers\RssParser.cs" />
     <Compile Include="Indexers\RssSyncCommand.cs" />