diff --git a/src/NzbDrone.Core.Test/Files/Indexers/TorrentRss/AnimeTosho_NoSize.xml b/src/NzbDrone.Core.Test/Files/Indexers/TorrentRss/AnimeTosho_NoSize.xml new file mode 100644 index 000000000..4befe325f --- /dev/null +++ b/src/NzbDrone.Core.Test/Files/Indexers/TorrentRss/AnimeTosho_NoSize.xml @@ -0,0 +1,41 @@ + + + + + + + Anime Tosho + https://animetosho.org/ + Latest releases feed + + en-us + 30 + Tue, 02 Aug 2016 13:48:04 +0000 + + + [FFF] Ore Monogatari!! - Vol.01 [BD][720p-AAC] + + Total Size: 1.366 GB
Download Links: 6 file(s)]]> + https://animetosho.org/view/fff-ore-monogatari-vol-01-bd-720p-aac.1009077 + https://animetosho.org/view/fff-ore-monogatari-vol-01-bd-720p-aac.1009077 + + + TokyoTosho + Tue, 02 Aug 2016 13:48:04 +0000 + https://animetosho.org/view/1009077 + + + DAYS - 05 (1280x720 HEVC2 AAC).mkv + + Total Size: 158.1 MB
Download Links: Go4Up | Jheberg | MultiUp | OpenLoad | Sendspace | SolidFiles | UsersCloud]]> + https://animetosho.org/view/days-05-1280x720-hevc2-aac-mkv.1009055 + https://animetosho.org/view/days-05-1280x720-hevc2-aac-mkv.1009055 + + + TokyoTosho + Tue, 02 Aug 2016 12:50:06 +0000 + https://animetosho.org/view/1009055 + + + + diff --git a/src/NzbDrone.Core.Test/IndexerTests/TorrentRssIndexerTests/TorrentRssIndexerFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/TorrentRssIndexerTests/TorrentRssIndexerFixture.cs index 01f689d91..77de77ab3 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/TorrentRssIndexerTests/TorrentRssIndexerFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/TorrentRssIndexerTests/TorrentRssIndexerFixture.cs @@ -175,5 +175,48 @@ public void should_parse_recent_feed_from_ExtraTorrents() torrentInfo.Peers.Should().NotHaveValue(); torrentInfo.Seeders.Should().NotHaveValue(); } + + [Test] + public void should_parse_recent_feed_from_AnimeTosho_without_size() + { + GivenRecentFeedResponse("TorrentRss/AnimeTosho_NoSize.xml"); + + var releases = Subject.FetchRecent(); + + releases.Should().HaveCount(2); + releases.First().Should().BeOfType(); + + var torrentInfo = releases.First() as TorrentInfo; + + torrentInfo.Title.Should().Be("[FFF] Ore Monogatari!! - Vol.01 [BD][720p-AAC]"); + torrentInfo.DownloadProtocol.Should().Be(DownloadProtocol.Torrent); + torrentInfo.DownloadUrl.Should().Be("http://storage.animetosho.org/torrents/85a570f25067f69b3c83b901ce6c00c491345288/%5BFFF%5D%20Ore%20Monogatari%21%21%20-%20Vol.01%20%5BBD%5D%5B720p-AAC%5D.torrent"); + torrentInfo.InfoUrl.Should().BeNullOrEmpty(); + torrentInfo.CommentUrl.Should().Be("https://animetosho.org/view/fff-ore-monogatari-vol-01-bd-720p-aac.1009077"); + torrentInfo.Indexer.Should().Be(Subject.Definition.Name); + torrentInfo.PublishDate.Should().Be(DateTime.Parse("Tue, 02 Aug 2016 13:48:04 +0000").ToUniversalTime()); + torrentInfo.Size.Should().Be((long)Math.Round((double)1.366m * 1024L * 1024L * 1024L)); + torrentInfo.InfoHash.Should().BeNull(); + torrentInfo.MagnetUrl.Should().BeNull(); + torrentInfo.Peers.Should().NotHaveValue(); + torrentInfo.Seeders.Should().NotHaveValue(); + } + + [Test] + public void should_parse_multi_enclosure_from_AnimeTosho() + { + GivenRecentFeedResponse("TorrentRss/AnimeTosho_NoSize.xml"); + + var releases = Subject.FetchRecent(); + + releases.Should().HaveCount(2); + releases.Last().Should().BeOfType(); + + var torrentInfo = releases.Last() as TorrentInfo; + + torrentInfo.Title.Should().Be("DAYS - 05 (1280x720 HEVC2 AAC).mkv"); + torrentInfo.DownloadProtocol.Should().Be(DownloadProtocol.Torrent); + torrentInfo.DownloadUrl.Should().Be("http://storage.animetosho.org/torrents/4b58360143d59a55cbd922397a3eaa378165f3ff/DAYS%20-%2005%20%281280x720%20HEVC2%20AAC%29.torrent"); + } } } diff --git a/src/NzbDrone.Core.Test/IndexerTests/TorrentRssIndexerTests/TorrentRssSettingsDetectorFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/TorrentRssIndexerTests/TorrentRssSettingsDetectorFixture.cs index 349f257bb..cf734c24d 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/TorrentRssIndexerTests/TorrentRssSettingsDetectorFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/TorrentRssIndexerTests/TorrentRssSettingsDetectorFixture.cs @@ -196,6 +196,26 @@ public void should_reject_rss_settings_for_AwesomeHD() settings.Should().BeNull(); } + [Test] + public void should_detect_rss_settings_for_AnimeTosho_without_size() + { + _indexerSettings.AllowZeroSize = true; + + GivenRecentFeedResponse("TorrentRss/AnimeTosho_NoSize.xml"); + + var settings = Subject.Detect(_indexerSettings); + + settings.ShouldBeEquivalentTo(new TorrentRssIndexerParserSettings + { + UseEZTVFormat = false, + UseEnclosureUrl = true, + UseEnclosureLength = false, + ParseSizeInDescription = true, + ParseSeedersInDescription = false, + SizeElementName = null + }); + } + [TestCase("BitMeTv/BitMeTv.xml")] [TestCase("Fanzub/fanzub.xml")] [TestCase("KickassTorrents/KickassTorrents.xml")] diff --git a/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj b/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj index f4e45987d..8edd74b9d 100644 --- a/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj +++ b/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj @@ -426,6 +426,9 @@ Always + + Always + Always diff --git a/src/NzbDrone.Core/Indexers/Newznab/NewznabRssParser.cs b/src/NzbDrone.Core/Indexers/Newznab/NewznabRssParser.cs index 1ebe8945d..16c4dea9b 100644 --- a/src/NzbDrone.Core/Indexers/Newznab/NewznabRssParser.cs +++ b/src/NzbDrone.Core/Indexers/Newznab/NewznabRssParser.cs @@ -11,6 +11,11 @@ public class NewznabRssParser : RssParser { public const string ns = "{http://www.newznab.com/DTD/2010/feeds/attributes/}"; + public NewznabRssParser() + { + PreferredEnclosureMimeType = "application/x-nzb"; + } + protected override bool PreProcess(IndexerResponse indexerResponse) { var xdoc = LoadXmlDocument(indexerResponse); @@ -52,7 +57,7 @@ protected override ReleaseInfo ProcessItem(XElement item, ReleaseInfo releaseInf protected override ReleaseInfo PostProcess(XElement item, ReleaseInfo releaseInfo) { - var enclosureType = item.Element("enclosure").Attribute("type").Value; + var enclosureType = GetEnclosure(item).Attribute("type").Value; if (enclosureType.Contains("application/x-bittorrent")) { throw new UnsupportedFeedException("Feed contains {0}, did you intend to add a Torznab indexer?", enclosureType); diff --git a/src/NzbDrone.Core/Indexers/RssParser.cs b/src/NzbDrone.Core/Indexers/RssParser.cs index 271c5da6b..d2c03cfa4 100644 --- a/src/NzbDrone.Core/Indexers/RssParser.cs +++ b/src/NzbDrone.Core/Indexers/RssParser.cs @@ -32,6 +32,8 @@ public class RssParser : IParseIndexerResponse // Parse "Size: 1.3 GB" or "1.3 GB" parts in the description element and use that as Size. public bool ParseSizeInDescription { get; set; } + public string PreferredEnclosureMimeType { get; set; } + private IndexerResponse _indexerResponse; public RssParser() @@ -192,7 +194,7 @@ protected virtual string GetDownloadUrl(XElement item) { if (UseEnclosureUrl) { - return ParseUrl((string)item.Element("enclosure").Attribute("url")); + return ParseUrl((string)GetEnclosure(item).Attribute("url")); } return ParseUrl((string)item.Element("link")); @@ -229,7 +231,7 @@ protected virtual long GetSize(XElement item) protected virtual long GetEnclosureLength(XElement item) { - var enclosure = item.Element("enclosure"); + var enclosure = GetEnclosure(item); if (enclosure != null) { @@ -239,6 +241,33 @@ protected virtual long GetEnclosureLength(XElement item) return 0; } + protected virtual XElement GetEnclosure(XElement item) + { + var enclosures = item.Elements("enclosure").ToArray(); + + if (enclosures.Length == 0) + { + return null; + } + + if (enclosures.Length == 1) + { + return enclosures.First(); + } + + if (PreferredEnclosureMimeType != null) + { + var preferredEnclosure = enclosures.FirstOrDefault(v => v.Attribute("type").Value == PreferredEnclosureMimeType); + + if (preferredEnclosure != null) + { + return preferredEnclosure; + } + } + + return item.Elements("enclosure").SingleOrDefault(); + } + protected IEnumerable GetItems(XDocument document) { var root = document.Root; @@ -278,7 +307,7 @@ protected virtual string ParseUrl(string value) } } - private static readonly Regex ParseSizeRegex = new Regex(@"(?(?:\d+,)*\d+(?:\.\d{1,2})?)\W?(?[KMG]i?B)(?![\w/])", + private static readonly Regex ParseSizeRegex = new Regex(@"(?(?[KMG]i?B)(?![\w/])", RegexOptions.IgnoreCase | RegexOptions.Compiled); public static long ParseSize(string sizeString, bool defaultToBinaryPrefix) diff --git a/src/NzbDrone.Core/Indexers/TorrentRss/TorrentRssSettingsDetector.cs b/src/NzbDrone.Core/Indexers/TorrentRss/TorrentRssSettingsDetector.cs index 919254abe..0c07e2a51 100644 --- a/src/NzbDrone.Core/Indexers/TorrentRss/TorrentRssSettingsDetector.cs +++ b/src/NzbDrone.Core/Indexers/TorrentRss/TorrentRssSettingsDetector.cs @@ -169,8 +169,12 @@ private TorrentRssIndexerParserSettings GetGenericTorrentRssParserSettings(Index releases = ParseResponse(parser, response); ValidateReleases(releases, indexerSettings); - if (!releases.Any(r => r.Size < ValidSizeThreshold)) + if (releases.Count(r => r.Size >= ValidSizeThreshold) > releases.Count() / 2) { + if (releases.Any(r => r.Size < ValidSizeThreshold)) + { + _logger.Debug("Feed {0} contains very small releases.", response.Request.Url); + } _logger.Trace("Feed has valid size in description."); return settings; } diff --git a/src/NzbDrone.Core/Indexers/TorrentRssParser.cs b/src/NzbDrone.Core/Indexers/TorrentRssParser.cs index af94ac174..e5451de86 100644 --- a/src/NzbDrone.Core/Indexers/TorrentRssParser.cs +++ b/src/NzbDrone.Core/Indexers/TorrentRssParser.cs @@ -17,7 +17,7 @@ public class TorrentRssParser : RssParser public TorrentRssParser() { - + PreferredEnclosureMimeType = "application/x-bittorrent"; } public IEnumerable GetItems(IndexerResponse indexerResponse)