From 6eb678cd921fca91b498f3b71984a1798c6c28c0 Mon Sep 17 00:00:00 2001 From: Leonardo Galli Date: Sun, 1 Apr 2018 20:21:24 +0200 Subject: [PATCH] Added: Radarr now stores cookies for PTP beyond restarts. Should limit active sessions correctly. (#2643) Fixes #2626 --- package-lock.json | 796 ++++++++++++++++++ .../.idea.NzbDrone/.idea/contentModel.xml | 34 +- src/.idea/.idea.NzbDrone/riderModule.iml | 100 +-- .../Http/HttpClientFixture.cs | 228 ++++- src/NzbDrone.Common/Http/HttpClient.cs | 85 +- src/NzbDrone.Common/Http/HttpRequest.cs | 4 + src/NzbDrone.Common/Http/HttpResponse.cs | 18 +- .../144_add_cookies_to_indexer_status.cs | 16 + src/NzbDrone.Core/Datastore/TableMapping.cs | 1 + .../AwesomeHD/AwesomeHDRequestGenerator.cs | 3 + .../Indexers/AwesomeHD/AwesomeHDRssParser.cs | 2 + .../Indexers/HDBits/HDBitsParser.cs | 5 +- .../Indexers/HDBits/HDBitsRequestGenerator.cs | 4 + src/NzbDrone.Core/Indexers/HttpIndexerBase.cs | 50 +- .../Indexers/IIndexerRequestGenerator.cs | 4 + .../IPTorrents/IPTorrentsRequestGenerator.cs | 3 + .../Indexers/IProcessIndexerResponse.cs | 4 +- src/NzbDrone.Core/Indexers/IndexerStatus.cs | 4 + .../Indexers/IndexerStatusService.cs | 24 + .../Newznab/NewznabRequestGenerator.cs | 3 + .../Indexers/Nyaa/NyaaRequestGenerator.cs | 3 + .../Omgwtfnzbs/OmgwtfnzbsRequestGenerator.cs | 3 + .../Indexers/PassThePopcorn/PassThePopcorn.cs | 20 +- .../PassThePopcorn/PassThePopcornParser.cs | 20 +- .../PassThePopcornRequestGenerator.cs | 62 +- .../Indexers/Rarbg/RarbgParser.cs | 3 + .../Indexers/Rarbg/RarbgRequestGenerator.cs | 4 + .../Indexers/RssIndexerRequestGenerator.cs | 4 + src/NzbDrone.Core/Indexers/RssParser.cs | 2 + .../TorrentPotato/TorrentPotatoParser.cs | 2 + .../TorrentPotatoRequestGenerator.cs | 3 + .../TorrentRssIndexerRequestGenerator.cs | 3 + src/NzbDrone.Core/NzbDrone.Core.csproj | 1 + 33 files changed, 1301 insertions(+), 217 deletions(-) create mode 100644 src/NzbDrone.Core/Datastore/Migration/144_add_cookies_to_indexer_status.cs diff --git a/package-lock.json b/package-lock.json index 774ae1f9c..c61e58f62 100644 --- a/package-lock.json +++ b/package-lock.json @@ -422,6 +422,7 @@ "requires": { "anymatch": "1.3.2", "async-each": "1.0.1", + "fsevents": "1.1.3", "glob-parent": "2.0.0", "inherits": "2.0.3", "is-binary-path": "1.0.1", @@ -1034,6 +1035,795 @@ "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" }, + "fsevents": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.1.3.tgz", + "integrity": "sha512-WIr7iDkdmdbxu/Gh6eKEZJL6KPE74/5MEsf2whTOFNxbIoIixogroLdKYqB6FDav4Wavh/lZdzzd3b2KxIXC5Q==", + "optional": true, + "requires": { + "nan": "2.10.0", + "node-pre-gyp": "0.6.39" + }, + "dependencies": { + "abbrev": { + "version": "1.1.0", + "bundled": true, + "optional": true + }, + "ajv": { + "version": "4.11.8", + "bundled": true, + "optional": true, + "requires": { + "co": "4.6.0", + "json-stable-stringify": "1.0.1" + } + }, + "ansi-regex": { + "version": "2.1.1", + "bundled": true + }, + "aproba": { + "version": "1.1.1", + "bundled": true, + "optional": true + }, + "are-we-there-yet": { + "version": "1.1.4", + "bundled": true, + "optional": true, + "requires": { + "delegates": "1.0.0", + "readable-stream": "2.2.9" + } + }, + "asn1": { + "version": "0.2.3", + "bundled": true, + "optional": true + }, + "assert-plus": { + "version": "0.2.0", + "bundled": true, + "optional": true + }, + "asynckit": { + "version": "0.4.0", + "bundled": true, + "optional": true + }, + "aws-sign2": { + "version": "0.6.0", + "bundled": true, + "optional": true + }, + "aws4": { + "version": "1.6.0", + "bundled": true, + "optional": true + }, + "balanced-match": { + "version": "0.4.2", + "bundled": true + }, + "bcrypt-pbkdf": { + "version": "1.0.1", + "bundled": true, + "optional": true, + "requires": { + "tweetnacl": "0.14.5" + } + }, + "block-stream": { + "version": "0.0.9", + "bundled": true, + "requires": { + "inherits": "2.0.3" + } + }, + "boom": { + "version": "2.10.1", + "bundled": true, + "requires": { + "hoek": "2.16.3" + } + }, + "brace-expansion": { + "version": "1.1.7", + "bundled": true, + "requires": { + "balanced-match": "0.4.2", + "concat-map": "0.0.1" + } + }, + "buffer-shims": { + "version": "1.0.0", + "bundled": true + }, + "caseless": { + "version": "0.12.0", + "bundled": true, + "optional": true + }, + "co": { + "version": "4.6.0", + "bundled": true, + "optional": true + }, + "code-point-at": { + "version": "1.1.0", + "bundled": true + }, + "combined-stream": { + "version": "1.0.5", + "bundled": true, + "requires": { + "delayed-stream": "1.0.0" + } + }, + "concat-map": { + "version": "0.0.1", + "bundled": true + }, + "console-control-strings": { + "version": "1.1.0", + "bundled": true + }, + "core-util-is": { + "version": "1.0.2", + "bundled": true + }, + "cryptiles": { + "version": "2.0.5", + "bundled": true, + "requires": { + "boom": "2.10.1" + } + }, + "dashdash": { + "version": "1.14.1", + "bundled": true, + "optional": true, + "requires": { + "assert-plus": "1.0.0" + }, + "dependencies": { + "assert-plus": { + "version": "1.0.0", + "bundled": true, + "optional": true + } + } + }, + "debug": { + "version": "2.6.8", + "bundled": true, + "optional": true, + "requires": { + "ms": "2.0.0" + } + }, + "deep-extend": { + "version": "0.4.2", + "bundled": true, + "optional": true + }, + "delayed-stream": { + "version": "1.0.0", + "bundled": true + }, + "delegates": { + "version": "1.0.0", + "bundled": true, + "optional": true + }, + "detect-libc": { + "version": "1.0.2", + "bundled": true, + "optional": true + }, + "ecc-jsbn": { + "version": "0.1.1", + "bundled": true, + "optional": true, + "requires": { + "jsbn": "0.1.1" + } + }, + "extend": { + "version": "3.0.1", + "bundled": true, + "optional": true + }, + "extsprintf": { + "version": "1.0.2", + "bundled": true + }, + "forever-agent": { + "version": "0.6.1", + "bundled": true, + "optional": true + }, + "form-data": { + "version": "2.1.4", + "bundled": true, + "optional": true, + "requires": { + "asynckit": "0.4.0", + "combined-stream": "1.0.5", + "mime-types": "2.1.15" + } + }, + "fs.realpath": { + "version": "1.0.0", + "bundled": true + }, + "fstream": { + "version": "1.0.11", + "bundled": true, + "requires": { + "graceful-fs": "4.1.11", + "inherits": "2.0.3", + "mkdirp": "0.5.1", + "rimraf": "2.6.1" + } + }, + "fstream-ignore": { + "version": "1.0.5", + "bundled": true, + "optional": true, + "requires": { + "fstream": "1.0.11", + "inherits": "2.0.3", + "minimatch": "3.0.4" + } + }, + "gauge": { + "version": "2.7.4", + "bundled": true, + "optional": true, + "requires": { + "aproba": "1.1.1", + "console-control-strings": "1.1.0", + "has-unicode": "2.0.1", + "object-assign": "4.1.1", + "signal-exit": "3.0.2", + "string-width": "1.0.2", + "strip-ansi": "3.0.1", + "wide-align": "1.1.2" + } + }, + "getpass": { + "version": "0.1.7", + "bundled": true, + "optional": true, + "requires": { + "assert-plus": "1.0.0" + }, + "dependencies": { + "assert-plus": { + "version": "1.0.0", + "bundled": true, + "optional": true + } + } + }, + "glob": { + "version": "7.1.2", + "bundled": true, + "requires": { + "fs.realpath": "1.0.0", + "inflight": "1.0.6", + "inherits": "2.0.3", + "minimatch": "3.0.4", + "once": "1.4.0", + "path-is-absolute": "1.0.1" + } + }, + "graceful-fs": { + "version": "4.1.11", + "bundled": true + }, + "har-schema": { + "version": "1.0.5", + "bundled": true, + "optional": true + }, + "har-validator": { + "version": "4.2.1", + "bundled": true, + "optional": true, + "requires": { + "ajv": "4.11.8", + "har-schema": "1.0.5" + } + }, + "has-unicode": { + "version": "2.0.1", + "bundled": true, + "optional": true + }, + "hawk": { + "version": "3.1.3", + "bundled": true, + "requires": { + "boom": "2.10.1", + "cryptiles": "2.0.5", + "hoek": "2.16.3", + "sntp": "1.0.9" + } + }, + "hoek": { + "version": "2.16.3", + "bundled": true + }, + "http-signature": { + "version": "1.1.1", + "bundled": true, + "optional": true, + "requires": { + "assert-plus": "0.2.0", + "jsprim": "1.4.0", + "sshpk": "1.13.0" + } + }, + "inflight": { + "version": "1.0.6", + "bundled": true, + "requires": { + "once": "1.4.0", + "wrappy": "1.0.2" + } + }, + "inherits": { + "version": "2.0.3", + "bundled": true + }, + "ini": { + "version": "1.3.4", + "bundled": true, + "optional": true + }, + "is-fullwidth-code-point": { + "version": "1.0.0", + "bundled": true, + "requires": { + "number-is-nan": "1.0.1" + } + }, + "is-typedarray": { + "version": "1.0.0", + "bundled": true, + "optional": true + }, + "isarray": { + "version": "1.0.0", + "bundled": true + }, + "isstream": { + "version": "0.1.2", + "bundled": true, + "optional": true + }, + "jodid25519": { + "version": "1.0.2", + "bundled": true, + "optional": true, + "requires": { + "jsbn": "0.1.1" + } + }, + "jsbn": { + "version": "0.1.1", + "bundled": true, + "optional": true + }, + "json-schema": { + "version": "0.2.3", + "bundled": true, + "optional": true + }, + "json-stable-stringify": { + "version": "1.0.1", + "bundled": true, + "optional": true, + "requires": { + "jsonify": "0.0.0" + } + }, + "json-stringify-safe": { + "version": "5.0.1", + "bundled": true, + "optional": true + }, + "jsonify": { + "version": "0.0.0", + "bundled": true, + "optional": true + }, + "jsprim": { + "version": "1.4.0", + "bundled": true, + "optional": true, + "requires": { + "assert-plus": "1.0.0", + "extsprintf": "1.0.2", + "json-schema": "0.2.3", + "verror": "1.3.6" + }, + "dependencies": { + "assert-plus": { + "version": "1.0.0", + "bundled": true, + "optional": true + } + } + }, + "mime-db": { + "version": "1.27.0", + "bundled": true + }, + "mime-types": { + "version": "2.1.15", + "bundled": true, + "requires": { + "mime-db": "1.27.0" + } + }, + "minimatch": { + "version": "3.0.4", + "bundled": true, + "requires": { + "brace-expansion": "1.1.7" + } + }, + "minimist": { + "version": "0.0.8", + "bundled": true + }, + "mkdirp": { + "version": "0.5.1", + "bundled": true, + "requires": { + "minimist": "0.0.8" + } + }, + "ms": { + "version": "2.0.0", + "bundled": true, + "optional": true + }, + "node-pre-gyp": { + "version": "0.6.39", + "bundled": true, + "optional": true, + "requires": { + "detect-libc": "1.0.2", + "hawk": "3.1.3", + "mkdirp": "0.5.1", + "nopt": "4.0.1", + "npmlog": "4.1.0", + "rc": "1.2.1", + "request": "2.81.0", + "rimraf": "2.6.1", + "semver": "5.3.0", + "tar": "2.2.1", + "tar-pack": "3.4.0" + } + }, + "nopt": { + "version": "4.0.1", + "bundled": true, + "optional": true, + "requires": { + "abbrev": "1.1.0", + "osenv": "0.1.4" + } + }, + "npmlog": { + "version": "4.1.0", + "bundled": true, + "optional": true, + "requires": { + "are-we-there-yet": "1.1.4", + "console-control-strings": "1.1.0", + "gauge": "2.7.4", + "set-blocking": "2.0.0" + } + }, + "number-is-nan": { + "version": "1.0.1", + "bundled": true + }, + "oauth-sign": { + "version": "0.8.2", + "bundled": true, + "optional": true + }, + "object-assign": { + "version": "4.1.1", + "bundled": true, + "optional": true + }, + "once": { + "version": "1.4.0", + "bundled": true, + "requires": { + "wrappy": "1.0.2" + } + }, + "os-homedir": { + "version": "1.0.2", + "bundled": true, + "optional": true + }, + "os-tmpdir": { + "version": "1.0.2", + "bundled": true, + "optional": true + }, + "osenv": { + "version": "0.1.4", + "bundled": true, + "optional": true, + "requires": { + "os-homedir": "1.0.2", + "os-tmpdir": "1.0.2" + } + }, + "path-is-absolute": { + "version": "1.0.1", + "bundled": true + }, + "performance-now": { + "version": "0.2.0", + "bundled": true, + "optional": true + }, + "process-nextick-args": { + "version": "1.0.7", + "bundled": true + }, + "punycode": { + "version": "1.4.1", + "bundled": true, + "optional": true + }, + "qs": { + "version": "6.4.0", + "bundled": true, + "optional": true + }, + "rc": { + "version": "1.2.1", + "bundled": true, + "optional": true, + "requires": { + "deep-extend": "0.4.2", + "ini": "1.3.4", + "minimist": "1.2.0", + "strip-json-comments": "2.0.1" + }, + "dependencies": { + "minimist": { + "version": "1.2.0", + "bundled": true, + "optional": true + } + } + }, + "readable-stream": { + "version": "2.2.9", + "bundled": true, + "requires": { + "buffer-shims": "1.0.0", + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "1.0.0", + "process-nextick-args": "1.0.7", + "string_decoder": "1.0.1", + "util-deprecate": "1.0.2" + } + }, + "request": { + "version": "2.81.0", + "bundled": true, + "optional": true, + "requires": { + "aws-sign2": "0.6.0", + "aws4": "1.6.0", + "caseless": "0.12.0", + "combined-stream": "1.0.5", + "extend": "3.0.1", + "forever-agent": "0.6.1", + "form-data": "2.1.4", + "har-validator": "4.2.1", + "hawk": "3.1.3", + "http-signature": "1.1.1", + "is-typedarray": "1.0.0", + "isstream": "0.1.2", + "json-stringify-safe": "5.0.1", + "mime-types": "2.1.15", + "oauth-sign": "0.8.2", + "performance-now": "0.2.0", + "qs": "6.4.0", + "safe-buffer": "5.0.1", + "stringstream": "0.0.5", + "tough-cookie": "2.3.2", + "tunnel-agent": "0.6.0", + "uuid": "3.0.1" + } + }, + "rimraf": { + "version": "2.6.1", + "bundled": true, + "requires": { + "glob": "7.1.2" + } + }, + "safe-buffer": { + "version": "5.0.1", + "bundled": true + }, + "semver": { + "version": "5.3.0", + "bundled": true, + "optional": true + }, + "set-blocking": { + "version": "2.0.0", + "bundled": true, + "optional": true + }, + "signal-exit": { + "version": "3.0.2", + "bundled": true, + "optional": true + }, + "sntp": { + "version": "1.0.9", + "bundled": true, + "requires": { + "hoek": "2.16.3" + } + }, + "sshpk": { + "version": "1.13.0", + "bundled": true, + "optional": true, + "requires": { + "asn1": "0.2.3", + "assert-plus": "1.0.0", + "bcrypt-pbkdf": "1.0.1", + "dashdash": "1.14.1", + "ecc-jsbn": "0.1.1", + "getpass": "0.1.7", + "jodid25519": "1.0.2", + "jsbn": "0.1.1", + "tweetnacl": "0.14.5" + }, + "dependencies": { + "assert-plus": { + "version": "1.0.0", + "bundled": true, + "optional": true + } + } + }, + "string_decoder": { + "version": "1.0.1", + "bundled": true, + "requires": { + "safe-buffer": "5.0.1" + } + }, + "string-width": { + "version": "1.0.2", + "bundled": true, + "requires": { + "code-point-at": "1.1.0", + "is-fullwidth-code-point": "1.0.0", + "strip-ansi": "3.0.1" + } + }, + "stringstream": { + "version": "0.0.5", + "bundled": true, + "optional": true + }, + "strip-ansi": { + "version": "3.0.1", + "bundled": true, + "requires": { + "ansi-regex": "2.1.1" + } + }, + "strip-json-comments": { + "version": "2.0.1", + "bundled": true, + "optional": true + }, + "tar": { + "version": "2.2.1", + "bundled": true, + "requires": { + "block-stream": "0.0.9", + "fstream": "1.0.11", + "inherits": "2.0.3" + } + }, + "tar-pack": { + "version": "3.4.0", + "bundled": true, + "optional": true, + "requires": { + "debug": "2.6.8", + "fstream": "1.0.11", + "fstream-ignore": "1.0.5", + "once": "1.4.0", + "readable-stream": "2.2.9", + "rimraf": "2.6.1", + "tar": "2.2.1", + "uid-number": "0.0.6" + } + }, + "tough-cookie": { + "version": "2.3.2", + "bundled": true, + "optional": true, + "requires": { + "punycode": "1.4.1" + } + }, + "tunnel-agent": { + "version": "0.6.0", + "bundled": true, + "optional": true, + "requires": { + "safe-buffer": "5.0.1" + } + }, + "tweetnacl": { + "version": "0.14.5", + "bundled": true, + "optional": true + }, + "uid-number": { + "version": "0.0.6", + "bundled": true, + "optional": true + }, + "util-deprecate": { + "version": "1.0.2", + "bundled": true + }, + "uuid": { + "version": "3.0.1", + "bundled": true, + "optional": true + }, + "verror": { + "version": "1.3.6", + "bundled": true, + "optional": true, + "requires": { + "extsprintf": "1.0.2" + } + }, + "wide-align": { + "version": "1.1.2", + "bundled": true, + "optional": true, + "requires": { + "string-width": "1.0.2" + } + }, + "wrappy": { + "version": "1.0.2", + "bundled": true + } + } + }, "fstream": { "version": "0.1.31", "resolved": "https://registry.npmjs.org/fstream/-/fstream-0.1.31.tgz", @@ -2783,6 +3573,12 @@ "duplexer2": "0.0.2" } }, + "nan": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.10.0.tgz", + "integrity": "sha512-bAdJv7fBLhWC+/Bls0Oza+mvTaNQtP+1RyhhhvD95pgUJz6XM5IzgmxOkItJ9tkoCiplvAnXI1tNmmUD/eScyA==", + "optional": true + }, "natives": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/natives/-/natives-1.1.0.tgz", diff --git a/src/.idea/.idea.NzbDrone/.idea/contentModel.xml b/src/.idea/.idea.NzbDrone/.idea/contentModel.xml index cbbd0c4e4..50671a366 100644 --- a/src/.idea/.idea.NzbDrone/.idea/contentModel.xml +++ b/src/.idea/.idea.NzbDrone/.idea/contentModel.xml @@ -1,24 +1,21 @@ - + - - - + + + - - - - - - - + + + + - + @@ -70,10 +67,10 @@ - - - - + + + + @@ -1332,6 +1329,7 @@ + @@ -3282,7 +3280,7 @@ - + @@ -3294,7 +3292,7 @@ - + diff --git a/src/.idea/.idea.NzbDrone/riderModule.iml b/src/.idea/.idea.NzbDrone/riderModule.iml index c84a9c732..c8b2ee068 100644 --- a/src/.idea/.idea.NzbDrone/riderModule.iml +++ b/src/.idea/.idea.NzbDrone/riderModule.iml @@ -1,103 +1,9 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + \ No newline at end of file diff --git a/src/NzbDrone.Common.Test/Http/HttpClientFixture.cs b/src/NzbDrone.Common.Test/Http/HttpClientFixture.cs index d9bde2f84..07b45c222 100644 --- a/src/NzbDrone.Common.Test/Http/HttpClientFixture.cs +++ b/src/NzbDrone.Common.Test/Http/HttpClientFixture.cs @@ -273,19 +273,74 @@ public void should_not_send_cookie_to_other_host() response.Resource.Headers.Should().NotContainKey("Cookie"); } + [Test] + public void should_not_store_request_cookie() + { + var requestGet = new HttpRequest($"http://{_httpBinHost}/get"); + requestGet.Cookies.Add("my", "cookie"); + requestGet.AllowAutoRedirect = false; + requestGet.StoreRequestCookie = false; + requestGet.StoreResponseCookie = false; + var responseGet = Subject.Get(requestGet); + + var requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies"); + requestCookies.AllowAutoRedirect = false; + var responseCookies = Subject.Get(requestCookies); + + responseCookies.Resource.Cookies.Should().BeEmpty(); + + ExceptionVerification.IgnoreErrors(); + } + + [Test] + public void should_store_request_cookie() + { + var requestGet = new HttpRequest($"http://{_httpBinHost}/get"); + requestGet.Cookies.Add("my", "cookie"); + requestGet.AllowAutoRedirect = false; + requestGet.StoreRequestCookie.Should().BeTrue(); + requestGet.StoreResponseCookie = false; + var responseGet = Subject.Get(requestGet); + + var requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies"); + requestCookies.AllowAutoRedirect = false; + var responseCookies = Subject.Get(requestCookies); + + responseCookies.Resource.Cookies.Should().HaveCount(1).And.Contain("my", "cookie"); + + ExceptionVerification.IgnoreErrors(); + } + + [Test] + public void should_delete_request_cookie() + { + var requestDelete = new HttpRequest($"http://{_httpBinHost}/cookies/delete?my"); + requestDelete.Cookies.Add("my", "cookie"); + requestDelete.AllowAutoRedirect = true; + requestDelete.StoreRequestCookie = false; + requestDelete.StoreResponseCookie = false; + + // Delete and redirect since that's the only way to check the internal temporary cookie container + var responseCookies = Subject.Get(requestDelete); + + responseCookies.Resource.Cookies.Should().BeEmpty(); + } + [Test] public void should_not_store_response_cookie() { var requestSet = new HttpRequest(string.Format("http://{0}/cookies/set?my=cookie", _httpBinHost)); requestSet.AllowAutoRedirect = false; + requestSet.StoreRequestCookie = false; + requestSet.StoreResponseCookie.Should().BeFalse(); var responseSet = Subject.Get(requestSet); - var request = new HttpRequest(string.Format("http://{0}/get", _httpBinHost)); + var requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies"); - var response = Subject.Get(request); + var responseCookies = Subject.Get(requestCookies); - response.Resource.Headers.Should().NotContainKey("Cookie"); + responseCookies.Resource.Cookies.Should().BeEmpty(); ExceptionVerification.IgnoreErrors(); } @@ -295,19 +350,31 @@ public void should_store_response_cookie() { var requestSet = new HttpRequest(string.Format("http://{0}/cookies/set?my=cookie", _httpBinHost)); requestSet.AllowAutoRedirect = false; + requestSet.StoreRequestCookie = false; requestSet.StoreResponseCookie = true; var responseSet = Subject.Get(requestSet); - var request = new HttpRequest(string.Format("http://{0}/get", _httpBinHost)); + var requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies"); - var response = Subject.Get(request); + var responseCookies = Subject.Get(requestCookies); - response.Resource.Headers.Should().ContainKey("Cookie"); + responseCookies.Resource.Cookies.Should().HaveCount(1).And.Contain("my", "cookie"); - var cookie = response.Resource.Headers["Cookie"].ToString(); + ExceptionVerification.IgnoreErrors(); + } - cookie.Should().Contain("my=cookie"); + [Test] + public void should_temp_store_response_cookie() + { + var requestSet = new HttpRequest($"http://{_httpBinHost}/cookies/set?my=cookie"); + requestSet.AllowAutoRedirect = true; + requestSet.StoreRequestCookie = false; + requestSet.StoreResponseCookie.Should().BeFalse(); + var responseSet = Subject.Get(requestSet); + + // Set and redirect since that's the only way to check the internal temporary cookie container + responseSet.Resource.Cookies.Should().HaveCount(1).And.Contain("my", "cookie"); ExceptionVerification.IgnoreErrors(); } @@ -315,26 +382,148 @@ public void should_store_response_cookie() [Test] public void should_overwrite_response_cookie() { - var requestSet = new HttpRequest(string.Format("http://{0}/cookies/set?my=cookie", _httpBinHost)); + var requestSet = new HttpRequest($"http://{_httpBinHost}/cookies/set?my=cookie"); + requestSet.Cookies.Add("my", "oldcookie"); requestSet.AllowAutoRedirect = false; + requestSet.StoreRequestCookie = false; requestSet.StoreResponseCookie = true; - requestSet.Cookies["my"] = "oldcookie"; var responseSet = Subject.Get(requestSet); - var request = new HttpRequest(string.Format("http://{0}/get", _httpBinHost)); + var requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies"); - var response = Subject.Get(request); + var responseCookies = Subject.Get(requestCookies); - response.Resource.Headers.Should().ContainKey("Cookie"); - - var cookie = response.Resource.Headers["Cookie"].ToString(); - - cookie.Should().Contain("my=cookie"); + responseCookies.Resource.Cookies.Should().HaveCount(1).And.Contain("my", "cookie"); ExceptionVerification.IgnoreErrors(); } + [Test] + public void should_overwrite_temp_response_cookie() + { + var requestSet = new HttpRequest($"http://{_httpBinHost}/cookies/set?my=cookie"); + requestSet.Cookies.Add("my", "oldcookie"); + requestSet.AllowAutoRedirect = true; + requestSet.StoreRequestCookie = true; + requestSet.StoreResponseCookie = false; + + var responseSet = Subject.Get(requestSet); + + responseSet.Resource.Cookies.Should().HaveCount(1).And.Contain("my", "cookie"); + + var requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies"); + + var responseCookies = Subject.Get(requestCookies); + + responseCookies.Resource.Cookies.Should().HaveCount(1).And.Contain("my", "oldcookie"); + + ExceptionVerification.IgnoreErrors(); + } + + [Test] + public void should_not_delete_response_cookie() + { + var requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies"); + requestCookies.Cookies.Add("my", "cookie"); + requestCookies.AllowAutoRedirect = false; + requestCookies.StoreRequestCookie = true; + requestCookies.StoreResponseCookie = false; + var responseCookies = Subject.Get(requestCookies); + + responseCookies.Resource.Cookies.Should().HaveCount(1).And.Contain("my", "cookie"); + + var requestDelete = new HttpRequest($"http://{_httpBinHost}/cookies/delete?my"); + requestDelete.AllowAutoRedirect = false; + requestDelete.StoreRequestCookie = false; + requestDelete.StoreResponseCookie = false; + + var responseDelete = Subject.Get(requestDelete); + + requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies"); + requestCookies.StoreRequestCookie = false; + requestCookies.StoreResponseCookie = false; + + responseCookies = Subject.Get(requestCookies); + + responseCookies.Resource.Cookies.Should().HaveCount(1).And.Contain("my", "cookie"); + + ExceptionVerification.IgnoreErrors(); + } + + [Test] + public void should_delete_response_cookie() + { + var requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies"); + requestCookies.Cookies.Add("my", "cookie"); + requestCookies.AllowAutoRedirect = false; + requestCookies.StoreRequestCookie = true; + requestCookies.StoreResponseCookie = false; + var responseCookies = Subject.Get(requestCookies); + + responseCookies.Resource.Cookies.Should().HaveCount(1).And.Contain("my", "cookie"); + + var requestDelete = new HttpRequest($"http://{_httpBinHost}/cookies/delete?my"); + requestDelete.AllowAutoRedirect = false; + requestDelete.StoreRequestCookie = false; + requestDelete.StoreResponseCookie = true; + + var responseDelete = Subject.Get(requestDelete); + + requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies"); + requestCookies.StoreRequestCookie = false; + requestCookies.StoreResponseCookie = false; + + responseCookies = Subject.Get(requestCookies); + + responseCookies.Resource.Cookies.Should().BeEmpty(); + + ExceptionVerification.IgnoreErrors(); + } + + [Test] + public void should_delete_temp_response_cookie() + { + var requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies"); + requestCookies.Cookies.Add("my", "cookie"); + requestCookies.AllowAutoRedirect = false; + requestCookies.StoreRequestCookie = true; + requestCookies.StoreResponseCookie = false; + var responseCookies = Subject.Get(requestCookies); + + responseCookies.Resource.Cookies.Should().HaveCount(1).And.Contain("my", "cookie"); + + var requestDelete = new HttpRequest($"http://{_httpBinHost}/cookies/delete?my"); + requestDelete.AllowAutoRedirect = true; + requestDelete.StoreRequestCookie = false; + requestDelete.StoreResponseCookie = false; + var responseDelete = Subject.Get(requestDelete); + + responseDelete.Resource.Cookies.Should().BeEmpty(); + + requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies"); + requestCookies.StoreRequestCookie = false; + requestCookies.StoreResponseCookie = false; + + responseCookies.Resource.Cookies.Should().HaveCount(1).And.Contain("my", "cookie"); + + ExceptionVerification.IgnoreErrors(); + } + + [Test] + public void should_not_send_old_cookie() + { + GivenOldCookie(); + + var requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies"); + requestCookies.IgnorePersistentCookies = true; + requestCookies.StoreRequestCookie = false; + requestCookies.StoreResponseCookie = false; + var responseCookies = Subject.Get(requestCookies); + + responseCookies.Resource.Cookies.Should().BeEmpty(); + } + [Test] public void should_throw_on_http429_too_many_requests() { @@ -442,4 +631,9 @@ public class HttpBinResource public string Url { get; set; } public string Data { get; set; } } + + public class HttpCookieResource + { + public Dictionary Cookies { get; set; } + } } diff --git a/src/NzbDrone.Common/Http/HttpClient.cs b/src/NzbDrone.Common/Http/HttpClient.cs index e567d5ed0..2f985d552 100644 --- a/src/NzbDrone.Common/Http/HttpClient.cs +++ b/src/NzbDrone.Common/Http/HttpClient.cs @@ -45,7 +45,9 @@ public HttpClient(IEnumerable requestInterceptors, ICac public HttpResponse Execute(HttpRequest request) { - var response = ExecuteRequest(request); + var cookieContainer = InitializeRequestCookies(request); + + var response = ExecuteRequest(request, cookieContainer); if (request.AllowAutoRedirect && response.HasHttpRedirect) { @@ -64,7 +66,7 @@ public HttpResponse Execute(HttpRequest request) throw new WebException($"Too many automatic redirections were attempted for {string.Join(" -> ", autoRedirectChain)}", WebExceptionStatus.ProtocolError); } - response = ExecuteRequest(request); + response = ExecuteRequest(request, cookieContainer); } while (response.HasHttpRedirect); } @@ -91,7 +93,7 @@ public HttpResponse Execute(HttpRequest request) return response; } - private HttpResponse ExecuteRequest(HttpRequest request) + private HttpResponse ExecuteRequest(HttpRequest request, CookieContainer cookieContainer) { foreach (var interceptor in _requestInterceptors) { @@ -107,11 +109,11 @@ private HttpResponse ExecuteRequest(HttpRequest request) var stopWatch = Stopwatch.StartNew(); - var cookies = PrepareRequestCookies(request); + PrepareRequestCookies(request, cookieContainer); - var response = _httpDispatcher.GetResponse(request, cookies); + var response = _httpDispatcher.GetResponse(request, cookieContainer); - HandleResponseCookies(request, cookies); + HandleResponseCookies(response, cookieContainer); stopWatch.Stop(); @@ -130,49 +132,84 @@ private HttpResponse ExecuteRequest(HttpRequest request) return response; } - private CookieContainer PrepareRequestCookies(HttpRequest request) + private CookieContainer InitializeRequestCookies(HttpRequest request) { lock (_cookieContainerCache) { - var persistentCookieContainer = _cookieContainerCache.Get("container", () => new CookieContainer()); + var sourceContainer = new CookieContainer(); + + var presistentContainer = _cookieContainerCache.Get("container", () => new CookieContainer()); + + if (!request.IgnorePersistentCookies) + { + var persistentCookies = presistentContainer.GetCookies((Uri)request.Url); + sourceContainer.Add(persistentCookies); + } if (request.Cookies.Count != 0) { foreach (var pair in request.Cookies) { - persistentCookieContainer.Add(new Cookie(pair.Key, pair.Value, "/", request.Url.Host) + var cookie = new Cookie(pair.Key, pair.Value, "/") { // Use Now rather than UtcNow to work around Mono cookie expiry bug. // See https://gist.github.com/ta264/7822b1424f72e5b4c961 Expires = DateTime.Now.AddHours(1) - }); + }; + + sourceContainer.Add((Uri)request.Url, cookie); + + if (request.StoreRequestCookie) + { + presistentContainer.Add((Uri)request.Url, cookie); + } } } - var requestCookies = persistentCookieContainer.GetCookies((Uri)request.Url); - - var cookieContainer = new CookieContainer(); - - cookieContainer.Add(requestCookies); - - return cookieContainer; + return sourceContainer; } } - private void HandleResponseCookies(HttpRequest request, CookieContainer cookieContainer) + private void PrepareRequestCookies(HttpRequest request, CookieContainer cookieContainer) { - if (!request.StoreResponseCookie) + // Don't collect persistnet cookies for intermediate/redirected urls. + /*lock (_cookieContainerCache) + { + var presistentContainer = _cookieContainerCache.Get("container", () => new CookieContainer()); + var persistentCookies = presistentContainer.GetCookies((Uri)request.Url); + var existingCookies = cookieContainer.GetCookies((Uri)request.Url); + + cookieContainer.Add(persistentCookies); + cookieContainer.Add(existingCookies); + }*/ + } + + private void HandleResponseCookies(HttpResponse response, CookieContainer cookieContainer) + { + var cookieHeaders = response.GetCookieHeaders(); + if (cookieHeaders.Empty()) { return; } - lock (_cookieContainerCache) + if (response.Request.StoreResponseCookie) { - var persistentCookieContainer = _cookieContainerCache.Get("container", () => new CookieContainer()); + lock (_cookieContainerCache) + { + var persistentCookieContainer = _cookieContainerCache.Get("container", () => new CookieContainer()); - var cookies = cookieContainer.GetCookies((Uri)request.Url); - - persistentCookieContainer.Add(cookies); + foreach (var cookieHeader in cookieHeaders) + { + try + { + persistentCookieContainer.SetCookies((Uri)response.Request.Url, cookieHeader); + } + catch (Exception ex) + { + _logger.Debug(ex, "Invalid cookie in {0}", response.Request.Url); + } + } + } } } diff --git a/src/NzbDrone.Common/Http/HttpRequest.cs b/src/NzbDrone.Common/Http/HttpRequest.cs index 3de009d0c..f0f2d053c 100644 --- a/src/NzbDrone.Common/Http/HttpRequest.cs +++ b/src/NzbDrone.Common/Http/HttpRequest.cs @@ -13,6 +13,8 @@ public HttpRequest(string url, HttpAccept httpAccept = null) Url = new HttpUri(url); Headers = new HttpHeader(); AllowAutoRedirect = true; + StoreRequestCookie = true; + IgnorePersistentCookies = false; Cookies = new Dictionary(); if (!RuntimeInfoBase.IsProduction) @@ -37,6 +39,8 @@ public HttpRequest(string url, HttpAccept httpAccept = null) public bool ConnectionKeepAlive { get; set; } public bool LogResponseContent { get; set; } public Dictionary Cookies { get; private set; } + public bool IgnorePersistentCookies { get; set; } + public bool StoreRequestCookie { get; set; } public bool StoreResponseCookie { get; set; } public TimeSpan RequestTimeout { get; set; } public TimeSpan RateLimit { get; set; } diff --git a/src/NzbDrone.Common/Http/HttpResponse.cs b/src/NzbDrone.Common/Http/HttpResponse.cs index 734238cfc..e0b11b51c 100644 --- a/src/NzbDrone.Common/Http/HttpResponse.cs +++ b/src/NzbDrone.Common/Http/HttpResponse.cs @@ -55,20 +55,22 @@ public string Content StatusCode == HttpStatusCode.MovedPermanently || StatusCode == HttpStatusCode.Found; + public string[] GetCookieHeaders() + { + return Headers.GetValues("Set-Cookie") ?? new string[0]; + } + public Dictionary GetCookies() { var result = new Dictionary(); - var setCookieHeaders = Headers.GetValues("Set-Cookie"); - if (setCookieHeaders != null) + var setCookieHeaders = GetCookieHeaders(); + foreach (var cookie in setCookieHeaders) { - foreach (var cookie in setCookieHeaders) + var match = RegexSetCookie.Match(cookie); + if (match.Success) { - var match = RegexSetCookie.Match(cookie); - if (match.Success) - { - result[match.Groups[1].Value] = match.Groups[2].Value; - } + result[match.Groups[1].Value] = match.Groups[2].Value; } } diff --git a/src/NzbDrone.Core/Datastore/Migration/144_add_cookies_to_indexer_status.cs b/src/NzbDrone.Core/Datastore/Migration/144_add_cookies_to_indexer_status.cs new file mode 100644 index 000000000..f47f0a2b2 --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/144_add_cookies_to_indexer_status.cs @@ -0,0 +1,16 @@ +using FluentMigrator; +using NzbDrone.Core.Datastore.Migration.Framework; + +namespace NzbDrone.Core.Datastore.Migration +{ + [Migration(144)] + public class add_cookies_to_indexer_status : NzbDroneMigrationBase + { + protected override void MainDbUpgrade() + { + Alter.Table("IndexerStatus").AddColumn("Cookies").AsString().Nullable() + .AddColumn("CookiesExpirationDate").AsDateTime().Nullable(); + + } + } +} diff --git a/src/NzbDrone.Core/Datastore/TableMapping.cs b/src/NzbDrone.Core/Datastore/TableMapping.cs index 05bda27ac..61b2eb760 100644 --- a/src/NzbDrone.Core/Datastore/TableMapping.cs +++ b/src/NzbDrone.Core/Datastore/TableMapping.cs @@ -146,6 +146,7 @@ private static void RegisterMappers() MapRepository.Instance.RegisterTypeConverter(typeof(List), new EmbeddedDocumentConverter(new QualityIntConverter())); MapRepository.Instance.RegisterTypeConverter(typeof(QualityModel), new EmbeddedDocumentConverter(new QualityIntConverter())); MapRepository.Instance.RegisterTypeConverter(typeof(Dictionary), new EmbeddedDocumentConverter()); + MapRepository.Instance.RegisterTypeConverter(typeof(IDictionary), new EmbeddedDocumentConverter()); MapRepository.Instance.RegisterTypeConverter(typeof(List), new EmbeddedDocumentConverter()); MapRepository.Instance.RegisterTypeConverter(typeof(List), new EmbeddedDocumentConverter()); MapRepository.Instance.RegisterTypeConverter(typeof(ParsedMovieInfo), new EmbeddedDocumentConverter()); diff --git a/src/NzbDrone.Core/Indexers/AwesomeHD/AwesomeHDRequestGenerator.cs b/src/NzbDrone.Core/Indexers/AwesomeHD/AwesomeHDRequestGenerator.cs index 4b60f4875..39177c98b 100644 --- a/src/NzbDrone.Core/Indexers/AwesomeHD/AwesomeHDRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/AwesomeHD/AwesomeHDRequestGenerator.cs @@ -25,6 +25,9 @@ public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchC return pageableRequests; } + public Func> GetCookies { get; set; } + public Action, DateTime?> CookiesUpdater { get; set; } + private IEnumerable GetRequest(string searchParameters) { if (searchParameters != null) diff --git a/src/NzbDrone.Core/Indexers/AwesomeHD/AwesomeHDRssParser.cs b/src/NzbDrone.Core/Indexers/AwesomeHD/AwesomeHDRssParser.cs index 19aa5e442..adb94af27 100644 --- a/src/NzbDrone.Core/Indexers/AwesomeHD/AwesomeHDRssParser.cs +++ b/src/NzbDrone.Core/Indexers/AwesomeHD/AwesomeHDRssParser.cs @@ -106,6 +106,8 @@ public IList ParseResponse(IndexerResponse indexerResponse) return torrentInfos.OrderByDescending(o => ((dynamic)o).Seeders).ToArray(); } + public Action, DateTime?> CookiesUpdater { get; set; } + private string GetDownloadUrl(string torrentId, string authKey, string passKey) { var url = new HttpUri(_settings.BaseUrl) diff --git a/src/NzbDrone.Core/Indexers/HDBits/HDBitsParser.cs b/src/NzbDrone.Core/Indexers/HDBits/HDBitsParser.cs index b2cae9a61..d4994b35a 100644 --- a/src/NzbDrone.Core/Indexers/HDBits/HDBitsParser.cs +++ b/src/NzbDrone.Core/Indexers/HDBits/HDBitsParser.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Net; using Newtonsoft.Json; using Newtonsoft.Json.Linq; @@ -85,6 +86,8 @@ public IList ParseResponse(IndexerResponse indexerResponse) return torrentInfos.ToArray(); } + public Action, DateTime?> CookiesUpdater { get; set; } + private string GetDownloadUrl(string torrentId) { var url = new HttpUri(_settings.BaseUrl) diff --git a/src/NzbDrone.Core/Indexers/HDBits/HDBitsRequestGenerator.cs b/src/NzbDrone.Core/Indexers/HDBits/HDBitsRequestGenerator.cs index 789cf0285..6d100bfa0 100644 --- a/src/NzbDrone.Core/Indexers/HDBits/HDBitsRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/HDBits/HDBitsRequestGenerator.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.Linq; using NzbDrone.Common.Http; @@ -44,6 +45,9 @@ private bool TryAddSearchParameters(TorrentQuery query, SearchCriteriaBase searc } return false; } + + public Func> GetCookies { get; set; } + public Action, DateTime?> CookiesUpdater { get; set; } private IEnumerable GetRequest(TorrentQuery query) { diff --git a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs index aab7af4cf..d193dbea9 100644 --- a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs @@ -45,10 +45,8 @@ public override IList FetchRecent() { return new List(); } - - var generator = GetRequestGenerator(); - - return FetchReleases(generator.GetRecentRequests(), true); + + return FetchReleases(GetRequestChain(), true); } public override IList Fetch(MovieSearchCriteria searchCriteria) @@ -58,17 +56,48 @@ public override IList Fetch(MovieSearchCriteria searchCriteria) return new List(); } - var generator = GetRequestGenerator(); - - return FetchReleases(generator.GetSearchRequests(searchCriteria)); + return FetchReleases(GetRequestChain(searchCriteria)); } + protected IndexerPageableRequestChain GetRequestChain(SearchCriteriaBase searchCriteria = null) + { + var generator = GetRequestGenerator(); + + //A func ensures cookies are always updated to the latest. This way, the first page could update the cookies and then can be reused by the second page. + + generator.GetCookies = () => + { + var cookies = _indexerStatusService.GetIndexerCookies(Definition.Id); + var expiration = _indexerStatusService.GetIndexerCookiesExpirationDate(Definition.Id); + if (expiration < DateTime.Now) + { + cookies = null; + } + + return cookies; + }; + + var requests = searchCriteria == null ? generator.GetRecentRequests() : generator.GetSearchRequests(searchCriteria as MovieSearchCriteria); + + generator.CookiesUpdater = (cookies, expiration) => + { + _indexerStatusService.UpdateCookies(Definition.Id, cookies, expiration); + }; + + return requests; + } + + protected virtual IList FetchReleases(IndexerPageableRequestChain pageableRequestChain, bool isRecent = false) { var releases = new List(); var url = string.Empty; var parser = GetParser(); + parser.CookiesUpdater = (cookies, expiration) => + { + _indexerStatusService.UpdateCookies(Definition.Id, cookies, expiration); + }; try { @@ -261,8 +290,11 @@ protected virtual ValidationFailure TestConnection() try { var parser = GetParser(); - var generator = GetRequestGenerator(); - var releases = FetchPage(generator.GetRecentRequests().GetAllTiers().First().First(), parser); + parser.CookiesUpdater = (cookies, expiration) => + { + _indexerStatusService.UpdateCookies(Definition.Id, cookies, expiration); + }; + var releases = FetchPage(GetRequestChain().GetAllTiers().First().First(), parser); if (releases.Empty()) { diff --git a/src/NzbDrone.Core/Indexers/IIndexerRequestGenerator.cs b/src/NzbDrone.Core/Indexers/IIndexerRequestGenerator.cs index 372125582..ef32b2ab0 100644 --- a/src/NzbDrone.Core/Indexers/IIndexerRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/IIndexerRequestGenerator.cs @@ -1,3 +1,5 @@ +using System; +using System.Collections.Generic; using NzbDrone.Core.IndexerSearch.Definitions; namespace NzbDrone.Core.Indexers @@ -6,5 +8,7 @@ public interface IIndexerRequestGenerator { IndexerPageableRequestChain GetRecentRequests(); IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria); + Func> GetCookies { get; set; } + Action, DateTime?> CookiesUpdater { get; set; } } } diff --git a/src/NzbDrone.Core/Indexers/IPTorrents/IPTorrentsRequestGenerator.cs b/src/NzbDrone.Core/Indexers/IPTorrents/IPTorrentsRequestGenerator.cs index 339f9471d..264ea5ab5 100644 --- a/src/NzbDrone.Core/Indexers/IPTorrents/IPTorrentsRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/IPTorrents/IPTorrentsRequestGenerator.cs @@ -27,5 +27,8 @@ private IEnumerable GetRssRequests() { yield return new IndexerRequest(Settings.BaseUrl, HttpAccept.Rss); } + + public Func> GetCookies { get; set; } + public Action, DateTime?> CookiesUpdater { get; set; } } } diff --git a/src/NzbDrone.Core/Indexers/IProcessIndexerResponse.cs b/src/NzbDrone.Core/Indexers/IProcessIndexerResponse.cs index 56a8bed4e..1dd5f02ca 100644 --- a/src/NzbDrone.Core/Indexers/IProcessIndexerResponse.cs +++ b/src/NzbDrone.Core/Indexers/IProcessIndexerResponse.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using NzbDrone.Core.Parser.Model; namespace NzbDrone.Core.Indexers @@ -6,5 +7,6 @@ namespace NzbDrone.Core.Indexers public interface IParseIndexerResponse { IList ParseResponse(IndexerResponse indexerResponse); + Action, DateTime?> CookiesUpdater { get; set; } } } diff --git a/src/NzbDrone.Core/Indexers/IndexerStatus.cs b/src/NzbDrone.Core/Indexers/IndexerStatus.cs index 662c9de64..aa137315b 100644 --- a/src/NzbDrone.Core/Indexers/IndexerStatus.cs +++ b/src/NzbDrone.Core/Indexers/IndexerStatus.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using NzbDrone.Core.Datastore; using NzbDrone.Core.Parser.Model; @@ -14,6 +15,9 @@ public class IndexerStatus : ModelBase public DateTime? DisabledTill { get; set; } public ReleaseInfo LastRssSyncReleaseInfo { get; set; } + + public IDictionary Cookies { get; set; } + public DateTime? CookiesExpirationDate { get; set; } public bool IsDisabled() { diff --git a/src/NzbDrone.Core/Indexers/IndexerStatusService.cs b/src/NzbDrone.Core/Indexers/IndexerStatusService.cs index 8e1bd1fe5..1d177fc87 100644 --- a/src/NzbDrone.Core/Indexers/IndexerStatusService.cs +++ b/src/NzbDrone.Core/Indexers/IndexerStatusService.cs @@ -12,11 +12,14 @@ public interface IIndexerStatusService { List GetBlockedIndexers(); ReleaseInfo GetLastRssSyncReleaseInfo(int indexerId); + IDictionary GetIndexerCookies(int indexerId); + DateTime GetIndexerCookiesExpirationDate(int indexerId); void RecordSuccess(int indexerId); void RecordFailure(int indexerId, TimeSpan minimumBackOff = default(TimeSpan)); void RecordConnectionFailure(int indexerId); void UpdateRssSyncStatus(int indexerId, ReleaseInfo releaseInfo); + void UpdateCookies(int indexerId, IDictionary cookies, DateTime? expiration); } public class IndexerStatusService : IIndexerStatusService, IHandleAsync> @@ -55,6 +58,16 @@ public ReleaseInfo GetLastRssSyncReleaseInfo(int indexerId) return GetIndexerStatus(indexerId).LastRssSyncReleaseInfo; } + public IDictionary GetIndexerCookies(int indexerId) + { + return GetIndexerStatus(indexerId).Cookies; + } + + public DateTime GetIndexerCookiesExpirationDate(int indexerId) + { + return GetIndexerStatus(indexerId).CookiesExpirationDate ?? DateTime.Now + TimeSpan.FromDays(12); + } + private IndexerStatus GetIndexerStatus(int indexerId) { return _indexerStatusRepository.FindByIndexerId(indexerId) ?? new IndexerStatus { IndexerId = indexerId }; @@ -141,6 +154,17 @@ public void UpdateRssSyncStatus(int indexerId, ReleaseInfo releaseInfo) } } + public void UpdateCookies(int indexerId, IDictionary cookies, DateTime? expiration) + { + lock (_syncRoot) + { + var status = GetIndexerStatus(indexerId); + status.Cookies = cookies; + status.CookiesExpirationDate = expiration; + _indexerStatusRepository.Upsert(status); + } + } + public void HandleAsync(ProviderDeletedEvent message) { var indexerStatus = _indexerStatusRepository.FindByIndexerId(message.ProviderId); diff --git a/src/NzbDrone.Core/Indexers/Newznab/NewznabRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Newznab/NewznabRequestGenerator.cs index 06a681ac5..2adbde4d7 100644 --- a/src/NzbDrone.Core/Indexers/Newznab/NewznabRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/Newznab/NewznabRequestGenerator.cs @@ -109,5 +109,8 @@ private IEnumerable GetPagedRequests(int maxPages, IEnumerable> GetCookies { get; set; } + public Action, DateTime?> CookiesUpdater { get; set; } } } diff --git a/src/NzbDrone.Core/Indexers/Nyaa/NyaaRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Nyaa/NyaaRequestGenerator.cs index 0a4553f3a..33bd7ba46 100644 --- a/src/NzbDrone.Core/Indexers/Nyaa/NyaaRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/Nyaa/NyaaRequestGenerator.cs @@ -60,5 +60,8 @@ public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchC { return new IndexerPageableRequestChain(); } + + public Func> GetCookies { get; set; } + public Action, DateTime?> CookiesUpdater { get; set; } } } diff --git a/src/NzbDrone.Core/Indexers/Omgwtfnzbs/OmgwtfnzbsRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Omgwtfnzbs/OmgwtfnzbsRequestGenerator.cs index 16e310190..314014761 100644 --- a/src/NzbDrone.Core/Indexers/Omgwtfnzbs/OmgwtfnzbsRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/Omgwtfnzbs/OmgwtfnzbsRequestGenerator.cs @@ -49,5 +49,8 @@ public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchC return pageableRequests; } + + public Func> GetCookies { get; set; } + public Action, DateTime?> CookiesUpdater { get; set; } } } diff --git a/src/NzbDrone.Core/Indexers/PassThePopcorn/PassThePopcorn.cs b/src/NzbDrone.Core/Indexers/PassThePopcorn/PassThePopcorn.cs index e9b9892a8..2b9158678 100644 --- a/src/NzbDrone.Core/Indexers/PassThePopcorn/PassThePopcorn.cs +++ b/src/NzbDrone.Core/Indexers/PassThePopcorn/PassThePopcorn.cs @@ -15,8 +15,8 @@ public class PassThePopcorn : HttpIndexerBase public override bool SupportsSearch => true; public override int PageSize => 50; - private readonly ICached> _authCookieCache; private readonly IHttpClient _httpClient; + private readonly IIndexerStatusService _indexerStatusService; private readonly Logger _logger; public PassThePopcorn(IHttpClient httpClient, ICacheManager cacheManager, IIndexerStatusService indexerStatusService, @@ -25,7 +25,7 @@ public PassThePopcorn(IHttpClient httpClient, ICacheManager cacheManager, IIndex { _httpClient = httpClient; _logger = logger; - _authCookieCache = cacheManager.GetCache>(GetType(), "authCookies"); + _indexerStatusService = indexerStatusService; } public override IIndexerRequestGenerator GetRequestGenerator() @@ -35,7 +35,6 @@ public override IIndexerRequestGenerator GetRequestGenerator() Settings = Settings, HttpClient = _httpClient, Logger = _logger, - AuthCookieCache = _authCookieCache }; } @@ -43,5 +42,20 @@ public override IParseIndexerResponse GetParser() { return new PassThePopcornParser(Settings); } + + /*protected override IndexerResponse FetchIndexerResponse(IndexerRequest request) + { + _logger.Debug("Downloading Feed " + request.HttpRequest.ToString(false)); + + if (request.HttpRequest.RateLimit < RateLimit) + { + request.HttpRequest.RateLimit = RateLimit; + } + + //Potentially dangerous though if ptp moves domains! + request.HttpRequest.AllowAutoRedirect = false; + + return new IndexerResponse(request, _httpClient.Execute(request.HttpRequest)); + }*/ } } diff --git a/src/NzbDrone.Core/Indexers/PassThePopcorn/PassThePopcornParser.cs b/src/NzbDrone.Core/Indexers/PassThePopcorn/PassThePopcornParser.cs index 50fc94414..5eada1d7b 100644 --- a/src/NzbDrone.Core/Indexers/PassThePopcorn/PassThePopcornParser.cs +++ b/src/NzbDrone.Core/Indexers/PassThePopcorn/PassThePopcornParser.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.Net; using Newtonsoft.Json; @@ -13,8 +14,6 @@ namespace NzbDrone.Core.Indexers.PassThePopcorn public class PassThePopcornParser : IParseIndexerResponse { private readonly PassThePopcornSettings _settings; - public ICached> AuthCookieCache { get; set; } - public PassThePopcornParser(PassThePopcornSettings settings) { _settings = settings; @@ -27,16 +26,23 @@ public IList ParseResponse(IndexerResponse indexerResponse) if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK) { // Remove cookie cache - AuthCookieCache.Remove(_settings.BaseUrl.Trim().TrimEnd('/')); - + if (indexerResponse.HttpResponse.HasHttpRedirect && indexerResponse.HttpResponse.Headers["Location"] + .ContainsIgnoreCase("login.php")) + { + CookiesUpdater(null, null); + throw new IndexerException(indexerResponse, "We are being redirected to the PTP login page. Most likely your session expired or was killed. Try testing the indexer in the settings."); + } throw new IndexerException(indexerResponse, $"Unexpected response status {indexerResponse.HttpResponse.StatusCode} code from API request"); } if (indexerResponse.HttpResponse.Headers.ContentType != HttpAccept.Json.Value) { + if (indexerResponse.HttpResponse.Request.Url.Path.ContainsIgnoreCase("login.php")) + { + CookiesUpdater(null, null); + throw new IndexerException(indexerResponse, "We are currently on the login page. Most likely your session expired or was killed. Try testing the indexer in the settings."); + } // Remove cookie cache - AuthCookieCache.Remove(_settings.BaseUrl.Trim().TrimEnd('/')); - throw new IndexerException(indexerResponse, $"Unexpected response header {indexerResponse.HttpResponse.Headers.ContentType} from API request, expected {HttpAccept.Json.Value}"); } @@ -97,6 +103,8 @@ public IList ParseResponse(IndexerResponse indexerResponse) } + public Action, DateTime?> CookiesUpdater { get; set; } + private string GetDownloadUrl(int torrentId, string authKey, string passKey) { var url = new HttpUri(_settings.BaseUrl) diff --git a/src/NzbDrone.Core/Indexers/PassThePopcorn/PassThePopcornRequestGenerator.cs b/src/NzbDrone.Core/Indexers/PassThePopcorn/PassThePopcornRequestGenerator.cs index 01133f1a3..e2099dbc5 100644 --- a/src/NzbDrone.Core/Indexers/PassThePopcorn/PassThePopcornRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/PassThePopcorn/PassThePopcornRequestGenerator.cs @@ -13,7 +13,8 @@ public class PassThePopcornRequestGenerator : IIndexerRequestGenerator public PassThePopcornSettings Settings { get; set; } - public ICached> AuthCookieCache { get; set; } + public IDictionary Cookies { get; set; } + public IHttpClient HttpClient { get; set; } public Logger Logger { get; set; } @@ -33,62 +34,62 @@ public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchC return pageableRequests; } + public Func> GetCookies { get; set; } + public Action, DateTime?> CookiesUpdater { get; set; } + private IEnumerable GetRequest(string searchParameters) { + Cookies = GetCookies(); + Authenticate(); - var filter = ""; - if (searchParameters == null) - { - - - } - var request = new IndexerRequest( - $"{Settings.BaseUrl.Trim().TrimEnd('/')}/torrents.php?action=advanced&json=noredirect&searchstr={searchParameters}{filter}", + $"{Settings.BaseUrl.Trim().TrimEnd('/')}/torrents.php?action=advanced&json=noredirect&searchstr={searchParameters}", HttpAccept.Json); - var cookies = AuthCookieCache.Find(Settings.BaseUrl.Trim().TrimEnd('/')); - foreach (var cookie in cookies) + foreach (var cookie in Cookies) { request.HttpRequest.Cookies[cookie.Key] = cookie.Value; } + CookiesUpdater(Cookies, DateTime.Now + TimeSpan.FromDays(30)); + yield return request; } private void Authenticate() { - var requestBuilder = new HttpRequestBuilder($"{Settings.BaseUrl.Trim().TrimEnd('/')}") + if (Cookies == null) { - LogResponseContent = true - }; + var requestBuilder = new HttpRequestBuilder($"{Settings.BaseUrl.Trim().TrimEnd('/')}") + { + LogResponseContent = true + }; - requestBuilder.Method = HttpMethod.POST; - requestBuilder.Resource("ajax.php?action=login"); - requestBuilder.PostProcess += r => r.RequestTimeout = TimeSpan.FromSeconds(15); - - var authKey = Settings.BaseUrl.Trim().TrimEnd('/'); - var cookies = AuthCookieCache.Find(authKey); - - if (cookies == null) - { - AuthCookieCache.Remove(authKey); + requestBuilder.Method = HttpMethod.POST; + requestBuilder.Resource("ajax.php?action=login"); + requestBuilder.PostProcess += r => r.RequestTimeout = TimeSpan.FromSeconds(15); + var authLoginRequest = requestBuilder .AddFormParameter("username", Settings.Username) .AddFormParameter("password", Settings.Password) .AddFormParameter("passkey", Settings.Passkey) .AddFormParameter("keeplogged", "1") - .AddFormParameter("login", "Log In!") .SetHeader("Content-Type", "multipart/form-data") .Accept(HttpAccept.Json) .Build(); + authLoginRequest.AllowAutoRedirect = true; + // We want clean cookies for the auth request. + authLoginRequest.StoreRequestCookie = false; + authLoginRequest.StoreResponseCookie = false; + authLoginRequest.Cookies.Clear(); + authLoginRequest.IgnorePersistentCookies = true; var response = HttpClient.Execute(authLoginRequest); var result = Json.Deserialize(response.Content); - if (result.Result != "Ok" || string.IsNullOrWhiteSpace(result.Result)) + if (result?.Result != "Ok" || string.IsNullOrWhiteSpace(result.Result)) { Logger.Debug("PassThePopcorn authentication failed."); throw new Exception("Failed to authenticate with PassThePopcorn."); @@ -96,13 +97,8 @@ private void Authenticate() Logger.Debug("PassThePopcorn authentication succeeded."); - cookies = response.GetCookies(); - AuthCookieCache.Set(authKey, cookies, new TimeSpan(7, 0, 0, 0, 0)); // re-auth every 7 days - requestBuilder.SetCookies(cookies); - } - else - { - requestBuilder.SetCookies(cookies); + Cookies = response.GetCookies(); + requestBuilder.SetCookies(Cookies); } } } diff --git a/src/NzbDrone.Core/Indexers/Rarbg/RarbgParser.cs b/src/NzbDrone.Core/Indexers/Rarbg/RarbgParser.cs index 223446ffb..1c3a816ea 100644 --- a/src/NzbDrone.Core/Indexers/Rarbg/RarbgParser.cs +++ b/src/NzbDrone.Core/Indexers/Rarbg/RarbgParser.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.Net; using System.Text.RegularExpressions; @@ -76,6 +77,8 @@ public IList ParseResponse(IndexerResponse indexerResponse) return results; } + public Action, DateTime?> CookiesUpdater { get; set; } + private string GetGuid(RarbgTorrent torrent) { var match = RegexGuid.Match(torrent.download); diff --git a/src/NzbDrone.Core/Indexers/Rarbg/RarbgRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Rarbg/RarbgRequestGenerator.cs index 59caed890..53c790e4d 100644 --- a/src/NzbDrone.Core/Indexers/Rarbg/RarbgRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/Rarbg/RarbgRequestGenerator.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; @@ -105,5 +106,8 @@ private IEnumerable GetMovieRequest(MovieSearchCriteria searchCr yield return new IndexerRequest(requestBuilder.Build()); } + + public Func> GetCookies { get; set; } + public Action, DateTime?> CookiesUpdater { get; set; } } } diff --git a/src/NzbDrone.Core/Indexers/RssIndexerRequestGenerator.cs b/src/NzbDrone.Core/Indexers/RssIndexerRequestGenerator.cs index 40cbaaaac..328bc49b2 100644 --- a/src/NzbDrone.Core/Indexers/RssIndexerRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/RssIndexerRequestGenerator.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using NzbDrone.Common.Http; using NzbDrone.Core.IndexerSearch.Definitions; @@ -27,5 +28,8 @@ public virtual IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria { return new IndexerPageableRequestChain(); } + + public Func> GetCookies { get; set; } + public Action, DateTime?> CookiesUpdater { get; set; } } } diff --git a/src/NzbDrone.Core/Indexers/RssParser.cs b/src/NzbDrone.Core/Indexers/RssParser.cs index f03513cd6..40445df22 100644 --- a/src/NzbDrone.Core/Indexers/RssParser.cs +++ b/src/NzbDrone.Core/Indexers/RssParser.cs @@ -73,6 +73,8 @@ public virtual IList ParseResponse(IndexerResponse indexerResponse) return releases; } + public Action, DateTime?> CookiesUpdater { get; set; } + protected virtual XDocument LoadXmlDocument(IndexerResponse indexerResponse) { try diff --git a/src/NzbDrone.Core/Indexers/TorrentPotato/TorrentPotatoParser.cs b/src/NzbDrone.Core/Indexers/TorrentPotato/TorrentPotatoParser.cs index 54b5f23c4..c9cf7f7df 100644 --- a/src/NzbDrone.Core/Indexers/TorrentPotato/TorrentPotatoParser.cs +++ b/src/NzbDrone.Core/Indexers/TorrentPotato/TorrentPotatoParser.cs @@ -48,6 +48,8 @@ public IList ParseResponse(IndexerResponse indexerResponse) return results; } + public Action, DateTime?> CookiesUpdater { get; set; } + private string GetGuid(Result torrent) { var match = RegexGuid.Match(torrent.download_url); diff --git a/src/NzbDrone.Core/Indexers/TorrentPotato/TorrentPotatoRequestGenerator.cs b/src/NzbDrone.Core/Indexers/TorrentPotato/TorrentPotatoRequestGenerator.cs index e4409d368..3d8a73f7b 100644 --- a/src/NzbDrone.Core/Indexers/TorrentPotato/TorrentPotatoRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/TorrentPotato/TorrentPotatoRequestGenerator.cs @@ -79,5 +79,8 @@ public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchC pageableRequests.Add(GetMovieRequest(searchCriteria)); return pageableRequests; } + + public Func> GetCookies { get; set; } + public Action, DateTime?> CookiesUpdater { get; set; } } } diff --git a/src/NzbDrone.Core/Indexers/TorrentRss/TorrentRssIndexerRequestGenerator.cs b/src/NzbDrone.Core/Indexers/TorrentRss/TorrentRssIndexerRequestGenerator.cs index f5843311b..4de64aa34 100644 --- a/src/NzbDrone.Core/Indexers/TorrentRss/TorrentRssIndexerRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/TorrentRss/TorrentRssIndexerRequestGenerator.cs @@ -38,5 +38,8 @@ private IEnumerable GetRssRequests(string searchParameters) yield return request; } + + public Func> GetCookies { get; set; } + public Action, DateTime?> CookiesUpdater { get; set; } } } diff --git a/src/NzbDrone.Core/NzbDrone.Core.csproj b/src/NzbDrone.Core/NzbDrone.Core.csproj index 9a851d1a2..efde3fb1e 100644 --- a/src/NzbDrone.Core/NzbDrone.Core.csproj +++ b/src/NzbDrone.Core/NzbDrone.Core.csproj @@ -128,6 +128,7 @@ +