mirror of
https://github.com/Radarr/Radarr.git
synced 2024-09-11 03:52:33 +02:00
Async HttpClient and list lookup
This commit is contained in:
parent
c64c2d9f27
commit
c8a2af867e
@ -1,18 +1,19 @@
|
||||
using System.IO;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace NzbDrone.Common.Extensions
|
||||
{
|
||||
public static class StreamExtensions
|
||||
{
|
||||
public static byte[] ToBytes(this Stream input)
|
||||
public static async Task<byte[]> ToBytes(this Stream input)
|
||||
{
|
||||
var buffer = new byte[16 * 1024];
|
||||
using (var ms = new MemoryStream())
|
||||
{
|
||||
int read;
|
||||
while ((read = input.Read(buffer, 0, buffer.Length)) > 0)
|
||||
while ((read = await input.ReadAsync(buffer, 0, buffer.Length)) > 0)
|
||||
{
|
||||
ms.Write(buffer, 0, read);
|
||||
await ms.WriteAsync(buffer, 0, read);
|
||||
}
|
||||
|
||||
return ms.ToArray();
|
||||
|
@ -1,9 +1,10 @@
|
||||
using System.Net;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace NzbDrone.Common.Http.Dispatchers
|
||||
{
|
||||
public interface IHttpDispatcher
|
||||
{
|
||||
HttpResponse GetResponse(HttpRequest request, CookieContainer cookies);
|
||||
Task<HttpResponse> GetResponseAsync(HttpRequest request, CookieContainer cookies);
|
||||
}
|
||||
}
|
||||
|
@ -3,6 +3,7 @@
|
||||
using System.IO.Compression;
|
||||
using System.Net;
|
||||
using System.Reflection;
|
||||
using System.Threading.Tasks;
|
||||
using NLog;
|
||||
using NLog.Fluent;
|
||||
using NzbDrone.Common.EnvironmentInfo;
|
||||
@ -28,7 +29,7 @@ public ManagedHttpDispatcher(IHttpProxySettingsProvider proxySettingsProvider, I
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public HttpResponse GetResponse(HttpRequest request, CookieContainer cookies)
|
||||
public async Task<HttpResponse> GetResponseAsync(HttpRequest request, CookieContainer cookies)
|
||||
{
|
||||
var webRequest = (HttpWebRequest)WebRequest.Create((Uri)request.Url);
|
||||
|
||||
@ -77,7 +78,7 @@ public HttpResponse GetResponse(HttpRequest request, CookieContainer cookies)
|
||||
}
|
||||
}
|
||||
|
||||
httpWebResponse = (HttpWebResponse)webRequest.GetResponse();
|
||||
httpWebResponse = (HttpWebResponse)await webRequest.GetResponseAsync();
|
||||
}
|
||||
catch (WebException e)
|
||||
{
|
||||
@ -120,7 +121,7 @@ public HttpResponse GetResponse(HttpRequest request, CookieContainer cookies)
|
||||
{
|
||||
try
|
||||
{
|
||||
data = responseStream.ToBytes();
|
||||
data = await responseStream.ToBytes();
|
||||
|
||||
if (PlatformInfo.IsMono && httpWebResponse.ContentEncoding == "gzip")
|
||||
{
|
||||
|
@ -4,6 +4,7 @@
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Threading.Tasks;
|
||||
using NLog;
|
||||
using NzbDrone.Common.Cache;
|
||||
using NzbDrone.Common.EnvironmentInfo;
|
||||
@ -24,6 +25,16 @@ HttpResponse<T> Get<T>(HttpRequest request)
|
||||
HttpResponse Post(HttpRequest request);
|
||||
HttpResponse<T> Post<T>(HttpRequest request)
|
||||
where T : new();
|
||||
|
||||
Task<HttpResponse> ExecuteAsync(HttpRequest request);
|
||||
Task DownloadFileAsync(string url, string fileName);
|
||||
Task<HttpResponse> GetAsync(HttpRequest request);
|
||||
Task<HttpResponse<T>> GetAsync<T>(HttpRequest request)
|
||||
where T : new();
|
||||
Task<HttpResponse> HeadAsync(HttpRequest request);
|
||||
Task<HttpResponse> PostAsync(HttpRequest request);
|
||||
Task<HttpResponse<T>> PostAsync<T>(HttpRequest request)
|
||||
where T : new();
|
||||
}
|
||||
|
||||
public class HttpClient : IHttpClient
|
||||
@ -54,11 +65,11 @@ public HttpClient(IEnumerable<IHttpRequestInterceptor> requestInterceptors,
|
||||
_cookieContainerCache = cacheManager.GetCache<CookieContainer>(typeof(HttpClient));
|
||||
}
|
||||
|
||||
public HttpResponse Execute(HttpRequest request)
|
||||
public async Task<HttpResponse> ExecuteAsync(HttpRequest request)
|
||||
{
|
||||
var cookieContainer = InitializeRequestCookies(request);
|
||||
|
||||
var response = ExecuteRequest(request, cookieContainer);
|
||||
var response = await ExecuteRequestAsync(request, cookieContainer);
|
||||
|
||||
if (request.AllowAutoRedirect && response.HasHttpRedirect)
|
||||
{
|
||||
@ -77,7 +88,7 @@ public HttpResponse Execute(HttpRequest request)
|
||||
throw new WebException($"Too many automatic redirections were attempted for {autoRedirectChain.Join(" -> ")}", WebExceptionStatus.ProtocolError);
|
||||
}
|
||||
|
||||
response = ExecuteRequest(request, cookieContainer);
|
||||
response = await ExecuteRequestAsync(request, cookieContainer);
|
||||
}
|
||||
while (response.HasHttpRedirect);
|
||||
}
|
||||
@ -104,7 +115,12 @@ public HttpResponse Execute(HttpRequest request)
|
||||
return response;
|
||||
}
|
||||
|
||||
private HttpResponse ExecuteRequest(HttpRequest request, CookieContainer cookieContainer)
|
||||
public HttpResponse Execute(HttpRequest request)
|
||||
{
|
||||
return ExecuteAsync(request).GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
private async Task<HttpResponse> ExecuteRequestAsync(HttpRequest request, CookieContainer cookieContainer)
|
||||
{
|
||||
foreach (var interceptor in _requestInterceptors)
|
||||
{
|
||||
@ -113,7 +129,7 @@ private HttpResponse ExecuteRequest(HttpRequest request, CookieContainer cookieC
|
||||
|
||||
if (request.RateLimit != TimeSpan.Zero)
|
||||
{
|
||||
_rateLimitService.WaitAndPulse(request.Url.Host, request.RateLimit);
|
||||
await _rateLimitService.WaitAndPulseAsync(request.Url.Host, request.RateLimit);
|
||||
}
|
||||
|
||||
_logger.Trace(request);
|
||||
@ -122,7 +138,7 @@ private HttpResponse ExecuteRequest(HttpRequest request, CookieContainer cookieC
|
||||
|
||||
PrepareRequestCookies(request, cookieContainer);
|
||||
|
||||
var response = _httpDispatcher.GetResponse(request, cookieContainer);
|
||||
var response = await _httpDispatcher.GetResponseAsync(request, cookieContainer);
|
||||
|
||||
HandleResponseCookies(response, cookieContainer);
|
||||
|
||||
@ -231,7 +247,7 @@ private void HandleResponseCookies(HttpResponse response, CookieContainer cookie
|
||||
}
|
||||
}
|
||||
|
||||
public void DownloadFile(string url, string fileName)
|
||||
public async Task DownloadFileAsync(string url, string fileName)
|
||||
{
|
||||
try
|
||||
{
|
||||
@ -247,7 +263,7 @@ public void DownloadFile(string url, string fileName)
|
||||
using (var webClient = new GZipWebClient())
|
||||
{
|
||||
webClient.Headers.Add(HttpRequestHeader.UserAgent, _userAgentBuilder.GetUserAgent());
|
||||
webClient.DownloadFile(url, fileName);
|
||||
await webClient.DownloadFileTaskAsync(url, fileName);
|
||||
stopWatch.Stop();
|
||||
_logger.Debug("Downloading Completed. took {0:0}s", stopWatch.Elapsed.Seconds);
|
||||
}
|
||||
@ -255,47 +271,92 @@ public void DownloadFile(string url, string fileName)
|
||||
catch (WebException e)
|
||||
{
|
||||
_logger.Warn("Failed to get response from: {0} {1}", url, e.Message);
|
||||
|
||||
if (File.Exists(fileName))
|
||||
{
|
||||
File.Delete(fileName);
|
||||
}
|
||||
|
||||
throw;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.Warn(e, "Failed to get response from: " + url);
|
||||
|
||||
if (File.Exists(fileName))
|
||||
{
|
||||
File.Delete(fileName);
|
||||
}
|
||||
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
public HttpResponse Get(HttpRequest request)
|
||||
public void DownloadFile(string url, string fileName)
|
||||
{
|
||||
// https://docs.microsoft.com/en-us/archive/msdn-magazine/2015/july/async-programming-brownfield-async-development#the-thread-pool-hack
|
||||
Task.Run(() => DownloadFileAsync(url, fileName)).GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
public Task<HttpResponse> GetAsync(HttpRequest request)
|
||||
{
|
||||
request.Method = HttpMethod.GET;
|
||||
return Execute(request);
|
||||
return ExecuteAsync(request);
|
||||
}
|
||||
|
||||
public HttpResponse Get(HttpRequest request)
|
||||
{
|
||||
return Task.Run(() => GetAsync(request)).GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
public async Task<HttpResponse<T>> GetAsync<T>(HttpRequest request)
|
||||
where T : new()
|
||||
{
|
||||
var response = await GetAsync(request);
|
||||
CheckResponseContentType(response);
|
||||
return new HttpResponse<T>(response);
|
||||
}
|
||||
|
||||
public HttpResponse<T> Get<T>(HttpRequest request)
|
||||
where T : new()
|
||||
{
|
||||
var response = Get(request);
|
||||
CheckResponseContentType(response);
|
||||
return new HttpResponse<T>(response);
|
||||
return Task.Run(() => GetAsync<T>(request)).GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
public Task<HttpResponse> HeadAsync(HttpRequest request)
|
||||
{
|
||||
request.Method = HttpMethod.HEAD;
|
||||
return ExecuteAsync(request);
|
||||
}
|
||||
|
||||
public HttpResponse Head(HttpRequest request)
|
||||
{
|
||||
request.Method = HttpMethod.HEAD;
|
||||
return Execute(request);
|
||||
return Task.Run(() => HeadAsync(request)).GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
public Task<HttpResponse> PostAsync(HttpRequest request)
|
||||
{
|
||||
request.Method = HttpMethod.POST;
|
||||
return ExecuteAsync(request);
|
||||
}
|
||||
|
||||
public HttpResponse Post(HttpRequest request)
|
||||
{
|
||||
request.Method = HttpMethod.POST;
|
||||
return Execute(request);
|
||||
return Task.Run(() => PostAsync(request)).GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
public async Task<HttpResponse<T>> PostAsync<T>(HttpRequest request)
|
||||
where T : new()
|
||||
{
|
||||
var response = await PostAsync(request);
|
||||
CheckResponseContentType(response);
|
||||
return new HttpResponse<T>(response);
|
||||
}
|
||||
|
||||
public HttpResponse<T> Post<T>(HttpRequest request)
|
||||
where T : new()
|
||||
{
|
||||
var response = Post(request);
|
||||
CheckResponseContentType(response);
|
||||
return new HttpResponse<T>(response);
|
||||
return Task.Run(() => PostAsync<T>(request)).GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
private void CheckResponseContentType(HttpResponse response)
|
||||
|
@ -1,5 +1,6 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Threading.Tasks;
|
||||
using NLog;
|
||||
using NzbDrone.Common.Cache;
|
||||
|
||||
@ -8,6 +9,7 @@ namespace NzbDrone.Common.TPL
|
||||
public interface IRateLimitService
|
||||
{
|
||||
void WaitAndPulse(string key, TimeSpan interval);
|
||||
Task WaitAndPulseAsync(string key, TimeSpan interval);
|
||||
}
|
||||
|
||||
public class RateLimitService : IRateLimitService
|
||||
@ -23,13 +25,7 @@ public RateLimitService(ICacheManager cacheManager, Logger logger)
|
||||
|
||||
public void WaitAndPulse(string key, TimeSpan interval)
|
||||
{
|
||||
var waitUntil = _rateLimitStore.AddOrUpdate(key,
|
||||
(s) => DateTime.UtcNow + interval,
|
||||
(s, i) => new DateTime(Math.Max(DateTime.UtcNow.Ticks, i.Ticks), DateTimeKind.Utc) + interval);
|
||||
|
||||
waitUntil -= interval;
|
||||
|
||||
var delay = waitUntil - DateTime.UtcNow;
|
||||
var delay = GetDelay(key, interval);
|
||||
|
||||
if (delay.TotalSeconds > 0.0)
|
||||
{
|
||||
@ -37,5 +33,27 @@ public void WaitAndPulse(string key, TimeSpan interval)
|
||||
System.Threading.Thread.Sleep(delay);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task WaitAndPulseAsync(string key, TimeSpan interval)
|
||||
{
|
||||
var delay = GetDelay(key, interval);
|
||||
|
||||
if (delay.TotalSeconds > 0.0)
|
||||
{
|
||||
_logger.Trace("Rate Limit triggered, delaying '{0}' for {1:0.000} sec", key, delay.TotalSeconds);
|
||||
await Task.Delay(delay);
|
||||
}
|
||||
}
|
||||
|
||||
private TimeSpan GetDelay(string key, TimeSpan interval)
|
||||
{
|
||||
var waitUntil = _rateLimitStore.AddOrUpdate(key,
|
||||
(s) => DateTime.UtcNow + interval,
|
||||
(s, i) => new DateTime(Math.Max(DateTime.UtcNow.Ticks, i.Ticks), DateTimeKind.Utc) + interval);
|
||||
|
||||
waitUntil -= interval;
|
||||
|
||||
return waitUntil - DateTime.UtcNow;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -1,5 +1,6 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using NzbDrone.Core.Movies;
|
||||
using NzbDrone.Core.Movies.Credits;
|
||||
|
||||
@ -10,5 +11,7 @@ public interface IProvideMovieInfo
|
||||
Movie GetMovieByImdbId(string imdbId);
|
||||
Tuple<Movie, List<Credit>> GetMovieInfo(int tmdbId);
|
||||
HashSet<int> GetChangedMovies(DateTime startTime);
|
||||
|
||||
Task<Tuple<Movie, List<Credit>>> GetMovieInfoAsync(int tmdbId);
|
||||
}
|
||||
}
|
||||
|
@ -1,4 +1,5 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using NzbDrone.Core.Movies;
|
||||
|
||||
namespace NzbDrone.Core.MetadataSource
|
||||
@ -8,5 +9,6 @@ public interface ISearchForNewMovie
|
||||
List<Movie> SearchForNewMovie(string title);
|
||||
|
||||
Movie MapMovieToTmdbMovie(Movie movie);
|
||||
Task<Movie> MapMovieToTmdbMovieAsync(Movie movie);
|
||||
}
|
||||
}
|
||||
|
@ -2,6 +2,7 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Threading.Tasks;
|
||||
using NLog;
|
||||
using NzbDrone.Common.Cloud;
|
||||
using NzbDrone.Common.Extensions;
|
||||
@ -14,7 +15,6 @@
|
||||
using NzbDrone.Core.Movies;
|
||||
using NzbDrone.Core.Movies.AlternativeTitles;
|
||||
using NzbDrone.Core.Movies.Credits;
|
||||
using NzbDrone.Core.NetImport.ImportExclusions;
|
||||
using NzbDrone.Core.NetImport.TMDb;
|
||||
using NzbDrone.Core.Parser;
|
||||
|
||||
@ -65,7 +65,7 @@ public HashSet<int> GetChangedMovies(DateTime startTime)
|
||||
return new HashSet<int>(response.Resource.Results.Select(c => c.id));
|
||||
}
|
||||
|
||||
public Tuple<Movie, List<Credit>> GetMovieInfo(int tmdbId)
|
||||
public async Task<Tuple<Movie, List<Credit>>> GetMovieInfoAsync(int tmdbId)
|
||||
{
|
||||
var httpRequest = _radarrMetadata.Create()
|
||||
.SetSegment("route", "movie")
|
||||
@ -75,7 +75,7 @@ public Tuple<Movie, List<Credit>> GetMovieInfo(int tmdbId)
|
||||
httpRequest.AllowAutoRedirect = true;
|
||||
httpRequest.SuppressHttpError = true;
|
||||
|
||||
var httpResponse = _httpClient.Get<MovieResource>(httpRequest);
|
||||
var httpResponse = await _httpClient.GetAsync<MovieResource>(httpRequest);
|
||||
|
||||
if (httpResponse.HasHttpError)
|
||||
{
|
||||
@ -98,6 +98,11 @@ public Tuple<Movie, List<Credit>> GetMovieInfo(int tmdbId)
|
||||
return new Tuple<Movie, List<Credit>>(movie, credits.ToList());
|
||||
}
|
||||
|
||||
public Tuple<Movie, List<Credit>> GetMovieInfo(int tmdbId)
|
||||
{
|
||||
return GetMovieInfoAsync(tmdbId).GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
public Movie GetMovieByImdbId(string imdbId)
|
||||
{
|
||||
var httpRequest = _radarrMetadata.Create()
|
||||
@ -236,14 +241,14 @@ private string StripTrailingTheFromTitle(string title)
|
||||
return title;
|
||||
}
|
||||
|
||||
public Movie MapMovieToTmdbMovie(Movie movie)
|
||||
public async Task<Movie> MapMovieToTmdbMovieAsync(Movie movie)
|
||||
{
|
||||
try
|
||||
{
|
||||
Movie newMovie = movie;
|
||||
if (movie.TmdbId > 0)
|
||||
{
|
||||
newMovie = GetMovieInfo(movie.TmdbId).Item1;
|
||||
newMovie = (await GetMovieInfoAsync(movie.TmdbId)).Item1;
|
||||
}
|
||||
else if (movie.ImdbId.IsNotNullOrWhiteSpace())
|
||||
{
|
||||
@ -283,6 +288,11 @@ public Movie MapMovieToTmdbMovie(Movie movie)
|
||||
}
|
||||
}
|
||||
|
||||
public Movie MapMovieToTmdbMovie(Movie movie)
|
||||
{
|
||||
return MapMovieToTmdbMovieAsync(movie).GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
public List<Movie> SearchForNewMovie(string title)
|
||||
{
|
||||
try
|
||||
|
@ -1,4 +1,5 @@
|
||||
using NLog;
|
||||
using System;
|
||||
using NLog;
|
||||
using NzbDrone.Common.Cloud;
|
||||
using NzbDrone.Common.Http;
|
||||
using NzbDrone.Core.Configuration;
|
||||
@ -11,6 +12,7 @@ public abstract class TMDbNetImportBase<TSettings> : HttpNetImportBase<TSettings
|
||||
where TSettings : TMDbSettingsBase<TSettings>, new()
|
||||
{
|
||||
public override NetImportType ListType => NetImportType.TMDB;
|
||||
public override TimeSpan RateLimit => TimeSpan.Zero;
|
||||
|
||||
public readonly ISearchForNewMovie _skyhookProxy;
|
||||
public readonly IHttpRequestBuilderFactory _requestBuilder;
|
||||
|
@ -1,5 +1,6 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using Nancy;
|
||||
using NzbDrone.Common.Extensions;
|
||||
using NzbDrone.Core.MediaCover;
|
||||
@ -27,22 +28,14 @@ private object Search()
|
||||
{
|
||||
var results = _fetchNetImport.FetchAndFilter((int)Request.Query.listId, false);
|
||||
|
||||
List<Movie> realResults = new List<Movie>();
|
||||
var tasks = results.Where(movie => movie.TmdbId == 0 || !movie.Images.Any() || movie.Overview.IsNullOrWhiteSpace())
|
||||
.Select(x => _movieSearch.MapMovieToTmdbMovieAsync(x));
|
||||
|
||||
foreach (var movie in results)
|
||||
{
|
||||
var mapped = movie;
|
||||
var realResults = results.Where(movie => movie.TmdbId != 0 && movie.Images.Any() && movie.Overview.IsNotNullOrWhiteSpace()).ToList();
|
||||
|
||||
if (movie.TmdbId == 0 || !movie.Images.Any() || movie.Overview.IsNullOrWhiteSpace())
|
||||
{
|
||||
mapped = _movieSearch.MapMovieToTmdbMovie(movie);
|
||||
}
|
||||
var mapped = Task.WhenAll(tasks).GetAwaiter().GetResult();
|
||||
|
||||
if (mapped != null)
|
||||
{
|
||||
realResults.Add(mapped);
|
||||
}
|
||||
}
|
||||
realResults.AddRange(mapped.Where(x => x != null));
|
||||
|
||||
return MapToResource(realResults);
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user