using System; using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Net; using System.Net.Http; using System.Threading.Tasks; using NLog; using NzbDrone.Common.Cache; using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http.Dispatchers; using NzbDrone.Common.TPL; namespace NzbDrone.Common.Http { public interface IHttpClient { HttpResponse Execute(HttpRequest request); void DownloadFile(string url, string fileName); HttpResponse Get(HttpRequest request); HttpResponse Get(HttpRequest request) where T : new(); HttpResponse Head(HttpRequest request); HttpResponse Post(HttpRequest request); HttpResponse Post(HttpRequest request) where T : new(); Task ExecuteAsync(HttpRequest request); Task DownloadFileAsync(string url, string fileName); Task GetAsync(HttpRequest request); Task> GetAsync(HttpRequest request) where T : new(); Task HeadAsync(HttpRequest request); Task PostAsync(HttpRequest request); Task> PostAsync(HttpRequest request) where T : new(); } public class HttpClient : IHttpClient { private const int MaxRedirects = 5; private readonly Logger _logger; private readonly IRateLimitService _rateLimitService; private readonly ICached _cookieContainerCache; private readonly List _requestInterceptors; private readonly IHttpDispatcher _httpDispatcher; public HttpClient(IEnumerable requestInterceptors, ICacheManager cacheManager, IRateLimitService rateLimitService, IHttpDispatcher httpDispatcher, Logger logger) { _requestInterceptors = requestInterceptors.ToList(); _rateLimitService = rateLimitService; _httpDispatcher = httpDispatcher; _logger = logger; ServicePointManager.DefaultConnectionLimit = 12; _cookieContainerCache = cacheManager.GetCache(typeof(HttpClient)); } public virtual async Task ExecuteAsync(HttpRequest request) { var cookieContainer = InitializeRequestCookies(request); var response = await ExecuteRequestAsync(request, cookieContainer); if (request.AllowAutoRedirect && response.HasHttpRedirect) { var autoRedirectChain = new List(); autoRedirectChain.Add(request.Url.ToString()); do { request.Url = new HttpUri(response.RedirectUrl); autoRedirectChain.Add(request.Url.ToString()); _logger.Trace("Redirected to {0}", request.Url); if (autoRedirectChain.Count > MaxRedirects) { throw new WebException($"Too many automatic redirections were attempted for {autoRedirectChain.Join(" -> ")}", WebExceptionStatus.ProtocolError); } // 302 or 303 should default to GET on redirect even if POST on original if (response.StatusCode == HttpStatusCode.Redirect || response.StatusCode == HttpStatusCode.RedirectMethod) { request.Method = HttpMethod.Get; request.ContentData = null; } response = await ExecuteRequestAsync(request, cookieContainer); } while (response.HasHttpRedirect); } if (response.HasHttpRedirect && !RuntimeInfo.IsProduction) { _logger.Error("Server requested a redirect to [{0}] while in developer mode. Update the request URL to avoid this redirect.", response.Headers["Location"]); } if (!request.SuppressHttpError && response.HasHttpError) { _logger.Warn("HTTP Error - {0}", response); if ((int)response.StatusCode == 429) { throw new TooManyRequestsException(request, response); } else { throw new HttpException(request, response); } } return response; } public HttpResponse Execute(HttpRequest request) { return ExecuteAsync(request).GetAwaiter().GetResult(); } private async Task ExecuteRequestAsync(HttpRequest request, CookieContainer cookieContainer) { foreach (var interceptor in _requestInterceptors) { request = interceptor.PreRequest(request); } if (request.RateLimit != TimeSpan.Zero) { await _rateLimitService.WaitAndPulseAsync(request.Url.Host, request.RateLimit); } _logger.Trace(request); var stopWatch = Stopwatch.StartNew(); PrepareRequestCookies(request, cookieContainer); var response = await _httpDispatcher.GetResponseAsync(request, cookieContainer); HandleResponseCookies(response, cookieContainer); stopWatch.Stop(); _logger.Trace("{0} ({1} ms)", response, stopWatch.ElapsedMilliseconds); foreach (var interceptor in _requestInterceptors) { response = interceptor.PostResponse(response); } if (request.LogResponseContent) { _logger.Trace("Response content ({0} bytes): {1}", response.ResponseData.Length, response.Content); } return response; } private CookieContainer InitializeRequestCookies(HttpRequest request) { lock (_cookieContainerCache) { var sourceContainer = new CookieContainer(); var presistentContainer = _cookieContainerCache.Get("container", () => new CookieContainer()); var persistentCookies = presistentContainer.GetCookies((Uri)request.Url); sourceContainer.Add(persistentCookies); if (request.Cookies.Count != 0) { foreach (var pair in request.Cookies) { Cookie cookie; if (pair.Value == null) { cookie = new Cookie(pair.Key, "", "/") { Expires = DateTime.Now.AddDays(-1) }; } else { cookie = new Cookie(pair.Key, pair.Value, "/") { // Use Now rather than UtcNow to work around Mono cookie expiry bug. // See https://gist.github.com/ta264/7822b1424f72e5b4c961 Expires = DateTime.Now.AddHours(1) }; } sourceContainer.Add((Uri)request.Url, cookie); if (request.StoreRequestCookie) { presistentContainer.Add((Uri)request.Url, cookie); } } } return sourceContainer; } } private void PrepareRequestCookies(HttpRequest request, CookieContainer cookieContainer) { // Don't collect persistnet cookies for intermediate/redirected urls. /*lock (_cookieContainerCache) { var presistentContainer = _cookieContainerCache.Get("container", () => new CookieContainer()); var persistentCookies = presistentContainer.GetCookies((Uri)request.Url); var existingCookies = cookieContainer.GetCookies((Uri)request.Url); cookieContainer.Add(persistentCookies); cookieContainer.Add(existingCookies); }*/ } private void HandleResponseCookies(HttpResponse response, CookieContainer cookieContainer) { var cookieHeaders = response.GetCookieHeaders(); if (cookieHeaders.Empty()) { return; } if (response.Request.StoreResponseCookie) { lock (_cookieContainerCache) { var persistentCookieContainer = _cookieContainerCache.Get("container", () => new CookieContainer()); foreach (var cookieHeader in cookieHeaders) { try { persistentCookieContainer.SetCookies((Uri)response.Request.Url, cookieHeader); } catch (Exception ex) { _logger.Debug(ex, "Invalid cookie in {0}", response.Request.Url); } } } } } public async Task DownloadFileAsync(string url, string fileName) { await _httpDispatcher.DownloadFileAsync(url, fileName); } public void DownloadFile(string url, string fileName) { // https://docs.microsoft.com/en-us/archive/msdn-magazine/2015/july/async-programming-brownfield-async-development#the-thread-pool-hack Task.Run(() => DownloadFileAsync(url, fileName)).GetAwaiter().GetResult(); } public Task GetAsync(HttpRequest request) { request.Method = HttpMethod.Get; return ExecuteAsync(request); } public HttpResponse Get(HttpRequest request) { return Task.Run(() => GetAsync(request)).GetAwaiter().GetResult(); } public async Task> GetAsync(HttpRequest request) where T : new() { var response = await GetAsync(request); CheckResponseContentType(response); return new HttpResponse(response); } public HttpResponse Get(HttpRequest request) where T : new() { return Task.Run(() => GetAsync(request)).GetAwaiter().GetResult(); } public Task HeadAsync(HttpRequest request) { request.Method = HttpMethod.Head; return ExecuteAsync(request); } public HttpResponse Head(HttpRequest request) { return Task.Run(() => HeadAsync(request)).GetAwaiter().GetResult(); } public Task PostAsync(HttpRequest request) { request.Method = HttpMethod.Post; return ExecuteAsync(request); } public HttpResponse Post(HttpRequest request) { return Task.Run(() => PostAsync(request)).GetAwaiter().GetResult(); } public async Task> PostAsync(HttpRequest request) where T : new() { var response = await PostAsync(request); CheckResponseContentType(response); return new HttpResponse(response); } public HttpResponse Post(HttpRequest request) where T : new() { return Task.Run(() => PostAsync(request)).GetAwaiter().GetResult(); } private void CheckResponseContentType(HttpResponse response) { if (response.Headers.ContentType != null && response.Headers.ContentType.Contains("text/html")) { throw new UnexpectedHtmlContentException(response); } } } }