Fixed: Cardigann Download block handling

src/NzbDrone.Common/Http/HttpRequestBuilder.cs

@@ -291,6 +291,16 @@ namespace NzbDrone.Common.Http
             return this;
         }
 
+        public virtual HttpRequestBuilder SetHeaders(Dictionary<string, string> headers)
+        {
+            foreach (var header in headers)
+            {
+                Headers.Set(header.Key, header.Value);
+            }
+
+            return this;
+        }
+
         public virtual HttpRequestBuilder AddPrefixQueryParam(string key, object value, bool replace = false)
         {
             if (replace)

src/NzbDrone.Core/Download/DownloadService.cs

@@ -112,7 +112,7 @@ namespace NzbDrone.Core.Download
 
         public async Task<byte[]> DownloadReport(string link, int indexerId, string source, string title)
         {
-            var url = new HttpUri(link);
+            var url = new Uri(link);
 
             // Limit grabs to 2 per second.
             if (link.IsNotNullOrWhiteSpace() && !link.StartsWith("magnet:"))

src/NzbDrone.Core/History/HistoryService.cs

@@ -142,7 +142,7 @@ namespace NzbDrone.Core.History
             if (message.Query is BookSearchCriteria)
             {
                 history.Data.Add("Author", ((BookSearchCriteria)message.Query).Author ?? string.Empty);
-                history.Data.Add("Title", ((BookSearchCriteria)message.Query).Title ?? string.Empty);
+                history.Data.Add("BookTitle", ((BookSearchCriteria)message.Query).Title ?? string.Empty);
             }
 
             history.Data.Add("ElapsedTime", message.Time.ToString());

src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs

@@ -37,6 +37,8 @@ namespace NzbDrone.Core.Indexers.Cardigann
                     Settings = Settings
                 });
 
+            generator = (CardigannRequestGenerator)SetCookieFunctions(generator);
+
             _generatorCache.ClearExpired();
 
             return generator;
@@ -134,6 +136,29 @@ namespace NzbDrone.Core.Indexers.Cardigann
             await generator.DoLogin();
         }
 
+        public override async Task<byte[]> Download(Uri link)
+        {
+            var generator = (CardigannRequestGenerator)GetRequestGenerator();
+
+            var request = await generator.DownloadRequest(link);
+            request.AllowAutoRedirect = true;
+
+            var downloadBytes = Array.Empty<byte>();
+
+            try
+            {
+                var response = await _httpClient.ExecuteAsync(request);
+                downloadBytes = response.ResponseData;
+            }
+            catch (Exception)
+            {
+                _indexerStatusService.RecordFailure(Definition.Id);
+                _logger.Error("Download failed");
+            }
+
+            return downloadBytes;
+        }
+
         protected override async Task Test(List<ValidationFailure> failures)
         {
             await base.Test(failures);

src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannBase.cs

@@ -6,6 +6,7 @@ using System.Security.Cryptography;
 using System.Text;
 using System.Text.RegularExpressions;
 using AngleSharp.Dom;
+using Microsoft.AspNetCore.WebUtilities;
 using Newtonsoft.Json.Linq;
 using NLog;
 using NzbDrone.Common.Extensions;
@@ -706,6 +707,44 @@ namespace NzbDrone.Core.Indexers.Cardigann
             return data;
         }
 
+        protected Dictionary<string, string> ParseCustomHeaders(Dictionary<string, List<string>> customHeaders,
+                                                      Dictionary<string, object> variables)
+        {
+            if (customHeaders == null)
+            {
+                return null;
+            }
+
+            // FIXME: fix jackett header handling (allow it to specifiy the same header multipe times)
+            var headers = new Dictionary<string, string>();
+            foreach (var header in customHeaders)
+            {
+                headers.Add(header.Key, ApplyGoTemplateText(header.Value[0], variables));
+            }
+
+            return headers;
+        }
+
+        protected IDictionary<string, object> AddTemplateVariablesFromUri(IDictionary<string, object> variables, Uri uri, string prefix = "")
+        {
+            variables[prefix + ".AbsoluteUri"] = uri.AbsoluteUri;
+            variables[prefix + ".AbsolutePath"] = uri.AbsolutePath;
+            variables[prefix + ".Scheme"] = uri.Scheme;
+            variables[prefix + ".Host"] = uri.Host;
+            variables[prefix + ".Port"] = uri.Port.ToString();
+            variables[prefix + ".PathAndQuery"] = uri.PathAndQuery;
+            variables[prefix + ".Query"] = uri.Query;
+            var queryString = QueryHelpers.ParseQuery(uri.Query);
+
+            foreach (var key in queryString.Keys)
+            {
+                //If we have supplied the same query string multiple time, just take the first.
+                variables[prefix + ".Query." + key] = queryString[key].First();
+            }
+
+            return variables;
+        }
+
         protected Uri ResolvePath(string path, Uri currentUrl = null)
         {
             return new Uri(currentUrl ?? new Uri(SiteLink), path);

src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs

@@ -607,7 +607,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
 
                     var request = new HttpRequestBuilder(captchaUrl.ToString())
                         .SetCookies(landingResult.GetCookies())
-                        .SetHeader("Referrer", loginUrl.AbsoluteUri)
+                        .SetHeader("Referer", loginUrl.AbsoluteUri)
                         .Build();
 
                     var response = await HttpClient.ExecuteAsync(request);
@@ -644,6 +644,139 @@ namespace NzbDrone.Core.Indexers.Cardigann
 
         protected string GetRedirectDomainHint(HttpResponse result) => GetRedirectDomainHint(result.Request.Url.ToString(), result.Headers.GetSingleValue("Location"));
 
+        protected async Task<HttpResponse> HandleRequest(RequestBlock request, Dictionary<string, object> variables = null, string referer = null)
+        {
+            var requestLinkStr = ResolvePath(ApplyGoTemplateText(request.Path, variables)).ToString();
+            _logger.Debug($"CardigannIndexer ({_definition.Id}): handleRequest() requestLinkStr= {requestLinkStr}");
+
+            Dictionary<string, string> pairs = null;
+            var queryCollection = new NameValueCollection();
+
+            var method = HttpMethod.GET;
+            if (string.Equals(request.Method, "post", StringComparison.OrdinalIgnoreCase))
+            {
+                method = HttpMethod.POST;
+                pairs = new Dictionary<string, string>();
+            }
+
+            foreach (var input in request.Inputs)
+            {
+                var value = ApplyGoTemplateText(input.Value, variables);
+                if (method == HttpMethod.GET)
+                {
+                    queryCollection.Add(input.Key, value);
+                }
+                else if (method == HttpMethod.POST)
+                {
+                    pairs.Add(input.Key, value);
+                }
+            }
+
+            if (queryCollection.Count > 0)
+            {
+                if (!requestLinkStr.Contains("?"))
+                {
+                    // TODO Need Encoding here if we add it back
+                    requestLinkStr += "?" + queryCollection.GetQueryString(separator: request.Queryseparator).Substring(1);
+                }
+                else
+                {
+                    requestLinkStr += queryCollection.GetQueryString(separator: request.Queryseparator);
+                }
+            }
+
+            var httpRequest = new HttpRequestBuilder(requestLinkStr)
+                .SetCookies(Cookies ?? new Dictionary<string, string>())
+                .SetHeaders(pairs ?? new Dictionary<string, string>())
+                .SetHeader("Referer", referer)
+                .Build();
+
+            httpRequest.Method = method;
+
+            var response = await HttpClient.ExecuteAsync(httpRequest);
+
+            _logger.Debug($"CardigannIndexer ({_definition.Id}): handleRequest() remote server returned {response.StatusCode.ToString()}");
+            return response;
+        }
+
+        public async Task<HttpRequest> DownloadRequest(Uri link)
+        {
+            Cookies = GetCookies();
+            var method = HttpMethod.GET;
+
+            if (_definition.Download != null)
+            {
+                var download = _definition.Download;
+                var variables = GetBaseTemplateVariables();
+
+                AddTemplateVariablesFromUri(variables, link, ".DownloadUri");
+
+                if (download.Before != null)
+                {
+                    await HandleRequest(download.Before, variables, link.ToString());
+                }
+
+                if (download.Method == "post")
+                {
+                    method = HttpMethod.POST;
+                }
+
+                if (download.Selector != null)
+                {
+                    var selector = ApplyGoTemplateText(download.Selector, variables);
+                    var headers = ParseCustomHeaders(_definition.Search?.Headers, variables);
+
+                    var request = new HttpRequestBuilder(link.ToString())
+                        .SetCookies(Cookies ?? new Dictionary<string, string>())
+                        .SetHeaders(headers ?? new Dictionary<string, string>())
+                        .Build();
+
+                    request.AllowAutoRedirect = true;
+
+                    var response = await HttpClient.ExecuteAsync(request);
+
+                    var results = response.Content;
+                    var searchResultParser = new HtmlParser();
+                    var searchResultDocument = searchResultParser.ParseDocument(results);
+                    var downloadElement = searchResultDocument.QuerySelector(selector);
+                    if (downloadElement != null)
+                    {
+                        _logger.Debug(string.Format("CardigannIndexer ({0}): Download selector {1} matched:{2}", _definition.Id, selector, downloadElement.ToHtmlPretty()));
+
+                        var href = "";
+                        if (download.Attribute != null)
+                        {
+                            href = downloadElement.GetAttribute(download.Attribute);
+                            if (href == null)
+                            {
+                                throw new Exception(string.Format("Attribute \"{0}\" is not set for element {1}", download.Attribute, downloadElement.ToHtmlPretty()));
+                            }
+                        }
+                        else
+                        {
+                            href = downloadElement.TextContent;
+                        }
+
+                        href = ApplyFilters(href, download.Filters, variables);
+                        link = ResolvePath(href, link);
+                    }
+                    else
+                    {
+                        _logger.Error(string.Format("CardigannIndexer ({0}): Download selector {1} didn't match:\n{2}", _definition.Id, download.Selector, results));
+                        throw new Exception(string.Format("Download selector {0} didn't match", download.Selector));
+                    }
+                }
+            }
+
+            var downloadRequest = new HttpRequestBuilder(link.AbsoluteUri)
+                .SetCookies(Cookies ?? new Dictionary<string, string>())
+                .Build();
+
+            downloadRequest.Method = method;
+
+            return downloadRequest;
+        }
+
         public bool CheckIfLoginIsNeeded(HttpResponse response)
         {
             if (response.HasHttpRedirect)

src/NzbDrone.Core/Indexers/Definitions/Headphones/Headphones.cs

@@ -50,9 +50,9 @@ namespace NzbDrone.Core.Indexers.Headphones
             }
         }
 
-        public override async Task<byte[]> Download(HttpUri link)
+        public override async Task<byte[]> Download(Uri link)
         {
-            var requestBuilder = new HttpRequestBuilder(link.FullUri);
+            var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri);
 
             var downloadBytes = Array.Empty<byte>();
 

src/NzbDrone.Core/Indexers/HttpIndexerBase.cs

@@ -97,11 +97,11 @@ namespace NzbDrone.Core.Indexers
             return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria));
         }
 
-        public override async Task<byte[]> Download(HttpUri link)
+        public override async Task<byte[]> Download(Uri link)
         {
             Cookies = GetCookies();
 
-            var requestBuilder = new HttpRequestBuilder(link.FullUri);
+            var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri);
 
             if (Cookies != null)
             {

src/NzbDrone.Core/Indexers/IIndexer.cs

@@ -1,5 +1,5 @@
+using System;
 using System.Threading.Tasks;
-using NzbDrone.Common.Http;
 using NzbDrone.Core.IndexerSearch.Definitions;
 using NzbDrone.Core.ThingiProvider;
 
@@ -22,7 +22,7 @@ namespace NzbDrone.Core.Indexers
         Task<IndexerPageableQueryResult> Fetch(BookSearchCriteria searchCriteria);
         Task<IndexerPageableQueryResult> Fetch(BasicSearchCriteria searchCriteria);
 
-        Task<byte[]> Download(HttpUri link);
+        Task<byte[]> Download(Uri link);
 
         IndexerCapabilities GetCapabilities();
     }

src/NzbDrone.Core/Indexers/IndexerBase.cs

@@ -74,7 +74,7 @@ namespace NzbDrone.Core.Indexers
         public abstract Task<IndexerPageableQueryResult> Fetch(TvSearchCriteria searchCriteria);
         public abstract Task<IndexerPageableQueryResult> Fetch(BookSearchCriteria searchCriteria);
         public abstract Task<IndexerPageableQueryResult> Fetch(BasicSearchCriteria searchCriteria);
-        public abstract Task<byte[]> Download(HttpUri searchCriteria);
+        public abstract Task<byte[]> Download(Uri searchCriteria);
 
         public abstract IndexerCapabilities GetCapabilities();