shpaq/Prowlarr-Prowlarr
1
0
Fork 0
mirror of https://github.com/Prowlarr/Prowlarr.git synced 2025-09-17 17:14:18 +02:00
Code Issues Packages Projects Releases Wiki Activity

Fixed: Cardigann Download block handling

Browse Source
This commit is contained in:
Qstick
2021-04-18 22:38:21 -04:00
parent 2938029400
commit 62977d0863
10 changed files with 217 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/NzbDrone.Common/Http/HttpRequestBuilder.cs
View File

@@ -291,6 +291,16 @@ namespace NzbDrone.Common.Http
return this;
}
public virtual HttpRequestBuilder SetHeaders(Dictionary<string, string> headers)
{
foreach (var header in headers)
{
Headers.Set(header.Key, header.Value);
}
return this;
}
public virtual HttpRequestBuilder AddPrefixQueryParam(string key, object value, bool replace = false)
{
if (replace)

2
src/NzbDrone.Core/Download/DownloadService.cs
View File

@@ -112,7 +112,7 @@ namespace NzbDrone.Core.Download
public async Task<byte[]> DownloadReport(string link, int indexerId, string source, string title)
{
var url = new HttpUri(link);
var url = new Uri(link);
// Limit grabs to 2 per second.
if (link.IsNotNullOrWhiteSpace() && !link.StartsWith("magnet:"))

2
src/NzbDrone.Core/History/HistoryService.cs
View File

@@ -142,7 +142,7 @@ namespace NzbDrone.Core.History
if (message.Query is BookSearchCriteria)
{
history.Data.Add("Author", ((BookSearchCriteria)message.Query).Author ?? string.Empty);
history.Data.Add("Title", ((BookSearchCriteria)message.Query).Title ?? string.Empty);
history.Data.Add("BookTitle", ((BookSearchCriteria)message.Query).Title ?? string.Empty);
}
history.Data.Add("ElapsedTime", message.Time.ToString());

25
src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs
View File

@@ -37,6 +37,8 @@ namespace NzbDrone.Core.Indexers.Cardigann
Settings = Settings
});
generator = (CardigannRequestGenerator)SetCookieFunctions(generator);
_generatorCache.ClearExpired();
return generator;
@@ -134,6 +136,29 @@ namespace NzbDrone.Core.Indexers.Cardigann
await generator.DoLogin();
}
public override async Task<byte[]> Download(Uri link)
{
var generator = (CardigannRequestGenerator)GetRequestGenerator();
var request = await generator.DownloadRequest(link);
request.AllowAutoRedirect = true;
var downloadBytes = Array.Empty<byte>();
try
{
var response = await _httpClient.ExecuteAsync(request);
downloadBytes = response.ResponseData;
}
catch (Exception)
{
_indexerStatusService.RecordFailure(Definition.Id);
_logger.Error("Download failed");
}
return downloadBytes;
}
protected override async Task Test(List<ValidationFailure> failures)
{
await base.Test(failures);

39
src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannBase.cs
View File

@@ -6,6 +6,7 @@ using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using AngleSharp.Dom;
using Microsoft.AspNetCore.WebUtilities;
using Newtonsoft.Json.Linq;
using NLog;
using NzbDrone.Common.Extensions;
@@ -706,6 +707,44 @@ namespace NzbDrone.Core.Indexers.Cardigann
return data;
}
protected Dictionary<string, string> ParseCustomHeaders(Dictionary<string, List<string>> customHeaders,
Dictionary<string, object> variables)
{
if (customHeaders == null)
{
return null;
}
// FIXME: fix jackett header handling (allow it to specifiy the same header multipe times)
var headers = new Dictionary<string, string>();
foreach (var header in customHeaders)
{
headers.Add(header.Key, ApplyGoTemplateText(header.Value[0], variables));
}
return headers;
}
protected IDictionary<string, object> AddTemplateVariablesFromUri(IDictionary<string, object> variables, Uri uri, string prefix = "")
{
variables[prefix + ".AbsoluteUri"] = uri.AbsoluteUri;
variables[prefix + ".AbsolutePath"] = uri.AbsolutePath;
variables[prefix + ".Scheme"] = uri.Scheme;
variables[prefix + ".Host"] = uri.Host;
variables[prefix + ".Port"] = uri.Port.ToString();
variables[prefix + ".PathAndQuery"] = uri.PathAndQuery;
variables[prefix + ".Query"] = uri.Query;
var queryString = QueryHelpers.ParseQuery(uri.Query);
foreach (var key in queryString.Keys)
{
//If we have supplied the same query string multiple time, just take the first.
variables[prefix + ".Query." + key] = queryString[key].First();
}
return variables;
}
protected Uri ResolvePath(string path, Uri currentUrl = null)
{
return new Uri(currentUrl ?? new Uri(SiteLink), path);

135
src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs
View File

@@ -607,7 +607,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
var request = new HttpRequestBuilder(captchaUrl.ToString())
.SetCookies(landingResult.GetCookies())
.SetHeader("Referrer", loginUrl.AbsoluteUri)
.SetHeader("Referer", loginUrl.AbsoluteUri)
.Build();
var response = await HttpClient.ExecuteAsync(request);
@@ -644,6 +644,139 @@ namespace NzbDrone.Core.Indexers.Cardigann
protected string GetRedirectDomainHint(HttpResponse result) => GetRedirectDomainHint(result.Request.Url.ToString(), result.Headers.GetSingleValue("Location"));
protected async Task<HttpResponse> HandleRequest(RequestBlock request, Dictionary<string, object> variables = null, string referer = null)
{
var requestLinkStr = ResolvePath(ApplyGoTemplateText(request.Path, variables)).ToString();
_logger.Debug($"CardigannIndexer ({_definition.Id}): handleRequest() requestLinkStr= {requestLinkStr}");
Dictionary<string, string> pairs = null;
var queryCollection = new NameValueCollection();
var method = HttpMethod.GET;
if (string.Equals(request.Method, "post", StringComparison.OrdinalIgnoreCase))
{
method = HttpMethod.POST;
pairs = new Dictionary<string, string>();
}
foreach (var input in request.Inputs)
{
var value = ApplyGoTemplateText(input.Value, variables);
if (method == HttpMethod.GET)
{
queryCollection.Add(input.Key, value);
}
else if (method == HttpMethod.POST)
{
pairs.Add(input.Key, value);
}
}
if (queryCollection.Count > 0)
{
if (!requestLinkStr.Contains("?"))
{
// TODO Need Encoding here if we add it back
requestLinkStr += "?" + queryCollection.GetQueryString(separator: request.Queryseparator).Substring(1);
}
else
{
requestLinkStr += queryCollection.GetQueryString(separator: request.Queryseparator);
}
}
var httpRequest = new HttpRequestBuilder(requestLinkStr)
.SetCookies(Cookies ?? new Dictionary<string, string>())
.SetHeaders(pairs ?? new Dictionary<string, string>())
.SetHeader("Referer", referer)
.Build();
httpRequest.Method = method;
var response = await HttpClient.ExecuteAsync(httpRequest);
_logger.Debug($"CardigannIndexer ({_definition.Id}): handleRequest() remote server returned {response.StatusCode.ToString()}");
return response;
}
public async Task<HttpRequest> DownloadRequest(Uri link)
{
Cookies = GetCookies();
var method = HttpMethod.GET;
if (_definition.Download != null)
{
var download = _definition.Download;
var variables = GetBaseTemplateVariables();
AddTemplateVariablesFromUri(variables, link, ".DownloadUri");
if (download.Before != null)
{
await HandleRequest(download.Before, variables, link.ToString());
}
if (download.Method == "post")
{
method = HttpMethod.POST;
}
if (download.Selector != null)
{
var selector = ApplyGoTemplateText(download.Selector, variables);
var headers = ParseCustomHeaders(_definition.Search?.Headers, variables);
var request = new HttpRequestBuilder(link.ToString())
.SetCookies(Cookies ?? new Dictionary<string, string>())
.SetHeaders(headers ?? new Dictionary<string, string>())
.Build();
request.AllowAutoRedirect = true;
var response = await HttpClient.ExecuteAsync(request);
var results = response.Content;
var searchResultParser = new HtmlParser();
var searchResultDocument = searchResultParser.ParseDocument(results);
var downloadElement = searchResultDocument.QuerySelector(selector);
if (downloadElement != null)
{
_logger.Debug(string.Format("CardigannIndexer ({0}): Download selector {1} matched:{2}", _definition.Id, selector, downloadElement.ToHtmlPretty()));
var href = "";
if (download.Attribute != null)
{
href = downloadElement.GetAttribute(download.Attribute);
if (href == null)
{
throw new Exception(string.Format("Attribute \"{0}\" is not set for element {1}", download.Attribute, downloadElement.ToHtmlPretty()));
}
}
else
{
href = downloadElement.TextContent;
}
href = ApplyFilters(href, download.Filters, variables);
link = ResolvePath(href, link);
}
else
{
_logger.Error(string.Format("CardigannIndexer ({0}): Download selector {1} didn't match:\n{2}", _definition.Id, download.Selector, results));
throw new Exception(string.Format("Download selector {0} didn't match", download.Selector));
}
}
}
var downloadRequest = new HttpRequestBuilder(link.AbsoluteUri)
.SetCookies(Cookies ?? new Dictionary<string, string>())
.Build();
downloadRequest.Method = method;
return downloadRequest;
}
public bool CheckIfLoginIsNeeded(HttpResponse response)
{
if (response.HasHttpRedirect)

4
src/NzbDrone.Core/Indexers/Definitions/Headphones/Headphones.cs
View File

@@ -50,9 +50,9 @@ namespace NzbDrone.Core.Indexers.Headphones
}
}
public override async Task<byte[]> Download(HttpUri link)
public override async Task<byte[]> Download(Uri link)
{
var requestBuilder = new HttpRequestBuilder(link.FullUri);
var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri);
var downloadBytes = Array.Empty<byte>();

4
src/NzbDrone.Core/Indexers/HttpIndexerBase.cs
View File

@@ -97,11 +97,11 @@ namespace NzbDrone.Core.Indexers
return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria));
}
public override async Task<byte[]> Download(HttpUri link)
public override async Task<byte[]> Download(Uri link)
{
Cookies = GetCookies();
var requestBuilder = new HttpRequestBuilder(link.FullUri);
var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri);
if (Cookies != null)
{

4
src/NzbDrone.Core/Indexers/IIndexer.cs
View File

@@ -1,5 +1,5 @@
using System;
using System.Threading.Tasks;
using NzbDrone.Common.Http;
using NzbDrone.Core.IndexerSearch.Definitions;
using NzbDrone.Core.ThingiProvider;
@@ -22,7 +22,7 @@ namespace NzbDrone.Core.Indexers
Task<IndexerPageableQueryResult> Fetch(BookSearchCriteria searchCriteria);
Task<IndexerPageableQueryResult> Fetch(BasicSearchCriteria searchCriteria);
Task<byte[]> Download(HttpUri link);
Task<byte[]> Download(Uri link);
IndexerCapabilities GetCapabilities();
}

2
src/NzbDrone.Core/Indexers/IndexerBase.cs
View File

@@ -74,7 +74,7 @@ namespace NzbDrone.Core.Indexers
public abstract Task<IndexerPageableQueryResult> Fetch(TvSearchCriteria searchCriteria);
public abstract Task<IndexerPageableQueryResult> Fetch(BookSearchCriteria searchCriteria);
public abstract Task<IndexerPageableQueryResult> Fetch(BasicSearchCriteria searchCriteria);
public abstract Task<byte[]> Download(HttpUri searchCriteria);
public abstract Task<byte[]> Download(Uri searchCriteria);
public abstract IndexerCapabilities GetCapabilities();