shpaq/Prowlarr-Prowlarr
1
0
Fork 0
mirror of https://github.com/Prowlarr/Prowlarr.git synced 2025-10-02 00:34:37 +02:00
Code Issues Packages Projects Releases Wiki Activity

New: Make indexer HTTP requests async

Browse Source
This commit is contained in:
ta264
2021-03-17 21:20:57 +00:00
committed by Qstick
parent 231d43e181
commit 29da0b7031
29 changed files with 216 additions and 213 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs
View File

@@ -4,7 +4,6 @@ using System.Linq;
using System.Threading.Tasks;
using NLog;
using NzbDrone.Common.Instrumentation.Extensions;
using NzbDrone.Common.TPL;
using NzbDrone.Core.Indexers;
using NzbDrone.Core.Indexers.Events;
using NzbDrone.Core.IndexerSearch.Definitions;
@@ -15,7 +14,7 @@ namespace NzbDrone.Core.IndexerSearch
{
public interface ISearchForNzb
{
NewznabResults Search(NewznabRequest request, List<int> indexerIds, bool interactiveSearch);
Task<NewznabResults> Search(NewznabRequest request, List<int> indexerIds, bool interactiveSearch);
}
public class NzbSearchService : ISearchForNzb
@@ -36,7 +35,7 @@ namespace NzbDrone.Core.IndexerSearch
_logger = logger;
}
public NewznabResults Search(NewznabRequest request, List<int> indexerIds, bool interactiveSearch)
public Task<NewznabResults> Search(NewznabRequest request, List<int> indexerIds, bool interactiveSearch)
{
switch (request.t)
{
@@ -53,7 +52,7 @@ namespace NzbDrone.Core.IndexerSearch
}
}
private NewznabResults MovieSearch(NewznabRequest request, List<int> indexerIds, bool interactiveSearch)
private async Task<NewznabResults> MovieSearch(NewznabRequest request, List<int> indexerIds, bool interactiveSearch)
{
var searchSpec = Get<MovieSearchCriteria>(request, indexerIds, interactiveSearch);
@@ -61,10 +60,10 @@ namespace NzbDrone.Core.IndexerSearch
searchSpec.TmdbId = request.tmdbid;
searchSpec.TraktId = request.traktid;
return new NewznabResults { Releases = MapReleases(Dispatch(indexer => indexer.Fetch(searchSpec), searchSpec), request.server) };
return new NewznabResults { Releases = MapReleases(await Dispatch(indexer => indexer.Fetch(searchSpec), searchSpec), request.server) };
}
private NewznabResults MusicSearch(NewznabRequest request, List<int> indexerIds, bool interactiveSearch)
private async Task<NewznabResults> MusicSearch(NewznabRequest request, List<int> indexerIds, bool interactiveSearch)
{
var searchSpec = Get<MusicSearchCriteria>(request, indexerIds, interactiveSearch);
@@ -72,10 +71,10 @@ namespace NzbDrone.Core.IndexerSearch
searchSpec.Album = request.album;
searchSpec.Label = request.label;
return new NewznabResults { Releases = MapReleases(Dispatch(indexer => indexer.Fetch(searchSpec), searchSpec), request.server) };
return new NewznabResults { Releases = MapReleases(await Dispatch(indexer => indexer.Fetch(searchSpec), searchSpec), request.server) };
}
private NewznabResults TvSearch(NewznabRequest request, List<int> indexerIds, bool interactiveSearch)
private async Task<NewznabResults> TvSearch(NewznabRequest request, List<int> indexerIds, bool interactiveSearch)
{
var searchSpec = Get<TvSearchCriteria>(request, indexerIds, interactiveSearch);
@@ -87,24 +86,24 @@ namespace NzbDrone.Core.IndexerSearch
searchSpec.RId = request.rid;
searchSpec.TvMazeId = request.tvmazeid;
return new NewznabResults { Releases = MapReleases(Dispatch(indexer => indexer.Fetch(searchSpec), searchSpec), request.server) };
return new NewznabResults { Releases = MapReleases(await Dispatch(indexer => indexer.Fetch(searchSpec), searchSpec), request.server) };
}
private NewznabResults BookSearch(NewznabRequest request, List<int> indexerIds, bool interactiveSearch)
private async Task<NewznabResults> BookSearch(NewznabRequest request, List<int> indexerIds, bool interactiveSearch)
{
var searchSpec = Get<BookSearchCriteria>(request, indexerIds, interactiveSearch);
searchSpec.Author = request.author;
searchSpec.Title = request.title;
return new NewznabResults { Releases = MapReleases(Dispatch(indexer => indexer.Fetch(searchSpec), searchSpec), request.server) };
return new NewznabResults { Releases = MapReleases(await Dispatch(indexer => indexer.Fetch(searchSpec), searchSpec), request.server) };
}
private NewznabResults BasicSearch(NewznabRequest request, List<int> indexerIds, bool interactiveSearch)
private async Task<NewznabResults> BasicSearch(NewznabRequest request, List<int> indexerIds, bool interactiveSearch)
{
var searchSpec = Get<BasicSearchCriteria>(request, indexerIds, interactiveSearch);
return new NewznabResults { Releases = MapReleases(Dispatch(indexer => indexer.Fetch(searchSpec), searchSpec), request.server) };
return new NewznabResults { Releases = MapReleases(await Dispatch(indexer => indexer.Fetch(searchSpec), searchSpec), request.server) };
}
private List<ReleaseInfo> MapReleases(List<ReleaseInfo> releases, string serverUrl)
@@ -145,7 +144,7 @@ namespace NzbDrone.Core.IndexerSearch
return spec;
}
private List<ReleaseInfo> Dispatch(Func<IIndexer, IndexerPageableQueryResult> searchAction, SearchCriteriaBase criteriaBase)
private async Task<List<ReleaseInfo>> Dispatch(Func<IIndexer, Task<IndexerPageableQueryResult>> searchAction, SearchCriteriaBase criteriaBase)
{
var indexers = _indexerFactory.GetAvailableProviders();
@@ -157,46 +156,39 @@ namespace NzbDrone.Core.IndexerSearch
.ToList();
}
var reports = new List<ReleaseInfo>();
_logger.ProgressInfo("Searching {0} indexers for {1}", indexers.Count, criteriaBase.SearchTerm);
var taskList = new List<Task>();
var taskFactory = new TaskFactory(TaskCreationOptions.LongRunning, TaskContinuationOptions.None);
var tasks = indexers.Select(x => DispatchIndexer(searchAction, x, criteriaBase));
foreach (var indexer in indexers)
{
var indexerLocal = indexer;
var batch = await Task.WhenAll(tasks);
taskList.Add(taskFactory.StartNew(() =>
{
try
{
var indexerReports = searchAction(indexerLocal);
lock (reports)
{
reports.AddRange(indexerReports.Releases);
}
foreach (var query in indexerReports.Queries)
{
_eventAggregator.PublishEvent(new IndexerQueryEvent(indexer.Definition.Id, criteriaBase, query.ElapsedTime, true, indexerReports.Releases.Count()));
}
}
catch (Exception e)
{
_eventAggregator.PublishEvent(new IndexerQueryEvent(indexer.Definition.Id, criteriaBase, 0, false));
_logger.Error(e, "Error while searching for {0}", criteriaBase);
}
}).LogExceptions());
}
Task.WaitAll(taskList.ToArray());
var reports = batch.SelectMany(x => x).ToList();
_logger.Debug("Total of {0} reports were found for {1} from {2} indexers", reports.Count, criteriaBase, indexers.Count);
return reports;
}
private async Task<IList<ReleaseInfo>> DispatchIndexer(Func<IIndexer, Task<IndexerPageableQueryResult>> searchAction, IIndexer indexer, SearchCriteriaBase criteriaBase)
{
try
{
var indexerReports = await searchAction(indexer);
foreach (var query in indexerReports.Queries)
{
_eventAggregator.PublishEvent(new IndexerQueryEvent(indexer.Definition.Id, criteriaBase, query.ElapsedTime, true, indexerReports.Releases.Count()));
}
return indexerReports.Releases;
}
catch (Exception e)
{
_eventAggregator.PublishEvent(new IndexerQueryEvent(indexer.Definition.Id, criteriaBase, 0, false));
_logger.Error(e, "Error while searching for {0}", criteriaBase);
}
return new List<ReleaseInfo>();
}
}
}

7
src/NzbDrone.Core/Indexers/Definitions/AnimeTorrents.cs
View File

@@ -4,6 +4,7 @@ using System.Collections.Specialized;
using System.Globalization;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using AngleSharp.Html.Parser;
using FluentValidation;
using NLog;
@@ -44,7 +45,7 @@ namespace NzbDrone.Core.Indexers.Definitions
return new AnimeTorrentsParser(Settings, Capabilities.Categories, BaseUrl);
}
protected override void DoLogin()
protected override async Task DoLogin()
{
UpdateCookies(null, null);
@@ -54,7 +55,7 @@ namespace NzbDrone.Core.Indexers.Definitions
AllowAutoRedirect = true
};
var loginPage = _httpClient.Execute(new HttpRequest(LoginUrl));
var loginPage = await _httpClient.ExecuteAsync(new HttpRequest(LoginUrl));
requestBuilder.Method = HttpMethod.POST;
requestBuilder.PostProcess += r => r.RequestTimeout = TimeSpan.FromSeconds(15);
requestBuilder.SetCookies(loginPage.GetCookies());
@@ -67,7 +68,7 @@ namespace NzbDrone.Core.Indexers.Definitions
.SetHeader("Content-Type", "multipart/form-data")
.Build();
var response = _httpClient.Execute(authLoginRequest);
var response = await _httpClient.ExecuteAsync(authLoginRequest);
if (response.Content != null && response.Content.Contains("logout.php"))
{

13
src/NzbDrone.Core/Indexers/Definitions/Avistaz/Avistaz.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.Net;
using System.Threading.Tasks;
using FluentValidation.Results;
using NLog;
using NzbDrone.Common.Http;
@@ -52,9 +53,9 @@ namespace NzbDrone.Core.Indexers.Definitions.Avistaz
return caps;
}
protected override void DoLogin()
protected override async Task DoLogin()
{
Settings.Token = GetToken();
Settings.Token = await GetToken();
if (Definition.Id > 0)
{
@@ -74,11 +75,11 @@ namespace NzbDrone.Core.Indexers.Definitions.Avistaz
return false;
}
protected override ValidationFailure TestConnection()
protected override async Task<ValidationFailure> TestConnection()
{
try
{
GetToken();
await GetToken();
}
catch (Exception ex)
{
@@ -90,7 +91,7 @@ namespace NzbDrone.Core.Indexers.Definitions.Avistaz
return null;
}
private string GetToken()
private async Task<string> GetToken()
{
var requestBuilder = new HttpRequestBuilder(LoginUrl)
{
@@ -108,7 +109,7 @@ namespace NzbDrone.Core.Indexers.Definitions.Avistaz
.Accept(HttpAccept.Json)
.Build();
var response = _httpClient.Post<AvistazAuthResponse>(authLoginRequest);
var response = await _httpClient.PostAsync<AvistazAuthResponse>(authLoginRequest);
var token = response.Resource.Token;
return token;

7
src/NzbDrone.Core/Indexers/Definitions/BakaBT.cs
View File

@@ -5,6 +5,7 @@ using System.Globalization;
using System.Linq;
using System.Net;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using AngleSharp.Dom;
using AngleSharp.Html.Parser;
using FluentValidation;
@@ -45,7 +46,7 @@ namespace NzbDrone.Core.Indexers.Definitions
return new BakaBTParser(Settings, Capabilities.Categories, BaseUrl);
}
protected override void DoLogin()
protected override async Task DoLogin()
{
UpdateCookies(null, null);
@@ -55,7 +56,7 @@ namespace NzbDrone.Core.Indexers.Definitions
AllowAutoRedirect = true
};
var loginPage = _httpClient.Execute(new HttpRequest(LoginUrl));
var loginPage = await _httpClient.ExecuteAsync(new HttpRequest(LoginUrl));
requestBuilder.Method = HttpMethod.POST;
requestBuilder.PostProcess += r => r.RequestTimeout = TimeSpan.FromSeconds(15);
@@ -77,7 +78,7 @@ namespace NzbDrone.Core.Indexers.Definitions
.SetHeader("Content-Type", "multipart/form-data")
.Build();
var response = _httpClient.Execute(authLoginRequest);
var response = await _httpClient.ExecuteAsync(authLoginRequest);
if (response.Content != null && response.Content.Contains("<a href=\"logout.php\">Logout</a>"))
{

9
src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs
View File

@@ -1,4 +1,5 @@
using System.Collections.Generic;
using System.Threading.Tasks;
using FluentValidation.Results;
using NLog;
using NzbDrone.Common.Http;
@@ -95,18 +96,18 @@ namespace NzbDrone.Core.Indexers.Cardigann
return generator.CheckIfLoginIsNeeded(httpResponse);
}
protected override void DoLogin()
protected override async Task DoLogin()
{
var generator = (CardigannRequestGenerator)GetRequestGenerator();
SetCookieFunctions(generator);
generator.DoLogin();
await generator.DoLogin();
}
protected override void Test(List<ValidationFailure> failures)
protected override async Task Test(List<ValidationFailure> failures)
{
base.Test(failures);
await base.Test(failures);
if (failures.HasErrors())
{
return;

25
src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs
View File

@@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Collections.Specialized;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using AngleSharp.Html.Dom;
using AngleSharp.Html.Parser;
using Newtonsoft.Json.Linq;
@@ -157,7 +158,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
return variables;
}
public void DoLogin()
public async Task DoLogin()
{
var login = _definition.Login;
@@ -190,7 +191,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
requestBuilder.Headers.Add("Referer", SiteLink);
var response = HttpClient.Execute(requestBuilder.Build());
var response = await HttpClient.ExecuteAsync(requestBuilder.Build());
Cookies = response.GetCookies();
@@ -214,7 +215,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
// landingResultDocument might not be initiated if the login is caused by a relogin during a query
if (landingResultDocument == null)
{
GetConfigurationForSetup(true);
await GetConfigurationForSetup(true);
}
var form = landingResultDocument.QuerySelector(formSelector);
@@ -327,7 +328,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
requestBuilder.Headers.Add("Referer", loginUrl);
var simpleCaptchaResult = HttpClient.Execute(requestBuilder.Build());
var simpleCaptchaResult = await HttpClient.ExecuteAsync(requestBuilder.Build());
var simpleCaptchaJSON = JObject.Parse(simpleCaptchaResult.Content);
var captchaSelection = simpleCaptchaJSON["images"][0]["hash"].ToString();
@@ -431,7 +432,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
var request = requestBuilder.Build();
request.SetContent(body);
loginResult = HttpClient.Execute(request);
loginResult = await HttpClient.ExecuteAsync(request);
}
else
{
@@ -451,7 +452,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
requestBuilder.AddFormParameter(pair.Key, pair.Value);
}
loginResult = HttpClient.Execute(requestBuilder.Build());
loginResult = await HttpClient.ExecuteAsync(requestBuilder.Build());
}
Cookies = loginResult.GetCookies();
@@ -486,7 +487,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
requestBuilder.Headers.Add("Referer", SiteLink);
var response = HttpClient.Execute(requestBuilder.Build());
var response = await HttpClient.ExecuteAsync(requestBuilder.Build());
Cookies = response.GetCookies();
@@ -510,7 +511,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
requestBuilder.Headers.Add("Referer", SiteLink);
var response = HttpClient.Execute(requestBuilder.Build());
var response = await HttpClient.ExecuteAsync(requestBuilder.Build());
Cookies = response.GetCookies();
@@ -556,7 +557,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
return true;
}
public void GetConfigurationForSetup(bool automaticlogin)
public async Task GetConfigurationForSetup(bool automaticlogin)
{
var login = _definition.Login;
@@ -587,7 +588,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
requestBuilder.SetCookies(Cookies);
}
landingResult = HttpClient.Execute(requestBuilder.Build());
landingResult = await HttpClient.ExecuteAsync(requestBuilder.Build());
Cookies = landingResult.GetCookies();
@@ -659,7 +660,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
return;
}
protected bool TestLogin()
protected async Task<bool> TestLogin()
{
var login = _definition.Login;
@@ -684,7 +685,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
requestBuilder.SetCookies(Cookies);
}
var testResult = HttpClient.Execute(requestBuilder.Build());
var testResult = await HttpClient.ExecuteAsync(requestBuilder.Build());
if (testResult.HasHttpRedirect)
{

5
src/NzbDrone.Core/Indexers/Definitions/Gazelle/Gazelle.cs
View File

@@ -1,4 +1,5 @@
using System;
using System.Threading.Tasks;
using NLog;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
@@ -47,7 +48,7 @@ namespace NzbDrone.Core.Indexers.Gazelle
return caps;
}
protected override void DoLogin()
protected override async Task DoLogin()
{
var requestBuilder = new HttpRequestBuilder(LoginUrl)
{
@@ -68,7 +69,7 @@ namespace NzbDrone.Core.Indexers.Gazelle
.Accept(HttpAccept.Json)
.Build();
var response = _httpClient.Execute(authLoginRequest);
var response = await _httpClient.ExecuteAsync(authLoginRequest);
cookies = response.GetCookies();

5
src/NzbDrone.Core/Indexers/Definitions/HDTorrents.cs
View File

@@ -4,6 +4,7 @@ using System.Collections.Specialized;
using System.Globalization;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using AngleSharp.Html.Parser;
using FluentValidation;
using NLog;
@@ -43,7 +44,7 @@ namespace NzbDrone.Core.Indexers.Definitions
return new HDTorrentsParser(Settings, Capabilities.Categories, BaseUrl);
}
protected override void DoLogin()
protected override async Task DoLogin()
{
var requestBuilder = new HttpRequestBuilder(LoginUrl)
{
@@ -62,7 +63,7 @@ namespace NzbDrone.Core.Indexers.Definitions
.SetHeader("Content-Type", "multipart/form-data")
.Build();
var response = _httpClient.Execute(authLoginRequest);
var response = await _httpClient.ExecuteAsync(authLoginRequest);
cookies = response.GetCookies();
UpdateCookies(cookies, DateTime.Now + TimeSpan.FromDays(30));

10
src/NzbDrone.Core/Indexers/Definitions/Headphones/Headphones.cs
View File

@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using FluentValidation.Results;
using NLog;
using NzbDrone.Common.Http;
@@ -38,9 +39,9 @@ namespace NzbDrone.Core.Indexers.Headphones
{
}
protected override void Test(List<ValidationFailure> failures)
protected override async Task Test(List<ValidationFailure> failures)
{
base.Test(failures);
await base.Test(failures);
if (failures.Any())
{
@@ -48,7 +49,7 @@ namespace NzbDrone.Core.Indexers.Headphones
}
}
public override byte[] Download(HttpUri link)
public override async Task<byte[]> Download(HttpUri link)
{
var requestBuilder = new HttpRequestBuilder(link.FullUri);
@@ -60,7 +61,8 @@ namespace NzbDrone.Core.Indexers.Headphones
try
{
downloadBytes = _httpClient.Execute(request).ResponseData;
var response = await _httpClient.ExecuteAsync(request);
downloadBytes = response.ResponseData;
}
catch (Exception)
{

5
src/NzbDrone.Core/Indexers/Definitions/Newznab/Newznab.cs
View File

@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using FluentValidation.Results;
using NLog;
using NzbDrone.Common.Extensions;
@@ -131,9 +132,9 @@ namespace NzbDrone.Core.Indexers.Newznab
return settings;
}
protected override void Test(List<ValidationFailure> failures)
protected override async Task Test(List<ValidationFailure> failures)
{
base.Test(failures);
await base.Test(failures);
if (failures.HasErrors())
{
return;

7
src/NzbDrone.Core/Indexers/Definitions/RevolutionTT.cs
View File

@@ -4,6 +4,7 @@ using System.Collections.Specialized;
using System.Globalization;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using AngleSharp.Html.Parser;
using FluentValidation;
using NLog;
@@ -45,7 +46,7 @@ namespace NzbDrone.Core.Indexers.Definitions
return new RevolutionTTParser(Settings, Capabilities.Categories, BaseUrl);
}
protected override void DoLogin()
protected override async Task DoLogin()
{
UpdateCookies(null, null);
@@ -55,7 +56,7 @@ namespace NzbDrone.Core.Indexers.Definitions
AllowAutoRedirect = true
};
var loginPage = _httpClient.Execute(new HttpRequest(BaseUrl + "login.php"));
var loginPage = await _httpClient.ExecuteAsync(new HttpRequest(BaseUrl + "login.php"));
requestBuilder.Method = HttpMethod.POST;
requestBuilder.PostProcess += r => r.RequestTimeout = TimeSpan.FromSeconds(15);
@@ -67,7 +68,7 @@ namespace NzbDrone.Core.Indexers.Definitions
.SetHeader("Content-Type", "multipart/form-data")
.Build();
var response = _httpClient.Execute(authLoginRequest);
var response = await _httpClient.ExecuteAsync(authLoginRequest);
if (response.Content != null && response.Content.Contains("/logout.php"))
{

5
src/NzbDrone.Core/Indexers/Definitions/TorrentLeech.cs
View File

@@ -4,6 +4,7 @@ using System.Globalization;
using System.Linq;
using System.Net;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using FluentValidation;
using Newtonsoft.Json;
using NLog;
@@ -44,7 +45,7 @@ namespace NzbDrone.Core.Indexers.Definitions
return new TorrentLeechParser(Settings, Capabilities.Categories, BaseUrl);
}
protected override void DoLogin()
protected override async Task DoLogin()
{
var requestBuilder = new HttpRequestBuilder(LoginUrl)
{
@@ -63,7 +64,7 @@ namespace NzbDrone.Core.Indexers.Definitions
.SetHeader("Content-Type", "multipart/form-data")
.Build();
var response = _httpClient.Execute(authLoginRequest);
var response = await _httpClient.ExecuteAsync(authLoginRequest);
cookies = response.GetCookies();
UpdateCookies(cookies, DateTime.Now + TimeSpan.FromDays(30));

7
src/NzbDrone.Core/Indexers/Definitions/TorrentSeeds.cs
View File

@@ -4,6 +4,7 @@ using System.Collections.Specialized;
using System.Linq;
using System.Net;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using AngleSharp.Html.Parser;
using FluentValidation;
using NLog;
@@ -44,14 +45,14 @@ namespace NzbDrone.Core.Indexers.Definitions
return new TorrentSeedsParser(Settings, Capabilities.Categories, BaseUrl);
}
protected override void DoLogin()
protected override async Task DoLogin()
{
var requestBuilder = new HttpRequestBuilder(LoginUrl)
{
LogResponseContent = true
};
var loginPage = _httpClient.Execute(new HttpRequest(TokenUrl));
var loginPage = await _httpClient.ExecuteAsync(new HttpRequest(TokenUrl));
var parser = new HtmlParser();
var dom = parser.ParseDocument(loginPage.Content);
var token = dom.QuerySelector("form.form-horizontal > span");
@@ -72,7 +73,7 @@ namespace NzbDrone.Core.Indexers.Definitions
.SetHeader("Content-Type", "multipart/form-data")
.Build();
var response = _httpClient.Execute(authLoginRequest);
var response = await _httpClient.ExecuteAsync(authLoginRequest);
cookies = response.GetCookies();
UpdateCookies(cookies, DateTime.Now + TimeSpan.FromDays(30));

5
src/NzbDrone.Core/Indexers/Definitions/Torznab/Torznab.cs
View File

@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using FluentValidation.Results;
using NLog;
using NzbDrone.Common.Extensions;
@@ -81,9 +82,9 @@ namespace NzbDrone.Core.Indexers.Torznab
return settings;
}
protected override void Test(List<ValidationFailure> failures)
protected override async Task Test(List<ValidationFailure> failures)
{
base.Test(failures);
await base.Test(failures);
if (failures.HasErrors())
{
return;

9
src/NzbDrone.Core/Indexers/DownloadService.cs
View File

@@ -1,4 +1,5 @@
using System;
using System.Threading.Tasks;
using NLog;
using NzbDrone.Common.EnsureThat;
using NzbDrone.Common.Extensions;
@@ -12,7 +13,7 @@ namespace NzbDrone.Core.Indexers
{
public interface IDownloadService
{
byte[] DownloadReport(string link, int indexerId, string source, string title);
Task<byte[]> DownloadReport(string link, int indexerId, string source, string title);
void RecordRedirect(string link, int indexerId, string source, string title);
}
@@ -37,14 +38,14 @@ namespace NzbDrone.Core.Indexers
_logger = logger;
}
public byte[] DownloadReport(string link, int indexerId, string source, string title)
public async Task<byte[]> DownloadReport(string link, int indexerId, string source, string title)
{
var url = new HttpUri(link);
// Limit grabs to 2 per second.
if (link.IsNotNullOrWhiteSpace() && !link.StartsWith("magnet:"))
{
_rateLimitService.WaitAndPulse(url.Host, TimeSpan.FromSeconds(2));
await _rateLimitService.WaitAndPulseAsync(url.Host, TimeSpan.FromSeconds(2));
}
var indexer = _indexerFactory.GetInstance(_indexerFactory.Get(indexerId));
@@ -53,7 +54,7 @@ namespace NzbDrone.Core.Indexers
try
{
downloadedBytes = indexer.Download(url);
downloadedBytes = await indexer.Download(url);
_indexerStatusService.RecordSuccess(indexerId);
success = true;
}

53
src/NzbDrone.Core/Indexers/HttpIndexerBase.cs
View File

@@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using FluentValidation.Results;
using NLog;
using NzbDrone.Common.Extensions;
@@ -42,57 +43,57 @@ namespace NzbDrone.Core.Indexers
_httpClient = httpClient;
}
public override IndexerPageableQueryResult Fetch(MovieSearchCriteria searchCriteria)
public override Task<IndexerPageableQueryResult> Fetch(MovieSearchCriteria searchCriteria)
{
//TODO: Re-Enable when All Indexer Caps are fixed and tests don't fail
//if (!SupportsSearch)
//{
// return new IndexerPageableQueryResult();
// return Task.FromResult(new Task<IndexerPageableQueryResult>());
//}
return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria));
}
public override IndexerPageableQueryResult Fetch(MusicSearchCriteria searchCriteria)
public override Task<IndexerPageableQueryResult> Fetch(MusicSearchCriteria searchCriteria)
{
if (!SupportsSearch)
{
return new IndexerPageableQueryResult();
return Task.FromResult(new IndexerPageableQueryResult());
}
return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria));
}
public override IndexerPageableQueryResult Fetch(TvSearchCriteria searchCriteria)
public override Task<IndexerPageableQueryResult> Fetch(TvSearchCriteria searchCriteria)
{
if (!SupportsSearch)
{
return new IndexerPageableQueryResult();
return Task.FromResult(new IndexerPageableQueryResult());
}
return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria));
}
public override IndexerPageableQueryResult Fetch(BookSearchCriteria searchCriteria)
public override Task<IndexerPageableQueryResult> Fetch(BookSearchCriteria searchCriteria)
{
if (!SupportsSearch)
{
return new IndexerPageableQueryResult();
return Task.FromResult(new IndexerPageableQueryResult());
}
return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria));
}
public override IndexerPageableQueryResult Fetch(BasicSearchCriteria searchCriteria)
public override Task<IndexerPageableQueryResult> Fetch(BasicSearchCriteria searchCriteria)
{
if (!SupportsSearch)
{
return new IndexerPageableQueryResult();
return Task.FromResult(new IndexerPageableQueryResult());
}
return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria));
}
public override byte[] Download(HttpUri link)
public override async Task<byte[]> Download(HttpUri link)
{
Cookies = GetCookies();
@@ -107,7 +108,8 @@ namespace NzbDrone.Core.Indexers
try
{
downloadBytes = _httpClient.Execute(requestBuilder.Build()).ResponseData;
var response = await _httpClient.ExecuteAsync(requestBuilder.Build());
downloadBytes = response.ResponseData;
}
catch (Exception)
{
@@ -159,7 +161,7 @@ namespace NzbDrone.Core.Indexers
_indexerStatusService.UpdateCookies(Definition.Id, cookies, expiration);
}
protected virtual IndexerPageableQueryResult FetchReleases(Func<IIndexerRequestGenerator, IndexerPageableRequestChain> pageableRequestChainSelector, bool isRecent = false)
protected virtual async Task<IndexerPageableQueryResult> FetchReleases(Func<IIndexerRequestGenerator, IndexerPageableRequestChain> pageableRequestChainSelector, bool isRecent = false)
{
var releases = new List<ReleaseInfo>();
var result = new IndexerPageableQueryResult();
@@ -195,7 +197,7 @@ namespace NzbDrone.Core.Indexers
{
url = request.Url.FullUri;
var page = FetchPage(request, parser);
var page = await FetchPage(request, parser);
result.Queries.Add(page);
@@ -360,9 +362,9 @@ namespace NzbDrone.Core.Indexers
return PageSize != 0 && page.Count >= PageSize;
}
protected virtual IndexerQueryResult FetchPage(IndexerRequest request, IParseIndexerResponse parser)
protected virtual async Task<IndexerQueryResult> FetchPage(IndexerRequest request, IParseIndexerResponse parser)
{
var response = FetchIndexerResponse(request);
var response = await FetchIndexerResponse(request);
try
{
@@ -398,11 +400,12 @@ namespace NzbDrone.Core.Indexers
return false;
}
protected virtual void DoLogin()
protected virtual Task DoLogin()
{
return Task.CompletedTask;
}
protected virtual IndexerResponse FetchIndexerResponse(IndexerRequest request)
protected virtual async Task<IndexerResponse> FetchIndexerResponse(IndexerRequest request)
{
_logger.Debug("Downloading Feed " + request.HttpRequest.ToString(false));
@@ -431,7 +434,7 @@ namespace NzbDrone.Core.Indexers
var stopWatch = Stopwatch.StartNew();
request.HttpRequest.SuppressHttpError = true;
var response = _httpClient.Execute(request.HttpRequest);
var response = await _httpClient.ExecuteAsync(request.HttpRequest);
stopWatch.Stop();
@@ -440,7 +443,7 @@ namespace NzbDrone.Core.Indexers
{
_logger.Trace("Attempting to re-auth based on indexer search response");
DoLogin();
await DoLogin();
request.HttpRequest.Cookies.Clear();
if (Cookies != null)
@@ -451,7 +454,7 @@ namespace NzbDrone.Core.Indexers
}
}
response = _httpClient.Execute(request.HttpRequest);
response = await _httpClient.ExecuteAsync(request.HttpRequest);
}
// Throw any other http error we get after attempting auth
@@ -474,12 +477,12 @@ namespace NzbDrone.Core.Indexers
return new IndexerResponse(request, response, stopWatch.ElapsedMilliseconds);
}
protected override void Test(List<ValidationFailure> failures)
protected override async Task Test(List<ValidationFailure> failures)
{
failures.AddIfNotNull(TestConnection());
failures.AddIfNotNull(await TestConnection());
}
protected virtual ValidationFailure TestConnection()
protected virtual async Task<ValidationFailure> TestConnection()
{
try
{
@@ -500,7 +503,7 @@ namespace NzbDrone.Core.Indexers
return new ValidationFailure(string.Empty, "No rss feed query available. This may be an issue with the indexer or your indexer category settings.");
}
var releases = FetchPage(firstRequest, parser);
var releases = await FetchPage(firstRequest, parser);
if (releases.Releases.Empty())
{

13
src/NzbDrone.Core/Indexers/IIndexer.cs
View File

@@ -1,3 +1,4 @@
using System.Threading.Tasks;
using NzbDrone.Common.Http;
using NzbDrone.Core.IndexerSearch.Definitions;
using NzbDrone.Core.ThingiProvider;
@@ -15,13 +16,13 @@ namespace NzbDrone.Core.Indexers
DownloadProtocol Protocol { get; }
IndexerPrivacy Privacy { get; }
IndexerPageableQueryResult Fetch(MovieSearchCriteria searchCriteria);
IndexerPageableQueryResult Fetch(MusicSearchCriteria searchCriteria);
IndexerPageableQueryResult Fetch(TvSearchCriteria searchCriteria);
IndexerPageableQueryResult Fetch(BookSearchCriteria searchCriteria);
IndexerPageableQueryResult Fetch(BasicSearchCriteria searchCriteria);
Task<IndexerPageableQueryResult> Fetch(MovieSearchCriteria searchCriteria);
Task<IndexerPageableQueryResult> Fetch(MusicSearchCriteria searchCriteria);
Task<IndexerPageableQueryResult> Fetch(TvSearchCriteria searchCriteria);
Task<IndexerPageableQueryResult> Fetch(BookSearchCriteria searchCriteria);
Task<IndexerPageableQueryResult> Fetch(BasicSearchCriteria searchCriteria);
byte[] Download(HttpUri link);
Task<byte[]> Download(HttpUri link);
IndexerCapabilities GetCapabilities();
}

17
src/NzbDrone.Core/Indexers/IndexerBase.cs
View File

@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using FluentValidation.Results;
using NLog;
using NzbDrone.Common.Extensions;
@@ -68,12 +69,12 @@ namespace NzbDrone.Core.Indexers
protected TSettings Settings => (TSettings)Definition.Settings;
public abstract IndexerPageableQueryResult Fetch(MovieSearchCriteria searchCriteria);
public abstract IndexerPageableQueryResult Fetch(MusicSearchCriteria searchCriteria);
public abstract IndexerPageableQueryResult Fetch(TvSearchCriteria searchCriteria);
public abstract IndexerPageableQueryResult Fetch(BookSearchCriteria searchCriteria);
public abstract IndexerPageableQueryResult Fetch(BasicSearchCriteria searchCriteria);
public abstract byte[] Download(HttpUri searchCriteria);
public abstract Task<IndexerPageableQueryResult> Fetch(MovieSearchCriteria searchCriteria);
public abstract Task<IndexerPageableQueryResult> Fetch(MusicSearchCriteria searchCriteria);
public abstract Task<IndexerPageableQueryResult> Fetch(TvSearchCriteria searchCriteria);
public abstract Task<IndexerPageableQueryResult> Fetch(BookSearchCriteria searchCriteria);
public abstract Task<IndexerPageableQueryResult> Fetch(BasicSearchCriteria searchCriteria);
public abstract Task<byte[]> Download(HttpUri searchCriteria);
public abstract IndexerCapabilities GetCapabilities();
@@ -98,7 +99,7 @@ namespace NzbDrone.Core.Indexers
try
{
Test(failures);
Test(failures).GetAwaiter().GetResult();
}
catch (Exception ex)
{
@@ -109,7 +110,7 @@ namespace NzbDrone.Core.Indexers
return new ValidationResult(failures);
}
protected abstract void Test(List<ValidationFailure> failures);
protected abstract Task Test(List<ValidationFailure> failures);
public override string ToString()
{