shpaq/Jackett-Jackett
1
0
Fork 0
mirror of https://github.com/Jackett/Jackett.git synced 2025-12-25 15:15:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

indexers: add SupportsPagination to prevent fetching the first page multiple times (#14122)

Browse Source
This commit is contained in:
Bogdan
2023-03-05 14:30:40 +02:00
committed by GitHub
parent dcaa587853
commit 8729652e86
11 changed files with 71 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/Jackett.Common/Indexers/Abstract/AvistazTracker.cs
View File

@@ -20,6 +20,8 @@ namespace Jackett.Common.Indexers.Abstract
[ExcludeFromCodeCoverage]
public abstract class AvistazTracker : BaseWebIndexer
{
public override bool SupportsPagination => true;
private readonly Dictionary<string, string> AuthHeaders = new Dictionary<string, string>
{
{"Accept", "application/json"},
@@ -39,12 +41,20 @@ namespace Jackett.Common.Indexers.Abstract
protected virtual List<KeyValuePair<string, string>> GetSearchQueryParameters(TorznabQuery query)
{
var categoryMapping = MapTorznabCapsToTrackers(query).Distinct().ToList();
var qc = new List<KeyValuePair<string, string>> // NameValueCollection don't support cat[]=19&cat[]=6
{
{"in", "1"},
{"type", categoryMapping.Any() ? categoryMapping.First() : "0"}
{ "in", "1" },
{ "type", categoryMapping.FirstIfSingleOrDefault("0") },
{ "limit", "50" }
};
if (query.Limit > 0 && query.Offset > 0)
{
var page = query.Offset / query.Limit + 1;
qc.Add("page", page.ToString());
}
// resolution filter to improve the search
if (!query.Categories.Contains(TorznabCatType.Movies.ID) && !query.Categories.Contains(TorznabCatType.TV.ID) &&
!query.Categories.Contains(TorznabCatType.Audio.ID))

14
src/Jackett.Common/Indexers/Abstract/SpeedAppTracker.cs
View File

@@ -20,6 +20,8 @@ namespace Jackett.Common.Indexers.Abstract
[ExcludeFromCodeCoverage]
public abstract class SpeedAppTracker : BaseWebIndexer
{
public override bool SupportsPagination => true;
protected virtual bool UseP2PReleaseName => false;
protected virtual int minimumSeedTime => 172800; // 48h
@@ -90,11 +92,17 @@ namespace Jackett.Common.Indexers.Abstract
//var categoryMapping = MapTorznabCapsToTrackers(query).Distinct().ToList();
var qc = new List<KeyValuePair<string, string>> // NameValueCollection don't support cat[]=19&cat[]=6
{
{"itemsPerPage", "100"},
{"sort", "torrent.createdAt"},
{"direction", "desc"}
{ "itemsPerPage", "100" },
{ "sort", "torrent.createdAt" },
{ "direction", "desc" }
};
if (query.Limit > 0 && query.Offset > 0)
{
var page = query.Offset / query.Limit + 1;
qc.Add("page", page.ToString());
}
foreach (var cat in MapTorznabCapsToTrackers(query))
qc.Add("categories[]", cat);

6
src/Jackett.Common/Indexers/BaseIndexer.cs
View File

@@ -29,6 +29,7 @@ namespace Jackett.Common.Indexers
public string Language { get; protected set; }
public string Type { get; protected set; }
public virtual bool SupportsPagination => false;
[JsonConverter(typeof(EncodingJsonConverter))]
public Encoding Encoding { get; protected set; }
@@ -297,7 +298,10 @@ namespace Jackett.Common.Indexers
var queryCopy = query.Clone();
if (!CanHandleQuery(queryCopy) || !CanHandleCategories(queryCopy, isMetaIndexer))
return new IndexerResult(this, new ReleaseInfo[0], false);
return new IndexerResult(this, Array.Empty<ReleaseInfo>(), false);
if (!SupportsPagination && queryCopy.Offset > 0)
return new IndexerResult(this, Array.Empty<ReleaseInfo>(), false);
if (queryCopy.Cache)
{

5
src/Jackett.Common/Indexers/BroadcasTheNet.cs
View File

@@ -19,6 +19,8 @@ namespace Jackett.Common.Indexers
[ExcludeFromCodeCoverage]
public class BroadcasTheNet : BaseWebIndexer
{
public override bool SupportsPagination => true;
// based on https://github.com/Prowlarr/Prowlarr/tree/develop/src/NzbDrone.Core/Indexers/Definitions/BroadcastheNet
private readonly string APIBASE = "https://api.broadcasthe.net";
@@ -71,8 +73,9 @@ namespace Jackett.Common.Indexers
try
{
var results = await PerformQuery(new TorznabQuery());
if (results.Count() == 0)
if (!results.Any())
throw new Exception("Testing returned no results!");
IsConfigured = true;
SaveConfig();
}

2
src/Jackett.Common/Indexers/IIndexer.cs
View File

@@ -36,6 +36,8 @@ namespace Jackett.Common.Indexers
string Id { get; }
Encoding Encoding { get; }
bool SupportsPagination { get; }
TorznabCapabilities TorznabCaps { get; }
// Whether this indexer has been configured, verified and saved in the past and has the settings required for functioning

8
src/Jackett.Common/Indexers/IPTorrents.cs
View File

@@ -21,6 +21,8 @@ namespace Jackett.Common.Indexers
[ExcludeFromCodeCoverage]
public class IPTorrents : BaseWebIndexer
{
public override bool SupportsPagination => true;
private string SearchUrl => SiteLink + "t";
public override string[] AlternativeSiteLinks { get; protected set; } = {
@@ -271,6 +273,12 @@ namespace Jackett.Common.Indexers
qc.Set("o", ((SingleSelectConfigurationItem)configData.GetDynamic("sort")).Value);
if (query.Limit > 0 && query.Offset > 0)
{
var page = query.Offset / query.Limit + 1;
qc.Add("p", page.ToString());
}
var searchUrl = SearchUrl + "?" + qc.GetQueryString();
var response = await RequestWithCookiesAndRetryAsync(searchUrl, referer: SearchUrl, headers: headers);
var results = response.ContentString;

17
src/Jackett.Common/Indexers/Libble.cs
View File

@@ -8,6 +8,7 @@ using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using AngleSharp.Html.Parser;
using Jackett.Common.Extensions;
using Jackett.Common.Models;
using Jackett.Common.Models.IndexerConfig;
using Jackett.Common.Services.Interfaces;
@@ -21,6 +22,8 @@ namespace Jackett.Common.Indexers
[ExcludeFromCodeCoverage]
public class Libble : BaseWebIndexer
{
public override bool SupportsPagination => true;
private string LandingUrl => SiteLink + "login.php";
private string LoginUrl => SiteLink + "login.php";
private string SearchUrl => SiteLink + "torrents.php";
@@ -141,16 +144,16 @@ namespace Jackett.Common.Indexers
foreach (var cat in MapTorznabCapsToTrackers(query))
queryCollection.Set($"filter_cat[{cat}]", "1");
if (query.Artist != null)
if (query.Artist.IsNotNullOrWhiteSpace() && query.Artist != "VA")
queryCollection.Set("artistname", query.Artist);
if (query.Label != null)
if (query.Label.IsNotNullOrWhiteSpace())
queryCollection.Set("recordlabel", query.Label);
if (query.Year != null)
if (query.Year.HasValue)
queryCollection.Set("year", query.Year.ToString());
if (query.Album != null)
if (query.Album.IsNotNullOrWhiteSpace())
queryCollection.Set("groupname", query.Album);
if (query.IsGenreQuery)
@@ -159,6 +162,12 @@ namespace Jackett.Common.Indexers
queryCollection.Set("tags_type", "0");
}
if (query.Limit > 0 && query.Offset > 0)
{
var page = query.Offset / query.Limit + 1;
queryCollection.Add("page", page.ToString());
}
searchUrl += "?" + queryCollection.GetQueryString();
var searchPage = await RequestWithCookiesAndRetryAsync(searchUrl);

2
src/Jackett.Common/Indexers/MyAnonamouse.cs
View File

@@ -20,6 +20,8 @@ namespace Jackett.Common.Indexers
[ExcludeFromCodeCoverage]
public class MyAnonamouse : BaseWebIndexer
{
public override bool SupportsPagination => true;
private string SearchUrl => SiteLink + "tor/js/loadSearchJSONbasic.php";
private new ConfigurationDataMyAnonamouse configData => (ConfigurationDataMyAnonamouse)base.configData;

2
src/Jackett.Common/Indexers/NebulanceAPI.cs
View File

@@ -19,6 +19,8 @@ namespace Jackett.Common.Indexers
[ExcludeFromCodeCoverage]
public class NebulanceAPI : BaseWebIndexer
{
public override bool SupportsPagination => true;
// Docs at https://nebulance.io/articles.php?topic=api_key
protected virtual string APIUrl => SiteLink + "api.php";
protected virtual int KeyLength => 32;

10
src/Jackett.Common/Indexers/SpeedCD.cs
View File

@@ -21,6 +21,8 @@ namespace Jackett.Common.Indexers
[ExcludeFromCodeCoverage]
public class SpeedCD : BaseWebIndexer
{
public override bool SupportsPagination => true;
private string LoginUrl1 => SiteLink + "checkpoint/API";
private string LoginUrl2 => SiteLink + "checkpoint/";
private string SearchUrl => SiteLink + "browse/";
@@ -192,6 +194,14 @@ namespace Jackett.Common.Indexers
qc.Add(WebUtilityHelpers.UrlEncode(term.Trim(), Encoding));
}
if (query.Limit > 0 && query.Offset > 0)
{
var page = query.Offset / query.Limit + 1;
qc.Add("p");
qc.Add(page.ToString());
}
var searchUrl = SearchUrl + string.Join("/", qc);
var response = await RequestWithCookiesAndRetryAsync(searchUrl);

2
src/Jackett.Test/Common/Utils/FilterFuncs/IndexerBaseStub.cs
View File

@@ -22,6 +22,8 @@ namespace Jackett.Test.Common.Utils.FilterFuncs
public virtual string Language => throw TestExceptions.UnexpectedInvocation;
public virtual bool SupportsPagination => false;
public virtual string LastError
{
get => throw TestExceptions.UnexpectedInvocation;