shpaq/Prowlarr-Prowlarr
1
0
Fork 0
mirror of https://github.com/Prowlarr/Prowlarr.git synced 2025-09-17 17:14:18 +02:00
Code Issues Packages Projects Releases Wiki Activity

Fixed: (Indexers) Add SupportsPagination to prevent fetching the first page multiple times

Browse Source
This commit is contained in:
Bogdan
2023-03-05 18:37:04 +02:00
parent d4c5e39c9c
commit 4348ebe187
22 changed files with 69 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/NzbDrone.Core/Datastore/TableMapping.cs
View File

@@ -55,6 +55,7 @@ namespace NzbDrone.Core.Datastore
.Ignore(i => i.SupportsRss)
.Ignore(i => i.SupportsSearch)
.Ignore(i => i.SupportsRedirect)
.Ignore(i => i.SupportsPagination)
.Ignore(i => i.Capabilities)
.HasOne(a => a.AppProfile, a => a.AppProfileId);

6
src/NzbDrone.Core/Indexers/Definitions/AnimeBytes.cs
View File

@@ -124,12 +124,6 @@ namespace NzbDrone.Core.Indexers.Definitions
{
var pageableRequests = new IndexerPageableRequestChain();
// TODO: Remove this once Prowlarr has proper support for non Pageable Indexers and can tell Sonarr that indexer doesn't support pagination in a proper way, for now just return empty release list on all request containing an offset
if (searchCriteria.Offset is > 0)
{
return pageableRequests;
}
pageableRequests.Add(GetRequest(searchType, searchCriteria.SanitizedSearchTerm, searchCriteria.Categories));
return pageableRequests;

1
src/NzbDrone.Core/Indexers/Definitions/BinSearch.cs
View File

@@ -29,6 +29,7 @@ namespace NzbDrone.Core.Indexers.Definitions
public override DownloadProtocol Protocol => DownloadProtocol.Usenet;
public override IndexerPrivacy Privacy => IndexerPrivacy.Public;
public override bool SupportsRss => false;
public override bool SupportsPagination => true;
public override IndexerCapabilities Capabilities => SetCapabilities();
public BinSearch(IIndexerHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, IValidateNzbs nzbValidationService, Logger logger)

1
src/NzbDrone.Core/Indexers/Definitions/BroadcastheNet/BroadcastheNet.cs
View File

@@ -15,6 +15,7 @@ namespace NzbDrone.Core.Indexers.BroadcastheNet
public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
public override bool SupportsRss => true;
public override bool SupportsSearch => true;
public override bool SupportsPagination => true;
public override int PageSize => 100;
public override IndexerCapabilities Capabilities => SetCapabilities();
public override TimeSpan RateLimit => TimeSpan.FromSeconds(5);

1
src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs
View File

@@ -165,6 +165,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
SupportsRss = SupportsRss,
SupportsSearch = SupportsSearch,
SupportsRedirect = SupportsRedirect,
SupportsPagination = SupportsPagination,
Capabilities = new IndexerCapabilities(),
ExtraFields = settings
};

9
src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs
View File

@@ -1022,15 +1022,6 @@ namespace NzbDrone.Core.Indexers.Cardigann
private IEnumerable<IndexerRequest> GetRequest(Dictionary<string, object> variables, SearchCriteriaBase searchCriteria)
{
var limit = searchCriteria.Limit ?? 100;
var offset = searchCriteria.Offset ?? 0;
if (offset > 0 && limit > 0 && offset / limit > 0)
{
// Pagination doesn't work yet, this is to prevent fetching the first page multiple times.
yield break;
}
var search = _definition.Search;
var mappedCategories = _categories.MapTorznabCapsToTrackers((int[])variables[".Query.Categories"]);

1
src/NzbDrone.Core/Indexers/Definitions/Headphones/Headphones.cs
View File

@@ -19,6 +19,7 @@ namespace NzbDrone.Core.Indexers.Headphones
public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
public override string[] IndexerUrls => new string[] { "https://indexer.codeshy.com" };
public override string Description => "A Private Usenet indexer for music";
public override bool SupportsPagination => true;
public override IndexerCapabilities Capabilities => SetCapabilities();
public override IIndexerRequestGenerator GetRequestGenerator()

19
src/NzbDrone.Core/Indexers/Definitions/IPTorrents.cs
View File

@@ -40,6 +40,7 @@ namespace NzbDrone.Core.Indexers.Definitions
public override string Description => "IPTorrents (IPT) is a Private Torrent Tracker for 0DAY / GENERAL.";
public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
public override bool SupportsPagination => true;
public override IndexerCapabilities Capabilities => SetCapabilities();
public IPTorrents(IIndexerHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger)
@@ -175,13 +176,13 @@ namespace NzbDrone.Core.Indexers.Definitions
public IPTorrentsSettings Settings { get; set; }
public IndexerCapabilities Capabilities { get; set; }
private IEnumerable<IndexerRequest> GetPagedRequests(string term, int[] categories, int limit, int offset, string imdbId = null)
private IEnumerable<IndexerRequest> GetPagedRequests(string term, SearchCriteriaBase searchCriteria, string imdbId = null)
{
var searchUrl = Settings.BaseUrl + "t";
var qc = new NameValueCollection();
foreach (var cat in Capabilities.Categories.MapTorznabCapsToTrackers(categories))
foreach (var cat in Capabilities.Categories.MapTorznabCapsToTrackers(searchCriteria.Categories))
{
qc.Add(cat, string.Empty);
}
@@ -204,9 +205,9 @@ namespace NzbDrone.Core.Indexers.Definitions
qc.Add("q", "+(" + term + ")");
}
if (offset > 0 && limit > 0)
if (searchCriteria.Limit is > 0 && searchCriteria.Offset is > 0)
{
var page = (int)(offset / limit) + 1;
var page = (int)(searchCriteria.Offset / searchCriteria.Limit) + 1;
qc.Add("p", page.ToString());
}
@@ -229,7 +230,7 @@ namespace NzbDrone.Core.Indexers.Definitions
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SearchTerm), searchCriteria.Categories, searchCriteria.Limit ?? 100, searchCriteria.Offset ?? 0, searchCriteria.FullImdbId));
pageableRequests.Add(GetPagedRequests($"{searchCriteria.SearchTerm}", searchCriteria, searchCriteria.FullImdbId));
return pageableRequests;
}
@@ -238,7 +239,7 @@ namespace NzbDrone.Core.Indexers.Definitions
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories, searchCriteria.Limit ?? 100, searchCriteria.Offset ?? 0));
pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}", searchCriteria));
return pageableRequests;
}
@@ -247,7 +248,7 @@ namespace NzbDrone.Core.Indexers.Definitions
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedTvSearchString), searchCriteria.Categories, searchCriteria.Limit ?? 100, searchCriteria.Offset ?? 0, searchCriteria.FullImdbId));
pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedTvSearchString}", searchCriteria, searchCriteria.FullImdbId));
return pageableRequests;
}
@@ -256,7 +257,7 @@ namespace NzbDrone.Core.Indexers.Definitions
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories, searchCriteria.Limit ?? 100, searchCriteria.Offset ?? 0));
pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}", searchCriteria));
return pageableRequests;
}
@@ -265,7 +266,7 @@ namespace NzbDrone.Core.Indexers.Definitions
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories, searchCriteria.Limit ?? 100, searchCriteria.Offset ?? 0));
pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}", searchCriteria));
return pageableRequests;
}

3
src/NzbDrone.Core/Indexers/Definitions/Libble.cs
View File

@@ -33,6 +33,7 @@ public class Libble : TorrentIndexerBase<LibbleSettings>
public override Encoding Encoding => Encoding.UTF8;
public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
public override bool SupportsPagination => true;
public override int PageSize => 50;
public override IndexerCapabilities Capabilities => SetCapabilities();
@@ -206,7 +207,7 @@ public class LibbleRequestGenerator : IIndexerRequestGenerator
queryCats.ForEach(cat => parameters.Set($"filter_cat[{cat}]", "1"));
}
if (searchCriteria.Offset.HasValue && searchCriteria.Limit.HasValue && searchCriteria.Offset > 0 && searchCriteria.Limit > 0)
if (searchCriteria.Limit is > 0 && searchCriteria.Offset is > 0)
{
var page = (int)(searchCriteria.Offset / searchCriteria.Limit) + 1;
parameters.Set("page", page.ToString());

1
src/NzbDrone.Core/Indexers/Definitions/MyAnonamouse.cs
View File

@@ -32,6 +32,7 @@ namespace NzbDrone.Core.Indexers.Definitions
public override string Description => "MyAnonaMouse (MAM) is a large ebook and audiobook tracker.";
public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
public override bool SupportsPagination => true;
public override int PageSize => 100;
public override IndexerCapabilities Capabilities => SetCapabilities();
private readonly ICacheManager _cacheManager;

1
src/NzbDrone.Core/Indexers/Definitions/Nebulance.cs
View File

@@ -29,6 +29,7 @@ namespace NzbDrone.Core.Indexers.Definitions
public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
public override bool SupportsRedirect => true;
public override bool SupportsPagination => true;
public override IndexerCapabilities Capabilities => SetCapabilities();
public Nebulance(IIndexerHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger)

2
src/NzbDrone.Core/Indexers/Definitions/Newznab/Newznab.cs
View File

@@ -23,6 +23,7 @@ namespace NzbDrone.Core.Indexers.Newznab
public override string Description => "Newznab is an API search specification for Usenet";
public override bool FollowRedirect => true;
public override bool SupportsRedirect => true;
public override bool SupportsPagination => true;
public override DownloadProtocol Protocol => DownloadProtocol.Usenet;
public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
@@ -135,6 +136,7 @@ namespace NzbDrone.Core.Indexers.Newznab
SupportsRss = SupportsRss,
SupportsSearch = SupportsSearch,
SupportsRedirect = SupportsRedirect,
SupportsPagination = SupportsPagination,
Capabilities = Capabilities
};
}

1
src/NzbDrone.Core/Indexers/Definitions/NzbIndex.cs
View File

@@ -25,6 +25,7 @@ namespace NzbDrone.Core.Indexers.Definitions
public override string Description => "A Usenet Indexer";
public override DownloadProtocol Protocol => DownloadProtocol.Usenet;
public override IndexerPrivacy Privacy => IndexerPrivacy.SemiPrivate;
public override bool SupportsPagination => true;
public override IndexerCapabilities Capabilities => SetCapabilities();
public NzbIndex(IIndexerHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger)

16
src/NzbDrone.Core/Indexers/Definitions/SpeedApp/SpeedAppBase.cs
View File

@@ -30,6 +30,7 @@ namespace NzbDrone.Core.Indexers.Definitions
private string LoginUrl => Settings.BaseUrl + "api/login";
public override Encoding Encoding => Encoding.UTF8;
public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
public override bool SupportsPagination => true;
public override int PageSize => 100;
public override IndexerCapabilities Capabilities => SetCapabilities();
protected virtual int MinimumSeedTime => 172800; // 48 hours
@@ -169,26 +170,23 @@ namespace NzbDrone.Core.Indexers.Definitions
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}", searchCriteria.Categories, searchCriteria.Limit ?? _pageSize, searchCriteria.Offset ?? 0, imdbId, season, episode));
pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}", searchCriteria, imdbId, season, episode));
return pageableRequests;
}
private IEnumerable<IndexerRequest> GetPagedRequests(string term, int[] categories, int limit, int offset, string imdbId = null, int? season = null, string episode = null)
private IEnumerable<IndexerRequest> GetPagedRequests(string term, SearchCriteriaBase searchCriteria, string imdbId = null, int? season = null, string episode = null)
{
limit = Math.Min(_pageSize, limit);
offset = Math.Max(0, offset);
var parameters = new NameValueCollection
{
{ "itemsPerPage", limit.ToString() },
{ "itemsPerPage", Math.Min(_pageSize, searchCriteria.Limit.GetValueOrDefault(_pageSize)).ToString() },
{ "sort", "torrent.createdAt" },
{ "direction", "desc" }
};
if (limit > 0 && offset > 0)
if (searchCriteria.Limit is > 0 && searchCriteria.Offset is > 0)
{
var page = (offset / limit) + 1;
var page = (int)(searchCriteria.Offset / searchCriteria.Limit) + 1;
parameters.Set("page", page.ToString());
}
@@ -211,7 +209,7 @@ namespace NzbDrone.Core.Indexers.Definitions
parameters.Set("episode", episode);
}
var cats = _capabilities.Categories.MapTorznabCapsToTrackers(categories);
var cats = _capabilities.Categories.MapTorznabCapsToTrackers(searchCriteria.Categories);
if (cats.Count > 0)
{
foreach (var cat in cats)

3
src/NzbDrone.Core/Indexers/Definitions/TorrentPotato/TorrentPotato.cs
View File

@@ -32,7 +32,8 @@ namespace NzbDrone.Core.Indexers.TorrentPotato
Protocol = DownloadProtocol.Torrent,
SupportsRss = SupportsRss,
SupportsSearch = SupportsSearch,
SupportsRedirect = SupportsRedirect
SupportsRedirect = SupportsRedirect,
SupportsPagination = SupportsPagination
};
}

1
src/NzbDrone.Core/Indexers/Definitions/Torznab/Torznab.cs
View File

@@ -113,6 +113,7 @@ namespace NzbDrone.Core.Indexers.Torznab
SupportsRss = SupportsRss,
SupportsSearch = SupportsSearch,
SupportsRedirect = SupportsRedirect,
SupportsPagination = SupportsPagination,
Capabilities = Capabilities
};
}

26
src/NzbDrone.Core/Indexers/HttpIndexerBase.cs
View File

@@ -33,6 +33,7 @@ namespace NzbDrone.Core.Indexers
public override bool SupportsRss => true;
public override bool SupportsSearch => true;
public override bool SupportsRedirect => false;
public override bool SupportsPagination => false;
public override Encoding Encoding => Encoding.UTF8;
public override string Language => "en-US";
@@ -60,6 +61,11 @@ namespace NzbDrone.Core.Indexers
return Task.FromResult(new IndexerPageableQueryResult());
}
if (!SupportsPagination && searchCriteria.Offset is > 0)
{
return Task.FromResult(new IndexerPageableQueryResult());
}
return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria), searchCriteria);
}
@@ -70,6 +76,11 @@ namespace NzbDrone.Core.Indexers
return Task.FromResult(new IndexerPageableQueryResult());
}
if (!SupportsPagination && searchCriteria.Offset is > 0)
{
return Task.FromResult(new IndexerPageableQueryResult());
}
return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria), searchCriteria);
}
@@ -80,6 +91,11 @@ namespace NzbDrone.Core.Indexers
return Task.FromResult(new IndexerPageableQueryResult());
}
if (!SupportsPagination && searchCriteria.Offset is > 0)
{
return Task.FromResult(new IndexerPageableQueryResult());
}
return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria), searchCriteria);
}
@@ -90,6 +106,11 @@ namespace NzbDrone.Core.Indexers
return Task.FromResult(new IndexerPageableQueryResult());
}
if (!SupportsPagination && searchCriteria.Offset is > 0)
{
return Task.FromResult(new IndexerPageableQueryResult());
}
return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria), searchCriteria);
}
@@ -100,6 +121,11 @@ namespace NzbDrone.Core.Indexers
return Task.FromResult(new IndexerPageableQueryResult());
}
if (!SupportsPagination && searchCriteria.Offset is > 0)
{
return Task.FromResult(new IndexerPageableQueryResult());
}
return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria), searchCriteria);
}

1
src/NzbDrone.Core/Indexers/IIndexer.cs
View File

@@ -11,6 +11,7 @@ namespace NzbDrone.Core.Indexers
bool SupportsRss { get; }
bool SupportsSearch { get; }
bool SupportsRedirect { get; }
bool SupportsPagination { get; }
IndexerCapabilities Capabilities { get; }
string[] IndexerUrls { get; }

8
src/NzbDrone.Core/Indexers/IndexerBase.cs
View File

@@ -36,6 +36,7 @@ namespace NzbDrone.Core.Indexers
public abstract bool SupportsRss { get; }
public abstract bool SupportsSearch { get; }
public abstract bool SupportsRedirect { get; }
public abstract bool SupportsPagination { get; }
public abstract IndexerCapabilities Capabilities { get; protected set; }
public IndexerBase(IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger)
@@ -51,12 +52,7 @@ namespace NzbDrone.Core.Indexers
{
var attributes = GetType().GetCustomAttributes(false);
foreach (ObsoleteAttribute attribute in attributes.OfType<ObsoleteAttribute>())
{
return true;
}
return false;
return attributes.OfType<ObsoleteAttribute>().Any();
}
public virtual ProviderMessage Message => null;

1
src/NzbDrone.Core/Indexers/IndexerDefinition.cs
View File

@@ -20,6 +20,7 @@ namespace NzbDrone.Core.Indexers
public bool SupportsRss { get; set; }
public bool SupportsSearch { get; set; }
public bool SupportsRedirect { get; set; }
public bool SupportsPagination { get; set; }
public IndexerCapabilities Capabilities { get; set; }
public int Priority { get; set; } = 25;
public bool Redirect { get; set; }

3
src/NzbDrone.Core/Indexers/IndexerFactory.cs
View File

@@ -201,9 +201,10 @@ namespace NzbDrone.Core.Indexers
definition.SupportsRss = provider.SupportsRss;
definition.SupportsSearch = provider.SupportsSearch;
definition.SupportsRedirect = provider.SupportsRedirect;
definition.SupportsPagination = provider.SupportsPagination;
//We want to use the definition Caps and Privacy for Cardigann instead of the provider.
if (definition.Implementation != typeof(Cardigann.Cardigann).Name)
if (definition.Implementation != nameof(Cardigann.Cardigann))
{
definition.IndexerUrls = provider.IndexerUrls;
definition.LegacyUrls = provider.LegacyUrls;

8
src/Prowlarr.Api.V1/Indexers/IndexerResource.cs
View File

@@ -25,6 +25,7 @@ namespace Prowlarr.Api.V1.Indexers
public bool SupportsRss { get; set; }
public bool SupportsSearch { get; set; }
public bool SupportsRedirect { get; set; }
public bool SupportsPagination { get; set; }
public int AppProfileId { get; set; }
public DownloadProtocol Protocol { get; set; }
public IndexerPrivacy Privacy { get; set; }
@@ -57,9 +58,9 @@ namespace Prowlarr.Api.V1.Indexers
var infoLinkName = definition.ImplementationName;
if (definition.Implementation == typeof(Cardigann).Name)
if (definition.Implementation == nameof(Cardigann))
{
var extraFields = definition.ExtraFields?.Select((x, i) => MapField(x, i)).ToList() ?? new List<Field>();
var extraFields = definition.ExtraFields?.Select(MapField).ToList() ?? new List<Field>();
resource.Fields.AddRange(extraFields);
@@ -79,7 +80,7 @@ namespace Prowlarr.Api.V1.Indexers
infoLinkName = settings.DefinitionFile;
}
resource.InfoLink = string.Format("https://wiki.servarr.com/prowlarr/supported-indexers#{0}", infoLinkName.ToLower().Replace(' ', '-'));
resource.InfoLink = $"https://wiki.servarr.com/prowlarr/supported-indexers#{infoLinkName.ToLower().Replace(' ', '-')}";
resource.AppProfileId = definition.AppProfileId;
resource.IndexerUrls = definition.IndexerUrls;
resource.LegacyUrls = definition.LegacyUrls;
@@ -91,6 +92,7 @@ namespace Prowlarr.Api.V1.Indexers
resource.SupportsRss = definition.SupportsRss;
resource.SupportsSearch = definition.SupportsSearch;
resource.SupportsRedirect = definition.SupportsRedirect;
resource.SupportsPagination = definition.SupportsPagination;
resource.Capabilities = definition.Capabilities.ToResource();
resource.Protocol = definition.Protocol;
resource.Privacy = definition.Privacy;