shpaq/Jackett-Jackett
1
0
Fork 0
mirror of https://github.com/Jackett/Jackett.git synced 2025-09-17 17:34:09 +02:00
Code Issues Packages Projects Releases Wiki Activity

core: redo search cache from scratch. resolves #10382 (#10447)

Browse Source 
In simple words, when you make a request in Jackett, the results are saved in memory (cache). The next request will return results form the cache improving response time and making fewer requests to the sites.
* We assume all indexers/sites are stateless, the same request return the same response. If you change the search term, categories or something in the query Jackett has to make a live request to the indexer.
* There are some situations when we don't want to use the cache:
** When we are testing the indexers => if query.IsTest results are not cached
** When the user updates the configuration of one indexer => We call CleanIndexerCache to remove cached results before testing the configuration
** When there is some error/exception in the indexer => The results are not cached so we can retry in the next request
* We want to limit the memory usage, so we try to remove elements from cache ASAP:
** Each indexer can have a maximum number of results in memory. If the limit is exceeded we remove old results
** Cached results expire after some time
* Users can configure the cache or even disable it
This commit is contained in:
Diego Heras
2020-12-11 23:14:21 +01:00
committed by GitHub
parent ec3787e803
commit 69125add3e
126 changed files with 744 additions and 364 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
src/Jackett.Common/Indexers/BaseIndexer.cs
View File

@@ -36,6 +36,7 @@ namespace Jackett.Common.Indexers
protected Logger logger;
protected IIndexerConfigurationService configurationService;
protected IProtectionService protectionService;
protected ICacheService cacheService;
protected ConfigurationData configData;
@@ -62,11 +63,12 @@ namespace Jackett.Common.Indexers
// standard constructor used by most indexers
public BaseIndexer(string link, string id, string name, string description,
IIndexerConfigurationService configService, Logger logger, ConfigurationData configData,
IProtectionService p)
IProtectionService p, ICacheService cs)
{
this.logger = logger;
configurationService = configService;
protectionService = p;
cacheService = cs;
if (!link.EndsWith("/", StringComparison.Ordinal))
throw new Exception("Site link must end with a slash.");
@@ -378,14 +380,19 @@ namespace Jackett.Common.Indexers
public virtual async Task<IndexerResult> ResultsForQuery(TorznabQuery query, bool isMetaIndexer)
{
if (!CanHandleQuery(query) || !CanHandleCategories(query, isMetaIndexer))
return new IndexerResult(this, new ReleaseInfo[0]);
return new IndexerResult(this, new ReleaseInfo[0], false);
var cachedReleases = cacheService.Search(this, query);
if (cachedReleases != null)
return new IndexerResult(this, cachedReleases, true);
try
{
var results = await PerformQuery(query);
results = FilterResults(query, results);
results = FixResults(query, results);
return new IndexerResult(this, results);
cacheService.CacheResults(this, query, results.ToList());
return new IndexerResult(this, results, false);
}
catch (Exception ex)
{
@@ -400,9 +407,9 @@ namespace Jackett.Common.Indexers
{
protected BaseWebIndexer(string link, string id, string name, string description,
IIndexerConfigurationService configService, WebClient client, Logger logger,
ConfigurationData configData, IProtectionService p, TorznabCapabilities caps,
string downloadBase = null)
: base(link, id, name, description, configService, logger, configData, p)
ConfigurationData configData, IProtectionService p, ICacheService cacheService,
TorznabCapabilities caps, string downloadBase = null)
: base(link, id, name, description, configService, logger, configData, p, cacheService)
{
webclient = client;
downloadUrlBase = downloadBase;
@@ -410,8 +417,9 @@ namespace Jackett.Common.Indexers
}
// minimal constructor used by e.g. cardigann generic indexer
protected BaseWebIndexer(IIndexerConfigurationService configService, WebClient client, Logger logger, IProtectionService p)
: base("/", "", "", "", configService, logger, null, p) => webclient = client;
protected BaseWebIndexer(IIndexerConfigurationService configService, WebClient client, Logger logger,
IProtectionService p, ICacheService cacheService)
: base("/", "", "", "", configService, logger, null, p, cacheService) => webclient = client;
public virtual async Task<byte[]> Download(Uri link)
{
@@ -658,7 +666,6 @@ namespace Jackett.Common.Indexers
{
var result = await base.ResultsForQuery(query, isMetaIndexer);
result.Releases = CleanLinks(result.Releases);
return result;
}
@@ -696,9 +703,9 @@ namespace Jackett.Common.Indexers
{
protected BaseCachingWebIndexer(string link,string id, string name, string description,
IIndexerConfigurationService configService, WebClient client, Logger logger,
ConfigurationData configData, IProtectionService p, TorznabCapabilities caps = null,
string downloadBase = null)
: base(link, id, name, description, configService, client, logger, configData, p, caps, downloadBase)
ConfigurationData configData, IProtectionService p, ICacheService cacheService,
TorznabCapabilities caps = null, string downloadBase = null)
: base(link, id, name, description, configService, client, logger, configData, p, cacheService, caps, downloadBase)
{
}
@@ -710,6 +717,7 @@ namespace Jackett.Common.Indexers
}
}
// TODO: remove this implementation and use gloal cache
protected static List<CachedQueryResult> cache = new List<CachedQueryResult>();
protected static readonly TimeSpan cacheTime = new TimeSpan(0, 9, 0);
}