shpaq/Jackett-Jackett
1
0
Fork 0
mirror of https://github.com/Jackett/Jackett.git synced 2025-09-29 21:49:13 +02:00
Code Issues Packages Projects Releases Wiki Activity

core: clean up indexer manager code (#13867)

Browse Source
This commit is contained in:
Diego Heras
2023-01-07 17:18:24 +01:00
committed by GitHub
parent e7384b73e4
commit 034ca38368
3 changed files with 82 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

157
src/Jackett.Common/Services/IndexerManagerService.cs
View File

@@ -9,7 +9,6 @@ using Jackett.Common.Indexers.Meta;
using Jackett.Common.Models; using Jackett.Common.Models;
using Jackett.Common.Models.Config; using Jackett.Common.Models.Config;
using Jackett.Common.Services.Interfaces; using Jackett.Common.Services.Interfaces;
using Jackett.Common.Utils;
using Jackett.Common.Utils.Clients; using Jackett.Common.Utils.Clients;
using NLog; using NLog;
using YamlDotNet.Serialization; using YamlDotNet.Serialization;
@@ -21,24 +20,24 @@ namespace Jackett.Common.Services
public class IndexerManagerService : IIndexerManagerService public class IndexerManagerService : IIndexerManagerService
{ {
private readonly ICacheService cacheService; private readonly ICacheService _cacheService;
private readonly IIndexerConfigurationService configService; private readonly IIndexerConfigurationService _configService;
private readonly IProtectionService protectionService; private readonly IProtectionService _protectionService;
private readonly WebClient webClient; private readonly WebClient _webClient;
private readonly IProcessService processService; private readonly IProcessService _processService;
private readonly IConfigurationService globalConfigService; private readonly IConfigurationService _globalConfigService;
private readonly ServerConfig serverConfig; private readonly ServerConfig _serverConfig;
private readonly Logger logger; private readonly Logger _logger;
private readonly Dictionary<string, IIndexer> indexers = new Dictionary<string, IIndexer>(); private readonly Dictionary<string, IIndexer> _indexers = new Dictionary<string, IIndexer>();
private AggregateIndexer aggregateIndexer; private AggregateIndexer _aggregateIndexer;
private ConcurrentDictionary<string, IWebIndexer> availableFilters = new ConcurrentDictionary<string, IWebIndexer>(); private ConcurrentDictionary<string, IWebIndexer> _availableFilters = new ConcurrentDictionary<string, IWebIndexer>();
// this map is used to maintain backward compatibility when renaming the id of an indexer // this map is used to maintain backward compatibility when renaming the id of an indexer
// (the id is used in the torznab/download/search urls and in the indexer configuration file) // (the id is used in the torznab/download/search urls and in the indexer configuration file)
// if the indexer is removed, remove it from this list too // if the indexer is removed, remove it from this list too
// use: {"<old id>", "<new id>"} // use: {"<old id>", "<new id>"}
private readonly Dictionary<string, string> renamedIndexers = new Dictionary<string, string> private readonly Dictionary<string, string> _renamedIndexers = new Dictionary<string, string>
{ {
{"audiobooktorrents", "abtorrents"}, {"audiobooktorrents", "abtorrents"},
{"baibako", "rudub"}, {"baibako", "rudub"},
@@ -69,19 +68,19 @@ namespace Jackett.Common.Services
public IndexerManagerService(IIndexerConfigurationService config, IProtectionService protectionService, WebClient webClient, Logger l, ICacheService cache, IProcessService processService, IConfigurationService globalConfigService, ServerConfig serverConfig) public IndexerManagerService(IIndexerConfigurationService config, IProtectionService protectionService, WebClient webClient, Logger l, ICacheService cache, IProcessService processService, IConfigurationService globalConfigService, ServerConfig serverConfig)
{ {
configService = config; _configService = config;
this.protectionService = protectionService; _protectionService = protectionService;
this.webClient = webClient; _webClient = webClient;
this.processService = processService; _processService = processService;
this.globalConfigService = globalConfigService; _globalConfigService = globalConfigService;
this.serverConfig = serverConfig; _serverConfig = serverConfig;
logger = l; _logger = l;
cacheService = cache; _cacheService = cache;
} }
public void InitIndexers(IEnumerable<string> path) public void InitIndexers(List<string> path)
{ {
logger.Info($"Using HTTP Client: {webClient.GetType().Name}"); _logger.Info($"Using HTTP Client: {_webClient.GetType().Name}");
MigrateRenamedIndexers(); MigrateRenamedIndexers();
InitIndexers(); InitIndexers();
@@ -92,14 +91,14 @@ namespace Jackett.Common.Services
private void MigrateRenamedIndexers() private void MigrateRenamedIndexers()
{ {
foreach (var oldId in renamedIndexers.Keys) foreach (var oldId in _renamedIndexers.Keys)
{ {
var oldPath = configService.GetIndexerConfigFilePath(oldId); var oldPath = _configService.GetIndexerConfigFilePath(oldId);
if (File.Exists(oldPath)) if (File.Exists(oldPath))
{ {
// if the old configuration exists, we rename it to be used by the renamed indexer // if the old configuration exists, we rename it to be used by the renamed indexer
logger.Info($"Old configuration detected: {oldPath}"); _logger.Info($"Old configuration detected: {oldPath}");
var newPath = configService.GetIndexerConfigFilePath(renamedIndexers[oldId]); var newPath = _configService.GetIndexerConfigFilePath(_renamedIndexers[oldId]);
if (File.Exists(newPath)) if (File.Exists(newPath))
File.Delete(newPath); File.Delete(newPath);
File.Move(oldPath, newPath); File.Move(oldPath, newPath);
@@ -112,14 +111,14 @@ namespace Jackett.Common.Services
File.Delete(newPathBak); File.Delete(newPathBak);
File.Move(oldPathBak, newPathBak); File.Move(oldPathBak, newPathBak);
} }
logger.Info($"Configuration renamed: {oldPath} => {newPath}"); _logger.Info($"Configuration renamed: {oldPath} => {newPath}");
} }
} }
} }
private void InitIndexers() private void InitIndexers()
{ {
logger.Info("Loading Native indexers ..."); _logger.Info("Loading Native indexers ...");
var allTypes = GetType().Assembly.GetTypes(); var allTypes = GetType().Assembly.GetTypes();
var allIndexerTypes = allTypes.Where(p => typeof(IIndexer).IsAssignableFrom(p)); var allIndexerTypes = allTypes.Where(p => typeof(IIndexer).IsAssignableFrom(p));
@@ -133,29 +132,29 @@ namespace Jackett.Common.Services
if (constructor != null) if (constructor != null)
{ {
// create own webClient instance for each indexer (separate cookies stores, etc.) // create own webClient instance for each indexer (separate cookies stores, etc.)
var indexerWebClientInstance = (WebClient)Activator.CreateInstance(webClient.GetType(), processService, logger, globalConfigService, serverConfig); var indexerWebClientInstance = (WebClient)Activator.CreateInstance(_webClient.GetType(), _processService, _logger, _globalConfigService, _serverConfig);
var arguments = new object[] { configService, indexerWebClientInstance, logger, protectionService, cacheService }; var arguments = new object[] { _configService, indexerWebClientInstance, _logger, _protectionService, _cacheService };
var indexer = (IIndexer)constructor.Invoke(arguments); var indexer = (IIndexer)constructor.Invoke(arguments);
return indexer; return indexer;
} }
logger.Error($"Cannot instantiate Native indexer: {type.Name}"); _logger.Error($"Cannot instantiate Native indexer: {type.Name}");
return null; return null;
}).Where(indexer => indexer != null).ToList(); }).Where(indexer => indexer != null).ToList();
foreach (var indexer in nativeIndexers) foreach (var indexer in nativeIndexers)
{ {
indexers.Add(indexer.Id, indexer); _indexers.Add(indexer.Id, indexer);
configService.Load(indexer); _configService.Load(indexer);
} }
logger.Info($"Loaded {nativeIndexers.Count} Native indexers: {string.Join(", ", nativeIndexers.Select(i => i.Id))}"); _logger.Info($"Loaded {nativeIndexers.Count} Native indexers: {string.Join(", ", nativeIndexers.Select(i => i.Id))}");
} }
private void InitCardigannIndexers(IEnumerable<string> path) private void InitCardigannIndexers(List<string> path)
{ {
logger.Info("Loading Cardigann indexers from: " + string.Join(", ", path)); _logger.Info("Loading Cardigann indexers from: " + string.Join(", ", path));
var deserializer = new DeserializerBuilder() var deserializer = new DeserializerBuilder()
.WithNamingConvention(CamelCaseNamingConvention.Instance) .WithNamingConvention(CamelCaseNamingConvention.Instance)
@@ -169,7 +168,7 @@ namespace Jackett.Common.Services
var files = existingDirectories.SelectMany(d => d.GetFiles("*.yml")); var files = existingDirectories.SelectMany(d => d.GetFiles("*.yml"));
var definitions = files.Select(file => var definitions = files.Select(file =>
{ {
logger.Debug("Loading Cardigann definition " + file.FullName); _logger.Debug("Loading Cardigann definition " + file.FullName);
try try
{ {
var definitionString = File.ReadAllText(file.FullName); var definitionString = File.ReadAllText(file.FullName);
@@ -178,7 +177,7 @@ namespace Jackett.Common.Services
} }
catch (Exception e) catch (Exception e)
{ {
logger.Error($"Error while parsing Cardigann definition {file.FullName}\n{e}"); _logger.Error($"Error while parsing Cardigann definition {file.FullName}\n{e}");
return null; return null;
} }
}).Where(definition => definition != null); }).Where(definition => definition != null);
@@ -187,16 +186,16 @@ namespace Jackett.Common.Services
{ {
try try
{ {
// create own webClient instance for each indexer (seperate cookies stores, etc.) // create own webClient instance for each indexer (separate cookies stores, etc.)
var indexerWebClientInstance = (WebClient)Activator.CreateInstance(webClient.GetType(), processService, logger, globalConfigService, serverConfig); var indexerWebClientInstance = (WebClient)Activator.CreateInstance(_webClient.GetType(), _processService, _logger, _globalConfigService, _serverConfig);
IIndexer indexer = new CardigannIndexer(configService, indexerWebClientInstance, logger, protectionService, cacheService, definition); IIndexer indexer = new CardigannIndexer(_configService, indexerWebClientInstance, _logger, _protectionService, _cacheService, definition);
configService.Load(indexer); _configService.Load(indexer);
return indexer; return indexer;
} }
catch (Exception e) catch (Exception e)
{ {
logger.Error($"Error while creating Cardigann instance from definition ID={definition.Id}: {e}"); _logger.Error($"Error while creating Cardigann instance from definition ID={definition.Id}: {e}");
return null; return null;
} }
}).Where(cardigannIndexer => cardigannIndexer != null).ToList(); // Explicit conversion to list to avoid repeated resource loading }).Where(cardigannIndexer => cardigannIndexer != null).ToList(); // Explicit conversion to list to avoid repeated resource loading
@@ -205,59 +204,59 @@ namespace Jackett.Common.Services
var cardiganIds = new List<string>(); var cardiganIds = new List<string>();
foreach (var indexer in cardigannIndexers) foreach (var indexer in cardigannIndexers)
{ {
if (indexers.ContainsKey(indexer.Id)) if (_indexers.ContainsKey(indexer.Id))
{ {
logger.Warn($"Ignoring definition ID={indexer.Id}: Indexer already exists"); _logger.Warn($"Ignoring definition ID={indexer.Id}: Indexer already exists");
continue; continue;
} }
indexers.Add(indexer.Id, indexer); _indexers.Add(indexer.Id, indexer);
cardigannCounter++; cardigannCounter++;
cardiganIds.Add(indexer.Id); cardiganIds.Add(indexer.Id);
} }
logger.Info($"Loaded {cardigannCounter} Cardigann indexers: {string.Join(", ", cardiganIds)}"); _logger.Info($"Loaded {cardigannCounter} Cardigann indexers: {string.Join(", ", cardiganIds)}");
} }
catch (Exception e) catch (Exception e)
{ {
logger.Error($"Error while loading Cardigann definitions: {e}"); _logger.Error($"Error while loading Cardigann definitions: {e}");
} }
logger.Info($"Loaded {indexers.Count} indexers in total"); _logger.Info($"Loaded {_indexers.Count} indexers in total");
} }
public void InitMetaIndexers() public void InitMetaIndexers()
{ {
var (fallbackStrategyProvider, resultFilterProvider) = GetStrategyProviders(); var (fallbackStrategyProvider, resultFilterProvider) = GetStrategyProviders();
logger.Info("Adding aggregate indexer ('all' indexer) ..."); _logger.Info("Adding aggregate indexer ('all' indexer) ...");
aggregateIndexer = new AggregateIndexer(fallbackStrategyProvider, resultFilterProvider, configService, webClient, logger, protectionService, cacheService) _aggregateIndexer = new AggregateIndexer(fallbackStrategyProvider, resultFilterProvider, _configService, _webClient, _logger, _protectionService, _cacheService)
{ {
Indexers = indexers.Values Indexers = _indexers.Values
}; };
var predefinedFilters = var predefinedFilters =
new[] { "public", "private", "semi-public" } new[] { "public", "private", "semi-public" }
.Select(type => (filter: FilterFunc.Type.ToFilter(type), func: FilterFunc.Type.ToFunc(type))) .Select(type => (filter: FilterFunc.Type.ToFilter(type), func: FilterFunc.Type.ToFunc(type)))
.Concat( .Concat(
indexers.Values.SelectMany(x => x.Tags).Distinct() _indexers.Values.SelectMany(x => x.Tags).Distinct()
.Select(tag => (filter: FilterFunc.Tag.ToFilter(tag), func: FilterFunc.Tag.ToFunc(tag))) .Select(tag => (filter: FilterFunc.Tag.ToFilter(tag), func: FilterFunc.Tag.ToFunc(tag)))
).Select(x => new KeyValuePair<string, IWebIndexer>(x.filter, CreateFilterIndexer(x.filter, x.func))); ).Select(x => new KeyValuePair<string, IWebIndexer>(x.filter, CreateFilterIndexer(x.filter, x.func)));
availableFilters = new ConcurrentDictionary<string, IWebIndexer>(predefinedFilters); _availableFilters = new ConcurrentDictionary<string, IWebIndexer>(predefinedFilters);
} }
public void RemoveLegacyConfigurations() public void RemoveLegacyConfigurations()
{ {
var directoryInfo = new DirectoryInfo(globalConfigService.GetIndexerConfigDir()); var directoryInfo = new DirectoryInfo(_globalConfigService.GetIndexerConfigDir());
if (!directoryInfo.Exists) if (!directoryInfo.Exists)
return; // the directory does not exist the first start return; // the directory does not exist the first start
var files = directoryInfo.GetFiles("*.json*"); var files = directoryInfo.GetFiles("*.json*");
foreach (var file in files) foreach (var file in files)
{ {
var indexerId = file.Name.Replace(".bak", "").Replace(".json", ""); var indexerId = file.Name.Replace(".bak", "").Replace(".json", "");
if (!indexers.ContainsKey(indexerId) && File.Exists(file.FullName)) if (!_indexers.ContainsKey(indexerId) && File.Exists(file.FullName))
{ {
logger.Info($"Removing old configuration file: {file.FullName}"); _logger.Info($"Removing old configuration file: {file.FullName}");
File.Delete(file.FullName); File.Delete(file.FullName);
} }
} }
@@ -268,10 +267,10 @@ namespace Jackett.Common.Services
// old id of renamed indexer is used to maintain backward compatibility // old id of renamed indexer is used to maintain backward compatibility
// both, the old id and the new one can be used until we remove it from renamedIndexers // both, the old id and the new one can be used until we remove it from renamedIndexers
var realName = name; var realName = name;
if (renamedIndexers.ContainsKey(name)) if (_renamedIndexers.ContainsKey(name))
{ {
realName = renamedIndexers[name]; realName = _renamedIndexers[name];
logger.Warn($@"Indexer {name} has been renamed to {realName}. Please, update the URL of the feeds. _logger.Warn($@"Indexer {name} has been renamed to {realName}. Please, update the URL of the feeds.
This may stop working in the future."); This may stop working in the future.");
} }
@@ -281,23 +280,23 @@ namespace Jackett.Common.Services
public IWebIndexer GetWebIndexer(string name) public IWebIndexer GetWebIndexer(string name)
{ {
if (indexers.ContainsKey(name)) if (_indexers.ContainsKey(name))
return indexers[name] as IWebIndexer; return _indexers[name] as IWebIndexer;
if (name == "all") if (name == "all")
return aggregateIndexer; return _aggregateIndexer;
if (availableFilters.TryGetValue(name, out var indexer)) if (_availableFilters.TryGetValue(name, out var indexer))
return indexer; return indexer;
if (FilterFunc.TryParse(name, out var filterFunc)) if (FilterFunc.TryParse(name, out var filterFunc))
return availableFilters.GetOrAdd(name, x => CreateFilterIndexer(name, filterFunc)); return _availableFilters.GetOrAdd(name, x => CreateFilterIndexer(name, filterFunc));
logger.Error($"Request for unknown indexer: {name}"); _logger.Error($"Request for unknown indexer: {name.Replace(Environment.NewLine, "")}");
throw new Exception($"Unknown indexer: {name}"); throw new Exception($"Unknown indexer: {name}");
} }
public IEnumerable<IIndexer> GetAllIndexers() => indexers.Values.OrderBy(_ => _.DisplayName); public List<IIndexer> GetAllIndexers() => _indexers.Values.OrderBy(_ => _.DisplayName).ToList();
public async Task TestIndexer(string name) public async Task TestIndexer(string name)
{ {
@@ -310,7 +309,7 @@ namespace Jackett.Common.Services
}; };
var result = await indexer.ResultsForQuery(query); var result = await indexer.ResultsForQuery(query);
logger.Info($"Test search in {indexer.DisplayName} => Found {result.Releases.Count()} releases"); _logger.Info($"Test search in {indexer.DisplayName} => Found {result.Releases.Count()} releases");
if (!result.Releases.Any()) if (!result.Releases.Any())
throw new Exception($"Test search in {indexer.DisplayName} => Found no results while trying to browse this tracker"); throw new Exception($"Test search in {indexer.DisplayName} => Found no results while trying to browse this tracker");
@@ -319,39 +318,39 @@ namespace Jackett.Common.Services
public void DeleteIndexer(string name) public void DeleteIndexer(string name)
{ {
var indexer = GetIndexer(name); var indexer = GetIndexer(name);
configService.Delete(indexer); _configService.Delete(indexer);
indexer.Unconfigure(); indexer.Unconfigure();
} }
private IWebIndexer CreateFilterIndexer(string filter, Func<IIndexer, bool> filterFunc) private IWebIndexer CreateFilterIndexer(string filter, Func<IIndexer, bool> filterFunc)
{ {
var (fallbackStrategyProvider, resultFilterProvider) = GetStrategyProviders(); var (fallbackStrategyProvider, resultFilterProvider) = GetStrategyProviders();
logger.Info($"Adding filter indexer ('{filter}' indexer) ..."); _logger.Info($"Adding filter indexer ('{filter.Replace(Environment.NewLine, "")}' indexer) ...");
return new FilterIndexer( return new FilterIndexer(
filter, filter,
fallbackStrategyProvider, fallbackStrategyProvider,
resultFilterProvider, resultFilterProvider,
configService, _configService,
webClient, _webClient,
logger, _logger,
protectionService, _protectionService,
cacheService, _cacheService,
filterFunc filterFunc
) )
{ {
Indexers = indexers.Values Indexers = _indexers.Values
}; };
} }
private (IFallbackStrategyProvider fallbackStrategyProvider, IResultFilterProvider resultFilterProvider) private (IFallbackStrategyProvider fallbackStrategyProvider, IResultFilterProvider resultFilterProvider)
GetStrategyProviders() GetStrategyProviders()
{ {
var omdbApiKey = serverConfig.OmdbApiKey; var omdbApiKey = _serverConfig.OmdbApiKey;
IFallbackStrategyProvider fallbackStrategyProvider; IFallbackStrategyProvider fallbackStrategyProvider;
IResultFilterProvider resultFilterProvider; IResultFilterProvider resultFilterProvider;
if (!string.IsNullOrWhiteSpace(omdbApiKey)) if (!string.IsNullOrWhiteSpace(omdbApiKey))
{ {
var imdbResolver = new OmdbResolver(webClient, omdbApiKey, serverConfig.OmdbApiUrl); var imdbResolver = new OmdbResolver(_webClient, omdbApiKey, _serverConfig.OmdbApiUrl);
fallbackStrategyProvider = new ImdbFallbackStrategyProvider(imdbResolver); fallbackStrategyProvider = new ImdbFallbackStrategyProvider(imdbResolver);
resultFilterProvider = new ImdbTitleResultFilterProvider(imdbResolver); resultFilterProvider = new ImdbTitleResultFilterProvider(imdbResolver);
} }

4
src/Jackett.Common/Services/Interfaces/IIndexerManagerService.cs
View File

@@ -10,9 +10,9 @@ namespace Jackett.Common.Services.Interfaces
void DeleteIndexer(string name); void DeleteIndexer(string name);
IIndexer GetIndexer(string name); IIndexer GetIndexer(string name);
IWebIndexer GetWebIndexer(string name); IWebIndexer GetWebIndexer(string name);
IEnumerable<IIndexer> GetAllIndexers(); List<IIndexer> GetAllIndexers();
void InitIndexers(IEnumerable<string> path); void InitIndexers(List<string> path);
void InitMetaIndexers(); void InitMetaIndexers();
} }
} }

4
src/Jackett.Test/TestHelpers/TestIIndexerManagerServiceHelper.cs
View File

@@ -13,13 +13,13 @@ namespace Jackett.Test.TestHelpers
public void DeleteIndexer(string name) => throw new NotImplementedException(); public void DeleteIndexer(string name) => throw new NotImplementedException();
public IEnumerable<IIndexer> GetAllIndexers() => throw new NotImplementedException(); public List<IIndexer> GetAllIndexers() => throw new NotImplementedException();
public IIndexer GetIndexer(string name) => throw new NotImplementedException(); public IIndexer GetIndexer(string name) => throw new NotImplementedException();
public IWebIndexer GetWebIndexer(string name) => throw new NotImplementedException(); public IWebIndexer GetWebIndexer(string name) => throw new NotImplementedException();
public void InitIndexers(IEnumerable<string> path) => throw new NotImplementedException(); public void InitIndexers(List<string> path) => throw new NotImplementedException();
public void SaveConfig(IIndexer indexer, JToken obj) => LastSavedConfig = obj; public void SaveConfig(IIndexer indexer, JToken obj) => LastSavedConfig = obj;