diff --git a/frontend/src/Indexer/Index/Table/CapabilitiesLabel.js b/frontend/src/Indexer/Index/Table/CapabilitiesLabel.js index 144b9e6ad..9fb216193 100644 --- a/frontend/src/Indexer/Index/Table/CapabilitiesLabel.js +++ b/frontend/src/Indexer/Index/Table/CapabilitiesLabel.js @@ -59,4 +59,13 @@ CapabilitiesLabel.propTypes = { capabilities: PropTypes.object.isRequired }; +CapabilitiesLabel.defaultProps = { + capabilities: { + movieSearchAvailable: false, + tvSearchAvailable: false, + musicSearchAvailable: false, + bookSearchAvailable: false + } +}; + export default CapabilitiesLabel; diff --git a/src/NzbDrone.Core/IndexerVersions/IndexerDefinitionUpdateCommand.cs b/src/NzbDrone.Core/IndexerVersions/IndexerDefinitionUpdateCommand.cs new file mode 100644 index 000000000..0d9ea54a5 --- /dev/null +++ b/src/NzbDrone.Core/IndexerVersions/IndexerDefinitionUpdateCommand.cs @@ -0,0 +1,9 @@ +using NzbDrone.Core.Messaging.Commands; + +namespace NzbDrone.Core.IndexerVersions +{ + public class IndexerDefinitionUpdateCommand : Command + { + public override bool SendUpdatesToClient => true; + } +} diff --git a/src/NzbDrone.Core/IndexerVersions/IndexerDefinitionUpdateService.cs b/src/NzbDrone.Core/IndexerVersions/IndexerDefinitionUpdateService.cs new file mode 100644 index 000000000..4654c8423 --- /dev/null +++ b/src/NzbDrone.Core/IndexerVersions/IndexerDefinitionUpdateService.cs @@ -0,0 +1,157 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using NLog; +using NzbDrone.Common.Cache; +using NzbDrone.Common.EnvironmentInfo; +using NzbDrone.Common.Extensions; +using NzbDrone.Common.Http; +using NzbDrone.Core.Indexers.Cardigann; +using NzbDrone.Core.Messaging.Commands; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace NzbDrone.Core.IndexerVersions +{ + public interface IIndexerDefinitionUpdateService + { + List All(); + CardigannDefinition GetDefinition(string fileKey); + } + + public class IndexerDefinitionUpdateService : IIndexerDefinitionUpdateService, IExecute + { + private const int DEFINITION_VERSION = 1; + + private readonly IHttpClient _httpClient; + private readonly IAppFolderInfo _appFolderInfo; + private readonly ICached _cache; + private readonly Logger _logger; + + private readonly IDeserializer _deserializer = new DeserializerBuilder() + .IgnoreUnmatchedProperties() + .WithNamingConvention(CamelCaseNamingConvention.Instance) + .Build(); + + public IndexerDefinitionUpdateService(IHttpClient httpClient, + IAppFolderInfo appFolderInfo, + ICacheManager cacheManager, + Logger logger) + { + _appFolderInfo = appFolderInfo; + _cache = cacheManager.GetCache(typeof(CardigannDefinition), "definitions"); + _httpClient = httpClient; + _logger = logger; + } + + public List All() + { + var request = new HttpRequest($"https://indexers.prowlarr.com/master/{DEFINITION_VERSION}"); + var response = _httpClient.Get>(request); + var remoteDefs = response.Resource.ToDictionary(x => x.File); + + var startupFolder = _appFolderInfo.StartUpFolder; + + var prefix = Path.Combine(startupFolder, "Definitions"); + + var directoryInfos = new List { new DirectoryInfo(prefix) }; + var existingDirectories = directoryInfos.Where(d => d.Exists); + var files = existingDirectories.SelectMany(d => d.GetFiles("*.yml")); + + var indexerList = new List(); + + foreach (var file in files) + { + indexerList.AddIfNotNull(remoteDefs[Path.GetFileNameWithoutExtension(file.Name)]); + } + + return indexerList; + } + + public CardigannDefinition GetDefinition(string file) + { + if (string.IsNullOrEmpty(file)) + { + throw new ArgumentNullException(nameof(file)); + } + + var definition = _cache.Get(file, () => LoadIndexerDef(file)); + + return definition; + } + + private CardigannDefinition GetHttpDefinition(string id) + { + var req = new HttpRequest($"https://indexers.prowlarr.com/master/{DEFINITION_VERSION}/{id}"); + var response = _httpClient.Get(req); + return _deserializer.Deserialize(response.Content); + } + + private CardigannDefinition LoadIndexerDef(string fileKey) + { + if (string.IsNullOrEmpty(fileKey)) + { + throw new ArgumentNullException(nameof(fileKey)); + } + + var definitionFolder = Path.Combine(_appFolderInfo.StartUpFolder, "Definitions"); + + var directoryInfo = new DirectoryInfo(definitionFolder); + + if (directoryInfo.Exists) + { + var files = directoryInfo.GetFiles($"{fileKey}.yml"); + + if (files.Any()) + { + var file = files.First(); + _logger.Trace("Loading Cardigann definition " + file.FullName); + try + { + var definitionString = File.ReadAllText(file.FullName); + var definition = _deserializer.Deserialize(definitionString); + return definition; + } + catch (Exception e) + { + _logger.Error($"Error while parsing Cardigann definition {file.FullName}\n{e}"); + } + } + } + + throw new ArgumentOutOfRangeException(nameof(fileKey)); + } + + public void Execute(IndexerDefinitionUpdateCommand message) + { + UpdateLocalDefinitions(); + } + + private void UpdateLocalDefinitions() + { + var request = new HttpRequest($"https://indexers.prowlarr.com/master/{DEFINITION_VERSION}"); + var response = _httpClient.Get>(request); + + foreach (var def in response.Resource) + { + try + { + var startupFolder = _appFolderInfo.StartUpFolder; + + var saveFile = Path.Combine(startupFolder, "Definitions", $"{def.File}.yml"); + + _httpClient.DownloadFile($"https://indexers.prowlarr.com/master/{DEFINITION_VERSION}/{def.File}", saveFile); + + _cache.Remove(def.File); + + _logger.Info("Updated definition: {0}", def.File); + } + catch (Exception ex) + { + _logger.Error("Definition download failed: {0}, {1}", def.File, ex.Message); + } + } + } + } +} diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs index 31c2b33d4..41ac192d7 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs @@ -4,6 +4,7 @@ using FluentValidation.Results; using NLog; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; +using NzbDrone.Core.IndexerVersions; using NzbDrone.Core.ThingiProvider; using NzbDrone.Core.Validation; @@ -11,7 +12,7 @@ namespace NzbDrone.Core.Indexers.Cardigann { public class Cardigann : HttpIndexerBase { - private readonly ICardigannDefinitionService _definitionService; + private readonly IIndexerDefinitionUpdateService _definitionService; public override string Name => "Cardigann"; @@ -33,14 +34,6 @@ namespace NzbDrone.Core.Indexers.Cardigann _logger); } - public override IndexerCapabilities GetCapabilities() - { - // TODO: This uses indexer capabilities when called so we don't have to keep up with all of them - // however, this is not pulled on a all pull from UI, doing so will kill the UI load if an indexer is down - // should we just purge and manage - return new IndexerCapabilities(); - } - public override IEnumerable DefaultDefinitions { get @@ -52,7 +45,7 @@ namespace NzbDrone.Core.Indexers.Cardigann } } - public Cardigann(ICardigannDefinitionService definitionService, + public Cardigann(IIndexerDefinitionUpdateService definitionService, IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, @@ -76,7 +69,7 @@ namespace NzbDrone.Core.Indexers.Cardigann Privacy = definition.Type == "private" ? IndexerPrivacy.Private : IndexerPrivacy.Public, SupportsRss = SupportsRss, SupportsSearch = SupportsSearch, - Capabilities = Capabilities, + Capabilities = new IndexerCapabilities(), ExtraFields = definition.Settings }; } diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/IndexerDefinition.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannDefinition.cs similarity index 100% rename from src/NzbDrone.Core/Indexers/Definitions/Cardigann/IndexerDefinition.cs rename to src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannDefinition.cs diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannDefinitionService.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannDefinitionService.cs deleted file mode 100644 index beb5bd202..000000000 --- a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannDefinitionService.cs +++ /dev/null @@ -1,44 +0,0 @@ -using System.Collections.Generic; -using NzbDrone.Common.Http; -using YamlDotNet.Serialization; -using YamlDotNet.Serialization.NamingConventions; - -namespace NzbDrone.Core.Indexers.Cardigann -{ - public interface ICardigannDefinitionService - { - List All(); - CardigannDefinition GetDefinition(string id); - } - - public class CardigannDefinitionService : ICardigannDefinitionService - { - private const int DEFINITION_VERSION = 1; - - private readonly IHttpClient _httpClient; - - private readonly IDeserializer _deserializer = new DeserializerBuilder() - .IgnoreUnmatchedProperties() - .WithNamingConvention(CamelCaseNamingConvention.Instance) - .Build(); - - public CardigannDefinitionService(IHttpClient httpClient) - { - _httpClient = httpClient; - } - - public List All() - { - var request = new HttpRequest($"https://indexers.prowlarr.com/master/{DEFINITION_VERSION}"); - var response = _httpClient.Get>(request); - return response.Resource; - } - - public CardigannDefinition GetDefinition(string id) - { - var req = new HttpRequest($"https://indexers.prowlarr.com/master/{DEFINITION_VERSION}/{id}"); - var response = _httpClient.Get(req); - return _deserializer.Deserialize(response.Content); - } - } -} diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannMetaDef.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannMetaDef.cs index a27e0c9dc..6fb1645be 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannMetaDef.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannMetaDef.cs @@ -14,5 +14,6 @@ namespace NzbDrone.Core.Indexers.Cardigann public List Links { get; set; } public List Legacylinks { get; set; } public List Settings { get; set; } + public string Sha { get; set; } } } diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/IndexerDefinitionUpdateCommand.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/IndexerDefinitionUpdateCommand.cs new file mode 100644 index 000000000..912eb955c --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/IndexerDefinitionUpdateCommand.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NzbDrone.Core.Messaging.Commands; + +namespace NzbDrone.Core.Indexers.Definitions.Cardigann +{ + public class IndexerDefinitionUpdateCommand : Command + { + public override bool SendUpdatesToClient => true; + } +} diff --git a/src/NzbDrone.Core/Indexers/Definitions/Newznab/Newznab.cs b/src/NzbDrone.Core/Indexers/Definitions/Newznab/Newznab.cs index 426fd891e..461d69285 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Newznab/Newznab.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Newznab/Newznab.cs @@ -20,6 +20,8 @@ namespace NzbDrone.Core.Indexers.Newznab public override DownloadProtocol Protocol => DownloadProtocol.Usenet; public override IndexerPrivacy Privacy => IndexerPrivacy.Private; + public override IndexerCapabilities Capabilities { get => new IndexerCapabilities(); protected set => base.Capabilities = value; } + public override int PageSize => _capabilitiesProvider.GetCapabilities(Settings).LimitsDefault.Value; public override IIndexerRequestGenerator GetRequestGenerator() @@ -36,14 +38,6 @@ namespace NzbDrone.Core.Indexers.Newznab return new NewznabRssParser(Settings); } - public override IndexerCapabilities GetCapabilities() - { - // TODO: This uses indexer capabilities when called so we don't have to keep up with all of them - // however, this is not pulled on a all pull from UI, doing so will kill the UI load if an indexer is down - // should we just purge and manage - return _capabilitiesProvider.GetCapabilities(Settings); - } - public override IEnumerable DefaultDefinitions { get diff --git a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs index 7dc2ea842..8e56e9f8f 100644 --- a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs @@ -23,7 +23,7 @@ namespace NzbDrone.Core.Indexers public override bool SupportsRss => true; public override bool SupportsSearch => true; - public override IndexerCapabilities Capabilities => new IndexerCapabilities(); + public override IndexerCapabilities Capabilities { get; protected set; } public bool SupportsPaging => PageSize > 0; @@ -86,11 +86,6 @@ namespace NzbDrone.Core.Indexers return requests; } - public override IndexerCapabilities GetCapabilities() - { - return Capabilities; - } - protected virtual IList FetchReleases(Func pageableRequestChainSelector, bool isRecent = false) { var releases = new List(); diff --git a/src/NzbDrone.Core/Indexers/IIndexer.cs b/src/NzbDrone.Core/Indexers/IIndexer.cs index d13506844..a05e42362 100644 --- a/src/NzbDrone.Core/Indexers/IIndexer.cs +++ b/src/NzbDrone.Core/Indexers/IIndexer.cs @@ -16,7 +16,5 @@ namespace NzbDrone.Core.Indexers IList FetchRecent(); IList Fetch(MovieSearchCriteria searchCriteria); - - IndexerCapabilities GetCapabilities(); } } diff --git a/src/NzbDrone.Core/Indexers/IndexerBase.cs b/src/NzbDrone.Core/Indexers/IndexerBase.cs index baeeff9b3..63788a825 100644 --- a/src/NzbDrone.Core/Indexers/IndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/IndexerBase.cs @@ -26,7 +26,7 @@ namespace NzbDrone.Core.Indexers public abstract bool SupportsRss { get; } public abstract bool SupportsSearch { get; } - public abstract IndexerCapabilities Capabilities { get; } + public abstract IndexerCapabilities Capabilities { get; protected set; } public IndexerBase(IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger) { @@ -69,8 +69,6 @@ namespace NzbDrone.Core.Indexers public abstract IList FetchRecent(); public abstract IList Fetch(MovieSearchCriteria searchCriteria); - public abstract IndexerCapabilities GetCapabilities(); - protected virtual IList CleanupReleases(IEnumerable releases) { var result = releases.DistinctBy(v => v.Guid).ToList(); diff --git a/src/NzbDrone.Core/Indexers/IndexerCapabilities.cs b/src/NzbDrone.Core/Indexers/IndexerCapabilities.cs index e7c8546f1..d9e2870b1 100644 --- a/src/NzbDrone.Core/Indexers/IndexerCapabilities.cs +++ b/src/NzbDrone.Core/Indexers/IndexerCapabilities.cs @@ -2,6 +2,8 @@ using System; using System.Collections.Generic; using System.Linq; using System.Xml.Linq; +using NzbDrone.Core.Indexers.Cardigann; +using NzbDrone.Core.Parser.Model; namespace NzbDrone.Core.Indexers { @@ -88,6 +90,47 @@ namespace NzbDrone.Core.Indexers Categories = new List(); } + public void ParseCardigannSearchModes(Dictionary> modes) + { + if (modes == null || !modes.Any()) + { + throw new Exception("At least one search mode is required"); + } + + if (!modes.ContainsKey("search")) + { + throw new Exception("The search mode 'search' is mandatory"); + } + + foreach (var entry in modes) + { + switch (entry.Key) + { + case "search": + if (entry.Value == null || entry.Value.Count != 1 || entry.Value[0] != "q") + { + throw new Exception("In search mode 'search' only 'q' parameter is supported and it's mandatory"); + } + + break; + case "tv-search": + ParseTvSearchParams(entry.Value); + break; + case "movie-search": + ParseMovieSearchParams(entry.Value); + break; + case "music-search": + ParseMusicSearchParams(entry.Value); + break; + case "book-search": + ParseBookSearchParams(entry.Value); + break; + default: + throw new Exception($"Unsupported search mode: {entry.Key}"); + } + } + } + public void ParseTvSearchParams(IEnumerable paramsList) { if (paramsList == null) @@ -169,6 +212,33 @@ namespace NzbDrone.Core.Indexers } } + private void ParseBookSearchParams(IEnumerable paramsList) + { + if (paramsList == null) + { + return; + } + + foreach (var paramStr in paramsList) + { + if (Enum.TryParse(paramStr, true, out BookSearchParam param)) + { + if (!BookSearchParams.Contains(param)) + { + BookSearchParams.Add(param); + } + else + { + throw new Exception($"Duplicate book-search param: {paramStr}"); + } + } + else + { + throw new Exception($"Not supported book-search param: {paramStr}"); + } + } + } + private string SupportedTvSearchParams() { var parameters = new List { "q" }; // q is always enabled diff --git a/src/NzbDrone.Core/Indexers/IndexerFactory.cs b/src/NzbDrone.Core/Indexers/IndexerFactory.cs index 875ee4588..f3cf51a95 100644 --- a/src/NzbDrone.Core/Indexers/IndexerFactory.cs +++ b/src/NzbDrone.Core/Indexers/IndexerFactory.cs @@ -5,6 +5,7 @@ using FluentValidation.Results; using NLog; using NzbDrone.Common.Composition; using NzbDrone.Core.Indexers.Cardigann; +using NzbDrone.Core.IndexerVersions; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.ThingiProvider; @@ -20,11 +21,11 @@ namespace NzbDrone.Core.Indexers public class IndexerFactory : ProviderFactory, IIndexerFactory { - private readonly ICardigannDefinitionService _definitionService; + private readonly IIndexerDefinitionUpdateService _definitionService; private readonly IIndexerStatusService _indexerStatusService; private readonly Logger _logger; - public IndexerFactory(ICardigannDefinitionService definitionService, + public IndexerFactory(IIndexerDefinitionUpdateService definitionService, IIndexerStatusService indexerStatusService, IIndexerRepository providerRepository, IEnumerable providers, @@ -41,14 +42,17 @@ namespace NzbDrone.Core.Indexers public override List All() { var definitions = base.All(); - var metaDefs = _definitionService.All().ToDictionary(x => x.File); foreach (var definition in definitions) { if (definition.Implementation == typeof(Cardigann.Cardigann).Name) { var settings = (CardigannSettings)definition.Settings; - definition.ExtraFields = metaDefs[settings.DefinitionFile].Settings; + var defFile = _definitionService.GetDefinition(settings.DefinitionFile); + definition.ExtraFields = defFile.Settings; + definition.Privacy = defFile.Type == "private" ? IndexerPrivacy.Private : IndexerPrivacy.Public; + definition.Capabilities = new IndexerCapabilities(); + definition.Capabilities.ParseCardigannSearchModes(defFile.Caps.Modes); } } @@ -58,12 +62,15 @@ namespace NzbDrone.Core.Indexers public override IndexerDefinition Get(int id) { var definition = base.Get(id); - var metaDefs = _definitionService.All().ToDictionary(x => x.File); if (definition.Implementation == typeof(Cardigann.Cardigann).Name) { var settings = (CardigannSettings)definition.Settings; - definition.ExtraFields = metaDefs[settings.DefinitionFile].Settings; + var defFile = _definitionService.GetDefinition(settings.DefinitionFile); + definition.ExtraFields = defFile.Settings; + definition.Privacy = defFile.Type == "private" ? IndexerPrivacy.Private : IndexerPrivacy.Public; + definition.Capabilities = new IndexerCapabilities(); + definition.Capabilities.ParseCardigannSearchModes(defFile.Caps.Modes); } return definition; @@ -79,7 +86,7 @@ namespace NzbDrone.Core.Indexers foreach (var provider in _providers) { var definitions = provider.DefaultDefinitions - .Where(v => v.Name != null && v.Name != provider.GetType().Name); + .Where(v => v.Name != null && (v.Name != typeof(Cardigann.Cardigann).Name || v.Name != typeof(Newznab.Newznab).Name)); foreach (IndexerDefinition definition in definitions) { @@ -99,10 +106,15 @@ namespace NzbDrone.Core.Indexers base.SetProviderCharacteristics(provider, definition); definition.Protocol = provider.Protocol; - definition.Privacy = provider.Privacy; definition.SupportsRss = provider.SupportsRss; definition.SupportsSearch = provider.SupportsSearch; - definition.Capabilities = provider.Capabilities; + + //We want to use the definition Caps and Privacy for Cardigann instead of the provider. + if (definition.Implementation != typeof(Cardigann.Cardigann).Name) + { + definition.Privacy = provider.Privacy; + definition.Capabilities = provider.Capabilities; + } } public List RssEnabled(bool filterBlockedIndexers = true) diff --git a/src/NzbDrone.Core/Jobs/TaskManager.cs b/src/NzbDrone.Core/Jobs/TaskManager.cs index 72e718ed1..ef93adee4 100644 --- a/src/NzbDrone.Core/Jobs/TaskManager.cs +++ b/src/NzbDrone.Core/Jobs/TaskManager.cs @@ -6,6 +6,7 @@ using NzbDrone.Core.Backup; using NzbDrone.Core.Configuration; using NzbDrone.Core.HealthCheck; using NzbDrone.Core.Housekeeping; +using NzbDrone.Core.IndexerVersions; using NzbDrone.Core.Lifecycle; using NzbDrone.Core.Messaging.Commands; using NzbDrone.Core.Messaging.Events; @@ -59,6 +60,7 @@ namespace NzbDrone.Core.Jobs new ScheduledTask { Interval = 6 * 60, TypeName = typeof(ApplicationCheckUpdateCommand).FullName }, new ScheduledTask { Interval = 6 * 60, TypeName = typeof(CheckHealthCommand).FullName }, new ScheduledTask { Interval = 24 * 60, TypeName = typeof(HousekeepingCommand).FullName }, + new ScheduledTask { Interval = 6 * 60, TypeName = typeof(IndexerDefinitionUpdateCommand).FullName }, new ScheduledTask { diff --git a/src/Prowlarr.Api.V1/Indexers/IndexerModule.cs b/src/Prowlarr.Api.V1/Indexers/IndexerModule.cs index c9859bdc7..9521c8126 100644 --- a/src/Prowlarr.Api.V1/Indexers/IndexerModule.cs +++ b/src/Prowlarr.Api.V1/Indexers/IndexerModule.cs @@ -59,7 +59,7 @@ namespace Prowlarr.Api.V1.Indexers switch (requestType) { case "caps": - Response response = indexerInstance.GetCapabilities().ToXml(); + Response response = indexer.Capabilities.ToXml(); response.ContentType = "application/rss+xml"; return response; case "tvsearch":