More Cardigann

frontend/src/Indexer/Index/Table/CapabilitiesLabel.js

@@ -59,4 +59,13 @@ CapabilitiesLabel.propTypes = {
   capabilities: PropTypes.object.isRequired
 };
 
+CapabilitiesLabel.defaultProps = {
+  capabilities: {
+    movieSearchAvailable: false,
+    tvSearchAvailable: false,
+    musicSearchAvailable: false,
+    bookSearchAvailable: false
+  }
+};
+
 export default CapabilitiesLabel;

src/NzbDrone.Core/IndexerVersions/IndexerDefinitionUpdateCommand.cs

@@ -0,0 +1,9 @@
+using NzbDrone.Core.Messaging.Commands;
+
+namespace NzbDrone.Core.IndexerVersions
+{
+    public class IndexerDefinitionUpdateCommand : Command
+    {
+        public override bool SendUpdatesToClient => true;
+    }
+}

src/NzbDrone.Core/IndexerVersions/IndexerDefinitionUpdateService.cs

@@ -0,0 +1,157 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using NLog;
+using NzbDrone.Common.Cache;
+using NzbDrone.Common.EnvironmentInfo;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.Indexers.Cardigann;
+using NzbDrone.Core.Messaging.Commands;
+using YamlDotNet.Serialization;
+using YamlDotNet.Serialization.NamingConventions;
+
+namespace NzbDrone.Core.IndexerVersions
+{
+    public interface IIndexerDefinitionUpdateService
+    {
+        List<CardigannMetaDefinition> All();
+        CardigannDefinition GetDefinition(string fileKey);
+    }
+
+    public class IndexerDefinitionUpdateService : IIndexerDefinitionUpdateService, IExecute<IndexerDefinitionUpdateCommand>
+    {
+        private const int DEFINITION_VERSION = 1;
+
+        private readonly IHttpClient _httpClient;
+        private readonly IAppFolderInfo _appFolderInfo;
+        private readonly ICached<CardigannDefinition> _cache;
+        private readonly Logger _logger;
+
+        private readonly IDeserializer _deserializer = new DeserializerBuilder()
+            .IgnoreUnmatchedProperties()
+            .WithNamingConvention(CamelCaseNamingConvention.Instance)
+            .Build();
+
+        public IndexerDefinitionUpdateService(IHttpClient httpClient,
+                                          IAppFolderInfo appFolderInfo,
+                                          ICacheManager cacheManager,
+                                          Logger logger)
+        {
+            _appFolderInfo = appFolderInfo;
+            _cache = cacheManager.GetCache<CardigannDefinition>(typeof(CardigannDefinition), "definitions");
+            _httpClient = httpClient;
+            _logger = logger;
+        }
+
+        public List<CardigannMetaDefinition> All()
+        {
+            var request = new HttpRequest($"https://indexers.prowlarr.com/master/{DEFINITION_VERSION}");
+            var response = _httpClient.Get<List<CardigannMetaDefinition>>(request);
+            var remoteDefs = response.Resource.ToDictionary(x => x.File);
+
+            var startupFolder = _appFolderInfo.StartUpFolder;
+
+            var prefix = Path.Combine(startupFolder, "Definitions");
+
+            var directoryInfos = new List<DirectoryInfo> { new DirectoryInfo(prefix) };
+            var existingDirectories = directoryInfos.Where(d => d.Exists);
+            var files = existingDirectories.SelectMany(d => d.GetFiles("*.yml"));
+
+            var indexerList = new List<CardigannMetaDefinition>();
+
+            foreach (var file in files)
+            {
+                indexerList.AddIfNotNull(remoteDefs[Path.GetFileNameWithoutExtension(file.Name)]);
+            }
+
+            return indexerList;
+        }
+
+        public CardigannDefinition GetDefinition(string file)
+        {
+            if (string.IsNullOrEmpty(file))
+            {
+                throw new ArgumentNullException(nameof(file));
+            }
+
+            var definition = _cache.Get(file, () => LoadIndexerDef(file));
+
+            return definition;
+        }
+
+        private CardigannDefinition GetHttpDefinition(string id)
+        {
+            var req = new HttpRequest($"https://indexers.prowlarr.com/master/{DEFINITION_VERSION}/{id}");
+            var response = _httpClient.Get(req);
+            return _deserializer.Deserialize<CardigannDefinition>(response.Content);
+        }
+
+        private CardigannDefinition LoadIndexerDef(string fileKey)
+        {
+            if (string.IsNullOrEmpty(fileKey))
+            {
+                throw new ArgumentNullException(nameof(fileKey));
+            }
+
+            var definitionFolder = Path.Combine(_appFolderInfo.StartUpFolder, "Definitions");
+
+            var directoryInfo = new DirectoryInfo(definitionFolder);
+
+            if (directoryInfo.Exists)
+            {
+                var files = directoryInfo.GetFiles($"{fileKey}.yml");
+
+                if (files.Any())
+                {
+                    var file = files.First();
+                    _logger.Trace("Loading Cardigann definition " + file.FullName);
+                    try
+                    {
+                        var definitionString = File.ReadAllText(file.FullName);
+                        var definition = _deserializer.Deserialize<CardigannDefinition>(definitionString);
+                        return definition;
+                    }
+                    catch (Exception e)
+                    {
+                        _logger.Error($"Error while parsing Cardigann definition {file.FullName}\n{e}");
+                    }
+                }
+            }
+
+            throw new ArgumentOutOfRangeException(nameof(fileKey));
+        }
+
+        public void Execute(IndexerDefinitionUpdateCommand message)
+        {
+            UpdateLocalDefinitions();
+        }
+
+        private void UpdateLocalDefinitions()
+        {
+            var request = new HttpRequest($"https://indexers.prowlarr.com/master/{DEFINITION_VERSION}");
+            var response = _httpClient.Get<List<CardigannMetaDefinition>>(request);
+
+            foreach (var def in response.Resource)
+            {
+                try
+                {
+                    var startupFolder = _appFolderInfo.StartUpFolder;
+
+                    var saveFile = Path.Combine(startupFolder, "Definitions", $"{def.File}.yml");
+
+                    _httpClient.DownloadFile($"https://indexers.prowlarr.com/master/{DEFINITION_VERSION}/{def.File}", saveFile);
+
+                    _cache.Remove(def.File);
+
+                    _logger.Info("Updated definition: {0}", def.File);
+                }
+                catch (Exception ex)
+                {
+                    _logger.Error("Definition download failed: {0}, {1}", def.File, ex.Message);
+                }
+            }
+        }
+    }
+}

src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs

@@ -4,6 +4,7 @@ using FluentValidation.Results;
 using NLog;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
+using NzbDrone.Core.IndexerVersions;
 using NzbDrone.Core.ThingiProvider;
 using NzbDrone.Core.Validation;
 
@@ -11,7 +12,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
 {
     public class Cardigann : HttpIndexerBase<CardigannSettings>
     {
-        private readonly ICardigannDefinitionService _definitionService;
+        private readonly IIndexerDefinitionUpdateService _definitionService;
 
         public override string Name => "Cardigann";
 
@@ -33,14 +34,6 @@ namespace NzbDrone.Core.Indexers.Cardigann
                                        _logger);
         }
 
-        public override IndexerCapabilities GetCapabilities()
-        {
-            // TODO: This uses indexer capabilities when called so we don't have to keep up with all of them
-            // however, this is not pulled on a all pull from UI, doing so will kill the UI load if an indexer is down
-            // should we just purge and manage
-            return new IndexerCapabilities();
-        }
-
         public override IEnumerable<ProviderDefinition> DefaultDefinitions
         {
             get
@@ -52,7 +45,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
             }
         }
 
-        public Cardigann(ICardigannDefinitionService definitionService,
+        public Cardigann(IIndexerDefinitionUpdateService definitionService,
                          IHttpClient httpClient,
                          IIndexerStatusService indexerStatusService,
                          IConfigService configService,
@@ -76,7 +69,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
                 Privacy = definition.Type == "private" ? IndexerPrivacy.Private : IndexerPrivacy.Public,
                 SupportsRss = SupportsRss,
                 SupportsSearch = SupportsSearch,
-                Capabilities = Capabilities,
+                Capabilities = new IndexerCapabilities(),
                 ExtraFields = definition.Settings
             };
         }

src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannDefinitionService.cs

@@ -1,44 +0,0 @@
-using System.Collections.Generic;
-using NzbDrone.Common.Http;
-using YamlDotNet.Serialization;
-using YamlDotNet.Serialization.NamingConventions;
-
-namespace NzbDrone.Core.Indexers.Cardigann
-{
-    public interface ICardigannDefinitionService
-    {
-        List<CardigannMetaDefinition> All();
-        CardigannDefinition GetDefinition(string id);
-    }
-
-    public class CardigannDefinitionService : ICardigannDefinitionService
-    {
-        private const int DEFINITION_VERSION = 1;
-
-        private readonly IHttpClient _httpClient;
-
-        private readonly IDeserializer _deserializer = new DeserializerBuilder()
-            .IgnoreUnmatchedProperties()
-            .WithNamingConvention(CamelCaseNamingConvention.Instance)
-            .Build();
-
-        public CardigannDefinitionService(IHttpClient httpClient)
-        {
-            _httpClient = httpClient;
-        }
-
-        public List<CardigannMetaDefinition> All()
-        {
-            var request = new HttpRequest($"https://indexers.prowlarr.com/master/{DEFINITION_VERSION}");
-            var response = _httpClient.Get<List<CardigannMetaDefinition>>(request);
-            return response.Resource;
-        }
-
-        public CardigannDefinition GetDefinition(string id)
-        {
-            var req = new HttpRequest($"https://indexers.prowlarr.com/master/{DEFINITION_VERSION}/{id}");
-            var response = _httpClient.Get(req);
-            return _deserializer.Deserialize<CardigannDefinition>(response.Content);
-        }
-    }
-}

src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannMetaDef.cs

@@ -14,5 +14,6 @@ namespace NzbDrone.Core.Indexers.Cardigann
         public List<string> Links { get; set; }
         public List<string> Legacylinks { get; set; }
         public List<SettingsField> Settings { get; set; }
+        public string Sha { get; set; }
     }
 }

src/NzbDrone.Core/Indexers/Definitions/Cardigann/IndexerDefinitionUpdateCommand.cs

@@ -0,0 +1,14 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using NzbDrone.Core.Messaging.Commands;
+
+namespace NzbDrone.Core.Indexers.Definitions.Cardigann
+{
+    public class IndexerDefinitionUpdateCommand : Command
+    {
+        public override bool SendUpdatesToClient => true;
+    }
+}

src/NzbDrone.Core/Indexers/Definitions/Newznab/Newznab.cs

@@ -20,6 +20,8 @@ namespace NzbDrone.Core.Indexers.Newznab
         public override DownloadProtocol Protocol => DownloadProtocol.Usenet;
         public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
 
+        public override IndexerCapabilities Capabilities { get => new IndexerCapabilities(); protected set => base.Capabilities = value; }
+
         public override int PageSize => _capabilitiesProvider.GetCapabilities(Settings).LimitsDefault.Value;
 
         public override IIndexerRequestGenerator GetRequestGenerator()
@@ -36,14 +38,6 @@ namespace NzbDrone.Core.Indexers.Newznab
             return new NewznabRssParser(Settings);
         }
 
-        public override IndexerCapabilities GetCapabilities()
-        {
-            // TODO: This uses indexer capabilities when called so we don't have to keep up with all of them
-            // however, this is not pulled on a all pull from UI, doing so will kill the UI load if an indexer is down
-            // should we just purge and manage
-            return _capabilitiesProvider.GetCapabilities(Settings);
-        }
-
         public override IEnumerable<ProviderDefinition> DefaultDefinitions
         {
             get

src/NzbDrone.Core/Indexers/HttpIndexerBase.cs

@@ -23,7 +23,7 @@ namespace NzbDrone.Core.Indexers
 
         public override bool SupportsRss => true;
         public override bool SupportsSearch => true;
-        public override IndexerCapabilities Capabilities => new IndexerCapabilities();
+        public override IndexerCapabilities Capabilities { get; protected set; }
 
         public bool SupportsPaging => PageSize > 0;
 
@@ -86,11 +86,6 @@ namespace NzbDrone.Core.Indexers
             return requests;
         }
 
-        public override IndexerCapabilities GetCapabilities()
-        {
-            return Capabilities;
-        }
-
         protected virtual IList<ReleaseInfo> FetchReleases(Func<IIndexerRequestGenerator, IndexerPageableRequestChain> pageableRequestChainSelector, bool isRecent = false)
         {
             var releases = new List<ReleaseInfo>();

src/NzbDrone.Core/Indexers/IIndexer.cs

@@ -16,7 +16,5 @@ namespace NzbDrone.Core.Indexers
 
         IList<ReleaseInfo> FetchRecent();
         IList<ReleaseInfo> Fetch(MovieSearchCriteria searchCriteria);
-
-        IndexerCapabilities GetCapabilities();
     }
 }

src/NzbDrone.Core/Indexers/IndexerBase.cs

@@ -26,7 +26,7 @@ namespace NzbDrone.Core.Indexers
 
         public abstract bool SupportsRss { get; }
         public abstract bool SupportsSearch { get; }
-        public abstract IndexerCapabilities Capabilities { get; }
+        public abstract IndexerCapabilities Capabilities { get; protected set; }
 
         public IndexerBase(IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger)
         {
@@ -69,8 +69,6 @@ namespace NzbDrone.Core.Indexers
         public abstract IList<ReleaseInfo> FetchRecent();
         public abstract IList<ReleaseInfo> Fetch(MovieSearchCriteria searchCriteria);
 
-        public abstract IndexerCapabilities GetCapabilities();
-
         protected virtual IList<ReleaseInfo> CleanupReleases(IEnumerable<ReleaseInfo> releases)
         {
             var result = releases.DistinctBy(v => v.Guid).ToList();

src/NzbDrone.Core/Indexers/IndexerCapabilities.cs

@@ -2,6 +2,8 @@ using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Xml.Linq;
+using NzbDrone.Core.Indexers.Cardigann;
+using NzbDrone.Core.Parser.Model;
 
 namespace NzbDrone.Core.Indexers
 {
@@ -88,6 +90,47 @@ namespace NzbDrone.Core.Indexers
             Categories = new List<IndexerCategory>();
         }
 
+        public void ParseCardigannSearchModes(Dictionary<string, List<string>> modes)
+        {
+            if (modes == null || !modes.Any())
+            {
+                throw new Exception("At least one search mode is required");
+            }
+
+            if (!modes.ContainsKey("search"))
+            {
+                throw new Exception("The search mode 'search' is mandatory");
+            }
+
+            foreach (var entry in modes)
+            {
+                switch (entry.Key)
+                {
+                    case "search":
+                        if (entry.Value == null || entry.Value.Count != 1 || entry.Value[0] != "q")
+                        {
+                            throw new Exception("In search mode 'search' only 'q' parameter is supported and it's mandatory");
+                        }
+
+                        break;
+                    case "tv-search":
+                        ParseTvSearchParams(entry.Value);
+                        break;
+                    case "movie-search":
+                        ParseMovieSearchParams(entry.Value);
+                        break;
+                    case "music-search":
+                        ParseMusicSearchParams(entry.Value);
+                        break;
+                    case "book-search":
+                        ParseBookSearchParams(entry.Value);
+                        break;
+                    default:
+                        throw new Exception($"Unsupported search mode: {entry.Key}");
+                }
+            }
+        }
+
         public void ParseTvSearchParams(IEnumerable<string> paramsList)
         {
             if (paramsList == null)
@@ -169,6 +212,33 @@ namespace NzbDrone.Core.Indexers
             }
         }
 
+        private void ParseBookSearchParams(IEnumerable<string> paramsList)
+        {
+            if (paramsList == null)
+            {
+                return;
+            }
+
+            foreach (var paramStr in paramsList)
+            {
+                if (Enum.TryParse(paramStr, true, out BookSearchParam param))
+                {
+                    if (!BookSearchParams.Contains(param))
+                    {
+                        BookSearchParams.Add(param);
+                    }
+                    else
+                    {
+                        throw new Exception($"Duplicate book-search param: {paramStr}");
+                    }
+                }
+                else
+                {
+                    throw new Exception($"Not supported book-search param: {paramStr}");
+                }
+            }
+        }
+
         private string SupportedTvSearchParams()
         {
             var parameters = new List<string> { "q" }; // q is always enabled

src/NzbDrone.Core/Indexers/IndexerFactory.cs

@@ -5,6 +5,7 @@ using FluentValidation.Results;
 using NLog;
 using NzbDrone.Common.Composition;
 using NzbDrone.Core.Indexers.Cardigann;
+using NzbDrone.Core.IndexerVersions;
 using NzbDrone.Core.Messaging.Events;
 using NzbDrone.Core.ThingiProvider;
 
@@ -20,11 +21,11 @@ namespace NzbDrone.Core.Indexers
 
     public class IndexerFactory : ProviderFactory<IIndexer, IndexerDefinition>, IIndexerFactory
     {
-        private readonly ICardigannDefinitionService _definitionService;
+        private readonly IIndexerDefinitionUpdateService _definitionService;
         private readonly IIndexerStatusService _indexerStatusService;
         private readonly Logger _logger;
 
-        public IndexerFactory(ICardigannDefinitionService definitionService,
+        public IndexerFactory(IIndexerDefinitionUpdateService definitionService,
                               IIndexerStatusService indexerStatusService,
                               IIndexerRepository providerRepository,
                               IEnumerable<IIndexer> providers,
@@ -41,14 +42,17 @@ namespace NzbDrone.Core.Indexers
         public override List<IndexerDefinition> All()
         {
             var definitions = base.All();
-            var metaDefs = _definitionService.All().ToDictionary(x => x.File);
 
             foreach (var definition in definitions)
             {
                 if (definition.Implementation == typeof(Cardigann.Cardigann).Name)
                 {
                     var settings = (CardigannSettings)definition.Settings;
-                    definition.ExtraFields = metaDefs[settings.DefinitionFile].Settings;
+                    var defFile = _definitionService.GetDefinition(settings.DefinitionFile);
+                    definition.ExtraFields = defFile.Settings;
+                    definition.Privacy = defFile.Type == "private" ? IndexerPrivacy.Private : IndexerPrivacy.Public;
+                    definition.Capabilities = new IndexerCapabilities();
+                    definition.Capabilities.ParseCardigannSearchModes(defFile.Caps.Modes);
                 }
             }
 
@@ -58,12 +62,15 @@ namespace NzbDrone.Core.Indexers
         public override IndexerDefinition Get(int id)
         {
             var definition = base.Get(id);
-            var metaDefs = _definitionService.All().ToDictionary(x => x.File);
 
             if (definition.Implementation == typeof(Cardigann.Cardigann).Name)
             {
                 var settings = (CardigannSettings)definition.Settings;
-                definition.ExtraFields = metaDefs[settings.DefinitionFile].Settings;
+                var defFile = _definitionService.GetDefinition(settings.DefinitionFile);
+                definition.ExtraFields = defFile.Settings;
+                definition.Privacy = defFile.Type == "private" ? IndexerPrivacy.Private : IndexerPrivacy.Public;
+                definition.Capabilities = new IndexerCapabilities();
+                definition.Capabilities.ParseCardigannSearchModes(defFile.Caps.Modes);
             }
 
             return definition;
@@ -79,7 +86,7 @@ namespace NzbDrone.Core.Indexers
             foreach (var provider in _providers)
             {
                 var definitions = provider.DefaultDefinitions
-                    .Where(v => v.Name != null && v.Name != provider.GetType().Name);
+                    .Where(v => v.Name != null && (v.Name != typeof(Cardigann.Cardigann).Name || v.Name != typeof(Newznab.Newznab).Name));
 
                 foreach (IndexerDefinition definition in definitions)
                 {
@@ -99,11 +106,16 @@ namespace NzbDrone.Core.Indexers
             base.SetProviderCharacteristics(provider, definition);
 
             definition.Protocol = provider.Protocol;
-            definition.Privacy = provider.Privacy;
             definition.SupportsRss = provider.SupportsRss;
             definition.SupportsSearch = provider.SupportsSearch;
+
+            //We want to use the definition Caps and Privacy for Cardigann instead of the provider.
+            if (definition.Implementation != typeof(Cardigann.Cardigann).Name)
+            {
+                definition.Privacy = provider.Privacy;
                 definition.Capabilities = provider.Capabilities;
             }
+        }
 
         public List<IIndexer> RssEnabled(bool filterBlockedIndexers = true)
         {

src/NzbDrone.Core/Jobs/TaskManager.cs

@@ -6,6 +6,7 @@ using NzbDrone.Core.Backup;
 using NzbDrone.Core.Configuration;
 using NzbDrone.Core.HealthCheck;
 using NzbDrone.Core.Housekeeping;
+using NzbDrone.Core.IndexerVersions;
 using NzbDrone.Core.Lifecycle;
 using NzbDrone.Core.Messaging.Commands;
 using NzbDrone.Core.Messaging.Events;
@@ -59,6 +60,7 @@ namespace NzbDrone.Core.Jobs
                     new ScheduledTask { Interval = 6 * 60, TypeName = typeof(ApplicationCheckUpdateCommand).FullName },
                     new ScheduledTask { Interval = 6 * 60, TypeName = typeof(CheckHealthCommand).FullName },
                     new ScheduledTask { Interval = 24 * 60, TypeName = typeof(HousekeepingCommand).FullName },
+                    new ScheduledTask { Interval = 6 * 60, TypeName = typeof(IndexerDefinitionUpdateCommand).FullName },
 
                     new ScheduledTask
                     {

src/Prowlarr.Api.V1/Indexers/IndexerModule.cs

@@ -59,7 +59,7 @@ namespace Prowlarr.Api.V1.Indexers
             switch (requestType)
             {
                 case "caps":
-                    Response response = indexerInstance.GetCapabilities().ToXml();
+                    Response response = indexer.Capabilities.ToXml();
                     response.ContentType = "application/rss+xml";
                     return response;
                 case "tvsearch":