[feature] Indexer status (#11706)

Co-authored-by: ilike2burnthing <59480337+ilike2burnthing@users.noreply.github.com>

README.md

@@ -596,6 +596,7 @@ Filter | Condition
 `tag:<tag>` | where the indexer tags contains `<tag>`
 `lang:<tag>` | where the indexer language start with `<lang>`
 `test:{passed\|failed}` | where the last indexer test performed `passed` or `failed`
+`status:{healthy\|failing\|unknown}` | where the indexer state is `healthy` (succesfully operates in the last minutes), `failing` (generates errors in the recent call) or `unknown` (unused for a while)
 
 Supported operators
 Operator | Condition
@@ -604,9 +605,12 @@ Operator | Condition
 `<expr1>+<expr2>[+<expr3>...]` | where `<expr1>` and `<expr2>` [and `<expr3>`...]
 `<expr1>,<expr2>[,<expr3>...]` | where `<expr1>` or `<expr2>` [or `<expr3>`...]
 
-Example:
+Example 1:
 The "filter" indexer at `/api/v2.0/indexers/tag:group1,!type:private+lang:en/results/torznab` will query all the configured indexers tagged with `group1` or all the indexers not private and with `en` language (`en-en`,`en-us`,...)
 
+Example 2:
+The "filter" indexer at `/api/v2.0/indexers/!status:failing,test:passed` will query all the configured indexers not `failing` or which `passed` its last test.
+
 ## Installation on Windows
 We recommend you install Jackett as a Windows service using the supplied installer. You may also download the zipped version if you would like to configure everything manually.
 

src/Jackett.Common/Indexers/BaseIndexer.cs

@@ -36,6 +36,14 @@ namespace Jackett.Common.Indexers
         public virtual bool IsConfigured { get; protected set; }
         public virtual string[] Tags { get; protected set; }
 
+        // https://github.com/Jackett/Jackett/issues/3292#issuecomment-838586679
+        private TimeSpan HealthyStatusValidity => cacheService.CacheTTL + cacheService.CacheTTL;
+        private static readonly TimeSpan ErrorStatusValidity = TimeSpan.FromMinutes(10);
+        private static readonly TimeSpan MaxStatusValidity = TimeSpan.FromDays(1);
+
+        private int errorCount;
+        private DateTime expireAt;
+
         protected Logger logger;
         protected IIndexerConfigurationService configurationService;
         protected IProtectionService protectionService;
@@ -61,6 +69,10 @@ namespace Jackett.Common.Indexers
             }
         }
 
+        public virtual bool IsHealthy => errorCount == 0 && expireAt > DateTime.Now;
+        public virtual bool IsFailing => errorCount > 0 && expireAt > DateTime.Now;
+
+
         public abstract TorznabCapabilities TorznabCaps { get; protected set; }
 
         // standard constructor used by most indexers
@@ -92,6 +104,8 @@ namespace Jackett.Common.Indexers
         {
             CookieHeader = string.Empty;
             IsConfigured = false;
+            errorCount = 0;
+            expireAt = DateTime.MinValue;
         }
 
         public virtual void SaveConfig() => configurationService.Save(this as IIndexer, configData.ToJson(protectionService, forDisplay: false));
@@ -386,10 +400,14 @@ namespace Jackett.Common.Indexers
                 results = FilterResults(query, results);
                 results = FixResults(query, results);
                 cacheService.CacheResults(this, query, results.ToList());
+                errorCount = 0;
+                expireAt = DateTime.Now.Add(HealthyStatusValidity);
                 return new IndexerResult(this, results, false);
             }
             catch (Exception ex)
             {
+                var delay = Math.Min(MaxStatusValidity.TotalSeconds, ErrorStatusValidity.TotalSeconds * Math.Pow(2, errorCount++));
+                expireAt = DateTime.Now.AddSeconds(delay);
                 throw new IndexerException(this, ex);
             }
         }

src/Jackett.Common/Indexers/IIndexer.cs

@@ -41,6 +41,8 @@ namespace Jackett.Common.Indexers
         bool IsConfigured { get; }
 
         string[] Tags { get; }
+        bool IsHealthy { get; }
+        bool IsFailing { get; }
 
         // Retrieved for starting setup for the indexer via web API
         Task<ConfigurationData> GetConfigurationForSetup();

src/Jackett.Common/Services/CacheService.cs

@@ -177,6 +177,8 @@ namespace Jackett.Common.Services
             }
         }
 
+        public TimeSpan CacheTTL => TimeSpan.FromSeconds(_serverConfig.CacheTtl);
+
         private bool IsCacheEnabled()
         {
             if (!_serverConfig.CacheEnabled)

src/Jackett.Common/Services/Interfaces/ICacheService.cs

@@ -1,3 +1,4 @@
+using System;
 using System.Collections.Generic;
 using Jackett.Common.Indexers;
 using Jackett.Common.Models;
@@ -11,5 +12,6 @@ namespace Jackett.Common.Services.Interfaces
         List<TrackerCacheResult> GetCachedResults();
         void CleanIndexerCache(IIndexer indexer);
         void CleanCache();
+        TimeSpan CacheTTL { get; }
     }
 }

src/Jackett.Common/Utils/FilterFunc.cs

@@ -17,10 +17,11 @@ namespace Jackett.Common.Utils
         public static readonly FilterFuncComponent Language = Component("lang", args => indexer => indexer.Language.StartsWith(args, StringComparison.InvariantCultureIgnoreCase));
         public static readonly FilterFuncComponent Type = Component("type", args => indexer => string.Equals(indexer.Type, args, StringComparison.InvariantCultureIgnoreCase));
         public static readonly FilterFuncComponent Test = TestFilterFunc.Default;
+        public static readonly FilterFuncComponent Status = StatusFilterFunc.Default;
 
         static FilterFunc()
         {
-            Expression = new FilterFuncExpression(Tag, Language, Type, Test);
+            Expression = new FilterFuncExpression(Tag, Language, Type, Test, Status);
         }
 
         public static bool TryParse(string source, out Func<IIndexer, bool> func)

src/Jackett.Common/Utils/FilterFuncs/StatusFilterFunc.cs

@@ -0,0 +1,30 @@
+using System;
+using Jackett.Common.Indexers;
+
+namespace Jackett.Common.Utils.FilterFuncs
+{
+    public class StatusFilterFunc : FilterFuncComponent
+    {
+        public static readonly StatusFilterFunc Default = new StatusFilterFunc();
+        public const string Healthy = "healthy";
+        public const string Failing = "failing";
+        public const string Unknown = "unknown";
+
+        private StatusFilterFunc() : base("status")
+        {
+        }
+
+        public override Func<IIndexer, bool> ToFunc(string args)
+        {
+            if (args == null)
+                throw new ArgumentNullException(nameof(args));
+            if (string.Equals(Healthy, args, StringComparison.InvariantCultureIgnoreCase))
+                return i => IsValid(i) && i.IsHealthy && !i.IsFailing;
+            if (string.Equals(Failing, args, StringComparison.InvariantCultureIgnoreCase))
+                return i => IsValid(i) && !i.IsHealthy && i.IsFailing;
+            if (string.Equals(Unknown, args, StringComparison.InvariantCultureIgnoreCase))
+                return i => IsValid(i) && ((!i.IsHealthy && !i.IsFailing) || (i.IsHealthy && i.IsFailing));
+            throw new ArgumentException($"Invalid filter. Status should be '{Healthy}', {Failing} or '{Unknown}'", nameof(args));
+        }
+    }
+}

src/Jackett.Test/Common/Utils/FilterFuncs/IndexerBaseStub.cs

@@ -38,6 +38,10 @@ namespace Jackett.Test.Common.Utils.FilterFuncs
 
         public virtual string[] Tags => throw TestExceptions.UnexpectedInvocation;
 
+        public virtual bool IsHealthy => throw TestExceptions.UnexpectedInvocation;
+
+        public virtual bool IsFailing => throw TestExceptions.UnexpectedInvocation;
+
         public virtual Task<ConfigurationData> GetConfigurationForSetup() => throw TestExceptions.UnexpectedInvocation;
 
         public virtual Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson) => throw TestExceptions.UnexpectedInvocation;

src/Jackett.Test/Common/Utils/FilterFuncs/StatusFuncTests.cs

@@ -0,0 +1,108 @@
+using System;
+using Jackett.Common.Utils;
+using Jackett.Common.Utils.FilterFuncs;
+using NUnit.Framework;
+
+namespace Jackett.Test.Common.Utils.FilterFuncs
+{
+    [TestFixture]
+    public class StatusFuncTests
+    {
+        private static readonly IndexerStub HealthyIndexer = new IndexerStub(isHealthy: true, isFailing: false);
+        private static readonly IndexerStub FailingIndexer = new IndexerStub(isHealthy: false, isFailing: true);
+        private static readonly IndexerStub UnknownIndexer = new IndexerStub(isHealthy: false, isFailing: false);
+        private static readonly IndexerStub InvalidIndexer = new IndexerStub(isHealthy: true, isFailing: true);
+
+        private class IndexerStub : IndexerBaseStub
+        {
+            public IndexerStub(bool isHealthy, bool isFailing)
+            {
+                IsHealthy = isHealthy;
+                IsFailing = isFailing;
+            }
+
+            public override bool IsConfigured => true;
+
+            public override bool IsHealthy { get; }
+            public override bool IsFailing { get; }
+        }
+
+        [Test]
+        public void NullStatus_ThrowsException()
+        {
+            Assert.Throws<ArgumentNullException>(() => FilterFunc.Status.ToFunc(null));
+        }
+
+        [Test]
+        public void EmptyStatus_ThrowsException()
+        {
+            Assert.Throws<ArgumentException>(() => FilterFunc.Status.ToFunc(string.Empty));
+        }
+
+        [Test]
+        public void InvalidStatus_ThrowsException()
+        {
+            Assert.Throws<ArgumentException>(() => FilterFunc.Status.ToFunc(StatusFilterFunc.Healthy + StatusFilterFunc.Failing));
+        }
+
+        [Test]
+        public void HealthyFilter()
+        {
+            var passedFilterFunc = FilterFunc.Status.ToFunc(StatusFilterFunc.Healthy);
+            Assert.IsTrue(passedFilterFunc(HealthyIndexer));
+            Assert.IsFalse(passedFilterFunc(FailingIndexer));
+            Assert.IsFalse(passedFilterFunc(UnknownIndexer));
+            Assert.IsFalse(passedFilterFunc(InvalidIndexer));
+        }
+
+        [Test]
+        public void FailingFilter()
+        {
+            var failingFilterFunc = FilterFunc.Status.ToFunc(StatusFilterFunc.Failing);
+            Assert.IsFalse(failingFilterFunc(HealthyIndexer));
+            Assert.IsTrue(failingFilterFunc(FailingIndexer));
+            Assert.IsFalse(failingFilterFunc(UnknownIndexer));
+            Assert.IsFalse(failingFilterFunc(InvalidIndexer));
+        }
+
+        [Test]
+        public void UnknownFilter()
+        {
+            var unknownFilterFunc = FilterFunc.Status.ToFunc(StatusFilterFunc.Unknown);
+            Assert.IsFalse(unknownFilterFunc(HealthyIndexer));
+            Assert.IsFalse(unknownFilterFunc(FailingIndexer));
+            Assert.IsTrue(unknownFilterFunc(UnknownIndexer));
+            Assert.IsTrue(unknownFilterFunc(InvalidIndexer));
+        }
+
+        [Test]
+        public void PassedFilter_CaseInsensitiveSource()
+        {
+            var upperFilterFunc = FilterFunc.Status.ToFunc(StatusFilterFunc.Healthy.ToUpper());
+            Assert.IsTrue(upperFilterFunc(HealthyIndexer));
+
+            var lowerFilterFunc = FilterFunc.Status.ToFunc(StatusFilterFunc.Healthy.ToLower());
+            Assert.IsTrue(lowerFilterFunc(HealthyIndexer));
+        }
+
+        [Test]
+        public void FailedFilter_CaseInsensitiveSource()
+        {
+            var upperFilterFunc = FilterFunc.Status.ToFunc(StatusFilterFunc.Failing.ToUpper());
+            Assert.IsTrue(upperFilterFunc(FailingIndexer));
+
+            var lowerFilterFunc = FilterFunc.Status.ToFunc(StatusFilterFunc.Failing.ToLower());
+            Assert.IsTrue(lowerFilterFunc(FailingIndexer));
+        }
+
+        [Test]
+        public void UnknownFilter_CaseInsensitiveSource()
+        {
+            var upperFilterFunc = FilterFunc.Status.ToFunc(StatusFilterFunc.Unknown.ToUpper());
+            Assert.IsTrue(upperFilterFunc(UnknownIndexer));
+
+            var lowerFilterFunc = FilterFunc.Status.ToFunc(StatusFilterFunc.Unknown.ToLower());
+            Assert.IsTrue(lowerFilterFunc(UnknownIndexer));
+        }
+    }
+}