core: refactor results post processing: filter and fixing results (#10082)

src/Jackett.Common/Indexers/BaseIndexer.cs

@@ -263,20 +263,44 @@ namespace Jackett.Common.Indexers
 
         protected virtual IEnumerable<ReleaseInfo> FilterResults(TorznabQuery query, IEnumerable<ReleaseInfo> results)
         {
-            if (query.Categories.Length == 0)
+            var filteredResults = results;
-                return results;
 
+            // filter results with wrong categories
+            if (query.Categories.Length > 0)
+            {
                 // expand parent categories from the query
                 var expandedQueryCats = TorznabCaps.Categories.ExpandTorznabQueryCategories(query);
 
-            var filteredResults = results.Where(result =>
+                filteredResults = filteredResults.Where(result =>
                     result.Category?.Any() != true ||
                     expandedQueryCats.Intersect(result.Category).Any()
                 );
+            }
+
+            // eliminate excess results
+            if (query.Limit > 0)
+                filteredResults = filteredResults.Take(query.Limit);
 
             return filteredResults;
         }
 
+        protected virtual IEnumerable<ReleaseInfo> FixResults(TorznabQuery query, IEnumerable<ReleaseInfo> results)
+        {
+            var fixedResults = results.Select(r =>
+            {
+                // add origin
+                r.Origin = this;
+
+                // fix publish date
+                // some trackers do not keep their clocks up to date and can be ~20 minutes out!
+                if (r.PublishDate > DateTime.Now)
+                    r.PublishDate = DateTime.Now;
+
+                return r;
+            });
+            return fixedResults;
+        }
+
         public virtual bool CanHandleQuery(TorznabQuery query)
         {
             if (query == null)
@@ -327,27 +351,15 @@ namespace Jackett.Common.Indexers
         public abstract Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson);
 
         public virtual async Task<IndexerResult> ResultsForQuery(TorznabQuery query)
-        {
-            try
         {
             if (!CanHandleQuery(query))
                 return new IndexerResult(this, new ReleaseInfo[0]);
+
+            try
+            {
                 var results = await PerformQuery(query);
                 results = FilterResults(query, results);
-                if (query.Limit > 0)
+                results = FixResults(query, results);
-                {
-                    results = results.Take(query.Limit);
-                }
-                results = results.Select(r =>
-                {
-                    r.Origin = this;
-
-                    // Some trackers do not keep their clocks up to date and can be ~20 minutes out!
-                    if (r.PublishDate > DateTime.Now)
-                        r.PublishDate = DateTime.Now;
-                    return r;
-                });
-
                 return new IndexerResult(this, results);
             }
             catch (Exception ex)

src/Jackett.Common/Indexers/Meta/BaseMetaIndexer.cs

@@ -47,20 +47,15 @@ namespace Jackett.Common.Indexers.Meta
         public override Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson) => Task.FromResult(IndexerConfigurationStatus.Completed);
 
         public override async Task<IndexerResult> ResultsForQuery(TorznabQuery query)
-        {
-            try
         {
             if (!CanHandleQuery(query))
                 return new IndexerResult(this, new ReleaseInfo[0]);
-                var results = await PerformQuery(query);
-                var correctedResults = results.Select(r =>
-                {
-                    if (r.PublishDate > DateTime.Now)
-                        r.PublishDate = DateTime.Now;
-                    return r;
-                });
 
-                return new IndexerResult(this, correctedResults);
+            try
+            {
+                var results = await PerformQuery(query);
+                // the results are already filtered and fixed by each indexer
+                return new IndexerResult(this, results);
             }
             catch (Exception ex)
             {

src/Jackett.Common/Models/ReleaseInfo.cs

@@ -35,7 +35,7 @@ namespace Jackett.Common.Models
         public long? MinimumSeedTime { get; set; }
         public double? DownloadVolumeFactor { get; set; }
         public double? UploadVolumeFactor { get; set; }
-        [JsonIgnore] // don't export the Origin to the manul search API, otherwise each result line contains a full recursive indexer JSON structure
+        [JsonIgnore] // don't export the Origin to the manual search API, otherwise each result line contains a full recursive indexer JSON structure
         public IIndexer Origin;
 
 

src/Jackett.Test/Common/Indexers/BaseWebIndexerTests.cs

@@ -1,3 +1,4 @@
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using Jackett.Common.Models;
@@ -42,7 +43,7 @@ namespace Jackett.Test.Common.Indexers
         }
 
         [Test]
-        public void TestFilterResults()
+        public void TestFilterResultsCategories()
         {
             var indexer = new TestWebIndexer();
             indexer.AddTestCategories();
@@ -100,6 +101,49 @@ namespace Jackett.Test.Common.Indexers
             Assert.AreEqual(null, filteredResults[1].Category);
         }
 
+        [Test]
+        public void TestFilterResultsLimit()
+        {
+            var indexer = new TestWebIndexer();
+            var results = new List<ReleaseInfo>
+            {
+                new ReleaseInfo(),
+                new ReleaseInfo()
+            };
+
+            var query = new TorznabQuery();
+            var filteredResults = indexer._FilterResults(query, results).ToList();
+            Assert.AreEqual(2, filteredResults.Count);
+
+            query = new TorznabQuery
+            {
+                Limit = 1
+            };
+            filteredResults = indexer._FilterResults(query, results).ToList();
+            Assert.AreEqual(1, filteredResults.Count);
+        }
+
+        [Test]
+        public void TestFixResults()
+        {
+            var indexer = new TestWebIndexer();
+            var query = new TorznabQuery();
+            var results = new List<ReleaseInfo>
+            {
+                new ReleaseInfo
+                {
+                    PublishDate = new DateTime(3000, 1, 1) // future date
+                }
+            };
+
+            // fix origin and publish date
+            Assert.AreEqual(null, results.First().Origin);
+            Assert.AreEqual(3000, results.First().PublishDate.Year);
+            var fixedResults = indexer._FixResults(query, results).ToList();
+            Assert.AreEqual(indexer.Id, fixedResults.First().Origin.Id);
+            Assert.AreEqual(DateTime.Now.Year, fixedResults.First().PublishDate.Year);
+        }
+
         [Test]
         public void TestAddCategoryMapping()
         {

src/Jackett.Test/TestHelpers/TestWebIndexer.cs

@@ -48,6 +48,9 @@ namespace Jackett.Test.TestHelpers
         public IEnumerable<ReleaseInfo> _FilterResults(TorznabQuery query, IEnumerable<ReleaseInfo> results) =>
             FilterResults(query, results);
 
+        public IEnumerable<ReleaseInfo> _FixResults(TorznabQuery query, IEnumerable<ReleaseInfo> results) =>
+            FixResults(query, results);
+
         public void _AddCategoryMapping(string trackerCategory, TorznabCategory newznabCategory, string trackerCategoryDesc = null) =>
             AddCategoryMapping(trackerCategory, newznabCategory, trackerCategoryDesc);