shpaq/Jackett-Jackett
1
0
Fork 0
mirror of https://github.com/Jackett/Jackett.git synced 2025-09-17 17:34:09 +02:00
Code Issues Packages Projects Releases Wiki Activity

indexers: filter invalid releases without categories or size (#15213)

Browse Source 
* indexers: filter invalid releases without categories or size

* Update BaseIndexer.cs

---------

Co-authored-by: garfield69 <garfield69@outlook.com>
This commit is contained in:
Bogdan
2024-04-01 03:46:50 +03:00
committed by GitHub
parent 00540e76a9
commit 10b3d84482
6 changed files with 87 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
src/Jackett.Common/Indexers/BaseIndexer.cs
View File

@@ -241,7 +241,21 @@ namespace Jackett.Common.Indexers
{ {
if (release.Title.IsNullOrWhiteSpace()) if (release.Title.IsNullOrWhiteSpace())
{ {
logger.Error("Invalid Release: '{0}' from indexer: {1}. No title provided.", release.Details, Name); logger.Error("[{0}] Invalid Release: '{1}'. No title provided.", Id, release.Details);
return false;
}
if (release.Size == null)
{
logger.Warn("[{0}] Invalid Release: '{1}'. No size provided.", Id, release.Details);
return false;
}
if (release.Category == null || !release.Category.Any())
{
logger.Warn("[{0}] Invalid Release: '{1}'. No categories provided.", Id, release.Details);
return false; return false;
} }

42
src/Jackett.Common/Indexers/CardigannIndexer.cs
View File

@@ -2193,34 +2193,52 @@ namespace Jackett.Common.Indexers
case "category": case "category":
if (FieldModifiers.Contains("noappend")) if (FieldModifiers.Contains("noappend"))
{ {
logger.Warn($"CardigannIndexer ({Id}): The \"noappend\" modifier is deprecated. Please switch to \"default\". See the Definition Format in the Wiki for more information."); logger.Warn("CardigannIndexer ({0}): The \"noappend\" modifier is deprecated. Please switch to \"default\". See the Definition Format in the Wiki for more information.", Id);
} }
var cats = MapTrackerCatToNewznab(value); var cats = MapTrackerCatToNewznab(value);
if (cats.Any()) if (cats.Any())
{ {
if (release.Category == null || FieldModifiers.Contains("noappend")) release.Category = release.Category == null || FieldModifiers.Contains("noappend")
release.Category = cats; ? cats
else : release.Category.Union(cats).ToList();
release.Category = release.Category.Union(cats).ToList();
} }
value = release.Category.ToString();
if (value.IsNotNullOrWhiteSpace() && !release.Category.Any())
{
logger.Warn("[{0}] Invalid category for value: '{1}'", Id, value);
}
else
{
value = release.Category.ToString();
}
break; break;
case "categorydesc": case "categorydesc":
if (FieldModifiers.Contains("noappend")) if (FieldModifiers.Contains("noappend"))
{ {
logger.Warn($"CardigannIndexer ({Id}): The \"noappend\" modifier is deprecated. Please switch to \"default\". See the Definition Format in the Wiki for more information."); logger.Warn("CardigannIndexer ({0}): The \"noappend\" modifier is deprecated. Please switch to \"default\". See the Definition Format in the Wiki for more information.", Id);
} }
var catsDesc = MapTrackerCatDescToNewznab(value); var catsDesc = MapTrackerCatDescToNewznab(value);
if (catsDesc.Any()) if (catsDesc.Any())
{ {
if (release.Category == null || FieldModifiers.Contains("noappend")) release.Category = release.Category == null || FieldModifiers.Contains("noappend")
release.Category = catsDesc; ? catsDesc
else : release.Category.Union(catsDesc).ToList();
release.Category = release.Category.Union(catsDesc).ToList();
} }
value = release.Category.ToString();
if (value.IsNotNullOrWhiteSpace() && !release.Category.Any())
{
logger.Warn("[{0}] Invalid category for value: '{1}'", Id, value);
}
else
{
value = release.Category.ToString();
}
break; break;
case "size": case "size":
release.Size = ParseUtil.GetBytes(value); release.Size = ParseUtil.GetBytes(value);

1
src/Jackett.Common/Models/ReleaseInfo.cs
View File

@@ -54,6 +54,7 @@ namespace Jackett.Common.Models
public ReleaseInfo() public ReleaseInfo()
{ {
Category = new List<int>();
Languages = new List<string>(); Languages = new List<string>();
Subs = new List<string>(); Subs = new List<string>();
} }

57
src/Jackett.Test/Common/Indexers/BaseWebIndexerTests.cs
View File

@@ -1,8 +1,10 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using Jackett.Common.Helpers;
using Jackett.Common.Models; using Jackett.Common.Models;
using Jackett.Test.TestHelpers; using Jackett.Test.TestHelpers;
using NLog;
using NUnit.Framework; using NUnit.Framework;
using Assert = NUnit.Framework.Assert; using Assert = NUnit.Framework.Assert;
@@ -11,10 +13,12 @@ namespace Jackett.Test.Common.Indexers
[TestFixture] [TestFixture]
public class BaseWebIndexerTests public class BaseWebIndexerTests
{ {
private readonly Logger _logger = LogManager.GetCurrentClassLogger();
[Test] [Test]
public void TestConstructor() public void TestConstructor()
{ {
var indexer = new TestWebIndexer(); var indexer = new TestWebIndexer(_logger);
var caps = indexer.TorznabCaps; var caps = indexer.TorznabCaps;
Assert.True(caps.SearchAvailable); Assert.True(caps.SearchAvailable);
@@ -45,7 +49,7 @@ namespace Jackett.Test.Common.Indexers
[Test] [Test]
public void TestFilterResultsCategories() public void TestFilterResultsCategories()
{ {
var indexer = new TestWebIndexer(); var indexer = new TestWebIndexer(_logger);
indexer.AddTestCategories(); indexer.AddTestCategories();
var results = new List<ReleaseInfo> var results = new List<ReleaseInfo>
@@ -53,76 +57,81 @@ namespace Jackett.Test.Common.Indexers
new ReleaseInfo new ReleaseInfo
{ {
Title = "Release 1", Title = "Release 1",
Category = new List<int> { TorznabCatType.Movies.ID } Category = new List<int> { TorznabCatType.Movies.ID },
Size = 2.Gigabytes()
}, },
new ReleaseInfo new ReleaseInfo
{ {
Title = "Release 2", Title = "Release 2",
Category = new List<int> { TorznabCatType.MoviesSD.ID } Category = new List<int> { TorznabCatType.MoviesSD.ID },
Size = 2.Gigabytes()
}, },
new ReleaseInfo new ReleaseInfo
{ {
Title = "Release 3", Title = "Release 3",
Category = new List<int> { TorznabCatType.BooksEBook.ID, 100004 } // torznab (mandatory) + custom cat Category = new List<int> { TorznabCatType.BooksEBook.ID, 100004 }, // torznab (mandatory) + custom cat
Size = 2.Gigabytes()
}, },
new ReleaseInfo new ReleaseInfo
{ {
Title = "Release 4", Title = "Release 4",
Category = new List<int> { TorznabCatType.AudioLossless.ID } // unsupported category in this indexer Category = new List<int> { TorznabCatType.AudioLossless.ID }, // unsupported category in this indexer
Size = 2.Gigabytes()
}, },
new ReleaseInfo new ReleaseInfo
{ {
Title = "Release 5", Title = "Release 5"
} }
}; };
var query = new TorznabQuery(); // without categories var query = new TorznabQuery(); // without categories
var filteredResults = indexer._FilterResults(query, results).ToList(); var filteredResults = indexer._FilterResults(query, results).ToList();
Assert.AreEqual(5, filteredResults.Count); Assert.AreEqual(4, filteredResults.Count);
query = new TorznabQuery // with child category query = new TorznabQuery // with child category
{ {
Categories = new[] { TorznabCatType.MoviesSD.ID } Categories = new[] { TorznabCatType.MoviesSD.ID }
}; };
filteredResults = indexer._FilterResults(query, results).ToList(); filteredResults = indexer._FilterResults(query, results).ToList();
Assert.AreEqual(2, filteredResults.Count); Assert.AreEqual(1, filteredResults.Count);
Assert.AreEqual(TorznabCatType.MoviesSD.ID, filteredResults[0].Category.First()); Assert.AreEqual(TorznabCatType.MoviesSD.ID, filteredResults[0].Category.First());
Assert.AreEqual(null, filteredResults[1].Category);
query = new TorznabQuery // with parent category query = new TorznabQuery // with parent category
{ {
Categories = new[] { TorznabCatType.Movies.ID } Categories = new[] { TorznabCatType.Movies.ID }
}; };
filteredResults = indexer._FilterResults(query, results).ToList(); filteredResults = indexer._FilterResults(query, results).ToList();
Assert.AreEqual(3, filteredResults.Count); Assert.AreEqual(2, filteredResults.Count);
Assert.AreEqual(TorznabCatType.Movies.ID, filteredResults[0].Category.First()); Assert.AreEqual(TorznabCatType.Movies.ID, filteredResults[0].Category.First());
Assert.AreEqual(TorznabCatType.MoviesSD.ID, filteredResults[1].Category.First()); Assert.AreEqual(TorznabCatType.MoviesSD.ID, filteredResults[1].Category.First());
Assert.AreEqual(null, filteredResults[2].Category);
query = new TorznabQuery // with custom category query = new TorznabQuery // with custom category
{ {
Categories = new[] { 100004 } Categories = new[] { 100004 }
}; };
filteredResults = indexer._FilterResults(query, results).ToList(); filteredResults = indexer._FilterResults(query, results).ToList();
Assert.AreEqual(2, filteredResults.Count); Assert.AreEqual(1, filteredResults.Count);
Assert.AreEqual(TorznabCatType.BooksEBook.ID, filteredResults[0].Category.First()); Assert.AreEqual(TorznabCatType.BooksEBook.ID, filteredResults[0].Category.First());
Assert.AreEqual(null, filteredResults[1].Category);
} }
[Test] [Test]
public void TestFilterResultsLimit() public void TestFilterResultsLimit()
{ {
var indexer = new TestWebIndexer(); var indexer = new TestWebIndexer(_logger);
var results = new List<ReleaseInfo> var results = new List<ReleaseInfo>
{ {
new ReleaseInfo new ReleaseInfo
{ {
Title = "Release 1" Title = "Release 1",
Category = new List<int> { TorznabCatType.Movies.ID },
Size = 2.Gigabytes()
}, },
new ReleaseInfo new ReleaseInfo
{ {
Title = "Release 2" Title = "Release 2",
Category = new List<int> { TorznabCatType.Movies.ID },
Size = 2.Gigabytes()
} }
}; };
@@ -141,7 +150,7 @@ namespace Jackett.Test.Common.Indexers
[Test] [Test]
public void TestFixResultsOriginPublishDate() public void TestFixResultsOriginPublishDate()
{ {
var indexer = new TestWebIndexer(); var indexer = new TestWebIndexer(_logger);
var query = new TorznabQuery(); var query = new TorznabQuery();
var results = new List<ReleaseInfo> var results = new List<ReleaseInfo>
{ {
@@ -163,7 +172,7 @@ namespace Jackett.Test.Common.Indexers
[Test] [Test]
public void TestFixResultsMagnet() public void TestFixResultsMagnet()
{ {
var indexer = new TestWebIndexer(); var indexer = new TestWebIndexer(_logger);
var query = new TorznabQuery(); var query = new TorznabQuery();
// get info_hash from magnet // get info_hash from magnet
@@ -202,7 +211,7 @@ namespace Jackett.Test.Common.Indexers
[Test] [Test]
public void TestAddCategoryMapping() public void TestAddCategoryMapping()
{ {
var indexer = new TestWebIndexer(); var indexer = new TestWebIndexer(_logger);
// you can find more complex tests in TorznabCapabilitiesCategoriesTests.cs // you can find more complex tests in TorznabCapabilitiesCategoriesTests.cs
indexer._AddCategoryMapping("11", TorznabCatType.MoviesSD, "MoviesSD"); indexer._AddCategoryMapping("11", TorznabCatType.MoviesSD, "MoviesSD");
@@ -222,7 +231,7 @@ namespace Jackett.Test.Common.Indexers
[Test] [Test]
public void TestAddMultiCategoryMapping() public void TestAddMultiCategoryMapping()
{ {
var indexer = new TestWebIndexer(); var indexer = new TestWebIndexer(_logger);
indexer._AddMultiCategoryMapping(TorznabCatType.MoviesHD, 19, 18); indexer._AddMultiCategoryMapping(TorznabCatType.MoviesHD, 19, 18);
Assert.AreEqual(1, indexer.TorznabCaps.Categories.GetTorznabCategoryTree().Count); Assert.AreEqual(1, indexer.TorznabCaps.Categories.GetTorznabCategoryTree().Count);
@@ -231,7 +240,7 @@ namespace Jackett.Test.Common.Indexers
[Test] [Test]
public void TestMapTorznabCapsToTrackers() public void TestMapTorznabCapsToTrackers()
{ {
var indexer = new TestWebIndexer(); var indexer = new TestWebIndexer(_logger);
indexer.AddTestCategories(); indexer.AddTestCategories();
// you can find more complex tests in TorznabCapabilitiesCategoriesTests.cs // you can find more complex tests in TorznabCapabilitiesCategoriesTests.cs
@@ -248,7 +257,7 @@ namespace Jackett.Test.Common.Indexers
[Test] [Test]
public void TestMapTrackerCatToNewznab() public void TestMapTrackerCatToNewznab()
{ {
var indexer = new TestWebIndexer(); var indexer = new TestWebIndexer(_logger);
indexer.AddTestCategories(); indexer.AddTestCategories();
// you can find more complex tests in TorznabCapabilitiesCategoriesTests.cs // you can find more complex tests in TorznabCapabilitiesCategoriesTests.cs
@@ -260,7 +269,7 @@ namespace Jackett.Test.Common.Indexers
[Test] [Test]
public void TestMapTrackerCatDescToNewznab() public void TestMapTrackerCatDescToNewznab()
{ {
var indexer = new TestWebIndexer(); var indexer = new TestWebIndexer(_logger);
indexer.AddTestCategories(); indexer.AddTestCategories();
// you can find more complex tests in TorznabCapabilitiesCategoriesTests.cs // you can find more complex tests in TorznabCapabilitiesCategoriesTests.cs

7
src/Jackett.Test/Common/Models/DTO/IndexerTests.cs
View File

@@ -1,6 +1,7 @@
using System.Linq; using System.Linq;
using Jackett.Common.Models.DTO; using Jackett.Common.Models.DTO;
using Jackett.Test.TestHelpers; using Jackett.Test.TestHelpers;
using NLog;
using NUnit.Framework; using NUnit.Framework;
using Assert = NUnit.Framework.Assert; using Assert = NUnit.Framework.Assert;
@@ -9,10 +10,12 @@ namespace Jackett.Test.Common.Models.DTO
[TestFixture] [TestFixture]
public class IndexerTests public class IndexerTests
{ {
private readonly Logger _logger = LogManager.GetCurrentClassLogger();
[Test] [Test]
public void TestConstructor() public void TestConstructor()
{ {
var indexer = new TestWebIndexer(); var indexer = new TestWebIndexer(_logger);
var dto = new Indexer(indexer); var dto = new Indexer(indexer);
Assert.AreEqual("test_id", dto.id); Assert.AreEqual("test_id", dto.id);
@@ -31,7 +34,7 @@ namespace Jackett.Test.Common.Models.DTO
[Test] [Test]
public void TestConstructorWithCategories() public void TestConstructorWithCategories()
{ {
var indexer = new TestWebIndexer(); var indexer = new TestWebIndexer(_logger);
indexer.AddTestCategories(); indexer.AddTestCategories();
// test Jackett UI categories (internal JSON) // test Jackett UI categories (internal JSON)

5
src/Jackett.Test/TestHelpers/TestWebIndexer.cs
View File

@@ -6,6 +6,7 @@ using Jackett.Common.Indexers;
using Jackett.Common.Models; using Jackett.Common.Models;
using Jackett.Common.Models.IndexerConfig; using Jackett.Common.Models.IndexerConfig;
using Newtonsoft.Json.Linq; using Newtonsoft.Json.Linq;
using NLog;
namespace Jackett.Test.TestHelpers namespace Jackett.Test.TestHelpers
{ {
@@ -30,8 +31,8 @@ namespace Jackett.Test.TestHelpers
public override TorznabCapabilities TorznabCaps { get; protected set; } = new TorznabCapabilities(); public override TorznabCapabilities TorznabCaps { get; protected set; } = new TorznabCapabilities();
public TestWebIndexer() public TestWebIndexer(Logger logger)
: base(client: null, configService: null, logger: null, configData: new ConfigurationData(), p: null, cacheService: null) : base(client: null, configService: null, logger: logger, configData: new ConfigurationData(), p: null, cacheService: null)
{ {
} }