New: Smarter Newznab category mapping

src/NzbDrone.Core.Test/Files/Indexers/Newznab/newznab_caps.xml

@@ -18,6 +18,7 @@
       <subcat id="5030" name="SD"/>
       <subcat id="5060" name="Sport"/>
       <subcat id="5010" name="WEB-DL"/>
+      <subcat id="5999" name="Other"/>
     </category>
     <category id="7000" name="Other">
       <subcat id="7010" name="Misc"/>

src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabCapabilitiesProviderFixture.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Linq;
 using System.Net;
 using System.Threading.Tasks;
 using System.Xml;
@@ -70,6 +71,19 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests
             caps.LimitsMax.Value.Should().Be(60);
         }
 
+        [Test]
+        public void should_map_different_categories()
+        {
+            GivenCapsResponse(_caps);
+
+            var caps = Subject.GetCapabilities(_settings, _definition);
+
+            var bookCats = caps.Categories.MapTorznabCapsToTrackers(new int[] { NewznabStandardCategory.Books.Id });
+
+            bookCats.Count.Should().Be(2);
+            bookCats.Should().Contain("8000");
+        }
+
         [Test]
         public void should_use_default_pagesize_if_missing()
         {

src/NzbDrone.Core/Indexers/Definitions/Newznab/Newznab.cs

@@ -42,7 +42,7 @@ namespace NzbDrone.Core.Indexers.Newznab
 
         public override IParseIndexerResponse GetParser()
         {
-            return new NewznabRssParser(Settings);
+            return new NewznabRssParser(Settings, Definition, _capabilitiesProvider);
         }
 
         public string[] GetBaseUrlFromSettings()

src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabCapabilitiesProvider.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Net.Http;
 using System.Xml;
 using System.Xml.Linq;
@@ -221,27 +222,37 @@ namespace NzbDrone.Core.Indexers.Newznab
             {
                 foreach (var xmlCategory in xmlCategories.Elements("category"))
                 {
-                    var cat = new IndexerCategory
+                    var parentName = xmlCategory.Attribute("name").Value;
+                    var parentId = int.Parse(xmlCategory.Attribute("id").Value);
+                    var mappedCat = NewznabStandardCategory.AllCats.FirstOrDefault(x => x.Name.ToLower() == parentName.ToLower());
+
+                    if (mappedCat == null)
                     {
-                        Id = int.Parse(xmlCategory.Attribute("id").Value),
+                        mappedCat = NewznabStandardCategory.AllCats.FirstOrDefault(x => x.Id == parentId);
-                        Name = xmlCategory.Attribute("name").Value,
+                    }
-                        Description = xmlCategory.Attribute("description") != null ? xmlCategory.Attribute("description").Value : string.Empty
-                    };
 
                     foreach (var xmlSubcat in xmlCategory.Elements("subcat"))
                     {
-                        var subCat = new IndexerCategory
+                        var subName = xmlSubcat.Attribute("name").Value;
-                        {
+                        var subId = int.Parse(xmlSubcat.Attribute("id").Value);
-                            Id = int.Parse(xmlSubcat.Attribute("id").Value),
+                        var mappingName = $"{parentName}/{subName}";
-                            Name = xmlSubcat.Attribute("name").Value,
+                        var mappedSubCat = NewznabStandardCategory.AllCats.FirstOrDefault(x => x.Name.ToLower() == mappingName.ToLower());
-                            Description = xmlSubcat.Attribute("description") != null ? xmlSubcat.Attribute("description").Value : string.Empty
-                        };
 
-                        cat.SubCategories.Add(subCat);
+                        if (mappedSubCat == null)
-                        capabilities.Categories.AddCategoryMapping(subCat.Name, subCat);
+                        {
+                            mappedSubCat = NewznabStandardCategory.AllCats.FirstOrDefault(x => x.Id == subId);
                         }
 
-                    capabilities.Categories.AddCategoryMapping(cat.Name, cat);
+                        if (mappedSubCat != null)
+                        {
+                            capabilities.Categories.AddCategoryMapping(subId, mappedSubCat, mappingName);
+                        }
+                    }
+
+                    if (mappedCat != null)
+                    {
+                        capabilities.Categories.AddCategoryMapping(parentId, mappedCat, parentName);
+                    }
                 }
             }
 

src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabRequestGenerator.cs

@@ -68,6 +68,7 @@ namespace NzbDrone.Core.Indexers.Newznab
             }
 
             pageableRequests.Add(GetPagedRequests(searchCriteria,
+                capabilities,
                 parameters));
 
             return pageableRequests;
@@ -109,6 +110,7 @@ namespace NzbDrone.Core.Indexers.Newznab
             }
 
             pageableRequests.Add(GetPagedRequests(searchCriteria,
+                capabilities,
                 parameters));
 
             return pageableRequests;
@@ -175,6 +177,7 @@ namespace NzbDrone.Core.Indexers.Newznab
             }
 
             pageableRequests.Add(GetPagedRequests(searchCriteria,
+                capabilities,
                 parameters));
 
             return pageableRequests;
@@ -216,6 +219,7 @@ namespace NzbDrone.Core.Indexers.Newznab
             }
 
             pageableRequests.Add(GetPagedRequests(searchCriteria,
+                capabilities,
                 parameters));
 
             return pageableRequests;
@@ -233,15 +237,15 @@ namespace NzbDrone.Core.Indexers.Newznab
                 parameters.Add("q", NewsnabifyTitle(searchCriteria.SearchTerm));
             }
 
-            pageableRequests.Add(GetPagedRequests(searchCriteria, parameters));
+            pageableRequests.Add(GetPagedRequests(searchCriteria, capabilities, parameters));
 
             return pageableRequests;
         }
 
-        private IEnumerable<IndexerRequest> GetPagedRequests(SearchCriteriaBase searchCriteria, NameValueCollection parameters)
+        private IEnumerable<IndexerRequest> GetPagedRequests(SearchCriteriaBase searchCriteria, IndexerCapabilities capabilities, NameValueCollection parameters)
         {
             var baseUrl = string.Format("{0}{1}?t={2}&extended=1", Settings.BaseUrl.TrimEnd('/'), Settings.ApiPath.TrimEnd('/'), searchCriteria.SearchType);
-            var categories = searchCriteria.Categories;
+            var categories = capabilities.Categories.MapTorznabCapsToTrackers(searchCriteria.Categories);
 
             if (categories != null && categories.Any())
             {

src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabRssParser.cs

@@ -5,6 +5,7 @@ using System.Xml.Linq;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Core.Indexers.Exceptions;
 using NzbDrone.Core.Parser.Model;
+using NzbDrone.Core.ThingiProvider;
 
 namespace NzbDrone.Core.Indexers.Newznab
 {
@@ -13,12 +14,16 @@ namespace NzbDrone.Core.Indexers.Newznab
         public const string ns = "{http://www.newznab.com/DTD/2010/feeds/attributes/}";
 
         private readonly NewznabSettings _settings;
+        private readonly ProviderDefinition _definition;
+        private readonly INewznabCapabilitiesProvider _capabilitiesProvider;
 
-        public NewznabRssParser(NewznabSettings settings)
+        public NewznabRssParser(NewznabSettings settings, ProviderDefinition definition, INewznabCapabilitiesProvider capabilitiesProvider)
         {
             PreferredEnclosureMimeTypes = UsenetEnclosureMimeTypes;
             UseEnclosureUrl = true;
             _settings = settings;
+            _definition = definition;
+            _capabilitiesProvider = capabilitiesProvider;
         }
 
         public static void CheckError(XDocument xdoc, IndexerResponse indexerResponse)
@@ -118,19 +123,17 @@ namespace NzbDrone.Core.Indexers.Newznab
 
         protected override ICollection<IndexerCategory> GetCategory(XElement item)
         {
+            var capabilities = _capabilitiesProvider.GetCapabilities(_settings, _definition);
             var cats = TryGetMultipleNewznabAttributes(item, "category");
             var results = new List<IndexerCategory>();
 
             foreach (var cat in cats)
             {
-                if (int.TryParse(cat, out var intCategory))
+                var indexerCat = capabilities.Categories.MapTrackerCatToNewznab(cat);
-                {
-                    var indexerCat = _settings.Categories?.FirstOrDefault(c => c.Id == intCategory) ?? null;
 
                 if (indexerCat != null)
                 {
-                        results.Add(indexerCat);
+                    results.AddRange(indexerCat);
-                    }
                 }
             }
 

src/NzbDrone.Core/Indexers/Definitions/Torznab/Torznab.cs

@@ -41,7 +41,7 @@ namespace NzbDrone.Core.Indexers.Torznab
 
         public override IParseIndexerResponse GetParser()
         {
-            return new TorznabRssParser(Settings);
+            return new TorznabRssParser(Settings, Definition, _capabilitiesProvider);
         }
 
         public string[] GetBaseUrlFromSettings()

src/NzbDrone.Core/Indexers/Definitions/Torznab/TorznabRssParser.cs

@@ -4,7 +4,9 @@ using System.Linq;
 using System.Xml.Linq;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Core.Indexers.Exceptions;
+using NzbDrone.Core.Indexers.Newznab;
 using NzbDrone.Core.Parser.Model;
+using NzbDrone.Core.ThingiProvider;
 
 namespace NzbDrone.Core.Indexers.Torznab
 {
@@ -13,10 +15,15 @@ namespace NzbDrone.Core.Indexers.Torznab
         public const string ns = "{http://torznab.com/schemas/2015/feed}";
 
         private readonly TorznabSettings _settings;
-        public TorznabRssParser(TorznabSettings settings)
+        private readonly ProviderDefinition _definition;
+        private readonly INewznabCapabilitiesProvider _capabilitiesProvider;
+
+        public TorznabRssParser(TorznabSettings settings, ProviderDefinition definition, INewznabCapabilitiesProvider capabilitiesProvider)
         {
             UseEnclosureUrl = true;
             _settings = settings;
+            _definition = definition;
+            _capabilitiesProvider = capabilitiesProvider;
         }
 
         protected override bool PreProcess(IndexerResponse indexerResponse)
@@ -157,19 +164,17 @@ namespace NzbDrone.Core.Indexers.Torznab
 
         protected override ICollection<IndexerCategory> GetCategory(XElement item)
         {
+            var capabilities = _capabilitiesProvider.GetCapabilities(_settings, _definition);
             var cats = TryGetMultipleNewznabAttributes(item, "category");
             var results = new List<IndexerCategory>();
 
             foreach (var cat in cats)
             {
-                if (int.TryParse(cat, out var intCategory))
+                var indexerCat = capabilities.Categories.MapTrackerCatToNewznab(cat);
-                {
-                    var indexerCat = _settings.Categories?.FirstOrDefault(c => c.Id == intCategory) ?? null;
 
                 if (indexerCat != null)
                 {
-                        results.Add(indexerCat);
+                    results.AddRange(indexerCat);
-                    }
                 }
             }