New: Support for Prowlarr Definitions v2

New: Support for Updated yml Definitions
Fixes: #298

src/NzbDrone.Core/IndexerVersions/IndexerDefinitionUpdateService.cs

@@ -24,7 +24,9 @@ namespace NzbDrone.Core.IndexerVersions
 
     public class IndexerDefinitionUpdateService : IIndexerDefinitionUpdateService, IExecute<IndexerDefinitionUpdateCommand>
     {
-        private const int DEFINITION_VERSION = 1;
+        /* Update Service will fall back if version # does not exist for an indexer  per Ta */
+
+        private const int DEFINITION_VERSION = 2;
         private readonly List<string> _defintionBlocklist = new List<string>()
         {
             "aither",

src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannDefinition.cs

@@ -39,6 +39,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
         public List<string> Links { get; set; }
         public List<string> Legacylinks { get; set; }
         public bool Followredirect { get; set; } = false;
+        public bool TestLinkTorrent { get; set; } = true;
         public List<string> Certificates { get; set; }
         public CapabilitiesBlock Caps { get; set; }
         public LoginBlock Login { get; set; }
@@ -167,11 +168,30 @@ namespace NzbDrone.Core.Indexers.Cardigann
     }
 
     public class DownloadBlock
+    {
+        public List<SelectorField> Selectors { get; set; }
+        public string Method { get; set; }
+        public BeforeBlock Before { get; set; }
+        public InfohashBlock Infohash { get; set; }
+    }
+
+    public class InfohashBlock
+    {
+        public SelectorField Hash { get; set; }
+        public SelectorField Title { get; set; }
+        public bool UseBeforeResponse { get; set; }
+    }
+
+    public class SelectorField
     {
         public string Selector { get; set; }
         public string Attribute { get; set; }
+        public bool UseBeforeResponse { get; set; }
         public List<FilterBlock> Filters { get; set; }
-        public string Method { get; set; }
+    }
-        public RequestBlock Before { get; set; }
+
+    public class BeforeBlock : RequestBlock
+    {
+        public SelectorField Pathselector { get; set; }
     }
 }

src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs

@@ -721,9 +721,26 @@ namespace NzbDrone.Core.Indexers.Cardigann
 
                 AddTemplateVariablesFromUri(variables, link, ".DownloadUri");
 
-                if (download.Before != null)
+                var headers = ParseCustomHeaders(_definition.Search?.Headers, variables);
+                HttpResponse response = null;
+
+                var request = new HttpRequestBuilder(link.ToString())
+                    .SetCookies(Cookies ?? new Dictionary<string, string>())
+                    .SetHeaders(headers ?? new Dictionary<string, string>())
+                    .Build();
+
+                request.AllowAutoRedirect = true;
+
+                var beforeBlock = download.Before;
+                if (beforeBlock != null)
                 {
-                    await HandleRequest(download.Before, variables, link.ToString());
+                    if (beforeBlock.Pathselector != null)
+                    {
+                        response = await HttpClient.ExecuteProxiedAsync(request, Definition);
+                        beforeBlock.Path = MatchSelector(response, beforeBlock.Pathselector, variables);
+                    }
+
+                    response = await HandleRequest(beforeBlock, variables, link.ToString());
                 }
 
                 if (download.Method == "post")
@@ -731,49 +748,105 @@ namespace NzbDrone.Core.Indexers.Cardigann
                     method = HttpMethod.POST;
                 }
 
-                if (download.Selector != null)
+                if (download.Infohash != null)
                 {
-                    var selector = ApplyGoTemplateText(download.Selector, variables);
+                    try
-                    var headers = ParseCustomHeaders(_definition.Search?.Headers, variables);
-
-                    var request = new HttpRequestBuilder(link.ToString())
-                        .SetCookies(Cookies ?? new Dictionary<string, string>())
-                        .SetHeaders(headers ?? new Dictionary<string, string>())
-                        .Build();
-
-                    request.AllowAutoRedirect = true;
-
-                    var response = await HttpClient.ExecuteProxiedAsync(request, Definition);
-
-                    var results = response.Content;
-                    var searchResultParser = new HtmlParser();
-                    var searchResultDocument = searchResultParser.ParseDocument(results);
-                    var downloadElement = searchResultDocument.QuerySelector(selector);
-                    if (downloadElement != null)
                     {
-                        _logger.Debug(string.Format("CardigannIndexer ({0}): Download selector {1} matched:{2}", _definition.Id, selector, downloadElement.ToHtmlPretty()));
+                        headers = ParseCustomHeaders(_definition.Search?.Headers, variables);
 
-                        var href = "";
+                        if (!download.Infohash.UseBeforeResponse || download.Before == null || response == null)
-                        if (download.Attribute != null)
                         {
-                            href = downloadElement.GetAttribute(download.Attribute);
+                            response = await HttpClient.ExecuteProxiedAsync(request, Definition);
+                        }
+
+                        var hash = MatchSelector(response, download.Infohash.Hash, variables);
+                        if (hash == null)
+                        {
+                            throw new Exception($"InfoHash selectors didn't match");
+                        }
+
+                        var title = MatchSelector(response, download.Infohash.Title, variables);
+                        if (title == null)
+                        {
+                            throw new Exception($"InfoHash selectors didn't match");
+                        }
+
+                        var magnet = MagnetLinkBuilder.BuildPublicMagnetLink(hash, title);
+                        var torrentLink = ResolvePath(magnet, link);
+
+                        var hashDownloadRequest = new HttpRequestBuilder(torrentLink.AbsoluteUri)
+                            .SetCookies(Cookies ?? new Dictionary<string, string>())
+                            .Build();
+
+                        hashDownloadRequest.Method = method;
+
+                        return hashDownloadRequest;
+                    }
+                    catch (Exception)
+                    {
+                        _logger.Error("CardigannIndexer ({0}): Exception with InfoHash block with hashSelector {1} and titleSelector {2}",
+                            _definition.Id,
+                            download.Infohash.Hash.Selector,
+                            download.Infohash.Title.Selector);
+                    }
+                }
+                else if (download.Selectors != null)
+                {
+                    headers = ParseCustomHeaders(_definition.Search?.Headers, variables);
+
+                    foreach (var selector in download.Selectors)
+                    {
+                        var queryselector = ApplyGoTemplateText(selector.Selector, variables);
+
+                        try
+                        {
+                            if (!selector.UseBeforeResponse || download.Before == null || response == null)
+                            {
+                                response = await HttpClient.ExecuteProxiedAsync(request, Definition);
+                            }
+
+                            var href = MatchSelector(response, selector, variables, debugMatch: true);
                             if (href == null)
                             {
-                                throw new Exception(string.Format("Attribute \"{0}\" is not set for element {1}", download.Attribute, downloadElement.ToHtmlPretty()));
+                                continue;
                             }
-                        }
-                        else
-                        {
-                            href = downloadElement.TextContent;
-                        }
 
-                        href = ApplyFilters(href, download.Filters, variables);
+                            var torrentLink = ResolvePath(href, link);
-                        link = ResolvePath(href, link);
+                            if (torrentLink.Scheme != "magnet" && _definition.TestLinkTorrent)
-                    }
+                            {
-                    else
+                                // Test link
-                    {
+                                var testLinkRequest = new HttpRequestBuilder(torrentLink.ToString())
-                        _logger.Error(string.Format("CardigannIndexer ({0}): Download selector {1} didn't match:\n{2}", _definition.Id, download.Selector, results));
+                                    .SetCookies(Cookies ?? new Dictionary<string, string>())
-                        throw new Exception(string.Format("Download selector {0} didn't match", download.Selector));
+                                    .SetHeaders(headers ?? new Dictionary<string, string>())
+                                    .Build();
+
+                                response = await HttpClient.ExecuteProxiedAsync(testLinkRequest, Definition);
+
+                                var content = response.Content;
+                                if (content.Length >= 1 && content[0] != 'd')
+                                {
+                                    _logger.Debug("CardigannIndexer ({0}): Download selector {1}'s torrent file is invalid, retrying with next available selector", _definition.Id, queryselector);
+
+                                    continue;
+                                }
+                            }
+
+                            link = torrentLink;
+
+                            var selectorDownloadRequest = new HttpRequestBuilder(link.AbsoluteUri)
+                                .SetCookies(Cookies ?? new Dictionary<string, string>())
+                                .Build();
+
+                            selectorDownloadRequest.Method = method;
+
+                            return selectorDownloadRequest;
+                        }
+                        catch (Exception e)
+                        {
+                            _logger.Error("{0} CardigannIndexer ({1}): An exception occurred while trying selector {2}, retrying with next available selector", e, _definition.Id, queryselector);
+
+                            throw new Exception(string.Format("An exception occurred while trying selector {0}", queryselector));
+                        }
                     }
                 }
             }
@@ -787,6 +860,44 @@ namespace NzbDrone.Core.Indexers.Cardigann
             return downloadRequest;
         }
 
+        protected string MatchSelector(HttpResponse response, SelectorField selector, Dictionary<string, object> variables, bool debugMatch = false)
+        {
+            var selectorText = ApplyGoTemplateText(selector.Selector, variables);
+            var parser = new HtmlParser();
+
+            var results = response.Content;
+            var resultDocument = parser.ParseDocument(results);
+
+            var element = resultDocument.QuerySelector(selectorText);
+            if (element == null)
+            {
+                _logger.Debug($"CardigannIndexer ({_definition.Id}): Selector {selectorText} could not match any elements.");
+                return null;
+            }
+
+            if (debugMatch)
+            {
+                _logger.Debug($"CardigannIndexer ({_definition.Id}): Download selector {selector} matched:{element.ToHtmlPretty()}");
+            }
+
+            string val;
+            if (selector.Attribute != null)
+            {
+                val = element.GetAttribute(selector.Attribute);
+                if (val == null)
+                {
+                    throw new Exception($"Attribute \"{selector.Attribute}\" is not set for element {element.ToHtmlPretty()}");
+                }
+            }
+            else
+            {
+                val = element.TextContent;
+            }
+
+            val = ApplyFilters(val, selector.Filters, variables);
+            return val;
+        }
+
         public bool CheckIfLoginIsNeeded(HttpResponse response)
         {
             if (response.HasHttpRedirect)