Add basic support for Cardigann definitions (#571)

* Add basic support for Cardigann definitions

* Add HDME definition

* Fix tests

* support split with negative indexes

* allow FromTimeAgo formats without spaces betwen value and unit

Example: 2h 3m

* Add basic support for Cardigann definitions

* Add HDME definition

* Fix tests

* support split with negative indexes

* allow FromTimeAgo formats without spaces betwen value and unit

Example: 2h 3m

src/Jackett.Test/TestIIndexerManagerServiceHelper.cs

@@ -42,5 +42,10 @@ namespace JackettTest
         {
             throw new NotImplementedException();
         }
+
+        public void InitCardigannIndexers(string path)
+        {
+            throw new NotImplementedException();
+        }
     }
 }

src/Jackett/Definitions/hdme.yml

@@ -0,0 +1,101 @@
+---
+  site: hdme
+  name: HDME
+  language: en-us
+  links:
+    - https://hdme.eu
+
+  caps:
+    categories:
+      24: TV/Anime       # Anime
+      25: PC/0day        # Appz
+      47: Movies/HD      # AVCHD
+      26: Movies/BluRay  # Bluray
+      54: Movies/HD      # dbREMUX
+      41: Movies/HD      # Documentaries
+      50: Movies/HD      # FourGHD
+      44: Movies/HD      # HDME
+      28: Audio/Lossless # HQ Music
+      48: Movies/HD      # iCandy
+      45: Movies/HD      # INtL
+      29: Other          # Misc
+      49: PC/Phone-Other # Mobile
+      30: Movies/HD      # Movie 1080i
+      31: Movies/HD      # Movie 1080p
+      32: Movies/HD      # Movie 720p
+      33: Audio/Video    # Music Videos
+      34: TV             # Packs
+      53: Movies/HD      # Remux
+      56: Movies/HD      # RUXi
+      55: Movies/HD      # SiNiSteR
+      36: TV/Sport       # Sports
+      37: TV/HD          # TV Series 1080i
+      38: TV/HD          # TV Series 1080p
+      39: TV/HD          # TV Series 720p
+      57: Movies         # UHD 2160p
+      40: XXX            # XXX
+
+    modes:
+      search: [q]
+      tv-search: [q, season, ep]
+
+  login:
+    path: /takelogin.php
+    method: post
+    form: form
+    inputs:
+      username: "{{ .Config.username }}"
+      password: "{{ .Config.password }}"
+    error:
+      - selector: td.embedded
+        message:
+          selector: td.text
+    test:
+      path: /my.php
+
+  ratio:
+    path: /my.php
+    selector: span.smallfont > font
+    filters:
+      - name: regexp
+        args: "Ratio:(.+?)Uploaded"
+      - name: replace
+        args: [",", ""]
+
+  search:
+    path: /browse.php
+    inputs:
+      $raw: "{{range .Categories}}c{{.}}=1&{{end}}"
+      search: "{{ .Keywords }}"
+      incldead: "1"
+      blah: "0"
+    rows:
+      selector: table[width="100%"] > tbody > tr:has(td.bottom[background="_images/bg_torrent.jpg"])
+    fields:
+      category:
+        selector: td:nth-child(2) a
+        attribute: href
+        filters:
+          - name: querystring
+            args: cat
+      title:
+        selector: td:nth-child(3) > a
+        attribute: title
+      comments:
+        selector: td:nth-child(3) > a
+        attribute: href
+      download:
+        selector: td:nth-child(11) > a
+        attribute: href
+      size:
+        selector: td:nth-child(6)
+        remove: br
+      date:
+        selector: td:nth-child(3)
+        filters:
+          - name: regexp
+            args: "Added: (.+?)\n"
+      seeders:
+        selector: td:nth-child(8)
+      leechers:
+        selector: td:nth-child(9)

src/Jackett/Indexers/BaseIndexer.cs

@@ -17,13 +17,13 @@ namespace Jackett.Indexers
 {
     public abstract class BaseIndexer
     {
-        public string SiteLink { get; private set; }
+        public string SiteLink { get; protected set; }
-        public string DisplayDescription { get; private set; }
+        public string DisplayDescription { get; protected set; }
-        public string DisplayName { get; private set; }
+        public string DisplayName { get; protected set; }
         public string ID { get { return GetIndexerID(GetType()); } }
 
         public bool IsConfigured { get; protected set; }
-        public TorznabCapabilities TorznabCaps { get; private set; }
+        public TorznabCapabilities TorznabCaps { get; protected set; }
         protected Logger logger;
         protected IIndexerManagerService indexerService;
         protected static List<CachedQueryResult> cache = new List<CachedQueryResult>();
@@ -44,7 +44,9 @@ namespace Jackett.Indexers
 
         private List<CategoryMapping> categoryMapping = new List<CategoryMapping>();
 
+        // standard constructor used by most indexers
         public BaseIndexer(string name, string link, string description, IIndexerManagerService manager, IWebClient client, Logger logger, ConfigurationData configData, IProtectionService p, TorznabCapabilities caps = null, string downloadBase = null)
+            : this(manager, client, logger, p)
         {
             if (!link.EndsWith("/"))
                 throw new Exception("Site link must end with a slash.");
@@ -52,12 +54,7 @@ namespace Jackett.Indexers
             DisplayName = name;
             DisplayDescription = description;
             SiteLink = link;
-            this.logger = logger;
-            indexerService = manager;
-            webclient = client;
-            protectionService = p;
             this.downloadUrlBase = downloadBase;
-
             this.configData = configData;
 
             if (caps == null)
@@ -66,6 +63,15 @@ namespace Jackett.Indexers
 
         }
 
+        // minimal constructor used by e.g. cardigann generic indexer
+        public BaseIndexer(IIndexerManagerService manager, IWebClient client, Logger logger, IProtectionService p)
+        {
+            this.logger = logger;
+            indexerService = manager;
+            webclient = client;
+            protectionService = p;
+        }
+
         public IEnumerable<ReleaseInfo> CleanLinks(IEnumerable<ReleaseInfo> releases)
         {
             if (string.IsNullOrEmpty(downloadUrlBase))

src/Jackett/Indexers/CardigannIndexer.cs

@@ -0,0 +1,636 @@
+using Jackett.Utils.Clients;
+using NLog;
+using Jackett.Services;
+using Jackett.Utils;
+using Jackett.Models;
+using System.Threading.Tasks;
+using Newtonsoft.Json.Linq;
+using System.Collections.Generic;
+using System;
+using Jackett.Models.IndexerConfig;
+using System.Collections.Specialized;
+using System.Text;
+using YamlDotNet.Serialization;
+using YamlDotNet.Serialization.NamingConventions;
+using static Jackett.Models.IndexerConfig.ConfigurationData;
+using AngleSharp.Parser.Html;
+using System.Text.RegularExpressions;
+using System.Web;
+
+namespace Jackett.Indexers
+{
+    public class CardigannIndexer : BaseIndexer, IIndexer
+    {
+        protected IndexerDefinition Definition;
+        public new string ID { get { return (Definition != null ? Definition.Site : GetIndexerID(GetType())); } }
+
+        new ConfigurationData configData
+        {
+            get { return (ConfigurationData)base.configData; }
+            set { base.configData = value; }
+        }
+
+        // Cardigann yaml classes
+        public class IndexerDefinition {
+            public string Site { get; set; }
+            public List<settingsField> Settings { get; set; }
+            public string Name { get; set; }
+            public string Description { get; set; }
+            public string Language { get; set; }
+            public List<string> Links { get; set; }
+            public capabilitiesBlock Caps { get; set; }
+            public loginBlock Login { get; set; }
+            public ratioBlock Ratio { get; set; }
+            public searchBlock Search { get; set; }
+            // IndexerDefinitionStats not needed/implemented
+        }
+        public class settingsField
+        {
+            public string Name { get; set; }
+            public string Type { get; set; }
+            public string Label { get; set; }
+        }
+
+        public class capabilitiesBlock
+        {
+            public Dictionary<string, string> Categories  { get; set; }
+            public Dictionary<string, List<string>> Modes { get; set; }
+        }
+
+        public class loginBlock
+        {
+            public string Path { get; set; }
+            public string Method { get; set; }
+            public string Form { get; set; }
+            public Dictionary<string, string> Inputs { get; set; }
+            public List<errorBlock> Error { get; set; }
+            public pageTestBlock Test { get; set; }
+        }
+
+        public class errorBlock
+        {
+            public string Path { get; set; }
+            public string Selector { get; set; }
+            public selectorBlock Message { get; set; }
+        }
+
+        public class selectorBlock
+        {
+            public string Selector { get; set; }
+            public string Text { get; set; }
+            public string Attribute { get; set; }
+            public string Remove { get; set; }
+            public List<filterBlock> Filters { get; set; }
+            public Dictionary<string, string> Case { get; set; }
+        }
+
+        public class filterBlock
+        {
+            public string Name { get; set; }
+            public dynamic Args { get; set; }
+        }
+
+        public class pageTestBlock
+        {
+            public string Path { get; set; }
+            public string Selector { get; set; }
+        }
+
+        public class ratioBlock : selectorBlock
+        {
+            public string Path { get; set; }
+        }
+
+        public class searchBlock
+        {
+            public string Path { get; set; }
+            public Dictionary<string, string> Inputs { get; set; }
+            public rowsBlock Rows { get; set; }
+            public Dictionary<string, selectorBlock> Fields { get; set; }
+        }
+
+        public class rowsBlock : selectorBlock
+        {
+            public int After { get; set; }
+            //public string Remove { get; set; } // already inherited
+            public string Dateheaders { get; set; }
+        }
+
+        public CardigannIndexer(IIndexerManagerService i, IWebClient wc, Logger l, IProtectionService ps)
+            : base(manager: i,
+                   client: wc,
+                   logger: l,
+                   p: ps)
+        {
+        }
+
+        public CardigannIndexer(IIndexerManagerService i, IWebClient wc, Logger l, IProtectionService ps, string DefinitionString)
+            : base(manager: i,
+                   client: wc,
+                   logger: l,
+                   p: ps)
+        {
+            Init(DefinitionString);
+        }
+
+        protected void Init(string DefinitionString)
+        {
+            var deserializer = new DeserializerBuilder()
+                .WithNamingConvention(new CamelCaseNamingConvention())
+                .IgnoreUnmatchedProperties()
+                .Build();
+            Definition = deserializer.Deserialize<IndexerDefinition>(DefinitionString);
+
+            // Add default data if necessary
+            if (Definition.Settings == null)
+                Definition.Settings = new List<settingsField>();
+
+            if (Definition.Settings.Count == 0)
+            {
+                Definition.Settings.Add(new settingsField { Name = "username", Label = "Username", Type = "text" });
+                Definition.Settings.Add(new settingsField { Name = "password", Label = "Password", Type = "password" });
+            }
+
+            // init missing mandatory attributes
+            DisplayName = Definition.Name;
+            DisplayDescription = Definition.Description;
+            SiteLink = Definition.Links[0]; // TODO: implement alternative links
+            if (!SiteLink.EndsWith("/"))
+                SiteLink += "/";
+            TorznabCaps = TorznabUtil.CreateDefaultTorznabTVCaps(); // TODO implement caps
+
+            // init config Data
+            configData = new ConfigurationData();
+            foreach (var Setting in Definition.Settings)
+            {
+                configData.AddDynamic(Setting.Name, new StringItem { Name = Setting.Label });
+            }
+
+            foreach (var Category in Definition.Caps.Categories)
+            {
+                var cat = TorznabCatType.GetCatByName(Category.Value);
+                if (cat == null)
+                {
+                    logger.Error(string.Format("CardigannIndexer ({0}): Can't find a category for {1}", ID, Category.Value));
+                    continue;
+                }
+                AddCategoryMapping(Category.Key, TorznabCatType.GetCatByName(Category.Value));
+                
+            }
+        }
+
+        protected Dictionary<string, object> getTemplateVariablesFromConfigData()
+        {
+            Dictionary<string, object> variables = new Dictionary<string, object>();
+
+            foreach (settingsField Setting in Definition.Settings)
+            {
+                variables[".Config."+Setting.Name] = ((StringItem)configData.GetDynamic(Setting.Name)).Value;
+            }
+            return variables;
+        }
+
+        // A very bad implementation of the golang template/text templating engine.
+        // But it should work for most basic constucts used by Cardigann definitions.
+        protected string applyGoTemplateText(string template, Dictionary<string, object> variables = null)
+        {
+            if (variables == null)
+            {
+                variables = getTemplateVariablesFromConfigData();
+            }
+
+            // handle if ... else ... expression
+            Regex IfElseRegex = new Regex(@"{{if\s*(.+?)\s*}}(.*?){{\s*else\s*}}(.*?){{\s*end\s*}}");
+            var IfElseRegexMatches = IfElseRegex.Match(template);
+
+            while (IfElseRegexMatches.Success)
+            {
+                string conditionResult = null;
+
+                string all = IfElseRegexMatches.Groups[0].Value;
+                string condition = IfElseRegexMatches.Groups[1].Value;
+                string onTrue = IfElseRegexMatches.Groups[2].Value;
+                string onFalse = IfElseRegexMatches.Groups[3].Value;
+
+                if (condition.StartsWith("."))
+                {
+                    string value = (string)variables[condition];
+                    if (!string.IsNullOrWhiteSpace(value))
+                    {
+                        conditionResult = onTrue;
+                    }
+                    else
+                    {
+                        conditionResult = onFalse;
+                    }
+                }
+                else
+                {
+                    throw new NotImplementedException("CardigannIndexer: Condition operation '" + condition + "' not implemented");
+                }
+                template = template.Replace(all, conditionResult);
+                IfElseRegexMatches = IfElseRegexMatches.NextMatch();
+            }
+
+            // handle range expression
+            Regex RangeRegex = new Regex(@"{{\s*range\s*(.+?)\s*}}(.*?){{\.}}(.*?){{end}}");
+            var RangeRegexMatches = RangeRegex.Match(template);
+
+            while (RangeRegexMatches.Success)
+            {
+                string expanded = string.Empty;
+
+                string all = RangeRegexMatches.Groups[0].Value;
+                string variable = RangeRegexMatches.Groups[1].Value;
+                string prefix = RangeRegexMatches.Groups[2].Value;
+                string postfix = RangeRegexMatches.Groups[3].Value;
+
+                foreach (string value in (List<string>)variables[variable])
+                {
+                    expanded += prefix + value + postfix;
+                }
+                template = template.Replace(all, expanded);
+                RangeRegexMatches = RangeRegexMatches.NextMatch();
+            }
+
+            // handle simple variables
+            Regex VariablesRegEx = new Regex(@"{{\s*(\..+?)\s*}}");
+            var VariablesRegExMatches = VariablesRegEx.Match(template);
+
+            while (VariablesRegExMatches.Success)
+            {
+                string expanded = string.Empty;
+
+                string all = VariablesRegExMatches.Groups[0].Value;
+                string variable = VariablesRegExMatches.Groups[1].Value;
+
+                string value = (string)variables[variable];
+                template = template.Replace(all, value);
+                VariablesRegExMatches = VariablesRegExMatches.NextMatch();
+            }
+
+            return template;
+        }
+
+        protected async Task<bool> DoLogin()
+        {
+            var Login = Definition.Login;
+
+            if (Login == null)
+                return false;
+
+            if (Login.Method == "post")
+            {
+                var pairs = new Dictionary<string, string>();
+                foreach (var Input in Definition.Login.Inputs)
+                {
+                    var value = applyGoTemplateText(Input.Value);
+                    pairs.Add(Input.Key, value);
+                }
+                foreach (var x in pairs)
+                {
+                    logger.Error(x.Key + ": " + x.Value);
+                }
+                var LoginUrl = SiteLink + Login.Path;
+                configData.CookieHeader.Value = null;
+                var loginResult = await RequestLoginAndFollowRedirect(LoginUrl, pairs, null, true, null, SiteLink, true);
+                configData.CookieHeader.Value = loginResult.Cookies;
+
+                if (Login.Error != null)
+                {
+                    var loginResultParser = new HtmlParser();
+                    var loginResultDocument = loginResultParser.Parse(loginResult.Content);
+                    foreach (errorBlock error in Login.Error)
+                    {
+                        var selection = loginResultDocument.QuerySelector(error.Selector);
+                        if (selection != null)
+                        {
+                            string errorMessage = selection.TextContent;
+                            if (error.Message != null)
+                            {
+                                var errorSubMessage = loginResultDocument.QuerySelector(error.Message.Selector);
+                                errorMessage = errorSubMessage.TextContent;
+                            }
+                            throw new ExceptionWithConfigData(string.Format("Login failed: {0}", errorMessage.Trim()), configData);
+                        }
+                    }
+                }
+            }
+            else if (Login.Method == "cookie")
+            {
+                configData.CookieHeader.Value = ((StringItem)configData.GetDynamic("cookie")).Value;
+            }
+            else
+            {
+                throw new NotImplementedException("Login method " + Definition.Login.Method + " not implemented");
+            }
+            return true;
+        }
+
+        protected async Task<bool> TestLogin()
+        {
+            var Login = Definition.Login;
+
+            if (Login == null || Login.Test == null)
+                return false;
+
+            // test if login was successful
+            var LoginTestUrl = SiteLink + Login.Test.Path;
+            var testResult = await RequestStringWithCookies(LoginTestUrl);
+
+            if (testResult.IsRedirect)
+            {
+                throw new ExceptionWithConfigData("Login Failed, got redirected", configData);
+            }
+
+            if (Login.Test.Selector != null)
+            {
+                var testResultParser = new HtmlParser();
+                var testResultDocument = testResultParser.Parse(testResult.Content);
+                var selection = testResultDocument.QuerySelectorAll(Login.Test.Selector);
+                if (selection.Length == 0)
+                {
+                    throw new ExceptionWithConfigData(string.Format("Login failed: Selector \"{0}\" didn't match", Login.Test.Selector), configData);
+                }
+            }
+            return true;
+        }
+
+        public async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
+        {
+            configData.LoadValuesFromJson(configJson);
+
+            await DoLogin();
+            await TestLogin();
+
+            SaveConfig();
+            IsConfigured = true;
+            return IndexerConfigurationStatus.Completed;
+        }
+
+        protected string applyFilters(string Data, List<filterBlock> Filters)
+        {
+            if (Filters == null)
+                return Data;
+
+            foreach(filterBlock Filter in Filters)
+            {
+                switch (Filter.Name)
+                {
+                    case "querystring":
+                        var param = (string)Filter.Args;
+                        var qsStr = Data.Split(new char[] { '?' }, 2)[1];
+                        qsStr = Data.Split(new char[] { '#' }, 2)[0];
+                        var qs = HttpUtility.ParseQueryString(qsStr);
+                        Data = qs.Get(param);
+                        break;
+                    case "timeparse":
+                    case "dateparse":
+                        throw new NotImplementedException("Filter " + Filter.Name + " not implemented");
+                        /*
+                        TODO: implement golang time format conversion, see http://fuckinggodateformat.com/
+                        if args == nil {
+			                return filterDateParse(nil, value)
+		                }
+		                if layout, ok := args.(string); ok {
+			                return filterDateParse([]string{layout}, value)
+		                }
+		                return "", fmt.Errorf("Filter argument type %T was invalid", args)
+                         */
+                        break;
+                    case "regexp":
+                        var pattern = (string)Filter.Args;
+                        var Regexp = new Regex(pattern);
+                        var Match = Regexp.Match(Data);
+                        Data = Match.Groups[1].Value;
+                        break;
+                    case "split":
+                        var sep = (string)Filter.Args[0];
+                        var pos = (string)Filter.Args[1];
+                        var posInt = int.Parse(pos);
+                        var strParts = Data.Split(sep[0]);
+                        if (posInt < 0)
+                        {
+                            posInt += strParts.Length;
+                        }
+                        Data = strParts[posInt];
+                        break;
+                    case "replace":
+                        var from = (string)Filter.Args[0];
+                        var to = (string)Filter.Args[1];
+                        Data = Data.Replace(from, to);
+                        break;
+                    case "trim":
+                        var cutset = (string)Filter.Args;
+                        Data = Data.Trim(cutset[0]);
+                        break;
+                    case "append":
+                        var str = (string)Filter.Args;
+                        Data += str;
+                        break;
+                    case "timeago":
+                    case "fuzzytime":
+                    case "reltime":
+                        var timestr = (string)Filter.Args;
+                        Data = DateTimeUtil.FromUnknown(timestr).ToString(DateTimeUtil.RFC1123ZPattern);
+                        break;
+                    default:
+                        break;
+                }
+            }
+            return Data;
+        }
+
+        protected string handleSelector(selectorBlock Selector, AngleSharp.Dom.IElement Dom)
+        {
+            if (Selector.Text != null)
+            {
+                return applyFilters(Selector.Text, Selector.Filters);
+            }
+            string value = null;
+            if (Selector.Selector != null)
+            {
+                AngleSharp.Dom.IElement selection = Dom.QuerySelector(Selector.Selector);
+                if (selection == null)
+                {
+                    throw new Exception(string.Format("Selector \"{0}\" didn't match {1}", Selector.Selector, Dom.OuterHtml));
+                }
+                if (Selector.Remove != null)
+                {
+                    foreach(var i in selection.QuerySelectorAll(Selector.Remove))
+                    {
+                        i.Remove();
+                    }
+                }
+                if (Selector.Attribute != null)
+                {
+                    value = selection.GetAttribute(Selector.Attribute);
+                }
+                else
+                {
+                    value = selection.TextContent;
+                }
+                
+            }
+            return applyFilters(value, Selector.Filters); ;
+        }
+
+        protected Uri resolvePath(string path)
+        {
+            return new Uri(SiteLink + path);
+        }
+
+        public async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
+        {
+            var releases = new List<ReleaseInfo>();
+
+            searchBlock Search = Definition.Search;
+
+            // init template context
+            var variables = getTemplateVariablesFromConfigData();
+
+            variables[".Query.Type"] = query.QueryType;
+            variables[".Query.Q"] = query.SearchTerm;
+            variables[".Query.Series"] = null;
+            variables[".Query.Ep"] = query.Episode;
+            variables[".Query.Season"] = query.Season;
+            variables[".Query.Movie"] = null;
+            variables[".Query.Year"] = null;
+            variables[".Query.Limit"] = query.Limit;
+            variables[".Query.Offset"] = query.Offset;
+            variables[".Query.Extended"] = query.Extended;
+            variables[".Query.Categories"] = query.Categories;
+            variables[".Query.APIKey"] = query.ApiKey;
+            variables[".Query.TVDBID"] = null;
+            variables[".Query.TVRageID"] = query.RageID;
+            variables[".Query.IMDBID"] = query.ImdbID;
+            variables[".Query.TVMazeID"] = null;
+            variables[".Query.TraktID"] = null;
+
+            variables[".Query.Episode"] = query.GetEpisodeSearchString();
+            variables[".Categories"] = MapTorznabCapsToTrackers(query);
+
+            var KeywordTokens = new List<string>();
+            var KeywordTokenKeys = new List<string> { "Q", "Series", "Movie", "Year" };
+            foreach (var key in KeywordTokenKeys)
+            {
+                var Value = (string)variables[".Query." + key];
+                if (!string.IsNullOrWhiteSpace(Value))
+                    KeywordTokens.Add(Value);
+            }
+
+            if (!string.IsNullOrWhiteSpace((string)variables[".Query.Episode"]))
+                KeywordTokens.Add((string)variables[".Query.Episode"]);
+            variables[".Query.Keywords"] = string.Join(" ", KeywordTokens);
+            variables[".Keywords"] = variables[".Query.Keywords"];
+
+            // build search URL
+            var searchUrl = SiteLink + applyGoTemplateText(Search.Path, variables) + "?";
+            var queryCollection = new NameValueCollection();
+            if (Search.Inputs != null)
+            { 
+                foreach (var Input in Search.Inputs)
+                {
+                    var value = applyGoTemplateText(Input.Value, variables);
+                    if (Input.Key == "$raw")
+                        searchUrl += value;
+                    else
+                        queryCollection.Add(Input.Key, value);
+                }
+            }
+            searchUrl += "&" + queryCollection.GetQueryString();
+
+            // send HTTP request
+            var response = await RequestBytesWithCookies(searchUrl);
+            var results = Encoding.GetEncoding("iso-8859-1").GetString(response.Content);
+            try
+            {
+                var SearchResultParser = new HtmlParser();
+                var SearchResultDocument = SearchResultParser.Parse(results);
+                
+                var Rows = SearchResultDocument.QuerySelectorAll(Search.Rows.Selector);
+                foreach (var Row in Rows)
+                {
+                    try
+                    {
+                        var release = new ReleaseInfo();
+                        release.MinimumRatio = 1;
+                        release.MinimumSeedTime = 48 * 60 * 60;
+
+                        // Parse fields
+                        foreach (var Field in Search.Fields)
+                        {
+                            string value = handleSelector(Field.Value, Row);
+                            try
+                            {
+                                switch (Field.Key)
+                                {
+                                    case "download":
+                                        release.Link = resolvePath(value);
+                                        break;
+                                    case "details":
+                                        var url = resolvePath(value);
+                                        release.Guid = url;
+                                        if (release.Comments == null)
+                                            release.Comments = url;
+                                        break;
+                                    case "comments":
+                                        release.Comments = resolvePath(value);
+                                        break;
+                                    case "title":
+                                        release.Title = value;
+                                        break;
+                                    case "description":
+                                        release.Description = value;
+                                        break;
+                                    case "category":
+                                        release.Category = MapTrackerCatToNewznab(value);
+                                        break;
+                                    case "size":
+                                        release.Size = ReleaseInfo.GetBytes(value);
+                                        break;
+                                    case "leechers":
+                                        if (release.Peers == null)
+                                            release.Peers = ParseUtil.CoerceInt(value);
+                                        else
+                                            release.Peers += ParseUtil.CoerceInt(value);
+                                        break;
+                                    case "seeders":
+                                        release.Seeders = ParseUtil.CoerceInt(value);
+                                        if (release.Peers == null)
+                                            release.Peers = release.Seeders;
+                                        else
+                                            release.Peers += release.Seeders;
+
+                                        break;
+                                    case "date":
+                                        release.PublishDate = DateTimeUtil.FromUnknown(value);
+                                        break;
+                                    default:
+                                        break;
+                                }
+                            }
+                            catch (Exception ex)
+                            {
+                                throw new Exception(string.Format("Error while parsing field={0}, selector={1}, value={2}: {3}", Field.Key, Field.Value.Selector, value, ex.Message));
+                            }
+                        }
+
+                        releases.Add(release);
+                    }
+                    catch (Exception ex)
+                    {
+                        logger.Error(string.Format("CardigannIndexer ({0}): Error while parsing row '{1}': {2}", ID, Row.OuterHtml, ex.Message));
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                OnParseError(results, ex);
+            }
+
+            return releases;
+        }
+    }
+}
+

src/Jackett/Jackett.csproj

@@ -54,6 +54,10 @@
       <HintPath>..\packages\CloudFlareUtilities.0.3.2-alpha\lib\portable45-net45+win8+wpa81\CloudFlareUtilities.dll</HintPath>
       <Private>True</Private>
     </Reference>
+    <Reference Include="DateTimeRoutines, Version=1.0.0.0, Culture=neutral, processorArchitecture=MSIL">
+      <HintPath>..\packages\DateTimeRoutines.1.0.16\lib\net40\DateTimeRoutines.dll</HintPath>
+      <Private>True</Private>
+    </Reference>
     <Reference Include="System" />
     <Reference Include="System.Configuration.Install" />
     <Reference Include="System.Core" />
@@ -147,6 +151,10 @@
     <Reference Include="System.Web.Http.Tracing, Version=5.2.3.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35">
       <HintPath>..\packages\Microsoft.AspNet.WebApi.Tracing.5.2.3\lib\net45\System.Web.Http.Tracing.dll</HintPath>
     </Reference>
+    <Reference Include="YamlDotNet, Version=4.0.0.0, Culture=neutral, processorArchitecture=MSIL">
+      <HintPath>..\packages\YamlDotNet.4.0.1-pre288\lib\net35\YamlDotNet.dll</HintPath>
+      <Private>True</Private>
+    </Reference>
   </ItemGroup>
   <ItemGroup>
     <Compile Include="AuthenticationException.cs" />
@@ -156,6 +164,7 @@
     <Compile Include="Controllers\TorznabController.cs" />
     <Compile Include="Controllers\DownloadController.cs" />
     <Compile Include="Engine.cs" />
+    <Compile Include="Indexers\CardigannIndexer.cs" />
     <Compile Include="Indexers\myAmity.cs" />
     <Compile Include="Indexers\TorrentNetwork.cs" />
     <Compile Include="Indexers\Andraste.cs" />
@@ -361,6 +370,9 @@
     <None Include="CurlSharp.dll.config">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
+    <None Include="Definitions\hdme.yml">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
     <None Include="packages.config">
       <SubType>Designer</SubType>
     </None>
@@ -687,6 +699,7 @@
   <ItemGroup>
     <Service Include="{508349B6-6B84-4DF5-91F0-309BEEBAD82D}" />
   </ItemGroup>
+  <ItemGroup />
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
        Other similar extension points exist, see Microsoft.Common.targets.

src/Jackett/Models/IndexerConfig/ConfigurationData.cs

@@ -10,9 +10,10 @@ using System.Threading.Tasks;
 
 namespace Jackett.Models.IndexerConfig
 {
-    public abstract class ConfigurationData
+    public class ConfigurationData
     {
         const string PASSWORD_REPLACEMENT = "|||%%PREVJACKPASSWD%%|||";
+        protected Dictionary<string, Item> dynamics = new Dictionary<string, Item>(); // list for dynamic items
 
         public enum ItemType
         {
@@ -132,6 +133,8 @@ namespace Jackett.Models.IndexerConfig
                 .Where(p => p.PropertyType.IsSubclassOf(typeof(Item)))
                 .Select(p => (Item)p.GetValue(this));
 
+            properties = properties.Concat(dynamics.Values).ToArray();
+
             if (!forDisplay)
             {
                 properties = properties
@@ -142,6 +145,16 @@ namespace Jackett.Models.IndexerConfig
             return properties.ToArray();
         }
 
+        public void AddDynamic(string ID, Item item)
+        {
+            dynamics.Add(ID, item);
+        }
+
+        public Item GetDynamic(string ID)
+        {
+            return dynamics[ID];
+        }
+
         public class Item
         {
             public ItemType ItemType { get; set; }

src/Jackett/Models/TorznabCatType.cs

@@ -27,5 +27,21 @@ namespace Jackett.Models
             return string.Empty;
         }
 
+        public static string NormalizeCatName(string name)
+        {
+            return name.Replace(" ", "").ToLower();
+        }
+
+        public static TorznabCategory GetCatByName(string name)
+        {
+            var cat = AllCats.FirstOrDefault(c => NormalizeCatName(c.Name) == NormalizeCatName(name));
+            if (cat != null)
+            {
+                return cat;
+            }
+
+            return null;
+        }
+
     }
 }

src/Jackett/Services/ConfigurationService.cs

@@ -25,6 +25,7 @@ namespace Jackett.Services
         T GetConfig<T>();
         void SaveConfig<T>(T config);
         string ApplicationFolder();
+        string GetCardigannDefinitionsFolder();
         void CreateOrMigrateSettings();
         void PerformMigration();
     }
@@ -197,6 +198,22 @@ namespace Jackett.Services
             return dir;
         }
 
+        public string GetCardigannDefinitionsFolder()
+        {
+            // If we are debugging we can use the non copied definitions.
+            string dir = Path.Combine(ApplicationFolder(), "Definitions"); ;
+
+#if DEBUG
+            // When we are running in debug use the source files
+            var sourcePath = Path.GetFullPath(Path.Combine(ApplicationFolder(), "..\\..\\..\\Jackett\\Definitions"));
+            if (Directory.Exists(sourcePath))
+            {
+                dir = sourcePath;
+            }
+#endif
+            return dir;
+        }
+
         public string GetVersion()
         {
             return Assembly.GetExecutingAssembly().GetName().Version.ToString();

src/Jackett/Services/IndexerManagerService.cs

@@ -22,6 +22,7 @@ namespace Jackett.Services
         IEnumerable<IIndexer> GetAllIndexers();
         void SaveConfig(IIndexer indexer, JToken obj);
         void InitIndexers();
+        void InitCardigannIndexers(string path);
     }
 
     public class IndexerManagerService : IIndexerManagerService
@@ -40,26 +41,53 @@ namespace Jackett.Services
             cacheService = cache;
         }
 
+        protected void LoadIndexerConfig(IIndexer idx)
+        {
+            var configFilePath = GetIndexerConfigFilePath(idx);
+            if (File.Exists(configFilePath))
+            {
+                var fileStr = File.ReadAllText(configFilePath);
+                var jsonString = JToken.Parse(fileStr);
+                try
+                {
+                    idx.LoadFromSavedConfiguration(jsonString);
+                }
+                catch (Exception ex)
+                {
+                    logger.Error(ex, "Failed loading configuration for {0}, you must reconfigure this indexer", idx.DisplayName);
+                }
+            }
+        }
+
         public void InitIndexers()
         {
             logger.Info("Using HTTP Client: " + container.Resolve<IWebClient>().GetType().Name);
 
-            foreach (var idx in container.Resolve<IEnumerable<IIndexer>>().OrderBy(_ => _.DisplayName))
+            foreach (var idx in container.Resolve<IEnumerable<IIndexer>>().Where(p => p.ID != "cardigannindexer").OrderBy(_ => _.DisplayName))
             {
                 indexers.Add(idx.ID, idx);
-                var configFilePath = GetIndexerConfigFilePath(idx);
+                LoadIndexerConfig(idx);
-                if (File.Exists(configFilePath))
+            }
+        }
+
+        public void InitCardigannIndexers(string path)
+        {
+            logger.Info("Loading Cardigann definitions from: " + path);
+
+            DirectoryInfo d = new DirectoryInfo(path);
+
+            foreach (var file in d.GetFiles("*.yml"))
+            {
+                string DefinitionString = File.ReadAllText(file.FullName);
+                CardigannIndexer idx = new CardigannIndexer(this, container.Resolve<IWebClient>(), logger, container.Resolve<IProtectionService>(), DefinitionString);
+                if (indexers.ContainsKey(idx.ID))
                 {
-                    var fileStr = File.ReadAllText(configFilePath);
+                    logger.Debug(string.Format("Ignoring definition ID={0}, file={1}: Indexer already exists", idx.ID, file.FullName));
-                    var jsonString = JToken.Parse(fileStr);
+                }
-                    try
+                else
-                    {
+                { 
-                        idx.LoadFromSavedConfiguration(jsonString);
+                    indexers.Add(idx.ID, idx);
-                    }
+                    LoadIndexerConfig(idx);
-                    catch (Exception ex)
-                    {
-                        logger.Error(ex, "Failed loading configuration for {0}, you must reconfigure this indexer", idx.DisplayName);
-                    }
                 }
             }
         }

src/Jackett/Services/ServerService.cs

@@ -150,6 +150,7 @@ namespace Jackett.Services
             CultureInfo.DefaultThreadCurrentCulture = new CultureInfo("en-US");
             // Load indexers
             indexerService.InitIndexers();
+            indexerService.InitCardigannIndexers(configService.GetCardigannDefinitionsFolder());
             client.Init();
         }
 

src/Jackett/Utils/DateTimeUtil.cs

@@ -1,13 +1,17 @@
-using System;
+using Cliver;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
+using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 
 namespace Jackett.Utils
 {
     public static class DateTimeUtil
     {
+        public static string RFC1123ZPattern = "ddd, dd MMM yyyy HH':'mm':'ss z";
+
         public static DateTime UnixTimestampToDateTime(double unixTime)
         {
             DateTime unixStart = new DateTime(1970, 1, 1, 0, 0, 0, 0, System.DateTimeKind.Utc);
@@ -32,33 +36,117 @@ namespace Jackett.Utils
                 return DateTime.SpecifyKind(DateTime.Now, DateTimeKind.Local);
             }
 
-            var dateParts = str.Split(new char[0], StringSplitOptions.RemoveEmptyEntries);
+            str = str.Replace(",", "");
+            str = str.Replace("ago", "");
+            str = str.Replace("and", "");
+
             TimeSpan timeAgo = TimeSpan.Zero;
-            for (var i = 0; i < dateParts.Length / 2; i++)
+            Regex TimeagoRegex = new Regex(@"\s*?([\d\.]+)\s*?([^\d\s\.]+)\s*?");
+            var TimeagoMatches = TimeagoRegex.Match(str);
+
+            while (TimeagoMatches.Success)
             {
-                var val = ParseUtil.CoerceFloat(dateParts[i * 2]);
+                string expanded = string.Empty;
-                var unit = dateParts[i * 2 + 1];
+
-                if (unit.Contains("sec"))
+                var val = ParseUtil.CoerceFloat(TimeagoMatches.Groups[1].Value);
+                var unit = TimeagoMatches.Groups[2].Value;
+                TimeagoMatches = TimeagoMatches.NextMatch();
+
+                if (unit.Contains("sec") || unit == "s")
                     timeAgo += TimeSpan.FromSeconds(val);
-                else if (unit.Contains("min"))
+                else if (unit.Contains("min") || unit == "m")
                     timeAgo += TimeSpan.FromMinutes(val);
-                else if (unit.Contains("hour") || unit.Contains("hr"))
+                else if (unit.Contains("hour") || unit.Contains("hr") || unit == "h")
                     timeAgo += TimeSpan.FromHours(val);
-                else if (unit.Contains("day"))
+                else if (unit.Contains("day") ||unit == "d")
                     timeAgo += TimeSpan.FromDays(val);
-                else if (unit.Contains("week") || unit.Contains("wk"))
+                else if (unit.Contains("week") || unit.Contains("wk") || unit == "w")
                     timeAgo += TimeSpan.FromDays(val * 7);
-                else if (unit.Contains("month"))
+                else if (unit.Contains("month") || unit == "mo")
                     timeAgo += TimeSpan.FromDays(val * 30);
-                else if (unit.Contains("year"))
+                else if (unit.Contains("year") || unit == "y")
                     timeAgo += TimeSpan.FromDays(val * 365);
                 else
                 {
-                    throw new Exception("TimeAgo parsing failed");
+                    throw new Exception("TimeAgo parsing failed, unknown unit: "+unit);
                 }
             }
 
             return DateTime.SpecifyKind(DateTime.Now - timeAgo, DateTimeKind.Local);
         }
+
+        // Uses the DateTimeRoutines library to parse the date
+        // http://www.codeproject.com/Articles/33298/C-Date-Time-Parser
+        public static DateTime FromFuzzyTime(string str, DateTimeRoutines.DateTimeFormat format = DateTimeRoutines.DateTimeFormat.USA_DATE)
+        {
+            DateTimeRoutines.ParsedDateTime dt;
+            if (DateTimeRoutines.TryParseDateOrTime(str, format, out dt))
+            {
+                return dt.DateTime;
+            }
+            throw new Exception("FromFuzzyTime parsing failed");
+        }
+
+        public static Regex timeAgoRegexp = new Regex(@"(?i)\bago", RegexOptions.Compiled);
+        public static Regex todayRegexp = new Regex(@"(?i)\btoday([\s,]+|$)", RegexOptions.Compiled);
+        public static Regex tomorrowRegexp = new Regex(@"(?i)\btomorrow([\s,]+|$)", RegexOptions.Compiled);
+        public static Regex yesterdayRegexp = new Regex(@"(?i)\byesterday([\s,]+|$)", RegexOptions.Compiled);
+        public static Regex missingYearRegexp = new Regex(@"^\d{1,2}-\d{1,2}\b", RegexOptions.Compiled);
+
+        public static DateTime FromUnknown(string str)
+        {
+            str = ParseUtil.NormalizeSpace(str);
+            Match match;
+
+            // ... ago
+            match = timeAgoRegexp.Match(str);
+            if (match.Success)
+            {
+                var timeago = str;
+                return FromTimeAgo(timeago);
+            }
+
+            // Today ...
+            match = todayRegexp.Match(str);
+            if (match.Success)
+            {
+                var time = str.Replace(match.Groups[0].Value, "");
+                DateTime dt = DateTime.SpecifyKind(DateTime.UtcNow.Date, DateTimeKind.Unspecified);
+                dt += TimeSpan.Parse(time);
+                return dt;
+            }
+
+            // Yesterday ...
+            match = yesterdayRegexp.Match(str);
+            if (match.Success)
+            {
+                var time = str.Replace(match.Groups[0].Value, "");
+                DateTime dt = DateTime.SpecifyKind(DateTime.UtcNow.Date, DateTimeKind.Unspecified);
+                dt += TimeSpan.Parse(time);
+                dt -= TimeSpan.FromDays(1);
+                return dt;
+            }
+
+            // Tomorrow ...
+            match = tomorrowRegexp.Match(str);
+            if (match.Success)
+            {
+                var time = str.Replace(match.Groups[0].Value, "");
+                DateTime dt = DateTime.SpecifyKind(DateTime.UtcNow.Date, DateTimeKind.Unspecified);
+                dt += TimeSpan.Parse(time);
+                dt += TimeSpan.FromDays(1);
+                return dt;
+            }
+
+            // add missing year
+            match = missingYearRegexp.Match(str);
+            if (match.Success)
+            {
+                var date = match.Groups[0].Value;
+                string newDate = date+"-"+DateTime.Now.Year.ToString();
+                str = str.Replace(date, newDate);
+            }
+            return FromFuzzyTime(str);
+        }
     }
 }

src/Jackett/Utils/ParseUtil.cs

@@ -9,45 +9,58 @@ namespace Jackett.Utils
 {
     public static class ParseUtil
     {
+        public static string NormalizeSpace(string s)
+        {
+            return s.Trim();
+        }
+
+        public static string NormalizeNumber(string s)
+        {
+            string normalized = NormalizeSpace(s);
+            normalized = normalized.Replace("-", "0");
+            normalized = normalized.Replace(",", "");
+            return normalized;
+        }
+
         public static double CoerceDouble(string str)
         {
-            return double.Parse(str.Trim(), NumberStyles.Any, CultureInfo.InvariantCulture);
+            return double.Parse(NormalizeNumber(str), NumberStyles.Any, CultureInfo.InvariantCulture);
         }
 
         public static float CoerceFloat(string str)
         {
-            return float.Parse(str.Trim(), NumberStyles.Any, CultureInfo.InvariantCulture);
+            return float.Parse(NormalizeNumber(str), NumberStyles.Any, CultureInfo.InvariantCulture);
         }
 
         public static int CoerceInt(string str)
         {
-            return int.Parse(str.Trim(), NumberStyles.Any, CultureInfo.InvariantCulture);
+            return int.Parse(NormalizeNumber(str), NumberStyles.Any, CultureInfo.InvariantCulture);
         }
 
         public static long CoerceLong(string str)
         {
-            return long.Parse(str.Trim(), NumberStyles.Any, CultureInfo.InvariantCulture);
+            return long.Parse(NormalizeNumber(str), NumberStyles.Any, CultureInfo.InvariantCulture);
         }
 
 
         public static bool TryCoerceDouble(string str, out double result)
         {
-            return double.TryParse(str.Trim(), NumberStyles.Any, CultureInfo.InvariantCulture, out result);
+            return double.TryParse(NormalizeNumber(str), NumberStyles.Any, CultureInfo.InvariantCulture, out result);
         }
 
         public static bool TryCoerceFloat(string str, out float result)
         {
-            return float.TryParse(str.Trim(), NumberStyles.Any, CultureInfo.InvariantCulture, out result);
+            return float.TryParse(NormalizeNumber(str), NumberStyles.Any, CultureInfo.InvariantCulture, out result);
         }
 
         public static bool TryCoerceInt(string str, out int result)
         {
-            return int.TryParse(str.Trim(), NumberStyles.Any, CultureInfo.InvariantCulture, out result);
+            return int.TryParse(NormalizeNumber(str), NumberStyles.Any, CultureInfo.InvariantCulture, out result);
         }
 
         public static bool TryCoerceLong(string str, out long result)
         {
-            return long.TryParse(str.Trim(), NumberStyles.Any, CultureInfo.InvariantCulture, out result);
+            return long.TryParse(NormalizeNumber(str), NumberStyles.Any, CultureInfo.InvariantCulture, out result);
         }
 
     }

src/Jackett/packages.config

@@ -9,6 +9,7 @@
   <package id="AutoMapper" version="4.1.1" targetFramework="net45" />
   <package id="CloudFlareUtilities" version="0.3.2-alpha" targetFramework="net45" />
   <package id="CsQuery" version="1.3.4" targetFramework="net45" />
+  <package id="DateTimeRoutines" version="1.0.16" targetFramework="net45" />
   <package id="Microsoft.AspNet.Identity.Core" version="2.2.1" targetFramework="net45" />
   <package id="Microsoft.AspNet.WebApi.Client" version="5.2.3" targetFramework="net45" />
   <package id="Microsoft.AspNet.WebApi.Core" version="5.2.3" targetFramework="net45" />
@@ -30,4 +31,5 @@
   <package id="NLog.Windows.Forms" version="4.2.3" targetFramework="net45" />
   <package id="Owin" version="1.0" targetFramework="net45" />
   <package id="SharpZipLib" version="0.86.0" targetFramework="net45" />
+  <package id="YamlDotNet" version="4.0.1-pre288" targetFramework="net45" />
 </packages>