From 5ee95e3cc29d1307192320eb82b5a8f1287f00d6 Mon Sep 17 00:00:00 2001 From: Qstick Date: Mon, 27 Jun 2022 20:39:15 -0500 Subject: [PATCH] V6 Cardigann Changes (#1045) * V6 Cardigann Changes * fixup! * !fixup range * !fixup more cardigann tests --- frontend/src/Search/QueryParameterModal.js | 1 + .../ApplyGoTemplateTextFixture.cs | 91 +++++++++++++++++++ src/NzbDrone.Core/History/HistoryService.cs | 1 + .../IndexerSearch/NewznabResults.cs | 1 + .../IndexerDefinitionUpdateService.cs | 2 +- .../Indexers/Definitions/Anthelion.cs | 4 - .../Definitions/Cardigann/CardigannBase.cs | 29 ++++-- .../Definitions/Cardigann/CardigannParser.cs | 11 ++- .../Cardigann/CardigannRequestGenerator.cs | 1 + .../Indexers/Definitions/Newznab/Newznab.cs | 2 +- .../Newznab/NewznabRequestGenerator.cs | 5 + .../Definitions/Newznab/NewznabRssParser.cs | 2 +- src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs | 1 + 13 files changed, 135 insertions(+), 16 deletions(-) create mode 100644 src/NzbDrone.Core.Test/IndexerTests/CardigannTests/ApplyGoTemplateTextFixture.cs diff --git a/frontend/src/Search/QueryParameterModal.js b/frontend/src/Search/QueryParameterModal.js index d684d32d8..df06648a2 100644 --- a/frontend/src/Search/QueryParameterModal.js +++ b/frontend/src/Search/QueryParameterModal.js @@ -24,6 +24,7 @@ const searchOptions = [ const seriesTokens = [ { token: '{ImdbId:tt1234567}', example: 'tt12345' }, { token: '{TvdbId:12345}', example: '12345' }, + { token: '{TmdbId:12345}', example: '12345' }, { token: '{TvMazeId:12345}', example: '54321' }, { token: '{Season:00}', example: '01' }, { token: '{Episode:00}', example: '01' } diff --git a/src/NzbDrone.Core.Test/IndexerTests/CardigannTests/ApplyGoTemplateTextFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/CardigannTests/ApplyGoTemplateTextFixture.cs new file mode 100644 index 000000000..0f585abc9 --- /dev/null +++ b/src/NzbDrone.Core.Test/IndexerTests/CardigannTests/ApplyGoTemplateTextFixture.cs @@ -0,0 +1,91 @@ +using System; +using System.Collections.Generic; +using FizzWare.NBuilder; +using FluentAssertions; +using NUnit.Framework; +using NzbDrone.Core.Indexers.Cardigann; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.IndexerTests.CardigannTests +{ + public class ApplyGoTemplateTextFixture : CoreTest + { + private Dictionary _variables; + private CardigannDefinition _definition; + + [SetUp] + public void SetUp() + { + _variables = new Dictionary + { + [".Config.sitelink"] = "https://somesite.com/", + [".True"] = "True", + [".False"] = null, + [".Today.Year"] = DateTime.Today.Year.ToString(), + [".Categories"] = new string[] { "tv", "movies" } + }; + + _definition = Builder.CreateNew() + .With(x => x.Encoding = "UTF-8") + .With(x => x.Links = new List + { + "https://somesite.com/" + }) + .With(x => x.Caps = new CapabilitiesBlock + { + Modes = new Dictionary> + { + { "search", new List { "q" } } + } + }) + .Build(); + + Mocker.SetConstant(_definition); + } + + [TestCase("{{ range .Categories}}&categories[]={{.}}{{end}}", "&categories[]=tv&categories[]=movies")] + [TestCase("{{ range $i, $e := .Categories}}&categories[{{$i}}]={{.}}{{end}}", "&categories[0]=tv&categories[1]=movies")] + [TestCase("{{ range $index, $element := .Categories}}&categories[{{$index}}]={{.}}+postIndex[{{$index}}]{{end}}", "&categories[0]=tv+postIndex[0]&categories[1]=movies+postIndex[1]")] + public void should_handle_range_statements(string template, string expected) + { + var result = Subject.ApplyGoTemplateText(template, _variables); + + result.Should().Be(expected); + } + + [TestCase("{{ re_replace .Query.Keywords \"[^a-zA-Z0-9]+\" \"%\" }}", "abc%def")] + public void should_handle_re_replace_statements(string template, string expected) + { + _variables[".Query.Keywords"] = string.Join(" ", new List { "abc", "def" }); + + var result = Subject.ApplyGoTemplateText(template, _variables); + + result.Should().Be(expected); + } + + [TestCase("{{ join .Categories \", \" }}", "tv, movies")] + public void should_handle_join_statements(string template, string expected) + { + var result = Subject.ApplyGoTemplateText(template, _variables); + + result.Should().Be(expected); + } + + [TestCase("{{ .Today.Year }}", "2022")] + public void should_handle_variables_statements(string template, string expected) + { + var result = Subject.ApplyGoTemplateText(template, _variables); + + result.Should().Be(expected); + } + + [TestCase("{{if .False }}0{{else}}1{{end}}", "1")] + [TestCase("{{if .True }}0{{else}}1{{end}}", "0")] + public void should_handle_if_statements(string template, string expected) + { + var result = Subject.ApplyGoTemplateText(template, _variables); + + result.Should().Be(expected); + } + } +} diff --git a/src/NzbDrone.Core/History/HistoryService.cs b/src/NzbDrone.Core/History/HistoryService.cs index 685a36882..a06636d42 100644 --- a/src/NzbDrone.Core/History/HistoryService.cs +++ b/src/NzbDrone.Core/History/HistoryService.cs @@ -136,6 +136,7 @@ namespace NzbDrone.Core.History { history.Data.Add("ImdbId", ((TvSearchCriteria)message.Query).FullImdbId ?? string.Empty); history.Data.Add("TvdbId", ((TvSearchCriteria)message.Query).TvdbId?.ToString() ?? string.Empty); + history.Data.Add("TmdbId", ((TvSearchCriteria)message.Query).TmdbId?.ToString() ?? string.Empty); history.Data.Add("TraktId", ((TvSearchCriteria)message.Query).TraktId?.ToString() ?? string.Empty); history.Data.Add("RId", ((TvSearchCriteria)message.Query).RId?.ToString() ?? string.Empty); history.Data.Add("TvMazeId", ((TvSearchCriteria)message.Query).TvMazeId?.ToString() ?? string.Empty); diff --git a/src/NzbDrone.Core/IndexerSearch/NewznabResults.cs b/src/NzbDrone.Core/IndexerSearch/NewznabResults.cs index 134bfe4d9..85af5b5f4 100644 --- a/src/NzbDrone.Core/IndexerSearch/NewznabResults.cs +++ b/src/NzbDrone.Core/IndexerSearch/NewznabResults.cs @@ -98,6 +98,7 @@ namespace NzbDrone.Core.IndexerSearch GetNabElement("imdb", r.ImdbId.ToString("D7"), protocol), GetNabElement("tmdbid", r.TmdbId, protocol), GetNabElement("traktid", r.TraktId, protocol), + GetNabElement("doubanid", r.DoubanId, protocol), GetNabElement("seeders", t.Seeders, protocol), GetNabElement("files", r.Files, protocol), GetNabElement("grabs", r.Grabs, protocol), diff --git a/src/NzbDrone.Core/IndexerVersions/IndexerDefinitionUpdateService.cs b/src/NzbDrone.Core/IndexerVersions/IndexerDefinitionUpdateService.cs index 2bc83aae7..b48b790b2 100644 --- a/src/NzbDrone.Core/IndexerVersions/IndexerDefinitionUpdateService.cs +++ b/src/NzbDrone.Core/IndexerVersions/IndexerDefinitionUpdateService.cs @@ -28,7 +28,7 @@ namespace NzbDrone.Core.IndexerVersions /* Update Service will fall back if version # does not exist for an indexer per Ta */ private const string DEFINITION_BRANCH = "master"; - private const int DEFINITION_VERSION = 5; + private const int DEFINITION_VERSION = 6; //Used when moving yml to C# private readonly List _defintionBlocklist = new List() diff --git a/src/NzbDrone.Core/Indexers/Definitions/Anthelion.cs b/src/NzbDrone.Core/Indexers/Definitions/Anthelion.cs index cb1cdde92..f1464f07a 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Anthelion.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Anthelion.cs @@ -6,11 +6,9 @@ using System.Net.Http; using System.Text; using System.Threading.Tasks; using AngleSharp.Html.Parser; -using FluentValidation; using NLog; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; -using NzbDrone.Core.Annotations; using NzbDrone.Core.Configuration; using NzbDrone.Core.Indexers.Exceptions; using NzbDrone.Core.Indexers.Settings; @@ -18,8 +16,6 @@ using NzbDrone.Core.IndexerSearch.Definitions; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Parser; using NzbDrone.Core.Parser.Model; -using NzbDrone.Core.ThingiProvider; -using NzbDrone.Core.Validation; namespace NzbDrone.Core.Indexers.Definitions { diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannBase.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannBase.cs index 387f45a58..3c725792f 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannBase.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannBase.cs @@ -28,7 +28,7 @@ namespace NzbDrone.Core.Indexers.Cardigann protected readonly IndexerCapabilitiesCategories _categories = new IndexerCapabilitiesCategories(); protected readonly List _defaultCategories = new List(); - protected readonly string[] OptionalFields = new string[] { "imdb", "imdbid", "rageid", "tmdbid", "tvdbid", "poster", "banner", "description" }; + protected readonly string[] OptionalFields = new string[] { "imdb", "imdbid", "rageid", "tmdbid", "tvdbid", "poster", "banner", "description", "doubanid" }; protected static readonly string[] _SupportedLogicFunctions = { @@ -338,9 +338,9 @@ namespace NzbDrone.Core.Indexers.Cardigann return variables; } - protected delegate string TemplateTextModifier(string str); + public delegate string TemplateTextModifier(string str); - protected string ApplyGoTemplateText(string template, Dictionary variables = null, TemplateTextModifier modifier = null) + public string ApplyGoTemplateText(string template, Dictionary variables = null, TemplateTextModifier modifier = null) { if (variables == null) { @@ -520,7 +520,7 @@ namespace NzbDrone.Core.Indexers.Cardigann } // handle range expression - var rangeRegex = new Regex(@"{{\s*range\s*(.+?)\s*}}(.*?){{\.}}(.*?){{end}}"); + var rangeRegex = new Regex(@"{{\s*range\s*(((?\$.+?),)((\s*(?.+?)\s*(:=)\s*)))?(?.+?)\s*}}(?.*?){{\.}}(?.*?){{end}}"); var rangeRegexMatches = rangeRegex.Match(template); while (rangeRegexMatches.Success) @@ -528,9 +528,13 @@ namespace NzbDrone.Core.Indexers.Cardigann var expanded = string.Empty; var all = rangeRegexMatches.Groups[0].Value; - var variable = rangeRegexMatches.Groups[1].Value; - var prefix = rangeRegexMatches.Groups[2].Value; - var postfix = rangeRegexMatches.Groups[3].Value; + var index = rangeRegexMatches.Groups["index"].Value; + var variable = rangeRegexMatches.Groups["variable"].Value; + var prefix = rangeRegexMatches.Groups["prefix"].Value; + var postfix = rangeRegexMatches.Groups["postfix"].Value; + + var arrayIndex = 0; + var indexReplace = "{{" + index + "}}"; foreach (var value in (ICollection)variables[variable]) { @@ -540,7 +544,16 @@ namespace NzbDrone.Core.Indexers.Cardigann newvalue = modifier(newvalue); } - expanded += prefix + newvalue + postfix; + var indexValue = arrayIndex++; + + if (index.IsNotNullOrWhiteSpace()) + { + expanded += prefix.Replace(indexReplace, indexValue.ToString()) + newvalue + postfix.Replace(indexReplace, indexValue.ToString()); + } + else + { + expanded += prefix + newvalue + postfix; + } } template = template.Replace(all, expanded); diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannParser.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannParser.cs index 930d5b45f..414ca8e53 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannParser.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannParser.cs @@ -60,7 +60,9 @@ namespace NzbDrone.Core.Indexers.Cardigann if (request.SearchPath.Response != null && request.SearchPath.Response.Type.Equals("json")) { - if (request.SearchPath.Response != null && request.SearchPath.Response.NoResultsMessage != null && (request.SearchPath.Response.NoResultsMessage.Equals(results) || (request.SearchPath.Response.NoResultsMessage == string.Empty && results == string.Empty))) + if (request.SearchPath.Response != null && + request.SearchPath.Response.NoResultsMessage != null && + ((request.SearchPath.Response.NoResultsMessage != string.Empty && results.Contains(request.SearchPath.Response.NoResultsMessage)) || (request.SearchPath.Response.NoResultsMessage == string.Empty && results == string.Empty))) { return releases; } @@ -575,6 +577,13 @@ namespace NzbDrone.Core.Indexers.Cardigann release.TvdbId = (int)ParseUtil.CoerceLong(tvdbId); value = release.TvdbId.ToString(); break; + case "doubanid": + var doubanIDRegEx = new Regex(@"(\d+)", RegexOptions.Compiled); + var doubanIDMatch = doubanIDRegEx.Match(value); + var doubanID = doubanIDMatch.Groups[1].Value; + release.DoubanId = (int)ParseUtil.CoerceLong(doubanID); + value = release.DoubanId.ToString(); + break; case "poster": if (!string.IsNullOrWhiteSpace(value)) { diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs index 079313e3d..dce7816d7 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs @@ -91,6 +91,7 @@ namespace NzbDrone.Core.Indexers.Cardigann variables[".Query.IMDBID"] = searchCriteria.FullImdbId; variables[".Query.IMDBIDShort"] = searchCriteria.ImdbId; variables[".Query.TVDBID"] = searchCriteria.TvdbId?.ToString() ?? null; + variables[".Query.TMDBID"] = searchCriteria.TmdbId?.ToString() ?? null; variables[".Query.TVRageID"] = searchCriteria.RId?.ToString() ?? null; variables[".Query.TVMazeID"] = searchCriteria.TvMazeId?.ToString() ?? null; variables[".Query.TraktID"] = searchCriteria.TraktId?.ToString() ?? null; diff --git a/src/NzbDrone.Core/Indexers/Definitions/Newznab/Newznab.cs b/src/NzbDrone.Core/Indexers/Definitions/Newznab/Newznab.cs index 367076cf1..9660c9391 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Newznab/Newznab.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Newznab/Newznab.cs @@ -187,7 +187,7 @@ namespace NzbDrone.Core.Indexers.Newznab } if (capabilities.TvSearchParams != null && - new[] { TvSearchParam.Q, TvSearchParam.TvdbId, TvSearchParam.RId }.Any(v => capabilities.TvSearchParams.Contains(v)) && + new[] { TvSearchParam.Q, TvSearchParam.TvdbId, TvSearchParam.TmdbId, TvSearchParam.RId }.Any(v => capabilities.TvSearchParams.Contains(v)) && new[] { TvSearchParam.Season, TvSearchParam.Ep }.All(v => capabilities.TvSearchParams.Contains(v))) { return null; diff --git a/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabRequestGenerator.cs index f8835e82a..08a05debb 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabRequestGenerator.cs @@ -125,6 +125,11 @@ namespace NzbDrone.Core.Indexers.Newznab parameters.Add("tvdbid", searchCriteria.TvdbId.Value.ToString()); } + if (searchCriteria.TmdbId.HasValue && capabilities.TvSearchTvdbAvailable) + { + parameters.Add("tmdbid", searchCriteria.TvdbId.Value.ToString()); + } + if (searchCriteria.ImdbId.IsNotNullOrWhiteSpace() && capabilities.TvSearchImdbAvailable) { parameters.Add("imdbid", searchCriteria.ImdbId); diff --git a/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabRssParser.cs b/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabRssParser.cs index b3792a47b..851951356 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabRssParser.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabRssParser.cs @@ -96,7 +96,7 @@ namespace NzbDrone.Core.Indexers.Newznab releaseInfo = base.ProcessItem(item, releaseInfo); releaseInfo.ImdbId = GetIntAttribute(item, "imdb"); - releaseInfo.TmdbId = GetIntAttribute(item, "tmdb"); + releaseInfo.TmdbId = GetIntAttribute(item, "tmdbid"); releaseInfo.TvdbId = GetIntAttribute(item, "tvdbid"); releaseInfo.TvRageId = GetIntAttribute(item, "rageid"); releaseInfo.Grabs = GetIntAttribute(item, "grabs"); diff --git a/src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs b/src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs index b8e77e328..0505c9e0d 100644 --- a/src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs +++ b/src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs @@ -33,6 +33,7 @@ namespace NzbDrone.Core.Parser.Model public int ImdbId { get; set; } public int TmdbId { get; set; } public int TraktId { get; set; } + public int DoubanId { get; set; } public int Year { get; set; } public string Author { get; set; } public string BookTitle { get; set; }